diff --git a/.env.example b/.env.example index 0acdeac..86bb653 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,6 @@ LOG_LEVEL=DEBUG RSPAMD_URL=http://127.0.0.1:11333 RSPAMD_CONTROLLER_URL=http://127.0.0.1:11334 RSPAMD_CONTROLLER_PASSWORD=devpass + +# Spelling/grammar check in the compose window (LanguageTool). Empty = off. +LANGUAGETOOL_URL=http://127.0.0.1:8010 diff --git a/Makefile b/Makefile index 495e093..c76b76c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ up: ## start services - docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch + docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool down: ## stop services (data kept) docker compose -f docker/docker-compose.yml down reset-db: ## DESTRUCTIVE: wipe db + search + blobs, re-migrate diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 3c9f0ff..41e40c5 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -23,6 +23,7 @@ def register_routers(app: FastAPI) -> None: from app.api.saved_searches import router as saved_searches_router from app.api.search import router as search_router from app.api.settings import router as settings_router + from app.api.spellcheck import router as spellcheck_router from app.api.threads import router as threads_router from app.identity.routes import router as auth_router from app.ws import router as ws_router @@ -42,6 +43,7 @@ def register_routers(app: FastAPI) -> None: avatar_router, plugins_router, admin_router, + spellcheck_router, ): app.include_router(router, prefix="/api") app.include_router(ws_router, prefix="/api") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index c3ad959..f07f929 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Annotated from fastapi import APIRouter, Depends @@ -24,6 +25,10 @@ _ALLOWED_INVITE_POLICY = {"add_update", "update_only", "none"} _ALLOWED_INVITE_FROM = {"contacts", "anyone"} +# "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de". +_LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*") + + class ReminderPref(BaseModel): method: str = "alert" # alert = in-app notification, email = email me minutes: int = 0 # minutes before the event start (0 = at time of event) @@ -42,6 +47,10 @@ class SettingsOut(BaseModel): invitation_calendar: str = "" # calendar id emailed events land in (blank => default) invitation_file: bool = False # also mark the invite message read + filed allowed_image_senders: list[str] = [] + # Spelling and grammar check in the compose window, and its language + # ("auto" detects it per message). + spellcheck_enabled: bool = True + spellcheck_language: str = "auto" class SettingsPatch(BaseModel): @@ -56,6 +65,8 @@ class SettingsPatch(BaseModel): invitation_from: str | None = None invitation_calendar: str | None = None invitation_file: bool | None = None + spellcheck_enabled: bool | None = None + spellcheck_language: str | None = None class QuotaOut(BaseModel): @@ -113,6 +124,8 @@ def _settings_out(user: User) -> SettingsOut: invitation_calendar=str(s.get("invitation_calendar", "")), invitation_file=bool(s.get("invitation_file", False)), allowed_image_senders=sorted(allowed_image_senders(user)), + spellcheck_enabled=bool(s.get("spellcheck_enabled", True)), + spellcheck_language=str(s.get("spellcheck_language", "auto")), ) @@ -144,9 +157,7 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession) if body.default_calendar is not None: updates["default_calendar"] = body.default_calendar if body.default_reminders_timed is not None: - updates["default_reminders_timed"] = [ - r.model_dump() for r in body.default_reminders_timed - ] + updates["default_reminders_timed"] = [r.model_dump() for r in body.default_reminders_timed] if body.default_reminders_allday is not None: updates["default_reminders_allday"] = [ r.model_dump() for r in body.default_reminders_allday @@ -159,6 +170,10 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession) updates["invitation_calendar"] = body.invitation_calendar if body.invitation_file is not None: updates["invitation_file"] = body.invitation_file + if body.spellcheck_enabled is not None: + updates["spellcheck_enabled"] = body.spellcheck_enabled + if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language): + updates["spellcheck_language"] = body.spellcheck_language user.settings = updates await db.commit() return _settings_out(user) diff --git a/backend/app/api/spellcheck.py b/backend/app/api/spellcheck.py new file mode 100644 index 0000000..ecb517d --- /dev/null +++ b/backend/app/api/spellcheck.py @@ -0,0 +1,187 @@ +"""Spelling and grammar check for the compose window, backed by LanguageTool. + +The browser sends the text it wants checked here, never to LanguageTool +itself: the engine stays on the internal network, only signed-in users reach +it, and nothing leaves the server. When no engine is configured (or it is +down), the check answers 503 and the compose window simply goes without. +""" + +from __future__ import annotations + +import asyncio +import time +from collections import Counter +from typing import Any + +import httpx +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field + +from app.api.errors import raise_api_error +from app.config import get_settings +from app.identity.deps import CurrentUser, require_csrf + +router = APIRouter(prefix="/spellcheck", tags=["compose"]) + +_TIMEOUT = 10.0 +# A message body is checked as a whole; past this it is a document, not mail. +MAX_TEXT = 20_000 +_MAX_SUGGESTIONS = 6 +# With "auto", LanguageTool settles on ONE language per request, so a message +# mixing Polish and English paragraphs would have every English word flagged as +# a Polish typo. Auto-detected text is therefore checked in chunks, each in its +# own language. A chunk is at least this many characters of consecutive lines: +# a greeting alone ("Witam,") is too short to detect reliably, so it rides with +# the lines after it. +_MIN_CHUNK = 60 +_PARALLEL = 4 + + +class CheckIn(BaseModel): + text: str = Field(max_length=MAX_TEXT) + # "auto" or a LanguageTool code ("pl-PL", "en-GB", "de-DE" ...). + language: str = Field(default="auto", max_length=16) + + +class Issue(BaseModel): + offset: int + length: int + message: str + short_message: str + replacements: list[str] + # spelling | grammar | style: what the underline colour follows. + kind: str + rule_id: str + + +class CheckOut(BaseModel): + language: str # the language checked (detected, with "auto") + language_name: str + issues: list[Issue] + + +class LanguageOut(BaseModel): + code: str + name: str + + +def _base() -> str: + url = get_settings().languagetool_url.rstrip("/") + if not url: + raise_api_error(503, "spellcheck_unavailable", "spell checking is not set up") + return url + + +def _kind(issue_type: str) -> str: + if issue_type == "misspelling": + return "spelling" + if issue_type in ("style", "typographical", "whitespace", "register", "locale-violation"): + return "style" + return "grammar" + + +def chunks(text: str) -> list[tuple[int, str]]: + """Split ``text`` into (offset, chunk) runs of whole lines, each at least + _MIN_CHUNK characters where the text allows; a blank line always ends one.""" + out: list[tuple[int, str]] = [] + start = pos = 0 + for line in text.split("\n"): + end = pos + len(line) + if not line.strip(): + if text[start:pos].strip(): + # Up to the line break before the blank line, not including it. + out.append((start, text[start:pos].rstrip("\n"))) + start = end + 1 + elif end - start >= _MIN_CHUNK: + out.append((start, text[start:end])) + start = end + 1 + pos = end + 1 + if text[start:].strip(): + out.append((start, text[start:])) + return out + + +async def _check_one( + client: httpx.AsyncClient, base: str, text: str, language: str +) -> dict[str, Any]: + resp = await client.post(f"{base}/v2/check", data={"text": text, "language": language}) + resp.raise_for_status() + data: dict[str, Any] = resp.json() + return data + + +def _issues(data: dict[str, Any], shift: int) -> list[Issue]: + return [ + Issue( + offset=int(m["offset"]) + shift, + length=int(m["length"]), + message=str(m.get("message", "")), + short_message=str(m.get("shortMessage", "")), + replacements=[str(r["value"]) for r in (m.get("replacements") or [])][ + :_MAX_SUGGESTIONS + ], + kind=_kind(str((m.get("rule") or {}).get("issueType", ""))), + rule_id=str((m.get("rule") or {}).get("id", "")), + ) + for m in data.get("matches") or [] + ] + + +def _detected(data: dict[str, Any]) -> tuple[str, str]: + lang = data.get("language") or {} + detected = lang.get("detectedLanguage") or lang + return str(detected.get("code", "")), str(detected.get("name", "")) + + +@router.post("", response_model=CheckOut, dependencies=[Depends(require_csrf)]) +async def check(body: CheckIn, user: CurrentUser) -> CheckOut: + base = _base() + if not body.text.strip(): + return CheckOut(language=body.language, language_name="", issues=[]) + parts = chunks(body.text) if body.language == "auto" else [(0, body.text)] + gate = asyncio.Semaphore(_PARALLEL) + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + + async def one(text: str) -> dict[str, Any]: + async with gate: + return await _check_one(client, base, text, body.language) + + results = await asyncio.gather(*(one(text) for _, text in parts)) + except (httpx.HTTPError, ValueError): + raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering") + + issues = [ + i for (shift, _), data in zip(parts, results, strict=True) for i in _issues(data, shift) + ] + # The language reported is the one most of the text was written in. + weight: Counter[tuple[str, str]] = Counter() + for (_, text), data in zip(parts, results, strict=True): + weight[_detected(data)] += len(text) + (code, name), _ = weight.most_common(1)[0] + return CheckOut(language=code or body.language, language_name=name, issues=issues) + + +_languages_cache: tuple[float, list[LanguageOut]] | None = None + + +@router.get("/languages", response_model=list[LanguageOut]) +async def languages(user: CurrentUser) -> list[LanguageOut]: + """What the engine can check, for the language picker in Settings.""" + global _languages_cache + if _languages_cache and time.monotonic() - _languages_cache[0] < 3600: + return _languages_cache[1] + base = _base() + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + resp = await client.get(f"{base}/v2/languages") + resp.raise_for_status() + rows: list[dict[str, Any]] = resp.json() + except (httpx.HTTPError, ValueError): + raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering") + seen: dict[str, str] = {} + for row in rows: + seen.setdefault(str(row["longCode"]), str(row["name"])) + out = sorted((LanguageOut(code=c, name=n) for c, n in seen.items()), key=lambda x: x.name) + _languages_cache = (time.monotonic(), out) + return out diff --git a/backend/app/config.py b/backend/app/config.py index 86b5622..a8654c5 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -68,6 +68,9 @@ class Settings(BaseSettings): # Server-side sender avatars (Gravatar + domain favicon). Fetched by the server # (never the browser) so senders can't see the user's IP or read activity. sender_avatars_enabled: bool = Field(default=True, alias="SENDER_AVATARS_ENABLED") + # Spelling and grammar checking in the compose window (LanguageTool server, + # e.g. http://languagetool:8010). Empty: the check is off for everyone. + languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL") # Auto-purge Trash/Spam messages older than this many days (0 disables). trash_retention_days: int = Field(default=30, alias="TRASH_RETENTION_DAYS") diff --git a/backend/tests/api/test_spellcheck.py b/backend/tests/api/test_spellcheck.py new file mode 100644 index 0000000..9216dfe --- /dev/null +++ b/backend/tests/api/test_spellcheck.py @@ -0,0 +1,182 @@ +"""Spelling/grammar check proxy (compose window -> LanguageTool). + +LanguageTool is replaced by a stub transport, so these run without the engine. +""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest +import pytest_asyncio +from httpx import ASGITransport +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool + +from app.api import spellcheck +from app.config import get_settings +from app.identity.passwords import hash_password +from app.identity.sessions import create_session +from app.main import create_app +from app.models import User + +LT_REPLY: dict[str, Any] = { + "language": { + "code": "auto", + "name": "Auto", + "detectedLanguage": {"code": "pl-PL", "name": "Polish"}, + }, + "matches": [ + { + "offset": 12, + "length": 6, + "message": "Wyraz „prosze” to najczęściej literówka", + "shortMessage": "Literówka", + "replacements": [{"value": "proszę"}], + "rule": {"id": "PL_PROSZE", "issueType": "misspelling"}, + }, + { + "offset": 40, + "length": 11, + "message": "Powtórzony wyraz", + "shortMessage": "", + "replacements": [{"value": v} for v in "abcdefgh"], + "rule": {"id": "WORD_REPEAT", "issueType": "duplication"}, + }, + ], +} + + +@pytest_asyncio.fixture +async def client(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[httpx.AsyncClient]: + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + session = async_sessionmaker(engine, expire_on_commit=False)() + user = User(handle=f"sp{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")) + session.add(user) + await session.commit() + token = await create_session(user.id, "pytest") + async with httpx.AsyncClient( + transport=ASGITransport(app=create_app()), + base_url="http://testserver", + cookies={"session": token, "csrf": "c"}, + headers={"X-CSRF": "c"}, + ) as c: + yield c + await session.execute(delete(User).where(User.id == user.id)) + await session.commit() + await session.close() + await engine.dispose() + + +def _stub(monkeypatch: pytest.MonkeyPatch, handler: Any) -> list[httpx.Request]: + """Route the module's outbound calls to `handler`; return what was sent.""" + seen: list[httpx.Request] = [] + real = httpx.AsyncClient + + def wrapped(request: httpx.Request) -> httpx.Response: + seen.append(request) + return handler(request) + + monkeypatch.setattr( + spellcheck.httpx, + "AsyncClient", + lambda **kw: real(transport=httpx.MockTransport(wrapped), **kw), + ) + monkeypatch.setattr(get_settings(), "languagetool_url", "http://lt.test:8010") + return seen + + +async def test_check_returns_trimmed_typed_issues( + client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + sent = _stub(monkeypatch, lambda _r: httpx.Response(200, json=LT_REPLY)) + resp = await client.post( + "/api/spellcheck", + json={"text": "Dzień dobry prosze o pliki ważne ważne", "language": "auto"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert (body["language"], body["language_name"]) == ("pl-PL", "Polish") + first, second = body["issues"] + assert first["kind"] == "spelling" and first["replacements"] == ["proszę"] + assert second["kind"] == "grammar" and len(second["replacements"]) == 6 + assert b"language=auto" in sent[0].content + + +async def test_unconfigured_or_down_engine_is_a_503( + client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def down(_r: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused") + + _stub(monkeypatch, down) + resp = await client.post("/api/spellcheck", json={"text": "hello"}) + assert resp.json()["error"]["code"] == "spellcheck_unavailable" + + monkeypatch.setattr(get_settings(), "languagetool_url", "") + resp = await client.post("/api/spellcheck", json={"text": "hello"}) + assert resp.status_code == 503 + + +async def test_empty_text_never_reaches_the_engine( + client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + sent = _stub(monkeypatch, lambda _r: httpx.Response(200, json=LT_REPLY)) + resp = await client.post("/api/spellcheck", json={"text": " \n "}) + assert resp.json()["issues"] == [] and sent == [] + + +async def test_settings_store_the_spellcheck_choice(client: httpx.AsyncClient) -> None: + out = (await client.patch("/api/settings", json={"spellcheck_language": "pl-PL"})).json() + assert out["spellcheck_language"] == "pl-PL" and out["spellcheck_enabled"] is True + out = (await client.patch("/api/settings", json={"spellcheck_enabled": False})).json() + assert out["spellcheck_enabled"] is False + bad = (await client.patch("/api/settings", json={"spellcheck_language": "