From 1f2e539eb2aa8286856aa56ea0f6c2f550dc396c Mon Sep 17 00:00:00 2001 From: Lukas Remis Date: Sun, 4 Oct 2026 23:51:31 +0200 Subject: [PATCH] Compose: spelling and grammar check with LanguageTool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A self-hosted LanguageTool (dev: docker/docker-compose.yml, started by `make up`) checks what the user writes in the compose window, through POST /api/spellcheck — the browser never talks to the engine directly. - Marks are CSS highlights over the text, never written into the message: red wavy for spelling, blue for grammar/style. Clicking one shows the message, up to five suggestions (applied as typed, so Ctrl+Z undoes them) and Ignore. The popup, like the link popup, opens above its anchor when there is no room below. - Checked after a pause in typing; the signature and quoted original are skipped; marks on words just edited drop at once; a check overtaken by a newer one, by turning checking off, or by closing the window is discarded. - With "auto", text is checked in chunks of lines, each in its own detected language — LanguageTool settles on one language per request, so a mixed Polish/English message had every English word flagged as a Polish typo. - Settings > Appearance: on/off and language (Auto-detect or any of the engine's languages); the compose window's More menu overrides it for one message. The browser's own spellcheck steps aside while ours runs, and comes back when it is off or the server has no engine (503 from the endpoint). Co-Authored-By: Claude Opus 5.5 --- .env.example | 3 + Makefile | 2 +- backend/app/api/__init__.py | 2 + backend/app/api/settings.py | 21 +- backend/app/api/spellcheck.py | 187 +++++++++++++ backend/app/config.py | 3 + backend/tests/api/test_spellcheck.py | 182 ++++++++++++ docker/docker-compose.yml | 17 ++ frontend/src/components/compose/Composer.tsx | 68 ++++- .../src/components/compose/RichTextEditor.tsx | 259 +++++++++++++++++- frontend/src/components/compose/spellcheck.ts | 143 ++++++++++ frontend/src/i18n/en.ts | 13 + frontend/src/index.css | 14 + frontend/src/lib/api.ts | 9 + frontend/src/lib/types.ts | 21 ++ frontend/src/routes/settings/Settings.tsx | 65 +++++ 16 files changed, 1000 insertions(+), 9 deletions(-) create mode 100644 backend/app/api/spellcheck.py create mode 100644 backend/tests/api/test_spellcheck.py create mode 100644 frontend/src/components/compose/spellcheck.ts diff --git a/.env.example b/.env.example index 0acdeac..86bb653 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,6 @@ LOG_LEVEL=DEBUG RSPAMD_URL=http://127.0.0.1:11333 RSPAMD_CONTROLLER_URL=http://127.0.0.1:11334 RSPAMD_CONTROLLER_PASSWORD=devpass + +# Spelling/grammar check in the compose window (LanguageTool). Empty = off. +LANGUAGETOOL_URL=http://127.0.0.1:8010 diff --git a/Makefile b/Makefile index 495e093..c76b76c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ up: ## start services - docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch + docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool down: ## stop services (data kept) docker compose -f docker/docker-compose.yml down reset-db: ## DESTRUCTIVE: wipe db + search + blobs, re-migrate diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 3c9f0ff..41e40c5 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -23,6 +23,7 @@ def register_routers(app: FastAPI) -> None: from app.api.saved_searches import router as saved_searches_router from app.api.search import router as search_router from app.api.settings import router as settings_router + from app.api.spellcheck import router as spellcheck_router from app.api.threads import router as threads_router from app.identity.routes import router as auth_router from app.ws import router as ws_router @@ -42,6 +43,7 @@ def register_routers(app: FastAPI) -> None: avatar_router, plugins_router, admin_router, + spellcheck_router, ): app.include_router(router, prefix="/api") app.include_router(ws_router, prefix="/api") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index c3ad959..f07f929 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Annotated from fastapi import APIRouter, Depends @@ -24,6 +25,10 @@ _ALLOWED_INVITE_POLICY = {"add_update", "update_only", "none"} _ALLOWED_INVITE_FROM = {"contacts", "anyone"} +# "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de". +_LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*") + + class ReminderPref(BaseModel): method: str = "alert" # alert = in-app notification, email = email me minutes: int = 0 # minutes before the event start (0 = at time of event) @@ -42,6 +47,10 @@ class SettingsOut(BaseModel): invitation_calendar: str = "" # calendar id emailed events land in (blank => default) invitation_file: bool = False # also mark the invite message read + filed allowed_image_senders: list[str] = [] + # Spelling and grammar check in the compose window, and its language + # ("auto" detects it per message). + spellcheck_enabled: bool = True + spellcheck_language: str = "auto" class SettingsPatch(BaseModel): @@ -56,6 +65,8 @@ class SettingsPatch(BaseModel): invitation_from: str | None = None invitation_calendar: str | None = None invitation_file: bool | None = None + spellcheck_enabled: bool | None = None + spellcheck_language: str | None = None class QuotaOut(BaseModel): @@ -113,6 +124,8 @@ def _settings_out(user: User) -> SettingsOut: invitation_calendar=str(s.get("invitation_calendar", "")), invitation_file=bool(s.get("invitation_file", False)), allowed_image_senders=sorted(allowed_image_senders(user)), + spellcheck_enabled=bool(s.get("spellcheck_enabled", True)), + spellcheck_language=str(s.get("spellcheck_language", "auto")), ) @@ -144,9 +157,7 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession) if body.default_calendar is not None: updates["default_calendar"] = body.default_calendar if body.default_reminders_timed is not None: - updates["default_reminders_timed"] = [ - r.model_dump() for r in body.default_reminders_timed - ] + updates["default_reminders_timed"] = [r.model_dump() for r in body.default_reminders_timed] if body.default_reminders_allday is not None: updates["default_reminders_allday"] = [ r.model_dump() for r in body.default_reminders_allday @@ -159,6 +170,10 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession) updates["invitation_calendar"] = body.invitation_calendar if body.invitation_file is not None: updates["invitation_file"] = body.invitation_file + if body.spellcheck_enabled is not None: + updates["spellcheck_enabled"] = body.spellcheck_enabled + if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language): + updates["spellcheck_language"] = body.spellcheck_language user.settings = updates await db.commit() return _settings_out(user) diff --git a/backend/app/api/spellcheck.py b/backend/app/api/spellcheck.py new file mode 100644 index 0000000..ecb517d --- /dev/null +++ b/backend/app/api/spellcheck.py @@ -0,0 +1,187 @@ +"""Spelling and grammar check for the compose window, backed by LanguageTool. + +The browser sends the text it wants checked here, never to LanguageTool +itself: the engine stays on the internal network, only signed-in users reach +it, and nothing leaves the server. When no engine is configured (or it is +down), the check answers 503 and the compose window simply goes without. +""" + +from __future__ import annotations + +import asyncio +import time +from collections import Counter +from typing import Any + +import httpx +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field + +from app.api.errors import raise_api_error +from app.config import get_settings +from app.identity.deps import CurrentUser, require_csrf + +router = APIRouter(prefix="/spellcheck", tags=["compose"]) + +_TIMEOUT = 10.0 +# A message body is checked as a whole; past this it is a document, not mail. +MAX_TEXT = 20_000 +_MAX_SUGGESTIONS = 6 +# With "auto", LanguageTool settles on ONE language per request, so a message +# mixing Polish and English paragraphs would have every English word flagged as +# a Polish typo. Auto-detected text is therefore checked in chunks, each in its +# own language. A chunk is at least this many characters of consecutive lines: +# a greeting alone ("Witam,") is too short to detect reliably, so it rides with +# the lines after it. +_MIN_CHUNK = 60 +_PARALLEL = 4 + + +class CheckIn(BaseModel): + text: str = Field(max_length=MAX_TEXT) + # "auto" or a LanguageTool code ("pl-PL", "en-GB", "de-DE" ...). + language: str = Field(default="auto", max_length=16) + + +class Issue(BaseModel): + offset: int + length: int + message: str + short_message: str + replacements: list[str] + # spelling | grammar | style: what the underline colour follows. + kind: str + rule_id: str + + +class CheckOut(BaseModel): + language: str # the language checked (detected, with "auto") + language_name: str + issues: list[Issue] + + +class LanguageOut(BaseModel): + code: str + name: str + + +def _base() -> str: + url = get_settings().languagetool_url.rstrip("/") + if not url: + raise_api_error(503, "spellcheck_unavailable", "spell checking is not set up") + return url + + +def _kind(issue_type: str) -> str: + if issue_type == "misspelling": + return "spelling" + if issue_type in ("style", "typographical", "whitespace", "register", "locale-violation"): + return "style" + return "grammar" + + +def chunks(text: str) -> list[tuple[int, str]]: + """Split ``text`` into (offset, chunk) runs of whole lines, each at least + _MIN_CHUNK characters where the text allows; a blank line always ends one.""" + out: list[tuple[int, str]] = [] + start = pos = 0 + for line in text.split("\n"): + end = pos + len(line) + if not line.strip(): + if text[start:pos].strip(): + # Up to the line break before the blank line, not including it. + out.append((start, text[start:pos].rstrip("\n"))) + start = end + 1 + elif end - start >= _MIN_CHUNK: + out.append((start, text[start:end])) + start = end + 1 + pos = end + 1 + if text[start:].strip(): + out.append((start, text[start:])) + return out + + +async def _check_one( + client: httpx.AsyncClient, base: str, text: str, language: str +) -> dict[str, Any]: + resp = await client.post(f"{base}/v2/check", data={"text": text, "language": language}) + resp.raise_for_status() + data: dict[str, Any] = resp.json() + return data + + +def _issues(data: dict[str, Any], shift: int) -> list[Issue]: + return [ + Issue( + offset=int(m["offset"]) + shift, + length=int(m["length"]), + message=str(m.get("message", "")), + short_message=str(m.get("shortMessage", "")), + replacements=[str(r["value"]) for r in (m.get("replacements") or [])][ + :_MAX_SUGGESTIONS + ], + kind=_kind(str((m.get("rule") or {}).get("issueType", ""))), + rule_id=str((m.get("rule") or {}).get("id", "")), + ) + for m in data.get("matches") or [] + ] + + +def _detected(data: dict[str, Any]) -> tuple[str, str]: + lang = data.get("language") or {} + detected = lang.get("detectedLanguage") or lang + return str(detected.get("code", "")), str(detected.get("name", "")) + + +@router.post("", response_model=CheckOut, dependencies=[Depends(require_csrf)]) +async def check(body: CheckIn, user: CurrentUser) -> CheckOut: + base = _base() + if not body.text.strip(): + return CheckOut(language=body.language, language_name="", issues=[]) + parts = chunks(body.text) if body.language == "auto" else [(0, body.text)] + gate = asyncio.Semaphore(_PARALLEL) + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + + async def one(text: str) -> dict[str, Any]: + async with gate: + return await _check_one(client, base, text, body.language) + + results = await asyncio.gather(*(one(text) for _, text in parts)) + except (httpx.HTTPError, ValueError): + raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering") + + issues = [ + i for (shift, _), data in zip(parts, results, strict=True) for i in _issues(data, shift) + ] + # The language reported is the one most of the text was written in. + weight: Counter[tuple[str, str]] = Counter() + for (_, text), data in zip(parts, results, strict=True): + weight[_detected(data)] += len(text) + (code, name), _ = weight.most_common(1)[0] + return CheckOut(language=code or body.language, language_name=name, issues=issues) + + +_languages_cache: tuple[float, list[LanguageOut]] | None = None + + +@router.get("/languages", response_model=list[LanguageOut]) +async def languages(user: CurrentUser) -> list[LanguageOut]: + """What the engine can check, for the language picker in Settings.""" + global _languages_cache + if _languages_cache and time.monotonic() - _languages_cache[0] < 3600: + return _languages_cache[1] + base = _base() + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + resp = await client.get(f"{base}/v2/languages") + resp.raise_for_status() + rows: list[dict[str, Any]] = resp.json() + except (httpx.HTTPError, ValueError): + raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering") + seen: dict[str, str] = {} + for row in rows: + seen.setdefault(str(row["longCode"]), str(row["name"])) + out = sorted((LanguageOut(code=c, name=n) for c, n in seen.items()), key=lambda x: x.name) + _languages_cache = (time.monotonic(), out) + return out diff --git a/backend/app/config.py b/backend/app/config.py index 86b5622..a8654c5 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -68,6 +68,9 @@ class Settings(BaseSettings): # Server-side sender avatars (Gravatar + domain favicon). Fetched by the server # (never the browser) so senders can't see the user's IP or read activity. sender_avatars_enabled: bool = Field(default=True, alias="SENDER_AVATARS_ENABLED") + # Spelling and grammar checking in the compose window (LanguageTool server, + # e.g. http://languagetool:8010). Empty: the check is off for everyone. + languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL") # Auto-purge Trash/Spam messages older than this many days (0 disables). trash_retention_days: int = Field(default=30, alias="TRASH_RETENTION_DAYS") diff --git a/backend/tests/api/test_spellcheck.py b/backend/tests/api/test_spellcheck.py new file mode 100644 index 0000000..9216dfe --- /dev/null +++ b/backend/tests/api/test_spellcheck.py @@ -0,0 +1,182 @@ +"""Spelling/grammar check proxy (compose window -> LanguageTool). + +LanguageTool is replaced by a stub transport, so these run without the engine. +""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest +import pytest_asyncio +from httpx import ASGITransport +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool + +from app.api import spellcheck +from app.config import get_settings +from app.identity.passwords import hash_password +from app.identity.sessions import create_session +from app.main import create_app +from app.models import User + +LT_REPLY: dict[str, Any] = { + "language": { + "code": "auto", + "name": "Auto", + "detectedLanguage": {"code": "pl-PL", "name": "Polish"}, + }, + "matches": [ + { + "offset": 12, + "length": 6, + "message": "Wyraz „prosze” to najczęściej literówka", + "shortMessage": "Literówka", + "replacements": [{"value": "proszę"}], + "rule": {"id": "PL_PROSZE", "issueType": "misspelling"}, + }, + { + "offset": 40, + "length": 11, + "message": "Powtórzony wyraz", + "shortMessage": "", + "replacements": [{"value": v} for v in "abcdefgh"], + "rule": {"id": "WORD_REPEAT", "issueType": "duplication"}, + }, + ], +} + + +@pytest_asyncio.fixture +async def client(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[httpx.AsyncClient]: + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + session = async_sessionmaker(engine, expire_on_commit=False)() + user = User(handle=f"sp{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")) + session.add(user) + await session.commit() + token = await create_session(user.id, "pytest") + async with httpx.AsyncClient( + transport=ASGITransport(app=create_app()), + base_url="http://testserver", + cookies={"session": token, "csrf": "c"}, + headers={"X-CSRF": "c"}, + ) as c: + yield c + await session.execute(delete(User).where(User.id == user.id)) + await session.commit() + await session.close() + await engine.dispose() + + +def _stub(monkeypatch: pytest.MonkeyPatch, handler: Any) -> list[httpx.Request]: + """Route the module's outbound calls to `handler`; return what was sent.""" + seen: list[httpx.Request] = [] + real = httpx.AsyncClient + + def wrapped(request: httpx.Request) -> httpx.Response: + seen.append(request) + return handler(request) + + monkeypatch.setattr( + spellcheck.httpx, + "AsyncClient", + lambda **kw: real(transport=httpx.MockTransport(wrapped), **kw), + ) + monkeypatch.setattr(get_settings(), "languagetool_url", "http://lt.test:8010") + return seen + + +async def test_check_returns_trimmed_typed_issues( + client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + sent = _stub(monkeypatch, lambda _r: httpx.Response(200, json=LT_REPLY)) + resp = await client.post( + "/api/spellcheck", + json={"text": "Dzień dobry prosze o pliki ważne ważne", "language": "auto"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert (body["language"], body["language_name"]) == ("pl-PL", "Polish") + first, second = body["issues"] + assert first["kind"] == "spelling" and first["replacements"] == ["proszę"] + assert second["kind"] == "grammar" and len(second["replacements"]) == 6 + assert b"language=auto" in sent[0].content + + +async def test_unconfigured_or_down_engine_is_a_503( + client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def down(_r: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused") + + _stub(monkeypatch, down) + resp = await client.post("/api/spellcheck", json={"text": "hello"}) + assert resp.json()["error"]["code"] == "spellcheck_unavailable" + + monkeypatch.setattr(get_settings(), "languagetool_url", "") + resp = await client.post("/api/spellcheck", json={"text": "hello"}) + assert resp.status_code == 503 + + +async def test_empty_text_never_reaches_the_engine( + client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + sent = _stub(monkeypatch, lambda _r: httpx.Response(200, json=LT_REPLY)) + resp = await client.post("/api/spellcheck", json={"text": " \n "}) + assert resp.json()["issues"] == [] and sent == [] + + +async def test_settings_store_the_spellcheck_choice(client: httpx.AsyncClient) -> None: + out = (await client.patch("/api/settings", json={"spellcheck_language": "pl-PL"})).json() + assert out["spellcheck_language"] == "pl-PL" and out["spellcheck_enabled"] is True + out = (await client.patch("/api/settings", json={"spellcheck_enabled": False})).json() + assert out["spellcheck_enabled"] is False + bad = (await client.patch("/api/settings", json={"spellcheck_language": "