Deploy: LanguageTool spell checker in production, with optional n-grams

- docker-compose.yml: a `languagetool` service (erikvl87/languagetool:6.8),
  internal only, 1 GB heap, one CPU, a 28 GB memory limit that is room for the
  n-gram page cache (reclaimable, not memory in use), and a low-priority
  warm-up read of the n-gram files at start so first checks hit RAM even on
  mechanical disks. `init` creates $DATA_ROOT/languagetool-ngrams; the app
  containers get LANGUAGETOOL_URL (empty turns the check off).
- `webmail.sh ngrams [langs]` (default en de fr es): deploy/ngrams.py, run in
  the backend image, downloads LanguageTool's n-gram data with resume, checks
  free space first (unpacked is ~2x the zip: ~26 GB for all four), unpacks into
  a temp folder and moves it into place, then recreates the engine to load it.
  languagetool.org refuses Python's default User-Agent, so it sends its own.
- `doctor` reports whether the spell checker answers and which n-gram
  languages are installed.
- docs/nas-install.md and .env.prod.example describe all of it.

Verified locally: German n-grams (1.58 GB -> 3.06 GB) load, and catch what the
rules alone do not ("liegt ihm Schrank" -> im, "Der Patenten" -> Patienten).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deploy/ngrams.py

@@ -0,0 +1,150 @@
+"""Download and unpack LanguageTool n-gram data (run by `webmail.sh ngrams`).
+
+Runs inside the backend image (any Python 3 will do) with the n-gram folder
+mounted at /ngrams; usage: ngrams.py [lang ...]  (default: en de fr es).
+
+Per language: skip it if already installed, check free space, download with
+resume (an 8 GB transfer that drops halfway picks up where it stopped), verify
+the size, unpack into a temporary folder and move it into place in one step —
+LanguageTool never sees half a language — then delete the zip. Python's zipfile
+rather than unzip: BusyBox unzip cannot be trusted with an 8 GB ZIP64 archive.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+import time
+import urllib.request
+import zipfile
+from pathlib import Path
+
+BASE = "https://languagetool.org/download/ngram-data/"
+FILES = {
+    "en": "ngrams-en-20150817.zip",
+    "de": "ngrams-de-20150819.zip",
+    "fr": "ngrams-fr-20150913.zip",
+    "es": "ngrams-es-20150915.zip",
+    "nl": "ngrams-nl-20181229.zip",
+}
+ROOT = Path(os.environ.get("NGRAMS_DIR", "/ngrams"))
+CHUNK = 1 << 20
+# languagetool.org answers Python's default User-Agent with 403 Forbidden.
+HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; webmail-ngrams/1.0)"}
+
+
+def say(msg: str) -> None:
+    print(msg, flush=True)
+
+
+def gib(n: float) -> str:
+    return f"{n / 1024**3:.2f} GB"
+
+
+def installed(lang: str) -> bool:
+    d = ROOT / lang
+    return all((d / f"{n}grams").is_dir() for n in (1, 2, 3))
+
+
+def remote_size(url: str) -> int:
+    req = urllib.request.Request(url, method="HEAD", headers=HEADERS)
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        return int(resp.headers["Content-Length"])
+
+
+def download(url: str, dest: Path, total: int) -> None:
+    have = dest.stat().st_size if dest.exists() else 0
+    if have == total:
+        say("  already downloaded")
+        return
+    req = urllib.request.Request(url, headers=HEADERS)
+    if have:
+        req.add_header("Range", f"bytes={have}-")
+        say(f"  resuming at {gib(have)}")
+    with (
+        urllib.request.urlopen(req, timeout=60) as resp,
+        open(dest, "ab" if have else "wb") as out,
+    ):
+        if have and resp.status != 206:  # the server ignored the range: start over
+            out.truncate(0)
+            have = 0
+        done, last, started = have, -1, time.monotonic()
+        while chunk := resp.read(CHUNK):
+            out.write(chunk)
+            done += len(chunk)
+            pct = done * 100 // total
+            if pct // 5 != last // 5:
+                rate = (done - have) / max(time.monotonic() - started, 0.001)
+                say(
+                    f"  {pct:3d}%  {gib(done)} of {gib(total)}  ({rate / 1024**2:.1f} MB/s)"
+                )
+                last = pct
+    if dest.stat().st_size != total:
+        raise SystemExit(
+            f"  download incomplete ({gib(dest.stat().st_size)} of {gib(total)}) — run again to resume"
+        )
+
+
+def unpack(zip_path: Path, lang: str) -> None:
+    tmp = ROOT / f".unpacking-{lang}"
+    shutil.rmtree(tmp, ignore_errors=True)
+    say("  unpacking…")
+    with zipfile.ZipFile(zip_path) as zf:
+        zf.extractall(tmp)
+    # The archive holds "<lang>/1grams …"; tolerate a different top folder.
+    src = next((p.parent for p in tmp.rglob("1grams") if p.is_dir()), None)
+    if src is None:
+        shutil.rmtree(tmp, ignore_errors=True)
+        raise SystemExit(
+            f"  {zip_path.name} has no 1grams folder — not LanguageTool n-gram data?"
+        )
+    for p in [src, *src.rglob("*")]:  # readable by the languagetool user (uid 100)
+        p.chmod(0o755 if p.is_dir() else 0o644)
+    final = ROOT / lang
+    shutil.rmtree(final, ignore_errors=True)
+    src.rename(final)
+    shutil.rmtree(tmp, ignore_errors=True)
+
+
+def size_of(path: Path) -> int:
+    return sum(p.stat().st_size for p in path.rglob("*") if p.is_file())
+
+
+def main(langs: list[str]) -> None:
+    ROOT.mkdir(parents=True, exist_ok=True)
+    unknown = [lang for lang in langs if lang not in FILES]
+    if unknown:
+        raise SystemExit(
+            f"no n-gram data for: {' '.join(unknown)} (available: {' '.join(FILES)})"
+        )
+    for lang in langs:
+        say(f"== {lang}")
+        if installed(lang):
+            say(f"  already installed ({gib(size_of(ROOT / lang))})")
+            continue
+        url = BASE + FILES[lang]
+        total = remote_size(url)
+        zip_path = ROOT / FILES[lang]
+        have = zip_path.stat().st_size if zip_path.exists() else 0
+        # Room for the rest of the zip plus the unpacked copy, which is about
+        # twice the zip's size (German: 1.58 GB download, 3.06 GB unpacked).
+        need = (total - have) + total * 2
+        free = shutil.disk_usage(ROOT).free
+        if free < need:
+            raise SystemExit(f"  not enough space: need ~{gib(need)}, {gib(free)} free")
+        say(f"  downloading {FILES[lang]} ({gib(total)})")
+        download(url, zip_path, total)
+        unpack(zip_path, lang)
+        zip_path.unlink()
+        say(f"  installed: {gib(size_of(ROOT / lang))}")
+    say(
+        "done: "
+        + ", ".join(
+            f"{lang} ({gib(size_of(ROOT / lang))})" for lang in FILES if installed(lang)
+        )
+    )
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:] or ["en", "de", "fr", "es"])

deploy/webmail.sh

@@ -12,6 +12,7 @@
 #   ./deploy/webmail.sh backup          take a verified database dump now
 #   ./deploy/webmail.sh restore <file>  DESTRUCTIVE: restore a dump
 #   ./deploy/webmail.sh reindex [args]  rebuild the search index from Postgres
+#   ./deploy/webmail.sh ngrams [langs]  download spell-check n-gram data (default: en de fr es)
 #   ./deploy/webmail.sh psql            open a database shell
 #   ./deploy/webmail.sh shell [svc]     shell inside a container (default: api)
 #   ./deploy/webmail.sh show-key        print MASTER_KEY so you can back it up
@@ -418,6 +419,23 @@ print('AUTHORISED')
         ylw "  antispam is not in ENABLED_PLUGINS — no spam filtering on this install" ;;
     esac
 
+    # Spelling/grammar check: is the engine answering, and with which n-grams?
+    lt="$(compose exec -T api python -c "
+import os, urllib.request
+url = os.environ.get('LANGUAGETOOL_URL', '')
+print('OFF' if not url else urllib.request.urlopen(url + '/v2/languages', timeout=5).status)
+" 2>&1 | tail -1)" || true
+    ngrams=""
+    for d in "${DATA_ROOT}/languagetool-ngrams"/*/; do
+        [ -d "${d}3grams" ] && ngrams="${ngrams} $(basename "$d")"
+    done
+    case "$lt" in
+        200) grn "  spell check engine answering; n-grams:${ngrams:- none (optional: webmail.sh ngrams)}" ;;
+        OFF) ylw "  spell check is off (LANGUAGETOOL_URL is empty)" ;;
+        *)   ylw "  spell check engine not answering — the compose window falls back to the browser's checker"
+             printf '    %s\n' "$lt" ;;
+    esac
+
     if command -v ss >/dev/null 2>&1; then
         local p
         for p in "WEB_PORT:${WEB_PORT:-8088}" "ADMIN_PORT:${ADMIN_PORT:-8089}"; do
@@ -675,6 +693,23 @@ cmd_show_key() {
 }
 
 cmd_reindex() { require_env; compose run --rm api reindex "$@"; }
+
+# Optional n-gram data for the spell checker (~13 GB download for en de fr es):
+# lets LanguageTool catch real words used wrongly ("loose weight", "das er
+# kommt"). Downloaded and unpacked by deploy/ngrams.py inside the backend image
+# (it resumes an interrupted download — just run it again), then the engine is
+# recreated to load it. Polish has no n-gram data; it is checked as before.
+cmd_ngrams() {
+    require_env
+    local dir="${DATA_ROOT}/languagetool-ngrams"
+    mkdir -p "$dir" || die "cannot create ${dir} (run as the owner of DATA_ROOT, or as root)"
+    echo "== downloading n-gram data into ${dir} (${*:-en de fr es})"
+    compose run --rm --no-deps -T --user 0 -v "${dir}:/ngrams" --entrypoint python api - "$@" \
+        < "${SCRIPT_DIR}/ngrams.py" || die "n-gram download failed — run the same command again to resume"
+    echo "== restarting the spell checker to load it"
+    compose up -d --no-deps --force-recreate languagetool
+    grn "n-grams installed. It reads them into memory in the background over the next minutes."
+}
 cmd_psql()    { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; }
 cmd_shell()   { require_env; compose exec "${1:-api}" /bin/sh; }
 
@@ -700,6 +735,7 @@ case "$cmd" in
     backup)   cmd_backup "$@" ;;
     restore)  cmd_restore "$@" ;;
     reindex)  cmd_reindex "$@" ;;
+    ngrams)   cmd_ngrams "$@" ;;
     show-key) cmd_show_key "$@" ;;
     psql)     cmd_psql "$@" ;;
     shell)    cmd_shell "$@" ;;

docker-compose.yml

@@ -69,6 +69,8 @@ x-backend-env: &backend-env
   # DOES NOT — the controller answers 401, the plugin fails open, and the only
   # visible symptom is "bayes": null on the stats endpoint.
   RSPAMD_CONTROLLER_PASSWORD: ${RSPAMD_CONTROLLER_PASSWORD}
+  # Spelling and grammar check in the compose window. Empty turns it off.
+  LANGUAGETOOL_URL: ${LANGUAGETOOL_URL-http://languagetool:8010}
   LOG_LEVEL: ${LOG_LEVEL:-INFO}
   UNDO_SECONDS: ${UNDO_SECONDS:-10}
   SESSION_TTL_DAYS: ${SESSION_TTL_DAYS:-30}
@@ -102,7 +104,8 @@ services:
       - -c
       - |
         set -e
-        mkdir -p /mnt/postgres /mnt/blobs /mnt/backups /mnt/redis /mnt/meili /mnt/state
+        mkdir -p /mnt/postgres /mnt/blobs /mnt/backups /mnt/redis /mnt/meili /mnt/state \
+          /mnt/languagetool-ngrams
         # Only recurse when ownership is actually wrong: `chown -R` over a large
         # blob store on every start would be minutes of pointless disk churn.
         [ "$$(stat -c %u /mnt/blobs)" = "10001" ] || chown -R 10001:10001 /mnt/blobs
@@ -262,6 +265,50 @@ services:
     mem_limit: ${RSPAMD_MEM_LIMIT:-256m}
     logging: *logging
 
+  # Spelling and grammar checking for the compose window. The app's API is its
+  # only client (POST /api/spellcheck); it is never published to the host.
+  #
+  # Optional n-gram data (`webmail.sh ngrams`: ~13 GB download, ~26 GB unpacked
+  # for en/de/fr/es) lets it catch real words used wrongly — "the affect was
+  # huge", "liegt ihm Schrank" — beyond its hand-written rules. It lives on disk
+  # and is read through the page cache, which Docker charges to this container:
+  # hence a memory limit far above the 1 GB Java heap, enough to keep all four
+  # languages cached. That room is file cache the kernel takes back whenever
+  # anything else needs memory, not memory in use. One CPU at most, so a burst
+  # of checking never slows sync, search or the API.
+  languagetool:
+    image: erikvl87/languagetool:6.8
+    # Read the n-gram files once at start (low priority, in the background), so
+    # even the first checks after a restart hit RAM instead of the disks.
+    command:
+      - bash
+      - -c
+      - |
+        if [ -d /ngrams ] && [ -n "$$(ls -A /ngrams 2>/dev/null)" ]; then
+          ( nice -n 19 find /ngrams -type f -exec cat {} + > /dev/null 2>&1 & )
+        fi
+        exec bash start.sh
+    environment:
+      Java_Xms: 256m
+      Java_Xmx: ${LANGUAGETOOL_HEAP:-1g}
+      # Without the data the folder is empty and LanguageTool simply skips it.
+      langtool_languageModel: /ngrams
+      TZ: ${TZ:-UTC}
+    volumes:
+      - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/languagetool-ngrams:/ngrams:ro
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8010/v2/languages || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
+    depends_on:
+      init: { condition: service_completed_successfully }
+    restart: unless-stopped
+    mem_limit: ${LANGUAGETOOL_MEM_LIMIT:-28g}
+    cpus: ${LANGUAGETOOL_CPUS:-1.0}
+    logging: *logging
+
   # Bayes tokens and fuzzy hashes. Separate from the app's redis on purpose
   # (§C.2): this data is derived and retrainable, so it trades durability for
   # room to grow — RDB snapshots every 5 min, and an eviction policy, where the

docker/.env.prod.example

@@ -59,6 +59,18 @@ RSPAMD_MEM_LIMIT=256m
 RSPAMD_REDIS_MEM_LIMIT=160m
 RSPAMD_REDIS_MAXMEMORY=128mb
 
+# --- Spelling and grammar check (LanguageTool) -------------------------------
+# The compose window's spell checker. Leave empty to turn it off for everyone;
+# unset uses the bundled container (http://languagetool:8010).
+#LANGUAGETOOL_URL=
+# Java heap. 1g covers a household or small team.
+LANGUAGETOOL_HEAP=1g
+# Mostly file cache for the optional n-gram data (`webmail.sh ngrams`, ~26 GB
+# unpacked for en/de/fr/es), which the kernel gives back under memory pressure.
+# Without n-grams it never comes near this. Lower it on a small NAS.
+LANGUAGETOOL_MEM_LIMIT=28g
+LANGUAGETOOL_CPUS=1.0
+
 # --- Search -----------------------------------------------------------------
 # pg   = Postgres full-text. No extra container, no extra RAM. Default.
 # meili = Meilisearch: typo tolerance, better ranking, faster on large mailboxes.

docs/nas-install.md

@@ -606,6 +606,38 @@ Its Bayes database lives in `$DATA_ROOT/rspamd-redis`. Like the search index,
 that is derived data: it rides along in backups but never needs restoring, and
 retraining rebuilds it.
 
+### The spell checker (LanguageTool)
+
+`languagetool` checks spelling and grammar in the compose window, in some 30
+languages. Like the anti-spam engine nothing waits for it: when it is down the
+compose window falls back to the browser's own spell checker. It is never
+published to the host — the app's API is its only client.
+
+It needs about **1.2 GB of RAM** (`LANGUAGETOOL_HEAP=1g` plus the JVM) and is
+capped at one CPU core (`LANGUAGETOOL_CPUS`), so checking can never slow sync,
+search or the API.
+
+**Optional n-gram data** teaches it to catch real words used wrongly — "the
+affect was huge", "Das Buch liegt ihm Schrank" — beyond its hand-written rules
+(English has ~780 such word pairs, German ~70). Install it once:
+
+```bash
+./deploy/webmail.sh ngrams            # en de fr es — or name languages: ngrams en de
+```
+
+That downloads about **13 GB** (English 8.4 GB, German/French/Spanish ~1.6 GB
+each) into `$DATA_ROOT/languagetool-ngrams`, unpacks it to about **26 GB**, and
+restarts the engine to load it. An interrupted download resumes when you run
+the command again. There is no n-gram data for Polish; Polish text is checked
+exactly as without it.
+
+The data is read from disk on demand and cached by the kernel, so the memory
+limit (`LANGUAGETOOL_MEM_LIMIT=28g`) is room for that cache, not memory in use.
+At start the container reads the files once in the background, at low
+priority, so even the first checks after a restart are served from RAM. On
+mechanical disks that warm-up takes several minutes; an NVMe cache in front of
+the volume makes it — and any later cache misses — fast.
+
 Two NAS-specific realities worth knowing:
 
 - **Disk hibernation won't happen.** Active accounts hold IMAP IDLE connections
@@ -839,6 +871,7 @@ directories are root-owned.
 ./deploy/webmail.sh backup           verified dump, right now
 ./deploy/webmail.sh restore <dump>   restore a dump (asks for confirmation)
 ./deploy/webmail.sh reindex [--purge] rebuild the search index
+./deploy/webmail.sh ngrams [langs]   spell-check n-gram data (default: en de fr es)
 ./deploy/webmail.sh psql             database shell
 ./deploy/webmail.sh shell [svc]      shell in a container
 ```