Deploy: LanguageTool spell checker in production, with optional n-grams

- docker-compose.yml: a `languagetool` service (erikvl87/languagetool:6.8),
  internal only, 1 GB heap, one CPU, a 28 GB memory limit that is room for the
  n-gram page cache (reclaimable, not memory in use), and a low-priority
  warm-up read of the n-gram files at start so first checks hit RAM even on
  mechanical disks. `init` creates $DATA_ROOT/languagetool-ngrams; the app
  containers get LANGUAGETOOL_URL (empty turns the check off).
- `webmail.sh ngrams [langs]` (default en de fr es): deploy/ngrams.py, run in
  the backend image, downloads LanguageTool's n-gram data with resume, checks
  free space first (unpacked is ~2x the zip: ~26 GB for all four), unpacks into
  a temp folder and moves it into place, then recreates the engine to load it.
  languagetool.org refuses Python's default User-Agent, so it sends its own.
- `doctor` reports whether the spell checker answers and which n-gram
  languages are installed.
- docs/nas-install.md and .env.prod.example describe all of it.

Verified locally: German n-grams (1.58 GB -> 3.06 GB) load, and catch what the
rules alone do not ("liegt ihm Schrank" -> im, "Der Patenten" -> Patienten).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deploy/ngrams.py

@@ -0,0 +1,150 @@
+"""Download and unpack LanguageTool n-gram data (run by `webmail.sh ngrams`).
+
+Runs inside the backend image (any Python 3 will do) with the n-gram folder
+mounted at /ngrams; usage: ngrams.py [lang ...]  (default: en de fr es).
+
+Per language: skip it if already installed, check free space, download with
+resume (an 8 GB transfer that drops halfway picks up where it stopped), verify
+the size, unpack into a temporary folder and move it into place in one step —
+LanguageTool never sees half a language — then delete the zip. Python's zipfile
+rather than unzip: BusyBox unzip cannot be trusted with an 8 GB ZIP64 archive.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+import time
+import urllib.request
+import zipfile
+from pathlib import Path
+
+BASE = "https://languagetool.org/download/ngram-data/"
+FILES = {
+    "en": "ngrams-en-20150817.zip",
+    "de": "ngrams-de-20150819.zip",
+    "fr": "ngrams-fr-20150913.zip",
+    "es": "ngrams-es-20150915.zip",
+    "nl": "ngrams-nl-20181229.zip",
+}
+ROOT = Path(os.environ.get("NGRAMS_DIR", "/ngrams"))
+CHUNK = 1 << 20
+# languagetool.org answers Python's default User-Agent with 403 Forbidden.
+HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; webmail-ngrams/1.0)"}
+
+
+def say(msg: str) -> None:
+    print(msg, flush=True)
+
+
+def gib(n: float) -> str:
+    return f"{n / 1024**3:.2f} GB"
+
+
+def installed(lang: str) -> bool:
+    d = ROOT / lang
+    return all((d / f"{n}grams").is_dir() for n in (1, 2, 3))
+
+
+def remote_size(url: str) -> int:
+    req = urllib.request.Request(url, method="HEAD", headers=HEADERS)
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        return int(resp.headers["Content-Length"])
+
+
+def download(url: str, dest: Path, total: int) -> None:
+    have = dest.stat().st_size if dest.exists() else 0
+    if have == total:
+        say("  already downloaded")
+        return
+    req = urllib.request.Request(url, headers=HEADERS)
+    if have:
+        req.add_header("Range", f"bytes={have}-")
+        say(f"  resuming at {gib(have)}")
+    with (
+        urllib.request.urlopen(req, timeout=60) as resp,
+        open(dest, "ab" if have else "wb") as out,
+    ):
+        if have and resp.status != 206:  # the server ignored the range: start over
+            out.truncate(0)
+            have = 0
+        done, last, started = have, -1, time.monotonic()
+        while chunk := resp.read(CHUNK):
+            out.write(chunk)
+            done += len(chunk)
+            pct = done * 100 // total
+            if pct // 5 != last // 5:
+                rate = (done - have) / max(time.monotonic() - started, 0.001)
+                say(
+                    f"  {pct:3d}%  {gib(done)} of {gib(total)}  ({rate / 1024**2:.1f} MB/s)"
+                )
+                last = pct
+    if dest.stat().st_size != total:
+        raise SystemExit(
+            f"  download incomplete ({gib(dest.stat().st_size)} of {gib(total)}) — run again to resume"
+        )
+
+
+def unpack(zip_path: Path, lang: str) -> None:
+    tmp = ROOT / f".unpacking-{lang}"
+    shutil.rmtree(tmp, ignore_errors=True)
+    say("  unpacking…")
+    with zipfile.ZipFile(zip_path) as zf:
+        zf.extractall(tmp)
+    # The archive holds "<lang>/1grams …"; tolerate a different top folder.
+    src = next((p.parent for p in tmp.rglob("1grams") if p.is_dir()), None)
+    if src is None:
+        shutil.rmtree(tmp, ignore_errors=True)
+        raise SystemExit(
+            f"  {zip_path.name} has no 1grams folder — not LanguageTool n-gram data?"
+        )
+    for p in [src, *src.rglob("*")]:  # readable by the languagetool user (uid 100)
+        p.chmod(0o755 if p.is_dir() else 0o644)
+    final = ROOT / lang
+    shutil.rmtree(final, ignore_errors=True)
+    src.rename(final)
+    shutil.rmtree(tmp, ignore_errors=True)
+
+
+def size_of(path: Path) -> int:
+    return sum(p.stat().st_size for p in path.rglob("*") if p.is_file())
+
+
+def main(langs: list[str]) -> None:
+    ROOT.mkdir(parents=True, exist_ok=True)
+    unknown = [lang for lang in langs if lang not in FILES]
+    if unknown:
+        raise SystemExit(
+            f"no n-gram data for: {' '.join(unknown)} (available: {' '.join(FILES)})"
+        )
+    for lang in langs:
+        say(f"== {lang}")
+        if installed(lang):
+            say(f"  already installed ({gib(size_of(ROOT / lang))})")
+            continue
+        url = BASE + FILES[lang]
+        total = remote_size(url)
+        zip_path = ROOT / FILES[lang]
+        have = zip_path.stat().st_size if zip_path.exists() else 0
+        # Room for the rest of the zip plus the unpacked copy, which is about
+        # twice the zip's size (German: 1.58 GB download, 3.06 GB unpacked).
+        need = (total - have) + total * 2
+        free = shutil.disk_usage(ROOT).free
+        if free < need:
+            raise SystemExit(f"  not enough space: need ~{gib(need)}, {gib(free)} free")
+        say(f"  downloading {FILES[lang]} ({gib(total)})")
+        download(url, zip_path, total)
+        unpack(zip_path, lang)
+        zip_path.unlink()
+        say(f"  installed: {gib(size_of(ROOT / lang))}")
+    say(
+        "done: "
+        + ", ".join(
+            f"{lang} ({gib(size_of(ROOT / lang))})" for lang in FILES if installed(lang)
+        )
+    )
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:] or ["en", "de", "fr", "es"])

deploy/webmail.sh

@@ -12,6 +12,7 @@
 #   ./deploy/webmail.sh backup          take a verified database dump now
 #   ./deploy/webmail.sh restore <file>  DESTRUCTIVE: restore a dump
 #   ./deploy/webmail.sh reindex [args]  rebuild the search index from Postgres
+#   ./deploy/webmail.sh ngrams [langs]  download spell-check n-gram data (default: en de fr es)
 #   ./deploy/webmail.sh psql            open a database shell
 #   ./deploy/webmail.sh shell [svc]     shell inside a container (default: api)
 #   ./deploy/webmail.sh show-key        print MASTER_KEY so you can back it up
@@ -418,6 +419,23 @@ print('AUTHORISED')
         ylw "  antispam is not in ENABLED_PLUGINS — no spam filtering on this install" ;;
     esac
 
+    # Spelling/grammar check: is the engine answering, and with which n-grams?
+    lt="$(compose exec -T api python -c "
+import os, urllib.request
+url = os.environ.get('LANGUAGETOOL_URL', '')
+print('OFF' if not url else urllib.request.urlopen(url + '/v2/languages', timeout=5).status)
+" 2>&1 | tail -1)" || true
+    ngrams=""
+    for d in "${DATA_ROOT}/languagetool-ngrams"/*/; do
+        [ -d "${d}3grams" ] && ngrams="${ngrams} $(basename "$d")"
+    done
+    case "$lt" in
+        200) grn "  spell check engine answering; n-grams:${ngrams:- none (optional: webmail.sh ngrams)}" ;;
+        OFF) ylw "  spell check is off (LANGUAGETOOL_URL is empty)" ;;
+        *)   ylw "  spell check engine not answering — the compose window falls back to the browser's checker"
+             printf '    %s\n' "$lt" ;;
+    esac
+
     if command -v ss >/dev/null 2>&1; then
         local p
         for p in "WEB_PORT:${WEB_PORT:-8088}" "ADMIN_PORT:${ADMIN_PORT:-8089}"; do
@@ -675,6 +693,23 @@ cmd_show_key() {
 }
 
 cmd_reindex() { require_env; compose run --rm api reindex "$@"; }
+
+# Optional n-gram data for the spell checker (~13 GB download for en de fr es):
+# lets LanguageTool catch real words used wrongly ("loose weight", "das er
+# kommt"). Downloaded and unpacked by deploy/ngrams.py inside the backend image
+# (it resumes an interrupted download — just run it again), then the engine is
+# recreated to load it. Polish has no n-gram data; it is checked as before.
+cmd_ngrams() {
+    require_env
+    local dir="${DATA_ROOT}/languagetool-ngrams"
+    mkdir -p "$dir" || die "cannot create ${dir} (run as the owner of DATA_ROOT, or as root)"
+    echo "== downloading n-gram data into ${dir} (${*:-en de fr es})"
+    compose run --rm --no-deps -T --user 0 -v "${dir}:/ngrams" --entrypoint python api - "$@" \
+        < "${SCRIPT_DIR}/ngrams.py" || die "n-gram download failed — run the same command again to resume"
+    echo "== restarting the spell checker to load it"
+    compose up -d --no-deps --force-recreate languagetool
+    grn "n-grams installed. It reads them into memory in the background over the next minutes."
+}
 cmd_psql()    { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; }
 cmd_shell()   { require_env; compose exec "${1:-api}" /bin/sh; }
 
@@ -700,6 +735,7 @@ case "$cmd" in
     backup)   cmd_backup "$@" ;;
     restore)  cmd_restore "$@" ;;
     reindex)  cmd_reindex "$@" ;;
+    ngrams)   cmd_ngrams "$@" ;;
     show-key) cmd_show_key "$@" ;;
     psql)     cmd_psql "$@" ;;
     shell)    cmd_shell "$@" ;;