diff --git a/deploy/ngrams.py b/deploy/ngrams.py new file mode 100644 index 0000000..b6a1fab --- /dev/null +++ b/deploy/ngrams.py @@ -0,0 +1,150 @@ +"""Download and unpack LanguageTool n-gram data (run by `webmail.sh ngrams`). + +Runs inside the backend image (any Python 3 will do) with the n-gram folder +mounted at /ngrams; usage: ngrams.py [lang ...] (default: en de fr es). + +Per language: skip it if already installed, check free space, download with +resume (an 8 GB transfer that drops halfway picks up where it stopped), verify +the size, unpack into a temporary folder and move it into place in one step — +LanguageTool never sees half a language — then delete the zip. Python's zipfile +rather than unzip: BusyBox unzip cannot be trusted with an 8 GB ZIP64 archive. +""" + +from __future__ import annotations + +import os +import shutil +import sys +import time +import urllib.request +import zipfile +from pathlib import Path + +BASE = "https://languagetool.org/download/ngram-data/" +FILES = { + "en": "ngrams-en-20150817.zip", + "de": "ngrams-de-20150819.zip", + "fr": "ngrams-fr-20150913.zip", + "es": "ngrams-es-20150915.zip", + "nl": "ngrams-nl-20181229.zip", +} +ROOT = Path(os.environ.get("NGRAMS_DIR", "/ngrams")) +CHUNK = 1 << 20 +# languagetool.org answers Python's default User-Agent with 403 Forbidden. +HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; webmail-ngrams/1.0)"} + + +def say(msg: str) -> None: + print(msg, flush=True) + + +def gib(n: float) -> str: + return f"{n / 1024**3:.2f} GB" + + +def installed(lang: str) -> bool: + d = ROOT / lang + return all((d / f"{n}grams").is_dir() for n in (1, 2, 3)) + + +def remote_size(url: str) -> int: + req = urllib.request.Request(url, method="HEAD", headers=HEADERS) + with urllib.request.urlopen(req, timeout=60) as resp: + return int(resp.headers["Content-Length"]) + + +def download(url: str, dest: Path, total: int) -> None: + have = dest.stat().st_size if dest.exists() else 0 + if have == total: + say(" already downloaded") + return + req = urllib.request.Request(url, headers=HEADERS) + if have: + req.add_header("Range", f"bytes={have}-") + say(f" resuming at {gib(have)}") + with ( + urllib.request.urlopen(req, timeout=60) as resp, + open(dest, "ab" if have else "wb") as out, + ): + if have and resp.status != 206: # the server ignored the range: start over + out.truncate(0) + have = 0 + done, last, started = have, -1, time.monotonic() + while chunk := resp.read(CHUNK): + out.write(chunk) + done += len(chunk) + pct = done * 100 // total + if pct // 5 != last // 5: + rate = (done - have) / max(time.monotonic() - started, 0.001) + say( + f" {pct:3d}% {gib(done)} of {gib(total)} ({rate / 1024**2:.1f} MB/s)" + ) + last = pct + if dest.stat().st_size != total: + raise SystemExit( + f" download incomplete ({gib(dest.stat().st_size)} of {gib(total)}) — run again to resume" + ) + + +def unpack(zip_path: Path, lang: str) -> None: + tmp = ROOT / f".unpacking-{lang}" + shutil.rmtree(tmp, ignore_errors=True) + say(" unpacking…") + with zipfile.ZipFile(zip_path) as zf: + zf.extractall(tmp) + # The archive holds "/1grams …"; tolerate a different top folder. + src = next((p.parent for p in tmp.rglob("1grams") if p.is_dir()), None) + if src is None: + shutil.rmtree(tmp, ignore_errors=True) + raise SystemExit( + f" {zip_path.name} has no 1grams folder — not LanguageTool n-gram data?" + ) + for p in [src, *src.rglob("*")]: # readable by the languagetool user (uid 100) + p.chmod(0o755 if p.is_dir() else 0o644) + final = ROOT / lang + shutil.rmtree(final, ignore_errors=True) + src.rename(final) + shutil.rmtree(tmp, ignore_errors=True) + + +def size_of(path: Path) -> int: + return sum(p.stat().st_size for p in path.rglob("*") if p.is_file()) + + +def main(langs: list[str]) -> None: + ROOT.mkdir(parents=True, exist_ok=True) + unknown = [lang for lang in langs if lang not in FILES] + if unknown: + raise SystemExit( + f"no n-gram data for: {' '.join(unknown)} (available: {' '.join(FILES)})" + ) + for lang in langs: + say(f"== {lang}") + if installed(lang): + say(f" already installed ({gib(size_of(ROOT / lang))})") + continue + url = BASE + FILES[lang] + total = remote_size(url) + zip_path = ROOT / FILES[lang] + have = zip_path.stat().st_size if zip_path.exists() else 0 + # Room for the rest of the zip plus the unpacked copy, which is about + # twice the zip's size (German: 1.58 GB download, 3.06 GB unpacked). + need = (total - have) + total * 2 + free = shutil.disk_usage(ROOT).free + if free < need: + raise SystemExit(f" not enough space: need ~{gib(need)}, {gib(free)} free") + say(f" downloading {FILES[lang]} ({gib(total)})") + download(url, zip_path, total) + unpack(zip_path, lang) + zip_path.unlink() + say(f" installed: {gib(size_of(ROOT / lang))}") + say( + "done: " + + ", ".join( + f"{lang} ({gib(size_of(ROOT / lang))})" for lang in FILES if installed(lang) + ) + ) + + +if __name__ == "__main__": + main(sys.argv[1:] or ["en", "de", "fr", "es"]) diff --git a/deploy/webmail.sh b/deploy/webmail.sh index 9369f5c..0aefc16 100755 --- a/deploy/webmail.sh +++ b/deploy/webmail.sh @@ -12,6 +12,7 @@ # ./deploy/webmail.sh backup take a verified database dump now # ./deploy/webmail.sh restore DESTRUCTIVE: restore a dump # ./deploy/webmail.sh reindex [args] rebuild the search index from Postgres +# ./deploy/webmail.sh ngrams [langs] download spell-check n-gram data (default: en de fr es) # ./deploy/webmail.sh psql open a database shell # ./deploy/webmail.sh shell [svc] shell inside a container (default: api) # ./deploy/webmail.sh show-key print MASTER_KEY so you can back it up @@ -418,6 +419,23 @@ print('AUTHORISED') ylw " antispam is not in ENABLED_PLUGINS — no spam filtering on this install" ;; esac + # Spelling/grammar check: is the engine answering, and with which n-grams? + lt="$(compose exec -T api python -c " +import os, urllib.request +url = os.environ.get('LANGUAGETOOL_URL', '') +print('OFF' if not url else urllib.request.urlopen(url + '/v2/languages', timeout=5).status) +" 2>&1 | tail -1)" || true + ngrams="" + for d in "${DATA_ROOT}/languagetool-ngrams"/*/; do + [ -d "${d}3grams" ] && ngrams="${ngrams} $(basename "$d")" + done + case "$lt" in + 200) grn " spell check engine answering; n-grams:${ngrams:- none (optional: webmail.sh ngrams)}" ;; + OFF) ylw " spell check is off (LANGUAGETOOL_URL is empty)" ;; + *) ylw " spell check engine not answering — the compose window falls back to the browser's checker" + printf ' %s\n' "$lt" ;; + esac + if command -v ss >/dev/null 2>&1; then local p for p in "WEB_PORT:${WEB_PORT:-8088}" "ADMIN_PORT:${ADMIN_PORT:-8089}"; do @@ -675,6 +693,23 @@ cmd_show_key() { } cmd_reindex() { require_env; compose run --rm api reindex "$@"; } + +# Optional n-gram data for the spell checker (~13 GB download for en de fr es): +# lets LanguageTool catch real words used wrongly ("loose weight", "das er +# kommt"). Downloaded and unpacked by deploy/ngrams.py inside the backend image +# (it resumes an interrupted download — just run it again), then the engine is +# recreated to load it. Polish has no n-gram data; it is checked as before. +cmd_ngrams() { + require_env + local dir="${DATA_ROOT}/languagetool-ngrams" + mkdir -p "$dir" || die "cannot create ${dir} (run as the owner of DATA_ROOT, or as root)" + echo "== downloading n-gram data into ${dir} (${*:-en de fr es})" + compose run --rm --no-deps -T --user 0 -v "${dir}:/ngrams" --entrypoint python api - "$@" \ + < "${SCRIPT_DIR}/ngrams.py" || die "n-gram download failed — run the same command again to resume" + echo "== restarting the spell checker to load it" + compose up -d --no-deps --force-recreate languagetool + grn "n-grams installed. It reads them into memory in the background over the next minutes." +} cmd_psql() { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; } cmd_shell() { require_env; compose exec "${1:-api}" /bin/sh; } @@ -700,6 +735,7 @@ case "$cmd" in backup) cmd_backup "$@" ;; restore) cmd_restore "$@" ;; reindex) cmd_reindex "$@" ;; + ngrams) cmd_ngrams "$@" ;; show-key) cmd_show_key "$@" ;; psql) cmd_psql "$@" ;; shell) cmd_shell "$@" ;; diff --git a/docker-compose.yml b/docker-compose.yml index ff839c4..b9a7820 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,6 +69,8 @@ x-backend-env: &backend-env # DOES NOT — the controller answers 401, the plugin fails open, and the only # visible symptom is "bayes": null on the stats endpoint. RSPAMD_CONTROLLER_PASSWORD: ${RSPAMD_CONTROLLER_PASSWORD} + # Spelling and grammar check in the compose window. Empty turns it off. + LANGUAGETOOL_URL: ${LANGUAGETOOL_URL-http://languagetool:8010} LOG_LEVEL: ${LOG_LEVEL:-INFO} UNDO_SECONDS: ${UNDO_SECONDS:-10} SESSION_TTL_DAYS: ${SESSION_TTL_DAYS:-30} @@ -102,7 +104,8 @@ services: - -c - | set -e - mkdir -p /mnt/postgres /mnt/blobs /mnt/backups /mnt/redis /mnt/meili /mnt/state + mkdir -p /mnt/postgres /mnt/blobs /mnt/backups /mnt/redis /mnt/meili /mnt/state \ + /mnt/languagetool-ngrams # Only recurse when ownership is actually wrong: `chown -R` over a large # blob store on every start would be minutes of pointless disk churn. [ "$$(stat -c %u /mnt/blobs)" = "10001" ] || chown -R 10001:10001 /mnt/blobs @@ -262,6 +265,50 @@ services: mem_limit: ${RSPAMD_MEM_LIMIT:-256m} logging: *logging + # Spelling and grammar checking for the compose window. The app's API is its + # only client (POST /api/spellcheck); it is never published to the host. + # + # Optional n-gram data (`webmail.sh ngrams`: ~13 GB download, ~26 GB unpacked + # for en/de/fr/es) lets it catch real words used wrongly — "the affect was + # huge", "liegt ihm Schrank" — beyond its hand-written rules. It lives on disk + # and is read through the page cache, which Docker charges to this container: + # hence a memory limit far above the 1 GB Java heap, enough to keep all four + # languages cached. That room is file cache the kernel takes back whenever + # anything else needs memory, not memory in use. One CPU at most, so a burst + # of checking never slows sync, search or the API. + languagetool: + image: erikvl87/languagetool:6.8 + # Read the n-gram files once at start (low priority, in the background), so + # even the first checks after a restart hit RAM instead of the disks. + command: + - bash + - -c + - | + if [ -d /ngrams ] && [ -n "$$(ls -A /ngrams 2>/dev/null)" ]; then + ( nice -n 19 find /ngrams -type f -exec cat {} + > /dev/null 2>&1 & ) + fi + exec bash start.sh + environment: + Java_Xms: 256m + Java_Xmx: ${LANGUAGETOOL_HEAP:-1g} + # Without the data the folder is empty and LanguageTool simply skips it. + langtool_languageModel: /ngrams + TZ: ${TZ:-UTC} + volumes: + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/languagetool-ngrams:/ngrams:ro + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8010/v2/languages || exit 1"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 60s + depends_on: + init: { condition: service_completed_successfully } + restart: unless-stopped + mem_limit: ${LANGUAGETOOL_MEM_LIMIT:-28g} + cpus: ${LANGUAGETOOL_CPUS:-1.0} + logging: *logging + # Bayes tokens and fuzzy hashes. Separate from the app's redis on purpose # (§C.2): this data is derived and retrainable, so it trades durability for # room to grow — RDB snapshots every 5 min, and an eviction policy, where the diff --git a/docker/.env.prod.example b/docker/.env.prod.example index 0fc22e2..d3a0a54 100644 --- a/docker/.env.prod.example +++ b/docker/.env.prod.example @@ -59,6 +59,18 @@ RSPAMD_MEM_LIMIT=256m RSPAMD_REDIS_MEM_LIMIT=160m RSPAMD_REDIS_MAXMEMORY=128mb +# --- Spelling and grammar check (LanguageTool) ------------------------------- +# The compose window's spell checker. Leave empty to turn it off for everyone; +# unset uses the bundled container (http://languagetool:8010). +#LANGUAGETOOL_URL= +# Java heap. 1g covers a household or small team. +LANGUAGETOOL_HEAP=1g +# Mostly file cache for the optional n-gram data (`webmail.sh ngrams`, ~26 GB +# unpacked for en/de/fr/es), which the kernel gives back under memory pressure. +# Without n-grams it never comes near this. Lower it on a small NAS. +LANGUAGETOOL_MEM_LIMIT=28g +LANGUAGETOOL_CPUS=1.0 + # --- Search ----------------------------------------------------------------- # pg = Postgres full-text. No extra container, no extra RAM. Default. # meili = Meilisearch: typo tolerance, better ranking, faster on large mailboxes. diff --git a/docs/nas-install.md b/docs/nas-install.md index f370963..fd010d3 100644 --- a/docs/nas-install.md +++ b/docs/nas-install.md @@ -606,6 +606,38 @@ Its Bayes database lives in `$DATA_ROOT/rspamd-redis`. Like the search index, that is derived data: it rides along in backups but never needs restoring, and retraining rebuilds it. +### The spell checker (LanguageTool) + +`languagetool` checks spelling and grammar in the compose window, in some 30 +languages. Like the anti-spam engine nothing waits for it: when it is down the +compose window falls back to the browser's own spell checker. It is never +published to the host — the app's API is its only client. + +It needs about **1.2 GB of RAM** (`LANGUAGETOOL_HEAP=1g` plus the JVM) and is +capped at one CPU core (`LANGUAGETOOL_CPUS`), so checking can never slow sync, +search or the API. + +**Optional n-gram data** teaches it to catch real words used wrongly — "the +affect was huge", "Das Buch liegt ihm Schrank" — beyond its hand-written rules +(English has ~780 such word pairs, German ~70). Install it once: + +```bash +./deploy/webmail.sh ngrams # en de fr es — or name languages: ngrams en de +``` + +That downloads about **13 GB** (English 8.4 GB, German/French/Spanish ~1.6 GB +each) into `$DATA_ROOT/languagetool-ngrams`, unpacks it to about **26 GB**, and +restarts the engine to load it. An interrupted download resumes when you run +the command again. There is no n-gram data for Polish; Polish text is checked +exactly as without it. + +The data is read from disk on demand and cached by the kernel, so the memory +limit (`LANGUAGETOOL_MEM_LIMIT=28g`) is room for that cache, not memory in use. +At start the container reads the files once in the background, at low +priority, so even the first checks after a restart are served from RAM. On +mechanical disks that warm-up takes several minutes; an NVMe cache in front of +the volume makes it — and any later cache misses — fast. + Two NAS-specific realities worth knowing: - **Disk hibernation won't happen.** Active accounts hold IMAP IDLE connections @@ -839,6 +871,7 @@ directories are root-owned. ./deploy/webmail.sh backup verified dump, right now ./deploy/webmail.sh restore restore a dump (asks for confirmation) ./deploy/webmail.sh reindex [--purge] rebuild the search index +./deploy/webmail.sh ngrams [langs] spell-check n-gram data (default: en de fr es) ./deploy/webmail.sh psql database shell ./deploy/webmail.sh shell [svc] shell in a container ```