From 95ca1b49f6d784c4b9c9a5a5c8d98192fcc643f2 Mon Sep 17 00:00:00 2001 From: Lukas Remis Date: Wed, 16 Sep 2026 22:14:43 +0200 Subject: [PATCH] Antispam plugin (M7 phase C1): classify, file, and stay out of the way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-pass spam filtering per SPEC Appendix C. The engine is Rspamd in its own container; this plugin is glue and policy, never a spam engine of its own. Classification (§C.4) hooks message.new, acts only on inbox arrivals of enabled accounts, and runs deny list -> allow list -> engine -> thresholds. Every failure path ends in "deliver normally, unclassified": an unreachable engine, a missing blob, an account with no Junk folder, a failed verdict write. Nothing here may delay or lose mail, so no service declares depends_on: rspamd. Two containers, started with everything else rather than behind a profile: rspamd, and a redis of its own. The separate redis is not incidental — the app's redis holds sync leases and arq queues under noeviction, and an ever-growing Bayes corpus must never be able to fail a job enqueue. This one takes RDB snapshots and evicts, because its data is derived and retrainable. Network checks (RBL/SURBL/fuzzy) are off: they leak message-derived hashes and URLs to third parties, which a self-hosted server's users did not ask for. The engine makes no outbound connections in the default configuration. The controller password is generated into .env like MASTER_KEY. It is internal; humans will reach Rspamd through the admin panel, authenticated as the ordinary administrator, rather than typing it. local.d/groups.conf zeroes six SMTP-time symbols — DATE_IN_PAST, PREVIOUSLY_DELIVERED, HFILTER_HOSTNAME_UNKNOWN, RCVD_NO_TLS_LAST, ARC_NA, DMARC_NA — and greylisting is disabled. Measured against 398 real delivered messages, those checks were firing on 44-71% of ordinary mail and pushing 11.3% of it over the banner threshold; scoring mail at rest for facts true of every delivered message is simply the wrong question. After: median 3.09 -> 0.70, p95 7.00 -> 3.40, banner rate 11.3% -> 1.0%. The spec's 12.0/6.0 thresholds were never the problem and are unchanged. Symbols are zeroed rather than removed so verdicts stay explainable, and they matter again at the M6 edge (§C.7). Still to come in C1: the §C.6 router and UI slots. C2 adds the learning loop — until Bayes is trained, genuine scams and legitimate bulk mail overlap in the 5-8 band and no threshold separates them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018SqJYt7jD7oUEkoY68sBZo --- backend/tests/plugins/test_antispam_policy.py | 94 +++++++++++ deploy/webmail.sh | 9 ++ docker-compose.yml | 72 +++++++++ docker/.env.prod.example | 12 ++ docker/Dockerfile.backend | 5 +- docker/entrypoint.sh | 2 +- docker/rspamd/local.d/classifier-bayes.conf | 6 + docker/rspamd/local.d/fuzzy_check.conf | 10 ++ docker/rspamd/local.d/greylist.conf | 5 + docker/rspamd/local.d/groups.conf | 34 ++++ docker/rspamd/local.d/options.inc | 5 + docker/rspamd/local.d/phishing.conf | 10 ++ docker/rspamd/local.d/rbl.conf | 10 ++ docker/rspamd/local.d/redis.conf | 4 + docker/rspamd/local.d/surbl.conf | 10 ++ .../webmail_plugin_antispam/pyproject.toml | 21 +++ .../webmail_plugin_antispam/__init__.py | 31 ++++ .../webmail_plugin_antispam/hook.py | 118 ++++++++++++++ .../migrations/versions/0001_init.py | 87 +++++++++++ .../webmail_plugin_antispam/policy.py | 122 +++++++++++++++ .../webmail_plugin_antispam/rspamd.py | 146 ++++++++++++++++++ 21 files changed, 810 insertions(+), 3 deletions(-) create mode 100644 backend/tests/plugins/test_antispam_policy.py create mode 100644 docker/rspamd/local.d/classifier-bayes.conf create mode 100644 docker/rspamd/local.d/fuzzy_check.conf create mode 100644 docker/rspamd/local.d/greylist.conf create mode 100644 docker/rspamd/local.d/groups.conf create mode 100644 docker/rspamd/local.d/options.inc create mode 100644 docker/rspamd/local.d/phishing.conf create mode 100644 docker/rspamd/local.d/rbl.conf create mode 100644 docker/rspamd/local.d/redis.conf create mode 100644 docker/rspamd/local.d/surbl.conf create mode 100644 plugins/webmail_plugin_antispam/pyproject.toml create mode 100644 plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py create mode 100644 plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py create mode 100644 plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py create mode 100644 plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py create mode 100644 plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py diff --git a/backend/tests/plugins/test_antispam_policy.py b/backend/tests/plugins/test_antispam_policy.py new file mode 100644 index 0000000..b61dbd8 --- /dev/null +++ b/backend/tests/plugins/test_antispam_policy.py @@ -0,0 +1,94 @@ +"""Antispam C1 decision logic (SPEC §C.4) — pure, no database, no engine.""" + +from __future__ import annotations + +import pytest +from webmail_plugin_antispam.policy import Settings, decide, matches + +SETTINGS = Settings( + enabled_accounts=[], + move_threshold=12.0, + banner_threshold=6.0, + network_checks=False, + learn_from_provider_junk=True, +) +NO_RULES: dict[str, set[str]] = {"allow": set(), "deny": set()} + + +def _decide(sender: str = "s@example.com", rules=None, score=None, settings=SETTINGS) -> str: + return decide(sender=sender, rules=rules or NO_RULES, score=score, settings=settings) + + +@pytest.mark.parametrize( + ("score", "expected"), + [ + (0.0, "none"), + (5.9, "none"), + (6.0, "banner"), + (11.9, "banner"), + (12.0, "moved"), + (99.0, "moved"), + ], +) +def test_thresholds_are_inclusive_lower_bounds(score: float, expected: str) -> None: + assert _decide(score=score) == expected + + +def test_unreachable_engine_fails_open() -> None: + """score=None is "could not classify" — mail must still be delivered plainly.""" + assert _decide(score=None) == "none" + + +def test_deny_list_moves_without_the_engine() -> None: + """Acceptance 5: a deny-listed sender is filed with no engine involvement.""" + rules = {"allow": set(), "deny": {"spammer@bad.test"}} + assert _decide(sender="spammer@bad.test", rules=rules, score=None) == "deny_listed" + + +def test_allow_list_beats_a_high_score() -> None: + """Acceptance 5: an allow-listed sender survives even an overwhelming score.""" + rules = {"allow": {"boss@work.test"}, "deny": set()} + assert _decide(sender="boss@work.test", rules=rules, score=100.0) == "allow_listed" + + +def test_deny_beats_allow() -> None: + """Both lists naming the same sender: the refusal wins.""" + rules = {"allow": {"x@y.test"}, "deny": {"x@y.test"}} + assert _decide(sender="x@y.test", rules=rules, score=0.0) == "deny_listed" + + +def test_domain_patterns_cover_every_sender_there() -> None: + rules = {"allow": set(), "deny": {"@bad.test"}} + assert _decide(sender="anyone@bad.test", rules=rules) == "deny_listed" + assert _decide(sender="anyone@good.test", rules=rules) == "none" + + +@pytest.mark.parametrize( + ("patterns", "email", "hit"), + [ + ({"a@b.test"}, "A@B.TEST", True), # case-insensitive + ({"a@b.test"}, " a@b.test ", True), # tolerant of stray whitespace + ({"@b.test"}, "anyone@b.test", True), + ({"@b.test"}, "anyone@other.test", False), + ({"a@b.test"}, "", False), # no sender at all + (set(), "a@b.test", False), + ], +) +def test_pattern_matching(patterns: set[str], email: str, hit: bool) -> None: + assert matches(patterns, email) is hit + + +def test_per_account_enablement() -> None: + """Empty list means every account; a populated one is an allow-list.""" + everywhere = SETTINGS + assert everywhere.covers("any-account-id") + + scoped = Settings( + enabled_accounts=["acc-1"], + move_threshold=12.0, + banner_threshold=6.0, + network_checks=False, + learn_from_provider_junk=True, + ) + assert scoped.covers("acc-1") + assert not scoped.covers("acc-2") diff --git a/deploy/webmail.sh b/deploy/webmail.sh index a77f358..a45ca96 100755 --- a/deploy/webmail.sh +++ b/deploy/webmail.sh @@ -187,6 +187,15 @@ ensure_secrets() { set_env_value POSTGRES_PASSWORD "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)" ylw "POSTGRES_PASSWORD was empty — generated one" fi + # Internal only: the antispam engine's controller. Nobody types this — the + # admin panel authenticates humans with the ordinary administrator account + # and holds this to talk to Rspamd. Nothing is lost if it is regenerated. + current="$(sed -n 's/^RSPAMD_CONTROLLER_PASSWORD=//p' "$ENV_FILE" | head -1)" + if [ -z "$current" ]; then + set_env_value RSPAMD_CONTROLLER_PASSWORD \ + "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)" + ylw "RSPAMD_CONTROLLER_PASSWORD was empty — generated one" + fi } cmd_init() { diff --git a/docker-compose.yml b/docker-compose.yml index bb172ab..b694b47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -209,6 +209,78 @@ services: mem_limit: ${MEILI_MEM_LIMIT:-1g} logging: *logging + # --- anti-spam engine (SPEC Appendix C) ------------------------------------ + # Not optional: it starts with everything else. Note what does NOT depend on + # it — api, workers and web have no `depends_on: rspamd`. That is deliberate + # and load-bearing: classification is best-effort, delivery is not, so an + # engine that is down, slow or mid-restart must never hold up mail (§C.4). + rspamd: + image: rspamd/rspamd:3.9 + # The controller password is an internal secret nobody types; humans reach + # Rspamd through the webmail admin panel, which authenticates with the + # ordinary administrator account. local.d is read-only (the shipped privacy + # defaults), so the generated password is written to override.d at start. + # Starts as root only to write that file — /etc/rspamd is root-owned and the + # image's default user (_rspamd, uid 11333) cannot write it. rspamd then + # drops to _rspamd itself via -u/-g, which is how it normally runs: a root + # master supervising unprivileged workers. + user: root + entrypoint: + - /bin/sh + - -c + - | + mkdir -p /etc/rspamd/override.d + printf 'password = "%s";\nenable_password = "%s";\n' \ + "$$RSPAMD_CONTROLLER_PASSWORD" "$$RSPAMD_CONTROLLER_PASSWORD" \ + > /etc/rspamd/override.d/worker-controller.inc + exec /usr/bin/rspamd -f -u _rspamd -g _rspamd + environment: + RSPAMD_CONTROLLER_PASSWORD: ${RSPAMD_CONTROLLER_PASSWORD} + TZ: ${TZ:-UTC} + volumes: + - ./docker/rspamd/local.d:/etc/rspamd/local.d:ro + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/rspamd:/var/lib/rspamd + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:11334/ping | grep -qi pong"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + depends_on: + redis-rspamd: { condition: service_healthy } + init: { condition: service_completed_successfully } + restart: unless-stopped + mem_limit: ${RSPAMD_MEM_LIMIT:-256m} + logging: *logging + + # Bayes tokens and fuzzy hashes. Separate from the app's redis on purpose + # (§C.2): this data is derived and retrainable, so it trades durability for + # room to grow — RDB snapshots every 5 min, and an eviction policy, where the + # app's redis uses AOF + noeviction because losing a sync lease loses work. + redis-rspamd: + image: redis:7-alpine + command: + - redis-server + - --save + - "300" + - "1" + - --maxmemory + - ${RSPAMD_REDIS_MAXMEMORY:-128mb} + - --maxmemory-policy + - allkeys-lru + volumes: + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/rspamd-redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 10 + depends_on: + init: { condition: service_completed_successfully } + restart: unless-stopped + mem_limit: ${RSPAMD_REDIS_MEM_LIMIT:-160m} + logging: *logging + # One-shot schema upgrade: core Alembic head + every plugin's own branch. # Everything else waits for this to exit 0, so a failed migration stops the # roll instead of starting code against an old schema. diff --git a/docker/.env.prod.example b/docker/.env.prod.example index 0021a1b..06ab4dd 100644 --- a/docker/.env.prod.example +++ b/docker/.env.prod.example @@ -47,6 +47,18 @@ ADMIN_BIND=0.0.0.0 TZ=Europe/Warsaw +# --- Anti-spam (SPEC Appendix C) --------------------------------------------- +# Generated by `webmail.sh init`. Internal only: it authenticates the app to the +# Rspamd controller. Humans never type it — the admin panel gates Rspamd behind +# the ordinary administrator account. +RSPAMD_CONTROLLER_PASSWORD= +# Network checks (RBL/SURBL/fuzzy) are OFF in docker/rspamd/local.d by default: +# they leak message-derived hashes and URLs to third parties. Enable the modules +# there first if you want them. +RSPAMD_MEM_LIMIT=256m +RSPAMD_REDIS_MEM_LIMIT=160m +RSPAMD_REDIS_MAXMEMORY=128mb + # --- Search ----------------------------------------------------------------- # pg = Postgres full-text. No extra container, no extra RAM. Default. # meili = Meilisearch: typo tolerance, better ranking, faster on large mailboxes. diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index 8f4210c..9ed9b63 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -2,7 +2,7 @@ # # One image, several commands (see docker/entrypoint.sh) — matches SPEC §6.3's # WORKER_ROLE split. Build context is the REPO ROOT, because the image needs -# backend/ *and* plugins/ (the five plugins are separate distributions that the +# backend/ *and* plugins/ (the plugins are separate distributions that the # loader discovers through the `webmail.plugins` entry-point group, SPEC §11). # # docker build -f docker/Dockerfile.backend -t webmail/backend . @@ -35,7 +35,8 @@ RUN pip install \ /src/plugins/webmail_plugin_calendar \ /src/plugins/webmail_plugin_contacts \ /src/plugins/webmail_plugin_labels \ - /src/plugins/webmail_plugin_filters + /src/plugins/webmail_plugin_filters \ + /src/plugins/webmail_plugin_antispam # ---------- runtime ----------------------------------------------------------- FROM python:3.11-slim diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 56e7419..df71e83 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -19,7 +19,7 @@ set -eu : "${API_WORKERS:=1}" # Plugins whose migrations are applied on every `migrate` run. Space-separated so # an operator can add a third-party plugin without editing the image. -: "${ENABLED_PLUGINS:=calendar contacts labels filters}" +: "${ENABLED_PLUGINS:=calendar contacts labels filters antispam}" cmd="${1:-api}" [ "$#" -gt 0 ] && shift || true diff --git a/docker/rspamd/local.d/classifier-bayes.conf b/docker/rspamd/local.d/classifier-bayes.conf new file mode 100644 index 0000000..b69410a --- /dev/null +++ b/docker/rspamd/local.d/classifier-bayes.conf @@ -0,0 +1,6 @@ +# Per-user Bayes, keyed by the Deliver-To header the plugin sends (SPEC §C.5). +# This is what makes acceptance 4 hold: one user's training cannot move another +# user's scores. +per_user = true; +autolearn = false; # only explicit/passive teaching from the app (C2) +backend = "redis"; diff --git a/docker/rspamd/local.d/fuzzy_check.conf b/docker/rspamd/local.d/fuzzy_check.conf new file mode 100644 index 0000000..d610348 --- /dev/null +++ b/docker/rspamd/local.d/fuzzy_check.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# fuzzy_check performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/docker/rspamd/local.d/greylist.conf b/docker/rspamd/local.d/greylist.conf new file mode 100644 index 0000000..242af9d --- /dev/null +++ b/docker/rspamd/local.d/greylist.conf @@ -0,0 +1,5 @@ +# Greylisting is an SMTP-time tactic: defer, see if the sender retries. We scan +# mail that has ALREADY been delivered and fetched over IMAP, so there is nothing +# to defer — the symbol fired on 62.6% of a real-mail sample and only added noise. +# It belongs at the M6 edge (§C.7), not in the second-pass filter. +enabled = false; diff --git a/docker/rspamd/local.d/groups.conf b/docker/rspamd/local.d/groups.conf new file mode 100644 index 0000000..aaf52f8 --- /dev/null +++ b/docker/rspamd/local.d/groups.conf @@ -0,0 +1,34 @@ +# Second-pass reality check (SPEC §C.1: this filter runs on mail the provider has +# already accepted and delivered). +# +# Every symbol below is a legitimate SMTP-time signal that is meaningless — or +# actively wrong — when scoring a message at rest, days after delivery, fetched +# over IMAP. Measured on a 398-message sample of real delivered mail, they fired +# on 44-71% of it and pushed 11.3% of ordinary mail over the banner threshold. +# +# They are zeroed, not deleted: they still appear in the symbol list, so a verdict +# stays explainable, they just no longer contribute score. When M6 lands, the edge +# scanner (§C.7) sees real SMTP state and these matter again — there, not here. +# +# Measured on 398 real delivered messages, before -> after this file: +# median 3.09 -> 0.70 +# p95 7.00 -> 3.40 +# >= 6.0 11.3% -> 1.0% (the banner threshold) +# max 11.66 -> 9.15 (move threshold is 12.0, so: headroom) +# The thresholds themselves were left at the spec's 12.0/6.0 — they were never +# the problem. Re-measure after the C2 learning loop has a trained Bayes: until +# then genuine scams and legitimate bulk mail overlap in the 5-8 band, and no +# threshold can separate them. +symbols { + # "This message is older than the scan" — always true for us. + "DATE_IN_PAST" { score = 0.0; } + # "Already delivered once" — by definition, for a second-pass filter. + "PREVIOUSLY_DELIVERED" { score = 0.0; } + # Properties of the connecting host, which we never saw. + "HFILTER_HOSTNAME_UNKNOWN" { score = 0.0; } + "RCVD_NO_TLS_LAST" { score = 0.0; } + # Absence-of-authentication penalties. We cannot verify SPF/DKIM/DMARC/ARC at + # rest anyway, so "not available" says nothing about this message. + "ARC_NA" { score = 0.0; } + "DMARC_NA" { score = 0.0; } +} diff --git a/docker/rspamd/local.d/options.inc b/docker/rspamd/local.d/options.inc new file mode 100644 index 0000000..a44fe79 --- /dev/null +++ b/docker/rspamd/local.d/options.inc @@ -0,0 +1,5 @@ +# No outbound DNS at all unless a network module asks for it. Belt to the braces +# of disabling the modules themselves (see rbl.conf, surbl.conf, fuzzy_check.conf). +dns { + enable_dnssec = false; +} diff --git a/docker/rspamd/local.d/phishing.conf b/docker/rspamd/local.d/phishing.conf new file mode 100644 index 0000000..46f5017 --- /dev/null +++ b/docker/rspamd/local.d/phishing.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# phishing performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/docker/rspamd/local.d/rbl.conf b/docker/rspamd/local.d/rbl.conf new file mode 100644 index 0000000..c2392d8 --- /dev/null +++ b/docker/rspamd/local.d/rbl.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# rbl performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/docker/rspamd/local.d/redis.conf b/docker/rspamd/local.d/redis.conf new file mode 100644 index 0000000..ac145d7 --- /dev/null +++ b/docker/rspamd/local.d/redis.conf @@ -0,0 +1,4 @@ +# Bayes and fuzzy storage live in the DEDICATED redis (SPEC §C.2), never the +# app's: that one holds sync leases and arq queues under `noeviction`, and an +# ever-growing Bayes corpus must never be able to fail a job enqueue. +servers = "redis-rspamd:6379"; diff --git a/docker/rspamd/local.d/surbl.conf b/docker/rspamd/local.d/surbl.conf new file mode 100644 index 0000000..1682c36 --- /dev/null +++ b/docker/rspamd/local.d/surbl.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# surbl performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/plugins/webmail_plugin_antispam/pyproject.toml b/plugins/webmail_plugin_antispam/pyproject.toml new file mode 100644 index 0000000..a08c17c --- /dev/null +++ b/plugins/webmail_plugin_antispam/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "webmail-plugin-antispam" +version = "0.1.0" +description = "Anti-spam — the third heavy plugin (SPEC Appendix C). Second-pass classification via Rspamd, allow/deny lists, per-user Bayes." +requires-python = ">=3.11" +# The engine is a container, not a library: the plugin is glue + policy (§C.2). +# httpx is already a core dependency, used here to reach the Rspamd container. +dependencies = [] + +[project.entry-points."webmail.plugins"] +antispam = "webmail_plugin_antispam:plugin" + +[tool.setuptools] +packages = ["webmail_plugin_antispam"] + +[tool.setuptools.package-data] +webmail_plugin_antispam = ["ui/*.js", "migrations/versions/*.py"] diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py new file mode 100644 index 0000000..5ee573a --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py @@ -0,0 +1,31 @@ +"""``antispam`` — second-pass spam classification (SPEC Appendix C, M7). + +For synced accounts the provider has usually filtered spam already, so this is a +*second pass*: it catches what a weak provider missed, gives one consistent +experience across accounts, and trains a per-user Bayes that belongs to the user +rather than to their mail host. + +Phases: **C1** (here) scores new inbox mail, moves high scorers to Junk through +the core move machinery so the verdict reaches the IMAP server, banners the +uncertain middle, and honours per-user allow/deny lists. C2 adds the learning +loop; C3 moves the same engine to the SMTP edge when M6 lands. + +The engine is Rspamd in its own container (§C.2) — this package is glue and +policy, never a spam engine of its own. +""" + +from __future__ import annotations + +from pathlib import Path + +from app.plugins.sdk import Plugin +from webmail_plugin_antispam.hook import on_message_new + +_HERE = Path(__file__).parent + +plugin = Plugin( + name="antispam", + version="0.1.0", + hooks={"message.new": on_message_new}, + migrations=str(_HERE / "migrations" / "versions"), +) diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py new file mode 100644 index 0000000..0adacfa --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py @@ -0,0 +1,118 @@ +"""``message.new`` hook — the C1 classification flow (SPEC §C.4). + +Runs in the sync worker wherever ``message.new`` is published. The invariant that +shapes every branch: **spam filtering must never delay or lose mail**. Every +failure path here ends in "deliver normally, unclassified" — an unreachable +engine, a missing blob, an account with no Junk folder, a database error while +recording the verdict. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import Any + +from sqlalchemy import text + +from app.plugins.sdk import sdk +from webmail_plugin_antispam import policy, rspamd + +logger = logging.getLogger("plg.antispam") + +_SDK = sdk("antispam") + + +async def _record( + user_id: uuid.UUID, + message_id: uuid.UUID, + *, + score: float, + symbols: list[str], + action: str, +) -> None: + """Write the verdict row. A failure here must not affect delivery.""" + try: + async with _SDK.session(user_id) as s: + await s.execute( + text( + "INSERT INTO plg_antispam_verdicts " + " (message_id, user_id, score, symbols, action) " + "VALUES (:m, :u, :sc, CAST(:sy AS jsonb), :a) " + "ON CONFLICT (message_id) DO NOTHING" + ), + { + "m": str(message_id), + "u": str(user_id), + "sc": score, + "sy": json.dumps(symbols), + "a": action, + }, + ) + await s.commit() + except Exception as exc: # noqa: BLE001 + logger.warning("could not record antispam verdict for %s: %s", message_id, exc) + + +async def on_message_new(payload: dict[str, Any]) -> None: + uid_raw = payload.get("user_id") + mid_raw = payload.get("message_id") + if not isinstance(uid_raw, str) or not isinstance(mid_raw, str): + return + user_id, message_id = uuid.UUID(uid_raw), uuid.UUID(mid_raw) + + env = await _SDK.messages.envelope(user_id, message_id) + if env is None: + return + + # Only newly-arrived inbox mail (§C.4). Never Sent/Drafts/Junk/Trash: scoring + # a message the user just sent, or one the provider already filed as spam, + # would be noise at best and a misfiled reply at worst. + if env.get("mailbox_special_use") != "inbox": + return + + settings = await policy.load_settings(user_id) + account_id = str(env.get("account_id") or "") + if not settings.covers(account_id): + return + + rules = await policy.load_rules(user_id) + sender = str(env.get("from_email") or "") + + # A deny-list hit needs no engine at all (acceptance 5), so check rules first + # and only pay for scoring when the lists have nothing to say. + action = policy.decide(sender=sender, rules=rules, score=None, settings=settings) + score, symbols = 0.0, [] + + if action == "none": # neither list matched — ask the engine + raw = await _SDK.messages.raw(user_id, message_id) + if raw is not None: + verdict = await rspamd.check( + raw, user_key=str(user_id), network_checks=settings.network_checks + ) + if verdict is not None: + score, symbols = verdict.score, verdict.symbols + action = policy.decide( + sender=sender, rules=rules, score=verdict.score, settings=settings + ) + + if action in ("moved", "deny_listed"): + moved = await _SDK.actions.move_to_special(user_id, message_id, "junk") + if not moved: + # No Junk folder on this account (or it is already there). Downgrade + # to a banner rather than dropping the verdict: the user still sees + # the warning, and nothing is lost. + action = "banner" if action == "moved" else "deny_listed" + + await _record(user_id, message_id, score=score, symbols=symbols, action=action) + + await _SDK.publish( + "plugin.antispam.classified", + { + "user_id": str(user_id), + "message_id": str(message_id), + "score": score, + "action": action, + }, + ) diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py new file mode 100644 index 0000000..a41fc9d --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py @@ -0,0 +1,87 @@ +"""antispam plugin: settings, allow/deny rules, verdicts — with RLS (SPEC §C.3) + +Revision ID: antispam0001 +Revises: +Create Date: 2026-09-16 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +from app.plugins.rls import enable_rls + +revision: str = "antispam0001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE plg_antispam_settings ( + user_id uuid PRIMARY KEY, + -- empty array = every account; §C.3 keeps the "all" case cheap + enabled_accounts uuid[] NOT NULL DEFAULT '{}', + move_threshold real NOT NULL DEFAULT 12.0, -- score >= : move to Junk + banner_threshold real NOT NULL DEFAULT 6.0, -- score >= : warn banner + -- Off by default on purpose (§C.2): RBL/SURBL/fuzzy lookups leak + -- message-derived hashes and URLs to third parties. + network_checks boolean NOT NULL DEFAULT false, + learn_from_provider_junk boolean NOT NULL DEFAULT true + ) + """ + ) + op.execute( + """ + CREATE TABLE plg_antispam_rules ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL, + kind text NOT NULL, -- allow|deny + pattern citext NOT NULL, -- an address, or @domain for a whole domain + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (user_id, kind, pattern), + CONSTRAINT ck_plg_antispam_rules_kind CHECK (kind IN ('allow', 'deny')) + ) + """ + ) + op.execute( + """ + CREATE TABLE plg_antispam_verdicts ( + message_id uuid PRIMARY KEY, + user_id uuid NOT NULL, + score real NOT NULL, + symbols jsonb NOT NULL DEFAULT '[]', -- rspamd symbols, for explainability + action text NOT NULL, -- none|banner|moved|allow_listed|deny_listed + learned text, -- spam|ham|NULL (C2 learning loop) + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT ck_plg_antispam_verdicts_action CHECK ( + action IN ('none', 'banner', 'moved', 'allow_listed', 'deny_listed') + ), + CONSTRAINT ck_plg_antispam_verdicts_learned CHECK ( + learned IS NULL OR learned IN ('spam', 'ham') + ) + ) + """ + ) + # The stats page counts by action, and the banner slot looks up one message. + op.execute( + "CREATE INDEX ix_plg_antispam_verdicts_user_action " + "ON plg_antispam_verdicts (user_id, action)" + ) + op.execute( + "CREATE INDEX ix_plg_antispam_rules_user_kind ON plg_antispam_rules (user_id, kind)" + ) + + enable_rls("plg_antispam_settings") + enable_rls("plg_antispam_rules") + enable_rls("plg_antispam_verdicts") + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS plg_antispam_verdicts") + op.execute("DROP TABLE IF EXISTS plg_antispam_rules") + op.execute("DROP TABLE IF EXISTS plg_antispam_settings") diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py new file mode 100644 index 0000000..fe10d84 --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py @@ -0,0 +1,122 @@ +"""Settings and allow/deny rules — the policy half of classification (SPEC §C.3). + +Kept apart from the hook so the decision logic is testable without a database or +an engine: :func:`decide` is a pure function of (rules, score, thresholds). +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import text + +from app.plugins.sdk import sdk + +_SDK = sdk("antispam") + +# §C.3 defaults, used when a user has never opened the settings page. +DEFAULTS = { + "enabled_accounts": [], # empty = every account + "move_threshold": 12.0, + "banner_threshold": 6.0, + "network_checks": False, + "learn_from_provider_junk": True, +} + + +@dataclass(frozen=True) +class Settings: + enabled_accounts: list[str] + move_threshold: float + banner_threshold: float + network_checks: bool + learn_from_provider_junk: bool + + def covers(self, account_id: str) -> bool: + """Empty list means every account — the common case stays cheap.""" + return not self.enabled_accounts or account_id in self.enabled_accounts + + +async def load_settings(user_id: uuid.UUID) -> Settings: + async with _SDK.session(user_id) as s: + row = ( + await s.execute( + text( + "SELECT enabled_accounts, move_threshold, banner_threshold, " + "network_checks, learn_from_provider_junk " + "FROM plg_antispam_settings WHERE user_id = :u" + ), + {"u": str(user_id)}, + ) + ).first() + if row is None: + d: Any = DEFAULTS + return Settings( + enabled_accounts=[], + move_threshold=d["move_threshold"], + banner_threshold=d["banner_threshold"], + network_checks=d["network_checks"], + learn_from_provider_junk=d["learn_from_provider_junk"], + ) + return Settings( + enabled_accounts=[str(a) for a in (row[0] or [])], + move_threshold=float(row[1]), + banner_threshold=float(row[2]), + network_checks=bool(row[3]), + learn_from_provider_junk=bool(row[4]), + ) + + +async def load_rules(user_id: uuid.UUID) -> dict[str, set[str]]: + """``{"allow": {...}, "deny": {...}}`` of lower-cased patterns.""" + async with _SDK.session(user_id) as s: + rows = ( + await s.execute( + text("SELECT kind, pattern FROM plg_antispam_rules WHERE user_id = :u"), + {"u": str(user_id)}, + ) + ).all() + out: dict[str, set[str]] = {"allow": set(), "deny": set()} + for kind, pattern in rows: + if kind in out: + out[kind].add(str(pattern).strip().lower()) + return out + + +def matches(patterns: set[str], email: str) -> bool: + """A pattern is either a whole address or ``@domain`` for everyone there.""" + addr = (email or "").strip().lower() + if not addr: + return False + if addr in patterns: + return True + _, _, domain = addr.partition("@") + return bool(domain) and f"@{domain}" in patterns + + +def decide( + *, + sender: str, + rules: dict[str, set[str]], + score: float | None, + settings: Settings, +) -> str: + """The §C.4 decision, in order. Returns the verdict action. + + Deny beats allow beats engine: an explicit user rule is a statement about a + sender that no score should override. ``score is None`` means the engine + could not be reached — fail open, deliver as ``none``. + """ + if matches(rules["deny"], sender): + return "deny_listed" + if matches(rules["allow"], sender): + return "allow_listed" + if score is None: + return "none" + if score >= settings.move_threshold: + return "moved" + if score >= settings.banner_threshold: + return "banner" + return "none" diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py new file mode 100644 index 0000000..bc894da --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py @@ -0,0 +1,146 @@ +"""Rspamd HTTP client (SPEC §C.2). + +The engine is a container, not a library — this module is the whole of our +coupling to it. Two rules shape everything here: + +* **Fail open.** Rspamd being down, slow or wrong must never delay or lose mail + (§C.4). Every entry point returns ``None`` instead of raising, and the caller + treats ``None`` as "deliver normally, unclassified". +* **Not through the SDK's HTTP.** ``sdk.http`` applies the §10.6 SSRF guard, + which blocks private addresses — correct for user-supplied CalDAV URLs, wrong + for an operator-configured service on the compose network. The URL here comes + from the environment, never from a user, so the guard protects nothing and + would only forbid the one address that works. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from typing import Any + +import httpx + +logger = logging.getLogger("plg.antispam") + +# The compose service name; operators override for an external engine. +# Rspamd splits its HTTP over two workers and they are NOT interchangeable: +# :11333 scan worker -> /checkv2, unauthenticated +# :11334 controller -> /learnspam, /learnham, /stat, password-gated +# Pointing learning at the scan port silently fails, so both are named here. +RSPAMD_URL = os.environ.get("RSPAMD_URL", "http://rspamd:11333") +RSPAMD_CONTROLLER_URL = os.environ.get("RSPAMD_CONTROLLER_URL", "http://rspamd:11334") +# Internal credential (generated into .env by `webmail.sh init`). Humans never +# present this: the admin panel authenticates them as the webmail administrator +# and talks to the controller on their behalf. +CONTROLLER_PASSWORD = os.environ.get("RSPAMD_CONTROLLER_PASSWORD", "") +# Generous enough for a large message, short enough that a hung engine cannot +# stall the sync worker: classification is best-effort, delivery is not. +TIMEOUT = float(os.environ.get("RSPAMD_TIMEOUT", "10")) + +# Settings profiles defined in docker/rspamd/local.d/settings.conf. The privacy +# default (§C.2) is enforced in that file, not here: "no network" is the profile +# that omits the RBL/SURBL/fuzzy modules entirely, so a bug in this client +# cannot silently start leaking URLs and hashes to third parties. +SETTINGS_OFFLINE = "webmail_offline" +SETTINGS_NETWORK = "webmail_network" + + +@dataclass(frozen=True) +class Verdict: + """What the engine thought. ``symbols`` is kept for explainability (§C.3).""" + + score: float + required_score: float + action: str + symbols: list[str] = field(default_factory=list) + + +def _parse(data: dict[str, Any]) -> Verdict | None: + try: + score = float(data["score"]) + except (KeyError, TypeError, ValueError): + return None + raw_symbols = data.get("symbols") + symbols = sorted(raw_symbols) if isinstance(raw_symbols, dict) else [] + try: + required = float(data.get("required_score", 0.0)) + except (TypeError, ValueError): + required = 0.0 + return Verdict( + score=score, + required_score=required, + action=str(data.get("action", "no action")), + symbols=symbols, + ) + + +def _headers(user_key: str, network_checks: bool) -> dict[str, str]: + # Deliver-To is what Rspamd keys per-user settings and per-user Bayes on, so + # one user's training never moves another user's scores (§C.5, acceptance 4). + return { + "Deliver-To": user_key, + "Settings-ID": SETTINGS_NETWORK if network_checks else SETTINGS_OFFLINE, + } + + +async def check( + raw: bytes, *, user_key: str, network_checks: bool = False +) -> Verdict | None: + """Score one raw RFC822 message. ``None`` means "could not classify".""" + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.post( + f"{RSPAMD_URL}/checkv2", + content=raw, + headers=_headers(user_key, network_checks), + ) + resp.raise_for_status() + return _parse(resp.json()) + except Exception as exc: # noqa: BLE001 - fail open, whatever went wrong + logger.warning("rspamd check unavailable (%s); delivering unclassified", exc) + return None + + +def _controller_headers(user_key: str) -> dict[str, str]: + headers = {"Deliver-To": user_key} + if CONTROLLER_PASSWORD: + headers["Password"] = CONTROLLER_PASSWORD + return headers + + +async def learn(raw: bytes, *, user_key: str, spam: bool) -> bool: + """Teach this user's Bayes. Returns False if the engine was unreachable. + + Goes to the CONTROLLER with the password, unlike :func:`check`. Used by the + C2 learning loop. + """ + endpoint = "/learnspam" if spam else "/learnham" + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.post( + f"{RSPAMD_CONTROLLER_URL}{endpoint}", + content=raw, + headers=_controller_headers(user_key), + ) + resp.raise_for_status() + return True + except Exception as exc: # noqa: BLE001 + logger.warning("rspamd learn failed (%s)", exc) + return False + + +async def stats(*, user_key: str) -> dict[str, Any] | None: + """Controller ``/stat`` — feeds the §C.6 stats page. None when unreachable.""" + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.get( + f"{RSPAMD_CONTROLLER_URL}/stat", headers=_controller_headers(user_key) + ) + resp.raise_for_status() + data = resp.json() + return data if isinstance(data, dict) else None + except Exception as exc: # noqa: BLE001 + logger.warning("rspamd stat unavailable (%s)", exc) + return None