diff --git a/backend/tests/plugins/test_antispam_policy.py b/backend/tests/plugins/test_antispam_policy.py new file mode 100644 index 0000000..b61dbd8 --- /dev/null +++ b/backend/tests/plugins/test_antispam_policy.py @@ -0,0 +1,94 @@ +"""Antispam C1 decision logic (SPEC §C.4) — pure, no database, no engine.""" + +from __future__ import annotations + +import pytest +from webmail_plugin_antispam.policy import Settings, decide, matches + +SETTINGS = Settings( + enabled_accounts=[], + move_threshold=12.0, + banner_threshold=6.0, + network_checks=False, + learn_from_provider_junk=True, +) +NO_RULES: dict[str, set[str]] = {"allow": set(), "deny": set()} + + +def _decide(sender: str = "s@example.com", rules=None, score=None, settings=SETTINGS) -> str: + return decide(sender=sender, rules=rules or NO_RULES, score=score, settings=settings) + + +@pytest.mark.parametrize( + ("score", "expected"), + [ + (0.0, "none"), + (5.9, "none"), + (6.0, "banner"), + (11.9, "banner"), + (12.0, "moved"), + (99.0, "moved"), + ], +) +def test_thresholds_are_inclusive_lower_bounds(score: float, expected: str) -> None: + assert _decide(score=score) == expected + + +def test_unreachable_engine_fails_open() -> None: + """score=None is "could not classify" — mail must still be delivered plainly.""" + assert _decide(score=None) == "none" + + +def test_deny_list_moves_without_the_engine() -> None: + """Acceptance 5: a deny-listed sender is filed with no engine involvement.""" + rules = {"allow": set(), "deny": {"spammer@bad.test"}} + assert _decide(sender="spammer@bad.test", rules=rules, score=None) == "deny_listed" + + +def test_allow_list_beats_a_high_score() -> None: + """Acceptance 5: an allow-listed sender survives even an overwhelming score.""" + rules = {"allow": {"boss@work.test"}, "deny": set()} + assert _decide(sender="boss@work.test", rules=rules, score=100.0) == "allow_listed" + + +def test_deny_beats_allow() -> None: + """Both lists naming the same sender: the refusal wins.""" + rules = {"allow": {"x@y.test"}, "deny": {"x@y.test"}} + assert _decide(sender="x@y.test", rules=rules, score=0.0) == "deny_listed" + + +def test_domain_patterns_cover_every_sender_there() -> None: + rules = {"allow": set(), "deny": {"@bad.test"}} + assert _decide(sender="anyone@bad.test", rules=rules) == "deny_listed" + assert _decide(sender="anyone@good.test", rules=rules) == "none" + + +@pytest.mark.parametrize( + ("patterns", "email", "hit"), + [ + ({"a@b.test"}, "A@B.TEST", True), # case-insensitive + ({"a@b.test"}, " a@b.test ", True), # tolerant of stray whitespace + ({"@b.test"}, "anyone@b.test", True), + ({"@b.test"}, "anyone@other.test", False), + ({"a@b.test"}, "", False), # no sender at all + (set(), "a@b.test", False), + ], +) +def test_pattern_matching(patterns: set[str], email: str, hit: bool) -> None: + assert matches(patterns, email) is hit + + +def test_per_account_enablement() -> None: + """Empty list means every account; a populated one is an allow-list.""" + everywhere = SETTINGS + assert everywhere.covers("any-account-id") + + scoped = Settings( + enabled_accounts=["acc-1"], + move_threshold=12.0, + banner_threshold=6.0, + network_checks=False, + learn_from_provider_junk=True, + ) + assert scoped.covers("acc-1") + assert not scoped.covers("acc-2") diff --git a/deploy/webmail.sh b/deploy/webmail.sh index a77f358..a45ca96 100755 --- a/deploy/webmail.sh +++ b/deploy/webmail.sh @@ -187,6 +187,15 @@ ensure_secrets() { set_env_value POSTGRES_PASSWORD "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)" ylw "POSTGRES_PASSWORD was empty — generated one" fi + # Internal only: the antispam engine's controller. Nobody types this — the + # admin panel authenticates humans with the ordinary administrator account + # and holds this to talk to Rspamd. Nothing is lost if it is regenerated. + current="$(sed -n 's/^RSPAMD_CONTROLLER_PASSWORD=//p' "$ENV_FILE" | head -1)" + if [ -z "$current" ]; then + set_env_value RSPAMD_CONTROLLER_PASSWORD \ + "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)" + ylw "RSPAMD_CONTROLLER_PASSWORD was empty — generated one" + fi } cmd_init() { diff --git a/docker-compose.yml b/docker-compose.yml index bb172ab..b694b47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -209,6 +209,78 @@ services: mem_limit: ${MEILI_MEM_LIMIT:-1g} logging: *logging + # --- anti-spam engine (SPEC Appendix C) ------------------------------------ + # Not optional: it starts with everything else. Note what does NOT depend on + # it — api, workers and web have no `depends_on: rspamd`. That is deliberate + # and load-bearing: classification is best-effort, delivery is not, so an + # engine that is down, slow or mid-restart must never hold up mail (§C.4). + rspamd: + image: rspamd/rspamd:3.9 + # The controller password is an internal secret nobody types; humans reach + # Rspamd through the webmail admin panel, which authenticates with the + # ordinary administrator account. local.d is read-only (the shipped privacy + # defaults), so the generated password is written to override.d at start. + # Starts as root only to write that file — /etc/rspamd is root-owned and the + # image's default user (_rspamd, uid 11333) cannot write it. rspamd then + # drops to _rspamd itself via -u/-g, which is how it normally runs: a root + # master supervising unprivileged workers. + user: root + entrypoint: + - /bin/sh + - -c + - | + mkdir -p /etc/rspamd/override.d + printf 'password = "%s";\nenable_password = "%s";\n' \ + "$$RSPAMD_CONTROLLER_PASSWORD" "$$RSPAMD_CONTROLLER_PASSWORD" \ + > /etc/rspamd/override.d/worker-controller.inc + exec /usr/bin/rspamd -f -u _rspamd -g _rspamd + environment: + RSPAMD_CONTROLLER_PASSWORD: ${RSPAMD_CONTROLLER_PASSWORD} + TZ: ${TZ:-UTC} + volumes: + - ./docker/rspamd/local.d:/etc/rspamd/local.d:ro + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/rspamd:/var/lib/rspamd + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:11334/ping | grep -qi pong"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + depends_on: + redis-rspamd: { condition: service_healthy } + init: { condition: service_completed_successfully } + restart: unless-stopped + mem_limit: ${RSPAMD_MEM_LIMIT:-256m} + logging: *logging + + # Bayes tokens and fuzzy hashes. Separate from the app's redis on purpose + # (§C.2): this data is derived and retrainable, so it trades durability for + # room to grow — RDB snapshots every 5 min, and an eviction policy, where the + # app's redis uses AOF + noeviction because losing a sync lease loses work. + redis-rspamd: + image: redis:7-alpine + command: + - redis-server + - --save + - "300" + - "1" + - --maxmemory + - ${RSPAMD_REDIS_MAXMEMORY:-128mb} + - --maxmemory-policy + - allkeys-lru + volumes: + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/rspamd-redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 10 + depends_on: + init: { condition: service_completed_successfully } + restart: unless-stopped + mem_limit: ${RSPAMD_REDIS_MEM_LIMIT:-160m} + logging: *logging + # One-shot schema upgrade: core Alembic head + every plugin's own branch. # Everything else waits for this to exit 0, so a failed migration stops the # roll instead of starting code against an old schema. diff --git a/docker/.env.prod.example b/docker/.env.prod.example index 0021a1b..06ab4dd 100644 --- a/docker/.env.prod.example +++ b/docker/.env.prod.example @@ -47,6 +47,18 @@ ADMIN_BIND=0.0.0.0 TZ=Europe/Warsaw +# --- Anti-spam (SPEC Appendix C) --------------------------------------------- +# Generated by `webmail.sh init`. Internal only: it authenticates the app to the +# Rspamd controller. Humans never type it — the admin panel gates Rspamd behind +# the ordinary administrator account. +RSPAMD_CONTROLLER_PASSWORD= +# Network checks (RBL/SURBL/fuzzy) are OFF in docker/rspamd/local.d by default: +# they leak message-derived hashes and URLs to third parties. Enable the modules +# there first if you want them. +RSPAMD_MEM_LIMIT=256m +RSPAMD_REDIS_MEM_LIMIT=160m +RSPAMD_REDIS_MAXMEMORY=128mb + # --- Search ----------------------------------------------------------------- # pg = Postgres full-text. No extra container, no extra RAM. Default. # meili = Meilisearch: typo tolerance, better ranking, faster on large mailboxes. diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index 8f4210c..9ed9b63 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -2,7 +2,7 @@ # # One image, several commands (see docker/entrypoint.sh) — matches SPEC §6.3's # WORKER_ROLE split. Build context is the REPO ROOT, because the image needs -# backend/ *and* plugins/ (the five plugins are separate distributions that the +# backend/ *and* plugins/ (the plugins are separate distributions that the # loader discovers through the `webmail.plugins` entry-point group, SPEC §11). # # docker build -f docker/Dockerfile.backend -t webmail/backend . @@ -35,7 +35,8 @@ RUN pip install \ /src/plugins/webmail_plugin_calendar \ /src/plugins/webmail_plugin_contacts \ /src/plugins/webmail_plugin_labels \ - /src/plugins/webmail_plugin_filters + /src/plugins/webmail_plugin_filters \ + /src/plugins/webmail_plugin_antispam # ---------- runtime ----------------------------------------------------------- FROM python:3.11-slim diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 56e7419..df71e83 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -19,7 +19,7 @@ set -eu : "${API_WORKERS:=1}" # Plugins whose migrations are applied on every `migrate` run. Space-separated so # an operator can add a third-party plugin without editing the image. -: "${ENABLED_PLUGINS:=calendar contacts labels filters}" +: "${ENABLED_PLUGINS:=calendar contacts labels filters antispam}" cmd="${1:-api}" [ "$#" -gt 0 ] && shift || true diff --git a/docker/rspamd/local.d/classifier-bayes.conf b/docker/rspamd/local.d/classifier-bayes.conf new file mode 100644 index 0000000..b69410a --- /dev/null +++ b/docker/rspamd/local.d/classifier-bayes.conf @@ -0,0 +1,6 @@ +# Per-user Bayes, keyed by the Deliver-To header the plugin sends (SPEC §C.5). +# This is what makes acceptance 4 hold: one user's training cannot move another +# user's scores. +per_user = true; +autolearn = false; # only explicit/passive teaching from the app (C2) +backend = "redis"; diff --git a/docker/rspamd/local.d/fuzzy_check.conf b/docker/rspamd/local.d/fuzzy_check.conf new file mode 100644 index 0000000..d610348 --- /dev/null +++ b/docker/rspamd/local.d/fuzzy_check.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# fuzzy_check performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/docker/rspamd/local.d/greylist.conf b/docker/rspamd/local.d/greylist.conf new file mode 100644 index 0000000..242af9d --- /dev/null +++ b/docker/rspamd/local.d/greylist.conf @@ -0,0 +1,5 @@ +# Greylisting is an SMTP-time tactic: defer, see if the sender retries. We scan +# mail that has ALREADY been delivered and fetched over IMAP, so there is nothing +# to defer — the symbol fired on 62.6% of a real-mail sample and only added noise. +# It belongs at the M6 edge (§C.7), not in the second-pass filter. +enabled = false; diff --git a/docker/rspamd/local.d/groups.conf b/docker/rspamd/local.d/groups.conf new file mode 100644 index 0000000..aaf52f8 --- /dev/null +++ b/docker/rspamd/local.d/groups.conf @@ -0,0 +1,34 @@ +# Second-pass reality check (SPEC §C.1: this filter runs on mail the provider has +# already accepted and delivered). +# +# Every symbol below is a legitimate SMTP-time signal that is meaningless — or +# actively wrong — when scoring a message at rest, days after delivery, fetched +# over IMAP. Measured on a 398-message sample of real delivered mail, they fired +# on 44-71% of it and pushed 11.3% of ordinary mail over the banner threshold. +# +# They are zeroed, not deleted: they still appear in the symbol list, so a verdict +# stays explainable, they just no longer contribute score. When M6 lands, the edge +# scanner (§C.7) sees real SMTP state and these matter again — there, not here. +# +# Measured on 398 real delivered messages, before -> after this file: +# median 3.09 -> 0.70 +# p95 7.00 -> 3.40 +# >= 6.0 11.3% -> 1.0% (the banner threshold) +# max 11.66 -> 9.15 (move threshold is 12.0, so: headroom) +# The thresholds themselves were left at the spec's 12.0/6.0 — they were never +# the problem. Re-measure after the C2 learning loop has a trained Bayes: until +# then genuine scams and legitimate bulk mail overlap in the 5-8 band, and no +# threshold can separate them. +symbols { + # "This message is older than the scan" — always true for us. + "DATE_IN_PAST" { score = 0.0; } + # "Already delivered once" — by definition, for a second-pass filter. + "PREVIOUSLY_DELIVERED" { score = 0.0; } + # Properties of the connecting host, which we never saw. + "HFILTER_HOSTNAME_UNKNOWN" { score = 0.0; } + "RCVD_NO_TLS_LAST" { score = 0.0; } + # Absence-of-authentication penalties. We cannot verify SPF/DKIM/DMARC/ARC at + # rest anyway, so "not available" says nothing about this message. + "ARC_NA" { score = 0.0; } + "DMARC_NA" { score = 0.0; } +} diff --git a/docker/rspamd/local.d/options.inc b/docker/rspamd/local.d/options.inc new file mode 100644 index 0000000..a44fe79 --- /dev/null +++ b/docker/rspamd/local.d/options.inc @@ -0,0 +1,5 @@ +# No outbound DNS at all unless a network module asks for it. Belt to the braces +# of disabling the modules themselves (see rbl.conf, surbl.conf, fuzzy_check.conf). +dns { + enable_dnssec = false; +} diff --git a/docker/rspamd/local.d/phishing.conf b/docker/rspamd/local.d/phishing.conf new file mode 100644 index 0000000..46f5017 --- /dev/null +++ b/docker/rspamd/local.d/phishing.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# phishing performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/docker/rspamd/local.d/rbl.conf b/docker/rspamd/local.d/rbl.conf new file mode 100644 index 0000000..c2392d8 --- /dev/null +++ b/docker/rspamd/local.d/rbl.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# rbl performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/docker/rspamd/local.d/redis.conf b/docker/rspamd/local.d/redis.conf new file mode 100644 index 0000000..ac145d7 --- /dev/null +++ b/docker/rspamd/local.d/redis.conf @@ -0,0 +1,4 @@ +# Bayes and fuzzy storage live in the DEDICATED redis (SPEC §C.2), never the +# app's: that one holds sync leases and arq queues under `noeviction`, and an +# ever-growing Bayes corpus must never be able to fail a job enqueue. +servers = "redis-rspamd:6379"; diff --git a/docker/rspamd/local.d/surbl.conf b/docker/rspamd/local.d/surbl.conf new file mode 100644 index 0000000..1682c36 --- /dev/null +++ b/docker/rspamd/local.d/surbl.conf @@ -0,0 +1,10 @@ +# DISABLED BY DEFAULT (SPEC §C.2, acceptance 6). +# surbl performs third-party lookups derived from message content — hashes +# of the body, the URLs it contains, the sending IP. For a self-hosted personal +# server that is a privacy leak the user did not ask for, so the module is off +# and the engine makes no outbound connections at all. +# +# Turning this on is an operator decision, not a per-message one: set +# enabled = true here, restart rspamd, and only then does the plugin's +# network_checks setting have anything to enable. +enabled = false; diff --git a/plugins/webmail_plugin_antispam/pyproject.toml b/plugins/webmail_plugin_antispam/pyproject.toml new file mode 100644 index 0000000..a08c17c --- /dev/null +++ b/plugins/webmail_plugin_antispam/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "webmail-plugin-antispam" +version = "0.1.0" +description = "Anti-spam — the third heavy plugin (SPEC Appendix C). Second-pass classification via Rspamd, allow/deny lists, per-user Bayes." +requires-python = ">=3.11" +# The engine is a container, not a library: the plugin is glue + policy (§C.2). +# httpx is already a core dependency, used here to reach the Rspamd container. +dependencies = [] + +[project.entry-points."webmail.plugins"] +antispam = "webmail_plugin_antispam:plugin" + +[tool.setuptools] +packages = ["webmail_plugin_antispam"] + +[tool.setuptools.package-data] +webmail_plugin_antispam = ["ui/*.js", "migrations/versions/*.py"] diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py new file mode 100644 index 0000000..5ee573a --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/__init__.py @@ -0,0 +1,31 @@ +"""``antispam`` — second-pass spam classification (SPEC Appendix C, M7). + +For synced accounts the provider has usually filtered spam already, so this is a +*second pass*: it catches what a weak provider missed, gives one consistent +experience across accounts, and trains a per-user Bayes that belongs to the user +rather than to their mail host. + +Phases: **C1** (here) scores new inbox mail, moves high scorers to Junk through +the core move machinery so the verdict reaches the IMAP server, banners the +uncertain middle, and honours per-user allow/deny lists. C2 adds the learning +loop; C3 moves the same engine to the SMTP edge when M6 lands. + +The engine is Rspamd in its own container (§C.2) — this package is glue and +policy, never a spam engine of its own. +""" + +from __future__ import annotations + +from pathlib import Path + +from app.plugins.sdk import Plugin +from webmail_plugin_antispam.hook import on_message_new + +_HERE = Path(__file__).parent + +plugin = Plugin( + name="antispam", + version="0.1.0", + hooks={"message.new": on_message_new}, + migrations=str(_HERE / "migrations" / "versions"), +) diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py new file mode 100644 index 0000000..0adacfa --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py @@ -0,0 +1,118 @@ +"""``message.new`` hook — the C1 classification flow (SPEC §C.4). + +Runs in the sync worker wherever ``message.new`` is published. The invariant that +shapes every branch: **spam filtering must never delay or lose mail**. Every +failure path here ends in "deliver normally, unclassified" — an unreachable +engine, a missing blob, an account with no Junk folder, a database error while +recording the verdict. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import Any + +from sqlalchemy import text + +from app.plugins.sdk import sdk +from webmail_plugin_antispam import policy, rspamd + +logger = logging.getLogger("plg.antispam") + +_SDK = sdk("antispam") + + +async def _record( + user_id: uuid.UUID, + message_id: uuid.UUID, + *, + score: float, + symbols: list[str], + action: str, +) -> None: + """Write the verdict row. A failure here must not affect delivery.""" + try: + async with _SDK.session(user_id) as s: + await s.execute( + text( + "INSERT INTO plg_antispam_verdicts " + " (message_id, user_id, score, symbols, action) " + "VALUES (:m, :u, :sc, CAST(:sy AS jsonb), :a) " + "ON CONFLICT (message_id) DO NOTHING" + ), + { + "m": str(message_id), + "u": str(user_id), + "sc": score, + "sy": json.dumps(symbols), + "a": action, + }, + ) + await s.commit() + except Exception as exc: # noqa: BLE001 + logger.warning("could not record antispam verdict for %s: %s", message_id, exc) + + +async def on_message_new(payload: dict[str, Any]) -> None: + uid_raw = payload.get("user_id") + mid_raw = payload.get("message_id") + if not isinstance(uid_raw, str) or not isinstance(mid_raw, str): + return + user_id, message_id = uuid.UUID(uid_raw), uuid.UUID(mid_raw) + + env = await _SDK.messages.envelope(user_id, message_id) + if env is None: + return + + # Only newly-arrived inbox mail (§C.4). Never Sent/Drafts/Junk/Trash: scoring + # a message the user just sent, or one the provider already filed as spam, + # would be noise at best and a misfiled reply at worst. + if env.get("mailbox_special_use") != "inbox": + return + + settings = await policy.load_settings(user_id) + account_id = str(env.get("account_id") or "") + if not settings.covers(account_id): + return + + rules = await policy.load_rules(user_id) + sender = str(env.get("from_email") or "") + + # A deny-list hit needs no engine at all (acceptance 5), so check rules first + # and only pay for scoring when the lists have nothing to say. + action = policy.decide(sender=sender, rules=rules, score=None, settings=settings) + score, symbols = 0.0, [] + + if action == "none": # neither list matched — ask the engine + raw = await _SDK.messages.raw(user_id, message_id) + if raw is not None: + verdict = await rspamd.check( + raw, user_key=str(user_id), network_checks=settings.network_checks + ) + if verdict is not None: + score, symbols = verdict.score, verdict.symbols + action = policy.decide( + sender=sender, rules=rules, score=verdict.score, settings=settings + ) + + if action in ("moved", "deny_listed"): + moved = await _SDK.actions.move_to_special(user_id, message_id, "junk") + if not moved: + # No Junk folder on this account (or it is already there). Downgrade + # to a banner rather than dropping the verdict: the user still sees + # the warning, and nothing is lost. + action = "banner" if action == "moved" else "deny_listed" + + await _record(user_id, message_id, score=score, symbols=symbols, action=action) + + await _SDK.publish( + "plugin.antispam.classified", + { + "user_id": str(user_id), + "message_id": str(message_id), + "score": score, + "action": action, + }, + ) diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py new file mode 100644 index 0000000..a41fc9d --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/migrations/versions/0001_init.py @@ -0,0 +1,87 @@ +"""antispam plugin: settings, allow/deny rules, verdicts — with RLS (SPEC §C.3) + +Revision ID: antispam0001 +Revises: +Create Date: 2026-09-16 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +from app.plugins.rls import enable_rls + +revision: str = "antispam0001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE plg_antispam_settings ( + user_id uuid PRIMARY KEY, + -- empty array = every account; §C.3 keeps the "all" case cheap + enabled_accounts uuid[] NOT NULL DEFAULT '{}', + move_threshold real NOT NULL DEFAULT 12.0, -- score >= : move to Junk + banner_threshold real NOT NULL DEFAULT 6.0, -- score >= : warn banner + -- Off by default on purpose (§C.2): RBL/SURBL/fuzzy lookups leak + -- message-derived hashes and URLs to third parties. + network_checks boolean NOT NULL DEFAULT false, + learn_from_provider_junk boolean NOT NULL DEFAULT true + ) + """ + ) + op.execute( + """ + CREATE TABLE plg_antispam_rules ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL, + kind text NOT NULL, -- allow|deny + pattern citext NOT NULL, -- an address, or @domain for a whole domain + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (user_id, kind, pattern), + CONSTRAINT ck_plg_antispam_rules_kind CHECK (kind IN ('allow', 'deny')) + ) + """ + ) + op.execute( + """ + CREATE TABLE plg_antispam_verdicts ( + message_id uuid PRIMARY KEY, + user_id uuid NOT NULL, + score real NOT NULL, + symbols jsonb NOT NULL DEFAULT '[]', -- rspamd symbols, for explainability + action text NOT NULL, -- none|banner|moved|allow_listed|deny_listed + learned text, -- spam|ham|NULL (C2 learning loop) + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT ck_plg_antispam_verdicts_action CHECK ( + action IN ('none', 'banner', 'moved', 'allow_listed', 'deny_listed') + ), + CONSTRAINT ck_plg_antispam_verdicts_learned CHECK ( + learned IS NULL OR learned IN ('spam', 'ham') + ) + ) + """ + ) + # The stats page counts by action, and the banner slot looks up one message. + op.execute( + "CREATE INDEX ix_plg_antispam_verdicts_user_action " + "ON plg_antispam_verdicts (user_id, action)" + ) + op.execute( + "CREATE INDEX ix_plg_antispam_rules_user_kind ON plg_antispam_rules (user_id, kind)" + ) + + enable_rls("plg_antispam_settings") + enable_rls("plg_antispam_rules") + enable_rls("plg_antispam_verdicts") + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS plg_antispam_verdicts") + op.execute("DROP TABLE IF EXISTS plg_antispam_rules") + op.execute("DROP TABLE IF EXISTS plg_antispam_settings") diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py new file mode 100644 index 0000000..fe10d84 --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/policy.py @@ -0,0 +1,122 @@ +"""Settings and allow/deny rules — the policy half of classification (SPEC §C.3). + +Kept apart from the hook so the decision logic is testable without a database or +an engine: :func:`decide` is a pure function of (rules, score, thresholds). +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import text + +from app.plugins.sdk import sdk + +_SDK = sdk("antispam") + +# §C.3 defaults, used when a user has never opened the settings page. +DEFAULTS = { + "enabled_accounts": [], # empty = every account + "move_threshold": 12.0, + "banner_threshold": 6.0, + "network_checks": False, + "learn_from_provider_junk": True, +} + + +@dataclass(frozen=True) +class Settings: + enabled_accounts: list[str] + move_threshold: float + banner_threshold: float + network_checks: bool + learn_from_provider_junk: bool + + def covers(self, account_id: str) -> bool: + """Empty list means every account — the common case stays cheap.""" + return not self.enabled_accounts or account_id in self.enabled_accounts + + +async def load_settings(user_id: uuid.UUID) -> Settings: + async with _SDK.session(user_id) as s: + row = ( + await s.execute( + text( + "SELECT enabled_accounts, move_threshold, banner_threshold, " + "network_checks, learn_from_provider_junk " + "FROM plg_antispam_settings WHERE user_id = :u" + ), + {"u": str(user_id)}, + ) + ).first() + if row is None: + d: Any = DEFAULTS + return Settings( + enabled_accounts=[], + move_threshold=d["move_threshold"], + banner_threshold=d["banner_threshold"], + network_checks=d["network_checks"], + learn_from_provider_junk=d["learn_from_provider_junk"], + ) + return Settings( + enabled_accounts=[str(a) for a in (row[0] or [])], + move_threshold=float(row[1]), + banner_threshold=float(row[2]), + network_checks=bool(row[3]), + learn_from_provider_junk=bool(row[4]), + ) + + +async def load_rules(user_id: uuid.UUID) -> dict[str, set[str]]: + """``{"allow": {...}, "deny": {...}}`` of lower-cased patterns.""" + async with _SDK.session(user_id) as s: + rows = ( + await s.execute( + text("SELECT kind, pattern FROM plg_antispam_rules WHERE user_id = :u"), + {"u": str(user_id)}, + ) + ).all() + out: dict[str, set[str]] = {"allow": set(), "deny": set()} + for kind, pattern in rows: + if kind in out: + out[kind].add(str(pattern).strip().lower()) + return out + + +def matches(patterns: set[str], email: str) -> bool: + """A pattern is either a whole address or ``@domain`` for everyone there.""" + addr = (email or "").strip().lower() + if not addr: + return False + if addr in patterns: + return True + _, _, domain = addr.partition("@") + return bool(domain) and f"@{domain}" in patterns + + +def decide( + *, + sender: str, + rules: dict[str, set[str]], + score: float | None, + settings: Settings, +) -> str: + """The §C.4 decision, in order. Returns the verdict action. + + Deny beats allow beats engine: an explicit user rule is a statement about a + sender that no score should override. ``score is None`` means the engine + could not be reached — fail open, deliver as ``none``. + """ + if matches(rules["deny"], sender): + return "deny_listed" + if matches(rules["allow"], sender): + return "allow_listed" + if score is None: + return "none" + if score >= settings.move_threshold: + return "moved" + if score >= settings.banner_threshold: + return "banner" + return "none" diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py new file mode 100644 index 0000000..bc894da --- /dev/null +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/rspamd.py @@ -0,0 +1,146 @@ +"""Rspamd HTTP client (SPEC §C.2). + +The engine is a container, not a library — this module is the whole of our +coupling to it. Two rules shape everything here: + +* **Fail open.** Rspamd being down, slow or wrong must never delay or lose mail + (§C.4). Every entry point returns ``None`` instead of raising, and the caller + treats ``None`` as "deliver normally, unclassified". +* **Not through the SDK's HTTP.** ``sdk.http`` applies the §10.6 SSRF guard, + which blocks private addresses — correct for user-supplied CalDAV URLs, wrong + for an operator-configured service on the compose network. The URL here comes + from the environment, never from a user, so the guard protects nothing and + would only forbid the one address that works. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from typing import Any + +import httpx + +logger = logging.getLogger("plg.antispam") + +# The compose service name; operators override for an external engine. +# Rspamd splits its HTTP over two workers and they are NOT interchangeable: +# :11333 scan worker -> /checkv2, unauthenticated +# :11334 controller -> /learnspam, /learnham, /stat, password-gated +# Pointing learning at the scan port silently fails, so both are named here. +RSPAMD_URL = os.environ.get("RSPAMD_URL", "http://rspamd:11333") +RSPAMD_CONTROLLER_URL = os.environ.get("RSPAMD_CONTROLLER_URL", "http://rspamd:11334") +# Internal credential (generated into .env by `webmail.sh init`). Humans never +# present this: the admin panel authenticates them as the webmail administrator +# and talks to the controller on their behalf. +CONTROLLER_PASSWORD = os.environ.get("RSPAMD_CONTROLLER_PASSWORD", "") +# Generous enough for a large message, short enough that a hung engine cannot +# stall the sync worker: classification is best-effort, delivery is not. +TIMEOUT = float(os.environ.get("RSPAMD_TIMEOUT", "10")) + +# Settings profiles defined in docker/rspamd/local.d/settings.conf. The privacy +# default (§C.2) is enforced in that file, not here: "no network" is the profile +# that omits the RBL/SURBL/fuzzy modules entirely, so a bug in this client +# cannot silently start leaking URLs and hashes to third parties. +SETTINGS_OFFLINE = "webmail_offline" +SETTINGS_NETWORK = "webmail_network" + + +@dataclass(frozen=True) +class Verdict: + """What the engine thought. ``symbols`` is kept for explainability (§C.3).""" + + score: float + required_score: float + action: str + symbols: list[str] = field(default_factory=list) + + +def _parse(data: dict[str, Any]) -> Verdict | None: + try: + score = float(data["score"]) + except (KeyError, TypeError, ValueError): + return None + raw_symbols = data.get("symbols") + symbols = sorted(raw_symbols) if isinstance(raw_symbols, dict) else [] + try: + required = float(data.get("required_score", 0.0)) + except (TypeError, ValueError): + required = 0.0 + return Verdict( + score=score, + required_score=required, + action=str(data.get("action", "no action")), + symbols=symbols, + ) + + +def _headers(user_key: str, network_checks: bool) -> dict[str, str]: + # Deliver-To is what Rspamd keys per-user settings and per-user Bayes on, so + # one user's training never moves another user's scores (§C.5, acceptance 4). + return { + "Deliver-To": user_key, + "Settings-ID": SETTINGS_NETWORK if network_checks else SETTINGS_OFFLINE, + } + + +async def check( + raw: bytes, *, user_key: str, network_checks: bool = False +) -> Verdict | None: + """Score one raw RFC822 message. ``None`` means "could not classify".""" + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.post( + f"{RSPAMD_URL}/checkv2", + content=raw, + headers=_headers(user_key, network_checks), + ) + resp.raise_for_status() + return _parse(resp.json()) + except Exception as exc: # noqa: BLE001 - fail open, whatever went wrong + logger.warning("rspamd check unavailable (%s); delivering unclassified", exc) + return None + + +def _controller_headers(user_key: str) -> dict[str, str]: + headers = {"Deliver-To": user_key} + if CONTROLLER_PASSWORD: + headers["Password"] = CONTROLLER_PASSWORD + return headers + + +async def learn(raw: bytes, *, user_key: str, spam: bool) -> bool: + """Teach this user's Bayes. Returns False if the engine was unreachable. + + Goes to the CONTROLLER with the password, unlike :func:`check`. Used by the + C2 learning loop. + """ + endpoint = "/learnspam" if spam else "/learnham" + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.post( + f"{RSPAMD_CONTROLLER_URL}{endpoint}", + content=raw, + headers=_controller_headers(user_key), + ) + resp.raise_for_status() + return True + except Exception as exc: # noqa: BLE001 + logger.warning("rspamd learn failed (%s)", exc) + return False + + +async def stats(*, user_key: str) -> dict[str, Any] | None: + """Controller ``/stat`` — feeds the §C.6 stats page. None when unreachable.""" + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.get( + f"{RSPAMD_CONTROLLER_URL}/stat", headers=_controller_headers(user_key) + ) + resp.raise_for_status() + data = resp.json() + return data if isinstance(data, dict) else None + except Exception as exc: # noqa: BLE001 + logger.warning("rspamd stat unavailable (%s)", exc) + return None