From dee1983b843ebc920887f1bfe35119c7a723d51b Mon Sep 17 00:00:00 2001 From: Lukas Remis Date: Thu, 17 Sep 2026 07:59:15 +0200 Subject: [PATCH] Antispam: never classify during backfill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit message.new fires for imported history as well as new arrivals, and EventBus awaits hooks inline. As written, adding an account would have scored every message in its backfill — filing years-old inbox mail into Junk on the server, and adding ~70 ms of serialized HTTP per message to the backfill loop (twelve minutes on a 10k-message account). It is also when the filter is least able to judge, because Bayes has learned nothing yet. The envelope now carries account_state, and the hook returns unless it is "live". The backfill window belongs to the C2 learning bootstrap (§C.5), which reads that history as training data instead of acting on it. Tests pin the gating: backfilling accounts are not even read from the blob store, non-inbox folders are ignored, a live inbox arrival still classifies, and an unreachable engine still delivers. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018SqJYt7jD7oUEkoY68sBZo --- backend/app/plugins/actions.py | 11 +- .../plugins/test_antispam_hook_gating.py | 138 ++++++++++++++++++ .../webmail_plugin_antispam/hook.py | 10 ++ 3 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 backend/tests/plugins/test_antispam_hook_gating.py diff --git a/backend/app/plugins/actions.py b/backend/app/plugins/actions.py index f339801..54a1dfe 100644 --- a/backend/app/plugins/actions.py +++ b/backend/app/plugins/actions.py @@ -23,7 +23,7 @@ from sqlalchemy import select from app.db import session_scope from app.events import get_event_bus from app.indexer import blobstore -from app.models import Mailbox, Message, Thread +from app.models import Account, Mailbox, Message, Thread class PluginMessages: @@ -92,14 +92,15 @@ class PluginMessages: async with session_scope() as session: row = ( await session.execute( - select(Message, Mailbox.special_use) + select(Message, Mailbox.special_use, Account.state) .join(Mailbox, Mailbox.id == Message.mailbox_id, isouter=True) + .join(Account, Account.id == Message.account_id, isouter=True) .where(Message.id == message_id, Message.user_id == user_id) ) ).first() if row is None: return None - msg, special_use = row + msg, special_use, account_state = row sender = msg.from_addr[0] if msg.from_addr else {} return { "message_id": str(msg.id), @@ -108,6 +109,10 @@ class PluginMessages: # with no SPECIAL-USE. A plugin that should only act on newly # arrived mail checks this — `message.new` itself does not say. "mailbox_special_use": special_use, + # "backfilling" | "live" | … — `message.new` fires for imported + # history too, so a plugin that should only react to genuinely + # new arrivals has to be able to tell the difference. + "account_state": account_state, "subject": msg.subject or "", "from_email": (sender.get("email") or "").lower(), "from_name": sender.get("name") or "", diff --git a/backend/tests/plugins/test_antispam_hook_gating.py b/backend/tests/plugins/test_antispam_hook_gating.py new file mode 100644 index 0000000..85ced80 --- /dev/null +++ b/backend/tests/plugins/test_antispam_hook_gating.py @@ -0,0 +1,138 @@ +"""What the antispam hook refuses to touch (SPEC §C.4). + +The gating matters more than the scoring: `message.new` fires for imported +history and for every folder, so a hook that acts on all of it would file +years-old mail into Junk and stall backfill with inline HTTP. +""" + +from __future__ import annotations + +import uuid +from typing import Any + +import pytest +from webmail_plugin_antispam import hook, policy + +LIVE_INBOX = { + "message_id": "m", + "account_id": "acc-1", + "mailbox_special_use": "inbox", + "account_state": "live", + "from_email": "someone@example.test", + "subject": "hello", + "to": [], + "flags": [], +} + + +class _FakeMessages: + def __init__(self, env: dict[str, Any] | None) -> None: + self._env = env + self.raw_calls = 0 + + async def envelope(self, *_a: object, **_k: object) -> dict[str, Any] | None: + return self._env + + async def raw(self, *_a: object, **_k: object) -> bytes | None: + self.raw_calls += 1 + return b"From: someone@example.test\r\n\r\nbody\r\n" + + +class _FakeActions: + def __init__(self) -> None: + self.moves: list[str] = [] + + async def move_to_special(self, _u: object, _m: object, special_use: str) -> bool: + self.moves.append(special_use) + return True + + +class _FakeSdk: + def __init__(self, env: dict[str, Any] | None) -> None: + self.messages = _FakeMessages(env) + self.actions = _FakeActions() + self.published: list[tuple[str, dict[str, Any]]] = [] + + async def publish(self, event: str, payload: dict[str, Any]) -> None: + self.published.append((event, payload)) + + +@pytest.fixture +def wired(monkeypatch: pytest.MonkeyPatch): # type: ignore[no-untyped-def] + """Swap the hook's SDK and its database/engine calls for fakes.""" + + recorded: list[dict[str, Any]] = [] + + async def _record(_u: object, _m: object, **kw: Any) -> None: + recorded.append(kw) + + async def _settings(_u: object) -> policy.Settings: + return policy.Settings( + enabled_accounts=[], + move_threshold=12.0, + banner_threshold=6.0, + network_checks=False, + learn_from_provider_junk=True, + ) + + async def _rules(_u: object) -> dict[str, set[str]]: + return {"allow": set(), "deny": set()} + + monkeypatch.setattr(hook, "_record", _record) + monkeypatch.setattr(policy, "load_settings", _settings) + monkeypatch.setattr(policy, "load_rules", _rules) + + def _install(env: dict[str, Any] | None, score: float | None = None) -> _FakeSdk: + fake = _FakeSdk(env) + monkeypatch.setattr(hook, "_SDK", fake) + + async def _check(*_a: object, **_k: object) -> object: + if score is None: + return None + return type("V", (), {"score": score, "symbols": ["X"]})() + + monkeypatch.setattr(hook.rspamd, "check", _check) + return fake + + return _install, recorded + + +async def _fire() -> None: + await hook.on_message_new({"user_id": str(uuid.uuid4()), "message_id": str(uuid.uuid4())}) + + +async def test_skips_while_the_account_is_backfilling(wired) -> None: # type: ignore[no-untyped-def] + """Imported history must not be scored, moved, or slowed down.""" + install, recorded = wired + fake = install({**LIVE_INBOX, "account_state": "backfilling"}, score=99.0) + await _fire() + assert fake.messages.raw_calls == 0 # never even fetched the body + assert fake.actions.moves == [] + assert recorded == [] + + +@pytest.mark.parametrize("special_use", ["sent", "junk", "trash", "drafts", None]) +async def test_skips_everything_outside_the_inbox(wired, special_use) -> None: # type: ignore[no-untyped-def] + install, recorded = wired + fake = install({**LIVE_INBOX, "mailbox_special_use": special_use}, score=99.0) + await _fire() + assert fake.actions.moves == [] + assert recorded == [] + + +async def test_classifies_a_live_inbox_arrival(wired) -> None: # type: ignore[no-untyped-def] + install, recorded = wired + fake = install(LIVE_INBOX, score=99.0) + await _fire() + assert fake.actions.moves == ["junk"] + assert recorded and recorded[0]["action"] == "moved" + assert fake.published and fake.published[0][0] == "plugin.antispam.classified" + + +async def test_unreachable_engine_delivers_normally(wired) -> None: # type: ignore[no-untyped-def] + """Acceptance 2: no engine, no interference.""" + install, recorded = wired + fake = install(LIVE_INBOX, score=None) + await _fire() + assert fake.actions.moves == [] + assert recorded and recorded[0]["action"] == "none" diff --git a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py index 0adacfa..1ecd742 100644 --- a/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py +++ b/plugins/webmail_plugin_antispam/webmail_plugin_antispam/hook.py @@ -72,6 +72,16 @@ async def on_message_new(payload: dict[str, Any]) -> None: if env.get("mailbox_special_use") != "inbox": return + # Never during backfill. message.new fires for every imported message too, and + # classifying history is wrong twice over: it would file years-old inbox mail + # into Junk on the server, and it would do so inline in the backfill loop — + # ~70 ms of HTTP per message, serialized, on an account with thousands of + # them. It is also when accuracy is lowest, because Bayes has learned nothing + # yet. The backfill window belongs to the C2 learning bootstrap (§C.5), not + # to classification. + if env.get("account_state") != "live": + return + settings = await policy.load_settings(user_id) account_id = str(env.get("account_id") or "") if not settings.covers(account_id):