Antispam: lessons refused by the engine no longer pass for learned
Found on the NAS, where nothing had been learned in weeks while the UI said "Marked as spam": the source tree predated v1.3.2, so api/worker-* had no controller password and every lesson got 401 — and the trail of state that left behind kept it broken after the password was fixed. - /learn records a lesson only when rspamd accepted it. Recording refused ones made the menu claim "Marked as spam" and its dedup blocked every retry. - Bootstrap: done-markers move to a new key. The old one may have been written by the "-0" bug (a total outage read as a clean run of zero), which left an account marked done with nothing taught and never retried; every older install re-runs the bootstrap once, harmlessly (rspamd skips what it knows). - "Learn from my Spam folder now" in Settings > Antispam filtering queues the bootstrap for the user's accounts at once, instead of the hourly sweep. - webmail.sh update now recreates rspamd: it reads the controller password only when created, so a changed .env reached the app but never the engine. - doctor's advice for the 401 names both real causes (a pre-v1.3.2 compose file, or an rspamd started with an old password) and the fix for each; the old "update v1.3.2" advice could not help. - docs/nas-install.md: how to refresh the source tree on a tarball install. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -7,6 +7,8 @@ tables. Requires the plugin installed and `install antispam` run.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import types
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
@@ -32,6 +34,29 @@ from app.models import User
|
||||
BASE = "/api/plugins/antispam"
|
||||
|
||||
|
||||
class _SdkProxy:
|
||||
"""The router's SDK handle with a few calls swapped out. The SDK object is a
|
||||
frozen dataclass, and `.messages` / `.accounts` hand out a fresh object on
|
||||
every access, so patching attributes on them does not stick."""
|
||||
|
||||
def __init__(self, real: object, **overrides: object) -> None:
|
||||
self._real = real
|
||||
self._overrides = overrides
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
if name in self._overrides:
|
||||
return self._overrides[name]
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
def _router_module() -> object:
|
||||
return importlib.import_module("webmail_plugin_antispam.router")
|
||||
|
||||
|
||||
async def _raw(*_a: object, **_k: object) -> bytes:
|
||||
return b"Subject: x\r\n\r\nbody"
|
||||
|
||||
|
||||
async def _mk_user(sm: async_sessionmaker) -> uuid.UUID: # type: ignore[type-arg]
|
||||
async with sm() as s:
|
||||
user = User(
|
||||
@@ -188,18 +213,82 @@ async def test_ui_bundles_are_served_and_gated(ctx: _Ctx) -> None:
|
||||
assert (await ctx.client_a.get(f"{BASE}/ui/../hook.py")).status_code == 404
|
||||
|
||||
|
||||
async def test_learn_records_even_without_a_prior_verdict(ctx: _Ctx) -> None:
|
||||
async def test_learn_records_even_without_a_prior_verdict(
|
||||
ctx: _Ctx, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Mail the provider filed as spam has no verdict row of ours to update.
|
||||
|
||||
The write must therefore upsert — and it must happen before the move, or the
|
||||
passive-learning hook re-teaches the lesson the move announces.
|
||||
"""
|
||||
router_mod = _router_module()
|
||||
|
||||
async def _accepted(_raw: bytes, **_kw: object) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
router_mod,
|
||||
"_SDK",
|
||||
_SdkProxy(router_mod._SDK, messages=_SdkProxy(router_mod._SDK.messages, raw=_raw)),
|
||||
)
|
||||
monkeypatch.setattr(rspamd_mod, "learn", _accepted)
|
||||
message_id = uuid.uuid4()
|
||||
r = await ctx.client_a.post(
|
||||
f"{BASE}/learn", json={"message_id": str(message_id), "verdict": "spam"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["taught"] is True
|
||||
assert r.json()["moved"] is False # not this user's message: nothing to move
|
||||
|
||||
body = (await ctx.client_a.get(f"{BASE}/context/{message_id}")).json()
|
||||
assert body["verdict"]["learned"] == "spam"
|
||||
|
||||
|
||||
async def test_a_refused_lesson_is_not_recorded(ctx: _Ctx, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The engine said no (a 401 from a mismatched controller password, say).
|
||||
|
||||
Recording it anyway made the menu say "Marked as spam" for a lesson never
|
||||
taught, and its dedup then blocked every retry.
|
||||
"""
|
||||
router_mod = _router_module()
|
||||
|
||||
async def _refused(_raw: bytes, **_kw: object) -> bool:
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(
|
||||
router_mod,
|
||||
"_SDK",
|
||||
_SdkProxy(router_mod._SDK, messages=_SdkProxy(router_mod._SDK.messages, raw=_raw)),
|
||||
)
|
||||
monkeypatch.setattr(rspamd_mod, "learn", _refused)
|
||||
message_id = uuid.uuid4()
|
||||
r = await ctx.client_a.post(
|
||||
f"{BASE}/learn", json={"message_id": str(message_id), "verdict": "spam"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["taught"] is False
|
||||
body = (await ctx.client_a.get(f"{BASE}/context/{message_id}")).json()
|
||||
assert body["verdict"] is None
|
||||
|
||||
|
||||
async def test_bootstrap_now_queues_only_this_users_accounts(
|
||||
ctx: _Ctx, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
router_mod = _router_module()
|
||||
mine, theirs = uuid.uuid4(), uuid.uuid4()
|
||||
queued: list[dict[str, object]] = []
|
||||
|
||||
async def _live() -> list[tuple[uuid.UUID, uuid.UUID]]:
|
||||
return [(ctx.user_a, mine), (ctx.user_b, theirs)]
|
||||
|
||||
async def _enqueue(_func: object, **kwargs: object) -> None:
|
||||
queued.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
router_mod,
|
||||
"_SDK",
|
||||
_SdkProxy(router_mod._SDK, accounts=types.SimpleNamespace(live=_live), enqueue=_enqueue),
|
||||
)
|
||||
r = await ctx.client_a.post(f"{BASE}/bootstrap")
|
||||
assert r.json() == {"queued": 1}
|
||||
assert queued == [{"user_id": str(ctx.user_a), "account_id": str(mine), "force": True}]
|
||||
|
||||
@@ -459,3 +459,85 @@ def test_compose_gives_the_app_the_controller_password() -> None:
|
||||
f"{rel}: the app containers do not receive RSPAMD_CONTROLLER_PASSWORD — "
|
||||
"antispam will classify but never learn"
|
||||
)
|
||||
|
||||
|
||||
class _BootKv:
|
||||
"""A kv store holding a given value per key; records what gets written."""
|
||||
|
||||
def __init__(self, values: dict[str, object]) -> None:
|
||||
self.values = values
|
||||
self.written: list[tuple[str, object]] = []
|
||||
|
||||
async def get(self, key: str, default: object = None) -> object:
|
||||
return self.values.get(key, default)
|
||||
|
||||
async def set(self, key: str, value: object) -> None:
|
||||
self.written.append((key, value))
|
||||
|
||||
|
||||
def _boot_fake(kv: _BootKv, taught: list[bool]) -> object:
|
||||
class _M:
|
||||
async def sample(self, *_a: object, **_kw: object) -> list[uuid.UUID]:
|
||||
return [uuid.uuid4() for _ in range(12)]
|
||||
|
||||
async def raw(self, *_a: object, **_k: object) -> bytes:
|
||||
return b"x"
|
||||
|
||||
class _Fake:
|
||||
messages = _M()
|
||||
|
||||
def kv(self, _u: object) -> _BootKv:
|
||||
return kv
|
||||
|
||||
async def publish(self, *_a: object, **_k: object) -> None:
|
||||
taught.append(True)
|
||||
|
||||
return _Fake()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_old_style_marker_does_not_block_the_bootstrap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Markers under the old key may come from the `-0` bug: done, nothing taught."""
|
||||
account = str(uuid.uuid4())
|
||||
kv = _BootKv({"_bootstrapped": [account]})
|
||||
ran: list[bool] = []
|
||||
|
||||
async def _settings(_u: object) -> policy.Settings:
|
||||
return SETTINGS
|
||||
|
||||
async def _ok(_raw: bytes, **_kw: object) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(policy, "load_settings", _settings)
|
||||
monkeypatch.setattr(jobs, "_SDK", _boot_fake(kv, ran))
|
||||
monkeypatch.setattr(jobs.rspamd, "learn", _ok)
|
||||
monkeypatch.setattr(jobs, "PAUSE_SECONDS", 0)
|
||||
await jobs.bootstrap_account({}, user_id=str(uuid.uuid4()), account_id=account)
|
||||
assert ran == [True]
|
||||
assert kv.written == [("bootstrap_done", [account])]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_done_accounts_are_skipped_unless_forced(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = str(uuid.uuid4())
|
||||
|
||||
async def _settings(_u: object) -> policy.Settings:
|
||||
return SETTINGS
|
||||
|
||||
async def _ok(_raw: bytes, **_kw: object) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(policy, "load_settings", _settings)
|
||||
monkeypatch.setattr(jobs.rspamd, "learn", _ok)
|
||||
monkeypatch.setattr(jobs, "PAUSE_SECONDS", 0)
|
||||
|
||||
kv, ran = _BootKv({"bootstrap_done": [account]}), []
|
||||
monkeypatch.setattr(jobs, "_SDK", _boot_fake(kv, ran))
|
||||
await jobs.bootstrap_account({}, user_id=str(uuid.uuid4()), account_id=account)
|
||||
assert ran == [] # the hourly sweep leaves it alone
|
||||
|
||||
await jobs.bootstrap_account({}, user_id=str(uuid.uuid4()), account_id=account, force=True)
|
||||
assert ran == [True] # the button re-runs it
|
||||
assert kv.written == [] # already listed: not appended twice
|
||||
|
||||
Reference in New Issue
Block a user