M6 hosted mailboxes (off by default); dark message mode; fixes

Hosted mailboxes (SPEC App. D), behind HOSTED_MAILBOXES (off: unchanged): - Stalwart mail server, provisioned by `python -m app.hosted.provision`; directory read from a Postgres view, app passwords only for mail apps - native account per user, the worker signs in as master for the user - raw messages of hosted mailboxes only cached, fetched back on demand - migrations 0025 (accounts.kind, directory view), 0026 (IMAP ids bigint) Mail: - dark message mode switch in the reading pane (Thunderbird's algorithm) - moves and folder creation quote the folder name ("Deleted Items"): swipe-to-delete / Trash no longer bounce back - thread unread counters: lock before recount, sorted lock order, UIDVALIDITY re-import recounts, nightly repair_threads - sidebar accounts sorted alphabetically, more space above headings Settings: Security headings renamed; antispam accounts are explicit (none ticked = none filtered), input widths, Activity hidden. Plugin bundles served with Cache-Control: no-cache. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 19:25:10 +02:00
parent d45fbf2c44
commit b88855931e
62 changed files with 2427 additions and 188 deletions
@@ -18,6 +18,19 @@ os.environ.setdefault(
 os.environ["REDIS_URL"] = "redis://127.0.0.1:6380/15"
 os.environ.setdefault("MASTER_KEY", base64.b64encode(b"\x00" * 32).decode())
 os.environ.setdefault("BLOB_ROOT", "./var/blobs-test")
+# Hosted mailboxes off: tests that need the mail server turn them on themselves
+# (tests/hosted/), against the dev stack's Stalwart container.
+os.environ["HOSTED_MAILBOXES"] = "false"
+for _key, _value in {
+    "STALWART_URL": "http://127.0.0.1:8180",
+    "STALWART_ADMIN_SECRET": "stalwart_admin_dev",
+    "STALWART_MASTER_SECRET": "stalwart_master_dev",
+    "STALWART_IMAP_HOST": "127.0.0.1",
+    "STALWART_IMAP_PORT": "1993",
+    "STALWART_SMTP_HOST": "127.0.0.1",
+    "STALWART_SMTP_PORT": "1465",
+}.items():
+    os.environ.setdefault(_key, _value)

 from collections.abc import AsyncIterator  # noqa: E402

@@ -0,0 +1,269 @@
+"""Hosted mailboxes (SPEC App. D) against the dev stack's Stalwart container.
+
+Sign-in through the directory (password, 2FA, block), app passwords made by the
+mail server, the permanent native account, and raw messages kept only by the
+mail server (fetched back on demand). Skipped when the container is not up.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import imaplib
+import smtplib
+import ssl
+import uuid
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from email.message import EmailMessage
+
+import httpx
+import pytest
+import pytest_asyncio
+from httpx import ASGITransport
+from sqlalchemy import delete, select
+
+from app.config import get_settings
+from app.db import session_scope
+from app.hosted import admin, native
+from app.identity import app_passwords
+from app.identity.passwords import hash_password
+from app.identity.sessions import create_session
+from app.identity.user_admin import delete_user, set_blocked
+from app.indexer import blobstore, rawstore
+from app.main import create_app
+from app.models import Account, Attachment, Mailbox, Message, User
+
+PASSWORD = "Correct-Horse-7"
+
+
+async def _server_domain() -> str | None:
+    try:
+        settings = await admin.get_one("SystemSettings")
+        domain = await admin.get_one("Domain", str(settings["defaultDomainId"]))
+    except admin.MailServerError:
+        return None
+    return str(domain["name"])
+
+
+@dataclass
+class Ctx:
+    user: User
+    web: httpx.AsyncClient
+
+
+@pytest_asyncio.fixture
+async def ctx(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Ctx]:
+    domain = await _server_domain()
+    if domain is None:
+        pytest.skip("the dev mail server (docker compose service stalwart) is not running")
+    settings = get_settings()
+    monkeypatch.setattr(settings, "hosted_mailboxes", True)
+    monkeypatch.setattr(settings, "app_domain", domain)
+    handle = f"hosted{uuid.uuid4().hex[:8]}@{domain}"
+    async with session_scope() as db:
+        user = User(handle=handle, password_hash=hash_password(PASSWORD))
+        db.add(user)
+    token = await create_session(user.id, "pytest")
+    web = httpx.AsyncClient(
+        transport=ASGITransport(app=create_app()),
+        base_url="http://testserver",
+        cookies={"session": token, "csrf": "c"},
+        headers={"X-CSRF": "c"},
+    )
+    try:
+        yield Ctx(user, web)
+    finally:
+        await web.aclose()
+        await admin.delete_account(handle)
+        async with session_scope() as db:
+            await db.execute(delete(User).where(User.id == user.id))
+
+
+def _imap_login(user: str, password: str) -> bool:
+    tls = ssl.create_default_context()
+    tls.check_hostname = False
+    tls.verify_mode = ssl.CERT_NONE
+    settings = get_settings()
+    try:
+        # A blocked account's sign-in is not answered: the server drops the
+        # connection after a minute. Not answering within seconds is a refusal.
+        conn = imaplib.IMAP4_SSL(
+            settings.stalwart_imap_host, settings.stalwart_imap_port, ssl_context=tls, timeout=5
+        )
+    except OSError:
+        return False
+    try:
+        conn.login(user, password)
+        conn.logout()
+        return True
+    except (imaplib.IMAP4.error, OSError):
+        return False
+
+
+async def _can_sign_in(user: str, password: str) -> bool:
+    return await asyncio.to_thread(_imap_login, user, password)
+
+
+def _deliver(to: str, subject: str, attachment: bytes) -> None:
+    msg = EmailMessage()
+    msg["From"] = "Anna Test <anna@example.org>"
+    msg["To"] = to
+    msg["Subject"] = subject
+    msg.set_content("Hello,\n\nthe report is attached.\n")
+    msg.add_attachment(attachment, maintype="application", subtype="octet-stream", filename="r.bin")
+    with smtplib.SMTP(get_settings().stalwart_imap_host, 1025, timeout=20) as smtp:
+        smtp.send_message(msg)
+
+
+async def test_the_webmail_password_never_opens_the_mailbox(ctx: Ctx) -> None:
+    """Mail apps sign in with app passwords only — 2FA or not."""
+    assert not await _can_sign_in(ctx.user.handle, PASSWORD)
+    assert not await _can_sign_in(ctx.user.handle, "wrong")
+
+
+async def test_app_passwords_work_for_mail_apps_and_dav_and_are_revoked_everywhere(
+    ctx: Ctx,
+) -> None:
+    handle = ctx.user.handle
+    async with session_scope() as db:
+        row, password = await app_passwords.create(db, ctx.user.id, "Thunderbird")
+    assert password.startswith("app_") and len(password) == app_passwords.SERVER_LENGTH
+    assert row.server_id
+    assert await _can_sign_in(handle, password)
+    async with session_scope() as db:
+        assert await app_passwords.verify(db, ctx.user.id, password) is not None  # DAV
+
+    # 2FA turned on changes nothing for it.
+    async with session_scope() as db:
+        user = await db.get(User, ctx.user.id)
+        assert user is not None
+        user.totp_secret = b"x"
+    assert await _can_sign_in(handle, password)
+
+    # Blocked: nothing opens the mailbox; unblocked: it does again.
+    async with session_scope() as db:
+        user = await db.get(User, ctx.user.id)
+        assert user is not None
+        await set_blocked(db, user, True)
+    assert not await _can_sign_in(handle, password)
+    async with session_scope() as db:
+        user = await db.get(User, ctx.user.id)
+        assert user is not None
+        await set_blocked(db, user, False)
+    assert await _can_sign_in(handle, password)
+
+    async with session_scope() as db:
+        assert await app_passwords.revoke(db, ctx.user.id, row.id)
+    assert not await _can_sign_in(handle, password)
+
+
+async def test_the_native_account_keeps_mail_once_and_cannot_be_removed(ctx: Ctx) -> None:
+    from app.sync.engine import bootstrap_and_backfill
+
+    async with session_scope() as db:
+        user = await db.get(User, ctx.user.id)
+        assert user is not None
+        account = await native.ensure(db, user)
+        assert account is not None
+        account_id = account.id
+    payload = uuid.uuid4().bytes * 5000
+    await asyncio.to_thread(_deliver, ctx.user.handle, "Report", payload)
+
+    # Synced like any IMAP account (delivery is asynchronous: retry briefly).
+    message = None
+    for _ in range(20):
+        await bootstrap_and_backfill(account_id)
+        async with session_scope() as db:
+            message = await db.scalar(
+                select(Message).where(Message.account_id == account_id, Message.subject == "Report")
+            )
+        if message is not None:
+            break
+        await asyncio.sleep(0.5)
+    assert message is not None
+
+    # A new mailbox has the plain folder names (not "Deleted Items", …).
+    async with session_scope() as db:
+        folders = dict(
+            (
+                await db.execute(
+                    select(Mailbox.special_use, Mailbox.path).where(
+                        Mailbox.account_id == account_id
+                    )
+                )
+            ).all()
+        )
+    assert folders["trash"] == "Trash" and folders["sent"] == "Sent"
+    assert folders["junk"] == "Junk" and folders["archive"] == "Archive"
+
+    # The raw message is not kept for good — only cached.
+    sha = message.blob_sha256
+    assert not blobstore.exists(ctx.user.id, sha)
+    assert rawstore.cache_path(ctx.user.id, sha).exists()
+
+    # Gone from the cache, the attachment is fetched back from the mail server.
+    rawstore.drop(ctx.user.id, sha)
+    async with session_scope() as db:
+        att = await db.scalar(select(Attachment).where(Attachment.message_id == message.id))
+    assert att is not None
+    got = await ctx.web.get(f"/api/messages/{message.id}/attachments/{att.id}?download=1")
+    assert got.status_code == 200 and got.content == payload
+    assert rawstore.cache_path(ctx.user.id, sha).exists()
+
+    # Permanent: not removable, its server settings not changeable.
+    listed = (await ctx.web.get("/api/accounts")).json()
+    assert [a["kind"] for a in listed] == ["native"]
+    assert (await ctx.web.delete(f"/api/accounts/{account_id}")).status_code == 409
+    patched = await ctx.web.patch(f"/api/accounts/{account_id}", json={"password": "x"})
+    assert patched.status_code == 400 and patched.json()["error"]["code"] == "native_account"
+
+
+async def test_deleting_the_user_deletes_the_mailbox(ctx: Ctx) -> None:
+    await admin.ensure_account(ctx.user.handle)  # brings it into the server
+    assert await admin.account_id(ctx.user.handle) is not None
+    async with session_scope() as db:
+        user = await db.get(User, ctx.user.id)
+        assert user is not None
+        await delete_user(db, user)
+    assert await admin.account_id(ctx.user.handle) is None
+    async with session_scope() as db:
+        assert await db.scalar(select(Account.id).where(Account.user_id == ctx.user.id)) is None
+
+
+async def test_the_cache_keeps_the_most_recently_used(ctx: Ctx) -> None:
+    import os
+    import time
+
+    uid = ctx.user.id
+    old = rawstore.put_cached(uid, b"a" * 1000)
+    new = rawstore.put_cached(uid, b"b" * 1000)
+    past = time.time() - 60
+    os.utime(rawstore.cache_path(uid, old), (past, past))
+    rawstore.prune(uid, max_bytes=1500)
+    assert not rawstore.cache_path(uid, old).exists()
+    assert rawstore.cache_path(uid, new).exists()
+    stale = time.time() - rawstore.KEEP_S - 10
+    os.utime(rawstore.cache_path(uid, new), (stale, stale))
+    rawstore.prune(uid)
+    assert not rawstore.cache_path(uid, new).exists()
+
+
+async def test_moves_reach_folders_with_spaces_in_their_names(ctx: Ctx) -> None:
+    """Older mailboxes (and many providers) say "Deleted Items": the name goes
+    over the wire quoted, or the move is refused and Trash never fills."""
+    from app.providers.imap_smtp import ImapSmtpProvider
+
+    await admin.ensure_account(ctx.user.handle)
+    msg = EmailMessage()
+    msg["From"] = "a@example.com"
+    msg["To"] = ctx.user.handle
+    msg["Subject"] = "Swipe me"
+    msg.set_content("x")
+    provider = ImapSmtpProvider(native.native_provider_config(ctx.user.handle))
+    try:
+        await provider.create_mailbox("Deleted Items")
+        uid = await provider.append("INBOX", msg.as_bytes(), frozenset())
+        moved = await provider.move(uid, "INBOX", "Deleted Items")
+        assert moved > 0
+    finally:
+        await provider.close()
@@ -8,6 +8,7 @@ from __future__ import annotations

 import uuid
 from email.message import EmailMessage
+from pathlib import Path

 from app.indexer import indexer
 from app.indexer.mime import parse_message, parse_message_file
@@ -50,6 +51,8 @@ async def test_parse_bounded_streams_large_messages(monkeypatch) -> None:  # typ

    monkeypatch.setattr(indexer, "parse_message_file", fake_file)
    monkeypatch.setattr(indexer, "parse_message", fake_bytes)
+    # The message's file, written before parsing (blob store or raw cache).
+    monkeypatch.setattr(indexer.rawstore, "locate", lambda user_id, sha: Path("/blob"))

    user_id = uuid.uuid4()
    small = await indexer._parse_bounded(user_id, "sha", b"x" * 16)
@@ -115,7 +115,15 @@ async def test_settings_defaults_then_merge(ctx: _Ctx) -> None:
    s = (await ctx.client_a.get(f"{BASE}/settings")).json()
    assert (s["move_threshold"], s["banner_threshold"]) == (12.0, 6.0)
    assert s["network_checks"] is False  # privacy default (§C.2)
-    assert s["enabled_accounts"] == []  # empty = every account
+    assert s["enabled_accounts"] is None  # every account
+
+    # Every account off is stored as such (not read back as "all on"), and
+    # all_accounts goes back to every account.
+    s = (await ctx.client_a.patch(f"{BASE}/settings", json={"enabled_accounts": []})).json()
+    assert s["enabled_accounts"] == []
+    assert (await ctx.client_a.get(f"{BASE}/settings")).json()["enabled_accounts"] == []
+    s = (await ctx.client_a.patch(f"{BASE}/settings", json={"all_accounts": True})).json()
+    assert s["enabled_accounts"] is None

    # PATCH merges: the field we leave out keeps its value.
    s = (await ctx.client_a.patch(f"{BASE}/settings", json={"move_threshold": 15.0})).json()
@@ -32,7 +32,7 @@ from webmail_plugin_antispam import policy, rspamd
 pytestmark = pytest.mark.asyncio

 SETTINGS = policy.Settings(
-    enabled_accounts=[],
+    enabled_accounts=None,
    move_threshold=12.0,
    banner_threshold=6.0,
    network_checks=False,
@@ -68,7 +68,7 @@ def wired(monkeypatch: pytest.MonkeyPatch):  # type: ignore[no-untyped-def]

    async def _settings(_u: object) -> policy.Settings:
        return policy.Settings(
-            enabled_accounts=[],
+            enabled_accounts=None,
            move_threshold=12.0,
            banner_threshold=6.0,
            network_checks=False,
@@ -14,7 +14,7 @@ import pytest
 from webmail_plugin_antispam import hook, jobs, policy

 SETTINGS = policy.Settings(
-    enabled_accounts=[],
+    enabled_accounts=None,
    move_threshold=12.0,
    banner_threshold=6.0,
    network_checks=False,
@@ -169,7 +169,7 @@ async def test_bootstrap_respects_the_setting(monkeypatch: pytest.MonkeyPatch) -

    async def _settings(_u: object) -> policy.Settings:
        return policy.Settings(
-            enabled_accounts=[],
+            enabled_accounts=None,
            move_threshold=12.0,
            banner_threshold=6.0,
            network_checks=False,
@@ -6,7 +6,7 @@ import pytest
 from webmail_plugin_antispam.policy import Settings, decide, matches

 SETTINGS = Settings(
-    enabled_accounts=[],
+    enabled_accounts=None,
    move_threshold=12.0,
    banner_threshold=6.0,
    network_checks=False,
@@ -79,7 +79,7 @@ def test_pattern_matching(patterns: set[str], email: str, hit: bool) -> None:


 def test_per_account_enablement() -> None:
-    """Empty list means every account; a populated one is an allow-list."""
+    """None means every account; a list is an allow-list — empty, none."""
    everywhere = SETTINGS
    assert everywhere.covers("any-account-id")

@@ -93,6 +93,15 @@ def test_per_account_enablement() -> None:
    assert scoped.covers("acc-1")
    assert not scoped.covers("acc-2")

+    nowhere = Settings(
+        enabled_accounts=[],
+        move_threshold=12.0,
+        banner_threshold=6.0,
+        network_checks=False,
+        learn_from_provider_junk=True,
+    )
+    assert not nowhere.covers("acc-1")
+

 # --- contacts are never filed as spam ----------------------------------------

@@ -0,0 +1,104 @@
+"""Thread counters that drifted from their messages are repaired (nightly safety
+net, app/threading_/threader.py repair_threads)."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from email.message import EmailMessage
+
+from sqlalchemy import delete, select, update
+
+from app.db import session_scope
+from app.identity.passwords import hash_password
+from app.indexer.indexer import persist_message
+from app.models import Account, Draft, Mailbox, Thread, User
+from app.providers.base import RemoteMessage
+from app.threading_.threader import repair_threads
+
+
+def _raw(subject: str) -> bytes:
+    m = EmailMessage()
+    m["Subject"] = subject
+    m["From"] = "billing@example.com"
+    m["To"] = "me@example.com"
+    m["Message-ID"] = f"<{uuid.uuid4().hex}@example.com>"
+    m.set_content("receipt")
+    return m.as_bytes()
+
+
+async def test_repair_recounts_stale_threads_and_drops_empty_ones() -> None:
+    async with session_scope() as db:
+        user = User(
+            handle=f"rep{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")
+        )
+        db.add(user)
+        await db.flush()
+        account = Account(
+            user_id=user.id,
+            email="me@example.com",
+            imap_config={},
+            smtp_config={},
+            secret_ct=b"",
+            discovery_src="manual",
+        )
+        db.add(account)
+        await db.flush()
+        inbox = Mailbox(account_id=account.id, user_id=user.id, path="INBOX", special_use="inbox")
+        db.add(inbox)
+        await db.flush()
+        result = await persist_message(
+            db,
+            user_id=user.id,
+            account_id=account.id,
+            mailbox_id=inbox.id,
+            mailbox_special_use="inbox",
+            remote=RemoteMessage(
+                uid=1,
+                raw=_raw("Your receipt"),
+                flags=frozenset({"\\Seen"}),
+                internal_date=datetime.now(UTC),
+            ),
+        )
+        assert result is not None
+        tid = result.message.thread_id
+        assert tid is not None
+        # A read message whose thread still says "unread" (the bug), and a
+        # thread left behind with no messages at all.
+        await db.execute(update(Thread).where(Thread.id == tid).values(unread_count=1))
+        empty = Thread(
+            user_id=user.id,
+            subject_norm="gone",
+            message_count=1,
+            unread_count=1,
+            last_message_at=datetime.now(UTC),
+        )
+        db.add(empty)
+        # An emptied thread a reply draft still points at: kept, counts zeroed.
+        replied = Thread(
+            user_id=user.id,
+            subject_norm="replying",
+            message_count=1,
+            unread_count=0,
+            last_message_at=datetime.now(UTC),
+        )
+        db.add(replied)
+        await db.flush()
+        db.add(Draft(user_id=user.id, account_id=account.id, thread_id=replied.id, payload={}))
+        await db.flush()
+        empty_id = empty.id
+        replied_id = replied.id
+    try:
+        async with session_scope() as db:
+            assert await repair_threads(db, user.id) == 3
+        async with session_scope() as db:
+            thread = await db.get(Thread, tid)
+            assert thread is not None and thread.unread_count == 0 and thread.message_count == 1
+            assert await db.get(Thread, empty_id) is None
+            kept = await db.get(Thread, replied_id)
+            assert kept is not None and kept.message_count == 0
+            assert await repair_threads(db, user.id) == 0  # nothing left to repair
+    finally:
+        async with session_scope() as db:
+            await db.execute(delete(User).where(User.id == user.id))  # cascades
+            assert await db.scalar(select(User.id).where(User.id == user.id)) is None