sync-v2 §10.7: fix +- sign in RSS-growth output (use :+.1f)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Makefile

@@ -9,8 +9,12 @@ reset-db:  ## DESTRUCTIVE: wipe db + search + blobs, re-migrate
 	cd backend && alembic upgrade head
 api:
 	uvicorn app.main:create_app --factory --reload --port 8000 --app-dir backend
-worker:
-	cd backend && arq workers.worker.WorkerSettings
+worker:      ## run both sync + jobs in one process (dev)
+	cd backend && WORKER_ROLE=all python -m workers.run
+worker-sync: ## run only the account-sync supervisor
+	cd backend && WORKER_ROLE=sync python -m workers.run
+worker-jobs: ## run only the arq job consumers (interactive + bulk)
+	cd backend && WORKER_ROLE=jobs python -m workers.run
 web:
 	cd frontend && npm run dev
 test:

backend/Dockerfile

@@ -0,0 +1,16 @@
+# Shared image for the worker roles (refactor §1). Built from backend/.
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install deps first for layer caching.
+COPY pyproject.toml ./
+COPY app ./app
+COPY workers ./workers
+COPY migrations ./migrations
+COPY alembic.ini ./alembic.ini
+RUN pip install --no-cache-dir .
+
+# Default to the combined role; docker-compose overrides WORKER_ROLE per service.
+ENV WORKER_ROLE=all
+CMD ["python", "-m", "workers.run"]

backend/app/accounts/service.py

@@ -80,6 +80,20 @@ async def delete_account(
     await session.execute(delete(Account).where(Account.id == account_id))
     await session.flush()
 
+    # Deregister from sync:live and drop any lease so the owning sync worker
+    # stops the account's runner on its next renewal (refactor §2 fencing).
+    from app.events import get_event_bus
+    from app.redis_client import get_redis
+    from app.sync import leases
+
+    await leases.mark_not_live(get_redis(), account_id)
+    # Sync v2 §3: signal the owning worker to tear the runner down now (≤30s),
+    # rather than waiting for the lease to lapse.
+    await get_event_bus().publish(
+        "account.state_changed",
+        {"user_id": str(user_id), "account_id": str(account_id), "state": "deleted"},
+    )
+
     # Content-addressed blobs are shared across the user's accounts (dedup): only
     # remove a blob when no remaining message/attachment references it.
     for sha in candidate_shas:

backend/app/api/accounts.py

@@ -30,7 +30,7 @@ from app.api.schemas import AccountOut
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
 from app.models import Account
-from app.queue import get_arq_pool
+from app.queue import enqueue_job
 from app.redis_client import get_redis
 from app.security.crypto import encrypt_secret
 
@@ -91,9 +91,8 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
 async def sync_all_accounts(user: CurrentUser, db: DbSession) -> dict[str, str]:
     """Poll every folder of each of the user's accounts (manual Refresh)."""
     account_ids = list(await db.scalars(select(Account.id).where(Account.user_id == user.id)))
-    pool = await get_arq_pool()
     for account_id in account_ids:
-        await pool.enqueue_job("refresh_account_job", str(account_id))
+        await enqueue_job("refresh_account_job", str(account_id))
     return {"status": "queued"}
 
 
@@ -145,8 +144,7 @@ async def create_new_account(
         raise_api_error(409, "account_exists", "an account for this email already exists")
     await db.commit()
 
-    pool = await get_arq_pool()
-    await pool.enqueue_job("sync_bootstrap_account", str(account.id))
+    await enqueue_job("sync_bootstrap_account", str(account.id))
 
     return _detail(account)
 
@@ -227,8 +225,7 @@ async def update_account(
     await db.commit()
 
     if conn_change:
-        pool = await get_arq_pool()
-        await pool.enqueue_job("sync_bootstrap_account", str(account.id))
+        await enqueue_job("sync_bootstrap_account", str(account.id))
 
     return _detail(account)
 

backend/app/api/actions.py

@@ -15,7 +15,7 @@ from app.db import get_session
 from app.events import get_event_bus
 from app.identity.deps import CurrentUser, require_csrf
 from app.models import Mailbox, Message, Thread
-from app.queue import get_arq_pool
+from app.queue import enqueue_job
 from app.sync.actions import apply_flags_to_messages, sanitize_flags
 
 router = APIRouter(tags=["mail"], dependencies=[Depends(require_csrf)])
@@ -26,10 +26,9 @@ DbSession = Annotated[AsyncSession, Depends(get_session)]
 async def _push_and_publish(
     user_id: uuid.UUID, changed: list[Message], add: frozenset[str], remove: frozenset[str]
 ) -> None:
-    pool = await get_arq_pool()
     bus = get_event_bus()
     for message in changed:
-        await pool.enqueue_job("push_flags_job", str(message.id), sorted(add), sorted(remove))
+        await enqueue_job("push_flags_job", str(message.id), sorted(add), sorted(remove))
         await bus.publish(
             "flags.changed",
             {
@@ -95,8 +94,7 @@ async def _move_thread(
     )
     if thread is None:
         raise_api_error(404, "not_found", "thread not found")
-    pool = await get_arq_pool()
-    await pool.enqueue_job("move_thread_to_folder_job", str(thread_id), special)
+    await enqueue_job("move_thread_to_folder_job", str(thread_id), special)
 
 
 @router.post("/threads/{thread_id}/archive")
@@ -126,8 +124,7 @@ async def delete_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession)
     )
     if thread is None:
         raise_api_error(404, "not_found", "thread not found")
-    pool = await get_arq_pool()
-    await pool.enqueue_job("delete_thread_job", str(thread_id))
+    await enqueue_job("delete_thread_job", str(thread_id))
     return {"status": "queued"}
 
 
@@ -154,6 +151,5 @@ async def move_message(
     message.mailbox_id = dst.id  # optimistic
     await db.commit()
 
-    pool = await get_arq_pool()
-    await pool.enqueue_job("move_job", str(message.id), str(dst.id), str(src_mailbox_id))
+    await enqueue_job("move_job", str(message.id), str(dst.id), str(src_mailbox_id))
     return {"id": str(message.id), "mailbox_id": str(dst.id)}

backend/app/api/compose.py

@@ -25,7 +25,7 @@ from app.outbox.schemas import (
     SendResponse,
 )
 from app.outbox.service import DraftError, cancel_outbox, create_outbox, upsert_draft
-from app.queue import get_arq_pool
+from app.queue import enqueue_job
 
 router = APIRouter(tags=["compose"], dependencies=[Depends(require_csrf)])
 
@@ -155,9 +155,7 @@ async def list_remote_drafts(user: CurrentUser, db: DbSession) -> list[RemoteDra
 
 
 @router.post("/drafts/adopt/{message_id}", response_model=DraftOut)
-async def adopt_remote_draft(
-    message_id: uuid.UUID, user: CurrentUser, db: DbSession
-) -> DraftOut:
+async def adopt_remote_draft(message_id: uuid.UUID, user: CurrentUser, db: DbSession) -> DraftOut:
     """Turn a Drafts-folder message into an editable local draft (linked by UID)."""
     msg = await db.scalar(
         select(Message)
@@ -235,8 +233,7 @@ async def publish_draft(draft_id: uuid.UUID, user: CurrentUser, db: DbSession) -
     draft = await db.scalar(select(Draft).where(Draft.id == draft_id, Draft.user_id == user.id))
     if draft is None:
         raise_api_error(404, "not_found", "draft not found")
-    pool = await get_arq_pool()
-    await pool.enqueue_job("push_draft_to_imap_job", str(draft_id))
+    await enqueue_job("push_draft_to_imap_job", str(draft_id))
     return {"status": "queued"}
 
 
@@ -250,8 +247,7 @@ async def delete_draft(draft_id: uuid.UUID, user: CurrentUser, db: DbSession) ->
     await db.commit()
     # Remove the mirrored copy from the server Drafts folder too.
     if imap_uid:
-        pool = await get_arq_pool()
-        await pool.enqueue_job("remove_imap_draft_job", str(account_id), imap_uid)
+        await enqueue_job("remove_imap_draft_job", str(account_id), imap_uid)
     return {"status": "deleted"}
 
 
@@ -265,8 +261,7 @@ async def send(body: SendRequest, user: CurrentUser, db: DbSession) -> SendRespo
     outbox = await create_outbox(db, user_id=user.id, draft=draft)
     await db.commit()
 
-    pool = await get_arq_pool()
-    await pool.enqueue_job(
+    await enqueue_job(
         "send_job",
         str(outbox.id),
         _defer_by=timedelta(seconds=get_settings().undo_seconds),

backend/app/concurrency.py

@@ -0,0 +1,32 @@
+"""Per-event-loop semaphores for bounded concurrency (refactor §4, §7).
+
+Connection and parse caps are process-local but must be bound to the running
+event loop: an :class:`asyncio.Semaphore` created on one loop cannot be awaited
+on another (a real hazard under pytest, which uses a fresh loop per test). These
+helpers hand back a semaphore keyed by ``(group, key)`` for the current loop,
+creating it on first use and dropping the whole set when the loop is collected.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import weakref
+
+# group -> {loop: {key: Semaphore}}. WeakKeyDictionary drops a loop's semaphores
+# once the loop is gone, so no state leaks across tests or restarts.
+_registry: dict[
+    str, weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Semaphore]]
+] = {}
+
+
+def loop_semaphore(group: str, key: str, cap: int) -> asyncio.Semaphore:
+    """Return the current loop's semaphore for ``(group, key)`` (cap on first use)."""
+    loop = asyncio.get_running_loop()
+    per_group = _registry.setdefault(group, weakref.WeakKeyDictionary())
+    per_loop = per_group.get(loop)
+    if per_loop is None:
+        per_loop = per_group[loop] = {}
+    sem = per_loop.get(key)
+    if sem is None:
+        sem = per_loop[key] = asyncio.Semaphore(cap)
+    return sem

backend/app/config.py

@@ -29,11 +29,32 @@ class Settings(BaseSettings):
     meili_key: str = Field(default="", alias="MEILI_KEY")
 
     undo_seconds: int = Field(default=10, alias="UNDO_SECONDS")
-    poll_interval: int = Field(default=120, alias="POLL_INTERVAL")
     session_ttl_days: int = Field(default=30, alias="SESSION_TTL_DAYS")
     max_attachment_mb: int = Field(default=25, alias="MAX_ATTACHMENT_MB")
     log_level: str = Field(default="INFO", alias="LOG_LEVEL")
 
+    # --- Sync engine v2 (workers layer) --------------------------------------
+    # Worker role: "sync" (lease-owned account runners) | "jobs" (arq consumer)
+    # | "all" (both, single-process dev default).
+    worker_role: str = Field(default="all", alias="WORKER_ROLE")
+    interactive_concurrency: int = Field(default=20, alias="INTERACTIVE_CONCURRENCY")
+    bulk_concurrency: int = Field(default=2, alias="BULK_CONCURRENCY")
+    # Lease protocol (§2): account ownership across sync workers.
+    max_accounts_per_worker: int = Field(default=400, alias="MAX_ACCOUNTS_PER_WORKER")
+    lease_ttl: int = Field(default=60, alias="LEASE_TTL")
+    lease_renew: int = Field(default=20, alias="LEASE_RENEW")
+    # Activity tiers (§5): hot = persistent IDLE, warm/cold = transient polls.
+    poll_warm: int = Field(default=180, alias="POLL_WARM")
+    poll_cold: int = Field(default=900, alias="POLL_COLD")
+    tier_hot_activity_h: int = Field(default=24, alias="TIER_HOT_ACTIVITY_H")
+    tier_session_grace_min: int = Field(default=30, alias="TIER_SESSION_GRACE_MIN")
+    tier_cold_after_d: int = Field(default=3, alias="TIER_COLD_AFTER_D")
+    hot_budget_per_worker: int = Field(default=250, alias="HOT_BUDGET_PER_WORKER")
+    # Memory/connection bounds (§7, §4): peak RAM ~= PARSE_CONCURRENCY x STREAM_PARSE_MB.
+    per_host_conn_cap: int = Field(default=50, alias="PER_HOST_CONN_CAP")
+    parse_concurrency: int = Field(default=4, alias="PARSE_CONCURRENCY")
+    stream_parse_mb: int = Field(default=8, alias="STREAM_PARSE_MB")
+
     oauth_google_client_id: str = Field(default="", alias="OAUTH_GOOGLE_CLIENT_ID")
     oauth_google_client_secret: str = Field(default="", alias="OAUTH_GOOGLE_CLIENT_SECRET")
     oauth_ms_client_id: str = Field(default="", alias="OAUTH_MS_CLIENT_ID")

backend/app/events.py

@@ -38,6 +38,9 @@ ALLOWED_EVENTS: frozenset[str] = frozenset(
         "account.state_changed",
         "draft.saved",
         "outbox.state_changed",
+        # Sync v2 §3: published on login and every WS connect; drives tier
+        # promotion in the sync workers (§5). Payload: {user_id}.
+        "user.active",
     }
 )
 

backend/app/identity/routes.py

@@ -12,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.api.errors import raise_api_error
 from app.config import get_settings
 from app.db import get_session
+from app.events import get_event_bus
 from app.identity import totp as totp_mod
 from app.identity.cookies import (
     clear_auth_cookies,
@@ -50,6 +51,8 @@ from app.identity.sessions import (
     revoke_session,
 )
 from app.models import User
+from app.redis_client import get_redis
+from app.sync import leases
 
 router = APIRouter(prefix="/auth", tags=["auth"])
 
@@ -116,6 +119,11 @@ async def login(body: LoginRequest, request: Request, response: Response, db: Db
     if user is None:
         raise_api_error(401, "invalid_credentials", "incorrect handle or password")
     await _establish_session(request, response, user)
+    # Sync v2 §3/§5: a login marks the user active (session heat) and promotes
+    # their accounts' tier to hot.
+    settings = get_settings()
+    await leases.mark_user_active(get_redis(), user.id, settings.tier_session_grace_min * 60)
+    await get_event_bus().publish("user.active", {"user_id": str(user.id)})
     return _user_out(user)
 
 

backend/app/indexer/indexer.py

@@ -17,8 +17,10 @@ from sqlalchemy import select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.concurrency import loop_semaphore
+from app.config import get_settings
 from app.indexer import blobstore
-from app.indexer.mime import ParsedEmail, parse_message
+from app.indexer.mime import ParsedEmail, parse_message, parse_message_file
 from app.indexer.sanitize import sanitize_html
 from app.models import Attachment, BodyPart, KnownRecipient, Message
 from app.providers.base import RemoteMessage
@@ -115,6 +117,21 @@ async def _resolve_message_id(
     return parsed_id
 
 
+async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmail:
+    """Parse under the PARSE_CONCURRENCY cap; stream large messages from disk (§7).
+
+    Peak RAM stays ~PARSE_CONCURRENCY x STREAM_PARSE_MB: only a few parses run at
+    once, and a message above the threshold is parsed from its (already written)
+    blob file instead of being held as an in-memory tree beside its raw bytes.
+    """
+    settings = get_settings()
+    threshold = settings.stream_parse_mb * 1024 * 1024
+    async with loop_semaphore("mime_parse", "parse", settings.parse_concurrency):
+        if len(raw) > threshold:
+            return parse_message_file(blobstore.blob_path(user_id, sha))
+        return parse_message(raw)
+
+
 async def persist_message(
     session: AsyncSession,
     *,
@@ -132,7 +149,7 @@ async def persist_message(
         return None
 
     sha = blobstore.store(user_id, remote.raw)
-    parsed = parse_message(remote.raw)
+    parsed = await _parse_bounded(user_id, sha, remote.raw)
 
     message_id_hdr = await _resolve_message_id(
         session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw

backend/app/indexer/mime.py

@@ -7,6 +7,7 @@ Header RFC2047 decoding is automatic under ``policy.default``.
 from __future__ import annotations
 
 import email
+import os
 from dataclasses import dataclass, field
 from datetime import datetime
 from email.headerregistry import AddressHeader
@@ -104,10 +105,21 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
 
 
 def parse_message(raw: bytes) -> ParsedEmail:
-    """Parse a raw RFC822 message into a :class:`ParsedEmail`."""
+    """Parse a raw RFC822 message (in memory) into a :class:`ParsedEmail`."""
     msg = email.message_from_bytes(raw, policy=default_policy)
     assert isinstance(msg, EmailMessage)
+    return _parse_email(msg)
 
+
+def parse_message_file(path: str | os.PathLike[str]) -> ParsedEmail:
+    """Parse a message straight from a blob file handle (large-message path, §7)."""
+    with open(path, "rb") as handle:
+        msg = email.message_from_binary_file(handle, policy=default_policy)
+    assert isinstance(msg, EmailMessage)
+    return _parse_email(msg)
+
+
+def _parse_email(msg: EmailMessage) -> ParsedEmail:
     message_id = normalize_message_id(msg.get("Message-ID"))
     in_reply_to_ids = normalize_reference_list(msg.get("In-Reply-To"))
     in_reply_to = in_reply_to_ids[0] if in_reply_to_ids else None

backend/app/outbox/draft_sync.py

@@ -34,9 +34,7 @@ async def _ensure_drafts_folder(provider: Any, account: Account, user_id: uuid.U
     uidvalidity = next((r.uidvalidity for r in remote if r.path == path), 0)
     async with session_scope() as session:
         existing = await session.scalar(
-            select(Mailbox).where(
-                Mailbox.account_id == account.id, Mailbox.special_use == "drafts"
-            )
+            select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )
         if existing is not None:
             return existing.path
@@ -72,9 +70,7 @@ async def push_draft_to_imap_job(ctx: dict[str, Any], draft_id: str) -> None:
         attachments = await load_attachments(session, user_id, draft)
         _, sent_raw, _, _ = build_mime(account, draft, reply_to, attachments)
         drafts_mb = await session.scalar(
-            select(Mailbox).where(
-                Mailbox.account_id == account.id, Mailbox.special_use == "drafts"
-            )
+            select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )
         drafts_path = drafts_mb.path if drafts_mb else None
         account_obj = account

backend/app/outbox/jobs.py

@@ -52,9 +52,7 @@ async def _harvest_recipients(session: AsyncSession, user_id: uuid.UUID, emails:
         await session.execute(stmt)
 
 
-async def _persist_sent_copy(
-    user_id: uuid.UUID, account: Account, raw: bytes, uid: int
-) -> None:
+async def _persist_sent_copy(user_id: uuid.UUID, account: Account, raw: bytes, uid: int) -> None:
     """Store the just-sent message in the DB (Sent mailbox) and notify the UI.
 
     No-op without a UIDPLUS append UID (uid <= 0); in that case a future Sent sync
@@ -64,9 +62,7 @@ async def _persist_sent_copy(
         return
     async with session_scope() as session:
         sent_mb = await session.scalar(
-            select(Mailbox).where(
-                Mailbox.account_id == account.id, Mailbox.special_use == "sent"
-            )
+            select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "sent")
         )
         if sent_mb is None:
             return
@@ -99,9 +95,7 @@ async def _persist_sent_copy(
         },
     )
     if thread_id is not None:
-        await bus.publish(
-            "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}
-        )
+        await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
 
 
 async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:

backend/app/providers/base.py

@@ -85,7 +85,7 @@ class MailProvider(Protocol):
 
     async def idle(self, mailbox: str) -> None:
         """Blocks until the server signals change or 25 min elapse (re-issue IDLE). May raise
-        ProviderNotSupported => sync loop falls back to poll() every POLL_INTERVAL (default 120 s).
+        ProviderNotSupported => sync loop falls back to poll() every POLL_WARM (default 180 s).
         """
         ...
 

backend/app/providers/imap_smtp/provider.py

@@ -17,6 +17,8 @@ from dataclasses import dataclass
 import aiosmtplib
 from aioimaplib import aioimaplib
 
+from app.concurrency import loop_semaphore
+from app.config import get_settings
 from app.providers.base import (
     FlagChange,
     MailboxDelta,
@@ -37,6 +39,7 @@ from app.providers.imap_smtp.parsing import (
 _FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
 _IDLE_TIMEOUT = 25 * 60  # SPEC §5.1: re-issue IDLE at least every 25 min
 _CONNECT_TIMEOUT = 20
+_PER_ACCOUNT_CONN_CAP = 4  # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
 _APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
 _COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")
 
@@ -59,34 +62,65 @@ class ImapSmtpProvider(MailProvider):
         self._imap: aioimaplib.IMAP4 | None = None
         self._selected: str | None = None
         self._uidvalidity_seen: dict[str, int] = {}
+        self._conn_slots: list[asyncio.Semaphore] = []
 
     # --- connection lifecycle ------------------------------------------------
 
+    async def _acquire_conn_slots(self) -> None:
+        """Bound simultaneous open sockets: ≤4 per account and ≤cap per host (§4).
+
+        Held for the connection's whole lifetime and released in :meth:`close`.
+        """
+        cfg = self._cfg
+        account = loop_semaphore(
+            "imap_account", f"{cfg.username}@{cfg.imap_host}", _PER_ACCOUNT_CONN_CAP
+        )
+        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
+        await account.acquire()  # consistent order (account, then host) avoids deadlock
+        try:
+            await host.acquire()
+        except BaseException:
+            account.release()
+            raise
+        self._conn_slots = [host, account]
+
+    def _release_conn_slots(self) -> None:
+        for sem in self._conn_slots:
+            sem.release()
+        self._conn_slots = []
+
     async def _connect(self) -> aioimaplib.IMAP4:
         if self._imap is not None:
             return self._imap
         cfg = self._cfg
+        await self._acquire_conn_slots()
         try:
-            if cfg.imap_tls == "ssl":
-                imap = aioimaplib.IMAP4_SSL(
-                    host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
-                )
-            else:
-                imap = aioimaplib.IMAP4(
-                    host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
-                )
-            await imap.wait_hello_from_server()
-            if cfg.imap_tls == "starttls":
-                await imap.starttls()
-        except (OSError, TimeoutError) as exc:
-            raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc
+            try:
+                if cfg.imap_tls == "ssl":
+                    imap = aioimaplib.IMAP4_SSL(
+                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
+                    )
+                else:
+                    imap = aioimaplib.IMAP4(
+                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
+                    )
+                await imap.wait_hello_from_server()
+                if cfg.imap_tls == "starttls":
+                    await imap.starttls()
+            except (OSError, TimeoutError) as exc:
+                raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc
 
-        resp = await imap.login(cfg.username, cfg.password)
-        if resp.result != "OK":
-            detail = (
-                b" ".join(resp.lines).decode("utf-8", "ignore") if resp.lines else "login failed"
-            )
-            raise ProviderAuthError(detail)
+            resp = await imap.login(cfg.username, cfg.password)
+            if resp.result != "OK":
+                detail = (
+                    b" ".join(resp.lines).decode("utf-8", "ignore")
+                    if resp.lines
+                    else "login failed"
+                )
+                raise ProviderAuthError(detail)
+        except BaseException:
+            self._release_conn_slots()  # never hold a slot without an open socket
+            raise
         self._imap = imap
         return imap
 
@@ -104,6 +138,7 @@ class ImapSmtpProvider(MailProvider):
                 await self._imap.logout()
             self._imap = None
             self._selected = None
+            self._release_conn_slots()  # free the per-account / per-host slots
 
     # --- MailProvider protocol ----------------------------------------------
 
@@ -293,19 +328,21 @@ class ImapSmtpProvider(MailProvider):
         cfg = self._cfg
         use_tls = cfg.smtp_tls == "ssl"
         start_tls = cfg.smtp_tls == "starttls"
+        host_slot = loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
         try:
-            await aiosmtplib.send(
-                raw,
-                sender=envelope_from,
-                recipients=envelope_to,
-                hostname=cfg.smtp_host,
-                port=cfg.smtp_port,
-                username=cfg.username,
-                password=cfg.password,
-                use_tls=use_tls,
-                start_tls=start_tls,
-                timeout=_CONNECT_TIMEOUT,
-            )
+            async with host_slot:  # bound simultaneous sockets to the SMTP host (§4)
+                await aiosmtplib.send(
+                    raw,
+                    sender=envelope_from,
+                    recipients=envelope_to,
+                    hostname=cfg.smtp_host,
+                    port=cfg.smtp_port,
+                    username=cfg.username,
+                    password=cfg.password,
+                    use_tls=use_tls,
+                    start_tls=start_tls,
+                    timeout=_CONNECT_TIMEOUT,
+                )
         except aiosmtplib.SMTPAuthenticationError as exc:
             raise ProviderAuthError(f"SMTP auth failed: {exc}") from exc
         except aiosmtplib.SMTPException as exc:

backend/app/queue.py

@@ -1,14 +1,33 @@
-"""arq queue helpers (enqueue side, used by the API)."""
+"""arq queue helpers (enqueue side, used by the API).
+
+Two named queues (refactor §6): ``interactive`` (fast user actions) and ``bulk``
+(backfills). Enqueue via :func:`enqueue_job`, which routes each job to its queue —
+the API never needs to know queue names.
+"""
 
 from __future__ import annotations
 
+from typing import Any
+
 from arq import create_pool
 from arq.connections import ArqRedis, RedisSettings
+from arq.jobs import Job
 
 from app.config import get_settings
 
 _pool: ArqRedis | None = None
 
+QUEUE_INTERACTIVE = "interactive"
+QUEUE_BULK = "bulk"
+
+# Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
+_BULK_JOBS = frozenset({"sync_bootstrap_account"})
+
+
+def queue_for(job_name: str) -> str:
+    """Return the queue a job should run on."""
+    return QUEUE_BULK if job_name in _BULK_JOBS else QUEUE_INTERACTIVE
+
 
 def redis_settings() -> RedisSettings:
     """Return arq RedisSettings derived from ``REDIS_URL``."""
@@ -23,6 +42,13 @@ async def get_arq_pool() -> ArqRedis:
     return _pool
 
 
+async def enqueue_job(name: str, *args: Any, **kwargs: Any) -> Job | None:
+    """Enqueue ``name`` onto its routed queue (interactive/bulk)."""
+    pool = await get_arq_pool()
+    kwargs.setdefault("_queue_name", queue_for(name))
+    return await pool.enqueue_job(name, *args, **kwargs)
+
+
 async def close_arq_pool() -> None:
     """Close the arq pool."""
     global _pool

backend/app/sync/actions.py

@@ -273,9 +273,7 @@ async def delete_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
                 .where(Draft.reply_to_message_id.in_(msg_ids))
                 .values(reply_to_message_id=None)
             )
-        await session.execute(
-            update(Draft).where(Draft.thread_id == tid).values(thread_id=None)
-        )
+        await session.execute(update(Draft).where(Draft.thread_id == tid).values(thread_id=None))
         for m in msgs:
             await session.delete(m)
         thread = await session.get(Thread, tid)

backend/app/sync/coordinator.py

@@ -0,0 +1,362 @@
+"""Sync coordinator: lease-owned account claimer (refactor §2, step 3).
+
+Replaces the startup-only supervisor. Each sync worker runs one coordinator that:
+
+* claims accounts from ``sync:live`` (up to ``MAX_ACCOUNTS_PER_WORKER``),
+  starting an :class:`AccountRunner` for each lease it wins;
+* renews its leases every ``LEASE_RENEW`` seconds and, on losing one (a renewal
+  that returns 0), stops that account's runner immediately (fencing);
+* reconciles ``sync:live`` against Postgres periodically as a safety net;
+* heartbeats a worker key so peers can compute a fair share and rebalance.
+
+All ownership is via :mod:`app.sync.leases`; two workers can never run the same
+account because claims are ``SET NX`` and renew/release are owner-checked.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import logging
+import os
+import random
+import secrets
+import socket
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import func, select
+
+from app.config import get_settings
+from app.db import session_scope
+from app.events import channel_for
+from app.models import Account, Message
+from app.redis_client import aw, get_redis
+from app.sync import leases
+from app.sync.runner import AccountRunner
+
+logger = logging.getLogger(__name__)
+
+
+def _new_worker_id() -> str:
+    return f"{socket.gethostname()}:{os.getpid()}:{secrets.token_hex(4)}"
+
+
+class SyncCoordinator:
+    """Owns the leased account runners for a single sync worker."""
+
+    def __init__(self, worker_id: str | None = None) -> None:
+        self.worker_id = worker_id or _new_worker_id()
+        self._redis = get_redis()
+        self._scripts = leases.LeaseScripts(self._redis)
+        self._runners: dict[uuid.UUID, AccountRunner] = {}
+        self._stop = asyncio.Event()
+        self._tasks: list[asyncio.Task[None]] = []
+
+        s = get_settings()
+        self.lease_ttl = s.lease_ttl
+        self.lease_renew = s.lease_renew
+        self.max_accounts = s.max_accounts_per_worker
+        # Operational cadences (not operator-tunable; §2 protocol timings).
+        self.claim_interval = 10.0
+        self.claim_jitter = 5.0
+        self.reconcile_interval = 300.0
+        self.heartbeat_ttl = 30
+        self.heartbeat_refresh = 10.0
+        self.rebalance_interval = 60.0
+        self.tier_eval_interval = 120.0
+        self.session_grace_s = s.tier_session_grace_min * 60
+        self.hot_activity = timedelta(hours=s.tier_hot_activity_h)
+        self.cold_after = timedelta(days=s.tier_cold_after_d)
+        self.hot_budget = s.hot_budget_per_worker
+
+    # --- lifecycle -----------------------------------------------------------
+
+    async def start(self) -> None:
+        self._stop.clear()
+        await self._reconcile_once()  # seed sync:live so the first claim isn't delayed
+        await self._heartbeat_once()
+        self._tasks = [
+            asyncio.create_task(
+                self._loop(self._claim_once, self.claim_interval, self.claim_jitter)
+            ),
+            asyncio.create_task(self._loop(self._renew_once, self.lease_renew)),
+            asyncio.create_task(self._loop(self._reconcile_once, self.reconcile_interval)),
+            asyncio.create_task(self._loop(self._heartbeat_once, self.heartbeat_refresh)),
+            asyncio.create_task(self._loop(self._rebalance_once, self.rebalance_interval)),
+            asyncio.create_task(self._loop(self._evaluate_tiers_once, self.tier_eval_interval)),
+            asyncio.create_task(self._event_subscriber()),
+        ]
+        logger.info("sync coordinator %s started", self.worker_id)
+
+    async def stop(self) -> None:
+        self._stop.set()
+        for task in self._tasks:
+            task.cancel()
+        if self._tasks:
+            await asyncio.gather(*self._tasks, return_exceptions=True)
+        self._tasks = []
+        # Release every lease so peers can pick the accounts up without waiting for TTL.
+        await asyncio.gather(
+            *(self._release(aid) for aid in list(self._runners)), return_exceptions=True
+        )
+        await self._redis.delete(leases.worker_key(self.worker_id))
+        logger.info("sync coordinator %s stopped", self.worker_id)
+
+    @property
+    def owned(self) -> set[uuid.UUID]:
+        return set(self._runners)
+
+    # --- periodic loop driver ------------------------------------------------
+
+    async def _loop(self, step: object, interval: float, jitter: float = 0.0) -> None:
+        while not self._stop.is_set():
+            delay = interval + random.uniform(0.0, jitter)
+            try:
+                await asyncio.wait_for(self._stop.wait(), timeout=delay)
+                return  # stop requested
+            except TimeoutError:
+                pass
+            try:
+                await step()  # type: ignore[operator]
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.exception("coordinator step failed on worker %s", self.worker_id)
+
+    # --- claim / renew / release ---------------------------------------------
+
+    async def _claim_once(self) -> None:
+        if len(self._runners) >= self.max_accounts:
+            return
+        async for raw in self._redis.sscan_iter(leases.LIVE_SET):
+            if self._stop.is_set() or len(self._runners) >= self.max_accounts:
+                break
+            account_id = uuid.UUID(raw)
+            if account_id in self._runners:
+                continue
+            if await leases.claim(self._redis, account_id, self.worker_id, self.lease_ttl):
+                await self._start_runner(account_id)
+
+    async def _renew_once(self) -> None:
+        for account_id in list(self._runners):
+            runner = self._runners.get(account_id)
+            if runner is None:
+                continue
+            # Reap a runner that self-verified it is no longer live (§3).
+            if runner.dead.is_set():
+                logger.info("reaping dead runner for account %s", account_id)
+                await self._release(account_id)
+                continue
+            if not await self._scripts.renew(account_id, self.worker_id, self.lease_ttl):
+                logger.warning("lost lease for account %s; fencing runner", account_id)
+                await self._drop(account_id)
+
+    async def _release(self, account_id: uuid.UUID) -> None:
+        """Graceful hand-off: stop the runner, then delete the lease if still ours."""
+        runner = self._runners.pop(account_id, None)
+        if runner is not None:
+            await runner.stop()
+        await self._scripts.release(account_id, self.worker_id)
+
+    async def _drop(self, account_id: uuid.UUID) -> None:
+        """Fencing: stop the runner without touching the lease (we no longer own it)."""
+        runner = self._runners.pop(account_id, None)
+        if runner is not None:
+            await runner.stop()
+
+    async def _start_runner(self, account_id: uuid.UUID) -> None:
+        # Inherit any tier a previous owner recorded; default warm on first claim.
+        tier = await aw(self._redis.get(leases.tier_key(account_id))) or "warm"
+        await aw(self._redis.set(leases.tier_key(account_id), tier))
+        runner = AccountRunner(account_id, tier)
+        runner.start()
+        self._runners[account_id] = runner
+
+    # --- reconcile / heartbeat / rebalance -----------------------------------
+
+    async def _reconcile_once(self) -> None:
+        """Make ``sync:live`` match the set of live accounts in Postgres.
+
+        Never touches leases — only set membership. Accounts that disappear here
+        stop being claimed; their runners are dropped when their lease is removed.
+        """
+        async with session_scope() as session:
+            live_db = {
+                str(a)
+                for a in await session.scalars(select(Account.id).where(Account.state == "live"))
+            }
+        live_redis = set(await aw(self._redis.smembers(leases.LIVE_SET)))
+        to_add = live_db - live_redis
+        to_remove = live_redis - live_db
+        if to_add:
+            await aw(self._redis.sadd(leases.LIVE_SET, *to_add))
+        if to_remove:
+            await aw(self._redis.srem(leases.LIVE_SET, *to_remove))
+            # Stop any runners for accounts we own that are no longer live.
+            for raw in to_remove:
+                account_id = uuid.UUID(raw)
+                if account_id in self._runners:
+                    await self._release(account_id)
+
+    async def _heartbeat_once(self) -> None:
+        """Publish the worker's load for peer rebalancing + observability (§2)."""
+        tiers = {"hot": 0, "warm": 0, "cold": 0}
+        sockets_open = 0
+        for runner in self._runners.values():
+            tiers[runner.tier] = tiers.get(runner.tier, 0) + 1
+            if runner.tier == "hot":
+                sockets_open += 1  # one persistent IDLE connection per hot account
+        payload = json.dumps(
+            {
+                "owned": len(self._runners),
+                "sockets_open": sockets_open,
+                "tiers": tiers,
+                "ts": time.time(),
+            }
+        )
+        await self._redis.set(leases.worker_key(self.worker_id), payload, ex=self.heartbeat_ttl)
+
+    async def _alive_worker_count(self) -> int:
+        count = 0
+        async for _ in self._redis.scan_iter(match=f"{leases.WORKERS_PREFIX}*"):
+            count += 1
+        return max(count, 1)
+
+    async def _rebalance_once(self) -> None:
+        """Shed accounts if we hold well above a fair share of the live total."""
+        total = int(await aw(self._redis.scard(leases.LIVE_SET)))
+        workers = await self._alive_worker_count()
+        target = total / workers
+        threshold = int(target * 1.2) + 5
+        excess = len(self._runners) - threshold
+        if excess <= 0:
+            return
+        # Release the newest claims first (least likely to be mid-backfill on us).
+        for account_id in list(self._runners)[-excess:]:
+            await self._release(account_id)
+        logger.info("worker %s shed %d account(s) to rebalance", self.worker_id, excess)
+
+    # --- event-driven lifecycle (§3) -----------------------------------------
+
+    async def _event_subscriber(self) -> None:
+        """React to ``account.state_changed`` across processes (Redis pub/sub).
+
+        Going live makes an account claimable at once; going away tears the
+        runner down immediately (≤30s from the API call) instead of waiting for
+        the lease to lapse.
+        """
+        pubsub = self._redis.pubsub()
+        state_ch = channel_for("account.state_changed")
+        active_ch = channel_for("user.active")
+        await pubsub.subscribe(state_ch, active_ch)
+        try:
+            async for message in pubsub.listen():
+                if self._stop.is_set():
+                    break
+                if message.get("type") != "message":
+                    continue
+                try:
+                    payload = json.loads(message["data"]).get("payload", {})
+                except (ValueError, TypeError):
+                    continue
+                if message.get("channel") == active_ch:
+                    await self._on_user_active(payload)
+                else:
+                    await self._on_state_changed(payload)
+        finally:
+            with contextlib.suppress(Exception):
+                await pubsub.aclose()  # type: ignore[no-untyped-call]
+
+    async def _on_state_changed(self, payload: dict[str, object]) -> None:
+        raw = payload.get("account_id")
+        if not isinstance(raw, str):
+            return
+        account_id = uuid.UUID(raw)
+        if payload.get("state") == "live":
+            await leases.mark_live(self._redis, account_id)  # claimable next cycle
+        elif account_id in self._runners:
+            await self._release(account_id)  # disabled/deleted/error -> stop now
+
+    async def _on_user_active(self, payload: dict[str, object]) -> None:
+        """Promote all this user's owned accounts to hot at once (≤5s, §5)."""
+        raw = payload.get("user_id")
+        if not isinstance(raw, str):
+            return
+        user_id = uuid.UUID(raw)
+        owned = list(self._runners)
+        if not owned:
+            return
+        async with session_scope() as session:
+            aids = list(
+                await session.scalars(
+                    select(Account.id).where(Account.id.in_(owned), Account.user_id == user_id)
+                )
+            )
+        for account_id in aids:
+            await self._set_tier(account_id, "hot")
+
+    # --- tier evaluation (§5) ------------------------------------------------
+
+    async def _set_tier(self, account_id: uuid.UUID, tier: str) -> None:
+        runner = self._runners.get(account_id)
+        if runner is None:
+            return
+        await aw(self._redis.set(leases.tier_key(account_id), tier))
+        await runner.apply_tier(tier)
+
+    async def _account_activity(
+        self, ids: list[uuid.UUID]
+    ) -> dict[uuid.UUID, tuple[uuid.UUID, datetime | None]]:
+        """Return {account_id: (user_id, last_mail_at)} for the owned accounts."""
+        async with session_scope() as session:
+            rows = (
+                await session.execute(
+                    select(Account.id, Account.user_id, func.max(Message.internal_date))
+                    .select_from(Account)
+                    .outerjoin(Message, Message.account_id == Account.id)
+                    .where(Account.id.in_(ids))
+                    .group_by(Account.id, Account.user_id)
+                )
+            ).all()
+        return {row[0]: (row[1], row[2]) for row in rows}
+
+    async def _evaluate_tiers_once(self) -> None:
+        """Recompute each owned account's tier from session heat + recent mail."""
+        owned = list(self._runners)
+        if not owned:
+            return
+        activity = await self._account_activity(owned)
+        now = datetime.now(UTC)
+        sessions: dict[uuid.UUID, bool] = {}
+        desired: dict[uuid.UUID, str] = {}
+        demotable: list[tuple[datetime, uuid.UUID]] = []  # activity-hot, no session
+        hot_count = 0
+        for account_id in owned:
+            user_id, last_mail = activity.get(account_id, (None, None))
+            session_active = False
+            if user_id is not None:
+                if user_id not in sessions:
+                    sessions[user_id] = bool(
+                        await aw(self._redis.exists(leases.session_key(user_id)))
+                    )
+                session_active = sessions[user_id]
+            recent = last_mail is not None and (now - last_mail) < self.hot_activity
+            if session_active or recent:
+                desired[account_id] = "hot"
+                hot_count += 1
+                if not session_active and last_mail is not None:
+                    demotable.append((last_mail, account_id))
+            elif last_mail is not None and (now - last_mail) < self.cold_after:
+                desired[account_id] = "warm"
+            else:
+                desired[account_id] = "cold"
+        # Global hot budget: shed the least-recently-active session-less hot accounts.
+        if hot_count > self.hot_budget:
+            demotable.sort(key=lambda item: item[0])  # oldest activity first
+            for _, account_id in demotable[: hot_count - self.hot_budget]:
+                desired[account_id] = "warm"
+        for account_id, tier in desired.items():
+            await self._set_tier(account_id, tier)

backend/app/sync/engine.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
+import random
 import uuid
 from typing import Any
 
@@ -41,6 +42,17 @@ async def _set_state(account_id: uuid.UUID, state: str, detail: str = "") -> Non
         await session.execute(
             update(Account).where(Account.id == account_id).values(state=state, state_detail=detail)
         )
+    # Keep the sync:live registry in step with the account's state so the sync
+    # coordinator claims it once live and drops it otherwise (refactor §2).
+    from app.redis_client import get_redis
+    from app.sync import leases
+
+    redis = get_redis()
+    if state == "live":
+        await leases.mark_live(redis, account_id)
+    else:
+        await leases.mark_not_live(redis, account_id)
+
     bus = get_event_bus()
     account = None
     async with session_scope() as session:
@@ -244,7 +256,7 @@ async def live_loop(
     from app.db import session_scope
 
     stop = stop or asyncio.Event()
-    poll_interval = get_settings().poll_interval
+    idle_fallback = get_settings().poll_warm
 
     async with session_scope() as session:
         account = await _load_account(session, account_id)
@@ -256,7 +268,7 @@ async def live_loop(
             try:
                 await provider.idle(mailbox_path)
             except ProviderNotSupported:
-                await asyncio.sleep(poll_interval)
+                await asyncio.sleep(idle_fallback)
             if stop.is_set():
                 break
             async with session_scope() as session:
@@ -330,19 +342,49 @@ async def refresh_account_job(ctx: dict[str, Any], account_id: str) -> None:
     await poll_folders_once(uuid.UUID(account_id), include_inbox=True)
 
 
-async def secondary_poll_loop(account_id: uuid.UUID, stop: asyncio.Event | None = None) -> None:
+async def secondary_poll_loop(
+    account_id: uuid.UUID, stop: asyncio.Event | None = None, interval: float | None = None
+) -> None:
     """Periodically CONDSTORE-poll non-INBOX folders (Sent/Drafts/Archive/Trash/Junk).
 
     INBOX has its own IDLE loop (real-time). This keeps the other folders in step
     with server-side changes (e.g. Gmail-filed sent mail) and changes made by other
-    clients. CONDSTORE/QRESYNC makes each pass a cheap MODSEQ delta.
+    clients. CONDSTORE/QRESYNC makes each pass a cheap MODSEQ delta. Used by hot-tier
+    runners alongside the INBOX IDLE loop.
     """
     stop = stop or asyncio.Event()
-    interval = get_settings().poll_interval
+    period = interval if interval is not None else get_settings().poll_warm
+    first = True
     while not stop.is_set():
-        # Wait first: the initial state was already covered by the backfill.
+        # Wait first (initial state was covered by backfill). Spread the fleet: the
+        # first wait is a random point in the period, later waits jitter ±10%, so
+        # accounts claimed together don't poll in lockstep (thundering herd).
+        wait = random.uniform(0, period) if first else period * random.uniform(0.9, 1.1)
+        first = False
         with contextlib.suppress(TimeoutError):
-            await asyncio.wait_for(stop.wait(), timeout=interval)
+            await asyncio.wait_for(stop.wait(), timeout=wait)
         if stop.is_set():
             break
         await _poll_secondary_once(account_id)
+
+
+async def poll_all_loop(
+    account_id: uuid.UUID, interval: float, stop: asyncio.Event | None = None
+) -> None:
+    """Transient CONDSTORE poll of INBOX + special folders every ``interval`` seconds.
+
+    The warm/cold tiers use this instead of a persistent IDLE connection: no socket
+    is held between passes. Correctness matches the hot tier — only latency differs.
+    """
+    stop = stop or asyncio.Event()
+    first = True
+    while not stop.is_set():
+        # Spread the fleet: first poll at a random point in the interval, then
+        # jitter ±10% so a batch claimed together doesn't poll in lockstep.
+        wait = random.uniform(0, interval) if first else interval * random.uniform(0.9, 1.1)
+        first = False
+        with contextlib.suppress(TimeoutError):
+            await asyncio.wait_for(stop.wait(), timeout=wait)
+        if stop.is_set():
+            break
+        await poll_folders_once(account_id, include_inbox=True)

backend/app/sync/leases.py

@@ -0,0 +1,99 @@
+"""Redis lease primitives for account ownership (refactor §2).
+
+A sync worker must hold the lease ``sync:lease:{account_id}`` before it runs an
+account's sync loops, and no two workers may hold the same lease. Claims are
+atomic (``SET NX EX``); renewal and release are owner-checked Lua so a worker can
+never renew or delete a lease that has been taken over by another worker.
+
+The set ``sync:live`` is the registry of accounts that *should* be synced; the
+claimer only ever claims ids drawn from it.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+import redis.asyncio as aioredis
+
+from app.redis_client import aw
+
+LIVE_SET = "sync:live"
+_LEASE_PREFIX = "sync:lease:"
+WORKERS_PREFIX = "sync:workers:"
+_TIER_PREFIX = "sync:tier:"
+_SESSION_PREFIX = "sess:active:"
+
+
+def lease_key(account_id: uuid.UUID) -> str:
+    return f"{_LEASE_PREFIX}{account_id}"
+
+
+def worker_key(worker_id: str) -> str:
+    return f"{WORKERS_PREFIX}{worker_id}"
+
+
+def tier_key(account_id: uuid.UUID) -> str:
+    return f"{_TIER_PREFIX}{account_id}"
+
+
+def session_key(user_id: uuid.UUID | str) -> str:
+    """Presence key the API refreshes while a user has an active session (§5)."""
+    return f"{_SESSION_PREFIX}{user_id}"
+
+
+async def mark_user_active(redis: aioredis.Redis, user_id: uuid.UUID | str, grace_s: int) -> None:
+    """Refresh the user's session-heat key; workers read it for tier promotion."""
+    await aw(redis.set(session_key(user_id), "1", ex=grace_s))
+
+
+# Renew only if we still own the lease (guards against a stale worker extending a
+# lease another worker has already claimed after expiry). Returns 1 on success.
+_RENEW_LUA = """
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+  return redis.call('EXPIRE', KEYS[1], ARGV[2])
+else
+  return 0
+end
+"""
+
+# Delete only if we still own the lease. Returns 1 if we released it, else 0.
+_RELEASE_LUA = """
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+  return redis.call('DEL', KEYS[1])
+else
+  return 0
+end
+"""
+
+
+class LeaseScripts:
+    """Server-registered Lua for owner-checked renew/release."""
+
+    def __init__(self, redis: aioredis.Redis) -> None:
+        self._renew = redis.register_script(_RENEW_LUA)
+        self._release = redis.register_script(_RELEASE_LUA)
+
+    async def renew(self, account_id: uuid.UUID, worker_id: str, ttl: int) -> bool:
+        result = await self._renew(keys=[lease_key(account_id)], args=[worker_id, ttl])
+        return bool(result)
+
+    async def release(self, account_id: uuid.UUID, worker_id: str) -> bool:
+        result = await self._release(keys=[lease_key(account_id)], args=[worker_id])
+        return bool(result)
+
+
+async def claim(redis: aioredis.Redis, account_id: uuid.UUID, worker_id: str, ttl: int) -> bool:
+    """Atomically claim the lease for ``account_id``; True iff we now own it."""
+    ok = await redis.set(lease_key(account_id), worker_id, nx=True, ex=ttl)
+    return bool(ok)
+
+
+async def mark_live(redis: aioredis.Redis, account_id: uuid.UUID) -> None:
+    """Register an account as one that should be synced (idempotent)."""
+    await aw(redis.sadd(LIVE_SET, str(account_id)))
+
+
+async def mark_not_live(redis: aioredis.Redis, account_id: uuid.UUID) -> None:
+    """Deregister an account and drop any lease so its owner stops it (fencing)."""
+    await aw(redis.srem(LIVE_SET, str(account_id)))
+    await aw(redis.delete(lease_key(account_id)))

backend/app/sync/runner.py

@@ -0,0 +1,141 @@
+"""Per-account sync runner (refactor §4, §5).
+
+An ``AccountRunner`` owns the long-lived sync work for a single account and
+exposes a ``start`` / ``stop`` / ``apply_tier`` lifecycle. The task set is shaped
+by the account's activity tier (§5):
+
+* **hot** — a persistent INBOX IDLE loop plus a secondary CONDSTORE poll of the
+  special folders every ``POLL_WARM`` seconds.
+* **warm** — no persistent socket; a single transient poll of INBOX + special
+  folders every ``POLL_WARM`` seconds.
+* **cold** — the same single transient poll, every ``POLL_COLD`` seconds.
+
+Correctness is identical across tiers — only latency and socket cost differ.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import uuid
+from collections.abc import Awaitable, Callable
+
+from sqlalchemy import select
+
+from app.config import get_settings
+from app.db import session_scope
+from app.models import Account
+from app.sync.engine import live_loop, poll_all_loop, secondary_poll_loop
+
+logger = logging.getLogger(__name__)
+
+_RECONNECT_BACKOFF = 30.0
+_VERIFY_INTERVAL = 60.0
+
+Tier = str  # "hot" | "warm" | "cold"
+
+
+class AccountRunner:
+    """Owns the tier-shaped sync work for one account until stopped."""
+
+    def __init__(self, account_id: uuid.UUID, tier: Tier = "warm") -> None:
+        self.account_id = account_id
+        self.tier = tier
+        self._stop = asyncio.Event()
+        self._tasks: list[asyncio.Task[None]] = []  # tier-independent (verify)
+        self._work: list[asyncio.Task[None]] = []  # tier-shaped loops
+        # Set when the account is found no longer live (§3 defense in depth); the
+        # coordinator reaps such runners and releases their lease.
+        self.dead = asyncio.Event()
+        self.verify_interval = _VERIFY_INTERVAL
+
+    def start(self) -> None:
+        """Launch the verify task and the current tier's work loops."""
+        self._stop.clear()
+        self._tasks = [asyncio.create_task(self._verify_loop())]
+        self._start_work()
+
+    def running(self) -> bool:
+        return any(not t.done() for t in (*self._tasks, *self._work))
+
+    async def stop(self) -> None:
+        """Signal and cancel every loop, waiting for teardown."""
+        self._stop.set()
+        await self._cancel(self._work)
+        await self._cancel(self._tasks)
+        self._work = []
+        self._tasks = []
+
+    async def apply_tier(self, tier: Tier) -> None:
+        """Reshape the work loops for a new tier (no-op if unchanged)."""
+        if tier == self.tier and self._work:
+            return
+        logger.info("account %s tier %s -> %s", self.account_id, self.tier, tier)
+        self.tier = tier
+        await self._cancel(self._work)
+        self._start_work()
+
+    # --- internals -----------------------------------------------------------
+
+    def _start_work(self) -> None:
+        settings = get_settings()
+        if self.tier == "hot":
+            self._work = [
+                asyncio.create_task(self._supervise(self._run_live, "live")),
+                asyncio.create_task(self._supervise(self._run_secondary, "secondary")),
+            ]
+        else:
+            interval = float(settings.poll_warm if self.tier == "warm" else settings.poll_cold)
+            self._work = [
+                asyncio.create_task(self._supervise(lambda: self._run_poll(interval), "poll"))
+            ]
+
+    async def _cancel(self, tasks: list[asyncio.Task[None]]) -> None:
+        for task in tasks:
+            task.cancel()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _run_live(self) -> None:
+        await live_loop(self.account_id, "INBOX", self._stop)
+
+    async def _run_secondary(self) -> None:
+        await secondary_poll_loop(self.account_id, self._stop, get_settings().poll_warm)
+
+    async def _run_poll(self, interval: float) -> None:
+        await poll_all_loop(self.account_id, interval, self._stop)
+
+    async def _verify_loop(self) -> None:
+        """Periodically confirm the account is still live; flag ``dead`` if not.
+
+        This makes zombie loops structurally impossible even if a lifecycle event
+        is lost: an account that was disabled/deleted stops being synced within a
+        verify cycle regardless of the lease/event paths.
+        """
+        while not self._stop.is_set():
+            with contextlib.suppress(TimeoutError):
+                await asyncio.wait_for(self._stop.wait(), timeout=self.verify_interval)
+            if self._stop.is_set():
+                return
+            if not await self._account_is_live():
+                logger.info("account %s no longer live; flagging runner dead", self.account_id)
+                self.dead.set()
+                return
+
+    async def _account_is_live(self) -> bool:
+        async with session_scope() as session:
+            state = await session.scalar(select(Account.state).where(Account.id == self.account_id))
+        return state == "live"
+
+    async def _supervise(self, run: Callable[[], Awaitable[None]], label: str) -> None:
+        """Run a loop, restarting it (with backoff) until stopped or cancelled."""
+        while not self._stop.is_set():
+            try:
+                await run()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.exception("%s loop for account %s crashed; retrying", label, self.account_id)
+                with contextlib.suppress(TimeoutError):
+                    await asyncio.wait_for(self._stop.wait(), timeout=_RECONNECT_BACKOFF)

backend/app/sync/supervisor.py

@@ -1,77 +0,0 @@
-"""Live-sync supervisor (SPEC §6.3).
-
-Runs inside the arq worker: on startup it launches one IDLE/poll loop per live
-account so new mail and flag changes flow to the UI over WebSocket. Loops
-self-heal (reconnect with backoff) and are cancelled on worker shutdown.
-
-NOTE(spec): accounts that become live *after* worker startup are picked up on the
-next worker restart; reacting to ``account.state_changed`` live is a later refinement.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import logging
-import uuid
-from typing import Any
-
-from sqlalchemy import select
-
-from app.db import session_scope
-from app.models import Account
-from app.sync.engine import live_loop, secondary_poll_loop
-
-logger = logging.getLogger(__name__)
-
-_RECONNECT_BACKOFF = 30.0
-
-
-async def _run_account(account_id: uuid.UUID, stop: asyncio.Event) -> None:
-    while not stop.is_set():
-        try:
-            await live_loop(account_id, "INBOX", stop)
-        except asyncio.CancelledError:
-            raise
-        except Exception:
-            logger.exception("live loop for account %s crashed; reconnecting", account_id)
-            with contextlib.suppress(TimeoutError):
-                await asyncio.wait_for(stop.wait(), timeout=_RECONNECT_BACKOFF)
-
-
-async def _run_secondary(account_id: uuid.UUID, stop: asyncio.Event) -> None:
-    while not stop.is_set():
-        try:
-            await secondary_poll_loop(account_id, stop)
-        except asyncio.CancelledError:
-            raise
-        except Exception:
-            logger.exception("secondary poll loop for %s crashed; retrying", account_id)
-            with contextlib.suppress(TimeoutError):
-                await asyncio.wait_for(stop.wait(), timeout=_RECONNECT_BACKOFF)
-
-
-async def start_supervisor(ctx: dict[str, Any]) -> None:
-    """Launch a live loop per live account (called from the worker on_startup)."""
-    async with session_scope() as session:
-        account_ids = list(await session.scalars(select(Account.id).where(Account.state == "live")))
-    stop = asyncio.Event()
-    ctx["supervisor_stop"] = stop
-    tasks: list[asyncio.Task[None]] = []
-    for aid in account_ids:
-        tasks.append(asyncio.create_task(_run_account(aid, stop)))  # INBOX IDLE
-        tasks.append(asyncio.create_task(_run_secondary(aid, stop)))  # other folders poll
-    ctx["supervisor_tasks"] = tasks
-    logger.info("live supervisor started for %d account(s)", len(account_ids))
-
-
-async def stop_supervisor(ctx: dict[str, Any]) -> None:
-    """Signal and cancel all live loops (called from the worker on_shutdown)."""
-    stop = ctx.get("supervisor_stop")
-    if stop is not None:
-        stop.set()
-    tasks = ctx.get("supervisor_tasks", [])
-    for task in tasks:
-        task.cancel()
-    if tasks:
-        await asyncio.gather(*tasks, return_exceptions=True)

backend/app/ws.py

@@ -13,9 +13,12 @@ import logging
 
 from fastapi import APIRouter, WebSocket, WebSocketDisconnect
 
+from app.config import get_settings
+from app.events import get_event_bus
 from app.identity.cookies import SESSION_COOKIE_INSECURE, SESSION_COOKIE_SECURE
 from app.identity.sessions import resolve_session
 from app.redis_client import get_redis
+from app.sync import leases
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +37,10 @@ async def ws_endpoint(websocket: WebSocket) -> None:
 
     await websocket.accept()
     user_id = str(session.user_id)
+    grace_s = get_settings().tier_session_grace_min * 60
+    # Sync v2 §3/§5: an open WS means the user is active -> session heat + promote.
+    await leases.mark_user_active(get_redis(), user_id, grace_s)
+    await get_event_bus().publish("user.active", {"user_id": user_id})
     pubsub = get_redis().pubsub()
     await pubsub.psubscribe("evt:*")
 
@@ -54,6 +61,8 @@ async def ws_endpoint(websocket: WebSocket) -> None:
             raw = await websocket.receive_text()
             try:
                 if json.loads(raw).get("type") == "ping":
+                    # A ping keeps the user's session heat alive (§5).
+                    await leases.mark_user_active(get_redis(), user_id, grace_s)
                     await websocket.send_json({"type": "pong"})
             except (ValueError, TypeError):
                 continue

backend/loadtest/mock_provider.py

@@ -0,0 +1,158 @@
+"""In-process mock ``MailProvider`` for the sync soak (acceptance §10.7).
+
+No sockets, no network: ``idle`` sleeps and returns "no change", ``poll`` returns
+an empty delta. Two process-global counters make the soak's assertions meaningful
+without real file descriptors:
+
+* ``persistent_open`` — providers currently open that have issued an IDLE, i.e. the
+  hot-tier persistent connections. Warm/cold runners never call ``idle`` so they
+  must not contribute here.
+* ``open`` — every currently-open provider (persistent + transient poll passes).
+
+``last_sync[account_id]`` records the wall-clock of the account's most recent poll,
+so the harness can compute the oldest sync lag across all accounts.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import time
+import uuid
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime
+from email.message import EmailMessage
+from email.utils import format_datetime
+
+from app.providers.base import MailboxDelta, RemoteMailbox, RemoteMessage
+
+
+class MockStats:
+    open = 0
+    persistent_open = 0
+    last_sync: dict[uuid.UUID, float] = {}
+
+    # --- optional mail injection (exercises parse/blob/DB/index under load) ----
+    mail_enabled = False
+    mail_per_poll = 0.0  # expected new INBOX messages per poll (fractional => prob)
+    msg_bytes = 10_000
+    big_pct = 0  # percent of messages that exceed STREAM_PARSE_MB (streaming path)
+    big_bytes = 10_000_000
+    delivered = 0  # total synthetic messages handed to the sync engine
+    _uid: dict[uuid.UUID, int] = {}
+
+    @classmethod
+    def reset(cls) -> None:
+        cls.open = 0
+        cls.persistent_open = 0
+        cls.last_sync = {}
+        cls.delivered = 0
+        cls._uid = {}
+
+
+def _make_raw(size_bytes: int) -> bytes:
+    """A unique, well-formed RFC822 message padded to roughly ``size_bytes``."""
+    msg = EmailMessage()
+    msg["Subject"] = f"soak {uuid.uuid4().hex[:12]}"
+    msg["From"] = "sender@load.local"
+    msg["To"] = "rcpt@load.local"
+    msg["Message-ID"] = f"<{uuid.uuid4().hex}@load.local>"
+    msg["Date"] = format_datetime(datetime.now(UTC))
+    msg.set_content("x" * max(0, size_bytes - 256))
+    return msg.as_bytes()
+
+
+def _draw_count() -> int:
+    n = int(MockStats.mail_per_poll)
+    frac = MockStats.mail_per_poll - n
+    return n + (1 if random.random() < frac else 0)
+
+
+class MockProvider:
+    """A MailProvider that simulates a quiet mailbox with no real I/O."""
+
+    def __init__(self, account_id: uuid.UUID, idle_s: float) -> None:
+        self.account_id = account_id
+        self.idle_s = idle_s
+        self._open = False
+        self._persistent = False
+
+    def _open_socket(self) -> None:
+        if not self._open:
+            self._open = True
+            MockStats.open += 1
+
+    async def idle(self, mailbox: str) -> None:
+        self._open_socket()
+        if not self._persistent:  # first IDLE marks this a persistent (hot) connection
+            self._persistent = True
+            MockStats.persistent_open += 1
+        await asyncio.sleep(self.idle_s)  # simulate the IDLE wait / 25-min re-issue
+
+    async def poll(
+        self, mailbox: str, *, last_seen_uid: int, highestmodseq: int | None
+    ) -> MailboxDelta:
+        self._open_socket()
+        MockStats.last_sync[self.account_id] = time.time()
+        new: list[RemoteMessage] = []
+        if MockStats.mail_enabled and mailbox == "INBOX":
+            for _ in range(_draw_count()):
+                uid = MockStats._uid.get(self.account_id, 0) + 1
+                MockStats._uid[self.account_id] = uid
+                big = random.random() < MockStats.big_pct / 100.0
+                size = MockStats.big_bytes if big else MockStats.msg_bytes
+                new.append(
+                    RemoteMessage(
+                        uid=uid,
+                        raw=_make_raw(size),
+                        flags=frozenset(),
+                        internal_date=datetime.now(UTC),
+                    )
+                )
+            MockStats.delivered += len(new)
+        return MailboxDelta(
+            new=new,
+            flag_changes=[],
+            expunged_uids=[],
+            new_uidvalidity=None,
+            new_highestmodseq=highestmodseq,
+        )
+
+    async def close(self) -> None:
+        if self._open:
+            self._open = False
+            MockStats.open -= 1
+        if self._persistent:
+            self._persistent = False
+            MockStats.persistent_open -= 1
+
+    # --- unused in the steady-state soak, defined for interface completeness ----
+
+    async def verify(self) -> None:
+        return None
+
+    async def list_mailboxes(self) -> list[RemoteMailbox]:
+        return []
+
+    async def backfill(
+        self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
+    ) -> AsyncIterator[list[RemoteMessage]]:
+        return
+        yield  # pragma: no cover - makes this an async generator
+
+    async def push_flags(
+        self, mailbox: str, uid: int, add: frozenset[str], remove: frozenset[str]
+    ) -> None:
+        return None
+
+    async def move(self, uid: int, src: str, dst: str) -> int:
+        return uid
+
+    async def delete_messages(self, mailbox: str, uids: list[int]) -> None:
+        return None
+
+    async def append(self, mailbox: str, raw: bytes, flags: frozenset[str]) -> int:
+        return 0
+
+    async def send(self, raw: bytes, envelope_from: str, envelope_to: list[str]) -> None:
+        return None

backend/loadtest/soak_sync.py

@@ -0,0 +1,471 @@
+"""Sync-engine soak / load test — acceptance criterion §10.7 (mock provider).
+
+Simulates ``SOAK_ACCOUNTS`` accounts (default 1000), a fraction with active
+sessions, sharded across ``SOAK_WORKERS`` in-process ``SyncCoordinator`` instances
+using a mock ``MailProvider`` (no real IMAP). It samples, over the run:
+
+* persistent sockets vs hot-account count   (target: persistent ≤ hot, total ≤ hot×1.05)
+* process RSS                                (target: stable — no per-account leak)
+* oldest sync lag across all accounts        (target: < POLL_COLD)
+
+**Fully isolated**: it creates its own Postgres database (``webmail_soak``) and uses
+a separate Redis DB, so it never touches the dev database, the dev Redis, or a
+running worker. Both are torn down at the end.
+
+Run (defaults = the §10.7 spec, ~1h):
+
+    cd backend && python -m loadtest.soak_sync
+
+Fast smoke run (a couple of minutes, shortened cadences):
+
+    cd backend && SOAK_ACCOUNTS=200 SOAK_DURATION_S=150 POLL_COLD=30 POLL_WARM=15 \
+        SOAK_TIER_EVAL_S=5 SOAK_IDLE_S=10 python -m loadtest.soak_sync
+
+Mail-injecting mode (SOAK_MAIL=1) delivers synthetic messages so the parse /
+blob / DB / index path actually runs and RSS + CPU move under throughput. Tunables:
+SOAK_MAIL_PER_POLL (expected new INBOX msgs per poll), SOAK_MSG_KB, SOAK_BIG_MSG_PCT
+(percent above STREAM_PARSE_MB, exercising the streaming/§8 path). Example:
+
+    cd backend && SOAK_MAIL=1 SOAK_MAIL_PER_POLL=0.5 SOAK_BIG_MSG_PCT=5 \
+        SOAK_ACCOUNTS=300 SOAK_DURATION_S=300 POLL_COLD=60 POLL_WARM=30 \
+        SOAK_TIER_EVAL_S=15 SOAK_IDLE_S=20 python -m loadtest.soak_sync
+
+Exit code 0 = all targets met, 1 = a target missed (usable in CI-nightly).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# --- isolation: point DB + Redis + blobs at throwaway locations BEFORE app import
+
+_DEV_DB = os.environ.get(
+    "DATABASE_URL", "postgresql+asyncpg://webmail:webmail_dev@127.0.0.1:5433/webmail_dev"
+)
+_SOAK_DB_NAME = os.environ.get("SOAK_DB_NAME", "webmail_soak")
+_SOAK_DB_URL = re.sub(r"/[^/?]+(\?|$)", f"/{_SOAK_DB_NAME}\\1", _DEV_DB)
+os.environ["DATABASE_URL"] = _SOAK_DB_URL
+os.environ.setdefault("SOAK_REDIS_URL", "redis://127.0.0.1:6380/14")
+os.environ["REDIS_URL"] = os.environ["SOAK_REDIS_URL"]
+# Isolate the blob store (mail injection writes real blobs) and force the in-DB
+# search backend so nothing leaks to the dev blob dir or an external Meilisearch.
+_SOAK_BLOBS = tempfile.mkdtemp(prefix="soak-blobs-")
+os.environ["BLOB_ROOT"] = _SOAK_BLOBS
+os.environ["SEARCH_BACKEND"] = "pg"
+
+import asyncio  # noqa: E402
+import logging  # noqa: E402
+import time  # noqa: E402
+import uuid  # noqa: E402
+from dataclasses import dataclass, field  # noqa: E402
+
+import asyncpg  # noqa: E402
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine  # noqa: E402
+from sqlalchemy.pool import NullPool  # noqa: E402
+
+from app.config import get_settings  # noqa: E402
+from app.identity.passwords import hash_password  # noqa: E402
+from app.models import Account, Mailbox, User  # noqa: E402
+from app.redis_client import close_redis, get_redis  # noqa: E402
+from app.sync import leases  # noqa: E402
+from app.sync.coordinator import SyncCoordinator  # noqa: E402
+from loadtest.mock_provider import MockProvider, MockStats  # noqa: E402
+
+# --- config ------------------------------------------------------------------
+
+
+def _int(name: str, default: int) -> int:
+    return int(os.environ.get(name, default))
+
+
+@dataclass
+class Config:
+    accounts: int = field(default_factory=lambda: _int("SOAK_ACCOUNTS", 1000))
+    session_pct: int = field(default_factory=lambda: _int("SOAK_SESSION_PCT", 20))
+    duration_s: int = field(default_factory=lambda: _int("SOAK_DURATION_S", 3600))
+    workers: int = field(default_factory=lambda: _int("SOAK_WORKERS", 3))
+    sample_s: int = field(default_factory=lambda: _int("SOAK_SAMPLE_S", 30))
+    idle_s: int = field(default_factory=lambda: _int("SOAK_IDLE_S", 60))
+    claim_s: int = field(default_factory=lambda: _int("SOAK_CLAIM_S", 5))
+    tier_eval_s: int = field(default_factory=lambda: _int("SOAK_TIER_EVAL_S", 120))
+    # Mail injection (off by default): synthetic messages exercise the
+    # parse/blob/DB/index path so RSS + CPU move under real throughput.
+    mail: bool = field(default_factory=lambda: bool(_int("SOAK_MAIL", 0)))
+    mail_per_poll: float = field(
+        default_factory=lambda: float(os.environ.get("SOAK_MAIL_PER_POLL", 0.3))
+    )
+    msg_kb: int = field(default_factory=lambda: _int("SOAK_MSG_KB", 12))
+    big_pct: int = field(default_factory=lambda: _int("SOAK_BIG_MSG_PCT", 0))
+
+    def __post_init__(self) -> None:
+        s = get_settings()
+        self.poll_cold = s.poll_cold
+        self.poll_warm = s.poll_warm
+        self.session_grace_s = s.tier_session_grace_min * 60
+        # Warm-up must outlast one cold poll so cold accounts have synced once, but
+        # never eat the whole run on a short smoke test.
+        default_warmup = min(self.poll_cold + 90, int(self.duration_s * 0.4))
+        self.warmup_s = _int("SOAK_WARMUP_S", default_warmup)
+
+
+# --- database lifecycle ------------------------------------------------------
+
+
+def _admin_dsn() -> str:
+    # asyncpg DSN for the maintenance DB (strip the +asyncpg driver + swap db name).
+    dsn = _DEV_DB.replace("postgresql+asyncpg://", "postgresql://")
+    return re.sub(r"/[^/?]+(\?|$)", "/postgres\\1", dsn)
+
+
+async def _create_database() -> None:
+    conn = await asyncpg.connect(_admin_dsn())
+    try:
+        await conn.execute(f'DROP DATABASE IF EXISTS "{_SOAK_DB_NAME}" WITH (FORCE)')
+        await conn.execute(f'CREATE DATABASE "{_SOAK_DB_NAME}"')
+    finally:
+        await conn.close()
+
+
+async def _drop_database() -> None:
+    conn = await asyncpg.connect(_admin_dsn())
+    try:
+        await conn.execute(f'DROP DATABASE IF EXISTS "{_SOAK_DB_NAME}" WITH (FORCE)')
+    finally:
+        await conn.close()
+
+
+def _migrate() -> None:
+    env = {**os.environ, "DATABASE_URL": _SOAK_DB_URL}
+    backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    subprocess.run(
+        [sys.executable, "-c", "from alembic.config import main; main(argv=['upgrade', 'head'])"],
+        cwd=backend_dir,
+        env=env,
+        check=True,
+    )
+
+
+_MAILBOXES = [
+    ("INBOX", "inbox"),
+    ("Sent", "sent"),
+    ("Drafts", "drafts"),
+    ("Archive", "archive"),
+    ("Trash", "trash"),
+]
+
+
+async def _seed(cfg: Config) -> list[uuid.UUID]:
+    """Create N users, each with one live account and a handful of mailboxes."""
+    engine = create_async_engine(_SOAK_DB_URL, poolclass=NullPool)
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    user_ids: list[uuid.UUID] = []
+    pw = hash_password("soak")
+    async with sm() as session:
+        for i in range(cfg.accounts):
+            user = User(handle=f"soak-{i:05d}@load.local", password_hash=pw)
+            session.add(user)
+            await session.flush()
+            user_ids.append(user.id)
+            account = Account(
+                user_id=user.id,
+                email=f"soak-{i:05d}@load.local",
+                display_name="Soak",
+                imap_config={"host": "mock", "port": 993, "tls": "ssl"},
+                smtp_config={"host": "mock", "port": 465, "tls": "ssl"},
+                secret_ct=b"",
+                discovery_src="manual",
+                state="live",
+            )
+            session.add(account)
+            await session.flush()
+            for path, special in _MAILBOXES:
+                session.add(
+                    Mailbox(account_id=account.id, user_id=user.id, path=path, special_use=special)
+                )
+            if i % 200 == 0:
+                await session.flush()
+        await session.commit()
+    await engine.dispose()
+    return user_ids
+
+
+# --- metrics -----------------------------------------------------------------
+
+
+def _rss_mb() -> float:
+    with open("/proc/self/status") as fh:
+        for line in fh:
+            if line.startswith("VmRSS:"):
+                return int(line.split()[1]) / 1024.0
+    return 0.0
+
+
+@dataclass
+class Sample:
+    t: float
+    owned: int
+    hot: int
+    warm: int
+    cold: int
+    persistent: int
+    total_open: int
+    max_lag: float
+    rss_mb: float
+    synced: int  # accounts that have completed >=1 poll (steady-state gate)
+    dead: int  # owned runners whose work loops have died (zombies)
+    delivered: int  # cumulative synthetic messages persisted (mail mode)
+
+
+async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, int, int]:
+    owned = hot = warm = cold = dead = 0
+    for c in coords:
+        for runner in list(c._runners.values()):
+            owned += 1
+            if runner.tier == "hot":
+                hot += 1
+            elif runner.tier == "warm":
+                warm += 1
+            else:
+                cold += 1
+            if not runner.running():  # owned but its work loops have died = zombie
+                dead += 1
+    return owned, hot, warm, cold, dead
+
+
+def _max_lag(now: float, seeded: int, start: float) -> float:
+    """Oldest sync lag across accounts. Never-synced accounts count from run start."""
+    worst = 0.0
+    synced = MockStats.last_sync
+    for ts in synced.values():
+        worst = max(worst, now - ts)
+    if len(synced) < seeded:  # some accounts have not completed a first poll yet
+        worst = max(worst, now - start)
+    return worst
+
+
+# --- run ---------------------------------------------------------------------
+
+
+def _percentile(values: list[float], pct: float) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    idx = min(len(ordered) - 1, int(pct / 100.0 * len(ordered)))
+    return ordered[idx]
+
+
+async def _soak(cfg: Config) -> int:
+    import app.sync.engine as engine_mod
+
+    # Route every provider build to the mock (no real IMAP for the whole run).
+    engine_mod.build_provider = lambda account: MockProvider(account.id, cfg.idle_s)  # type: ignore[assignment]
+    MockStats.reset()
+    if cfg.mail:
+        MockStats.mail_enabled = True
+        MockStats.mail_per_poll = cfg.mail_per_poll
+        MockStats.msg_bytes = cfg.msg_kb * 1024
+        MockStats.big_pct = cfg.big_pct
+        MockStats.big_bytes = (get_settings().stream_parse_mb + 2) * 1024 * 1024
+
+    redis = get_redis()
+    await redis.flushdb()
+
+    user_ids = await _seed(cfg)
+    active_users = user_ids[: cfg.accounts * cfg.session_pct // 100]
+
+    async def refresh_sessions() -> None:
+        for uid in active_users:
+            await leases.mark_user_active(redis, uid, cfg.session_grace_s)
+
+    await refresh_sessions()
+
+    # Size the per-worker cap so total capacity is just above the fleet: claims
+    # fill each worker roughly evenly and the rebalancer barely has to shed
+    # (mirrors how you'd tune MAX_ACCOUNTS_PER_WORKER in a real deployment).
+    import math
+
+    cap = max(50, math.ceil(cfg.accounts / cfg.workers * 1.2))
+    coords = [SyncCoordinator(f"soak-w{i}") for i in range(cfg.workers)]
+    for c in coords:
+        c.max_accounts = cap
+        c.claim_interval = cfg.claim_s
+        c.claim_jitter = 1.0
+        c.tier_eval_interval = cfg.tier_eval_s
+        c.heartbeat_refresh = 5.0
+        c.rebalance_interval = 20.0
+        c.reconcile_interval = 60.0
+    await asyncio.gather(*(c.start() for c in coords))
+
+    start = time.time()
+    deadline = start + cfg.duration_s
+    next_session_refresh = start + cfg.session_grace_s / 2
+    samples: list[Sample] = []
+    mail_desc = (
+        f"mail ON ~{cfg.mail_per_poll}/poll {cfg.msg_kb}KB big={cfg.big_pct}%"
+        if cfg.mail
+        else "mail OFF"
+    )
+    print(
+        f"soak: {cfg.accounts} accounts, {len(active_users)} active sessions, "
+        f"{cfg.workers} workers, duration {cfg.duration_s}s, warmup {cfg.warmup_s}s, "
+        f"POLL_COLD={cfg.poll_cold}s POLL_WARM={cfg.poll_warm}s, {mail_desc}",
+        flush=True,
+    )
+    print(
+        f"{'t':>6} {'owned':>6} {'hot':>5} {'warm':>5} {'cold':>5} "
+        f"{'persist':>8} {'open':>5} {'lag_s':>7} {'rss_mb':>8} {'dead':>5} "
+        f"{'msgs':>8} {'msg/s':>7}",
+        flush=True,
+    )
+    try:
+        while time.time() < deadline:
+            await asyncio.sleep(cfg.sample_s)
+            now = time.time()
+            if now >= next_session_refresh:
+                await refresh_sessions()
+                next_session_refresh = now + cfg.session_grace_s / 2
+            owned, hot, warm, cold, dead = await _tier_counts(coords)
+            s = Sample(
+                t=now - start,
+                owned=owned,
+                hot=hot,
+                warm=warm,
+                cold=cold,
+                persistent=MockStats.persistent_open,
+                total_open=MockStats.open,
+                max_lag=_max_lag(now, cfg.accounts, start),
+                rss_mb=_rss_mb(),
+                synced=len(MockStats.last_sync),
+                dead=dead,
+                delivered=MockStats.delivered,
+            )
+            samples.append(s)
+            prev = samples[-2] if len(samples) > 1 else None
+            rate = (s.delivered - prev.delivered) / cfg.sample_s if prev else 0.0
+            print(
+                f"{s.t:6.0f} {s.owned:6d} {s.hot:5d} {s.warm:5d} {s.cold:5d} "
+                f"{s.persistent:8d} {s.total_open:5d} {s.max_lag:7.0f} {s.rss_mb:8.1f} "
+                f"{s.dead:5d} {s.delivered:8d} {rate:7.1f}",
+                flush=True,
+            )
+    finally:
+        await asyncio.gather(*(c.stop() for c in coords), return_exceptions=True)
+
+    return _report(cfg, samples)
+
+
+def _report(cfg: Config, samples: list[Sample]) -> int:
+    # Steady state begins once every account has completed its first sync (and at
+    # least the configured warm-up has elapsed) — before that, lag legitimately
+    # reflects accounts still ramping up, not a starved steady state.
+    all_synced_t = next((s.t for s in samples if s.synced >= cfg.accounts), None)
+    if all_synced_t is None:
+        worst = max((s.synced for s in samples), default=0)
+        print(
+            f"\nFAIL: not all accounts synced (best {worst}/{cfg.accounts}); "
+            "increase SOAK_DURATION_S",
+            flush=True,
+        )
+        return 1
+    floor = max(cfg.warmup_s, all_synced_t)
+    steady = [s for s in samples if s.t >= floor]
+    if not steady:
+        print("\nFAIL: no steady-state samples (increase SOAK_DURATION_S)", flush=True)
+        return 1
+
+    owned = max(s.owned for s in steady)
+    hot = max(s.hot for s in steady)
+    persistent_p95 = _percentile([s.persistent for s in steady], 95)
+    open_p95 = _percentile([float(s.total_open) for s in steady], 95)
+    lag_max = max(s.max_lag for s in steady)
+    # RSS trend: compare the mean of the first third of steady samples to the last
+    # third. This detects a genuine upward drift (leak) while ignoring transient
+    # parse spikes from large messages (min-to-max would flake in mail mode).
+    third = max(1, len(steady) // 3)
+    rss_early = sum(s.rss_mb for s in steady[:third]) / third
+    rss_late = sum(s.rss_mb for s in steady[-third:]) / third
+    rss_max = max(s.rss_mb for s in steady)
+    rss_growth = (rss_late - rss_early) / rss_early * 100 if rss_early else 0.0
+
+    socket_budget = hot * 1.05  # §10.7: persistent sockets <= hot count + 5%
+    lag_budget = cfg.poll_cold * 1.15
+    rss_budget_pct = 15.0
+
+    # §10.7 hard criteria.
+    checks = [
+        (
+            "all accounts owned",
+            owned >= cfg.accounts,
+            f"owned={owned}/{cfg.accounts}",
+        ),
+        (
+            "persistent sockets <= hot+5%",
+            persistent_p95 <= socket_budget,
+            f"persist_p95={persistent_p95:.0f} budget={socket_budget:.0f} (hot={hot})",
+        ),
+        (
+            "oldest lag < POLL_COLD*1.15",
+            lag_max < lag_budget,
+            f"lag_max={lag_max:.0f}s budget={lag_budget:.0f}s",
+        ),
+        (
+            "RSS trend stable (<15%)",
+            rss_growth <= rss_budget_pct,
+            f"rss early {rss_early:.0f}MB -> late {rss_late:.0f}MB "
+            f"({rss_growth:+.1f}%), peak {rss_max:.0f}MB",
+        ),
+    ]
+
+    print("\n=== soak result (steady-state) ===", flush=True)
+    ok = True
+    for name, passed, detail in checks:
+        ok = ok and passed
+        print(f"  [{'PASS' if passed else 'FAIL'}] {name:30} {detail}", flush=True)
+    # Observational (transient poll connections, not a §10.7 gate).
+    print(
+        f"  [info] total open incl. transient   open_p95={open_p95:.0f}",
+        flush=True,
+    )
+    if cfg.mail:
+        delivered = samples[-1].delivered
+        span = samples[-1].t - samples[0].t or 1.0
+        rss_peak = max(s.rss_mb for s in samples)
+        print(
+            f"  [info] mail delivered={delivered} (~{delivered / span:.1f} msg/s), "
+            f"peak RSS={rss_peak:.0f}MB",
+            flush=True,
+        )
+    print(f"\nsteady samples: {len(steady)}  hot~{hot}  owned~{owned}", flush=True)
+    print("RESULT:", "PASS" if ok else "FAIL", flush=True)
+    return 0 if ok else 1
+
+
+async def _main() -> int:
+    if not os.environ.get("SOAK_DEBUG"):
+        logging.disable(logging.WARNING)  # quiet the per-account loops; keep ERROR+
+    else:
+        logging.basicConfig(level=logging.INFO)
+    print(f"provisioning isolated soak DB '{_SOAK_DB_NAME}' + Redis {os.environ['REDIS_URL']} ...")
+    await _create_database()
+    _migrate()
+    try:
+        rc = await _soak(Config())
+    finally:
+        await get_redis().flushdb()
+        await close_redis()
+        from app.db import dispose_engine
+
+        await dispose_engine()
+        await _drop_database()
+        shutil.rmtree(_SOAK_BLOBS, ignore_errors=True)
+        print(f"torn down soak DB '{_SOAK_DB_NAME}' + blob dir", flush=True)
+    return rc
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(_main()))

backend/migrations/versions/0006_user_quota.py

@@ -21,9 +21,7 @@ depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
-    op.execute(
-        "ALTER TABLE users ADD COLUMN quota_bytes bigint NOT NULL DEFAULT 5368709120"
-    )
+    op.execute("ALTER TABLE users ADD COLUMN quota_bytes bigint NOT NULL DEFAULT 5368709120")
 
 
 def downgrade() -> None:

backend/tests/accounts/test_account_api.py

@@ -13,6 +13,7 @@ from sqlalchemy import delete, select
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.pool import NullPool
 
+from app import queue as app_queue
 from app.api import accounts as accounts_api
 from app.config import get_settings
 from app.identity.passwords import hash_password
@@ -114,14 +115,14 @@ async def test_create_account_enqueues_sync(ctx: Ctx, monkeypatch) -> None:  # t
     enqueued: list[tuple[str, str]] = []
 
     class FakePool:
-        async def enqueue_job(self, name: str, arg: str) -> None:
+        async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
             enqueued.append((name, arg))
 
     async def fake_pool() -> FakePool:
         return FakePool()
 
     monkeypatch.setattr(accounts_api, "test_account", fake_test)
-    monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
 
     resp = await ctx.client.post(
         "/api/accounts",
@@ -151,14 +152,14 @@ async def _make_account(ctx: Ctx, monkeypatch) -> str:  # type: ignore[no-untype
         return None
 
     class FakePool:
-        async def enqueue_job(self, name: str, arg: str) -> None:
+        async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
             return None
 
     async def fake_pool() -> FakePool:
         return FakePool()
 
     monkeypatch.setattr(accounts_api, "test_account", fake_test)
-    monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
     resp = await ctx.client.post(
         "/api/accounts",
         json={**GOOD_CFG, "email": "me@example.com", "display_name": "Me", "password": "secret"},
@@ -174,13 +175,13 @@ async def test_sync_all_enqueues_refresh_per_account(ctx: Ctx, monkeypatch) -> N
     enqueued: list[tuple[str, str]] = []
 
     class FakePool:
-        async def enqueue_job(self, name: str, arg: str) -> None:
+        async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
             enqueued.append((name, arg))
 
     async def fake_pool() -> FakePool:
         return FakePool()
 
-    monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
     resp = await ctx.client.post("/api/accounts/sync", headers=_hdr())
     assert resp.status_code == 200
     assert enqueued == [("refresh_account_job", aid)]
@@ -221,7 +222,7 @@ async def test_update_password_reencrypts_and_resyncs(ctx: Ctx, monkeypatch) ->
     enqueued: list[tuple[str, str]] = []
 
     class FakePool:
-        async def enqueue_job(self, name: str, arg: str) -> None:
+        async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
             enqueued.append((name, arg))
 
     async def fake_pool() -> FakePool:
@@ -231,7 +232,7 @@ async def test_update_password_reencrypts_and_resyncs(ctx: Ctx, monkeypatch) ->
         return None
 
     monkeypatch.setattr(accounts_api, "test_account", fake_test)
-    monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
 
     resp = await ctx.client.patch(
         f"/api/accounts/{aid}", json={"password": "newsecret"}, headers=_hdr()
@@ -259,14 +260,14 @@ async def test_delete_account(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no
         return None
 
     class FakePool:
-        async def enqueue_job(self, name: str, arg: str) -> None:
+        async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
             return None
 
     async def fake_pool() -> FakePool:
         return FakePool()
 
     monkeypatch.setattr(accounts_api, "test_account", fake_test)
-    monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
 
     created = await ctx.client.post(
         "/api/accounts",

backend/tests/indexer/test_streaming_parse.py

@@ -0,0 +1,60 @@
+"""Bounded / streaming MIME parse (refactor §7).
+
+Large messages are parsed from their blob file rather than an in-memory tree, and
+all parsing passes through the PARSE_CONCURRENCY semaphore.
+"""
+
+from __future__ import annotations
+
+import uuid
+from email.message import EmailMessage
+
+from app.indexer import indexer
+from app.indexer.mime import parse_message, parse_message_file
+
+
+def _raw(body: str = "hello") -> bytes:
+    msg = EmailMessage()
+    msg["Subject"] = "Streamed"
+    msg["From"] = "a@example.com"
+    msg["To"] = "b@example.com"
+    msg["Message-ID"] = "<s1@example.com>"
+    msg.set_content(body)
+    msg.add_attachment(
+        b"x" * 1024, maintype="application", subtype="octet-stream", filename="a.bin"
+    )
+    return msg.as_bytes()
+
+
+def test_parse_file_matches_bytes(tmp_path) -> None:  # type: ignore[no-untyped-def]
+    raw = _raw()
+    path = tmp_path / "msg.eml"
+    path.write_bytes(raw)
+    # Parsing from the file handle yields exactly the same ParsedEmail as from bytes.
+    assert parse_message_file(path) == parse_message(raw)
+
+
+async def test_parse_bounded_streams_large_messages(monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    from app.config import get_settings
+
+    threshold = get_settings().stream_parse_mb * 1024 * 1024
+    calls: list[str] = []
+
+    def fake_file(path: object) -> str:
+        calls.append("file")
+        return "file"
+
+    def fake_bytes(raw: bytes) -> str:
+        calls.append("bytes")
+        return "bytes"
+
+    monkeypatch.setattr(indexer, "parse_message_file", fake_file)
+    monkeypatch.setattr(indexer, "parse_message", fake_bytes)
+
+    user_id = uuid.uuid4()
+    small = await indexer._parse_bounded(user_id, "sha", b"x" * 16)
+    big = await indexer._parse_bounded(user_id, "sha", b"x" * (threshold + 1))
+
+    assert small == "bytes"  # small stays in memory
+    assert big == "file"  # over threshold => parsed from the blob file
+    assert calls == ["bytes", "file"]

backend/tests/mail/test_actions.py

@@ -15,7 +15,7 @@ from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.pool import NullPool
 
-from app.api import actions as actions_api
+from app import queue as app_queue
 from app.config import get_settings
 from app.db import session_scope
 from app.identity.passwords import hash_password
@@ -112,13 +112,13 @@ async def ctx(monkeypatch) -> AsyncIterator[Ctx]:  # type: ignore[no-untyped-def
 
     # capture enqueues instead of hitting a real arq pool
     class FakePool:
-        async def enqueue_job(self, *args: object) -> None:
+        async def enqueue_job(self, *args: object, **kwargs: object) -> None:
             c.enqueued.append(args)
 
     async def fake_pool() -> FakePool:
         return FakePool()
 
-    monkeypatch.setattr(actions_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
 
     token = await create_session(user.id, "pytest")
     c.client = httpx.AsyncClient(

backend/tests/outbox/test_draft_sync.py

@@ -127,9 +127,7 @@ async def test_draft_messages_are_not_threaded(ctx: Ctx) -> None:
     m.set_content("body")
 
     drafts_mb = await ctx.session.scalar(
-        select(Mailbox).where(
-            Mailbox.account_id == ctx.account_id, Mailbox.special_use == "drafts"
-        )
+        select(Mailbox).where(Mailbox.account_id == ctx.account_id, Mailbox.special_use == "drafts")
     )
     assert drafts_mb is not None
     remote = RemoteMessage(

backend/tests/outbox/test_outbox.py

@@ -16,7 +16,7 @@ from sqlalchemy import delete, select
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.pool import NullPool
 
-from app.api import compose as compose_api
+from app import queue as app_queue
 from app.config import get_settings
 from app.db import session_scope
 from app.identity.passwords import hash_password
@@ -181,7 +181,7 @@ async def ctx(monkeypatch) -> AsyncIterator[Ctx]:  # type: ignore[no-untyped-def
     async def fake_pool() -> FakePool:
         return FakePool()
 
-    monkeypatch.setattr(compose_api, "get_arq_pool", fake_pool)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
 
     token = await create_session(user.id, "pytest")
     c.client = httpx.AsyncClient(

backend/tests/providers/test_conn_limits.py

@@ -0,0 +1,58 @@
+"""Per-account / per-host connection caps (refactor §4).
+
+The provider holds a per-account (≤4) and per-host semaphore slot for the whole
+lifetime of an open socket, so a worker can never open hundreds of simultaneous
+connections to one account or host.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from app.providers.imap_smtp.provider import _PER_ACCOUNT_CONN_CAP, ImapSmtpProvider, ProviderConfig
+
+
+def _cfg(username: str, host: str) -> ProviderConfig:
+    return ProviderConfig(
+        imap_host=host,
+        imap_port=993,
+        imap_tls="ssl",
+        smtp_host=host,
+        smtp_port=465,
+        smtp_tls="ssl",
+        username=username,
+        password="pw",
+    )
+
+
+async def test_per_account_connection_cap() -> None:
+    cfg = _cfg("user@example.com", "imap.example.com")
+    providers = [ImapSmtpProvider(cfg) for _ in range(_PER_ACCOUNT_CONN_CAP + 1)]
+
+    # Fill the account's slots (host cap defaults to 50, so account is the limiter).
+    for provider in providers[:_PER_ACCOUNT_CONN_CAP]:
+        await provider._acquire_conn_slots()
+
+    # The next acquire blocks until a slot frees.
+    with pytest.raises(TimeoutError):
+        await asyncio.wait_for(providers[-1]._acquire_conn_slots(), timeout=0.2)
+
+    providers[0]._release_conn_slots()
+    await asyncio.wait_for(providers[-1]._acquire_conn_slots(), timeout=0.5)  # now succeeds
+
+    for provider in providers[1:]:
+        provider._release_conn_slots()
+
+
+async def test_distinct_accounts_do_not_share_slots() -> None:
+    # Saturating account A must not block account B (independent semaphores).
+    a = [ImapSmtpProvider(_cfg("a@example.com", "imap.example.com")) for _ in range(4)]
+    for provider in a:
+        await provider._acquire_conn_slots()
+    b = ImapSmtpProvider(_cfg("b@example.com", "imap.example.com"))
+    await asyncio.wait_for(b._acquire_conn_slots(), timeout=0.5)
+    b._release_conn_slots()
+    for provider in a:
+        provider._release_conn_slots()

backend/tests/sync/test_acceptance.py

@@ -0,0 +1,415 @@
+"""Sync-engine v2 acceptance tests 1-3 (refactor §10).
+
+These drive the real :class:`SyncCoordinator` against the test Redis with the
+per-account IMAP loops replaced by fakes, so the lease protocol — single
+ownership, crash failover, and create/delete lifecycle — is exercised without a
+live mail server. Intervals are shortened so the tests run in seconds.
+
+  1. Two workers, N accounts: every account owned by exactly one worker, exactly
+     one live (IDLE) connection each.
+  2. Kill a worker: its orphaned accounts are re-owned by the survivor once the
+     leases expire, still one connection each (no double ownership).
+  3. A newly-live account is claimed without a restart; a removed account's
+     connection is torn down (lease fencing), with no further ownership.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from collections import defaultdict
+from collections.abc import AsyncIterator, Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest_asyncio
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+from app.db import session_scope
+from app.events import get_event_bus
+from app.identity.passwords import hash_password
+from app.models import Account, Mailbox, Message, User
+from app.redis_client import get_redis
+from app.sync import coordinator as coordinator_mod
+from app.sync import leases, runner
+from app.sync.coordinator import SyncCoordinator
+
+# --- fakes: replace the real IMAP loops with connection-counting stand-ins ----
+
+
+@pytest_asyncio.fixture
+async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]:  # type: ignore[no-untyped-def]
+    """Patch the runner's loops; yield a live (hot-tier IDLE) connection count."""
+    conns: dict[uuid.UUID, int] = defaultdict(int)
+
+    async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None:
+        conns[account_id] += 1  # one persistent IDLE connection while alive (hot)
+        try:
+            await stop.wait()
+        finally:
+            conns[account_id] -= 1
+
+    async def fake_secondary(
+        account_id: uuid.UUID, stop: asyncio.Event | None = None, interval: float | None = None
+    ) -> None:
+        assert stop is not None
+        await stop.wait()
+
+    async def fake_poll_all(
+        account_id: uuid.UUID, interval: float, stop: asyncio.Event | None = None
+    ) -> None:
+        assert stop is not None  # warm/cold: no persistent connection, just wait
+        await stop.wait()
+
+    monkeypatch.setattr(runner, "live_loop", fake_live_loop)
+    monkeypatch.setattr(runner, "secondary_poll_loop", fake_secondary)
+    monkeypatch.setattr(runner, "poll_all_loop", fake_poll_all)
+    yield conns
+
+
+@pytest_asyncio.fixture
+async def make_accounts() -> AsyncIterator[Callable[[int, str], object]]:
+    """Factory creating N live accounts; cleans up its users afterwards."""
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    user_ids: list[uuid.UUID] = []
+
+    async def _make(n: int, state: str = "live") -> list[uuid.UUID]:
+        async with sm() as session:
+            user = User(
+                handle=f"acc{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")
+            )
+            session.add(user)
+            await session.flush()
+            user_ids.append(user.id)
+            ids: list[uuid.UUID] = []
+            for _ in range(n):
+                account = Account(
+                    user_id=user.id,
+                    email=f"{uuid.uuid4().hex[:10]}@example.com",
+                    display_name="Acct",
+                    imap_config={},
+                    smtp_config={},
+                    secret_ct=b"",
+                    discovery_src="manual",
+                    state=state,
+                )
+                session.add(account)
+                await session.flush()
+                ids.append(account.id)
+            await session.commit()
+            return ids
+
+    try:
+        yield _make
+    finally:
+        async with sm() as session:
+            for uid in user_ids:
+                await session.execute(delete(User).where(User.id == uid))
+            await session.commit()
+        await engine.dispose()
+
+
+async def _no_reconcile() -> None:
+    """Stub: keep the shared dev DB's real live accounts out of the test."""
+    return None
+
+
+def _fast(c: SyncCoordinator) -> SyncCoordinator:
+    """Compress cadences and drive sync:live explicitly (no DB reconcile).
+
+    The tests share the dev Postgres, which holds real live accounts; reconciling
+    from it would pull them into the test. We seed ``sync:live`` ourselves instead.
+    """
+    c.claim_interval = 0.05
+    c.claim_jitter = 0.0
+    c.lease_renew = 0.3
+    c.lease_ttl = 2  # short so a crashed worker's leases expire quickly
+    c.reconcile_interval = 1000.0
+    c.heartbeat_refresh = 1000.0
+    c.rebalance_interval = 1000.0
+    c.tier_eval_interval = 1000.0  # lease tests pin the tier; no eval churn
+    c._reconcile_once = _no_reconcile  # type: ignore[method-assign]
+    return c
+
+
+async def _seed(ids: set[uuid.UUID], tier: str = "hot") -> None:
+    """Register accounts as live and pin their tier (hot => persistent IDLE)."""
+    redis = get_redis()
+    for account_id in ids:
+        await leases.mark_live(redis, account_id)
+        await redis.set(leases.tier_key(account_id), tier)
+
+
+async def _user_of(account_id: uuid.UUID) -> uuid.UUID:
+    async with session_scope() as session:
+        user_id = await session.scalar(select(Account.user_id).where(Account.id == account_id))
+    assert user_id is not None
+    return user_id
+
+
+async def _add_recent_mail(account_id: uuid.UUID, when: datetime) -> None:
+    """Insert one INBOX message dated ``when`` so the account reads as active."""
+    user_id = await _user_of(account_id)
+    async with session_scope() as session:
+        mailbox = Mailbox(account_id=account_id, user_id=user_id, path="INBOX", special_use="inbox")
+        session.add(mailbox)
+        await session.flush()
+        session.add(
+            Message(
+                user_id=user_id,
+                account_id=account_id,
+                mailbox_id=mailbox.id,
+                uid=1,
+                internal_date=when,
+                blob_sha256="0" * 64,
+            )
+        )
+        await session.commit()
+
+
+async def _until(pred: Callable[[], bool], timeout_s: float = 6.0) -> None:
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + timeout_s
+    while loop.time() < deadline:
+        if pred():
+            return
+        await asyncio.sleep(0.05)
+    assert pred(), "condition not met within timeout"
+
+
+# --- acceptance 1: single ownership, one connection per account ---------------
+
+
+async def test_acceptance_1_single_ownership(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    ids = set(await make_accounts(20))
+    await _seed(ids)
+    c1 = _fast(SyncCoordinator("w1"))
+    c2 = _fast(SyncCoordinator("w2"))
+    c1.rebalance_interval = c2.rebalance_interval = 0.3  # let the load even out
+    # Start concurrently so the two workers race for the leases.
+    await asyncio.gather(c1.start(), c2.start())
+    try:
+        # Both workers end up holding accounts (via the initial race + rebalance),
+        # together covering all 20 with no account owned twice.
+        await _until(lambda: c1.owned | c2.owned == ids and bool(c1.owned and c2.owned))
+        assert c1.owned & c2.owned == set()
+        # At steady state each account has exactly one live IDLE connection.
+        await _until(lambda: all(connections[a] == 1 for a in ids))
+    finally:
+        await c1.stop()
+        await c2.stop()
+    await _until(lambda: all(connections[a] == 0 for a in ids))
+
+
+# --- acceptance 2: crash failover --------------------------------------------
+
+
+async def test_acceptance_2_failover(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    ids = set(await make_accounts(10))
+    await _seed(ids)
+    c1 = _fast(SyncCoordinator("w1"))
+    await c1.start()
+    try:
+        await _until(lambda: c1.owned == ids)  # sole worker owns everything
+
+        c2 = _fast(SyncCoordinator("w2"))
+        await c2.start()
+
+        # Simulate `kill -9` on w1: connections drop and renewals stop, but the
+        # leases are NOT released — they must expire before w2 can take over.
+        for task in c1._tasks:
+            task.cancel()
+        await asyncio.gather(*c1._tasks, return_exceptions=True)
+        c1._tasks = []
+        for r in list(c1._runners.values()):
+            await r.stop()
+        c1._runners.clear()
+
+        try:
+            # w2 re-owns every orphan once the (short) leases expire.
+            await _until(lambda: c2.owned == ids, timeout_s=8.0)
+            assert all(connections[a] == 1 for a in ids)  # one connection each, no dupes
+        finally:
+            await c2.stop()
+    finally:
+        # w1 already crashed; clear any residual leases it left behind.
+        await get_redis().flushdb()
+    await _until(lambda: all(connections[a] == 0 for a in ids))
+
+
+# --- acceptance 3: create / delete lifecycle without restart ------------------
+
+
+async def test_acceptance_3_create_delete(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    c = _fast(SyncCoordinator("w1"))
+    await c.start()  # starts with no live accounts
+    try:
+        assert c.owned == set()
+
+        # A newly-live account (as the bootstrap job's _set_state('live') would do):
+        # register it in sync:live; the running coordinator must claim it, no restart.
+        (aid,) = await make_accounts(1)
+        await get_redis().set(leases.tier_key(aid), "hot")  # persistent IDLE
+        await leases.mark_live(get_redis(), aid)
+        await _until(lambda: aid in c.owned)
+        assert connections[aid] == 1
+
+        # Removing it (as delete_account does) fences the runner: lease gone ->
+        # next renewal fails -> connection torn down, no further ownership.
+        await leases.mark_not_live(get_redis(), aid)
+        await _until(lambda: aid not in c.owned)
+        await _until(lambda: connections[aid] == 0)
+    finally:
+        await c.stop()
+
+
+async def test_event_driven_teardown(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    """§3: an account.state_changed event tears the runner down at once.
+
+    Leases are kept long (no fencing) and the verify loop slow, so the only path
+    that can stop the runner in time is the cross-process event subscriber.
+    """
+    (aid,) = await make_accounts(1)
+    user_id = uuid.uuid4()
+    await _seed({aid})
+    c = _fast(SyncCoordinator("w1"))
+    c.lease_ttl = 60  # lease stays valid: fencing cannot explain the teardown
+    c.lease_renew = 30
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        assert connections[aid] == 1
+
+        # As delete_account does: drop it from sync:live (so it isn't re-claimed),
+        # then emit the event. The lease is left intact, so only the event — not
+        # fencing — can explain the teardown.
+        await get_redis().srem(leases.LIVE_SET, str(aid))
+        await get_event_bus().publish(
+            "account.state_changed",
+            {"user_id": str(user_id), "account_id": str(aid), "state": "deleted"},
+        )
+        await _until(lambda: aid not in c.owned, timeout_s=3.0)
+        await _until(lambda: connections[aid] == 0, timeout_s=3.0)
+    finally:
+        await c.stop()
+
+
+# --- acceptance 6: graceful shutdown -----------------------------------------
+
+
+async def test_acceptance_6_graceful_shutdown(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    """A graceful stop releases every lease so a peer adopts the accounts fast.
+
+    Leases are long-lived, so adoption within a couple of seconds proves the
+    accounts were handed over by explicit release — not by waiting out the TTL.
+    """
+    ids = set(await make_accounts(6))
+    await _seed(ids)  # hot => persistent IDLE
+    c1 = _fast(SyncCoordinator("w1"))
+    c2 = _fast(SyncCoordinator("w2"))
+    c1.lease_ttl = c2.lease_ttl = 30  # adoption must come from release, not expiry
+    await c1.start()
+    await _until(lambda: c1.owned == ids)  # sole owner
+    await c2.start()  # nothing to claim yet — all leased by w1
+    try:
+        await c1.stop()  # graceful: stops claimer, cancels runners, releases leases
+        assert c1.owned == set()
+        assert await get_redis().exists(leases.worker_key("w1")) == 0  # heartbeat gone
+        # w2 adopts every account well within the 30s TTL, one connection each.
+        await _until(lambda: c2.owned == ids, timeout_s=4.0)
+        await _until(lambda: all(connections[a] == 1 for a in ids))
+    finally:
+        await c2.stop()
+    await _until(lambda: all(connections[a] == 0 for a in ids))
+
+
+# --- §5 activity tiers -------------------------------------------------------
+
+
+async def test_tier_session_promotes_to_hot(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    (aid,) = await make_accounts(1)
+    user_id = await _user_of(aid)
+    await leases.mark_live(get_redis(), aid)  # claimed as warm by default
+    await leases.mark_user_active(get_redis(), user_id, 300)  # active session -> hot
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        await _until(lambda: connections[aid] == 1)  # hot => persistent IDLE
+        assert c._runners[aid].tier == "hot"
+    finally:
+        await c.stop()
+
+
+async def test_tier_no_session_no_mail_goes_cold(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    (aid,) = await make_accounts(1)
+    await leases.mark_live(get_redis(), aid)  # no session, no mail
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        await _until(lambda: c._runners[aid].tier == "cold")
+        assert connections[aid] == 0  # cold holds no persistent connection
+    finally:
+        await c.stop()
+
+
+async def test_tier_recent_mail_promotes_to_hot(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    (aid,) = await make_accounts(1)
+    await _add_recent_mail(aid, datetime.now(UTC))  # activity within TIER_HOT_ACTIVITY_H
+    await leases.mark_live(get_redis(), aid)
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        await _until(lambda: connections[aid] == 1)  # recent mail => hot
+        assert c._runners[aid].tier == "hot"
+    finally:
+        await c.stop()
+
+
+async def test_user_active_event_promotes_immediately(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    aid0, aid1 = await make_accounts(2)  # both under one user
+    user_id = await _user_of(aid0)
+    await leases.mark_live(get_redis(), aid0)
+    await leases.mark_live(get_redis(), aid1)
+    c = _fast(SyncCoordinator("w1"))  # tier eval disabled: only the event can promote
+    await c.start()
+    try:
+        await _until(lambda: {aid0, aid1} <= c.owned)
+        assert connections[aid0] == 0 and connections[aid1] == 0  # warm to start
+        await get_event_bus().publish("user.active", {"user_id": str(user_id)})
+        await _until(lambda: connections[aid0] == 1 and connections[aid1] == 1, timeout_s=3.0)
+    finally:
+        await c.stop()
+
+
+async def test_hot_budget_demotes_least_active(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    aid_old, aid_new = await make_accounts(2)  # activity-hot, same user, NO session
+    now = datetime.now(UTC)
+    await _add_recent_mail(aid_old, now - timedelta(hours=1))  # least recently active
+    await _add_recent_mail(aid_new, now)
+    await leases.mark_live(get_redis(), aid_old)
+    await leases.mark_live(get_redis(), aid_new)
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    c.hot_budget = 1  # only one may be hot
+    await c.start()
+    try:
+        await _until(lambda: {aid_old, aid_new} <= c.owned)
+        # The newer account stays hot; the older is shed to warm by the budget.
+        await _until(lambda: c._runners[aid_new].tier == "hot")
+        await _until(lambda: c._runners[aid_old].tier == "warm")
+        assert connections[aid_new] == 1 and connections[aid_old] == 0
+    finally:
+        await c.stop()
+
+
+def test_worker_id_is_unique() -> None:
+    assert coordinator_mod._new_worker_id() != coordinator_mod._new_worker_id()

backend/tests/test_queue_routing.py

@@ -0,0 +1,53 @@
+"""Interactive/bulk queue isolation (refactor §1, acceptance §10.5).
+
+Bulk backfills and interactive user actions land on separate arq queues (run by
+separate consumers with their own concurrency), so a burst of backfills can never
+starve sends/flag changes. Here we assert the routing that guarantees it.
+"""
+
+from __future__ import annotations
+
+from app import queue as app_queue
+from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, queue_for
+
+_INTERACTIVE_JOBS = [
+    "push_flags_job",
+    "move_job",
+    "move_thread_to_folder_job",
+    "delete_thread_job",
+    "send_job",
+    "push_draft_to_imap_job",
+    "remove_imap_draft_job",
+    "refresh_account_job",
+]
+
+
+def test_bulk_backfill_routes_to_bulk_queue() -> None:
+    assert queue_for("sync_bootstrap_account") == QUEUE_BULK
+
+
+def test_interactive_jobs_route_to_interactive_queue() -> None:
+    for name in _INTERACTIVE_JOBS:
+        assert queue_for(name) == QUEUE_INTERACTIVE
+
+
+async def test_enqueue_job_sets_queue_name(monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    captured: list[tuple[str, str | None]] = []
+
+    class FakePool:
+        async def enqueue_job(self, name: str, *args: object, **kwargs: object) -> None:
+            captured.append((name, kwargs.get("_queue_name")))  # type: ignore[arg-type]
+            return None
+
+    async def fake_pool() -> FakePool:
+        return FakePool()
+
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
+
+    await app_queue.enqueue_job("sync_bootstrap_account", "a")
+    await app_queue.enqueue_job("send_job", "b")
+
+    assert captured == [
+        ("sync_bootstrap_account", QUEUE_BULK),
+        ("send_job", QUEUE_INTERACTIVE),
+    ]

backend/workers/run.py

@@ -0,0 +1,99 @@
+"""Worker entrypoint (refactor §1, §7).
+
+One image, two roles selected by ``WORKER_ROLE``:
+
+- ``jobs`` — arq consumer for the ``interactive`` and ``bulk`` queues (both in one
+  process, per-queue concurrency/timeout).
+- ``sync`` — runs the lease-owned account-sync coordinator (§2) directly.
+- ``all``  — both, for single-process development (the default).
+
+Graceful shutdown (SIGINT/SIGTERM): stop the coordinator first — it stops the
+claimer, cancels the runners, and releases every lease so a peer adopts the
+accounts immediately — then drain the arq consumers. Target < 10 s.
+
+Run with: ``python -m workers.run``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import signal
+from typing import Any
+
+from arq.worker import Worker
+
+from app.config import get_settings
+from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, redis_settings
+from app.sync.coordinator import SyncCoordinator
+from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS
+
+logger = logging.getLogger(__name__)
+
+_SHUTDOWN_DRAIN_S = 10.0  # §7: release leases + drain within this budget
+
+
+def _worker(functions: list[Any], queue_name: str, max_jobs: int, job_timeout: int) -> Worker:
+    return Worker(
+        functions=functions,
+        queue_name=queue_name,
+        redis_settings=redis_settings(),
+        max_jobs=max_jobs,
+        job_timeout=job_timeout,
+        handle_signals=False,  # the runner owns signal handling (graceful shutdown)
+    )
+
+
+def build_arq_workers(role: str) -> list[Worker]:
+    """Construct the arq Worker instances for the given role (jobs only)."""
+    s = get_settings()
+    if role not in ("jobs", "all"):
+        return []
+    return [
+        _worker(INTERACTIVE_FUNCTIONS, QUEUE_INTERACTIVE, s.interactive_concurrency, 120),
+        _worker(BULK_FUNCTIONS, QUEUE_BULK, s.bulk_concurrency, 3600),
+    ]
+
+
+async def _run() -> None:
+    logging.basicConfig(level=get_settings().log_level)
+    role = get_settings().worker_role
+
+    coordinator = SyncCoordinator() if role in ("sync", "all") else None
+    if coordinator is not None:
+        await coordinator.start()
+    arq_tasks = [asyncio.create_task(w.async_run()) for w in build_arq_workers(role)]
+    logger.info("worker starting: role=%s (%d arq worker(s))", role, len(arq_tasks))
+
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        loop.add_signal_handler(sig, stop.set)
+    await stop.wait()
+
+    logger.info("worker stopping: releasing leases + draining")
+    if coordinator is not None:
+        try:
+            await asyncio.wait_for(coordinator.stop(), timeout=_SHUTDOWN_DRAIN_S)
+        except TimeoutError:
+            logger.warning("coordinator stop exceeded %.0fs drain budget", _SHUTDOWN_DRAIN_S)
+    for task in arq_tasks:
+        task.cancel()
+    await asyncio.gather(*arq_tasks, return_exceptions=True)
+
+    from app.db import dispose_engine
+    from app.queue import close_arq_pool
+    from app.redis_client import close_redis
+
+    await dispose_engine()
+    await close_redis()
+    await close_arq_pool()
+    logger.info("worker stopped")
+
+
+def main() -> None:
+    asyncio.run(_run())
+
+
+if __name__ == "__main__":
+    main()

backend/workers/worker.py

@@ -1,6 +1,9 @@
-"""arq worker entrypoint (SPEC §2 ``workers/worker.py``).
+"""Worker job callables and queue function-sets (refactor §1, §6).
 
-Run with: ``arq workers.worker.WorkerSettings`` (from ``backend/``).
+The process is launched via ``workers.run`` (``WORKER_ROLE`` decides which arq
+workers run). This module only defines the job callables and the two queue
+function-sets; the runner wires them onto the ``interactive`` / ``bulk`` queues.
+The sync role runs the coordinator directly (see ``workers.run``), not via arq.
 """
 
 from __future__ import annotations
@@ -9,10 +12,8 @@ import logging
 import uuid
 from typing import Any
 
-from app.config import get_settings
 from app.outbox.draft_sync import push_draft_to_imap_job, remove_imap_draft_job
 from app.outbox.jobs import send_job
-from app.queue import redis_settings as arq_redis_settings
 from app.sync.actions import (
     delete_thread_job,
     move_job,
@@ -32,41 +33,17 @@ async def sync_bootstrap_account(ctx: dict[str, Any], account_id: str) -> int:
     return count
 
 
-async def on_startup(ctx: dict[str, Any]) -> None:
-    logging.basicConfig(level=get_settings().log_level)
-    from app.sync.supervisor import start_supervisor
+# Fast, interactive user actions (§6): short timeout, high concurrency.
+INTERACTIVE_FUNCTIONS = [
+    push_flags_job,
+    move_job,
+    move_thread_to_folder_job,
+    delete_thread_job,
+    send_job,
+    push_draft_to_imap_job,
+    remove_imap_draft_job,
+    refresh_account_job,
+]
 
-    await start_supervisor(ctx)
-    logger.info("webmail worker started")
-
-
-async def on_shutdown(ctx: dict[str, Any]) -> None:
-    from app.db import dispose_engine
-    from app.redis_client import close_redis
-    from app.sync.supervisor import stop_supervisor
-
-    await stop_supervisor(ctx)
-    await dispose_engine()
-    await close_redis()
-    logger.info("webmail worker stopped")
-
-
-class WorkerSettings:
-    """arq worker configuration."""
-
-    functions = [
-        sync_bootstrap_account,
-        push_flags_job,
-        move_job,
-        move_thread_to_folder_job,
-        delete_thread_job,
-        send_job,
-        push_draft_to_imap_job,
-        remove_imap_draft_job,
-        refresh_account_job,
-    ]
-    on_startup = on_startup
-    on_shutdown = on_shutdown
-    max_jobs = 10  # global worker cap (SPEC §6.3)
-    job_timeout = 3600  # backfill of a large account can be long
-    redis_settings = arq_redis_settings()
+# Bulk work (§6): long timeout, low concurrency, isolated from interactive jobs.
+BULK_FUNCTIONS = [sync_bootstrap_account]

docker/docker-compose.yml

@@ -36,6 +36,36 @@ services:
       timeout: 3s
       retries: 10
 
+  # Sync-engine v2 workers (refactor §1): two roles sharing one image. Scale the
+  # sync role with `--scale worker-sync=N` (lease protocol prevents double-owning).
+  worker-sync:
+    build: { context: ../backend }
+    command: ["python", "-m", "workers.run"]
+    environment: &worker-env
+      WORKER_ROLE: sync
+      APP_DOMAIN: dev.webmail.local
+      DATABASE_URL: postgresql+asyncpg://webmail:webmail_dev@postgres:5432/webmail_dev
+      REDIS_URL: redis://redis:6379/0
+      MASTER_KEY: CHANGE_ME_base64_32_bytes
+      BLOB_ROOT: /var/blobs
+    volumes: [blobs:/var/blobs]
+    ulimits: { nofile: 65536 }
+    depends_on:
+      postgres: { condition: service_healthy }
+      redis: { condition: service_healthy }
+
+  worker-jobs:
+    build: { context: ../backend }
+    command: ["python", "-m", "workers.run"]
+    environment:
+      <<: *worker-env
+      WORKER_ROLE: jobs
+    volumes: [blobs:/var/blobs]
+    ulimits: { nofile: 65536 }
+    depends_on:
+      postgres: { condition: service_healthy }
+      redis: { condition: service_healthy }
+
   # Test mail servers (integration tests + manual dev against a throwaway mailbox)
   greenmail:
     image: greenmail/standalone:2.0.1
@@ -51,3 +81,4 @@ services:
 volumes:
   pgdata:
   meilidata:
+  blobs:

sync-engine-v2-refactor.md

@@ -0,0 +1,119 @@
+# Sync Engine v2 — Refactor Instructions (replaces SPEC §6.3)
+
+This document is a self-contained repair/rebuild task for the implementing agent. It replaces the current single-supervisor sync design with a sharded, lease-owned, tiered sync engine targeting **1000+ accounts**. Read it fully, then execute the migration steps in §9 **in order**.
+
+## 0. Ground rules
+
+- This is a **workers-layer refactor only**. Frozen and untouchable: the `MailProvider` interface (SPEC §5.1), the database schema (no migration is needed or allowed by this task), the REST/WS API surface, invariant **I-1** (API never opens IMAP connections), and the idempotency discipline (`UNIQUE (mailbox_id, uid)` upserts).
+- Existing tests must stay green after every migration step. Each step ends with its own acceptance checks (§10 maps them).
+- All new tunables go through `config.py` (pydantic-settings) — no literals in code. New env vars are listed in §8.
+- Redis is already the queue and event bus; it now also carries leases, the live-account set, tier state, and worker heartbeats. No new infrastructure.
+
+## 1. Process model: two roles
+
+The single worker process splits into two entrypoints (same codebase, `WORKER_ROLE` env):
+
+- **`role=jobs`** — pure arq consumer for the discrete jobs (send, flags, move, drafts, backfill). Stateless, horizontally scalable; run N instances freely.
+- **`role=sync`** — runs the lease claimer + per-account runners (long-lived loops). Scalable via the lease protocol in §2; run 1..N instances.
+
+Default dev/prod-small topology: 2× sync + 2× jobs. docker-compose gets two services (`worker-sync`, `worker-jobs`) sharing the image, each with `ulimits: { nofile: 65536 }`.
+
+## 2. Lease protocol (account ownership)
+
+Exactly one sync worker owns each live account at any moment. Ownership is a Redis lease:
+
+- **Registry:** `sync:live` — a Redis SET of account ids with `state='live'`. Maintained by the API on every account state change AND reconciled against Postgres by each sync worker every 300 s (belt and braces; reconcile only adds/removes ids, never touches leases).
+- **Lease key:** `sync:lease:{account_id}` = `{worker_id}`, TTL `LEASE_TTL` (60 s). `worker_id` = `{hostname}:{pid}:{random8}` generated at startup.
+- **Claim:** `SET sync:lease:{id} {worker_id} NX EX {LEASE_TTL}` — success starts the account runner (§4).
+- **Claimer loop:** every 10 s + random jitter 0–5 s, while `owned < MAX_ACCOUNTS_PER_WORKER`: iterate `sync:live` (SSCAN), attempt claims on ids with no lease. Jitter prevents thundering-herd on mass restart.
+- **Renewal:** every `LEASE_RENEW` (20 s), for each owned account run this Lua atomically (never a plain EXPIRE):
+  ```lua
+  if redis.call('GET', KEYS[1]) == ARGV[1]
+  then return redis.call('EXPIRE', KEYS[1], ARGV[2]) else return 0 end
+  ```
+- **Fencing (hard rule):** if renewal returns 0 for an account (lease lost or taken), the worker cancels that account's runner tasks **immediately, before any further provider or DB operation for it**. In-flight operations are tolerable because all writes are idempotent upserts.
+- **Release:** on graceful stop or account disable, cancel the runner, then delete the lease with the owner-checked Lua (`GET==me → DEL`).
+- **Worker registry & rebalance:** each sync worker heartbeats `sync:workers:{worker_id}` = JSON `{owned, sockets_open, tiers:{hot,warm,cold}, ts}` (TTL 30 s, refresh 10 s). Every 60 s: `target = |sync:live| / |alive workers|`; if `owned > target*1.2 + 5`, release the excess, choosing least-recently-active accounts first — released accounts are claimed by under-loaded workers within one claimer cycle.
+
+**Failover story (emergent, no extra code):** worker crash → leases expire ≤ `LEASE_TTL` → survivors claim orphans on the next claimer tick. Total re-ownership within ~90 s.
+
+## 3. Event-driven account lifecycle (kills startup-only discovery + zombie loops)
+
+Sync workers subscribe to the event bus:
+
+- `account.state_changed → live` (new account finished backfill, or re-enabled): API adds the id to `sync:live`; some worker claims it within one claimer cycle. **No restart, ever.**
+- `account.state_changed → disabled|deleted`: API removes the id from `sync:live`; the owning worker (it can check `GET sync:lease:{id}==me` cheaply, or simply receives the event and checks ownership) stops the runner, closes connections, releases the lease — target ≤ 30 s from the API call.
+- Defense in depth: every runner poll cycle re-verifies the account row exists with `state='live'`; if not, self-terminate and release. This makes zombie loops structurally impossible even if an event is lost.
+- **New core event `user.active`** (add to SPEC §8.3 catalog): published by the API on successful login and on every WS connect, payload `{user_id}`. Consumed for tier promotion (§5).
+
+## 4. Per-account runner (replaces the fixed 2-task model)
+
+One `AccountRunner` per owned account, whose task set depends on the current tier:
+
+- **hot:** task A = INBOX IDLE loop (one persistent connection; re-issue IDLE ≤ 25 min; on wake → `poll()` INBOX). Task B = secondary CONDSTORE poll of Sent/Drafts/Archive/Trash/Junk every `POLL_WARM` s on a transient connection.
+- **warm:** no persistent connection. Single task: transient CONDSTORE poll of INBOX + special folders every `POLL_WARM` s.
+- **cold:** same single task, every `POLL_COLD` s.
+
+Delta application, UIDVALIDITY handling, expunge soft-handling, and outbound flag write-then-enqueue semantics are **unchanged** from the current implementation.
+
+**Connection budgets:**
+- ≤ 2 concurrent IMAP connections per account from the sync side (IDLE + one transient); jobs may add short-lived connections but must acquire the same per-account semaphore (cap 4 total, as today).
+- Per-destination-host semaphore (process-local), cap `PER_HOST_CONN_CAP` (default 50): acquired around every IMAP/SMTP connect, so a worker never opens hundreds of simultaneous sockets to one provider. Sharding by account makes process-local sufficient.
+
+## 5. Activity tiers
+
+Tier state: `sync:tier:{account_id}` = `hot|warm|cold` (TTL none; default `warm` on claim). Evaluated by the owning worker every 120 s and on events:
+
+- **→ hot** when: `user.active` received for the owning user (all their accounts promote within ≤ 5 s, IDLE established), OR the account received new mail in the last `TIER_HOT_ACTIVITY_H` (24 h). Session-based heat persists while WS pings continue + `TIER_SESSION_GRACE_MIN` (30 min) after the last one (API refreshes `sess:active:{user_id}` TTL; workers just read it).
+- **→ warm** when neither hot condition holds and last activity < `TIER_COLD_AFTER_D` (3 d).
+- **→ cold** when no user session and no new mail for ≥ `TIER_COLD_AFTER_D` days. Any new mail found by a cold poll immediately re-promotes to hot (mail arriving = activity).
+- Optional global guard: if a worker's hot count exceeds `HOT_BUDGET_PER_WORKER` (default 250), the least-recently-active hot accounts without live sessions degrade to warm. Session-hot accounts are never degraded by the budget.
+
+Expected steady state at ~3000 accounts: roughly 15–25 % hot → 450–750 persistent sockets across the pool, the rest on cheap transient polls. Correctness is identical in all tiers — only latency differs (seconds / ≤ `POLL_WARM` / ≤ `POLL_COLD`).
+
+## 6. Queue split (interactive vs bulk)
+
+Two arq queues, consumed by `role=jobs` workers:
+
+| Queue | Jobs | `job_timeout` | Concurrency per worker |
+|---|---|---|---|
+| `interactive` | send_job, push_flags_job, move_job, move_thread_to_folder_job, delete_thread_job, push_draft_to_imap_job, remove_imap_draft_job, refresh_account_job | 120 s | 20 |
+| `bulk` | sync_bootstrap_account (and future re-backfills) | 3600 s | `BULK_CONCURRENCY` (default 2) |
+
+Every jobs worker consumes both queues (arq supports it) with the above per-queue caps — a storm of backfills can no longer delay a send. Retry/backoff and failure-revert semantics per job are unchanged.
+
+## 7. Backfill memory bounds
+
+- Global backfill parallelism is bounded by the `bulk` queue concurrency (above) — remove any other ad-hoc caps.
+- Per-process parse semaphore `PARSE_CONCURRENCY` (default 4) around MIME parsing.
+- Streaming threshold: a raw message larger than `STREAM_PARSE_MB` (default 8) is written to the blob store first and parsed from the file handle; smaller messages may parse in memory. Result: peak RAM ≈ `PARSE_CONCURRENCY × STREAM_PARSE_MB` per process, independent of mailbox sizes.
+- `sync.progress` publication cadence unchanged (per batch).
+
+## 8. New configuration (add to SPEC §14; all via `config.py`)
+
+`WORKER_ROLE=jobs|sync` · `MAX_ACCOUNTS_PER_WORKER=400` · `LEASE_TTL=60` · `LEASE_RENEW=20` · `POLL_WARM=180` · `POLL_COLD=900` · `TIER_HOT_ACTIVITY_H=24` · `TIER_SESSION_GRACE_MIN=30` · `TIER_COLD_AFTER_D=3` · `HOT_BUDGET_PER_WORKER=250` · `PER_HOST_CONN_CAP=50` · `BULK_CONCURRENCY=2` · `PARSE_CONCURRENCY=4` · `STREAM_PARSE_MB=8`. Remove `POLL_INTERVAL` (superseded by POLL_WARM/POLL_COLD).
+
+## 9. Migration steps (execute in order; suite green after each)
+
+1. **Roles + queues.** Add `WORKER_ROLE`; move job registration to the two named queues with the §6 routing/timeouts; compose gets `worker-sync` / `worker-jobs` services with `nofile` ulimits. Supervisor behavior untouched.
+2. **Extract `AccountRunner`.** Refactor the current per-account loops into a self-contained runner class (start/stop/teardown API), still launched by the old startup supervisor. Pure refactor, no behavior change.
+3. **Lease manager + claimer.** Implement §2 (worker_id, claim, Lua renewal, fencing, release, heartbeats, rebalance). Replace the startup supervisor with the claimer. Verify single-worker behavior is equivalent, then run two sync workers against the same accounts.
+4. **Event-driven lifecycle.** Implement §3 including the new `user.active` event in the API and the per-cycle state re-verification. Delete the startup-only discovery code path.
+5. **Tiers.** Implement §5 tier storage, evaluation, promotions/demotions, and the tier-shaped task sets in `AccountRunner`.
+6. **Memory bounds.** Implement §7 streaming threshold and semaphores.
+7. **Observability + graceful shutdown.** Structured log fields (`worker_id`, `account_id`, `tier`), heartbeat JSON per §2, SIGTERM handler: stop claimer → cancel runners → release leases → drain (target < 10 s).
+
+## 10. Acceptance criteria (all must pass; add as automated tests where marked)
+
+1. **[test]** Two sync workers, 20 mock accounts: at no point do two workers hold the same lease; each account has exactly one IDLE connection (assert via mock provider connection counter).
+2. **[test]** `kill -9` one of two sync workers: every orphaned account is re-owned and syncing within 90 s; message table has zero duplicate `(mailbox_id, uid)` rows afterwards.
+3. **[test]** Create an account via the API: its live loop starts within 30 s **without any worker restart**. Delete an account: all its connections closed and no further provider calls within 30 s (zombie regression test — mock provider records post-deletion calls, must be zero after the 30 s cutoff).
+4. **[test]** Publish `user.active`: all of that user's accounts reach `hot` (IDLE established on mock) within 5 s. After session grace expiry with no activity, they demote to `warm` on the next evaluation cycle.
+5. **[test]** Queue isolation: enqueue 10 bulk backfills + 20 interactive sends; all sends complete in < 5 s p95 while backfills run.
+6. **[test]** SIGTERM a sync worker: leases released and adopted by the peer within 15 s; no fencing violations logged.
+7. **[load, manual/CI-nightly]** 1000 simulated accounts (mock provider), 20 % with active sessions: steady-state persistent sockets ≤ hot count + 5 %; worker RSS stable over 1 h; oldest-sync-lag < `POLL_COLD` for every account.
+8. Backfilling a mailbox containing a 40 MB message never raises process RSS by more than ~`PARSE_CONCURRENCY × STREAM_PARSE_MB` above baseline.
+
+## 11. Out of scope for this task
+
+Cross-worker (Redis-based) per-host connection accounting, pgbouncer deployment, JMAP provider, multi-region operation, per-user rate plans. Note them in `docs/open-questions.md` if the need is observed.

webmail-client-spec_v2.md

@@ -0,0 +1,744 @@
+# Webmail Client — Implementation Specification (v1.3, agent-ready)
+
+This document is a complete, self-contained build specification. It is written to be handed to an AI coding agent. Read it fully before writing code. The companion document `webmail-client-concept.md` gives background; where they differ, **this spec wins**.
+
+---
+
+## 0. Instructions to the implementing agent
+
+1. **Build milestone by milestone** (§12), in order. Do not start a milestone before the previous one's acceptance criteria pass.
+2. **Do not add features beyond this spec.** Anything not specified here is out of scope; leave `TODO(spec)` comments instead of inventing behavior.
+3. **Definition of done** for every task: code + type hints + unit tests + passing `ruff check`, `ruff format --check`, `mypy --strict` (backend) / `tsc --noEmit`, `eslint` (frontend) + updated docs in `docs/` if an interface changed.
+4. **Stability over novelty:** use exactly the pinned dependencies in §3. Do not swap libraries or add dependencies without a `# DEPENDENCY:` comment explaining why, and never for something the stdlib does adequately.
+5. **All interfaces in §5 are frozen contracts.** Implementations may evolve; signatures may not (without bumping the version comment at the top of the interface file).
+6. **Security rules in §10 are hard requirements**, not suggestions.
+7. Every module publishing or consuming events must use the event names in §8 verbatim.
+8. Commit in small, working increments; every commit must leave the test suite green.
+
+## 1. Architecture invariants (never violate)
+
+- **I-1 Sync-first:** the web UI and REST API never open IMAP connections. Only sync workers talk to mail servers. The API reads exclusively from Postgres/Meilisearch.
+- **I-2 Independent identity:** login uses app accounts (`localpart@our-web-client.example`) with Argon2id hashes. Mailbox credentials are payload data, encrypted at rest, decrypted only inside sync workers.
+- **I-3 One index, many views:** unified inbox, per-account views, virtual mailboxes, and search results are all parameterized queries over the same index. No duplicated storage per view.
+- **I-4 Threading is ours:** JWZ in our backend (§7). Never use the IMAP `THREAD` extension.
+- **I-5 Direction-agnostic thread UI:** thread ordering (oldest-first/newest-first) is a data-ordering concern. One layout, one composer-anchor slot (§9.4).
+- **I-6 Plugins extend, core stays closed:** new features beyond this spec must be expressible through the plugin surfaces (§11).
+- **I-7 Multi-user isolation:** every query (SQL and search) carries a mandatory `user_id` filter. No exceptions, including admin tooling.
+- **I-8 Mail HTML is hostile input:** sanitize with nh3 server-side at index time AND render in a sandboxed iframe. Both, always.
+
+## 2. Repository layout (monorepo)
+
+```
+webmail/
+├── backend/
+│   ├── pyproject.toml
+│   ├── app/
+│   │   ├── main.py                  # FastAPI app factory
+│   │   ├── config.py                # pydantic-settings; all config via env
+│   │   ├── db.py                    # SQLAlchemy async engine/session
+│   │   ├── events.py                # EventBus (Redis pub/sub + arq enqueue)
+│   │   ├── identity/                # §6.1 auth, sessions, handles, TOTP
+│   │   ├── accounts/                # §6.2 wizard, autodiscovery, credential vault
+│   │   ├── providers/
+│   │   │   ├── base.py              # MailProvider interface (§5.1)
+│   │   │   └── imap_smtp/           # first implementation
+│   │   ├── sync/                    # §6.3 backfill, IDLE loops, reconciliation
+│   │   ├── indexer/                 # §6.4 MIME parsing → DB rows + blobs
+│   │   ├── threading_/              # §7 JWZ (trailing _ avoids stdlib clash)
+│   │   ├── search/
+│   │   │   ├── base.py              # SearchBackend interface (§5.2)
+│   │   │   ├── queryparser.py       # operator grammar (§8.2)
+│   │   │   ├── pg.py                # Postgres FTS backend (M0–M1)
+│   │   │   └── meili.py             # Meilisearch backend (M2)
+│   │   ├── outbox/                  # §6.5 drafts, MIME assembly, send queue, undo
+│   │   ├── api/                     # routers, one file per §9.1 group
+│   │   ├── ws.py                    # WebSocket hub (§9.2)
+│   │   └── plugins/                 # §11 loader + plugin packages
+│   ├── workers/worker.py            # arq worker entrypoint
+│   ├── migrations/                  # alembic
+│   └── tests/
+├── frontend/
+│   ├── package.json
+│   └── src/
+│       ├── api/                     # generated client from OpenAPI + WS layer
+│       ├── routes/                  # §9.3
+│       ├── components/
+│       │   ├── thread/ThreadView.tsx    # §9.4 direction-agnostic
+│       │   ├── thread/Composer.tsx
+│       │   ├── list/MessageList.tsx     # virtualized
+│       │   └── slots/               # plugin UI slots (§11)
+│       └── lib/
+├── docker/
+│   ├── docker-compose.yml           # app, worker, postgres:16, redis:7, meilisearch:v1.8, dovecot (test)
+│   └── docker-compose.ci.yml        # + greenmail for integration tests
+└── docs/
+```
+
+## 3. Pinned stack
+
+**Backend (Python 3.11):** fastapi ~0.115, uvicorn ~0.30, sqlalchemy ~2.0 (async, asyncpg), alembic ~1.13, pydantic ~2.8, pydantic-settings, arq ~0.26, redis ~5.0, aioimaplib ~1.1, imapclient ~3.0 (utility/fallback), aiosmtplib ~3.0, argon2-cffi ~23.1, pyotp ~2.9, dnspython ~2.6, httpx ~0.27, nh3 ~0.2, cryptography ~43.0, mail-parser ~4.0 (helpers only; primary parsing via stdlib `email.message.EmailMessage` with `policy.default`).
+
+**Frontend:** TypeScript ~5.5, React ~18.3, Vite ~5, TanStack Query ~5, TanStack Virtual ~3, Radix UI, Tailwind ~3.4, zod (WS payload validation). API client generated from OpenAPI (`openapi-typescript`).
+
+**Infra:** PostgreSQL 16 (with `citext` extension), Redis 7, Meilisearch v1.8 (M2+), MinIO or plain filesystem for the blob store (config-switchable; filesystem default).
+
+## 4. Database schema (Postgres 16, authoritative DDL)
+
+All PKs are `uuid DEFAULT gen_random_uuid()`. All tables carrying user data include `user_id` with an index. Timestamps are `timestamptz`.
+
+```sql
+CREATE EXTENSION IF NOT EXISTS citext;
+
+CREATE TABLE users (
+  id            uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  handle        citext UNIQUE NOT NULL,        -- full form: localpart@<APP_DOMAIN>
+  password_hash text   NOT NULL,               -- argon2id
+  totp_secret   bytea,                         -- encrypted; NULL = 2FA off
+  recovery_email citext,
+  settings      jsonb  NOT NULL DEFAULT '{}',  -- incl. {"thread_order":"asc"|"desc"}
+  created_at    timestamptz NOT NULL DEFAULT now()
+);
+-- Handle rules (enforce in code, mail-grade from day one):
+--   localpart: 3–30 chars, [a-z0-9._-], no leading/trailing/double dot,
+--   case-insensitive, reserved list (postmaster, abuse, admin, root, noreply, ...).
+
+CREATE TABLE accounts (
+  id             uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id        uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  email          citext NOT NULL,
+  display_name   text NOT NULL DEFAULT '',
+  color          text NOT NULL DEFAULT '#888888',
+  imap_config    jsonb NOT NULL,   -- {host,port,tls:"ssl"|"starttls",auth:"password"|"oauth2"}
+  smtp_config    jsonb NOT NULL,   -- same shape
+  secret_ct      bytea NOT NULL,   -- envelope-encrypted password or OAuth token bundle (§10.2)
+  discovery_src  text  NOT NULL,   -- preset|autoconfig|ispdb|srv|probe|manual
+  state          text  NOT NULL DEFAULT 'backfilling',  -- backfilling|live|error|disabled
+  state_detail   text  NOT NULL DEFAULT '',
+  last_sync_at   timestamptz,
+  created_at     timestamptz NOT NULL DEFAULT now(),
+  UNIQUE (user_id, email)
+);
+
+CREATE TABLE mailboxes (
+  id             uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  account_id     uuid NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
+  user_id        uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  path           text NOT NULL,                 -- IMAP mailbox name, UTF-7 decoded
+  special_use    text,                          -- inbox|sent|drafts|trash|junk|archive|NULL
+  uidvalidity    bigint NOT NULL DEFAULT 0,
+  highestmodseq  bigint,                        -- NULL if no CONDSTORE
+  last_seen_uid  bigint NOT NULL DEFAULT 0,
+  UNIQUE (account_id, path)
+);
+
+CREATE TABLE threads (
+  id              uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id         uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  subject_norm    text NOT NULL,                -- normalized subject (§7.3)
+  snippet         text NOT NULL DEFAULT '',
+  participants    jsonb NOT NULL DEFAULT '[]',  -- [{name,email}], deduped, ordered by recency
+  account_ids     uuid[] NOT NULL DEFAULT '{}',
+  message_count   int NOT NULL DEFAULT 0,
+  unread_count    int NOT NULL DEFAULT 0,
+  has_attachments boolean NOT NULL DEFAULT false,
+  last_message_at timestamptz NOT NULL
+);
+CREATE INDEX ON threads (user_id, last_message_at DESC);
+
+CREATE TABLE messages (
+  id              uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id         uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  account_id      uuid NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
+  mailbox_id      uuid NOT NULL REFERENCES mailboxes(id) ON DELETE CASCADE,
+  uid             bigint NOT NULL,              -- IMAP UID within mailbox
+  thread_id       uuid REFERENCES threads(id),
+  message_id_hdr  text,                         -- Message-ID, angle brackets stripped
+  in_reply_to     text,
+  references_hdr  text[] NOT NULL DEFAULT '{}',
+  subject         text NOT NULL DEFAULT '',
+  from_addr       jsonb NOT NULL DEFAULT '[]',
+  to_addr         jsonb NOT NULL DEFAULT '[]',
+  cc_addr         jsonb NOT NULL DEFAULT '[]',
+  date_hdr        timestamptz,
+  internal_date   timestamptz NOT NULL,
+  flags           text[] NOT NULL DEFAULT '{}', -- \Seen,\Flagged,\Answered,\Draft,\Deleted
+  size_bytes      int NOT NULL DEFAULT 0,
+  has_attachments boolean NOT NULL DEFAULT false,
+  snippet         text NOT NULL DEFAULT '',     -- first ~160 chars of text body
+  blob_sha256     text NOT NULL,                -- raw RFC822 in blob store (§6.4)
+  body_indexed    tsvector,                     -- Postgres FTS (M0–M1 SearchBackend)
+  UNIQUE (mailbox_id, uid)
+);
+CREATE INDEX ON messages (user_id, thread_id);
+CREATE INDEX ON messages (account_id, message_id_hdr);
+CREATE INDEX ON messages USING gin (body_indexed);
+
+CREATE TABLE body_parts (
+  id          uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  message_id  uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
+  kind        text NOT NULL,        -- text|html_sanitized
+  content     text NOT NULL
+);
+
+CREATE TABLE attachments (
+  id          uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  message_id  uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
+  user_id     uuid NOT NULL,
+  filename    text NOT NULL,
+  mime_type   text NOT NULL,
+  size_bytes  int  NOT NULL,
+  content_id  text,                 -- for inline cid: images
+  blob_sha256 text NOT NULL
+);
+
+CREATE TABLE drafts (
+  id          uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id     uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  account_id  uuid NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
+  thread_id   uuid REFERENCES threads(id),      -- NULL = new mail
+  reply_to_message_id uuid REFERENCES messages(id),
+  kind        text NOT NULL DEFAULT 'reply',    -- reply|reply_all|forward|new
+  payload     jsonb NOT NULL,       -- {to,cc,bcc,subject,html,text,attachment_ids}
+  updated_at  timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE TABLE outbox (
+  id          uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id     uuid NOT NULL,
+  draft_id    uuid NOT NULL,
+  state       text NOT NULL DEFAULT 'undo_window',  -- undo_window|sending|sent|failed|cancelled
+  send_after  timestamptz NOT NULL,                 -- now() + undo window
+  error       text
+);
+
+CREATE TABLE saved_searches (
+  id       uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id  uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  name     text NOT NULL,
+  query    text NOT NULL,           -- raw operator-syntax string (§8.2)
+  pinned   boolean NOT NULL DEFAULT true,
+  UNIQUE (user_id, name)
+);
+
+-- Core recipient autocomplete (Gmail-style, derived from mail history).
+-- The Contacts plugin (Appendix B) ENRICHES suggestions; this table works without it.
+CREATE TABLE known_recipients (
+  user_id      uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  email        citext NOT NULL,
+  name         text NOT NULL DEFAULT '',      -- last seen display name
+  times_used   int NOT NULL DEFAULT 0,        -- weighted: sent-to counts 3, received-from counts 1
+  last_used_at timestamptz NOT NULL,
+  PRIMARY KEY (user_id, email)
+);
+CREATE INDEX ON known_recipients (user_id, email text_pattern_ops);
+```
+
+**Blob store:** raw RFC822 messages and attachment payloads are stored content-addressed at `${BLOB_ROOT}/{user_id}/{sha256[:2]}/{sha256}` (filesystem) or the equivalent object key (MinIO). Postgres holds metadata only. Deleting a user removes their blob subtree.
+
+**Sessions** live in Redis (`sess:{token}` → JSON `{user_id, created, ua}`, TTL 30 days sliding), not Postgres.
+
+## 5. Frozen core interfaces (Python)
+
+### 5.1 `providers/base.py` — MailProvider
+
+```python
+# INTERFACE VERSION: 1
+from dataclasses import dataclass
+from typing import AsyncIterator, Protocol
+from datetime import datetime
+
+@dataclass(frozen=True)
+class RemoteMailbox:
+    path: str
+    special_use: str | None      # "inbox"|"sent"|"drafts"|"trash"|"junk"|"archive"|None
+    uidvalidity: int
+
+@dataclass(frozen=True)
+class RemoteMessage:
+    uid: int
+    raw: bytes                   # full RFC822
+    flags: frozenset[str]
+    internal_date: datetime
+
+@dataclass(frozen=True)
+class FlagChange:
+    uid: int
+    flags: frozenset[str]
+
+@dataclass(frozen=True)
+class MailboxDelta:
+    new: list[RemoteMessage]
+    flag_changes: list[FlagChange]
+    expunged_uids: list[int]
+    new_uidvalidity: int | None  # non-None => UIDVALIDITY changed, caller must full-resync
+    new_highestmodseq: int | None
+
+class MailProvider(Protocol):
+    async def verify(self) -> None: ...
+        # Raises ProviderAuthError / ProviderConnectError with a human-readable .detail
+
+    async def list_mailboxes(self) -> list[RemoteMailbox]: ...
+
+    def backfill(self, mailbox: str, *, newest_first: bool = True,
+                 batch_size: int = 200) -> AsyncIterator[list[RemoteMessage]]: ...
+        # Yields batches until the mailbox is exhausted. Headers+body in one pass.
+
+    async def poll(self, mailbox: str, *, last_seen_uid: int,
+                   highestmodseq: int | None) -> MailboxDelta: ...
+        # One reconciliation pass. Uses QRESYNC/CONDSTORE when highestmodseq is not None,
+        # otherwise UID SEARCH + FETCH FLAGS diff.
+
+    async def idle(self, mailbox: str) -> None: ...
+        # Blocks until the server signals change or 25 min elapse (re-issue IDLE). May raise
+        # ProviderNotSupported => sync loop falls back to poll() every POLL_INTERVAL (default 120 s).
+
+    async def push_flags(self, mailbox: str, uid: int,
+                         add: frozenset[str], remove: frozenset[str]) -> None: ...
+    async def move(self, uid: int, src: str, dst: str) -> int: ...   # returns new UID
+    async def append(self, mailbox: str, raw: bytes,
+                     flags: frozenset[str]) -> int: ...               # Sent/Drafts upload
+    async def send(self, raw: bytes, envelope_from: str,
+                   envelope_to: list[str]) -> None: ...               # SMTP submission
+```
+
+`ImapSmtpProvider` is the only M0–M2 implementation. A future `NativeProvider` (hosted mailboxes) or `JmapProvider` implements the same protocol — nothing above this interface may import IMAP libraries.
+
+### 5.2 `search/base.py` — SearchBackend
+
+```python
+# INTERFACE VERSION: 1
+@dataclass(frozen=True)
+class SearchQuery:
+    text: str | None                      # free-text remainder after operator extraction
+    filters: dict[str, object]            # parsed operators, keys per §8.2
+    user_id: UUID                         # ALWAYS set (invariant I-7)
+
+@dataclass(frozen=True)
+class SearchHit:
+    message_id: UUID
+    thread_id: UUID
+    score: float
+    highlights: dict[str, str]            # field -> HTML-escaped snippet with <mark>
+
+class SearchBackend(Protocol):
+    async def index(self, doc: SearchDoc) -> None: ...
+    async def delete(self, message_id: UUID) -> None: ...
+    async def delete_user(self, user_id: UUID) -> None: ...
+    async def query(self, q: SearchQuery, *, limit: int, cursor: str | None
+                    ) -> tuple[list[SearchHit], str | None]: ...
+```
+
+`SearchDoc` fields: `message_id, thread_id, user_id, account_id, mailbox_special_use, from_ (name+email), to, cc, subject, body_text, attachment_names[], has_attachments, is_unread, is_starred, date (unix), size_bytes`. Both backends (pg.py, meili.py) index the same doc. Meilisearch: one index `messages`, filterable = every non-body field incl. `user_id`, searchable = `subject, body_text, from_, to, cc, attachment_names`, sortable = `date`; ranking rules default + `date:desc` tiebreak.
+
+### 5.3 `events.py` — EventBus
+
+```python
+# INTERFACE VERSION: 1
+class EventBus(Protocol):
+    async def publish(self, event: str, payload: dict) -> None:
+        # Fan-out to Redis pub/sub channel "evt:{event}" AND enqueue arq job
+        # "on_{event}" for durable consumers (indexer, plugins).
+    def subscribe(self, event: str, handler: Handler) -> None: ...
+```
+
+Payloads always include `user_id`. Event names are listed in §8.3 and are the only ones allowed.
+
+## 6. Backend module specifications
+
+### 6.1 Identity (`app/identity/`)
+- Registration: validate localpart per §4 rules against reserved list in `identity/reserved.py`; store full handle `localpart@{APP_DOMAIN}` (env `APP_DOMAIN`).
+- Login: constant-time verify (argon2id, `time_cost=3, memory_cost=64MiB, parallelism=4`); if `totp_secret` set, require valid TOTP (±1 window). Rate limit: 10 attempts / 15 min per handle AND per IP (Redis counters); lockout returns 429 with `Retry-After`.
+- Session cookie: `__Host-session`, httpOnly, Secure, SameSite=Lax. CSRF: double-submit token header `X-CSRF` for all mutating requests.
+- TOTP setup: `POST /auth/totp/setup` returns provisioning URI + QR payload; enable requires one valid code; secret stored envelope-encrypted.
+- Password reset: only if `recovery_email` verified; single-use token, 30 min TTL, Redis-backed.
+
+### 6.2 Accounts & autodiscovery (`app/accounts/`)
+`discover(email) -> list[CandidateConfig]` runs the chain, **returning on first success** but recording every step's outcome in the response (`steps: [{method, status, detail}]`) for wizard UX:
+
+1. **Presets** — `accounts/presets.yaml` (config, not code): match by domain; ships with gmail.com, googlemail.com, outlook.com/hotmail/live, office365, yahoo, icloud/me.com, gmx, wp.pl, onet.pl, o2.pl, interia.pl. Each entry: imap/smtp host+port+tls, auth kind, oauth provider id if applicable.
+2. **Autoconfig** — GET `https://autoconfig.{domain}/mail/config-v1.1.xml?emailaddress={email}` then `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml`; parse Thunderbird XML (incomingServer type=imap, outgoingServer type=smtp; `%EMAILADDRESS%`/`%EMAILLOCALPART%` substitution).
+3. **ISPDB** — GET `https://autoconfig.thunderbird.net/v1.1/{domain}`; same XML format.
+4. **DNS SRV (RFC 6186)** — `_imaps._tcp.{domain}` (SSL 993), `_imap._tcp` (STARTTLS 143), `_submissions._tcp` (465), `_submission._tcp` (587); lowest priority wins.
+5. **Probe** — try `{imap,mail}.{domain}` × ports 993(ssl)/143(starttls) and `{smtp,mail}.{domain}` × 465(ssl)/587(starttls); TCP connect + TLS handshake + IMAP/SMTP greeting, 5 s timeout each, all concurrently.
+
+All HTTP/DNS lookups: 5 s timeout, no redirects off-domain, HTTPS only. `test(config, credentials)`: full IMAP LOGIN + `LIST`, and SMTP EHLO+AUTH (no message sent); errors mapped to precise user-facing strings (`auth_failed`, `tls_failed`, `connect_timeout`, `unknown_host`). On save: encrypt credentials (§10.2), create account row (`state=backfilling`), enqueue `sync.bootstrap_account`.
+
+### 6.3 Sync engine v2 (`app/sync/`) — sharded, lease-owned, tiered
+(Authoritative detail: `sync-engine-v2-refactor.md`; this section summarizes the binding rules.)
+
+- **Two worker roles** (`WORKER_ROLE`): `jobs` = stateless arq consumer, N instances; `sync` = lease claimer + per-account runners, 1..N instances. Compose services `worker-jobs` / `worker-sync`, `nofile` ulimit 65536.
+- **Lease ownership:** exactly one sync worker owns each live account. Redis SET `sync:live` (API-maintained, DB-reconciled every 300 s); lease `sync:lease:{account_id}` claimed `SET NX EX LEASE_TTL`, renewed every `LEASE_RENEW` via owner-checked Lua; **fencing** — on renewal failure, cancel that account's runner before any further provider/DB op. Worker heartbeats in `sync:workers:{id}`; rebalance releases excess above `target*1.2+5`. Crash failover = lease expiry + reclaim (≤ ~90 s), no coordinator.
+- **Event-driven lifecycle:** `account.state_changed` adds/removes ids from `sync:live` at runtime — accounts go live ≤ 30 s after creation and stop ≤ 30 s after deletion with no restarts; runners re-verify `state='live'` each cycle (zombie-proof). New core event **`user.active`** (API publishes on login/WS connect) drives tier promotion.
+- **Bootstrap (unchanged from v1, now a `bulk` job):** `list_mailboxes()` → upsert `mailboxes` (SPECIAL-USE mapping with name heuristics — Sent/Wysłane, Trash/Kosz, etc. in `sync/folder_names.py`); backfill newest-first per mailbox (INBOX first, then sent, then rest); `sync.progress` published per batch; on completion set `state='live'` → the API adds the id to `sync:live` and a sync worker claims it.
+- **AccountRunner per tier:** *hot* = persistent INBOX IDLE + transient CONDSTORE poll of special folders every `POLL_WARM`; *warm* = transient poll of INBOX+specials every `POLL_WARM`; *cold* = every `POLL_COLD`. Promotion to hot on `user.active` (≤ 5 s) or new mail within `TIER_HOT_ACTIVITY_H`; demotion by inactivity; `HOT_BUDGET_PER_WORKER` degrades least-active non-session accounts. Delta application, UIDVALIDITY wipe-and-rebackfill, expunge soft-handling, and write-then-enqueue outbound flags are unchanged from v1.
+- **Connection budgets:** ≤ 2 sync-side connections per account (4 total incl. jobs, shared semaphore); process-local per-destination-host semaphore cap `PER_HOST_CONN_CAP`.
+- **Queues:** `interactive` (send/flags/move/draft jobs, timeout 120 s, concurrency 20) and `bulk` (backfills, timeout 3600 s, concurrency `BULK_CONCURRENCY`) — bulk work can never delay a send.
+- **Memory bounds:** parse semaphore `PARSE_CONCURRENCY`; raw messages > `STREAM_PARSE_MB` stream to blob and parse from file — peak RAM ≈ `PARSE_CONCURRENCY × STREAM_PARSE_MB` per process regardless of mailbox size.
+- **Shutdown:** SIGTERM → stop claimer → cancel runners → release leases → drain in < 10 s.
+
+### 6.4 Indexer (`app/indexer/`)
+Input: `RemoteMessage`. Steps: store raw blob (sha256, dedupe) → parse with `email` stdlib (`policy.default`) → extract: headers (decode RFC2047), `references_hdr` = parsed `References` + `In-Reply-To` message-ids in order; best text body and best HTML body (multipart/alternative preference: html for display, text for indexing; if only HTML, derive text via nh3 strip); sanitize HTML with nh3 (allowlist: structural tags, inline styles filtered, `cid:` images rewritten to `/api/messages/{id}/cid/{content_id}`, remote `src` rewritten to blocked-placeholder attributes for FR-23) → write `messages`, `body_parts` (kinds `text` and `html_sanitized`), `attachments` → hand to Threader (§7) → `bus.publish("message.new", ...)` (search indexing + thread denormalization are consumers). For non-junk received messages, upsert the `From` address into `known_recipients` (weight 1).
+
+### 6.5 Outbox (`app/outbox/`)
+- Drafts: autosave endpoint upserts `drafts` (client debounces 2 s); a background job mirrors drafts to the account's Drafts folder via `append()` at most every 60 s per draft (replace previous UID).
+- Send: `POST /send {draft_id}` → build MIME (html + auto-generated text alternative via html-to-text; `In-Reply-To`/`References` from `reply_to_message_id`; attachments from blob store) → create `outbox` row with `send_after = now()+UNDO_SECONDS` (env, default 10) → schedule arq job at `send_after`. Undo endpoint flips state to `cancelled` if still in window. The job: SMTP `send()`, then `append()` to Sent (skip for Gmail preset — Gmail self-appends), mark `sent`, publish `message.sent`, and upsert every To/Cc/Bcc address into `known_recipients` (weight 3). Failure → `failed` + WS notification, draft preserved.
+
+## 7. Threading specification (JWZ, `app/threading_/`)
+
+### 7.1 Normalization
+- `message_id_hdr`, `in_reply_to`, each `References` entry: strip angle brackets and whitespace, lowercase the domain part only. Discard obviously invalid ids (no `@`).
+- Missing/duplicate `Message-ID`: synthesize `sha256(raw)[:32]@synthetic.local` and mark it non-authoritative (never used to join other messages to it).
+
+### 7.2 Incremental algorithm (per new message)
+1. Candidate set = all normalized ids in `references_hdr` + `in_reply_to` + own `message_id_hdr`.
+2. `SELECT DISTINCT thread_id FROM messages WHERE user_id=$1 AND message_id_hdr = ANY($2) AND thread_id IS NOT NULL`.
+3. 0 threads → subject fallback (§7.3); still none → create new thread.
+4. 1 thread → attach.
+5. ≥2 threads → **merge**: keep the thread with the oldest `last_message_at` as canonical, repoint messages of the others, delete emptied thread rows, publish `thread.updated` for all involved.
+6. Recompute denormalized thread fields (count, unread, participants, snippet = newest message's snippet, has_attachments, account_ids, last_message_at) in one UPDATE.
+
+Threads are cross-account by design: the candidate lookup is scoped by `user_id`, not `account_id` (invariant I-3). Sent-folder messages join threads exactly like received ones (FR-15).
+
+### 7.3 Subject fallback
+Only when reference joining fails: normalize subject (strip repeated `Re:/Fw:/Fwd:/Odp:/PD:` prefixes case-insensitively, collapse whitespace); join to a thread with the same `subject_norm` that has activity within 7 days AND shares ≥1 participant email; never join two messages that both lack references AND have empty subjects.
+
+### 7.4 Test vectors (must pass in `tests/threading/`)
+Linear chain; chain arriving out of order; reply missing `In-Reply-To` but carrying `References`; thread merge on late-arriving parent; broken client (subject-only) joins; distinct-topic same-subject ("hi") does NOT join; cross-account merge (message in account A references message in account B); duplicate Message-ID; synthetic-ID isolation.
+
+## 8. Search specification
+
+### 8.1 Pipeline
+Consumer of `message.new` / `message.changed` / `message.deleted` → builds `SearchDoc` → active `SearchBackend` (env `SEARCH_BACKEND=pg|meili`). Backfill indexes inline with sync (per batch), never as a separate pass the user waits on (FR-30).
+
+### 8.2 Query grammar (`search/queryparser.py`, engine-independent)
+Tokenization: whitespace-split, respecting `"quoted phrases"`. A token matching `key:value` (or `-key:value`) with a known key becomes a filter; everything else joins free text. `-` prefix negates a term or filter. `OR` (uppercase) groups adjacent terms/filters into a disjunction; default combinator is AND.
+
+| Operator | Filter semantics |
+|---|---|
+| `from:` `to:` `cc:` | substring match on name or email of that field |
+| `subject:` | match constrained to subject |
+| `account:` | account by `display_name` or email substring |
+| `in:` | `inbox\|sent\|drafts\|trash\|junk\|archive` (special_use) or folder path |
+| `is:` | `unread\|read\|starred\|answered` |
+| `has:attachment` | boolean |
+| `filename:` | attachment name substring |
+| `before:` `after:` | `YYYY-MM-DD` |
+| `newer_than:` `older_than:` | `Nd\|Nw\|Nm\|Ny` |
+| `larger:` `smaller:` | `NK\|NM` bytes |
+
+Parser output = `SearchQuery` (§5.2). Unknown `key:` tokens are treated as free text (forward compatibility). Unit tests must cover every operator, negation, quoting, `OR`, and garbage input (parser never raises).
+
+### 8.3 Event catalog (complete)
+`message.new`, `message.changed`, `message.deleted`, `message.sent`, `flags.changed`, `thread.updated`, `sync.progress`, `account.state_changed`, `draft.saved`, `outbox.state_changed`, `user.active` (published by the API on login and WS connect; consumed by sync workers for tier promotion — not forwarded to browsers). Payloads: `{user_id, ...ids, minimal-diff fields}` — documented in `docs/events.md` (agent: generate it from code).
+
+Plugins may additionally publish events namespaced `plugin.{name}.*` (e.g. `plugin.calendar.event_updated`); core code never subscribes to plugin events, but the WS hub forwards them to the owning user like any other event.
+
+## 9. API, realtime, frontend
+
+### 9.1 REST API (prefix `/api`, session-cookie auth, JSON; errors: `{error:{code,message}}`)
+
+**Auth:** `POST /auth/register` · `POST /auth/login` · `POST /auth/logout` · `GET /auth/me` · `GET /auth/sessions` · `DELETE /auth/sessions/{id}` · `POST /auth/totp/setup` · `POST /auth/totp/enable` · `POST /auth/password/reset-request` · `POST /auth/password/reset`
+
+**Accounts:** `POST /accounts/discover {email}` → `{candidates:[{imap,smtp,source}], steps:[...]}` · `POST /accounts/test {config, credentials}` → `{ok}|{error:{code,detail}}` · `POST /accounts` · `GET /accounts` (incl. `state`, sync progress) · `PATCH /accounts/{id}` · `DELETE /accounts/{id}` (cascades rows, blobs, search docs)
+
+**Mail:** `GET /threads?view=unified|account:{id}|mailbox:{id}|search&q=&cursor=&limit=50` → thread summaries (denormalized fields only; cursor = `last_message_at,id` keyset) · `GET /threads/{id}?sort=asc|desc` → ordered full messages (bodies of the N newest expanded per client hint `expand=`; older = envelope only) · `POST /threads/{id}/flags {add,remove}` (whole thread) · `POST /messages/{id}/flags` · `POST /messages/{id}/move {mailbox_id}` · `GET /messages/{id}/html` (sanitized, for iframe `srcdoc`) · `GET /messages/{id}/cid/{content_id}` · `GET /attachments/{id}` (Content-Disposition attachment; images/PDF also inline for preview) · `POST /messages/{id}/allow-remote-images` (per-sender allowlist in user settings)
+
+**Compose:** `POST /drafts` · `PUT /drafts/{id}` · `DELETE /drafts/{id}` · `POST /send {draft_id}` → `{outbox_id, undo_until}` · `POST /send/{outbox_id}/undo` · `GET /recipients/suggest?q=` → top 10 from `known_recipients` (prefix match on email or name, ranked by `times_used` decayed by recency); the composer merges these with any plugin capability providers (§11.3)
+
+**Search:** `GET /search?q=&cursor=` → `{threads:[...], hits_by_thread, highlights}` (grouped by thread, FR-28; `&group=message` for flat) · `GET/POST/DELETE /saved-searches`
+
+**Settings:** `GET/PATCH /settings` (incl. `thread_order: "asc"|"desc"`, `tz` IANA timezone — default from browser at registration, undo window, per-sender image allowlist)
+
+### 9.2 WebSocket `/api/ws`
+Auth via session cookie. Server → client frames: `{event, payload}` for every §8.3 event belonging to the user (that's the whole contract — the UI refetches via TanStack Query invalidation keyed on ids in the payload). Client → server: `{type:"ping"}` only. Zod-validate all frames.
+
+### 9.3 Frontend routes
+`/login`, `/register`, `/mail` (unified inbox), `/mail/a/:accountId`, `/mail/a/:accountId/f/:mailboxId`, `/mail/t/:threadId` (thread pane; on desktop rendered beside the list), `/mail/search?q=`, `/settings/{profile|security|accounts|appearance}`; `/settings/accounts/new` = wizard.
+
+**Account wizard flow:** step 1 email (+ password field revealed unless preset says oauth) → calls `discover` while typing pauses; step 2 shows resolved config with source badge ("found via ISPDB") and a Test button (auto-run); failure expands the manual form pre-filled with the best guess and the precise error; step 3 = created, live backfill progress bar (WS `sync.progress`) with "start reading now" (list fills as backfill proceeds).
+
+### 9.4 ThreadView contract (invariant I-5)
+```
+ThreadView(props: { messages: Message[] /* ASC always */, order: 'asc'|'desc',
+                    composerState, ... })
+```
+- Rendering order = `order==='asc' ? messages : [...messages].reverse()`. No other branching on `order` is allowed in layout code.
+- **Composer anchor** = a slot component rendered adjacent to the newest message: after it when `asc` (bottom of thread), directly after it when `desc` (top of thread, above older messages). Same component, one placement rule: "composer is always attached to the newest message."
+- Initial scroll: to the newest message (bottom for asc, top for desc). Collapse: all read messages except the newest are collapsed; "Expand all" toggles. Unread messages are never collapsed.
+- Keyboard: `j`/`k` move in *visual* order; `o`/`Enter` expand; `r` reply, `a` reply-all, `f` forward (open composer at anchor); `e` archive; `/` focus search; `gi` go to unified inbox; `esc` close composer (draft persists). Toolbar has the per-thread order toggle; it updates local state only, and long-presses/menu offers "make default" (PATCH /settings).
+- Message HTML renders in `<iframe sandbox="" srcdoc={...}>` sized via ResizeObserver postMessage shim; remote images show a "load images / always from this sender" bar when the sanitizer flagged any.
+
+## 10. Security requirements (hard)
+
+1. Argon2id params per §6.1; no other password hashing anywhere.
+2. **Credential vault:** mailbox secrets encrypted with AES-256-GCM under per-account data keys, wrapped by a master key from env `MASTER_KEY` (32 B, base64) — `cryptography` library, key id embedded for future rotation. Decryption code path exists only in `workers/` and `accounts/test`. Secrets never appear in logs, API responses, or exceptions (custom repr).
+3. CSRF double-submit on all mutating routes; `__Host-` cookie prefix; CSP: `default-src 'self'; frame-src 'self'; img-src 'self' data:` (remote mail images are proxied? **No** — v1 loads allowed remote images inside the sandboxed iframe only, which has its own relaxed CSP via srcdoc; document this in code).
+4. nh3 sanitization at index time; re-sanitize on render if sanitizer version changed (store `sanitizer_v` on body_parts).
+5. Rate limits: login (§6.1), `POST /accounts/discover` 10/min/user, `POST /send` 30/h/user (env-tunable).
+6. All outbound autodiscovery HTTP: HTTPS only, 5 s timeout, response size cap 256 KB, no off-domain redirects (SSRF guard: resolve + reject private IP ranges).
+7. Every SQL query and search query includes `user_id` (enforced by repository-layer helpers; raw session use in API code is a review error).
+8. Uploads (attachments to drafts): 25 MB/file, 40 MB/message total, content-type sniffed server-side, stored to blob store, never executed/served with original content type unless in safe list (images/pdf/text) — others force `application/octet-stream` + attachment disposition.
+
+## 11. Plugin API (freeze in M3 — including the heavy-plugin surfaces below)
+
+A plugin = Python package `webmail_plugin_<name>` exposing:
+
+```python
+plugin = Plugin(
+    name, version,
+    hooks       = {event: handler},          # subscribe to §8.3 events
+    router      = APIRouter | None,          # mounted at /api/plugins/{name}
+    ui_manifest = UiManifest | None,         # slots + pages, see below
+    migrations  = "migrations/" | None,      # 11.1
+    jobs        = [JobSpec, ...] | None,     # 11.2
+)
+```
+
+Loader discovers plugins via entry points group `webmail.plugins`, forwards declared events, and serves `GET /api/plugins/ui-manifest` for the frontend loader. Plugins may not import from `app.*` internals other than `app.plugins.sdk` (which exposes: DB session factory scoped to plugin tables, EventBus publish under `plugin.{name}.*`, blob store, outbox draft/send helpers, settings accessor, HTTP client with the §10.6 SSRF guard built in).
+
+Every plugin gets the namespaced KV store (`plugin_kv(plugin, user_id, key)`) for light state. Heavy plugins additionally get:
+
+### 11.1 Plugin-owned tables
+A plugin may ship Alembic migrations in its own version branch (`alembic -n plugin_{name}`). All tables MUST be prefixed `plg_{name}_` and every user-data table MUST carry `user_id` (invariant I-7 applies to plugins). Core migrations never reference plugin tables and vice versa. Uninstalling a plugin offers `--drop-data` to run its downgrade chain.
+
+### 11.2 Plugin background jobs
+`JobSpec(func, schedule: cron_str | None, singleton: bool)` registers tasks on the shared arq worker: cron-scheduled (e.g. a sync loop trigger) or enqueued on demand from hooks/routes. `singleton=True` guarantees at most one running instance per (plugin, user) via a Redis lock. Plugin jobs run in the same worker process pool with the same retry semantics as core jobs and are visible in worker logs under `plg.{name}.*`.
+
+### 11.3 Frontend surfaces
+- **Slots** (lazy-loaded ESM injected into mail UI): `thread.toolbar`, `message.actions`, `sidebar.section`, `settings.page`.
+- **Pages** (`app.page`): a plugin may declare full views: `{route: "/calendar", nav: {icon, label, order}, bundle}`. The shell adds the nav entry, and the route (including sub-routes under it) lazy-loads the bundle into the **entire content area** — no mail list/thread components are rendered. Page bundles receive the SDK context (API client, WS event stream filtered to `plugin.{name}.*` + requested core events, design-system components) and are sandboxed to their route: importing mail-component internals is a review error. Removing the plugin removes the nav entry and route cleanly.
+- **Capabilities**: the manifest may declare named capabilities mapping to plugin endpoints, e.g. `capabilities: {"recipients.suggest": "/api/plugins/contacts/suggest"}`. Core UI features query all providers of a capability they support and merge results client-side (defined capabilities in v1: `recipients.suggest` — used by the composer To/Cc/Bcc fields and by any other plugin needing person-picking, such as calendar attendees). This keeps core backend code plugin-unaware.
+
+Acceptance test for the API: implement **filters** (rules on `message.new`: match → move/flag) and **labels** entirely as plugins using only slots + KV; then implement the **calendar** (Appendix A) using tables + jobs + a page — all with zero core diffs.
+
+## 12. Milestones with acceptance criteria
+
+**M0 — Architecture spike**
+Login/register (no 2FA), one hardcoded-config IMAP account, backfill+IDLE for INBOX, JWZ, read-only unified list + thread view with working asc/desc toggle and correct composer-anchor placeholder, Postgres FTS free-text search.
+✅ Accept: sync a real 10k-message Gmail account via app password; threads match Gmail's grouping on a 50-thread sample ≥ 90%; toggle order and the newest message + anchor swap ends correctly; search returns in < 500 ms.
+
+**M1 — MVP**
+Full identity (2FA, sessions UI, reset), account wizard with full discovery chain, multi-account + unified inbox with account chips, flags/move/archive both directions, inline reply + reply-all/forward + SMTP send + Sent append, recipient autocomplete from `known_recipients`, drafts autosave, sanitized HTML + blocked remote images, operator search (full §8.2 grammar) on PG FTS, WS live updates.
+✅ Accept: add gmail.com, outlook.com, and one autoconfig-only domain through the wizard typing only address+password; reply from a thread lands correctly threaded in the recipient's client; all §8.2 operators pass integration tests; `docker-compose up` from clean checkout works.
+
+**M2 — Hardening / v1.0**
+OAuth2 (Google + Microsoft, XOAUTH2 for IMAP/SMTP, server-side refresh), QRESYNC fast resync, drafts mirrored to server, undo-send, attachment upload/preview, Meilisearch backend + instant search-as-you-type UI + saved searches as virtual mailboxes, session revocation, backfill progress UX, folder-name i18n heuristics.
+✅ Accept: typo query (`invocie`) finds "invoice" mail < 200 ms on a 100k-message corpus; kill -9 the worker mid-backfill and restart resumes without duplicates (idempotency via `(mailbox_id, uid)` unique key); undo-send within window results in no SMTP traffic.
+
+**M3 — Plugin platform**
+Loader, full SDK per §11 **including** plugin migrations (11.1), plugin jobs (11.2), and `app.page` (11.3); filters + labels shipped as plugins; plugin docs.
+✅ Accept: both plugins install/uninstall cleanly with zero core diffs; a third trivial plugin ("Unsubscribe header button" on `message.actions`) can be written from `docs/plugins.md` alone; a skeleton page-plugin (route + nav entry + one plugin table + one cron job) round-trips install → run → uninstall `--drop-data` leaving no residue.
+
+**M4 — Calendar plugin (Appendix A)**
+Phases A1–A2: local calendars + full page views + iMIP invite handling; then A3: CalDAV client sync + ICS subscriptions.
+✅ Accept: criteria in Appendix A §A.7.
+
+**M5 — Contacts plugin (Appendix B)**
+Phases B1–B2: contact books, full page, groups, add-from-mail, `recipients.suggest` capability, vCard import/export; then B3: CardDAV client sync. May be built in parallel with M4 — both depend only on the M3 SDK.
+✅ Accept: criteria in Appendix B §B.7.
+
+**M6 (optional) — Hosted mailboxes**
+Inbound MTA (Stalwart) delivering to a `NativeProvider`; handle becomes receiving address. Specified separately when reached.
+
+## 13. Testing & CI
+
+- **Unit:** pytest + pytest-asyncio; threading vectors (§7.4); query parser (§8.2); autodiscovery resolver with `respx`-mocked HTTP + `dnspython` stub; MIME corpus in `tests/fixtures/mime/` (agent: include ≥ 25 nasty real-world-style fixtures — encoded headers incl. RFC2047 Polish diacritics, nested multiparts, missing Message-ID, malformed dates, HTML-only, inline cid images, 8-bit subjects, winmail.dat placeholder).
+- **Integration:** docker-compose.ci.yml with Greenmail (IMAP+SMTP) and Dovecot; full loop test: append message to Greenmail → appears in API < 10 s → reply via API → arrives in Greenmail SMTP sink with correct References.
+- **Frontend:** vitest + React Testing Library; ThreadView ordering/anchor tests for both directions; Playwright smoke: login → add Greenmail account via wizard → read → inline reply → search.
+- **CI gates:** ruff, mypy --strict, pytest, tsc, eslint, vitest, Playwright smoke; coverage floor 80% on `threading_`, `search/queryparser`, `accounts/autodiscovery`, `indexer`.
+
+## 14. Configuration (env, complete list)
+
+`APP_DOMAIN`, `DATABASE_URL`, `REDIS_URL`, `MASTER_KEY`, `BLOB_ROOT` or `S3_*`, `SEARCH_BACKEND=pg|meili`, `MEILI_URL`, `MEILI_KEY`, `UNDO_SECONDS=10`, `SESSION_TTL_DAYS=30`, `OAUTH_GOOGLE_CLIENT_ID/SECRET`, `OAUTH_MS_CLIENT_ID/SECRET`, `MAX_ATTACHMENT_MB=25`, `LOG_LEVEL`; sync engine v2 (§6.3): `WORKER_ROLE=jobs|sync`, `MAX_ACCOUNTS_PER_WORKER=400`, `LEASE_TTL=60`, `LEASE_RENEW=20`, `POLL_WARM=180`, `POLL_COLD=900`, `TIER_HOT_ACTIVITY_H=24`, `TIER_SESSION_GRACE_MIN=30`, `TIER_COLD_AFTER_D=3`, `HOT_BUDGET_PER_WORKER=250`, `PER_HOST_CONN_CAP=50`, `BULK_CONCURRENCY=2`, `PARSE_CONCURRENCY=4`, `STREAM_PARSE_MB=8`. All read once in `config.py` (pydantic-settings); no `os.environ` elsewhere.
+
+---
+
+## Appendix A — Calendar plugin (`webmail_plugin_calendar`)
+
+The calendar is the reference "heavy plugin": it lives in the same app and repo, uses the §11 SDK only, and must build with **zero core diffs**. It ships as a page (`/calendar`) beside Mail — one product, one login.
+
+### A.1 Scope by phase
+- **A1 — Local calendar:** create/edit/delete events, multiple calendars per user, month/week/day/agenda views, recurrence, reminders (WS notification), timezone-correct display.
+- **A2 — Invites over mail (iMIP):** detect `text/calendar` parts on `message.new`; render an invite card in the thread (`message.actions` + inline summary); Accept/Tentative/Decline sends an iTIP REPLY through the outbox; organizer flow sends REQUEST/CANCEL to attendees; updates (SEQUENCE bumps) modify the stored event.
+- **A3 — External sources:** `CalendarProvider` interface with **CalDAV client** (first implementation; covers iCloud, Fastmail, Nextcloud) and read-only **ICS URL subscriptions** (holidays, team feeds); sync loop as a plugin job.
+- **A4 (deferred, spec later):** Google/Microsoft API providers reusing M2 OAuth machinery; secret **ICS export feed** `GET /api/plugins/calendar/feed/{token}.ics`; CalDAV **server** endpoint for native phone sync — architecturally anticipated, not built now.
+
+### A.2 Dependencies
+`icalendar ~5.0` (parse/serialize RFC 5545), `python-dateutil` (RRULE expansion), `caldav ~1.3` (A3), `tzdata`. No other additions.
+
+### A.3 Data model (plugin migrations, prefix `plg_calendar_`)
+```sql
+CREATE TABLE plg_calendar_calendars (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  name text NOT NULL,
+  color text NOT NULL DEFAULT '#3b82f6',
+  kind text NOT NULL DEFAULT 'local',     -- local|caldav|ics_subscription
+  source_config jsonb NOT NULL DEFAULT '{}',  -- caldav url/collection, ics url
+  secret_ct bytea,                        -- caldav credentials, §10.2 vault scheme
+  sync_token text,                        -- CalDAV sync-token / ICS etag
+  read_only boolean NOT NULL DEFAULT false,
+  UNIQUE (user_id, name)
+);
+
+CREATE TABLE plg_calendar_events (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  calendar_id uuid NOT NULL REFERENCES plg_calendar_calendars(id) ON DELETE CASCADE,
+  ical_uid text NOT NULL,                 -- UID from iCalendar; join key for iMIP updates
+  sequence int NOT NULL DEFAULT 0,
+  summary text NOT NULL DEFAULT '',
+  description text NOT NULL DEFAULT '',
+  location text NOT NULL DEFAULT '',
+  dtstart timestamptz NOT NULL,
+  dtend timestamptz,
+  all_day boolean NOT NULL DEFAULT false,
+  tz text NOT NULL DEFAULT 'UTC',         -- original TZID; render in user tz, store UTC
+  rrule text,                             -- raw RRULE string, NULL = single
+  exdates timestamptz[] NOT NULL DEFAULT '{}',
+  recurrence_id timestamptz,              -- non-NULL = detached exception instance
+  organizer jsonb,                        -- {name,email}
+  attendees jsonb NOT NULL DEFAULT '[]',  -- [{name,email,partstat,role}]
+  status text NOT NULL DEFAULT 'confirmed',   -- confirmed|tentative|cancelled
+  my_partstat text,                       -- needs-action|accepted|declined|tentative
+  reminders int[] NOT NULL DEFAULT '{}',  -- minutes before start
+  origin_message_id uuid,                 -- link back to the invite mail thread
+  raw_ical text,                          -- round-trip fidelity for iMIP/CalDAV
+  UNIQUE (calendar_id, ical_uid, recurrence_id)
+);
+CREATE INDEX ON plg_calendar_events (user_id, dtstart);
+```
+
+**Recurrence rule:** store the RRULE string; expand instances at query time with `dateutil.rrule` over the requested view range only (no materialization table). Exceptions: `exdates` remove instances; a row with `recurrence_id` overrides one instance. Range queries must account for events starting before the window but recurring into it (expand from `dtstart`, cap iterations at 1000 per event).
+
+### A.4 CalendarProvider interface (inside the plugin, mirrors §5.1)
+```python
+# PLUGIN INTERFACE VERSION: 1
+class CalendarProvider(Protocol):
+    async def verify(self) -> None: ...
+    async def list_collections(self) -> list[RemoteCalendar]: ...
+    async def pull(self, collection: str, *, sync_token: str | None
+                   ) -> tuple[list[RemoteEvent], list[str], str]: ...
+        # returns (changed_events_as_ical, deleted_uids, new_sync_token)
+    async def push(self, collection: str, ical: str, *, uid: str,
+                   etag: str | None) -> str: ...        # returns new etag
+    async def delete(self, collection: str, uid: str, etag: str | None) -> None: ...
+```
+Implementations in A3: `CaldavProvider`, `IcsSubscriptionProvider` (pull-only, `push`/`delete` raise `ReadOnlyCalendar`). Sync loop = plugin cron job (`schedule="*/5 * * * *"`, `singleton=True` per user) doing pull → upsert by `(calendar_id, ical_uid, recurrence_id)` → local-dirty push with etag conflict handling (conflict ⇒ server wins, local copy saved as a duplicate marked `status='tentative'`, user notified).
+
+### A.5 iMIP flow (A2)
+- **Inbound:** hook on `message.new`; if a `text/calendar` part exists, parse METHOD:
+  `REQUEST` → upsert event (match `ical_uid`; apply only if `SEQUENCE` ≥ stored) into the user's default calendar with `my_partstat='needs-action'`, set `origin_message_id`; `CANCEL` → `status='cancelled'`; `REPLY` → update that attendee's `partstat` on the organizer's copy.
+- **RSVP:** invite card buttons call `POST /api/plugins/calendar/events/{id}/rsvp {partstat}` → update row → build iTIP REPLY ics → send via SDK outbox helper to the organizer (subject `Accepted: {summary}` etc.) → publish `plugin.calendar.event_updated`.
+- **Organizer:** creating an event with attendees sends METHOD:REQUEST to each; edits bump `SEQUENCE` and re-send; deletion sends CANCEL. All outgoing invite mail uses the account whose address matches the organizer identity (picker if ambiguous).
+
+### A.6 UI (page `/calendar`, nav icon between Mail and Settings)
+Routes: `/calendar` (month default), `/calendar/{month|week|day|agenda}/:date`, `/calendar/event/:id`, `/calendar/settings` (calendars CRUD, CalDAV/ICS wizard reusing the account-wizard UX pattern: URL+credentials → test → collection picker). Interactions: click-drag create, drag-move/resize (writes optimistically, reverts on API error), event popover with edit/RSVP, mini-month navigator, calendar visibility toggles (client state), `t` = today, arrows = prev/next period. Invite linkage both ways: invite card in thread links to `/calendar/event/:id`; event view links back to the mail thread when `origin_message_id` is set. Live updates via WS `plugin.calendar.*` events. Timezone: render in user setting `settings.tz` (add to §9.1 settings; default from browser at registration).
+
+### A.7 Acceptance criteria
+1. Plugin installs/uninstalls (`--drop-data`) with zero diffs in `app/` and `frontend/src/` outside `plugins/`/`slots/` loader code.
+2. A weekly-recurring event with two EXDATEs and one moved instance renders correctly in month and week views across a DST boundary (Europe/Warsaw fixtures).
+3. Round-trip with Google Calendar **via iMIP mail only**: receive a Google invite → Accept in our UI → organizer's Google Calendar shows attendee accepted; our organizer invite → Google attendee accepts → our event shows `partstat=accepted`.
+4. CalDAV sync against a Nextcloud/Radicale test container: create/edit/delete propagate both directions in ≤ 5 min (one cron cycle); conflict case produces the A.4 tentative-duplicate, never silent loss.
+5. ICS subscription to a public holiday feed renders read-only events; editing is blocked in UI and API (403 `read_only_calendar`).
+6. Killing the worker mid-sync and restarting causes no duplicate events (upsert key discipline).
+
+---
+
+## Appendix B — Contacts plugin (`webmail_plugin_contacts`)
+
+The second heavy plugin: same rules as the calendar — §11 SDK only, zero core diffs, own page. Division of labor with core: core owns *implicit* recipient history (`known_recipients`, §4/§9.1) so autocomplete works even without this plugin; the Contacts plugin owns *explicit*, user-curated contact records and enriches suggestions via the `recipients.suggest` capability (§11.3).
+
+### B.1 Scope by phase
+- **B1 — Local contacts:** address books, contact CRUD with structured fields, groups, photo, search, dedicated page `/contacts`; "Add sender to contacts" from the thread view; suggestion capability wired into the composer.
+- **B2 — Portability:** vCard 3.0/4.0 import (file, multi-card) and export (single contact / book), duplicate detection + merge tool.
+- **B3 — External sources:** `ContactProvider` interface with **CardDAV client** sync (iCloud, Fastmail, Nextcloud, Radicale); per-book sync job.
+- **B4 (deferred, spec later):** Google People / Microsoft Graph providers (reusing M2 OAuth), CardDAV **server** endpoint for native phone sync, book sharing.
+
+### B.2 Dependencies
+`vobject ~0.9` (vCard parse/serialize). Photos stored via the SDK blob store (JPEG/PNG, re-encoded server-side, ≤ 512×512). Nothing else.
+
+### B.3 Data model (plugin migrations, prefix `plg_contacts_`)
+```sql
+CREATE TABLE plg_contacts_books (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  name text NOT NULL,
+  kind text NOT NULL DEFAULT 'local',       -- local|carddav
+  source_config jsonb NOT NULL DEFAULT '{}',-- carddav url/collection
+  secret_ct bytea,                          -- carddav credentials, §10.2 vault scheme
+  sync_token text,
+  read_only boolean NOT NULL DEFAULT false,
+  UNIQUE (user_id, name)
+);
+
+CREATE TABLE plg_contacts_contacts (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  book_id uuid NOT NULL REFERENCES plg_contacts_books(id) ON DELETE CASCADE,
+  vcard_uid text NOT NULL,                  -- UID property; join key for CardDAV
+  etag text,                                -- server etag (carddav books)
+  name_display text NOT NULL DEFAULT '',    -- FN
+  name_parts jsonb NOT NULL DEFAULT '{}',   -- {given,family,middle,prefix,suffix} (N)
+  emails jsonb NOT NULL DEFAULT '[]',       -- [{value,type,pref}]
+  phones jsonb NOT NULL DEFAULT '[]',
+  addresses jsonb NOT NULL DEFAULT '[]',    -- structured ADR
+  org text NOT NULL DEFAULT '',
+  title text NOT NULL DEFAULT '',
+  birthday date,
+  urls jsonb NOT NULL DEFAULT '[]',
+  notes text NOT NULL DEFAULT '',
+  photo_sha256 text,
+  raw_vcard text,                           -- round-trip fidelity: preserve unknown props
+  updated_at timestamptz NOT NULL DEFAULT now(),
+  UNIQUE (book_id, vcard_uid)
+);
+CREATE INDEX ON plg_contacts_contacts (user_id, name_display);
+CREATE INDEX ON plg_contacts_contacts USING gin (emails jsonb_path_ops);
+
+CREATE TABLE plg_contacts_groups (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  name text NOT NULL,
+  UNIQUE (user_id, name)
+);
+CREATE TABLE plg_contacts_group_members (
+  group_id uuid NOT NULL REFERENCES plg_contacts_groups(id) ON DELETE CASCADE,
+  contact_id uuid NOT NULL REFERENCES plg_contacts_contacts(id) ON DELETE CASCADE,
+  PRIMARY KEY (group_id, contact_id)
+);
+```
+
+vCard mapping discipline: parse known properties into columns, keep the full source in `raw_vcard`, and on export/push regenerate from columns **merged over** `raw_vcard` so properties we don't model (e.g. `X-` extensions) survive round-trips.
+
+### B.4 ContactProvider interface (inside the plugin, mirrors A.4)
+```python
+# PLUGIN INTERFACE VERSION: 1
+class ContactProvider(Protocol):
+    async def verify(self) -> None: ...
+    async def list_books(self) -> list[RemoteBook]: ...
+    async def pull(self, book: str, *, sync_token: str | None
+                   ) -> tuple[list[RemoteVcard], list[str], str]: ...
+        # (changed vcards, deleted uids, new sync_token)
+    async def push(self, book: str, vcard: str, *, uid: str,
+                   etag: str | None) -> str: ...
+    async def delete(self, book: str, uid: str, etag: str | None) -> None: ...
+```
+Sync loop = plugin cron job (`*/5 * * * *`, singleton per user), upsert by `(book_id, vcard_uid)`, etag conflict ⇒ server wins + local copy kept as a duplicate flagged for the merge tool (never silent loss) — same policy as A.4.
+
+### B.5 API & capability (`/api/plugins/contacts/`)
+`GET /contacts?book=&group=&q=&cursor=` · `POST /contacts` · `GET/PUT/DELETE /contacts/{id}` · `POST /contacts/from-message {message_id}` (prefill from From header) · `GET/POST/DELETE /books`, `/groups`, group membership ops · `POST /import` (vCard file → report `{created, updated, skipped}`) · `GET /export?book=|contact=` (`.vcf`) · `GET /duplicates` (candidate pairs: same email, or same normalized name + fuzzy) · `POST /merge {keep_id, remove_id}` (field union, keep wins conflicts, group memberships merged) · **`GET /suggest?q=`** — the `recipients.suggest` capability endpoint: matches name/email prefix and substring, returns `[{name, email, source:"contact", contact_id, photo_url?}]`. Composer merge rule (client-side, defined here so behavior is deterministic): contact hits rank above `known_recipients` hits at equal prefix quality; dedupe by email, contact entry wins and carries the photo/display name.
+
+### B.6 UI
+Page `/contacts` (nav entry after Calendar): three-pane layout — books/groups sidebar, alphabetically grouped virtualized list with search-as-you-type, detail pane with edit-in-place form; import/export and merge review under `/contacts/tools`; CardDAV book wizard reusing the account-wizard UX pattern (URL + credentials → test → book picker). Mail integration via `message.actions` slot: "Add to Contacts" (hidden when the sender already matches a contact; then it becomes "View contact" linking to `/contacts/{id}`). Group compose: selecting a group in the composer expands to its members' preferred emails.
+
+### B.7 Acceptance criteria
+1. Zero core diffs; install/uninstall `--drop-data` round-trip clean.
+2. Import a 500-card vCard file mixing v3.0/v4.0, quoted-printable encoding, and Polish diacritics in names (`Grzegorz Brzęczyszczykiewicz`) — all names, emails, and photos correct; re-export → re-import is lossless including `X-` properties (raw_vcard merge discipline).
+3. Composer typing a 3-char prefix returns merged suggestions in < 100 ms p95 (contacts ranked above history, deduped by email, photos shown for contacts).
+4. CardDAV sync against Radicale: create/edit/delete propagate both directions within one cron cycle; concurrent edit conflict produces a flagged duplicate, never silent loss; worker kill mid-sync ⇒ no duplicate contacts on restart.
+5. "Add sender to contacts" from a thread creates a prefilled contact; the same sender's messages immediately render the contact's display name, and the action flips to "View contact".
+6. Merge tool: merging two contacts unions emails/phones/groups, keeps the chosen primary's conflicting fields, and repoints nothing else in the system (calendar attendees and mail remain keyed by email, by design).
+
+---
+*End of specification. Ambiguities: prefer the simplest behavior consistent with the invariants in §1, leave a `TODO(spec)` comment, and list open questions in `docs/open-questions.md` rather than guessing silently.*