Loading splash: animated logo over a dark overlay on desktop

index.html carries the overlay markup and styles inline so it paints before
any script; public/splash.js (external, for the CSP) plays the logo
animation from icons/ultimo-splash.html, holds, fades out after ~3.25 s and
removes itself. Skipped at phone sizes (same query as useIsMobile) and in
pop-out compose windows; a click skips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,30 @@
+# Build context hygiene: the backend image builds from the repo root (it needs
+# backend/ *and* plugins/), so everything heavy or secret is excluded here.
+.git
+.github
+.gitea
+.claude
+**/node_modules
+**/dist
+**/__pycache__
+**/*.py[cod]
+**/.mypy_cache
+**/.ruff_cache
+**/.pytest_cache
+**/*.egg-info
+# Runtime data: mail content and local blobs must never enter an image layer.
+var/
+backend/var/
+**/var/blobs
+# Secrets
+.env.dev
+.env.*.local
+gitea-server-credentials.md
+docker/.env.prod
+deploy/*.env
+# Docs / screenshots / scratch
+screenshots/
+docs/
+*.md
+!README.md
+claude-session.txt

.env.example

@@ -11,3 +11,15 @@ POLL_INTERVAL=120
 SESSION_TTL_DAYS=30
 MAX_ATTACHMENT_MB=25
 LOG_LEVEL=DEBUG
+
+# --- Anti-spam (SPEC Appendix C) ---------------------------------------------
+# `make up` starts rspamd + redis-rspamd. The API and worker run on the host in
+# dev, so they use the published loopback ports rather than the service name.
+RSPAMD_URL=http://127.0.0.1:11333
+RSPAMD_CONTROLLER_URL=http://127.0.0.1:11334
+RSPAMD_CONTROLLER_PASSWORD=devpass
+
+# Spelling/grammar check in the compose window (LanguageTool). Empty = off.
+LANGUAGETOOL_URL=http://127.0.0.1:8010
+# Email translation (the translator service, translator/). Empty = off.
+TRANSLATOR_URL=http://127.0.0.1:5005

.gitignore

@@ -1,5 +1,8 @@
 # Secrets / local env
 .env.dev
+docker/.env.prod
+docker/.env
+/.env
 .env.*.local
 gitea-server-credentials.md
 
@@ -31,3 +34,9 @@ frontend/dist/
 .DS_Store
 .idea/
 .vscode/
+
+# Editor backups
+*~
+
+# Local session helper (Claude Code resume command)
+/claude-session.txt

Makefile

@@ -1,5 +1,5 @@
 up:        ## start services
-	docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch
+	docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool translator
 down:      ## stop services (data kept)
 	docker compose -f docker/docker-compose.yml down
 reset-db:  ## DESTRUCTIVE: wipe db + search + blobs, re-migrate
@@ -21,3 +21,22 @@ test:
 	cd backend && pytest -q && cd ../frontend && npx vitest run
 mail-test:  ## start throwaway IMAP/SMTP server
 	docker compose -f docker/docker-compose.yml up -d greenmail
+
+# --- production (NAS / single-host Docker deploy; see docs/nas-install.md) ---
+prod-init:    ## generate docker/.env.prod + data dirs
+	./deploy/webmail.sh init
+prod-up:      ## start the production stack
+	./deploy/webmail.sh up
+prod-down:    ## stop the production stack (data kept)
+	./deploy/webmail.sh down
+prod-status:  ## what is running, versions, disk, backups
+	./deploy/webmail.sh status
+prod-update:  ## backup -> migrate -> roll -> health-gate (auto-rollback on failure)
+	./deploy/webmail.sh update
+prod-backup:  ## take a verified database dump now
+	./deploy/webmail.sh backup
+prod-doctor:  ## pre-flight checks on a deployment
+	./deploy/webmail.sh doctor
+prod-images:  ## build production images locally (no push)
+	docker build -f docker/Dockerfile.backend -t webmail/backend:local .
+	docker build -f docker/Dockerfile.frontend -t webmail/web:local .

backend/app/accounts/discovery.py

@@ -18,7 +18,13 @@ import yaml
 from defusedxml import ElementTree as SafeET
 
 from app.accounts.ssrf import SsrfError, assert_host_is_public, guarded_get
-from app.accounts.types import CandidateConfig, DiscoveryResult, DiscoveryStep, ServerConfig
+from app.accounts.types import (
+    CandidateConfig,
+    DiscoveryResult,
+    DiscoveryStep,
+    HelpBlock,
+    ServerConfig,
+)
 
 _PROBE_TIMEOUT = 5.0
 
@@ -28,6 +34,26 @@ _presets_cache: dict[str, CandidateConfig] | None = None
 # --- presets -----------------------------------------------------------------
 
 
+def _strategy_from_auth(auth: str) -> tuple[str, ...]:
+    """Default credential flow when a preset doesn't declare one (SPEC §6.2.1)."""
+    return ("oauth2",) if auth == "oauth2" else ("password",)
+
+
+def _help_block(raw: object) -> HelpBlock | None:
+    if not isinstance(raw, dict):
+        return None
+    steps = raw.get("steps", [])
+    return HelpBlock(
+        help_id=str(raw["help_id"]),
+        deep_link=str(raw["deep_link"]),
+        app_name_hint=str(raw.get("app_name_hint", "")),
+        steps=tuple(str(s) for s in steps) if isinstance(steps, list) else (),
+        docs_url=str(raw.get("docs_url", "")),
+        setup_link=str(raw.get("setup_link", "")),
+        button_after_step=int(raw.get("button_after_step", 0)),
+    )
+
+
 def _load_presets() -> dict[str, CandidateConfig]:
     global _presets_cache
     if _presets_cache is None:
@@ -41,7 +67,15 @@ def _load_presets() -> dict[str, CandidateConfig]:
             smtp = ServerConfig(
                 prov["smtp"]["host"], prov["smtp"]["port"], prov["smtp"]["tls"], auth
             )
-            candidate = CandidateConfig(imap, smtp, "preset", prov.get("oauth_provider"))
+            strategy = tuple(prov.get("auth_strategy", [])) or _strategy_from_auth(auth)
+            candidate = CandidateConfig(
+                imap,
+                smtp,
+                "preset",
+                prov.get("oauth_provider"),
+                strategy,
+                _help_block(prov.get("help")),
+            )
             for domain in prov["domains"]:
                 mapping[domain.lower()] = candidate
         _presets_cache = mapping
@@ -91,7 +125,7 @@ def parse_autoconfig_xml(xml_text: str) -> CandidateConfig | None:
     smtp = _server("outgoingServer", "smtp")
     if imap is None or smtp is None:
         return None
-    return CandidateConfig(imap, smtp, "autoconfig")
+    return CandidateConfig(imap, smtp, "autoconfig", auth_strategy=_strategy_from_auth(imap.auth))
 
 
 # --- DNS SRV (RFC 6186) ------------------------------------------------------
@@ -191,7 +225,14 @@ async def _try_http_config(url: str, source: str) -> CandidateConfig | None:
     candidate = parse_autoconfig_xml(xml_text)
     if candidate is None:
         return None
-    return CandidateConfig(candidate.imap, candidate.smtp, source, candidate.oauth_provider)
+    return CandidateConfig(
+        candidate.imap,
+        candidate.smtp,
+        source,
+        candidate.oauth_provider,
+        candidate.auth_strategy,
+        candidate.help,
+    )
 
 
 async def discover(email: str) -> DiscoveryResult:

backend/app/accounts/presets.yaml

@@ -2,27 +2,62 @@
 # Matched by the domain of the email address. Each entry gives IMAP + SMTP
 # host/port/tls, the auth kind, and an oauth provider id when applicable.
 # tls: "ssl" (implicit TLS) | "starttls"; auth: "password" | "oauth2".
+#
+# auth_strategy (SPEC §6.2.1): ordered list of the credential flows the wizard
+# offers, first = default. Values: app_password | oauth2 | password.
+#   - app_password → renders the ProviderCredentialPanel keyed by help.help_id
+#     (deep-link + guide + paste + test); the value is used as a normal IMAP/SMTP
+#     password, so it reuses the plain-password sync path (zero new sync code).
+#   - oauth2 → XOAUTH2 (M2); for Google it is the *fallback* offered when the app
+#     password path fails and GOOGLE_OAUTH_ENABLED is set.
+#   - password → the generic password box (self-hosted / ISP accounts).
+# help (SPEC §6.2.2/6.2.3): data for the panel. steps are i18n message keys so
+# the copy is translatable; app_name_hint is a suggested app-password label.
 
 providers:
   google:
     domains: [gmail.com, googlemail.com]
     imap: { host: imap.gmail.com, port: 993, tls: ssl }
     smtp: { host: smtp.gmail.com, port: 465, tls: ssl }
-    auth: oauth2
+    auth: password
+    auth_strategy: [app_password, oauth2]
     oauth_provider: google
+    help:
+      help_id: gmail
+      deep_link: https://myaccount.google.com/apppasswords
+      setup_link: https://myaccount.google.com/signinoptions/two-step-verification
+      app_name_hint: "ultimo.email"
+      button_after_step: 2
+      docs_url: https://support.google.com/mail/answer/185833
+      steps:
+        - wizard.help.gmail.step1
+        - wizard.help.gmail.step2
+        - wizard.help.gmail.step3
+        - wizard.help.gmail.step4
 
   microsoft:
     domains: [outlook.com, hotmail.com, live.com, msn.com]
     imap: { host: outlook.office365.com, port: 993, tls: ssl }
     smtp: { host: smtp.office365.com, port: 587, tls: starttls }
-    auth: oauth2
+    auth: password
+    auth_strategy: [app_password, oauth2]
     oauth_provider: microsoft
+    help:
+      help_id: outlook
+      deep_link: https://account.microsoft.com/security
+      app_name_hint: "Webmail"
+      docs_url: https://support.microsoft.com/account-billing/using-app-passwords-with-apps-that-don-t-support-two-step-verification-5896ed9b-4263-e681-128a-a6f2979a7944
+      steps:
+        - wizard.help.outlook.step1
+        - wizard.help.outlook.step2
+        - wizard.help.outlook.step3
 
   office365:
     domains: [office365.com]
     imap: { host: outlook.office365.com, port: 993, tls: ssl }
     smtp: { host: smtp.office365.com, port: 587, tls: starttls }
     auth: oauth2
+    auth_strategy: [oauth2]
     oauth_provider: microsoft
 
   yahoo:
@@ -30,39 +65,64 @@ providers:
     imap: { host: imap.mail.yahoo.com, port: 993, tls: ssl }
     smtp: { host: smtp.mail.yahoo.com, port: 465, tls: ssl }
     auth: password
+    auth_strategy: [app_password]
+    help:
+      help_id: yahoo
+      deep_link: https://login.yahoo.com/account/security/app-passwords
+      app_name_hint: "Webmail"
+      docs_url: https://help.yahoo.com/kb/SLN15241.html
+      steps:
+        - wizard.help.yahoo.step1
+        - wizard.help.yahoo.step2
+        - wizard.help.yahoo.step3
 
   icloud:
     domains: [icloud.com, me.com, mac.com]
     imap: { host: imap.mail.me.com, port: 993, tls: ssl }
     smtp: { host: smtp.mail.me.com, port: 587, tls: starttls }
     auth: password
+    auth_strategy: [app_password]
+    help:
+      help_id: icloud
+      deep_link: https://account.apple.com/account/manage
+      app_name_hint: "Webmail"
+      docs_url: https://support.apple.com/102654
+      steps:
+        - wizard.help.icloud.step1
+        - wizard.help.icloud.step2
+        - wizard.help.icloud.step3
 
   gmx:
     domains: [gmx.com, gmx.net, gmx.de]
     imap: { host: imap.gmx.com, port: 993, tls: ssl }
     smtp: { host: mail.gmx.com, port: 587, tls: starttls }
     auth: password
+    auth_strategy: [password]
 
   wp:
     domains: [wp.pl]
     imap: { host: imap.wp.pl, port: 993, tls: ssl }
     smtp: { host: smtp.wp.pl, port: 465, tls: ssl }
     auth: password
+    auth_strategy: [password]
 
   onet:
     domains: [onet.pl, poczta.onet.pl, op.pl]
     imap: { host: imap.poczta.onet.pl, port: 993, tls: ssl }
     smtp: { host: smtp.poczta.onet.pl, port: 465, tls: ssl }
     auth: password
+    auth_strategy: [password]
 
   o2:
     domains: [o2.pl, tlen.pl]
     imap: { host: imap.poczta.o2.pl, port: 993, tls: ssl }
     smtp: { host: smtp.poczta.o2.pl, port: 465, tls: ssl }
     auth: password
+    auth_strategy: [password]
 
   interia:
     domains: [interia.pl, interia.eu]
     imap: { host: imap.poczta.interia.pl, port: 993, tls: ssl }
     smtp: { host: smtp.poczta.interia.pl, port: 465, tls: ssl }
     auth: password
+    auth_strategy: [password]

backend/app/accounts/schemas.py

@@ -3,9 +3,13 @@
 from __future__ import annotations
 
 import uuid
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
+# How much history to import at account setup (SPEC §12 M2).
+SyncWindow = Literal["all", "3months", "new"]
+
 
 class ServerConfigIn(BaseModel):
     host: str
@@ -18,11 +22,23 @@ class DiscoverRequest(BaseModel):
     email: str
 
 
+class HelpBlockOut(BaseModel):
+    help_id: str
+    deep_link: str
+    app_name_hint: str = ""
+    steps: list[str] = []
+    docs_url: str = ""
+    setup_link: str = ""
+    button_after_step: int = 0
+
+
 class CandidateOut(BaseModel):
     imap: ServerConfigIn
     smtp: ServerConfigIn
     source: str
     oauth_provider: str | None = None
+    auth_strategy: list[str] = ["password"]
+    help: HelpBlockOut | None = None
 
 
 class DiscoveryStepOut(BaseModel):
@@ -34,6 +50,8 @@ class DiscoveryStepOut(BaseModel):
 class DiscoverResponse(BaseModel):
     candidates: list[CandidateOut]
     steps: list[DiscoveryStepOut]
+    # Whether the wizard may offer the Google OAuth fallback (SPEC §6.2.1).
+    google_oauth_enabled: bool = False
 
 
 class TestRequest(BaseModel):
@@ -56,12 +74,15 @@ class CreateAccountRequest(BaseModel):
     smtp: ServerConfigIn
     username: str | None = None
     password: str = Field(min_length=1)
+    sync_window: SyncWindow = "all"
+    sync_spam: bool = False
 
 
 class UpdateAccountRequest(BaseModel):
     display_name: str | None = None
     color: str | None = None
     signature: str | None = None
+    sync_window: SyncWindow | None = None
     # Connection/credential changes require the password (re-validated + re-encrypted).
     imap: ServerConfigIn | None = None
     smtp: ServerConfigIn | None = None
@@ -76,5 +97,9 @@ class AccountDetailOut(BaseModel):
     state: str
     state_detail: str
     signature: str
+    sync_window: str
+    backfill_total: int
+    backfill_done: int
     imap_config: dict[str, object]
     smtp_config: dict[str, object]
+    kind: str = "external"

backend/app/accounts/service.py

@@ -4,14 +4,14 @@ from __future__ import annotations
 
 import uuid
 
-from sqlalchemy import delete, select
+from sqlalchemy import delete, func, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.accounts.schemas import CreateAccountRequest
 from app.accounts.types import ServerConfig
 from app.indexer import blobstore
-from app.models import Account, Attachment, Message
+from app.models import Account, Attachment, ComposeAttachment, Message, Thread
 from app.security.crypto import encrypt_secret
 
 
@@ -39,6 +39,8 @@ async def create_account(
         secret_ct=encrypt_secret(req.password.encode("utf-8")),
         discovery_src="manual",
         state="backfilling",
+        sync_window=req.sync_window,
+        sync_spam=req.sync_spam,
     )
     session.add(account)
     try:
@@ -80,12 +82,36 @@ async def delete_account(
     await session.execute(delete(Account).where(Account.id == account_id))
     await session.flush()
 
+    # Threads are user-scoped and can span accounts (Thread.account_ids), so the
+    # account cascade only removes its messages — not the threads. Clean up: drop
+    # threads left with no messages (else they linger as empty rows the unified list
+    # and label views still show but can't open), and de-list the account from
+    # threads that survive on another account's messages.
+    await session.execute(
+        delete(Thread).where(
+            Thread.user_id == user_id,
+            ~select(Message.id).where(Message.thread_id == Thread.id).exists(),
+        )
+    )
+    await session.execute(
+        update(Thread)
+        .where(Thread.user_id == user_id, Thread.account_ids.contains([account_id]))
+        .values(account_ids=func.array_remove(Thread.account_ids, account_id))
+    )
+
     # Deregister from sync:live and drop any lease so the owning sync worker
     # stops the account's runner on its next renewal (refactor §2 fencing).
+    from app.events import get_event_bus
     from app.redis_client import get_redis
     from app.sync import leases
 
     await leases.mark_not_live(get_redis(), account_id)
+    # Sync v2 §3: signal the owning worker to tear the runner down now (≤30s),
+    # rather than waiting for the lease to lapse.
+    await get_event_bus().publish(
+        "account.state_changed",
+        {"user_id": str(user_id), "account_id": str(account_id), "state": "deleted"},
+    )
 
     # Content-addressed blobs are shared across the user's accounts (dedup): only
     # remove a blob when no remaining message/attachment references it.
@@ -101,6 +127,13 @@ async def delete_account(
                 .where(Attachment.user_id == user_id, Attachment.blob_sha256 == sha)
                 .limit(1)
             )
+        if still_used is None:
+            # A forwarded file waiting in a draft shares the hash: keep it for that.
+            still_used = await session.scalar(
+                select(ComposeAttachment.id)
+                .where(ComposeAttachment.user_id == user_id, ComposeAttachment.blob_sha256 == sha)
+                .limit(1)
+            )
         if still_used is None:
             blobstore.remove(user_id, sha)
     return True

backend/app/accounts/ssrf.py

@@ -2,9 +2,10 @@
 
 All autodiscovery HTTP is HTTPS-only, 5 s timeout, response capped at 256 KB, no
 off-domain redirects, and the target host must resolve exclusively to public IPs
-(reject private/loopback/link-local/reserved ranges). A residual TOCTOU gap
-(DNS rebinding between our resolve and httpx's own resolve) is accepted for M1;
-a pinned-IP transport can close it later.
+(reject private/loopback/link-local/reserved ranges). The request then goes to
+the very address that was checked — the name is not looked up a second time —
+so a DNS answer that changes in between (DNS rebinding) cannot point it inward.
+TLS is still verified against the host name.
 """
 
 from __future__ import annotations
@@ -12,6 +13,7 @@ from __future__ import annotations
 import asyncio
 import ipaddress
 import socket
+from dataclasses import dataclass
 from urllib.parse import urlsplit
 
 import httpx
@@ -25,15 +27,12 @@ class SsrfError(Exception):
 
 
 def _is_public_ip(ip: str) -> bool:
-    addr = ipaddress.ip_address(ip)
-    return not (
-        addr.is_private
-        or addr.is_loopback
-        or addr.is_link_local
-        or addr.is_multicast
-        or addr.is_reserved
-        or addr.is_unspecified
-    )
+    addr = ipaddress.ip_address(ip.split("%", 1)[0])  # drop an IPv6 zone index
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
+        addr = addr.ipv4_mapped  # ::ffff:127.0.0.1 is 127.0.0.1
+    # is_global: not private, loopback, link-local, reserved, unspecified, nor
+    # shared/carrier-grade NAT (100.64.0.0/10, which is_private misses).
+    return addr.is_global and not addr.is_multicast
 
 
 async def _resolve_all(host: str) -> list[str]:
@@ -42,8 +41,10 @@ async def _resolve_all(host: str) -> list[str]:
     return [info[4][0] for info in infos]
 
 
-async def assert_host_is_public(host: str) -> None:
-    """Resolve ``host`` and reject if any address is non-public (SPEC §10.6)."""
+async def assert_host_is_public(host: str) -> list[str]:
+    """Resolve ``host`` and reject if any address is non-public (SPEC §10.6).
+
+    Returns the addresses checked, for the caller to connect to."""
     try:
         addresses = await _resolve_all(host)
     except socket.gaierror as exc:
@@ -53,6 +54,31 @@ async def assert_host_is_public(host: str) -> None:
     for ip in addresses:
         if not _is_public_ip(ip):
             raise SsrfError(f"{host} resolves to non-public address {ip}")
+    return addresses
+
+
+async def _pinned(url: str) -> tuple[str, dict[str, str], dict[str, str]]:
+    """``url`` aimed at the address just checked: (url, headers, extensions).
+
+    The URL names the IP, the Host header keeps the name (virtual hosting), and
+    TLS gets the name for SNI and for checking the certificate."""
+    parts = urlsplit(url)
+    if parts.scheme != "https":
+        raise SsrfError("only https is permitted")
+    hostname = parts.hostname
+    if not hostname:
+        raise SsrfError("missing host")
+    addresses = await assert_host_is_public(hostname)
+    # IPv4 first: containers often have no IPv6 route, though DNS returns AAAA.
+    ip = next((a for a in addresses if ":" not in a), addresses[0])
+    literal = f"[{ip}]" if ":" in ip else ip
+    netloc = f"{literal}:{parts.port}" if parts.port else literal
+    host_header = f"{hostname}:{parts.port}" if parts.port else hostname
+    return (
+        parts._replace(netloc=netloc).geturl(),
+        {"Host": host_header},
+        {"sni_hostname": hostname},
+    )
 
 
 async def guarded_get(url: str) -> str | None:
@@ -62,12 +88,7 @@ async def guarded_get(url: str) -> str | None:
     (any non-200 response). Raises :class:`SsrfError` on a policy violation
     (bad scheme, private IP, oversize); lets httpx network errors propagate.
     """
-    parts = urlsplit(url)
-    if parts.scheme != "https":
-        raise SsrfError("only https is permitted")
-    if not parts.hostname:
-        raise SsrfError("missing host")
-    await assert_host_is_public(parts.hostname)
+    target, pin_headers, extensions = await _pinned(url)
 
     async with (
         httpx.AsyncClient(
@@ -75,7 +96,7 @@ async def guarded_get(url: str) -> str | None:
             timeout=HTTP_TIMEOUT,
             headers={"User-Agent": "webmail-autodiscovery/1.0"},
         ) as client,
-        client.stream("GET", url) as resp,
+        client.stream("GET", target, headers=pin_headers, extensions=extensions) as resp,
     ):
         if resp.status_code != 200:
             return None
@@ -87,3 +108,55 @@ async def guarded_get(url: str) -> str | None:
                 raise SsrfError("response exceeds 256 KB cap")
             chunks.append(chunk)
     return b"".join(chunks).decode("utf-8", "replace")
+
+
+@dataclass(frozen=True)
+class GuardedResponse:
+    status: int
+    headers: dict[str, str]
+    content: bytes
+    text: str
+
+
+async def guarded_request(
+    method: str,
+    url: str,
+    *,
+    headers: dict[str, str] | None = None,
+    content: bytes | str | None = None,
+    auth: tuple[str, str] | None = None,
+    timeout: float = 15.0,  # noqa: ASYNC109 - passed straight to httpx's own timeout
+    max_bytes: int = 25 * 1024 * 1024,
+) -> GuardedResponse:
+    """Arbitrary-method HTTPS request under the same SSRF policy as :func:`guarded_get`.
+
+    Used by plugins that speak WebDAV/CalDAV/CardDAV (PROPFIND/REPORT/PUT/DELETE) or
+    fetch ICS feeds. HTTPS-only, public host required, no off-domain redirects, body
+    capped at ``max_bytes``. ``auth`` is an (user, password) basic-auth pair. Returns
+    status + headers + body (all statuses; the caller decides what's an error)."""
+    target, pin_headers, extensions = await _pinned(url)
+
+    async with (
+        httpx.AsyncClient(follow_redirects=False, timeout=timeout, auth=auth) as client,
+        client.stream(
+            method,
+            target,
+            headers={**(headers or {}), **pin_headers},
+            content=content,
+            extensions=extensions,
+        ) as resp,
+    ):
+        chunks: list[bytes] = []
+        total = 0
+        async for chunk in resp.aiter_bytes():
+            total += len(chunk)
+            if total > max_bytes:
+                raise SsrfError("response exceeds size cap")
+            chunks.append(chunk)
+        body = b"".join(chunks)
+        return GuardedResponse(
+            status=resp.status_code,
+            headers=dict(resp.headers),
+            content=body,
+            text=body.decode("utf-8", "replace"),
+        )

backend/app/accounts/testing.py

@@ -14,6 +14,7 @@ import aiosmtplib
 
 from app.accounts.ssrf import SsrfError, assert_host_is_public
 from app.accounts.types import ServerConfig
+from app.config import smtp_helo_name
 from app.providers.base import ProviderAuthError, ProviderConnectError
 from app.providers.imap_smtp import ImapSmtpProvider, ProviderConfig
 
@@ -53,6 +54,7 @@ async def _test_smtp(config: ServerConfig, username: str, password: str) -> None
         use_tls=config.tls == "ssl",
         start_tls=config.tls == "starttls",
         timeout=_SMTP_TIMEOUT,
+        local_hostname=smtp_helo_name(),
     )
     try:
         await smtp.connect()
@@ -89,6 +91,7 @@ async def test_account(
             smtp_tls=smtp.tls,
             username=username,
             password=password,
+            public_only=True,
         )
     )
     try:

backend/app/accounts/types.py

@@ -13,12 +13,29 @@ class ServerConfig:
     auth: str = "password"  # "password" | "oauth2"
 
 
+@dataclass(frozen=True)
+class HelpBlock:
+    """Data-driven content for the wizard's ProviderCredentialPanel (SPEC §6.2.2)."""
+
+    help_id: str  # gmail | yahoo | icloud | outlook
+    deep_link: str  # provider app-password page (opened from a user gesture)
+    app_name_hint: str  # suggested app-password label
+    steps: tuple[str, ...]  # i18n message keys
+    docs_url: str = ""
+    setup_link: str = ""  # optional 2FA/security page linked from the first step
+    button_after_step: int = 0  # 1-based step to place the app-password button after; 0 = top
+
+
 @dataclass(frozen=True)
 class CandidateConfig:
     imap: ServerConfig
     smtp: ServerConfig
     source: str  # preset | autoconfig | ispdb | srv | probe | manual
     oauth_provider: str | None = None
+    # Ordered credential flows the wizard offers (SPEC §6.2.1): first = default.
+    # Values: app_password | oauth2 | password.
+    auth_strategy: tuple[str, ...] = ("password",)
+    help: HelpBlock | None = None
 
 
 @dataclass(frozen=True)

backend/app/api/__init__.py

@@ -13,13 +13,25 @@ def register_routers(app: FastAPI) -> None:
     """
     from app.api.accounts import router as accounts_router
     from app.api.actions import router as actions_router
+    from app.api.admin import router as admin_router
+    from app.api.avatar import router as avatar_router
+    from app.api.completion import router as completion_router
     from app.api.compose import router as compose_router
+    from app.api.invitations import router as invitations_router
     from app.api.mailboxes import router as mailboxes_router
     from app.api.messages import router as messages_router
+    from app.api.plugins import router as plugins_router
+    from app.api.push import router as push_router
     from app.api.recipients import router as recipients_router
+    from app.api.saved_searches import router as saved_searches_router
     from app.api.search import router as search_router
     from app.api.settings import router as settings_router
+    from app.api.snooze import router as snooze_router
+    from app.api.spellcheck import router as spellcheck_router
+    from app.api.subscriptions import router as subscriptions_router
+    from app.api.templates import router as templates_router
     from app.api.threads import router as threads_router
+    from app.api.translation import router as translation_router
     from app.identity.routes import router as auth_router
     from app.ws import router as ws_router
 
@@ -34,6 +46,28 @@ def register_routers(app: FastAPI) -> None:
         recipients_router,
         settings_router,
         mailboxes_router,
+        saved_searches_router,
+        avatar_router,
+        plugins_router,
+        admin_router,
+        spellcheck_router,
+        push_router,
+        invitations_router,
+        templates_router,
+        snooze_router,
+        subscriptions_router,
+        completion_router,
+        translation_router,
     ):
         app.include_router(router, prefix="/api")
     app.include_router(ws_router, prefix="/api")
+    # CalDAV / CardDAV (SPEC Appendix E): at the root, where clients look for it.
+    from app.dav.routes import router as dav_router
+
+    app.include_router(dav_router)
+
+    # Mount discovered plugin routers at /api/plugins/{name} (SPEC §11). Hooks are
+    # subscribed in the lifespan startup so they register once per process.
+    from app.plugins import loader
+
+    loader.mount_routers(app)

backend/app/api/accounts.py

@@ -17,6 +17,7 @@ from app.accounts.schemas import (
     DiscoverRequest,
     DiscoverResponse,
     DiscoveryStepOut,
+    HelpBlockOut,
     ServerConfigIn,
     TestRequest,
     TestResponse,
@@ -27,6 +28,7 @@ from app.accounts.testing import AccountTestError, test_account
 from app.accounts.types import CandidateConfig, ServerConfig
 from app.api.errors import raise_api_error
 from app.api.schemas import AccountOut
+from app.config import get_settings
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
 from app.models import Account
@@ -62,6 +64,20 @@ def _candidate_out(c: CandidateConfig) -> CandidateOut:
         smtp=_server_in(c.smtp),
         source=c.source,
         oauth_provider=c.oauth_provider,
+        auth_strategy=list(c.auth_strategy),
+        help=(
+            HelpBlockOut(
+                help_id=c.help.help_id,
+                deep_link=c.help.deep_link,
+                app_name_hint=c.help.app_name_hint,
+                steps=list(c.help.steps),
+                docs_url=c.help.docs_url,
+                setup_link=c.help.setup_link,
+                button_after_step=c.help.button_after_step,
+            )
+            if c.help
+            else None
+        ),
     )
 
 
@@ -82,6 +98,13 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
             color=a.color,
             state=a.state,
             signature=a.signature,
+            sync_window=a.sync_window,
+            backfill_total=a.backfill_total,
+            backfill_done=a.backfill_done,
+            sync_error=a.sync_error,
+            sync_error_since=a.sync_error_since.isoformat() if a.sync_error_since else None,
+            state_detail=a.state_detail,
+            kind=a.kind,
         )
         for a in accounts
     ]
@@ -106,6 +129,7 @@ async def discover_account(body: DiscoverRequest, user: CurrentUser) -> Discover
             DiscoveryStepOut(method=s.method, status=s.status, detail=s.detail)
             for s in result.steps
         ],
+        google_oauth_enabled=get_settings().google_oauth_enabled,
     )
 
 
@@ -186,10 +210,24 @@ async def update_account(
     if body.signature is not None:
         account.signature = body.signature
 
+    # Broadening the sync window (e.g. "new" -> "all", "Download all mail") means
+    # importing history not fetched before, so re-run the backfill (SPEC §12 M2).
+    # persist_message is idempotent on (mailbox_id, uid), so already-synced mail
+    # is skipped. Narrowing keeps existing mail — nothing to fetch.
+    _COVERAGE = {"new": 0, "3months": 1, "all": 2}
+    broaden = False
+    if body.sync_window is not None and body.sync_window != account.sync_window:
+        broaden = _COVERAGE[body.sync_window] > _COVERAGE.get(account.sync_window, 0)
+        account.sync_window = body.sync_window
+        if broaden:
+            account.state = "backfilling"
+
     # Connection settings and/or password: re-validate against the servers, then
     # re-encrypt the secret and re-sync. The password must be supplied (§10.2: the
     # stored secret is only decrypted in the worker, never here).
     conn_change = body.imap is not None or body.smtp is not None or body.password is not None
+    if conn_change and account.kind == "native":
+        raise_api_error(400, "native_account", "the settings of your own mailbox cannot be changed")
     if conn_change:
         if not body.password:
             raise_api_error(
@@ -224,7 +262,7 @@ async def update_account(
 
     await db.commit()
 
-    if conn_change:
+    if conn_change or broaden:
         await enqueue_job("sync_bootstrap_account", str(account.id))
 
     return _detail(account)
@@ -232,10 +270,20 @@ async def update_account(
 
 @router.delete("/{account_id}", dependencies=[Depends(require_csrf)])
 async def remove_account(account_id: uuid.UUID, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    kind = await db.scalar(
+        select(Account.kind).where(Account.id == account_id, Account.user_id == user.id)
+    )
+    if kind == "native":
+        raise_api_error(409, "native_account", "your own mailbox cannot be removed")
     ok = await delete_account(db, user_id=user.id, account_id=account_id)
     if not ok:
         raise_api_error(404, "not_found", "account not found")
     await db.commit()
+    # Removing an account for the user's own address (the mail at its old
+    # provider) makes room for their hosted mailbox's (App. D §D.4).
+    from app.hosted import native
+
+    await native.ensure_and_sync(db, user)
     return {"status": "deleted"}
 
 
@@ -248,6 +296,10 @@ def _detail(account: Account) -> AccountDetailOut:
         state=account.state,
         state_detail=account.state_detail,
         signature=account.signature,
+        sync_window=account.sync_window,
+        backfill_total=account.backfill_total,
+        backfill_done=account.backfill_done,
         imap_config=account.imap_config,
         smtp_config=account.smtp_config,
+        kind=account.kind,
     )

backend/app/api/actions.py

@@ -10,7 +10,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.errors import raise_api_error
-from app.api.schemas import FlagChangeRequest, MoveRequest
+from app.api.schemas import FlagChangeRequest, MarkReadRequest, MoveRequest
 from app.db import get_session
 from app.events import get_event_bus
 from app.identity.deps import CurrentUser, require_csrf
@@ -86,6 +86,59 @@ async def thread_flags(
     return {"thread_id": str(thread_id), "changed": len(changed)}
 
 
+@router.post("/threads/mark-read")
+async def mark_all_read(
+    body: MarkReadRequest, user: CurrentUser, db: DbSession
+) -> dict[str, object]:
+    """Mark every unread thread in the given view as read (adds \\Seen). Scopes the
+    same way as the thread list: unified, folder:<special>, or account:<uuid>."""
+    view = body.view
+    conditions = [Thread.user_id == user.id, Thread.unread_count > 0]
+    if view.startswith("account:"):
+        try:
+            account_id = uuid.UUID(view.split(":", 1)[1])
+        except ValueError:
+            raise_api_error(400, "bad_view", "invalid account id in view")
+        in_account_inbox = (
+            select(Message.thread_id)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(
+                Message.user_id == user.id,
+                Message.account_id == account_id,
+                Mailbox.special_use == "inbox",
+            )
+        )
+        conditions.append(Thread.id.in_(in_account_inbox))
+    elif view.startswith("folder:"):
+        special = view.split(":", 1)[1]
+        in_folder = (
+            select(Message.thread_id)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(Message.user_id == user.id, Mailbox.special_use == special)
+        )
+        conditions.append(Thread.id.in_(in_folder))
+    elif view != "unified":
+        raise_api_error(400, "bad_view", "unsupported view")
+
+    thread_ids = list(await db.scalars(select(Thread.id).where(*conditions)))
+    if not thread_ids:
+        return {"changed": 0}
+    messages = list(
+        await db.scalars(
+            select(Message).where(Message.thread_id.in_(thread_ids), Message.user_id == user.id)
+        )
+    )
+    add, remove = sanitize_flags(["\\Seen"]), sanitize_flags([])
+    changed = [m for m in messages if _would_change(m, add, remove)]
+    await apply_flags_to_messages(db, messages, add, remove)
+    await db.commit()
+    await _push_and_publish(user.id, changed, add, remove)
+    bus = get_event_bus()
+    for tid in thread_ids:
+        await bus.publish("thread.updated", {"user_id": str(user.id), "thread_id": str(tid)})
+    return {"changed": len(changed)}
+
+
 async def _move_thread(
     user_id: uuid.UUID, thread_id: uuid.UUID, db: AsyncSession, special: str
 ) -> None:
@@ -111,8 +164,13 @@ async def trash_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -
 
 @router.post("/threads/{thread_id}/restore")
 async def restore_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -> dict[str, str]:
-    """Move a thread back to the Inbox (from Archive or Trash)."""
-    await _move_thread(user.id, thread_id, db, "inbox")
+    """Restore a thread (from Archive or Trash) to each message's original folder."""
+    thread = await db.scalar(
+        select(Thread).where(Thread.id == thread_id, Thread.user_id == user.id)
+    )
+    if thread is None:
+        raise_api_error(404, "not_found", "thread not found")
+    await enqueue_job("restore_thread_job", str(thread_id))
     return {"status": "queued"}
 
 
@@ -128,6 +186,19 @@ async def delete_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession)
     return {"status": "queued"}
 
 
+# Folders whose "Empty" action permanently deletes everything they hold.
+_EMPTYABLE_FOLDERS = frozenset({"trash", "junk", "archive"})
+
+
+@router.post("/folders/{special}/empty")
+async def empty_folder(special: str, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    """Permanently delete every message in a Trash/Spam/Archive folder."""
+    if special not in _EMPTYABLE_FOLDERS:
+        raise_api_error(400, "bad_folder", "this folder cannot be emptied")
+    await enqueue_job("empty_folder_job", str(user.id), special)
+    return {"status": "queued"}
+
+
 @router.post("/messages/{message_id}/move")
 async def move_message(
     message_id: uuid.UUID, body: MoveRequest, user: CurrentUser, db: DbSession

backend/app/api/admin.py

@@ -0,0 +1,400 @@
+"""Instance administration (SPEC §6.1).
+
+The backend seam for the admin panel: the registration gate — the first account
+on a fresh install becomes the administrator and closes signup behind itself,
+and this is how that gets reopened — the default quota, and managing users
+(search, per-user quota, block, delete). An administrator can change their own
+quota but cannot block or delete their own account, so the instance can never
+lock out its last admin that way.
+"""
+
+from __future__ import annotations
+
+import base64
+import datetime as dt
+import re
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Path, Query
+from pydantic import BaseModel
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import translation
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.db import get_session
+from app.hosted.admin import MailServerError
+from app.identity import invitations
+from app.identity.deps import CurrentAdmin, require_csrf
+from app.identity.mailer import (
+    LOGO_CID,
+    LOGO_DATA_URL,
+    LOGO_HTML,
+    MailerError,
+    mailer_config,
+    send_mail,
+)
+from app.identity.schemas import (
+    AdminInvitePreviewIn,
+    AdminInvitePreviewOut,
+    AdminInviteTemplateIn,
+    AdminInviteTemplateOut,
+    AdminMailIn,
+    AdminMailOut,
+    AdminMailTestIn,
+    AdminSettingsIn,
+    AdminSettingsOut,
+    AdminUserOut,
+    AdminUserPatch,
+    MessageOut,
+)
+from app.identity.service import app_settings
+from app.identity.user_admin import delete_user, search_users, set_blocked
+from app.models import Account, AppSettings, Message, User
+from app.security.crypto import encrypt_secret
+
+router = APIRouter(prefix="/admin", tags=["admin"])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+def _out(row: AppSettings) -> AdminSettingsOut:
+    return AdminSettingsOut(
+        registration_open=row.registration_open,
+        default_quota_bytes=row.default_quota_bytes,
+        invitations_enabled=row.invitations_enabled,
+    )
+
+
+@router.get("/settings", response_model=AdminSettingsOut)
+async def read_settings(admin: CurrentAdmin, db: DbSession) -> AdminSettingsOut:
+    return _out(await app_settings(db))
+
+
+@router.put("/settings", response_model=AdminSettingsOut, dependencies=[Depends(require_csrf)])
+async def write_settings(
+    body: AdminSettingsIn, admin: CurrentAdmin, db: DbSession
+) -> AdminSettingsOut:
+    """Merge the supplied fields into the settings row.
+
+    Changing ``default_quota_bytes`` affects users created afterwards only —
+    nothing here rewrites an existing ``users.quota_bytes``.
+    """
+    row = await app_settings(db, for_update=True)
+    if body.registration_open is not None:
+        row.registration_open = body.registration_open
+    if body.default_quota_bytes is not None:
+        row.default_quota_bytes = body.default_quota_bytes
+    if body.invitations_enabled is not None:
+        row.invitations_enabled = body.invitations_enabled
+    row.updated_at = dt.datetime.now(dt.UTC)
+    await db.commit()
+    return _out(row)
+
+
+# --- Outgoing system mail -------------------------------------------------------
+
+
+def _mail_out(row: AppSettings) -> AdminMailOut:
+    return AdminMailOut(
+        smtp_host=row.smtp_host,
+        smtp_port=row.smtp_port,
+        smtp_security=row.smtp_security,
+        smtp_username=row.smtp_username,
+        has_password=row.smtp_secret_ct is not None,
+        smtp_from=row.smtp_from,
+        public_url=row.public_url,
+        configured=bool(row.smtp_host and row.smtp_from and row.public_url),
+    )
+
+
+@router.get("/mail", response_model=AdminMailOut)
+async def read_mail(admin: CurrentAdmin, db: DbSession) -> AdminMailOut:
+    return _mail_out(await app_settings(db))
+
+
+@router.put("/mail", response_model=AdminMailOut, dependencies=[Depends(require_csrf)])
+async def write_mail(body: AdminMailIn, admin: CurrentAdmin, db: DbSession) -> AdminMailOut:
+    """Merge the supplied SMTP fields. The password is stored vault-encrypted."""
+    row = await app_settings(db, for_update=True)
+    for name in ("smtp_host", "smtp_username", "smtp_from"):
+        value = getattr(body, name)
+        if value is not None:
+            setattr(row, name, value.strip())
+    if body.smtp_port is not None:
+        row.smtp_port = body.smtp_port
+    if body.smtp_security is not None:
+        row.smtp_security = body.smtp_security
+    if body.smtp_password is not None:
+        row.smtp_secret_ct = (
+            encrypt_secret(body.smtp_password.encode()) if body.smtp_password else None
+        )
+    if body.public_url is not None:
+        url = body.public_url.strip().rstrip("/")
+        if url and not url.startswith(("http://", "https://")):
+            raise_api_error(400, "invalid_url", "the address must start with http:// or https://")
+        row.public_url = url
+    row.updated_at = dt.datetime.now(dt.UTC)
+    await db.commit()
+    return _mail_out(row)
+
+
+@router.post("/mail/test", response_model=MessageOut, dependencies=[Depends(require_csrf)])
+async def test_mail(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -> MessageOut:
+    """Send a test message with the saved settings, reporting the server's error."""
+    cfg = mailer_config(await app_settings(db))
+    if cfg is None:
+        raise_api_error(
+            409, "mail_not_configured", "set the SMTP server, From address and public address first"
+        )
+    text = (
+        "This is a test message from Ultimo Email.\n\n"
+        "Outgoing system mail works: password-reset links will be sent from this account.\n"
+    )
+    # Same logo header as the reset mail, so the test shows what users will get.
+    html = LOGO_HTML + f"<p>{text.replace(chr(10) * 2, '</p><p>').strip()}</p>"
+    try:
+        await send_mail(cfg, body.to.strip(), "Ultimo Email test message", text, html)
+    except MailerError as exc:
+        raise_api_error(502, "mail_failed", f"sending failed: {exc}")
+    return MessageOut()
+
+
+# --- Invitation email ------------------------------------------------------------
+
+_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+")
+_SAMPLE = {"invitee_name": "Anna Nowak", "inviter_name": "Jan Kowalski"}
+
+
+def _template_out(row: AppSettings) -> AdminInviteTemplateOut:
+    return AdminInviteTemplateOut(
+        subject=row.invite_subject,
+        html=row.invite_template,
+        from_address=row.invite_from,
+        default_subject=invitations.DEFAULT_SUBJECT,
+        default_html=invitations.DEFAULT_TEMPLATE,
+        default_from=row.smtp_from,
+        placeholders=list(invitations.PLACEHOLDERS),
+    )
+
+
+@router.get("/invitations/template", response_model=AdminInviteTemplateOut)
+async def read_invite_template(admin: CurrentAdmin, db: DbSession) -> AdminInviteTemplateOut:
+    return _template_out(await app_settings(db))
+
+
+@router.put(
+    "/invitations/template",
+    response_model=AdminInviteTemplateOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def write_invite_template(
+    body: AdminInviteTemplateIn, admin: CurrentAdmin, db: DbSession
+) -> AdminInviteTemplateOut:
+    """Upload the invitation email's HTML template, subject and sender."""
+    row = await app_settings(db, for_update=True)
+    if body.html is not None:
+        html = body.html.strip()
+        if html and (problem := invitations.template_problem(html)):
+            raise_api_error(400, "invalid_template", problem)
+        row.invite_template = html
+    if body.subject is not None:
+        row.invite_subject = " ".join(body.subject.split())
+    if body.from_address is not None:
+        sender = body.from_address.strip()
+        if sender and not _EMAIL_RE.fullmatch(sender):
+            raise_api_error(400, "invalid_email", "the sender must be an email address")
+        row.invite_from = sender
+    row.updated_at = dt.datetime.now(dt.UTC)
+    await db.commit()
+    return _template_out(row)
+
+
+def _sample(row: AppSettings, subject: str, html: str) -> invitations.RenderedMail:
+    """The invitation as a friend would get it, with sample names."""
+    if html.strip() and (problem := invitations.template_problem(html)):
+        raise_api_error(400, "invalid_template", problem)
+    draft = AppSettings(invite_subject=subject, invite_template=html.strip())
+    base = row.public_url.rstrip("/") or f"https://{get_settings().app_domain}"
+    return invitations.render(draft, link=f"{base}/register#invite=sample", **_SAMPLE)
+
+
+@router.post(
+    "/invitations/preview",
+    response_model=AdminInvitePreviewOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def preview_invite(
+    body: AdminInvitePreviewIn, admin: CurrentAdmin, db: DbSession
+) -> AdminInvitePreviewOut:
+    """Render a template (saved or not) with sample names, for the panel to show."""
+    mail = _sample(await app_settings(db), body.subject, body.html)
+    html = mail.html
+    # The preview frame has no message parts: show the images it would carry.
+    for part in mail.inline:
+        data = base64.b64encode(part.data).decode()
+        html = html.replace(f"cid:{part.cid}", f"data:{part.mime_type};base64,{data}")
+    html = html.replace(f"cid:{LOGO_CID}", LOGO_DATA_URL)
+    return AdminInvitePreviewOut(subject=mail.subject, html=html)
+
+
+@router.post("/invitations/test", response_model=MessageOut, dependencies=[Depends(require_csrf)])
+async def test_invite(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -> MessageOut:
+    """Send the saved invitation email, with sample names and a dummy link."""
+    row = await app_settings(db)
+    cfg = mailer_config(row)
+    if cfg is None:
+        raise_api_error(
+            409, "mail_not_configured", "set the SMTP server, From address and public address first"
+        )
+    mail = _sample(row, row.invite_subject, row.invite_template)
+    try:
+        await send_mail(
+            cfg,
+            body.to.strip(),
+            mail.subject,
+            mail.text,
+            mail.html,
+            sender=invitations.sender_address(row),
+            inline=mail.inline,
+        )
+    except MailerError as exc:
+        raise_api_error(502, "mail_failed", f"sending failed: {exc}")
+    return MessageOut()
+
+
+# --- Users --------------------------------------------------------------------
+
+
+def _user_out(user: User, used_bytes: int, accounts: list[str]) -> AdminUserOut:
+    return AdminUserOut(
+        id=user.id,
+        handle=user.handle,
+        is_admin=user.is_admin,
+        blocked_at=user.blocked_at.isoformat() if user.blocked_at else None,
+        quota_bytes=user.quota_bytes,
+        used_bytes=used_bytes,
+        created_at=user.created_at.isoformat(),
+        accounts=accounts,
+    )
+
+
+async def _target(
+    db: AsyncSession, admin: User, user_id: uuid.UUID, *, allow_self: bool = False
+) -> User:
+    """The user an admin action is aimed at. Blocking and deleting are never
+    allowed on the admin themselves; a quota change is (``allow_self``)."""
+    user = await db.scalar(select(User).where(User.id == user_id))
+    if user is None:
+        raise_api_error(404, "not_found", "user not found")
+    if user.id == admin.id and not allow_self:
+        raise_api_error(409, "cannot_modify_self", "you cannot do this to your own account")
+    return user
+
+
+@router.get("/users", response_model=list[AdminUserOut])
+async def list_users(
+    admin: CurrentAdmin, db: DbSession, q: str = Query("", max_length=200)
+) -> list[AdminUserOut]:
+    """Search users by login, or by a mail account's address or display name."""
+    return [_user_out(u, used, emails) for u, used, emails in await search_users(db, q)]
+
+
+@router.patch("/users/{user_id}", response_model=AdminUserOut, dependencies=[Depends(require_csrf)])
+async def patch_user(
+    user_id: uuid.UUID, body: AdminUserPatch, admin: CurrentAdmin, db: DbSession
+) -> AdminUserOut:
+    """Change a user's quota and/or block or unblock them."""
+    # Your own quota is fine to change; blocking yourself is not.
+    user = await _target(db, admin, user_id, allow_self=body.blocked is None)
+    if body.quota_bytes is not None:
+        user.quota_bytes = body.quota_bytes
+    if body.blocked is not None:
+        try:
+            await set_blocked(db, user, body.blocked)
+        except MailServerError as exc:
+            raise_api_error(
+                502, "mail_server_unavailable", f"the mail server did not answer: {exc}"
+            )
+    await db.commit()
+    used = await db.scalar(
+        select(func.coalesce(func.sum(Message.size_bytes), 0)).where(Message.user_id == user.id)
+    )
+    emails = list(
+        await db.scalars(
+            select(Account.email).where(Account.user_id == user.id).order_by(Account.email)
+        )
+    )
+    return _user_out(user, int(used or 0), emails)
+
+
+@router.delete("/users/{user_id}", dependencies=[Depends(require_csrf)])
+async def remove_user(user_id: uuid.UUID, admin: CurrentAdmin, db: DbSession) -> dict[str, str]:
+    """Delete a user and all of their data. Cannot be undone."""
+    user = await _target(db, admin, user_id)
+    try:
+        await delete_user(db, user)
+    except MailServerError as exc:
+        raise_api_error(502, "mail_server_unavailable", f"the mail server did not answer: {exc}")
+    return {"status": "deleted"}
+
+
+# --- email translation languages (translator service) ------------------------------
+
+
+class InstallProgressOut(BaseModel):
+    done: int
+    total: int
+
+
+class AdminTranslationOut(BaseModel):
+    enabled: bool  # TRANSLATOR_URL is set
+    reachable: bool
+    installed: list[str] = []
+    installing: dict[str, InstallProgressOut] = {}
+    pending: list[str] = []  # waiting to be tried again (no network last time)
+    available: list[str] = []  # what Mozilla has models for (empty: offline)
+    errors: dict[str, str] = {}  # why a language's last install failed
+
+
+_LANG = Path(pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$")
+
+
+@router.get("/translation", response_model=AdminTranslationOut)
+async def translation_status(admin: CurrentAdmin) -> AdminTranslationOut:
+    if not translation.enabled():
+        return AdminTranslationOut(enabled=False, reachable=False)
+    try:
+        data = await translation.status()
+    except translation.TranslationError:
+        return AdminTranslationOut(enabled=True, reachable=False)
+    return AdminTranslationOut(enabled=True, reachable=True, **data)
+
+
+@router.post(
+    "/translation/languages/{code}",
+    response_model=AdminTranslationOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def install_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
+    """Download a language's models (to and from English) in the background."""
+    try:
+        await translation.install(code)
+    except translation.TranslationError as exc:
+        raise_api_error(502, "translator_failed", str(exc))
+    return await translation_status(admin)
+
+
+@router.delete(
+    "/translation/languages/{code}",
+    response_model=AdminTranslationOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def remove_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
+    try:
+        await translation.remove(code)
+    except translation.TranslationError as exc:
+        raise_api_error(502, "translator_failed", str(exc))
+    return await translation_status(admin)

backend/app/api/avatar.py

@@ -0,0 +1,97 @@
+"""Sender avatars — Gravatar then the sender domain's favicon (SPEC §9.1).
+
+The browser only ever calls ``GET /api/avatar`` (same origin), so a sender can never
+see the user's IP or read activity: the server does the outbound fetch under the
+§10.6 SSRF guard and caches the result (both hits and misses) in Redis. A miss
+returns 204 No Content: the <img> fails to decode it and the UI falls back to its
+initials, without the browser logging an error for every sender who has no avatar
+(a 404 did, filling the console). Disable with ``SENDER_AVATARS_ENABLED=false``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+import httpx
+from fastapi import APIRouter, Query, Response
+
+from app.accounts.ssrf import SsrfError, guarded_request
+from app.config import get_settings
+from app.identity.deps import CurrentUser
+from app.redis_client import get_redis_bytes
+
+router = APIRouter(prefix="/avatar", tags=["mail"])
+
+_CACHE_TTL_HIT = 7 * 24 * 3600  # remember a found avatar for a week
+_CACHE_TTL_MISS = 24 * 3600  # re-check "no avatar" daily
+_MAX_BYTES = 200 * 1024
+_NONE = b"none"
+# The browser keeps an answer as long as we would, so it stops re-asking per page.
+_BROWSER_CACHE = {"Cache-Control": "private, max-age=86400"}
+
+
+def _no_avatar() -> Response:
+    return Response(status_code=204, headers=_BROWSER_CACHE)
+
+
+def _email_md5(email: str) -> str:
+    # Gravatar keys on md5(lowercased email); reused as our cache key. Not security.
+    return hashlib.md5(email.encode()).hexdigest()  # noqa: S324
+
+
+async def _fetch_image(url: str) -> tuple[str, bytes] | None:
+    """GET ``url`` under the SSRF guard; return (content_type, bytes) for a 200 image."""
+    try:
+        resp = await guarded_request("GET", url, timeout=5.0, max_bytes=_MAX_BYTES)
+    except (SsrfError, httpx.HTTPError, OSError):
+        return None
+    if resp.status != 200 or not resp.content:
+        return None
+    ctype = resp.headers.get("content-type", "").split(";")[0].strip().lower()
+    if not ctype.startswith("image/"):
+        return None
+    return ctype, resp.content
+
+
+async def _resolve(email: str, md5: str) -> tuple[str, bytes] | None:
+    """Gravatar first (``d=404`` so it 404s when the user has none), then the sender
+    domain's ``/favicon.ico`` as a brand-ish fallback."""
+    gravatar = await _fetch_image(f"https://www.gravatar.com/avatar/{md5}?d=404&s=80")
+    if gravatar is not None:
+        return gravatar
+    domain = email.rsplit("@", 1)[-1]
+    if domain:
+        favicon = await _fetch_image(f"https://{domain}/favicon.ico")
+        if favicon is not None:
+            return favicon
+    return None
+
+
+@router.get("")
+async def sender_avatar(_user: CurrentUser, email: str = Query(...)) -> Response:
+    email = email.strip().lower()
+    if not get_settings().sender_avatars_enabled or "@" not in email:
+        return _no_avatar()
+
+    # Binary-safe client: the cache holds raw image bytes (and a b"none" marker).
+    redis = get_redis_bytes()
+    md5 = _email_md5(email)
+    key = f"avatar:{md5}"
+    cached: bytes | None = await redis.get(key)
+    if cached == _NONE:
+        return _no_avatar()
+    if cached:
+        cached_ctype, _, cached_body = cached.partition(b"\n")
+        return Response(
+            content=cached_body,
+            media_type=cached_ctype.decode("ascii", "ignore"),
+            headers=_BROWSER_CACHE,
+        )
+
+    result = await _resolve(email, md5)
+    if result is None:
+        await redis.set(key, _NONE, ex=_CACHE_TTL_MISS)
+        return _no_avatar()
+    ctype, body = result
+    await redis.set(key, ctype.encode("ascii", "ignore") + b"\n" + body, ex=_CACHE_TTL_HIT)
+    return Response(content=body, media_type=ctype, headers=_BROWSER_CACHE)

backend/app/api/completion.py

@@ -0,0 +1,44 @@
+"""Sentence completion for the compose window (app/completion.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import completion
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import CompletionIndexState
+from app.queue import enqueue_job
+
+router = APIRouter(prefix="/completion", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class CompletionIn(BaseModel):
+    # The text before the cursor (its last line is all that matters). A POST,
+    # not a query string: what the user is typing stays out of access logs.
+    before: str = Field(max_length=2000)
+
+
+class CompletionOut(BaseModel):
+    suggestion: str | None
+
+
+@router.post("", response_model=CompletionOut)
+async def suggest(body: CompletionIn, user: CurrentUser, db: DbSession) -> CompletionOut:
+    before = body.before[-300:]
+    built = await db.scalar(
+        select(CompletionIndexState.built_at).where(CompletionIndexState.user_id == user.id)
+    )
+    if built is None or built < dt.datetime.now(dt.UTC) - completion.REBUILD_AFTER:
+        # In the background; until it is built, the starter phrases still answer.
+        await enqueue_job("build_index_job", str(user.id), _job_id=f"complete:{user.id}")
+    personal = await completion.personal_phrases(db, user.id, completion.keys_for(before))
+    return CompletionOut(suggestion=completion.complete(personal, before))

backend/app/api/compose.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import uuid
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, File, Query, UploadFile
@@ -15,16 +17,26 @@ from app.api.schemas import AttachmentOut
 from app.config import get_settings
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
-from app.indexer import blobstore
-from app.models import BodyPart, ComposeAttachment, Draft, Mailbox, Message
+from app.indexer import attachments as att_store
+from app.indexer import blobstore, rawstore
+from app.models import Attachment, BodyPart, ComposeAttachment, Draft, Mailbox, Message, Outbox
+from app.outbox.jobs import RETRY_WINDOW
 from app.outbox.schemas import (
     DraftIn,
     DraftOut,
+    OutboxItemOut,
     RemoteDraftOut,
     SendRequest,
     SendResponse,
 )
-from app.outbox.service import DraftError, cancel_outbox, create_outbox, upsert_draft
+from app.outbox.service import (
+    DraftError,
+    cancel_outbox,
+    create_outbox,
+    delete_mirrored_draft_row,
+    not_being_sent,
+    upsert_draft,
+)
 from app.queue import enqueue_job
 
 router = APIRouter(tags=["compose"], dependencies=[Depends(require_csrf)])
@@ -103,7 +115,9 @@ def _draft_out(draft: Draft) -> DraftOut:
 @router.get("/drafts", response_model=list[DraftOut])
 async def list_drafts(user: CurrentUser, db: DbSession) -> list[DraftOut]:
     drafts = await db.scalars(
-        select(Draft).where(Draft.user_id == user.id).order_by(Draft.updated_at.desc())
+        select(Draft)
+        .where(Draft.user_id == user.id, not_being_sent())
+        .order_by(Draft.updated_at.desc())
     )
     return [_draft_out(d) for d in drafts]
 
@@ -183,6 +197,38 @@ async def adopt_remote_draft(message_id: uuid.UUID, user: CurrentUser, db: DbSes
     parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == msg.id)))
     html = next((p.content for p in parts if p.kind == "html_sanitized"), "")
     text = next((p.content for p in parts if p.kind == "text"), "")
+
+    # Carry the draft's real (non-inline) attachments over as compose attachments,
+    # so they show in the editor and are re-attached on save/send. The underlying
+    # blobs are shared by sha256 — we only add new ComposeAttachment references.
+    att_rows = list(
+        await db.scalars(
+            select(Attachment)
+            .where(Attachment.message_id == msg.id, Attachment.content_id.is_(None))
+            .order_by(Attachment.filename)
+        )
+    )
+    attachment_ids: list[str] = []
+    if att_rows:
+        with contextlib.suppress(FileNotFoundError):  # fetched if a hosted mailbox's
+            await rawstore.ensure(msg)
+    for a in att_rows:
+        # Kept only inside the message: give the compose attachment a blob of its own.
+        if not await asyncio.to_thread(
+            att_store.materialize, user.id, msg.blob_sha256, a.blob_sha256, a.part_index
+        ):
+            continue
+        ca = ComposeAttachment(
+            user_id=user.id,
+            filename=a.filename,
+            mime_type=a.mime_type,
+            size_bytes=a.size_bytes,
+            blob_sha256=a.blob_sha256,
+        )
+        db.add(ca)
+        await db.flush()
+        attachment_ids.append(str(ca.id))
+
     payload = {
         "to": [_fmt_addr(a) for a in msg.to_addr],
         "cc": [_fmt_addr(a) for a in msg.cc_addr],
@@ -190,7 +236,7 @@ async def adopt_remote_draft(message_id: uuid.UUID, user: CurrentUser, db: DbSes
         "subject": msg.subject,
         "html": html,
         "text": text,
-        "attachment_ids": [],
+        "attachment_ids": attachment_ids,
     }
     draft = Draft(
         user_id=user.id,
@@ -244,6 +290,10 @@ async def delete_draft(draft_id: uuid.UUID, user: CurrentUser, db: DbSession) ->
         raise_api_error(404, "not_found", "draft not found")
     account_id, imap_uid = draft.account_id, draft.imap_uid
     await db.delete(draft)
+    # Drop the mirrored Drafts-folder message row now so the draft doesn't
+    # resurface as an "other client" remote draft after the local row is gone.
+    if imap_uid:
+        await delete_mirrored_draft_row(db, account_id=account_id, imap_uid=imap_uid)
     await db.commit()
     # Remove the mirrored copy from the server Drafts folder too.
     if imap_uid:
@@ -266,7 +316,10 @@ async def send(body: SendRequest, user: CurrentUser, db: DbSession) -> SendRespo
         str(outbox.id),
         _defer_by=timedelta(seconds=get_settings().undo_seconds),
     )
-    return SendResponse(outbox_id=outbox.id, undo_until=outbox.send_after.isoformat())
+    left = max(0.0, (outbox.send_after - datetime.now(UTC)).total_seconds())
+    return SendResponse(
+        outbox_id=outbox.id, undo_until=outbox.send_after.isoformat(), undo_seconds=left
+    )
 
 
 @router.post("/send/{outbox_id}/undo")
@@ -276,3 +329,72 @@ async def undo_send(outbox_id: uuid.UUID, user: CurrentUser, db: DbSession) -> d
     if not ok:
         raise_api_error(409, "undo_too_late", "the message has already left the undo window")
     return {"status": "cancelled"}
+
+
+# --- the Scheduled folder's server side: sends being retried, or failed ------------
+
+
+@router.get("/outbox", response_model=list[OutboxItemOut])
+async def list_outbox(user: CurrentUser, db: DbSession) -> list[OutboxItemOut]:
+    """Sends that did not go out: retrying (the server could not take them yet)
+    or failed for good. They wait in Scheduled, not in Drafts."""
+    rows = await db.execute(
+        select(Outbox, Draft)
+        .join(Draft, Draft.id == Outbox.draft_id)
+        .where(Outbox.user_id == user.id, Outbox.state.in_(("retrying", "failed")))
+        .order_by(Outbox.send_after.desc())
+    )
+    out: list[OutboxItemOut] = []
+    for ob, draft in rows.tuples():
+        payload = draft.payload or {}
+        to = payload.get("to")
+        out.append(
+            OutboxItemOut(
+                outbox_id=ob.id,
+                draft_id=draft.id,
+                account_id=draft.account_id,
+                state=ob.state,
+                attempts=ob.attempts,
+                error=ob.error,
+                next_attempt_at=ob.next_attempt_at.isoformat() if ob.next_attempt_at else None,
+                retry_until=ob.retry_until.isoformat() if ob.retry_until else None,
+                sent_at=ob.send_after.isoformat(),
+                to=[str(t) for t in to] if isinstance(to, list) else [],
+                subject=str(payload.get("subject") or ""),
+                preview=str(payload.get("text") or "")[:200],
+            )
+        )
+    return out
+
+
+async def _waiting_send(db: AsyncSession, user_id: uuid.UUID, outbox_id: uuid.UUID) -> Outbox:
+    ob = await db.scalar(select(Outbox).where(Outbox.id == outbox_id, Outbox.user_id == user_id))
+    if ob is None or ob.state not in ("retrying", "failed"):
+        raise_api_error(409, "not_waiting", "this message is not waiting to be sent")
+    return ob
+
+
+@router.post("/outbox/{outbox_id}/retry")
+async def retry_outbox(outbox_id: uuid.UUID, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    """Retry now. A send that had failed for good gets a fresh 24-hour window."""
+    ob = await _waiting_send(db, user.id, outbox_id)
+    now = datetime.now(UTC)
+    if ob.state == "failed" or ob.retry_until is None or ob.retry_until <= now:
+        ob.retry_until = now + RETRY_WINDOW
+    ob.state = "retrying"
+    ob.next_attempt_at = now
+    await db.commit()
+    await enqueue_job("send_job", str(ob.id), _job_id=f"send:{ob.id}:now:{ob.attempts}")
+    return {"status": "retrying"}
+
+
+@router.post("/outbox/{outbox_id}/cancel")
+async def cancel_waiting_send(
+    outbox_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    """Stop trying: the message goes back to Drafts (Edit, or before Delete)."""
+    ob = await _waiting_send(db, user.id, outbox_id)
+    ob.state = "cancelled"
+    ob.next_attempt_at = None
+    await db.commit()
+    return {"status": "cancelled", "draft_id": str(ob.draft_id)}

backend/app/api/errors.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from typing import NoReturn
 
 from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
@@ -29,7 +30,23 @@ async def _handler(request: Request, exc: StarletteHTTPException) -> JSONRespons
     return JSONResponse(status_code=exc.status_code, content=body, headers=headers)
 
 
+async def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+    """A malformed request, in the usual error shape. Names the first field at
+    fault and what is wrong with it — enough to show the user — but not the
+    whole request model the framework's own answer would spell out."""
+    errors = exc.errors()
+    first = errors[0] if errors else {}
+    loc = [str(p) for p in first.get("loc", ()) if p not in ("body", "query", "path")]
+    field = loc[-1] if loc else ""
+    reason = str(first.get("msg", "invalid value"))
+    message = f"{field}: {reason}" if field else reason
+    return JSONResponse(
+        status_code=422, content={"error": {"code": "validation_error", "message": message}}
+    )
+
+
 def register_error_handlers(app: FastAPI) -> None:
     """Install the error-shape handler for all HTTP exceptions."""
     app.add_exception_handler(StarletteHTTPException, _handler)  # type: ignore[arg-type]
     app.add_exception_handler(HTTPException, _handler)  # type: ignore[arg-type]
+    app.add_exception_handler(RequestValidationError, _validation_handler)  # type: ignore[arg-type]

backend/app/api/invitations.py

@@ -0,0 +1,123 @@
+"""Invite a friend (app/identity/invitations.py): what the sidebar entry and its
+dialog need — how many invitations are left, and sending one."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity import invitations
+from app.identity.deps import CurrentUser, require_csrf
+from app.identity.mailer import MailerError, mailer_config, send_mail
+from app.identity.service import app_settings
+from app.models import Invitation
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/invitations", tags=["invitations"])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+")
+
+
+class SentInvitationOut(BaseModel):
+    invitee_name: str
+    invitee_email: str
+    created_at: str
+    used: bool
+    expired: bool
+
+
+class InvitationsOut(BaseModel):
+    # Invitations are switched on and system mail can send them.
+    available: bool
+    remaining: int
+    per_user: int
+    sent: list[SentInvitationOut]
+
+
+class InvitationIn(BaseModel):
+    name: str = Field(min_length=1, max_length=100)
+    email: str = Field(min_length=3, max_length=320)
+
+
+@router.get("", response_model=InvitationsOut)
+async def read(user: CurrentUser, db: DbSession) -> InvitationsOut:
+    row = await app_settings(db)
+    now = dt.datetime.now(dt.UTC)
+    return InvitationsOut(
+        available=row.invitations_enabled and mailer_config(row) is not None,
+        remaining=await invitations.remaining(db, user.id),
+        per_user=invitations.PER_USER,
+        sent=[
+            SentInvitationOut(
+                invitee_name=i.invitee_name,
+                invitee_email=i.invitee_email,
+                created_at=i.created_at.isoformat(),
+                used=i.used_at is not None,
+                expired=i.used_at is None and i.expires_at <= now,
+            )
+            for i in await invitations.sent(db, user.id)
+        ],
+    )
+
+
+@router.post("", response_model=InvitationsOut, dependencies=[Depends(require_csrf)])
+async def send(body: InvitationIn, user: CurrentUser, db: DbSession) -> InvitationsOut:
+    """Email a friend a single-use sign-up link. Sent before answering, so the
+    user hears at once if it could not go out (and keeps the invitation)."""
+    name = " ".join(body.name.split())
+    email = body.email.strip()
+    if not name:
+        raise_api_error(400, "invalid_name", "enter your friend's name")
+    if not _EMAIL_RE.fullmatch(email):
+        raise_api_error(400, "invalid_email", "enter a valid email address")
+    row = await app_settings(db)
+    if not row.invitations_enabled:
+        raise_api_error(403, "invitations_disabled", "invitations are switched off")
+    cfg = mailer_config(row)
+    if cfg is None:
+        raise_api_error(503, "mail_not_configured", "the server cannot send email yet")
+    try:
+        invitation, token = await invitations.create(db, user, name=name, email=email)
+    except invitations.InvitationError as exc:
+        raise_api_error(409, exc.code, exc.message)
+    await db.commit()
+
+    # The token rides in the fragment: never sent to a server, so it stays out
+    # of access logs and Referer headers (as with password-reset links).
+    link = f"{cfg.public_url}/register#invite={token}"
+    mail = invitations.render(
+        row,
+        invitee_name=name,
+        inviter_name=await invitations.inviter_name(db, user),
+        link=link,
+    )
+    try:
+        await send_mail(
+            cfg,
+            email,
+            mail.subject,
+            mail.text,
+            mail.html,
+            sender=invitations.sender_address(row),
+            reply_to=user.handle,  # an answer reaches the friend who invited
+            inline=mail.inline,
+        )
+    except MailerError as exc:
+        logger.warning("invitation from %s could not be sent: %s", user.handle, exc)
+        # Not sent: the user keeps the invitation.
+        await db.execute(delete(Invitation).where(Invitation.id == invitation.id))
+        await db.commit()
+        raise_api_error(502, "mail_failed", "the invitation could not be sent — try again later")
+    return await read(user, db)

backend/app/api/mailboxes.py

@@ -7,12 +7,13 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Mailbox
+from app.models import Draft, Mailbox, Message, Thread
+from app.outbox.service import not_being_sent
 
 router = APIRouter(prefix="/mailboxes", tags=["mail"])
 
@@ -26,6 +27,12 @@ class MailboxOut(BaseModel):
     special_use: str | None
 
 
+class FolderCounts(BaseModel):
+    inbox_unread: int  # unread messages sitting in Inbox mailboxes
+    drafts: int  # drafts shown in the Drafts folder (local + un-adopted remote)
+    snoozed: int = 0  # conversations snoozed out of the Inbox
+
+
 @router.get("", response_model=list[MailboxOut])
 async def list_mailboxes(user: CurrentUser, db: DbSession) -> list[MailboxOut]:
     rows = await db.scalars(
@@ -35,3 +42,53 @@ async def list_mailboxes(user: CurrentUser, db: DbSession) -> list[MailboxOut]:
         MailboxOut(id=m.id, account_id=m.account_id, path=m.path, special_use=m.special_use)
         for m in rows
     ]
+
+
+@router.get("/counts", response_model=FolderCounts)
+async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
+    """Badge counts for the sidebar: unread Inbox messages and Drafts-folder items."""
+    inbox_unread = await db.scalar(
+        select(func.count())
+        .select_from(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.user_id == user.id,
+            Mailbox.special_use == "inbox",
+            ~Message.flags.contains(["\\Seen"]),
+            # A snoozed conversation is out of the Inbox until it wakes.
+            ~Message.thread_id.in_(
+                select(Thread.id).where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+            ),
+        )
+    )
+    snoozed = await db.scalar(
+        select(func.count())
+        .select_from(Thread)
+        .where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+    )
+
+    # Drafts folder = local drafts + IMAP-Drafts messages we haven't adopted (mirror
+    # UIDs excluded), matching what list_drafts + list_remote_drafts return.
+    local = await db.scalar(
+        select(func.count()).select_from(Draft).where(Draft.user_id == user.id, not_being_sent())
+    )
+    owned = {
+        (r.account_id, r.imap_uid)
+        for r in await db.execute(
+            select(Draft.account_id, Draft.imap_uid).where(
+                Draft.user_id == user.id, Draft.imap_uid.isnot(None)
+            )
+        )
+    }
+    remote_rows = await db.execute(
+        select(Message.account_id, Message.uid)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(Message.user_id == user.id, Mailbox.special_use == "drafts")
+    )
+    remote = sum(1 for r in remote_rows if (r.account_id, r.uid) not in owned)
+
+    return FolderCounts(
+        inbox_unread=int(inbox_unread or 0),
+        drafts=int(local or 0) + remote,
+        snoozed=int(snoozed or 0),
+    )

backend/app/api/messages.py

@@ -2,26 +2,50 @@
 
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import html as html_lib
 import io
+import logging
 import uuid
 import zipfile
 from typing import Annotated
+from urllib.parse import quote
 
 from fastapi import APIRouter, Depends, Query
-from fastapi.responses import HTMLResponse, Response
+from fastapi.responses import FileResponse, HTMLResponse, Response
 from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import translation
 from app.api.errors import raise_api_error
 from app.api.schemas import AttachmentOut
-from app.api.settings import add_allowed_image_sender, allowed_image_senders
+from app.api.settings import (
+    add_allowed_image_sender,
+    allowed_image_senders,
+    blocks_trackers,
+)
+from app.completion import html_to_text
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
-from app.indexer import blobstore
+from app.indexer import attachments as att_store
+from app.indexer import rawstore, thumbs
 from app.indexer.mime import parse_message
 from app.indexer.sanitize import SANITIZER_VERSION, sanitize_html
-from app.models import Attachment, BodyPart, Message
+from app.indexer.thumbs import thumb_type
+from app.models import (
+    Account,
+    Attachment,
+    BodyPart,
+    ComposeAttachment,
+    Message,
+    MessageTranslation,
+    User,
+)
+from app.translation_prep import find_names_in_signature
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/messages", tags=["mail"])
 
@@ -30,12 +54,40 @@ DbSession = Annotated[AsyncSession, Depends(get_session)]
 # Content types safe to serve inline (SPEC §10.8); everything else is forced to
 # an attachment download with a neutral type.
 _INLINE_SAFE = {"image/png", "image/jpeg", "image/gif", "image/webp", "application/pdf"}
+# Videos the browser can play, by extension — senders often label them
+# application/octet-stream. A QuickTime .mov is usually H.264 in an MP4-style
+# container: offered as video/mp4, browsers play it (an HEVC one stays unplayable,
+# and the preview offers a download instead). Media cannot run script: safe inline.
+_VIDEO_EXT_MIME = {
+    "mp4": "video/mp4",
+    "m4v": "video/mp4",
+    "mov": "video/mp4",
+    "webm": "video/webm",
+    "ogv": "video/ogg",
+}
+
+
+def _inline_type(mime_type: str, filename: str) -> str | None:
+    """The type to serve an attachment as for in-app viewing, or None."""
+    if mime_type in _INLINE_SAFE:
+        return mime_type
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    return _VIDEO_EXT_MIME.get(ext)
+
 
 # Base styles injected around the sanitized fragment so email content renders on a
 # readable light canvas with images that fit the pane (the iframe is sandboxed).
 _DOC_TEMPLATE = """<!doctype html><html><head><meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <style>
+  /* Self-hosted Lato so plain-text bodies use the app font (the sandboxed iframe
+     can't inherit the parent's web fonts). Served from the frontend's /fonts. */
+  @font-face{{font-family:"Lato";src:url("/fonts/Lato-Regular.ttf") format("truetype");
+    font-weight:400;font-style:normal;font-display:swap}}
+  @font-face{{font-family:"Lato";src:url("/fonts/Lato-Bold.ttf") format("truetype");
+    font-weight:700;font-style:normal;font-display:swap}}
+  @font-face{{font-family:"Lato";src:url("/fonts/Lato-Italic.ttf") format("truetype");
+    font-weight:400;font-style:italic;font-display:swap}}
   html,body{{margin:0;padding:0}}
   body{{font-family:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;font-size:14px;
     line-height:1.5;color:#1a1a1a;background:#fff;padding:14px;word-break:break-word}}
@@ -44,6 +96,10 @@ _DOC_TEMPLATE = """<!doctype html><html><head><meta charset="utf-8">
   blockquote{{margin:0 0 0 12px;padding-left:12px;border-left:3px solid #e0e0e0;color:#555}}
   a{{color:#2563eb}}
   pre{{white-space:pre-wrap;word-wrap:break-word}}
+  /* Plain-text emails: proportional Lato instead of the monospace <pre> default. */
+  .wm-plaintext{{white-space:pre-wrap;word-wrap:break-word;
+    font-family:"Lato",system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;
+    font-size:15px;line-height:1.55}}
 </style></head><body>{body}</body></html>"""
 
 
@@ -51,6 +107,31 @@ def _wrap(fragment: str) -> str:
     return _DOC_TEMPLATE.format(body=fragment)
 
 
+# The sanitized message as a document of its own. The app shows it in a
+# sandboxed frame (frontend MessageBody), but this URL can also be opened
+# directly in a tab, on our own origin: there, this policy is what keeps any
+# script the sanitizer might ever miss from running — no scripts, no plugins,
+# no forms, a sandbox of its own origin, and nothing framing it elsewhere.
+_MESSAGE_CSP = (
+    "default-src 'none'; script-src 'none'; object-src 'none'; base-uri 'none'; "
+    "form-action 'none'; frame-ancestors 'self'; img-src 'self' data: https: http:; "
+    "style-src 'self' 'unsafe-inline'; font-src 'self' data: https:; media-src 'self'; "
+    "sandbox allow-popups allow-popups-to-escape-sandbox"
+)
+
+
+def _html(document: str, trackers: int = 0, extra: dict[str, str] | None = None) -> HTMLResponse:
+    # The page reads what it needs from headers: the body is the message.
+    return HTMLResponse(
+        content=document,
+        headers={
+            "Content-Security-Policy": _MESSAGE_CSP,
+            "X-Trackers-Blocked": str(trackers),
+            **(extra or {}),
+        },
+    )
+
+
 async def _owned_message(db: AsyncSession, user_id: uuid.UUID, message_id: uuid.UUID) -> Message:
     msg = await db.scalar(
         select(Message).where(Message.id == message_id, Message.user_id == user_id)
@@ -64,47 +145,167 @@ def _sender_email(message: Message) -> str:
     return message.from_addr[0].get("email", "").lower() if message.from_addr else ""
 
 
-@router.get("/{message_id}/html", response_class=HTMLResponse)
-async def message_html(message_id: uuid.UUID, user: CurrentUser, db: DbSession) -> HTMLResponse:
-    """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
+async def _render_fragment(
+    db: AsyncSession, message: Message, allow_remote: bool, block: bool
+) -> tuple[str, int]:
+    """The sanitized body the user sees, and how many trackers came out of it.
 
-    Re-sanitizes from the raw blob when the sender is image-approved, or when the
-    stored copy predates the current sanitizer version (SPEC §10.4).
+    Re-sanitizes from the raw blob when remote images are allowed, the stored
+    copy predates the current sanitizer (SPEC §10.4), or trackers are to stay
+    in; otherwise the stored copy is served as is.
     """
-    message = await _owned_message(db, user.id, message_id)
-    allow_remote = _sender_email(message) in allowed_image_senders(user)
-
-    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
+    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message.id)))
     html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
 
     stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
-    if html_part is not None and not allow_remote and not stale:
-        return HTMLResponse(content=_wrap(html_part.content))
+    # The stored copy has its trackers taken out; with blocking switched off, a
+    # message that had some is rendered afresh with them left in.
+    trackers_kept = not block and html_part is not None and html_part.trackers_blocked > 0
+    if html_part is not None and not allow_remote and not stale and not trackers_kept:
+        return html_part.content, html_part.trackers_blocked if block else 0
 
-    # Re-sanitize the original HTML (image-approved sender, or a stale stored copy).
-    # If the raw blob is unavailable, fall back to whatever is already stored.
-    if allow_remote or stale:
+    raw = None
+    if html_part is not None or allow_remote:  # a plain-text message has nothing to redo
         try:
-            raw = blobstore.load(user.id, message.blob_sha256)
+            raw = await rawstore.ensure_bytes(message)
         except FileNotFoundError:
             raw = None
-        if raw is not None:
-            parsed = parse_message(raw)
-            if parsed.html_body is not None:
-                result = sanitize_html(
-                    parsed.html_body, message.id, allow_remote_images=allow_remote
-                )
-                if html_part is not None and not allow_remote:
-                    html_part.content = result.html  # persist the upgraded sanitization
-                    html_part.sanitizer_v = result.sanitizer_v
+    parsed = parse_message(raw) if raw is not None else None
+    if parsed is not None and parsed.html_body is not None:
+        if stale and html_part is not None:
+            # The stored copy is always the plain one: no remote images, trackers out.
+            upgraded = sanitize_html(parsed.html_body, message.id)
+            html_part.content = upgraded.html
+            html_part.sanitizer_v = upgraded.sanitizer_v
+            html_part.trackers_blocked = upgraded.trackers_blocked
             await db.commit()
-                return HTMLResponse(content=_wrap(result.html))
+            if block and not allow_remote:
+                return upgraded.html, upgraded.trackers_blocked
+        result = sanitize_html(
+            parsed.html_body, message.id, allow_remote_images=allow_remote, block_trackers=block
+        )
+        return result.html, result.trackers_blocked
 
     if html_part is not None:
-        return HTMLResponse(content=_wrap(html_part.content))
+        return html_part.content, html_part.trackers_blocked
     text_part = next((p for p in parts if p.kind == "text"), None)
     body = html_lib.escape(text_part.content) if text_part else ""
-    return HTMLResponse(content=_wrap(f"<pre>{body}</pre>"))
+    return f'<div class="wm-plaintext">{body}</div>', 0
+
+
+async def _content_language(db: AsyncSession, message: Message) -> str:
+    """The message's language, found (and kept) the first time it is shown."""
+    if message.content_language is not None or not translation.enabled():
+        return message.content_language or ""
+    text = await db.scalar(
+        select(BodyPart.content).where(BodyPart.message_id == message.id, BodyPart.kind == "text")
+    )
+    try:
+        found = await translation.detect(text or message.snippet or "")
+    except translation.TranslationError as exc:
+        logger.info("language detection unavailable: %s", exc)
+        return ""  # not kept: tried again once the service is back
+    message.content_language = found
+    await db.commit()
+    return found
+
+
+@router.get("/{message_id}/html", response_class=HTMLResponse)
+async def message_html(
+    message_id: uuid.UUID,
+    user: CurrentUser,
+    db: DbSession,
+    images: bool = Query(False),
+    translate: str | None = Query(None, pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$"),
+) -> HTMLResponse:
+    """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
+
+    ``images=1`` is a one-time "Load images" — render remote images for this
+    request only, without persisting the sender as trusted. ``translate=pl``
+    returns it translated (app/translation.py). Headers tell the page what it
+    got: X-Trackers-Blocked, X-Content-Language (the message's own language,
+    when known) and, translated, X-Translated-From / X-Translated-To.
+    """
+    message = await _owned_message(db, user.id, message_id)
+    allow_remote = images or _sender_email(message) in allowed_image_senders(user)
+    block = blocks_trackers(user)
+    fragment, trackers = await _render_fragment(db, message, allow_remote, block)
+
+    headers: dict[str, str] = {}
+    language = await _content_language(db, message)
+    if language:
+        headers["X-Content-Language"] = language
+    if translate:
+        fragment, source = await _translated(
+            db, user, message, fragment, language, translate, allow_remote, block
+        )
+        headers["X-Translated-From"] = source
+        headers["X-Translated-To"] = translate
+    return _html(_wrap(fragment), trackers, headers)
+
+
+async def _names(db: AsyncSession, user: User, message: Message) -> list[str]:
+    """Names a translation leaves as they are if the model would change them:
+    the people on the message (and the sender's single names, as signed:
+    "Milan"), the user's own names (accounts, signatures)."""
+    names: list[str] = []
+    for people in (message.from_addr, message.to_addr, message.cc_addr):
+        names += [p.get("name", "") for p in people if p.get("name")]
+    for p in message.from_addr:
+        names += [w for w in str(p.get("name", "")).split() if len(w) >= 3 and w[0].isupper()]
+    rows = await db.execute(
+        select(Account.display_name, Account.signature).where(Account.user_id == user.id)
+    )
+    for display_name, signature in rows:
+        if display_name:
+            names.append(display_name)
+        if signature:
+            names += find_names_in_signature(html_to_text(signature))
+    return names
+
+
+async def _translated(
+    db: AsyncSession,
+    user: User,
+    message: Message,
+    fragment: str,
+    language: str,
+    target: str,
+    allow_remote: bool,
+    block: bool,
+) -> tuple[str, str]:
+    """The fragment in ``target`` — from the cache, or translated and kept."""
+    version = SANITIZER_VERSION * 1000 + translation.PIPELINE_VERSION
+    cached = await db.get(MessageTranslation, (message.id, target, allow_remote))
+    if cached is not None and cached.sanitizer_v == version:
+        return cached.html, cached.source
+    try:
+        out = await translation.translate_html(
+            fragment, language or "auto", target, names=await _names(db, user, message)
+        )
+    except translation.TranslationError as exc:
+        raise_api_error(503, "translation_failed", str(exc))
+    # The translation comes back as HTML: cleaned like any HTML we show.
+    clean = sanitize_html(
+        out, message.id, allow_remote_images=allow_remote, block_trackers=block
+    ).html
+    source = language or "auto"
+    stmt = insert(MessageTranslation).values(
+        message_id=message.id,
+        target=target,
+        images=allow_remote,
+        source=source,
+        html=clean,
+        sanitizer_v=version,
+    )
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["message_id", "target", "images"],
+            set_={"html": clean, "source": source, "sanitizer_v": version},
+        )
+    )
+    await db.commit()
+    return clean, source
 
 
 @router.post("/{message_id}/allow-remote-images", dependencies=[Depends(require_csrf)])
@@ -134,11 +335,38 @@ async def message_cid(
     )
     if att is None:
         raise_api_error(404, "not_found", "inline part not found")
-    data = blobstore.load(msg.user_id, att.blob_sha256)
+    data = await _attachment_bytes(msg, att)
     media_type = att.mime_type if att.mime_type in _INLINE_SAFE else "application/octet-stream"
     return Response(content=data, media_type=media_type)
 
 
+def _content_disposition(disposition: str, filename: str) -> str:
+    """RFC 6266 Content-Disposition safe for non-ASCII names. HTTP header values
+    are Latin-1, so a raw Polish/CJK filename (ł, ń, …) crashes the response with
+    a 500; emit an ASCII fallback plus a UTF-8 ``filename*`` for modern clients."""
+    ascii_name = filename.encode("ascii", "ignore").decode("ascii").replace('"', "").strip()
+    fallback = ascii_name or "attachment"
+    return f"{disposition}; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename)}"
+
+
+async def _raw_ready(msg: Message) -> None:
+    """Have the message's raw source here (a hosted mailbox's is fetched from
+    the mail server when not cached); a miss is left to the reader to report."""
+    with contextlib.suppress(FileNotFoundError):
+        await rawstore.ensure(msg)
+
+
+async def _attachment_bytes(msg: Message, att: Attachment) -> bytes:
+    """An attachment's bytes, taken out of its message (404 if they cannot be)."""
+    await _raw_ready(msg)
+    try:
+        return await asyncio.to_thread(
+            att_store.load, msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index
+        )
+    except FileNotFoundError:
+        raise_api_error(404, "not_found", "attachment content missing")
+
+
 async def _real_attachments(db: AsyncSession, message_id: uuid.UUID) -> list[Attachment]:
     """Downloadable (non-inline) attachments for a message, in a stable order."""
     rows = await db.scalars(
@@ -161,6 +389,41 @@ async def list_attachments(
     ]
 
 
+@router.post(
+    "/{message_id}/forward-attachments",
+    response_model=list[AttachmentOut],
+    dependencies=[Depends(require_csrf)],
+)
+async def forward_attachments(
+    message_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> list[AttachmentOut]:
+    """Attach a message's files to a forward of it: one compose attachment per
+    real (non-inline) attachment. Each is taken out of the message into a blob
+    of its own (shared, by hash, with any other use of the same file).
+    Returns them as the composer lists its attachments."""
+    msg = await _owned_message(db, user.id, message_id)
+    await _raw_ready(msg)
+    rows = [
+        ComposeAttachment(
+            user_id=user.id,
+            filename=a.filename,
+            mime_type=a.mime_type,
+            size_bytes=a.size_bytes,
+            blob_sha256=a.blob_sha256,
+        )
+        for a in await _real_attachments(db, message_id)
+        if await asyncio.to_thread(
+            att_store.materialize, user.id, msg.blob_sha256, a.blob_sha256, a.part_index
+        )
+    ]
+    db.add_all(rows)
+    await db.commit()
+    return [
+        AttachmentOut(id=r.id, filename=r.filename, mime_type=r.mime_type, size_bytes=r.size_bytes)
+        for r in rows
+    ]
+
+
 @router.get("/{message_id}/attachments.zip")
 async def download_attachments_zip(
     message_id: uuid.UUID,
@@ -194,7 +457,7 @@ async def download_attachments_zip(
                 name = f"{stem}-{seen[name]}{dot}{ext}" if dot else f"{name}-{seen[name]}"
             else:
                 seen[name] = 1
-            zf.writestr(name, blobstore.load(msg.user_id, att.blob_sha256))
+            zf.writestr(name, await _attachment_bytes(msg, att))
 
     return Response(
         content=buffer.getvalue(),
@@ -203,6 +466,42 @@ async def download_attachments_zip(
     )
 
 
+@router.get("/{message_id}/attachments/{attachment_id}/thumb")
+async def attachment_thumbnail(
+    message_id: uuid.UUID, attachment_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> Response:
+    """A small preview of an image or a PDF's first page (WebP, at most
+    480 px, aspect kept). 404 when the file cannot have one: the reading pane
+    then shows its type icon instead."""
+    msg = await _owned_message(db, user.id, message_id)
+    att = await db.scalar(
+        select(Attachment).where(
+            Attachment.id == attachment_id, Attachment.message_id == message_id
+        )
+    )
+    kind = thumb_type(att.mime_type, att.filename) if att is not None else None
+    if att is None or kind is None:
+        raise_api_error(404, "not_found", "no thumbnail")
+    if not thumbs.made(msg.user_id, att.blob_sha256):
+        await _raw_ready(msg)
+    data = await asyncio.to_thread(
+        thumbs.thumbnail,
+        msg.user_id,
+        att.blob_sha256,
+        kind,
+        att.size_bytes,
+        lambda: att_store.load(msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index),
+    )
+    if data is None:
+        raise_api_error(404, "not_found", "no thumbnail")
+    # An attachment never changes under its id: the browser may keep this.
+    return Response(
+        content=data,
+        media_type="image/webp",
+        headers={"Cache-Control": "private, max-age=31536000, immutable"},
+    )
+
+
 @router.get("/{message_id}/attachments/{attachment_id}")
 async def message_attachment(
     message_id: uuid.UUID,
@@ -220,12 +519,22 @@ async def message_attachment(
     )
     if att is None:
         raise_api_error(404, "not_found", "attachment not found")
-    data = blobstore.load(msg.user_id, att.blob_sha256)
-    inline = att.mime_type in _INLINE_SAFE and not download
-    media_type = att.mime_type if inline else "application/octet-stream"
-    disposition = "inline" if inline else "attachment"
-    return Response(
-        content=data,
-        media_type=media_type,
-        headers={"Content-Disposition": f'{disposition}; filename="{att.filename}"'},
+    if not att_store.has_file(msg.user_id, att.blob_sha256):
+        await _raw_ready(msg)
+    try:
+        # Taken out of the message into a short-lived file (kept a day after
+        # its last use), so it can be streamed with HTTP Range like before.
+        path = await asyncio.to_thread(
+            att_store.file, msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index
+        )
+    except FileNotFoundError:
+        raise_api_error(404, "not_found", "attachment content missing")
+    inline_type = None if download else _inline_type(att.mime_type, att.filename)
+    disposition = "inline" if inline_type else "attachment"
+    # Streamed from disk, with HTTP Range: a video can be played and seeked
+    # without being sent whole first (Safari will not play one otherwise).
+    return FileResponse(
+        path,
+        media_type=inline_type or "application/octet-stream",
+        headers={"Content-Disposition": _content_disposition(disposition, att.filename)},
     )

backend/app/api/plugins.py

@@ -0,0 +1,23 @@
+"""Plugin platform HTTP surface (SPEC §11).
+
+Currently just the aggregated UI manifest the frontend loader fetches on boot.
+Individual plugin routers are mounted separately at ``/api/plugins/{name}`` by
+:func:`app.plugins.loader.mount_routers`.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter
+
+from app.identity.deps import CurrentUser
+from app.plugins import loader
+
+router = APIRouter(prefix="/plugins", tags=["plugins"])
+
+
+@router.get("/ui-manifest")
+async def ui_manifest(user: CurrentUser) -> dict[str, Any]:
+    """Slots, pages, and capabilities declared by all installed plugins (SPEC §11.3)."""
+    return loader.build_ui_manifest()

backend/app/api/push.py

@@ -0,0 +1,80 @@
+"""Push notifications about new mail: the server's public key, and the devices
+that want them (Settings → Accounts → Notifications)."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel, Field
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentSession, CurrentUser, require_csrf
+from app.push import service, webpush
+
+router = APIRouter(prefix="/push", tags=["push"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class PushKeyOut(BaseModel):
+    public_key: str
+
+
+class PushKeys(BaseModel):
+    p256dh: str = Field(min_length=1, max_length=200)
+    auth: str = Field(min_length=1, max_length=100)
+
+
+class SubscriptionIn(BaseModel):
+    """The browser's ``PushSubscription.toJSON()``."""
+
+    endpoint: str = Field(min_length=1, max_length=2000)
+    keys: PushKeys
+
+
+class EndpointIn(BaseModel):
+    endpoint: str = Field(min_length=1, max_length=2000)
+
+
+@router.get("/key", response_model=PushKeyOut)
+async def push_key(user: CurrentUser, db: DbSession) -> PushKeyOut:
+    """The key a browser subscribes with (``applicationServerKey``)."""
+    return PushKeyOut(public_key=(await service.vapid(db)).public_b64)
+
+
+@router.post("/subscriptions")
+async def add_subscription(
+    body: SubscriptionIn,
+    request: Request,
+    user: CurrentUser,
+    session: CurrentSession,
+    db: DbSession,
+) -> dict[str, str]:
+    if not webpush.allowed_endpoint(body.endpoint):
+        raise_api_error(400, "push_service", "not a known push service")
+    try:
+        p256dh = webpush.b64u_decode(body.keys.p256dh)
+        auth = webpush.b64u_decode(body.keys.auth)
+        if len(p256dh) != 65 or len(auth) != 16:
+            raise ValueError
+    except ValueError:
+        raise_api_error(400, "push_keys", "invalid subscription keys")
+    await service.subscribe(
+        db,
+        user_id=user.id,
+        session_sid=session.sid,
+        endpoint=body.endpoint,
+        p256dh=body.keys.p256dh,
+        auth=body.keys.auth,
+        user_agent=request.headers.get("user-agent", ""),
+    )
+    return {"status": "ok"}
+
+
+@router.post("/subscriptions/remove")
+async def remove_subscription(body: EndpointIn, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    await service.unsubscribe(db, user_id=user.id, endpoint=body.endpoint)
+    return {"status": "ok"}

backend/app/api/saved_searches.py

@@ -0,0 +1,82 @@
+"""Saved searches — stored queries shown as virtual mailboxes (SPEC §12 M2)."""
+
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import SavedSearch
+
+router = APIRouter(prefix="/saved-searches", tags=["search"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class SavedSearchOut(BaseModel):
+    id: uuid.UUID
+    name: str
+    query: str
+    pinned: bool
+
+
+class SavedSearchIn(BaseModel):
+    name: str = Field(min_length=1, max_length=80)
+    query: str = Field(min_length=1, max_length=500)
+    pinned: bool = True
+
+
+def _out(s: SavedSearch) -> SavedSearchOut:
+    return SavedSearchOut(id=s.id, name=s.name, query=s.query, pinned=s.pinned)
+
+
+@router.get("", response_model=list[SavedSearchOut])
+async def list_saved_searches(user: CurrentUser, db: DbSession) -> list[SavedSearchOut]:
+    rows = await db.scalars(
+        select(SavedSearch).where(SavedSearch.user_id == user.id).order_by(SavedSearch.name)
+    )
+    return [_out(s) for s in rows]
+
+
+@router.post("", response_model=SavedSearchOut)
+async def create_saved_search(
+    body: SavedSearchIn, user: CurrentUser, db: DbSession
+) -> SavedSearchOut:
+    name = body.name.strip()
+    query = body.query.strip()
+    if not name or not query:
+        raise_api_error(400, "invalid", "name and query are required")
+    # Upsert by (user, name): re-saving under an existing name updates its query.
+    existing = await db.scalar(
+        select(SavedSearch).where(SavedSearch.user_id == user.id, SavedSearch.name == name)
+    )
+    if existing is not None:
+        existing.query = query
+        existing.pinned = body.pinned
+        saved = existing
+    else:
+        saved = SavedSearch(user_id=user.id, name=name, query=query, pinned=body.pinned)
+        db.add(saved)
+    await db.commit()
+    return _out(saved)
+
+
+@router.delete("/{search_id}")
+async def delete_saved_search(
+    search_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    saved = await db.scalar(
+        select(SavedSearch).where(SavedSearch.id == search_id, SavedSearch.user_id == user.id)
+    )
+    if saved is None:
+        raise_api_error(404, "not_found", "saved search not found")
+    await db.delete(saved)
+    await db.commit()
+    return {"status": "deleted"}

backend/app/api/schemas.py

@@ -18,17 +18,23 @@ class ThreadSummary(BaseModel):
     subject: str
     snippet: str
     participants: list[Address]
+    # Distinct From addresses of the thread's visible messages, oldest first —
+    # what the list's sender column shows (recipients excluded).
+    senders: list[Address] = []
     account_ids: list[uuid.UUID]
     message_count: int
     unread_count: int
     has_attachments: bool
     pinned: bool
     last_message_at: dt.datetime
+    # Set while the thread is snoozed: when it comes back to the Inbox.
+    snoozed_until: dt.datetime | None = None
 
 
 class ThreadListOut(BaseModel):
     threads: list[ThreadSummary]
     cursor: str | None = None
+    total: int = 0
 
 
 class AttachmentOut(BaseModel):
@@ -54,7 +60,19 @@ class MessageEnvelope(BaseModel):
     is_starred: bool
     has_attachments: bool
     has_remote_images: bool
+    # The sender is on the user's trusted list, so remote images load without a prompt.
+    images_allowed: bool = False
+    size_bytes: int
     snippet: str
+    # Other accounts this same delivery also landed in (collapsed duplicates).
+    also_in_account_ids: list[uuid.UUID] = []
+    # Bulk mail: the Subscriptions group it belongs to, how it can be left
+    # (one_click | mailto | web), and whether the user already did.
+    subscription_key: str | None = None
+    unsubscribe_via: str | None = None
+    unsubscribed: bool = False
+    # Its language, once found (the first time it is shown): "" unknown.
+    content_language: str = ""
 
 
 class ThreadDetail(BaseModel):
@@ -63,6 +81,7 @@ class ThreadDetail(BaseModel):
     order: str  # "asc" | "desc"
     pinned: bool
     messages: list[MessageEnvelope]  # always ASC (SPEC §9.4)
+    snoozed_until: dt.datetime | None = None
 
 
 class FlagChangeRequest(BaseModel):
@@ -70,6 +89,10 @@ class FlagChangeRequest(BaseModel):
     remove: list[str] = []
 
 
+class MarkReadRequest(BaseModel):
+    view: str = "unified"  # unified | folder:<special> | account:<uuid>
+
+
 class MoveRequest(BaseModel):
     mailbox_id: uuid.UUID
 
@@ -81,6 +104,16 @@ class AccountOut(BaseModel):
     color: str
     state: str
     signature: str = ""
+    sync_window: str = "all"
+    backfill_total: int = 0
+    backfill_done: int = 0
+    # "unreachable" while the mail server has not answered for a while.
+    sync_error: str | None = None
+    sync_error_since: str | None = None
+    # Why the state is what it is (e.g. the server's reply to a refused sign-in).
+    state_detail: str = ""
+    # "native": the user's own hosted mailbox — cannot be removed (App. D §D.4).
+    kind: str = "external"
 
 
 class SearchThreadHit(BaseModel):
@@ -91,3 +124,5 @@ class SearchThreadHit(BaseModel):
 class SearchOut(BaseModel):
     threads: list[SearchThreadHit]
     cursor: str | None = None
+    # Conversations matching in all (first page only; None where not counted).
+    total: int | None = None

backend/app/api/search.py

@@ -10,7 +10,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.schemas import SearchOut, SearchThreadHit
-from app.api.threads import _summary
+from app.api.threads import _summary, thread_senders
 from app.db import get_session
 from app.identity.deps import CurrentUser
 from app.models import Thread
@@ -29,13 +29,22 @@ async def search(
     q: str = Query(""),
     cursor: str | None = None,
     limit: int = Query(50, ge=1, le=200),
+    filter: str | None = Query(None, pattern="^(unread|pinned)$"),
 ) -> SearchOut:
+    """One page of matching messages, grouped into conversations. ``filter``
+    keeps conversations with unread mail / pinned ones (the list's Unread and
+    Pinned tabs). The first page also says how many conversations match in
+    all, where the search engine can count them."""
     if not q.strip():
         return SearchOut(threads=[], cursor=None)
 
     backend = get_search_backend()
     query = to_search_query(q, user.id)
     hits, next_cursor = await backend.query(query, limit=limit, cursor=cursor)
+    total: int | None = None
+    count = getattr(backend, "count_threads", None)
+    if cursor is None and count is not None:
+        total = await count(query, thread_filter=filter)
 
     # Group hits by thread, preserving order and keeping the first highlight.
     order: list[uuid.UUID] = []
@@ -48,7 +57,7 @@ async def search(
             highlights[hit.thread_id] = hit.highlights
 
     if not order:
-        return SearchOut(threads=[], cursor=next_cursor)
+        return SearchOut(threads=[], cursor=next_cursor, total=total)
 
     threads = {
         t.id: t
@@ -56,9 +65,17 @@ async def search(
             select(Thread).where(Thread.id.in_(order), Thread.user_id == user.id)
         )
     }
+    senders = await thread_senders(db, user.id, list(threads))
     results = [
-        SearchThreadHit(thread=_summary(threads[tid]), highlights=highlights.get(tid, {}))
+        SearchThreadHit(
+            thread=_summary(threads[tid], senders=senders.get(tid)),
+            highlights=highlights.get(tid, {}),
+        )
         for tid in order
         if tid in threads
+        and (
+            filter is None
+            or (threads[tid].unread_count > 0 if filter == "unread" else threads[tid].pinned)
+        )
     ]
-    return SearchOut(threads=results, cursor=next_cursor)
+    return SearchOut(threads=results, cursor=next_cursor, total=total)

backend/app/api/settings.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import re
 from typing import Annotated
 
 from fastapi import APIRouter, Depends
@@ -18,17 +19,70 @@ router = APIRouter(prefix="/settings", tags=["settings"])
 DbSession = Annotated[AsyncSession, Depends(get_session)]
 
 _ALLOWED_ORDER = {"asc", "desc"}
+_ALLOWED_TIME_FORMAT = {"12", "24"}
+_ALLOWED_REMINDER_METHOD = {"alert", "email"}
+_ALLOWED_INVITE_POLICY = {"add_update", "update_only", "none"}
+_ALLOWED_INVITE_FROM = {"contacts", "anyone"}
+
+
+# "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de".
+_LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*")
+_UI_LANGUAGE = re.compile(r"[a-z]{2,3}(-[A-Za-z]{2,4})?")
+
+
+class ReminderPref(BaseModel):
+    method: str = "alert"  # alert = in-app notification, email = email me
+    minutes: int = 0  # minutes before the event start (0 = at time of event)
 
 
 class SettingsOut(BaseModel):
     thread_order: str = "asc"
     tz: str = "UTC"
+    time_format: str = "24"  # 12 = AM/PM, 24 = 24-hour (default); applies app-wide
+    jitsi_server: str = ""  # blank => https://meet.jit.si/
+    default_calendar: str = ""  # calendar id new events default to (blank => first)
+    default_reminders_timed: list[ReminderPref] = []  # defaults for timed events
+    default_reminders_allday: list[ReminderPref] = []  # defaults for all-day events
+    # How emailed events update the calendar: none | update_only | add_update.
+    invitation_policy: str = "none"
+    invitation_from: str = "contacts"  # whose emailed events auto-add (contacts|anyone)
+    invitation_calendar: str = ""  # calendar id emailed events land in (blank => default)
+    invitation_file: bool = False  # also mark the invite message read + filed
     allowed_image_senders: list[str] = []
+    # Spelling and grammar check in the compose window, and its language
+    # ("auto" detects it per message).
+    spellcheck_enabled: bool = True
+    spellcheck_language: str = "auto"
+    # Gmail-style keyboard shortcuts in the mail view (Settings → Keyboard shortcuts).
+    shortcuts_enabled: bool = True
+    # Remove tracking pixels from mail (Settings → Privacy).
+    block_trackers: bool = True
+    # Grey sentence endings in the compose window, taken with Tab.
+    completion_enabled: bool = True
+    # Interface language ("en", "pl", …); "" until chosen — the web app then
+    # picks one from the browser on first sign-in and saves it here.
+    ui_language: str = ""
+    # Names and words email translation leaves as they are.
 
 
 class SettingsPatch(BaseModel):
     thread_order: str | None = None
     tz: str | None = None
+    time_format: str | None = None
+    jitsi_server: str | None = None
+    default_calendar: str | None = None
+    default_reminders_timed: list[ReminderPref] | None = None
+    default_reminders_allday: list[ReminderPref] | None = None
+    invitation_policy: str | None = None
+    invitation_from: str | None = None
+    invitation_calendar: str | None = None
+    invitation_file: bool | None = None
+    spellcheck_enabled: bool | None = None
+    spellcheck_language: str | None = None
+    shortcuts_enabled: bool | None = None
+    block_trackers: bool | None = None
+    completion_enabled: bool | None = None
+    ui_language: str | None = None
 
 
 class QuotaOut(BaseModel):
@@ -43,6 +97,11 @@ def allowed_image_senders(user: User) -> set[str]:
     return set()
 
 
+def blocks_trackers(user: User) -> bool:
+    """Whether tracking pixels are taken out of this user's mail (on unless switched off)."""
+    return bool(user.settings.get("block_trackers", True))
+
+
 def add_allowed_image_sender(user: User, email: str) -> None:
     current = sorted(allowed_image_senders(user))
     if email.lower() not in current:
@@ -51,12 +110,47 @@ def add_allowed_image_sender(user: User, email: str) -> None:
     user.settings = {**user.settings, "allowed_image_senders": current}
 
 
+def _reminder_list(value: object) -> list[ReminderPref]:
+    """Normalize stored reminders. Accepts the legacy ``list[int]`` (minutes only,
+    method defaults to alert) as well as the ``{method, minutes}`` object form."""
+    if not isinstance(value, list):
+        return []
+    out: list[ReminderPref] = []
+    for x in value:
+        if isinstance(x, dict):
+            method = str(x.get("method", "alert"))
+            out.append(
+                ReminderPref(
+                    method=method if method in _ALLOWED_REMINDER_METHOD else "alert",
+                    minutes=int(x.get("minutes", 0)),
+                )
+            )
+        else:
+            out.append(ReminderPref(method="alert", minutes=int(x)))
+    return out
+
+
 def _settings_out(user: User) -> SettingsOut:
     s = user.settings
     return SettingsOut(
         thread_order=str(s.get("thread_order", "asc")),
         tz=str(s.get("tz", "UTC")),
+        time_format=str(s.get("time_format", "24")),
+        jitsi_server=str(s.get("jitsi_server", "")),
+        default_calendar=str(s.get("default_calendar", "")),
+        default_reminders_timed=_reminder_list(s.get("default_reminders_timed")),
+        default_reminders_allday=_reminder_list(s.get("default_reminders_allday")),
+        invitation_policy=str(s.get("invitation_policy", "none")),
+        invitation_from=str(s.get("invitation_from", "contacts")),
+        invitation_calendar=str(s.get("invitation_calendar", "")),
+        invitation_file=bool(s.get("invitation_file", False)),
         allowed_image_senders=sorted(allowed_image_senders(user)),
+        spellcheck_enabled=bool(s.get("spellcheck_enabled", True)),
+        spellcheck_language=str(s.get("spellcheck_language", "auto")),
+        shortcuts_enabled=bool(s.get("shortcuts_enabled", True)),
+        block_trackers=blocks_trackers(user),
+        completion_enabled=bool(s.get("completion_enabled", True)),
+        ui_language=str(s.get("ui_language", "")),
     )
 
 
@@ -81,6 +175,38 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession)
         updates["thread_order"] = body.thread_order
     if body.tz is not None:
         updates["tz"] = body.tz
+    if body.time_format is not None and body.time_format in _ALLOWED_TIME_FORMAT:
+        updates["time_format"] = body.time_format
+    if body.jitsi_server is not None:
+        updates["jitsi_server"] = body.jitsi_server.strip()
+    if body.default_calendar is not None:
+        updates["default_calendar"] = body.default_calendar
+    if body.default_reminders_timed is not None:
+        updates["default_reminders_timed"] = [r.model_dump() for r in body.default_reminders_timed]
+    if body.default_reminders_allday is not None:
+        updates["default_reminders_allday"] = [
+            r.model_dump() for r in body.default_reminders_allday
+        ]
+    if body.invitation_policy is not None and body.invitation_policy in _ALLOWED_INVITE_POLICY:
+        updates["invitation_policy"] = body.invitation_policy
+    if body.invitation_from is not None and body.invitation_from in _ALLOWED_INVITE_FROM:
+        updates["invitation_from"] = body.invitation_from
+    if body.invitation_calendar is not None:
+        updates["invitation_calendar"] = body.invitation_calendar
+    if body.invitation_file is not None:
+        updates["invitation_file"] = body.invitation_file
+    if body.spellcheck_enabled is not None:
+        updates["spellcheck_enabled"] = body.spellcheck_enabled
+    if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language):
+        updates["spellcheck_language"] = body.spellcheck_language
+    if body.shortcuts_enabled is not None:
+        updates["shortcuts_enabled"] = body.shortcuts_enabled
+    if body.block_trackers is not None:
+        updates["block_trackers"] = body.block_trackers
+    if body.completion_enabled is not None:
+        updates["completion_enabled"] = body.completion_enabled
+    if body.ui_language is not None and _UI_LANGUAGE.fullmatch(body.ui_language):
+        updates["ui_language"] = body.ui_language
     user.settings = updates
     await db.commit()
     return _settings_out(user)

backend/app/api/snooze.py

@@ -0,0 +1,76 @@
+"""Snooze endpoints: hide a conversation from the Inbox until a set time."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import snooze
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.events import get_event_bus
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Thread
+
+router = APIRouter(prefix="/threads", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class SnoozeIn(BaseModel):
+    until: dt.datetime
+
+
+class SnoozeOut(BaseModel):
+    thread_id: uuid.UUID
+    snoozed_until: dt.datetime | None
+
+
+async def _thread(db: AsyncSession, user_id: uuid.UUID, thread_id: uuid.UUID) -> Thread:
+    thread = await db.scalar(
+        select(Thread).where(Thread.id == thread_id, Thread.user_id == user_id)
+    )
+    if thread is None:
+        raise_api_error(404, "not_found", "thread not found")
+    return thread
+
+
+async def _announce(user_id: uuid.UUID, thread_id: uuid.UUID) -> None:
+    await get_event_bus().publish(
+        "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}
+    )
+
+
+@router.post("/{thread_id}/snooze", response_model=SnoozeOut)
+async def snooze_thread(
+    thread_id: uuid.UUID, body: SnoozeIn, user: CurrentUser, db: DbSession
+) -> SnoozeOut:
+    until = body.until
+    if until.tzinfo is None:
+        raise_api_error(400, "invalid_time", "the time needs a time zone")
+    now = dt.datetime.now(dt.UTC)
+    if until <= now:
+        raise_api_error(400, "invalid_time", "pick a time in the future")
+    if until > now + snooze.MAX_SNOOZE:
+        raise_api_error(400, "invalid_time", "snooze for at most a year")
+    thread = await _thread(db, user.id, thread_id)
+    snooze.snooze(thread, until, now)
+    await db.commit()
+    await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=thread.snoozed_until)
+
+
+@router.post("/{thread_id}/unsnooze", response_model=SnoozeOut)
+async def unsnooze_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -> SnoozeOut:
+    thread = await _thread(db, user.id, thread_id)
+    if thread.snoozed_until is not None:
+        snooze.wake(thread, dt.datetime.now(dt.UTC))
+        await db.commit()
+        await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=None)

backend/app/api/spellcheck.py

@@ -0,0 +1,187 @@
+"""Spelling and grammar check for the compose window, backed by LanguageTool.
+
+The browser sends the text it wants checked here, never to LanguageTool
+itself: the engine stays on the internal network, only signed-in users reach
+it, and nothing leaves the server. When no engine is configured (or it is
+down), the check answers 503 and the compose window simply goes without.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections import Counter
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.identity.deps import CurrentUser, require_csrf
+
+router = APIRouter(prefix="/spellcheck", tags=["compose"])
+
+_TIMEOUT = 10.0
+# A message body is checked as a whole; past this it is a document, not mail.
+MAX_TEXT = 20_000
+_MAX_SUGGESTIONS = 6
+# With "auto", LanguageTool settles on ONE language per request, so a message
+# mixing Polish and English paragraphs would have every English word flagged as
+# a Polish typo. Auto-detected text is therefore checked in chunks, each in its
+# own language. A chunk is at least this many characters of consecutive lines:
+# a greeting alone ("Witam,") is too short to detect reliably, so it rides with
+# the lines after it.
+_MIN_CHUNK = 60
+_PARALLEL = 4
+
+
+class CheckIn(BaseModel):
+    text: str = Field(max_length=MAX_TEXT)
+    # "auto" or a LanguageTool code ("pl-PL", "en-GB", "de-DE" ...).
+    language: str = Field(default="auto", max_length=16)
+
+
+class Issue(BaseModel):
+    offset: int
+    length: int
+    message: str
+    short_message: str
+    replacements: list[str]
+    # spelling | grammar | style: what the underline colour follows.
+    kind: str
+    rule_id: str
+
+
+class CheckOut(BaseModel):
+    language: str  # the language checked (detected, with "auto")
+    language_name: str
+    issues: list[Issue]
+
+
+class LanguageOut(BaseModel):
+    code: str
+    name: str
+
+
+def _base() -> str:
+    url = get_settings().languagetool_url.rstrip("/")
+    if not url:
+        raise_api_error(503, "spellcheck_unavailable", "spell checking is not set up")
+    return url
+
+
+def _kind(issue_type: str) -> str:
+    if issue_type == "misspelling":
+        return "spelling"
+    if issue_type in ("style", "typographical", "whitespace", "register", "locale-violation"):
+        return "style"
+    return "grammar"
+
+
+def chunks(text: str) -> list[tuple[int, str]]:
+    """Split ``text`` into (offset, chunk) runs of whole lines, each at least
+    _MIN_CHUNK characters where the text allows; a blank line always ends one."""
+    out: list[tuple[int, str]] = []
+    start = pos = 0
+    for line in text.split("\n"):
+        end = pos + len(line)
+        if not line.strip():
+            if text[start:pos].strip():
+                # Up to the line break before the blank line, not including it.
+                out.append((start, text[start:pos].rstrip("\n")))
+            start = end + 1
+        elif end - start >= _MIN_CHUNK:
+            out.append((start, text[start:end]))
+            start = end + 1
+        pos = end + 1
+    if text[start:].strip():
+        out.append((start, text[start:]))
+    return out
+
+
+async def _check_one(
+    client: httpx.AsyncClient, base: str, text: str, language: str
+) -> dict[str, Any]:
+    resp = await client.post(f"{base}/v2/check", data={"text": text, "language": language})
+    resp.raise_for_status()
+    data: dict[str, Any] = resp.json()
+    return data
+
+
+def _issues(data: dict[str, Any], shift: int) -> list[Issue]:
+    return [
+        Issue(
+            offset=int(m["offset"]) + shift,
+            length=int(m["length"]),
+            message=str(m.get("message", "")),
+            short_message=str(m.get("shortMessage", "")),
+            replacements=[str(r["value"]) for r in (m.get("replacements") or [])][
+                :_MAX_SUGGESTIONS
+            ],
+            kind=_kind(str((m.get("rule") or {}).get("issueType", ""))),
+            rule_id=str((m.get("rule") or {}).get("id", "")),
+        )
+        for m in data.get("matches") or []
+    ]
+
+
+def _detected(data: dict[str, Any]) -> tuple[str, str]:
+    lang = data.get("language") or {}
+    detected = lang.get("detectedLanguage") or lang
+    return str(detected.get("code", "")), str(detected.get("name", ""))
+
+
+@router.post("", response_model=CheckOut, dependencies=[Depends(require_csrf)])
+async def check(body: CheckIn, user: CurrentUser) -> CheckOut:
+    base = _base()
+    if not body.text.strip():
+        return CheckOut(language=body.language, language_name="", issues=[])
+    parts = chunks(body.text) if body.language == "auto" else [(0, body.text)]
+    gate = asyncio.Semaphore(_PARALLEL)
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+
+            async def one(text: str) -> dict[str, Any]:
+                async with gate:
+                    return await _check_one(client, base, text, body.language)
+
+            results = await asyncio.gather(*(one(text) for _, text in parts))
+    except (httpx.HTTPError, ValueError):
+        raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering")
+
+    issues = [
+        i for (shift, _), data in zip(parts, results, strict=True) for i in _issues(data, shift)
+    ]
+    # The language reported is the one most of the text was written in.
+    weight: Counter[tuple[str, str]] = Counter()
+    for (_, text), data in zip(parts, results, strict=True):
+        weight[_detected(data)] += len(text)
+    (code, name), _ = weight.most_common(1)[0]
+    return CheckOut(language=code or body.language, language_name=name, issues=issues)
+
+
+_languages_cache: tuple[float, list[LanguageOut]] | None = None
+
+
+@router.get("/languages", response_model=list[LanguageOut])
+async def languages(user: CurrentUser) -> list[LanguageOut]:
+    """What the engine can check, for the language picker in Settings."""
+    global _languages_cache
+    if _languages_cache and time.monotonic() - _languages_cache[0] < 3600:
+        return _languages_cache[1]
+    base = _base()
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(f"{base}/v2/languages")
+            resp.raise_for_status()
+            rows: list[dict[str, Any]] = resp.json()
+    except (httpx.HTTPError, ValueError):
+        raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering")
+    seen: dict[str, str] = {}
+    for row in rows:
+        seen.setdefault(str(row["longCode"]), str(row["name"]))
+    out = sorted((LanguageOut(code=c, name=n) for c, n in seen.items()), key=lambda x: x.name)
+    _languages_cache = (time.monotonic(), out)
+    return out

backend/app/api/subscriptions.py

@@ -0,0 +1,372 @@
+"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import case, func, literal, or_, select
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.elements import ColumnElement
+
+from app import subscriptions as subs
+from app.accounts.ssrf import guarded_request
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Mailbox, Message, SubscriptionState
+from app.outbox.schemas import DraftIn, DraftPayload
+from app.outbox.service import DraftError, create_outbox, upsert_draft
+from app.queue import enqueue_job
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+# RFC 8058 senders should stop within two days of a one-click request.
+GRACE = dt.timedelta(days=2)
+# Not counted: mail the user sent, and mail already thrown away (Trash, Spam) —
+# a sender whose mail was all deleted drops off the list.
+_LEFT_OUT = ("sent", "drafts", "trash", "junk")
+
+
+class SubscriptionOut(BaseModel):
+    key: str
+    name: str
+    email: str
+    list_id: str | None
+    total: int
+    unread: int
+    last_30_days: int
+    per_month: float
+    first_at: dt.datetime
+    last_at: dt.datetime
+    last_subject: str
+    one_click: bool
+    has_mailto: bool
+    has_web: bool
+    unsubscribed_at: dt.datetime | None
+    method: str | None
+    still_sending: bool
+
+
+class SubscriptionListOut(BaseModel):
+    subscriptions: list[SubscriptionOut]  # one page
+    total: int  # senders in all
+    pending: int  # messages whose headers are still being read
+
+
+class KeyIn(BaseModel):
+    key: str
+
+
+class UnsubscribeIn(KeyIn):
+    # From the reading pane: this message's own header, wherever it is filed
+    # (Spam included, which the page's groups leave out).
+    message_id: uuid.UUID | None = None
+
+
+class UnsubscribeOut(BaseModel):
+    status: str  # done | open
+    method: str  # one_click | mailto | web
+    url: str | None = None
+
+
+def _sender() -> ColumnElement[Any]:
+    return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, ""))
+
+
+def _key() -> ColumnElement[Any]:
+    return case(
+        (Message.list_id.isnot(None), literal("list:") + Message.list_id),
+        else_=literal("from:") + _sender(),
+    )
+
+
+def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]:
+    return [
+        Message.user_id == user_id,
+        Message.list_unsubscribe.isnot(None),
+        Message.list_unsubscribe != "",
+        or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)),
+    ]
+
+
+async def _start_backfill(user_id: uuid.UUID) -> None:
+    # One job per user at a time: arq refuses a second while the first exists.
+    await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}")
+
+
+@router.get("", response_model=SubscriptionListOut)
+async def list_subscriptions(
+    user: CurrentUser,
+    db: DbSession,
+    offset: int = Query(0, ge=0),
+    limit: int = Query(50, ge=1, le=200),
+    order: str = Query("desc"),  # desc: heard from most recently first | asc: longest ago
+) -> SubscriptionListOut:
+    """One page of senders, by when their latest mail came."""
+    pending = await subs.pending_count(db, user.id)
+    if pending:
+        await _start_backfill(user.id)
+
+    now = dt.datetime.now(dt.UTC)
+    key = _key().label("key")
+    seen = Message.flags.contains(["\\Seen"])
+    groups = (
+        select(
+            key,
+            func.count().label("total"),
+            func.count().filter(~seen).label("unread"),
+            func.count()
+            .filter(Message.internal_date > now - dt.timedelta(days=30))
+            .label("recent"),
+            func.min(Message.internal_date).label("first_at"),
+            func.max(Message.internal_date).label("last_at"),
+        )
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id))
+        .group_by(key)
+    )
+    total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0
+    agg_rows = (
+        await db.execute(
+            groups.order_by(
+                func.max(Message.internal_date).asc()
+                if order == "asc"
+                else func.max(Message.internal_date).desc(),
+                key,
+            )
+            .offset(offset)
+            .limit(limit)
+        )
+    ).all()
+    page_keys = [r.key for r in agg_rows]
+    if not page_keys:
+        return SubscriptionListOut(subscriptions=[], total=total, pending=pending)
+
+    in_page = _key().in_(page_keys)
+    latest_rows = (
+        await db.execute(
+            select(
+                key,
+                Message.from_addr,
+                Message.subject,
+                Message.list_id,
+                Message.list_unsubscribe,
+                Message.list_unsubscribe_post,
+            )
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(*_bulk(user.id), in_page)
+            .distinct(key)
+            .order_by(key, Message.internal_date.desc())
+        )
+    ).all()
+    latest = {r.key: r for r in latest_rows}
+    states = {
+        s.key: s
+        for s in await db.scalars(
+            select(SubscriptionState).where(
+                SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys)
+            )
+        )
+    }
+    # Senders still mailing after the grace period, for the whole page at once.
+    still_sending = set(
+        await db.scalars(
+            select(key)
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .join(
+                SubscriptionState,
+                (SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()),
+            )
+            .where(
+                *_bulk(user.id),
+                in_page,
+                Message.internal_date > SubscriptionState.unsubscribed_at + GRACE,
+            )
+            .distinct()
+        )
+    )
+
+    out: list[SubscriptionOut] = []
+    for row in agg_rows:
+        last = latest.get(row.key)
+        if last is None:
+            continue
+        sender = last.from_addr[0] if last.from_addr else {}
+        targets = subs.unsubscribe_targets(last.list_unsubscribe or "")
+        state = states.get(row.key)
+        months = max(1.0, (row.last_at - row.first_at).days / 30.44)
+        out.append(
+            SubscriptionOut(
+                key=row.key,
+                name=sender.get("name") or sender.get("email", ""),
+                email=sender.get("email", ""),
+                list_id=last.list_id,
+                total=row.total,
+                unread=row.unread,
+                last_30_days=row.recent,
+                per_month=round(row.total / months, 1),
+                first_at=row.first_at,
+                last_at=row.last_at,
+                last_subject=last.subject,
+                one_click=bool(last.list_unsubscribe_post and targets.https),
+                has_mailto=targets.mailto is not None,
+                has_web=targets.https is not None,
+                unsubscribed_at=state.unsubscribed_at if state else None,
+                method=state.method if state else None,
+                still_sending=row.key in still_sending,
+            )
+        )
+    return SubscriptionListOut(subscriptions=out, total=total, pending=pending)
+
+
+@router.get("/threads", response_model=list[uuid.UUID])
+async def subscription_threads(
+    user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200)
+) -> list[uuid.UUID]:
+    """The group's conversations, newest first (for the reading pane)."""
+    rows = await db.execute(
+        select(Message.thread_id, func.max(Message.internal_date).label("at"))
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None))
+        .group_by(Message.thread_id)
+        .order_by(func.max(Message.internal_date).desc())
+        .limit(limit)
+    )
+    return [r.thread_id for r in rows if r.thread_id is not None]
+
+
+async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message:
+    message = await db.scalar(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user_id), _key() == key)
+        .order_by(Message.internal_date.desc())
+        .limit(1)
+    )
+    if message is None:
+        raise_api_error(404, "not_found", "no mail from this sender")
+    return message
+
+
+async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None:
+    stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method)
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["user_id", "key"],
+            set_={"method": method, "unsubscribed_at": func.now()},
+        )
+    )
+    await db.commit()
+
+
+async def _one_click(url: str) -> bool:
+    try:
+        resp = await guarded_request(
+            "POST",
+            url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            content="List-Unsubscribe=One-Click",
+            max_bytes=256 * 1024,
+        )
+    except Exception as exc:  # SSRF refusal, network, TLS: any failure tries the next way
+        logger.info("one-click unsubscribe failed: %s", exc)
+        return False
+    return 200 <= resp.status < 400
+
+
+async def _send_mailto(
+    db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets
+) -> bool:
+    assert targets.mailto is not None
+    body = DraftIn(
+        account_id=message.account_id,
+        kind="new",
+        payload=DraftPayload(
+            to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body
+        ),
+    )
+    try:
+        draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None)
+    except DraftError:
+        return False
+    outbox = await create_outbox(db, user_id=user_id, draft=draft)
+    await db.commit()
+    await enqueue_job(
+        "send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds)
+    )
+    return True
+
+
+@router.post("/unsubscribe", response_model=UnsubscribeOut)
+async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut:
+    """Unsubscribe from a sender: one-click if offered, else by mail, else the web page."""
+    if body.message_id is not None:
+        found = await db.scalar(
+            select(Message).where(
+                Message.id == body.message_id,
+                Message.user_id == user.id,
+                Message.list_unsubscribe.isnot(None),
+                Message.list_unsubscribe != "",
+            )
+        )
+        if found is None:
+            raise_api_error(404, "not_found", "this message offers no way to unsubscribe")
+        message = found
+    else:
+        message = await _latest_with_header(db, user.id, body.key)
+    targets = subs.unsubscribe_targets(message.list_unsubscribe or "")
+    if message.list_unsubscribe_post and targets.https and await _one_click(targets.https):
+        await _record(db, user.id, body.key, "one_click")
+        return UnsubscribeOut(status="done", method="one_click")
+    if targets.mailto and await _send_mailto(db, user.id, message, targets):
+        await _record(db, user.id, body.key, "mailto")
+        return UnsubscribeOut(status="done", method="mailto")
+    if targets.https:
+        # The sender wants it done on their page: the user opens it.
+        await _record(db, user.id, body.key, "web")
+        return UnsubscribeOut(status="open", method="web", url=targets.https)
+    raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe")
+
+
+@router.post("/trash")
+async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]:
+    """Move every conversation from this sender to Trash.
+
+    Only conversations made up entirely of this sender's mail: one where the
+    user replied, or someone else wrote, stays where it is.
+    """
+    rows = (
+        await db.execute(
+            select(Message.thread_id, _key().label("key"), Mailbox.special_use)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(
+                Message.user_id == user.id,
+                Message.thread_id.in_(
+                    select(Message.thread_id)
+                    .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                    .where(*_bulk(user.id), _key() == body.key)
+                ),
+            )
+        )
+    ).all()
+    own: dict[uuid.UUID, bool] = {}
+    for thread_id, key, special in rows:
+        if thread_id is None or special == "trash":
+            continue
+        own[thread_id] = own.get(thread_id, True) and key == body.key
+    moved = [tid for tid, only_theirs in own.items() if only_theirs]
+    for tid in moved:
+        await enqueue_job("move_thread_to_folder_job", str(tid), "trash")
+    return {"moved": len(moved)}

backend/app/api/templates.py

@@ -0,0 +1,129 @@
+"""Templates: saved replies, inserted from the compose window's More menu."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+import nh3
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import MailTemplate
+
+router = APIRouter(prefix="/templates", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+MAX_TEMPLATES = 200
+MAX_BODY_BYTES = 256 * 1024
+
+# What the compose editor writes. A template goes back into the editor as HTML,
+# so it is cleaned like any HTML the app renders: no scripts, no handlers.
+_TAGS = {
+    "a", "b", "blockquote", "br", "code", "div", "em", "font", "h1", "h2", "h3",
+    "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strike", "strong",
+    "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "u", "ul",
+}  # fmt: skip
+_GENERIC = {"style", "class", "dir", "align"}
+_SPECIFIC = {
+    "a": {"href", "title"},
+    "img": {"src", "alt", "width", "height"},
+    "font": {"color", "face", "size"},
+    "td": {"colspan", "rowspan"},
+    "th": {"colspan", "rowspan"},
+}
+# nh3 takes no wildcard: the generic attributes go onto every tag.
+_ATTRS = {tag: _GENERIC | _SPECIFIC.get(tag, set()) for tag in _TAGS}
+
+
+class TemplateOut(BaseModel):
+    id: uuid.UUID
+    name: str
+    subject: str
+    body_html: str
+    updated_at: dt.datetime
+
+
+class TemplateIn(BaseModel):
+    name: str = Field(min_length=1, max_length=80)
+    subject: str = Field(default="", max_length=500)
+    body_html: str = ""
+
+
+def _out(t: MailTemplate) -> TemplateOut:
+    return TemplateOut(
+        id=t.id, name=t.name, subject=t.subject, body_html=t.body_html, updated_at=t.updated_at
+    )
+
+
+def clean_body(html: str) -> str:
+    return nh3.clean(
+        html,
+        tags=_TAGS,
+        attributes=_ATTRS,
+        url_schemes={"http", "https", "mailto", "data"},
+        link_rel="noopener noreferrer",
+    )
+
+
+@router.get("", response_model=list[TemplateOut])
+async def list_templates(user: CurrentUser, db: DbSession) -> list[TemplateOut]:
+    rows = await db.scalars(
+        select(MailTemplate)
+        .where(MailTemplate.user_id == user.id)
+        .order_by(func.lower(MailTemplate.name))
+    )
+    return [_out(t) for t in rows]
+
+
+@router.post("", response_model=TemplateOut)
+async def save_template(body: TemplateIn, user: CurrentUser, db: DbSession) -> TemplateOut:
+    """Save a template. Saving under a name that exists replaces that template."""
+    name = " ".join(body.name.split())
+    if not name:
+        raise_api_error(400, "invalid", "a template needs a name")
+    if len(body.body_html.encode()) > MAX_BODY_BYTES:
+        raise_api_error(400, "too_large", "the template is too large")
+    html = clean_body(body.body_html)
+    subject = " ".join(body.subject.split())
+    existing = await db.scalar(
+        select(MailTemplate).where(
+            MailTemplate.user_id == user.id, func.lower(MailTemplate.name) == name.lower()
+        )
+    )
+    if existing is not None:
+        existing.name, existing.subject, existing.body_html = name, subject, html
+        existing.updated_at = dt.datetime.now(dt.UTC)
+        saved = existing
+    else:
+        count = await db.scalar(
+            select(func.count()).select_from(MailTemplate).where(MailTemplate.user_id == user.id)
+        )
+        if (count or 0) >= MAX_TEMPLATES:
+            raise_api_error(409, "too_many", f"at most {MAX_TEMPLATES} templates")
+        saved = MailTemplate(user_id=user.id, name=name, subject=subject, body_html=html)
+        db.add(saved)
+    await db.commit()
+    await db.refresh(saved)
+    return _out(saved)
+
+
+@router.delete("/{template_id}")
+async def delete_template(
+    template_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    row = await db.scalar(
+        select(MailTemplate).where(MailTemplate.id == template_id, MailTemplate.user_id == user.id)
+    )
+    if row is None:
+        raise_api_error(404, "not_found", "template not found")
+    await db.delete(row)
+    await db.commit()
+    return {"status": "deleted"}

backend/app/api/threads.py

@@ -3,13 +3,15 @@
 from __future__ import annotations
 
 import uuid
-from typing import Annotated
+from collections.abc import Collection
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Query
-from sqlalchemy import literal, select, tuple_
+from sqlalchemy import Text, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.elements import ColumnElement
 
-from app.api.cursors import decode_cursor, encode_cursor
+from app import subscriptions as subs
 from app.api.errors import raise_api_error
 from app.api.schemas import (
     Address,
@@ -18,9 +20,10 @@ from app.api.schemas import (
     ThreadListOut,
     ThreadSummary,
 )
+from app.api.settings import allowed_image_senders, blocks_trackers
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Mailbox, Message, Thread, User
+from app.models import BodyPart, Mailbox, Message, SubscriptionState, Thread, User
 
 router = APIRouter(prefix="/threads", tags=["mail"])
 
@@ -31,22 +34,29 @@ def _addresses(raw: list[dict[str, str]]) -> list[Address]:
     return [Address(name=a.get("name", ""), email=a.get("email", "")) for a in raw]
 
 
-def _summary(thread: Thread) -> ThreadSummary:
+def _summary(
+    thread: Thread, message_count: int | None = None, senders: list[Address] | None = None
+) -> ThreadSummary:
     return ThreadSummary(
         id=thread.id,
         subject=thread.subject_norm or "(no subject)",
         snippet=thread.snippet,
         participants=_addresses(thread.participants),
+        senders=senders or [],
         account_ids=list(thread.account_ids),
-        message_count=thread.message_count,
+        # Count only the messages this view actually shows (deleted/spam excluded
+        # elsewhere); falls back to the denormalized total.
+        message_count=message_count if message_count is not None else thread.message_count,
         unread_count=thread.unread_count,
         has_attachments=thread.has_attachments,
         pinned=thread.pinned,
         last_message_at=thread.last_message_at,
+        snoozed_until=thread.snoozed_until,
     )
 
 
-def _envelope(msg: Message) -> MessageEnvelope:
+def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnvelope:
+    sender = msg.from_addr[0].get("email", "").lower() if msg.from_addr else ""
     return MessageEnvelope(
         id=msg.id,
         account_id=msg.account_id,
@@ -63,7 +73,10 @@ def _envelope(msg: Message) -> MessageEnvelope:
         is_starred="\\Flagged" in msg.flags,
         has_attachments=msg.has_attachments,
         has_remote_images=msg.has_remote_images,
+        images_allowed=bool(sender) and sender in allowed_senders,
+        size_bytes=msg.size_bytes,
         snippet=msg.snippet,
+        content_language=msg.content_language or "",
     )
 
 
@@ -72,29 +85,44 @@ async def list_threads(
     user: CurrentUser,
     db: DbSession,
     view: str = Query("unified"),
-    cursor: str | None = None,
     limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    filter: str = Query("all"),  # all | unread | pinned
+    order: str = Query("desc"),  # desc (newest first) | asc (oldest first)
+    ids: str = Query(""),  # comma-separated thread ids (e.g. a plugin's label view)
 ) -> ThreadListOut:
-    stmt = select(Thread).where(Thread.user_id == user.id)
+    # Collect WHERE conditions once so the page query and the total-count query
+    # stay in lockstep (offset pagination needs an accurate grand total).
+    conditions: list[ColumnElement[bool]] = [Thread.user_id == user.id]
 
-    if view.startswith("account:"):
-        # A single account's inbox: threads with a message in THIS account's
-        # inbox mailbox. Keeps the per-account view consistent with the unified
-        # Inbox (archived/trashed/sent threads don't leak in).
+    if ids:
+        # Explicit id set (ignores view/folder) — used by plugin-driven views like
+        # "all mail with label X", which supply the matching thread ids.
+        try:
+            wanted = [uuid.UUID(x) for x in ids.split(",") if x.strip()]
+        except ValueError:
+            raise_api_error(400, "bad_ids", "invalid thread id in ids")
+        conditions.append(Thread.id.in_(wanted or [uuid.uuid4()]))
+    elif view.startswith("account:"):
+        # A single account's mail: threads with a message in ANY of THIS account's
+        # mailboxes except Trash/Spam (those are only seen via their own folders).
         try:
             account_id = uuid.UUID(view.split(":", 1)[1])
         except ValueError:
             raise_api_error(400, "bad_view", "invalid account id in view")
-        in_account_inbox = (
+        in_account = (
             select(Message.thread_id)
             .join(Mailbox, Mailbox.id == Message.mailbox_id)
             .where(
                 Message.user_id == user.id,
                 Message.account_id == account_id,
-                Mailbox.special_use == "inbox",
+                or_(
+                    Mailbox.special_use.is_(None),
+                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
+                ),
             )
         )
-        stmt = stmt.where(Thread.id.in_(in_account_inbox))
+        conditions.append(Thread.id.in_(in_account))
     elif view.startswith("folder:"):
         # Threads with at least one message in a mailbox of this special_use,
         # across all of the user's accounts (unified per-folder view).
@@ -104,25 +132,144 @@ async def list_threads(
             .join(Mailbox, Mailbox.id == Message.mailbox_id)
             .where(Message.user_id == user.id, Mailbox.special_use == special)
         )
-        stmt = stmt.where(Thread.id.in_(in_folder))
+        conditions.append(Thread.id.in_(in_folder))
+        if special == "inbox":
+            conditions.append(Thread.snoozed_until.is_(None))  # back when it wakes
+    elif view == "snoozed":
+        conditions.append(Thread.snoozed_until.isnot(None))
     elif view != "unified":
         raise_api_error(400, "bad_view", "unsupported view")
 
-    decoded = decode_cursor(cursor)
-    if decoded is not None:
-        cur_date, cur_id = decoded
-        stmt = stmt.where(
-            tuple_(Thread.last_message_at, Thread.id) < tuple_(literal(cur_date), literal(cur_id))
-        )
-    stmt = stmt.order_by(Thread.last_message_at.desc(), Thread.id.desc()).limit(limit + 1)
+    if filter == "unread":
+        conditions.append(Thread.unread_count > 0)
+    elif filter == "pinned":
+        conditions.append(Thread.pinned.is_(True))
 
-    threads = list(await db.scalars(stmt))
-    has_more = len(threads) > limit
-    threads = threads[:limit]
-    next_cursor = (
-        encode_cursor(threads[-1].last_message_at, threads[-1].id) if has_more and threads else None
+    total = (
+        await db.scalar(
+            select(func.count()).select_from(select(Thread.id).where(*conditions).subquery())
         )
-    return ThreadListOut(threads=[_summary(t) for t in threads], cursor=next_cursor)
+    ) or 0
+
+    # A thread woken from snooze sorts as if it had just arrived.
+    sort_at = func.greatest(
+        Thread.last_message_at, func.coalesce(Thread.unsnoozed_at, Thread.last_message_at)
+    )
+    if view == "snoozed":
+        # Soonest back first, whatever the list order.
+        order_by: tuple[ColumnElement[Any], ...] = (
+            Thread.snoozed_until.asc(),
+            Thread.id.asc(),
+        )
+    elif order == "asc":
+        order_by = (sort_at.asc(), Thread.id.asc())
+    else:
+        order_by = (sort_at.desc(), Thread.id.desc())
+    stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit)
+    threads = list(await db.scalars(stmt))
+
+    # Per-thread visible message count matching what get_thread shows for this view:
+    # Trash/Spam count only their own messages; every other view excludes them. This
+    # keeps the list count in step with a single deleted message.
+    current_special = view.split(":", 1)[1] if view.startswith("folder:") else None
+    counts: dict[uuid.UUID, int] = {}
+    if threads:
+        # count DISTINCT deliveries so collapsed cross-account copies count once,
+        # matching what get_thread renders.
+        delivery = func.coalesce(Message.delivery_id, func.cast(Message.id, Text))
+        count_stmt = (
+            select(Message.thread_id, func.count(func.distinct(delivery)))
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(Message.user_id == user.id, Message.thread_id.in_([t.id for t in threads]))
+            .group_by(Message.thread_id)
+        )
+        count_stmt = count_stmt.where(_visible_in(current_special))
+        counts = {r[0]: r[1] for r in (await db.execute(count_stmt)).all() if r[0] is not None}
+    senders = await thread_senders(db, user.id, [t.id for t in threads], current_special)
+
+    return ThreadListOut(
+        threads=[_summary(t, counts.get(t.id), senders.get(t.id)) for t in threads],
+        total=total,
+    )
+
+
+async def _mark_subscriptions(
+    db: AsyncSession,
+    user_id: uuid.UUID,
+    messages: list[Message],
+    envelopes: list[MessageEnvelope],
+) -> None:
+    """Tell the reading pane which messages can be unsubscribed from, and how."""
+    keys: dict[uuid.UUID, str] = {}
+    for m, env in zip(messages, envelopes, strict=True):
+        if not m.list_unsubscribe:
+            continue
+        targets = subs.unsubscribe_targets(m.list_unsubscribe)
+        if m.list_unsubscribe_post and targets.https:
+            env.unsubscribe_via = "one_click"
+        elif targets.mailto:
+            env.unsubscribe_via = "mailto"
+        elif targets.https:
+            env.unsubscribe_via = "web"
+        else:
+            continue
+        sender = m.from_addr[0].get("email", "") if m.from_addr else ""
+        env.subscription_key = keys[m.id] = subs.group_key(m.list_id, sender)
+    if keys:
+        done = set(
+            await db.scalars(
+                select(SubscriptionState.key).where(
+                    SubscriptionState.user_id == user_id,
+                    SubscriptionState.key.in_(set(keys.values())),
+                )
+            )
+        )
+        for env in envelopes:
+            env.unsubscribed = env.subscription_key in done
+
+
+def _visible_in(current_special: str | None) -> ColumnElement[bool]:
+    """Which of a thread's messages a view shows (see ``_REMOVAL_FOLDERS``): Trash/Spam
+    show only their own messages; every other view hides them."""
+    if current_special in _REMOVAL_FOLDERS:
+        return Mailbox.special_use == current_special
+    return or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_REMOVAL_FOLDERS))
+
+
+async def thread_senders(
+    db: AsyncSession,
+    user_id: uuid.UUID,
+    thread_ids: list[uuid.UUID],
+    current_special: str | None = None,
+) -> dict[uuid.UUID, list[Address]]:
+    """Distinct From addresses per thread, oldest message first, over the messages
+    the view shows. Recipients are left out: the list's sender column names who
+    wrote, not everyone copied."""
+    if not thread_ids:
+        return {}
+    rows = await db.execute(
+        select(Message.thread_id, Message.from_addr)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.user_id == user_id,
+            Message.thread_id.in_(thread_ids),
+            _visible_in(current_special),
+        )
+        .order_by(Message.internal_date.asc(), Message.id.asc())
+    )
+    senders: dict[uuid.UUID, list[Address]] = {}
+    seen: dict[uuid.UUID, set[str]] = {}
+    for thread_id, from_addr in rows.all():
+        if thread_id is None:
+            continue
+        for addr in from_addr:
+            email = (addr.get("email") or "").lower()
+            if email and email not in seen.setdefault(thread_id, set()):
+                seen[thread_id].add(email)
+                senders.setdefault(thread_id, []).append(
+                    Address(name=addr.get("name", ""), email=addr.get("email", ""))
+                )
+    return senders
 
 
 async def _resolve_thread(db: AsyncSession, user: User, thread_id: uuid.UUID) -> Thread:
@@ -134,28 +281,78 @@ async def _resolve_thread(db: AsyncSession, user: User, thread_id: uuid.UUID) ->
     return thread
 
 
+# Trash/Spam are message-scoped folders: an individually deleted message lives
+# there on its own. Viewing a thread from Trash/Spam shows ONLY the messages filed
+# there; every other view shows the conversation with trashed/spam messages hidden.
+_REMOVAL_FOLDERS = frozenset({"trash", "junk"})
+
+
 @router.get("/{thread_id}", response_model=ThreadDetail)
 async def get_thread(
     thread_id: uuid.UUID,
     user: CurrentUser,
     db: DbSession,
     sort: str = Query("asc"),
+    view: str = Query("unified"),
 ) -> ThreadDetail:
     thread = await _resolve_thread(db, user, thread_id)
+    current_special = view.split(":", 1)[1] if view.startswith("folder:") else None
     # Messages are always returned ASC (oldest first); the client reverses for
     # display per the requested order (invariant I-5 / §9.4).
-    messages = list(
-        await db.scalars(
+    stmt = (
         select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
         .where(Message.thread_id == thread_id, Message.user_id == user.id)
         .order_by(Message.internal_date.asc(), Message.id.asc())
     )
+    # Trash/Spam: only the messages actually filed there. Conversation view: hide
+    # messages that were individually trashed or spammed.
+    stmt = stmt.where(_visible_in(current_special))
+    messages = list(await db.scalars(stmt))
+
+    # Collapse duplicate deliveries: the same email (same delivery_id) that landed
+    # in more than one of the user's accounts shows once, noting the other accounts.
+    # Messages without a real Message-ID (delivery_id is NULL) never collapse.
+    reps: dict[str, Message] = {}
+    also: dict[str, list[uuid.UUID]] = {}
+    ordered: list[Message] = []
+    for m in messages:
+        key = m.delivery_id or f"id:{m.id}"
+        rep = reps.get(key)
+        if rep is None:
+            reps[key] = m
+            also[key] = []
+            ordered.append(m)
+        elif m.account_id != rep.account_id and m.account_id not in also[key]:
+            also[key].append(m.account_id)
+
+    allowed_senders = allowed_image_senders(user)
+    envelopes = []
+    for m in ordered:
+        env = _envelope(m, allowed_senders)
+        env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"]
+        envelopes.append(env)
+    await _mark_subscriptions(db, user.id, ordered, envelopes)
+    if not blocks_trackers(user) and ordered:
+        # Trackers left in are remote images: offer "Load images" for them too.
+        with_trackers = set(
+            await db.scalars(
+                select(BodyPart.message_id).where(
+                    BodyPart.message_id.in_([m.id for m in ordered]),
+                    BodyPart.kind == "html_sanitized",
+                    BodyPart.trackers_blocked > 0,
                 )
+            )
+        )
+        for env in envelopes:
+            env.has_remote_images = env.has_remote_images or env.id in with_trackers
+
     order = "desc" if sort == "desc" else "asc"
     return ThreadDetail(
         id=thread.id,
         subject=thread.subject_norm or "(no subject)",
         order=order,
         pinned=thread.pinned,
-        messages=[_envelope(m) for m in messages],
+        messages=envelopes,
+        snoozed_until=thread.snoozed_until,
     )

backend/app/api/translation.py

@@ -0,0 +1,30 @@
+"""What email translation offers (app/translation.py), for the reading pane."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+from app import translation
+from app.identity.deps import CurrentUser
+
+router = APIRouter(prefix="/translation", tags=["mail"])
+
+
+class LanguageOut(BaseModel):
+    code: str
+    name: str
+
+
+class TranslationInfoOut(BaseModel):
+    available: bool
+    languages: list[LanguageOut]
+
+
+@router.get("/languages", response_model=TranslationInfoOut)
+async def translation_languages(user: CurrentUser) -> TranslationInfoOut:
+    found = await translation.languages()
+    return TranslationInfoOut(
+        available=len(found) > 1,  # English alone translates nothing
+        languages=[LanguageOut(code=x.code, name=x.name) for x in found],
+    )

backend/app/completion.py

@@ -0,0 +1,391 @@
+"""Sentence completion in the compose window: phrases the user writes, finished.
+
+No AI: a per-user index of the word sequences the user has written at least
+twice, built from their own sent mail (quoted replies and signatures cut off),
+plus a few common email phrases per language to start from. Typing "I look
+forward" finds what usually follows those words — "to hearing from you." — and
+the compose window shows it in grey until Tab takes it.
+
+It works in any language the user writes in: matching is on the words just
+typed, so a Polish sentence only ever meets Polish phrases.
+
+The index lives in ``completion_phrases`` (context → continuation, with how
+often it was seen) and is rebuilt in the background at most once a day, when
+the compose window asks for suggestions (:func:`build_index_job`).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import html as html_lib
+import logging
+import re
+import uuid
+from collections import Counter
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import delete, select
+from sqlalchemy.dialects.postgresql import insert
+
+from app.db import session_scope
+from app.models import (
+    Account,
+    BodyPart,
+    CompletionIndexState,
+    CompletionPhrase,
+    Mailbox,
+    Message,
+)
+
+logger = logging.getLogger(__name__)
+
+START = "^"  # a sentence's start, as a word: "^ pozdr" completes the first word
+MAX_CONTEXT = 3  # words before the cursor matched on
+MAX_CONTINUATION = 6  # words suggested at most
+MIN_COUNT = 2  # seen this often to be offered; one-offs are noise (and private)
+EXTEND_SHARE = 0.6  # a longer suggestion only while it is what usually follows
+CORPUS_MESSAGES = 500  # most recent sent messages read
+CORPUS_CHARS = 3000  # of each
+REBUILD_AFTER = dt.timedelta(days=1)
+
+# Where the user's own words end and quoted mail or a signature begins: a reply
+# header ("On … wrote:", Polish Gmail's "… napisał(a):", …), a forwarded or
+# original-message marker, or an Outlook-style header block.
+_QUOTE_START = re.compile(
+    r"^\s*(?:"
+    r"-{2,}\s*(?:original message|forwarded message|wiadomość oryginalna"
+    r"|wiadomość przekazana dalej|ursprüngliche nachricht|weitergeleitete nachricht)"
+    r"|(?:from|od|von|de):\s.+"
+    r"|sent from my\s"
+    r")",
+    re.IGNORECASE,
+)
+_QUOTE_HEADER_END = re.compile(
+    # German names the writer after the verb: "Am … schrieb Anna:".
+    r"(?:wrote|napisał\(a\)|napisała?|pisze|schrieb(?:\s+.{0,100})?|a écrit|escribió)\s*:\s*$",
+    re.IGNORECASE,
+)
+_SIGNATURE = re.compile(r"^--\s*$")
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+|\n+")
+_WORD = re.compile(r"\S+")
+_EDGE_PUNCT = re.compile(r"^[\"'“”„«»(\[–—-]+|[\"'“”„«»)\],.;:!?…–—-]+$")
+# Words a phrase is never learned across: numbers (phones, dates, prices),
+# addresses and links — useless to complete, and often private.
+_NOT_A_WORD = re.compile(r"\d|@|www\.|https?:", re.IGNORECASE)
+
+
+def html_to_text(html: str) -> str:
+    """An HTML signature as text lines (block tags and <br> break lines)."""
+    text = re.sub(r"(?i)<br\s*/?>|</(?:p|div|li|tr|h[1-6])\s*>", "\n", html)
+    text = re.sub(r"<[^>]+>", "", text)
+    return html_lib.unescape(text)
+
+
+def own_text(text: str, signatures: Iterable[str] = ()) -> str:
+    """The part of a sent message the user wrote: no quoted mail, no signature
+    (the ``--`` kind, or one of ``signatures`` — the accounts' own, as text)."""
+    sig_lines = {norm_line(x) for sig in signatures for x in sig.splitlines() if norm_line(x)}
+    kept: list[str] = []
+    for line in text.splitlines():
+        if _SIGNATURE.match(line) or _QUOTE_START.match(line) or _QUOTE_HEADER_END.search(line):
+            break
+        if norm_line(line) in sig_lines:
+            continue
+        if line.lstrip().startswith(">"):
+            continue  # an inline-quoted line
+        kept.append(line)
+    return "\n".join(kept)
+
+
+def norm_line(line: str) -> str:
+    return " ".join(line.lower().split())
+
+
+def norm(word: str) -> str:
+    """A word as matched: lower case, outer punctuation off ("Hello," -> "hello")."""
+    return _EDGE_PUNCT.sub("", word.lower())
+
+
+def sentences(text: str) -> Iterable[list[str]]:
+    for chunk in _SENTENCE_END.split(text):
+        words = _WORD.findall(chunk)
+        if words:
+            yield words
+
+
+@dataclass(frozen=True)
+class Phrase:
+    context: str  # normalized words before the cursor, space-joined ("^ i look")
+    continuation: str  # the words that follow, as written ("forward to hearing")
+    count: int
+
+
+def build_phrases(texts: Iterable[str], weight: int = 1) -> dict[tuple[str, str], int]:
+    """(context, continuation) -> count, for every sequence seen MIN_COUNT times.
+
+    Counts each sentence's word sequences (with the start marker), keeps the
+    frequent ones, and splits each into "what was typed" and "what follows".
+    """
+    counts: Counter[tuple[str, ...]] = Counter()
+    surface: dict[tuple[str, ...], tuple[str, ...]] = {}
+    longest = MAX_CONTEXT + MAX_CONTINUATION
+    for text in texts:
+        for words in sentences(text):
+            seq = [START, *words]
+            keys = [START, *("" if _NOT_A_WORD.search(w) else norm(w) for w in words)]
+            for i in range(len(seq)):
+                for n in range(2, min(longest, len(seq) - i) + 1):
+                    gram = tuple(keys[i : i + n])
+                    if "" in gram:
+                        break  # a lone punctuation mark: no phrase across it
+                    counts[gram] += weight
+                    surface.setdefault(gram, tuple(seq[i : i + n]))
+    out: dict[tuple[str, str], int] = {}
+    for gram, count in counts.items():
+        if count < MIN_COUNT:
+            continue
+        written = surface[gram]
+        for k in range(1, min(MAX_CONTEXT, len(gram) - 1) + 1):
+            rest = len(gram) - k
+            if rest > MAX_CONTINUATION:
+                continue
+            key = (" ".join(gram[:k]), " ".join(written[k:]))
+            out[key] = max(out.get(key, 0), count)
+    return out
+
+
+# A few common phrases per language, for users with little sent mail yet.
+STARTER_PHRASES: dict[str, list[str]] = {
+    "en": [
+        "I look forward to hearing from you.",
+        "I'm looking forward to hearing from you.",
+        "Looking forward to hearing from you.",
+        "Looking forward to working with you.",
+        "Thank you for your email.",
+        "Thank you for your help.",
+        "Thank you for your quick reply.",
+        "Thanks in advance.",
+        "Please let me know if you have any questions.",
+        "Let me know if you have any questions.",
+        "Let me know what you think.",
+        "Please find attached the",
+        "I hope this email finds you well.",
+        "I hope you're doing well.",
+        "Sorry for the late reply.",
+        "I would be happy to help.",
+        "Feel free to contact me if you need anything else.",
+        "Could you please send me the",
+        "Please confirm at your earliest convenience.",
+        "Have a great day!",
+        "Have a nice weekend!",
+        "Best regards,",
+        "Kind regards,",
+    ],
+    "pl": [
+        "Z poważaniem,",
+        "Pozdrawiam serdecznie,",
+        "Dziękuję za wiadomość.",
+        "Dziękuję za szybką odpowiedź.",
+        "Dziękuję za pomoc.",
+        "Z góry dziękuję.",
+        "W załączniku przesyłam",
+        "Proszę o informację zwrotną.",
+        "W razie pytań proszę o kontakt.",
+        "Czekam na odpowiedź.",
+        "Przepraszam za opóźnienie w odpowiedzi.",
+        "Będę wdzięczny za odpowiedź.",
+        "Będę wdzięczna za odpowiedź.",
+        "Zgodnie z naszą rozmową",
+        "Proszę o potwierdzenie.",
+        "Daj znać, co myślisz.",
+        "Miłego dnia!",
+        "Miłego weekendu!",
+    ],
+    "de": [
+        "Mit freundlichen Grüßen",
+        "Viele Grüße",
+        "Beste Grüße",
+        "Vielen Dank für Ihre E-Mail.",
+        "Vielen Dank für Ihre schnelle Antwort.",
+        "Vielen Dank im Voraus.",
+        "Ich freue mich auf Ihre Antwort.",
+        "Bei Fragen stehe ich Ihnen gerne zur Verfügung.",
+        "Im Anhang finden Sie",
+        "Entschuldigen Sie die späte Antwort.",
+        "Ich hoffe, es geht Ihnen gut.",
+        "Schönen Tag noch!",
+        "Schönes Wochenende!",
+    ],
+}
+
+# Built once: every starter phrase counts as seen MIN_COUNT times.
+_STARTERS: dict[str, Counter[str]] = {}
+for (_ctx, _cont), _n in build_phrases(
+    (p for phrases in STARTER_PHRASES.values() for p in phrases), weight=MIN_COUNT
+).items():
+    _STARTERS.setdefault(_ctx, Counter())[_cont] = _n
+
+_LEADING_PUNCT = re.compile(r"^[\"'“”„«»(\[]+")
+
+
+def _candidates(phrases: dict[str, Counter[str]], context: str, partial: str) -> Counter[str]:
+    """Continuations of ``context`` whose first word starts with ``partial``."""
+    out: Counter[str] = Counter()
+    for continuation, count in phrases.get(context, Counter()).items():
+        first = continuation.split(" ", 1)[0]
+        if partial and not first.lower().startswith(partial.lower()):
+            continue
+        if partial and len(first) <= len(partial):
+            continue  # that word is typed in full already
+        out[continuation] = count
+    return out
+
+
+def choose(candidates: Counter[str]) -> str | None:
+    """The suggestion: the most frequent next word, extended word by word only
+    while the longer phrase is what usually follows (EXTEND_SHARE)."""
+    if not candidates:
+        return None
+    by_first: Counter[str] = Counter()
+    for continuation, count in candidates.items():
+        first = continuation.split(" ", 1)[0]
+        by_first[first] = max(by_first[first], count)
+    chosen, chosen_count = by_first.most_common(1)[0]
+    while True:
+        depth = chosen.count(" ") + 1
+        longer = [
+            (c, n)
+            for c, n in candidates.items()
+            if c.startswith(chosen + " ") and c.count(" ") == depth
+        ]
+        if not longer:
+            return chosen
+        best, best_count = max(longer, key=lambda x: x[1])
+        if best_count < max(MIN_COUNT, EXTEND_SHARE * chosen_count):
+            return chosen
+        chosen, chosen_count = best, best_count
+
+
+def contexts(before: str) -> tuple[list[str], str]:
+    """For the text before the cursor: the contexts to try, longest first, and
+    the partly typed word ('' after a space). No contexts: nothing to suggest
+    (an empty sentence, or the cursor right after a punctuation mark)."""
+    if not before or not before.strip():
+        return [], ""
+    last = before[-1]
+    if not last.isspace() and not last.isalnum() and last not in "'’-":
+        return [], ""  # "you," or "end." — not mid-word, and no space typed yet
+    tail = _SENTENCE_END.split(before)[-1]
+    words = _WORD.findall(tail)
+    partial = ""
+    if words and not last.isspace():
+        partial = _LEADING_PUNCT.sub("", words.pop())
+    if not words and not partial:
+        return [], ""  # a new sentence with nothing in it yet
+    keys = [START, *(norm(w) for w in words)]
+    if "" in keys:  # a lone punctuation mark: the phrase starts after it
+        keys = keys[len(keys) - keys[::-1].index("") :]
+    if not keys:
+        return [], ""
+    return [" ".join(keys[-k:]) for k in range(min(MAX_CONTEXT, len(keys)), 0, -1)], partial
+
+
+def suggest(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """What to show after ``before`` (the text up to the cursor), or None.
+
+    The returned text starts where the cursor is: the rest of a partly typed
+    word, or (after a space) the next words. The user's own phrases and the
+    starter phrases count together.
+    """
+    tried, partial = contexts(before)
+    for context in tried:
+        merged = _candidates(personal, context, partial)
+        for cont, count in _candidates(_STARTERS, context, partial).items():
+            merged[cont] = merged.get(cont, 0) + count
+        if " " not in context and not partial:
+            # A single word before a space is a weak hint: only a strong habit.
+            merged = Counter({c: n for c, n in merged.items() if n >= 2 * MIN_COUNT})
+        best = choose(merged)
+        if best:
+            return best[len(partial) :] if partial else best
+    return None
+
+
+def complete(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """:func:`suggest`, and for a word typed in full but no space yet ("I look
+    forward"), what follows it — with the space ("␣to hearing from you.")."""
+    found = suggest(personal, before)
+    if found is None and before[-1:].isalnum():
+        after_space = suggest(personal, before + " ")
+        if after_space:
+            return " " + after_space
+    return found
+
+
+def keys_for(before: str) -> list[str]:
+    """The contexts :func:`complete` may look up, to fetch them from the index."""
+    keys = contexts(before)[0]
+    if before[-1:].isalnum():
+        keys += contexts(before + " ")[0]
+    return keys
+
+
+async def personal_phrases(
+    session: Any, user_id: uuid.UUID, keys: list[str]
+) -> dict[str, Counter[str]]:
+    rows = await session.execute(
+        select(
+            CompletionPhrase.context, CompletionPhrase.continuation, CompletionPhrase.count
+        ).where(CompletionPhrase.user_id == user_id, CompletionPhrase.context.in_(keys))
+    )
+    out: dict[str, Counter[str]] = {}
+    for context, continuation, count in rows:
+        out.setdefault(context, Counter())[continuation] = count
+    return out
+
+
+async def build_index_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Rebuild a user's phrase index from their recent sent mail. Returns its size."""
+    uid = uuid.UUID(user_id)
+    async with session_scope() as session:
+        texts = list(
+            await session.scalars(
+                select(BodyPart.content)
+                .join(Message, Message.id == BodyPart.message_id)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.user_id == uid,
+                    Mailbox.special_use == "sent",
+                    BodyPart.kind == "text",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(CORPUS_MESSAGES)
+            )
+        )
+        signatures = [
+            html_to_text(sig)
+            for sig in await session.scalars(
+                select(Account.signature).where(Account.user_id == uid, Account.signature != "")
+            )
+        ]
+        phrases = build_phrases(own_text(t, signatures)[:CORPUS_CHARS] for t in texts)
+        await session.execute(delete(CompletionPhrase).where(CompletionPhrase.user_id == uid))
+        rows = [
+            {"user_id": uid, "context": c, "continuation": cont, "count": n}
+            for (c, cont), n in phrases.items()
+        ]
+        for start in range(0, len(rows), 1000):
+            await session.execute(insert(CompletionPhrase), rows[start : start + 1000])
+        stmt = insert(CompletionIndexState).values(user_id=uid, built_at=dt.datetime.now(dt.UTC))
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=["user_id"], set_={"built_at": stmt.excluded.built_at}
+            )
+        )
+        await session.commit()
+    logger.info(
+        "completion index for %s: %d phrases from %d messages", user_id, len(rows), len(texts)
+    )
+    return len(rows)

backend/app/concurrency.py

@@ -0,0 +1,32 @@
+"""Per-event-loop semaphores for bounded concurrency (refactor §4, §7).
+
+Connection and parse caps are process-local but must be bound to the running
+event loop: an :class:`asyncio.Semaphore` created on one loop cannot be awaited
+on another (a real hazard under pytest, which uses a fresh loop per test). These
+helpers hand back a semaphore keyed by ``(group, key)`` for the current loop,
+creating it on first use and dropping the whole set when the loop is collected.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import weakref
+
+# group -> {loop: {key: Semaphore}}. WeakKeyDictionary drops a loop's semaphores
+# once the loop is gone, so no state leaks across tests or restarts.
+_registry: dict[
+    str, weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Semaphore]]
+] = {}
+
+
+def loop_semaphore(group: str, key: str, cap: int) -> asyncio.Semaphore:
+    """Return the current loop's semaphore for ``(group, key)`` (cap on first use)."""
+    loop = asyncio.get_running_loop()
+    per_group = _registry.setdefault(group, weakref.WeakKeyDictionary())
+    per_loop = per_group.get(loop)
+    if per_loop is None:
+        per_loop = per_group[loop] = {}
+    sem = per_loop.get(key)
+    if sem is None:
+        sem = per_loop[key] = asyncio.Semaphore(cap)
+    return sem

backend/app/config.py

@@ -29,10 +29,12 @@ class Settings(BaseSettings):
     meili_key: str = Field(default="", alias="MEILI_KEY")
 
     undo_seconds: int = Field(default=10, alias="UNDO_SECONDS")
-    poll_interval: int = Field(default=120, alias="POLL_INTERVAL")
     session_ttl_days: int = Field(default=30, alias="SESSION_TTL_DAYS")
     max_attachment_mb: int = Field(default=25, alias="MAX_ATTACHMENT_MB")
     log_level: str = Field(default="INFO", alias="LOG_LEVEL")
+    # Release identifier baked into the container image at build time; surfaced
+    # by GET /api/version so a deploy can verify what is actually running.
+    app_version: str = Field(default="dev", alias="APP_VERSION")
 
     # --- Sync engine v2 (workers layer) --------------------------------------
     # Worker role: "sync" (lease-owned account runners) | "jobs" (arq consumer)
@@ -44,14 +46,78 @@ class Settings(BaseSettings):
     max_accounts_per_worker: int = Field(default=400, alias="MAX_ACCOUNTS_PER_WORKER")
     lease_ttl: int = Field(default=60, alias="LEASE_TTL")
     lease_renew: int = Field(default=20, alias="LEASE_RENEW")
+    # Activity tiers (§5): hot = persistent IDLE, warm/cold = transient polls.
+    poll_warm: int = Field(default=180, alias="POLL_WARM")
+    poll_cold: int = Field(default=900, alias="POLL_COLD")
+    tier_hot_activity_h: int = Field(default=24, alias="TIER_HOT_ACTIVITY_H")
+    tier_session_grace_min: int = Field(default=30, alias="TIER_SESSION_GRACE_MIN")
+    tier_cold_after_d: int = Field(default=3, alias="TIER_COLD_AFTER_D")
+    hot_budget_per_worker: int = Field(default=250, alias="HOT_BUDGET_PER_WORKER")
+    # Memory/connection bounds (§7, §4): peak RAM ~= PARSE_CONCURRENCY x STREAM_PARSE_MB.
+    per_host_conn_cap: int = Field(default=50, alias="PER_HOST_CONN_CAP")
+    parse_concurrency: int = Field(default=4, alias="PARSE_CONCURRENCY")
+    stream_parse_mb: int = Field(default=8, alias="STREAM_PARSE_MB")
 
     oauth_google_client_id: str = Field(default="", alias="OAUTH_GOOGLE_CLIENT_ID")
     oauth_google_client_secret: str = Field(default="", alias="OAUTH_GOOGLE_CLIENT_SECRET")
     oauth_ms_client_id: str = Field(default="", alias="OAUTH_MS_CLIENT_ID")
     oauth_ms_client_secret: str = Field(default="", alias="OAUTH_MS_CLIENT_SECRET")
+    # Gates the "Sign in with Google instead" OAuth fallback in the wizard (SPEC
+    # §6.2.1). Off until an operator configures a verified Cloud project.
+    google_oauth_enabled: bool = Field(default=False, alias="GOOGLE_OAUTH_ENABLED")
+    # Server-side sender avatars (Gravatar + domain favicon). Fetched by the server
+    # (never the browser) so senders can't see the user's IP or read activity.
+    sender_avatars_enabled: bool = Field(default=True, alias="SENDER_AVATARS_ENABLED")
+    # Spelling and grammar checking in the compose window (LanguageTool server,
+    # e.g. http://languagetool:8010). Empty: the check is off for everyone.
+    languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL")
+    # Email translation: the translator service (translator/, app/translation.py),
+    # e.g. http://translator:5000. Empty: translation is off.
+    translator_url: str = Field(default="", alias="TRANSLATOR_URL")
+    # New passwords are checked against known data breaches (Have I Been Pwned's
+    # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server).
+    password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK")
+    # The name the webmail gives itself when it connects to a mail server to
+    # send (SMTP EHLO). Empty: APP_DOMAIN — not the container's random name.
+    smtp_helo_name: str = Field(default="", alias="SMTP_HELO_NAME")
+    # Auto-purge Trash/Spam messages older than this many days (0 disables).
+    trash_retention_days: int = Field(default=30, alias="TRASH_RETENTION_DAYS")
+
+    # --- Hosted mailboxes (SPEC Appendix D, app/hosted/) -----------------------
+    # Off: no mail server, byte-identical to before (every account external).
+    hosted_mailboxes: bool = Field(default=False, alias="HOSTED_MAILBOXES")
+    # The mail server's management API (JMAP), signed in to as its recovery admin.
+    stalwart_url: str = Field(default="http://stalwart:8080", alias="STALWART_URL")
+    stalwart_admin_secret: str = Field(default="", alias="STALWART_ADMIN_SECRET")
+    # The webmail worker's own sign-in, which may act for any mailbox
+    # (``user%worker``): no user's password is ever stored (§D.3).
+    stalwart_master_secret: str = Field(default="", alias="STALWART_MASTER_SECRET")
+    # The mail server's role on Postgres (its database, and the directory view).
+    stalwart_db_password: str = Field(default="", alias="STALWART_DB_PASSWORD")
+    stalwart_db_name: str = Field(default="stalwart", alias="STALWART_DB_NAME")
+    # Where the mail server reaches Postgres (its own network view of it).
+    stalwart_pg_host: str = Field(default="postgres", alias="STALWART_PG_HOST")
+    # IMAP/SMTP as the webmail reaches them (TLS, certificate not checked: the
+    # traffic stays on the Docker network or loopback).
+    stalwart_imap_host: str = Field(default="stalwart", alias="STALWART_IMAP_HOST")
+    stalwart_imap_port: int = Field(default=993, alias="STALWART_IMAP_PORT")
+    stalwart_smtp_host: str = Field(default="stalwart", alias="STALWART_SMTP_HOST")
+    stalwart_smtp_port: int = Field(default=465, alias="STALWART_SMTP_PORT")
+    # The mail server's public name (SMTP greeting, reverse DNS, what mail apps
+    # are told to connect to). Default: mail.APP_DOMAIN.
+    mail_hostname: str = Field(default="", alias="MAIL_HOSTNAME")
+    # Raw messages of hosted mailboxes are kept only in a cache (§D.6 item 2);
+    # beyond this per user, the least recently used go first.
+    raw_cache_mb: int = Field(default=256, alias="RAW_CACHE_MB")
 
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
     """Return the process-wide settings singleton."""
     return Settings()  # values are sourced from the environment
+
+
+def smtp_helo_name() -> str:
+    """The name to introduce ourselves with to SMTP servers (EHLO)."""
+    settings = get_settings()
+    return settings.smtp_helo_name.strip() or settings.app_domain

backend/app/dav/__init__.py

@@ -0,0 +1 @@
+"""CalDAV / CardDAV server (SPEC Appendix E)."""

backend/app/dav/auth.py

@@ -0,0 +1,93 @@
+"""Signing in a DAV client: HTTP Basic with an app password (SPEC App. E.2).
+
+Only app passwords: the account password (with or without 2FA) never works
+here, so a lost phone is one revocation away and 2FA cannot be bypassed. A
+sync makes many requests, each with the credentials; a successful check is
+remembered for a few minutes under a hash of the password, and forgotten as
+soon as the user's app passwords change (``app_passwords`` bumps a counter).
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import datetime as dt
+import hashlib
+import uuid
+
+from sqlalchemy import select, update
+
+from app.db import session_scope
+from app.identity import app_passwords
+from app.identity.service import resolve_handle
+from app.models import User, UserAppPassword
+from app.redis_client import get_redis
+
+CACHE_S = 300
+FAIL_WINDOW_S = 900
+MAX_FAILS_PER_ADDRESS = 10
+MAX_FAILS_PER_IP = 30
+_LAST_USED_EVERY = dt.timedelta(minutes=5)
+
+
+class Throttled(Exception):
+    """Too many failed sign-ins recently."""
+
+
+def _credentials(header: str | None) -> tuple[str, str] | None:
+    if not header or not header.lower().startswith("basic "):
+        return None
+    try:
+        decoded = base64.b64decode(header[6:].strip(), validate=True).decode("utf-8")
+    except (binascii.Error, UnicodeDecodeError):
+        return None
+    login, sep, password = decoded.partition(":")
+    return (login.strip(), password) if sep and login.strip() else None
+
+
+async def authenticate(header: str | None, ip: str) -> User | None:
+    """The user these Basic credentials sign in, or None. Raises Throttled."""
+    creds = _credentials(header)
+    if creds is None:
+        return None
+    login, password = creds
+    redis = get_redis()
+    handle = resolve_handle(login)
+    fail_keys = (f"dav:fail:addr:{handle}", f"dav:fail:ip:{ip}")
+    fails = [int(await redis.get(k) or 0) for k in fail_keys]
+    if fails[0] >= MAX_FAILS_PER_ADDRESS or fails[1] >= MAX_FAILS_PER_IP:
+        raise Throttled
+
+    async with session_scope() as db:
+        user = await db.scalar(select(User).where(User.handle == handle))
+        if user is None or user.blocked_at is not None:
+            await _failed(fail_keys)
+            return None
+        gen = int(await redis.get(app_passwords.generation_key(user.id)) or 0)
+        digest = hashlib.sha256(f"{user.id}:{password}".encode()).hexdigest()
+        cache_key = f"dav:auth:{user.id}:{gen}:{digest}"
+        if await redis.get(cache_key):
+            return user
+        row = await app_passwords.verify(db, user.id, password)
+        if row is None:
+            await _failed(fail_keys)
+            return None
+        await redis.set(cache_key, str(row.id), ex=CACHE_S)
+        now = dt.datetime.now(dt.UTC)
+        if row.last_used_at is None or now - row.last_used_at > _LAST_USED_EVERY:
+            await db.execute(
+                update(UserAppPassword).where(UserAppPassword.id == row.id).values(last_used_at=now)
+            )
+        db.expunge(user)
+        return user
+
+
+async def _failed(keys: tuple[str, ...]) -> None:
+    redis = get_redis()
+    for key in keys:
+        await redis.incr(key)
+        await redis.expire(key, FAIL_WINDOW_S)
+
+
+def user_id_of(user: User) -> uuid.UUID:
+    return user.id

backend/app/dav/provider.py

@@ -0,0 +1,118 @@
+"""What a plugin implements to serve its collections over DAV (SPEC App. E.1).
+
+The core (app/dav/routes.py) speaks the protocol; a provider only knows its
+own data: collections (calendars, address books) and the resources in them
+(one iCalendar object / one vCard each), named, with an ETag. Every method
+gets the signed-in user's id; providers open the user's RLS session
+(``sdk(name).session(user_id)``) themselves.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Literal, Protocol
+
+
+class DavError(Exception):
+    """A request the provider refuses: ``status`` is the HTTP answer
+    (403 read-only, 404 missing, 409 conflict, 412 precondition failed, 415
+    unsupported data); ``condition`` an optional DAV precondition element
+    (e.g. ``{urn:ietf:params:xml:ns:caldav}valid-calendar-data``)."""
+
+    def __init__(self, status: int, message: str = "", condition: str | None = None) -> None:
+        super().__init__(message or str(status))
+        self.status = status
+        self.condition = condition
+
+
+@dataclass
+class Collection:
+    id: str
+    displayname: str
+    sync_token: int  # the latest change number; also the CTag
+    description: str = ""
+    color: str | None = None  # "#rrggbb"
+    read_only: bool = False
+    components: list[str] = field(default_factory=lambda: ["VEVENT"])  # calendars only
+
+
+@dataclass
+class Resource:
+    name: str  # last path segment, e.g. "abc.ics"
+    etag: str  # without quotes
+    data: str | None = None  # the iCalendar / vCard text, when asked for
+
+
+@dataclass
+class Changes:
+    sync_token: int
+    changed: list[str]  # resource names changed or added since the old token
+    deleted: list[str]  # resource names gone since the old token
+
+
+class DavProvider(Protocol):
+    kind: Literal["calendar", "addressbook"]
+    content_type: str  # "text/calendar; charset=utf-8" / "text/vcard; charset=utf-8"
+    extension: str  # ".ics" / ".vcf"
+
+    async def collections(self, user_id: uuid.UUID) -> list[Collection]: ...
+
+    async def resources(self, user_id: uuid.UUID, collection_id: str) -> list[Resource]: ...
+
+    async def get(
+        self, user_id: uuid.UUID, collection_id: str, names: list[str] | None
+    ) -> list[Resource]: ...
+
+    async def put(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        name: str,
+        data: str,
+        *,
+        if_match: str | None,
+        if_none_match: bool,
+    ) -> tuple[str, bool]:
+        """Store ``data`` as ``name``; returns (new ETag, created)."""
+        ...
+
+    async def delete(
+        self, user_id: uuid.UUID, collection_id: str, name: str, *, if_match: str | None
+    ) -> None: ...
+
+    async def changes(self, user_id: uuid.UUID, collection_id: str, since: int) -> Changes | None:
+        """What changed after change number ``since``; None when that token is
+        too old to answer (the client then resyncs from scratch)."""
+        ...
+
+    async def create_collection(
+        self, user_id: uuid.UUID, collection_id: str, displayname: str, color: str | None
+    ) -> None:
+        """Create a collection at the id the client chose (MKCALENDAR / MKCOL);
+        DavError(403) when that id cannot be used."""
+        ...
+
+    async def update_collection(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        *,
+        displayname: str | None = None,
+        color: str | None = None,
+        description: str | None = None,
+    ) -> None: ...
+
+    async def delete_collection(self, user_id: uuid.UUID, collection_id: str) -> None: ...
+
+    async def query(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        start: datetime | None,
+        end: datetime | None,
+    ) -> list[str] | None:
+        """Calendars: names of resources with an occurrence overlapping the
+        range; None when the provider cannot filter (the core then returns all)."""
+        ...

backend/app/dav/routes.py

@@ -0,0 +1,595 @@
+"""The CalDAV / CardDAV endpoints (SPEC Appendix E).
+
+    /.well-known/caldav, /.well-known/carddav   → 301 /dav/
+    /dav/                                        root: points at the principal
+    /dav/principals/{user}/                      the signed-in user
+    /dav/calendars/{user}/[{calendar}/[{name}]]  calendar home, calendars, events
+    /dav/addressbooks/{user}/[{book}/[{name}]]   address-book home, books, contacts
+
+The data comes from the plugins' providers (app/dav/provider.py): the calendar
+plugin's for ``calendars``, the contacts plugin's for ``addressbooks``. Signed
+in with HTTP Basic and an app password only (app/dav/auth.py); session cookies
+are never looked at, so a web page cannot drive these endpoints.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+import xml.etree.ElementTree as ET
+from typing import Any
+from urllib.parse import quote, unquote, urlsplit
+
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import RedirectResponse
+from sqlalchemy import select
+
+from app.dav import auth
+from app.dav.provider import Collection, DavError, DavProvider, Resource
+from app.dav.xml import (
+    CR,
+    CS,
+    A,
+    BadRequest,
+    C,
+    D,
+    Multistatus,
+    PropValue,
+    el,
+    error_body,
+    href,
+    mkcol_props,
+    parse,
+    propfind_request,
+    proppatch_request,
+    q,
+    report_request,
+)
+from app.db import session_scope
+from app.models import Account, User
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+METHODS = [
+    "OPTIONS",
+    "PROPFIND",
+    "PROPPATCH",
+    "REPORT",
+    "GET",
+    "HEAD",
+    "PUT",
+    "DELETE",
+    "MKCALENDAR",
+    "MKCOL",
+]
+DAV_HEADER = "1, 3, calendar-access, addressbook, extended-mkcol"
+XML_TYPE = "application/xml; charset=utf-8"
+HOMES = {"calendars": "calendar", "addressbooks": "addressbook"}
+SYNC_PREFIX = "http://ultimo.email/ns/sync/"
+
+
+def _providers() -> dict[str, DavProvider]:
+    from app.plugins import loader
+
+    out: dict[str, DavProvider] = {}
+    for plugin in loader.get_plugins():
+        if plugin.dav is not None:
+            out[plugin.dav.kind] = plugin.dav
+    return out
+
+
+# ---- entry points ------------------------------------------------------------------
+
+
+@router.api_route("/.well-known/caldav", methods=["GET", "HEAD", "PROPFIND", "OPTIONS"])
+@router.api_route("/.well-known/carddav", methods=["GET", "HEAD", "PROPFIND", "OPTIONS"])
+async def well_known() -> Response:
+    return RedirectResponse("/dav/", status_code=301)
+
+
+@router.api_route("/dav", methods=METHODS)
+@router.api_route("/dav/{path:path}", methods=METHODS)
+async def dav(request: Request, path: str = "") -> Response:
+    method = request.method.upper()
+    if method == "OPTIONS":
+        return Response(
+            status_code=200,
+            headers={"DAV": DAV_HEADER, "Allow": ", ".join(METHODS), "Content-Length": "0"},
+        )
+    ip = request.client.host if request.client else ""
+    try:
+        user = await auth.authenticate(request.headers.get("authorization"), ip)
+    except auth.Throttled:
+        return Response(status_code=429, headers={"Retry-After": "900"})
+    if user is None:
+        return Response(
+            status_code=401,
+            headers={"WWW-Authenticate": 'Basic realm="Ultimo", charset="UTF-8"'},
+        )
+    parts = [unquote(p) for p in path.split("/") if p]
+    ctx = _Ctx(request, user, parts)
+    try:
+        return await ctx.handle(method)
+    except BadRequest as err:
+        return Response(status_code=400, content=str(err))
+    except DavError as err:
+        body = error_body(err.condition) if err.condition else b""
+        return Response(
+            status_code=err.status,
+            content=body,
+            media_type=XML_TYPE if body else None,
+        )
+
+
+class _Ctx:
+    def __init__(self, request: Request, user: User, parts: list[str]) -> None:
+        self.request = request
+        self.user = user
+        self.uid = str(user.id)
+        self.parts = parts
+        self.providers = _providers()
+
+    # -- paths -----------------------------------------------------------------------
+
+    def principal(self) -> str:
+        return f"/dav/principals/{self.uid}/"
+
+    def home(self, segment: str) -> str:
+        return f"/dav/{segment}/{self.uid}/"
+
+    def coll_href(self, segment: str, coll_id: str) -> str:
+        return f"{self.home(segment)}{quote(coll_id, safe='')}/"
+
+    def res_href(self, segment: str, coll_id: str, name: str) -> str:
+        return f"{self.coll_href(segment, coll_id)}{quote(name, safe='')}"
+
+    # -- dispatch --------------------------------------------------------------------
+
+    async def handle(self, method: str) -> Response:
+        p = self.parts
+        if not p:
+            return await self.root(method)
+        if p[0] == "principals":
+            if len(p) != 2 or p[1] != self.uid:
+                raise DavError(404)
+            return await self.principal_req(method)
+        if p[0] in HOMES and len(p) >= 2:
+            if p[1] != self.uid:
+                raise DavError(403)
+            provider = self.providers.get(HOMES[p[0]])
+            if provider is None:
+                raise DavError(404)
+            if len(p) == 2:
+                return await self.home_req(method, p[0], provider)
+            if len(p) == 3:
+                return await self.collection_req(method, p[0], provider, p[2])
+            if len(p) == 4:
+                return await self.resource_req(method, p[0], provider, p[2], p[3])
+        raise DavError(404)
+
+    async def body(self) -> ET.Element | None:
+        return parse(await self.request.body())
+
+    def depth(self) -> int:
+        value = (self.request.headers.get("depth") or "1").strip().lower()
+        return 0 if value == "0" else 1
+
+    @staticmethod
+    def multistatus(ms: Multistatus) -> Response:
+        return Response(status_code=207, content=ms.to_bytes(), media_type=XML_TYPE)
+
+    async def respond_props(
+        self, items: list[tuple[str, dict[str, PropValue]]], method_root: ET.Element | None
+    ) -> Response:
+        req = propfind_request(method_root)
+        ms = Multistatus()
+        for path, props in items:
+            if req.names_only:
+                ms.response(path, dict.fromkeys(props))
+            elif req.names is None:
+                ms.response(path, props)
+            else:
+                found = {n: props[n] for n in req.names if n in props}
+                missing = [n for n in req.names if n not in props]
+                ms.response(path, found, missing)
+        return self.multistatus(ms)
+
+    # -- root and principal ----------------------------------------------------------
+
+    def common(self) -> dict[str, PropValue]:
+        return {
+            q(D, "current-user-principal"): [href(self.principal())],
+            q(D, "principal-URL"): [href(self.principal())],
+            q(D, "owner"): [href(self.principal())],
+        }
+
+    async def root(self, method: str) -> Response:
+        if method != "PROPFIND":
+            return Response(status_code=200 if method in ("GET", "HEAD") else 405)
+        props = {**self.common(), q(D, "resourcetype"): [el(q(D, "collection"))]}
+        return await self.respond_props([("/dav/", props)], await self.body())
+
+    async def addresses(self) -> list[str]:
+        async with session_scope() as db:
+            emails = list(
+                await db.scalars(select(Account.email).where(Account.user_id == self.user.id))
+            )
+        out = [self.user.handle, *emails]
+        return list(dict.fromkeys(e.lower() for e in out))
+
+    async def principal_props(self) -> dict[str, PropValue]:
+        props = {
+            **self.common(),
+            q(D, "resourcetype"): [el(q(D, "principal")), el(q(D, "collection"))],
+            q(D, "displayname"): self.user.handle,
+            q(C, "calendar-user-address-set"): [
+                href(f"mailto:{a}") for a in await self.addresses()
+            ],
+            q(C, "calendar-user-type"): "INDIVIDUAL",
+            q(D, "principal-collection-set"): [href("/dav/principals/")],
+            q(D, "supported-report-set"): [],
+        }
+        if "calendar" in self.providers:
+            props[q(C, "calendar-home-set")] = [href(self.home("calendars"))]
+        if "addressbook" in self.providers:
+            props[q(CR, "addressbook-home-set")] = [href(self.home("addressbooks"))]
+        return props
+
+    async def principal_req(self, method: str) -> Response:
+        if method == "PROPFIND":
+            return await self.respond_props(
+                [(self.principal(), await self.principal_props())], await self.body()
+            )
+        if method == "REPORT":  # principal-property-search, expand-property: nothing to add
+            return self.multistatus(Multistatus())
+        return Response(status_code=405)
+
+    # -- homes -------------------------------------------------------------------------
+
+    def privileges(self, read_only: bool) -> list[ET.Element]:
+        names = ["read", "read-current-user-privilege-set"]
+        if not read_only:
+            names += ["write", "write-content", "write-properties", "bind", "unbind"]
+        return [el(q(D, "privilege"), children=[el(q(D, n))]) for n in names]
+
+    def reports(self, kind: str) -> list[ET.Element]:
+        ns, reports = (
+            (C, ["calendar-query", "calendar-multiget"])
+            if kind == "calendar"
+            else (CR, ["addressbook-query", "addressbook-multiget"])
+        )
+        items = [(ns, r) for r in reports] + [(D, "sync-collection")]
+        return [
+            el(q(D, "supported-report"), children=[el(q(D, "report"), children=[el(q(n, r))])])
+            for n, r in items
+        ]
+
+    async def home_req(self, method: str, segment: str, provider: DavProvider) -> Response:
+        if method == "PROPFIND":
+            home_props: dict[str, PropValue] = {
+                **self.common(),
+                q(D, "resourcetype"): [el(q(D, "collection"))],
+                q(D, "displayname"): "Calendars" if provider.kind == "calendar" else "Contacts",
+                q(D, "current-user-privilege-set"): self.privileges(False),
+            }
+            items = [(self.home(segment), home_props)]
+            if self.depth():
+                for coll in await provider.collections(self.user.id):
+                    items.append(
+                        (self.coll_href(segment, coll.id), self.collection_props(provider, coll))
+                    )
+            return await self.respond_props(items, await self.body())
+        if method == "REPORT":
+            return self.multistatus(Multistatus())
+        if method in ("GET", "HEAD"):
+            return Response(status_code=200)
+        return Response(status_code=405)
+
+    # -- collections -------------------------------------------------------------------
+
+    def collection_props(self, provider: DavProvider, coll: Collection) -> dict[str, PropValue]:
+        token = f"{SYNC_PREFIX}{coll.sync_token}"
+        props: dict[str, PropValue] = {
+            **self.common(),
+            q(D, "displayname"): coll.displayname,
+            q(D, "current-user-privilege-set"): self.privileges(coll.read_only),
+            q(D, "supported-report-set"): self.reports(provider.kind),
+            q(D, "sync-token"): token,
+            q(CS, "getctag"): str(coll.sync_token),
+        }
+        if provider.kind == "calendar":
+            props[q(D, "resourcetype")] = [el(q(D, "collection")), el(q(C, "calendar"))]
+            props[q(C, "supported-calendar-component-set")] = [
+                el(q(C, "comp"), attrs={"name": comp}) for comp in coll.components
+            ]
+            props[q(C, "supported-calendar-data")] = [
+                el(q(C, "calendar-data"), attrs={"content-type": "text/calendar", "version": "2.0"})
+            ]
+            props[q(C, "calendar-description")] = coll.description
+            if coll.color:
+                props[q(A, "calendar-color")] = coll.color
+        else:
+            props[q(D, "resourcetype")] = [el(q(D, "collection")), el(q(CR, "addressbook"))]
+            props[q(CR, "supported-address-data")] = [
+                el(
+                    q(CR, "address-data-type"),
+                    attrs={"content-type": "text/vcard", "version": "3.0"},
+                ),
+                el(
+                    q(CR, "address-data-type"),
+                    attrs={"content-type": "text/vcard", "version": "4.0"},
+                ),
+            ]
+            props[q(CR, "addressbook-description")] = coll.description
+        return props
+
+    async def find(self, provider: DavProvider, coll_id: str) -> Collection:
+        for coll in await provider.collections(self.user.id):
+            if coll.id == coll_id:
+                return coll
+        raise DavError(404)
+
+    def resource_props(
+        self, provider: DavProvider, res: Resource, with_data: bool
+    ) -> dict[str, PropValue]:
+        props: dict[str, PropValue] = {
+            q(D, "getetag"): f'"{res.etag}"',
+            q(D, "getcontenttype"): provider.content_type,
+            q(D, "resourcetype"): [],
+        }
+        if with_data and res.data is not None:
+            data_prop = (
+                q(C, "calendar-data") if provider.kind == "calendar" else q(CR, "address-data")
+            )
+            props[data_prop] = res.data
+        return props
+
+    async def collection_req(
+        self, method: str, segment: str, provider: DavProvider, coll_id: str
+    ) -> Response:
+        if method == "MKCALENDAR" or method == "MKCOL":
+            return await self.make_collection(segment, provider, coll_id)
+        coll = await self.find(provider, coll_id)
+        if method == "PROPFIND":
+            items = [(self.coll_href(segment, coll.id), self.collection_props(provider, coll))]
+            if self.depth():
+                for res in await provider.resources(self.user.id, coll.id):
+                    items.append(
+                        (
+                            self.res_href(segment, coll.id, res.name),
+                            self.resource_props(provider, res, False),
+                        )
+                    )
+            return await self.respond_props(items, await self.body())
+        if method == "REPORT":
+            return await self.report(segment, provider, coll)
+        if method == "PROPPATCH":
+            return await self.proppatch(segment, provider, coll)
+        if method == "DELETE":
+            if coll.read_only:
+                raise DavError(403)
+            await provider.delete_collection(self.user.id, coll.id)
+            return Response(status_code=204)
+        if method in ("GET", "HEAD"):
+            return Response(status_code=200)
+        return Response(status_code=405)
+
+    async def make_collection(self, segment: str, provider: DavProvider, coll_id: str) -> Response:
+        if any(c.id == coll_id for c in await provider.collections(self.user.id)):
+            raise DavError(405)  # exists already
+        root = await self.body()
+        props = mkcol_props(root)
+        rtype = props.get(q(D, "resourcetype"))
+        if self.request.method.upper() == "MKCOL" and (
+            rtype is None
+            or rtype.find(q(CR, "addressbook")) is None
+            or provider.kind != "addressbook"
+        ):
+            raise DavError(403)
+        if self.request.method.upper() == "MKCALENDAR" and provider.kind != "calendar":
+            raise DavError(403)
+        name_el = props.get(q(D, "displayname"))
+        color_el = props.get(q(A, "calendar-color"))
+        await provider.create_collection(
+            self.user.id,
+            coll_id,
+            (name_el.text or "").strip() if name_el is not None else "",
+            (color_el.text or "").strip() if color_el is not None else None,
+        )
+        return Response(status_code=201)
+
+    async def proppatch(self, segment: str, provider: DavProvider, coll: Collection) -> Response:
+        patch = proppatch_request(await self.body())
+        if coll.read_only:
+            raise DavError(403)
+        values: dict[str, Any] = {}
+        for name, value in patch.set.items():
+            text = (value.text or "").strip()
+            if name == q(D, "displayname"):
+                values["displayname"] = text
+            elif name == q(A, "calendar-color"):
+                values["color"] = text
+            elif name in (q(C, "calendar-description"), q(CR, "addressbook-description")):
+                values["description"] = text
+        if values:
+            await provider.update_collection(self.user.id, coll.id, **values)
+        # Properties not kept (calendar order, …) are accepted and ignored:
+        # clients treat a refusal as a failed sync.
+        ms = Multistatus()
+        ms.response(self.coll_href(segment, coll.id), dict.fromkeys([*patch.set, *patch.remove]))
+        return self.multistatus(ms)
+
+    async def report(self, segment: str, provider: DavProvider, coll: Collection) -> Response:
+        root = await self.body()
+        if root is None:
+            raise BadRequest("empty report")
+        rep = report_request(root)
+        wants_data = rep.props is None or any(
+            p in (q(C, "calendar-data"), q(CR, "address-data")) for p in rep.props
+        )
+        ms = Multistatus()
+
+        def add(res: Resource) -> None:
+            props = self.resource_props(provider, res, wants_data)
+            if rep.props is None:
+                ms.response(self.res_href(segment, coll.id, res.name), props)
+            else:
+                found = {n: props[n] for n in rep.props if n in props}
+                missing = [n for n in rep.props if n not in props]
+                ms.response(self.res_href(segment, coll.id, res.name), found, missing)
+
+        if rep.kind in ("calendar-multiget", "addressbook-multiget"):
+            names = [_last_segment(h) for h in rep.hrefs]
+            got = {r.name: r for r in await provider.get(self.user.id, coll.id, names)}
+            for h, name in zip(rep.hrefs, names, strict=True):
+                if name in got:
+                    add(got[name])
+                else:
+                    ms.response(h, {}, status=404)
+            return self.multistatus(ms)
+
+        if rep.kind in ("calendar-query", "addressbook-query"):
+            names_q = (
+                await provider.query(self.user.id, coll.id, rep.start, rep.end)
+                if rep.kind == "calendar-query"
+                else None
+            )
+            for res in (
+                await provider.get(self.user.id, coll.id, names_q)
+                if wants_data
+                else (
+                    await provider.resources(self.user.id, coll.id)
+                    if names_q is None
+                    else [
+                        r
+                        for r in await provider.resources(self.user.id, coll.id)
+                        if r.name in set(names_q)
+                    ]
+                )
+            ):
+                add(res)
+            return self.multistatus(ms)
+
+        # sync-collection (RFC 6578)
+        since = _token_number(rep.sync_token)
+        if rep.sync_token and since is None:
+            raise DavError(403, condition=q(D, "valid-sync-token"))
+        if since is None:
+            changed = None
+            deleted: list[str] = []
+            token = coll.sync_token
+        else:
+            changes = await provider.changes(self.user.id, coll.id, since)
+            if changes is None:
+                raise DavError(403, condition=q(D, "valid-sync-token"))
+            changed, deleted, token = changes.changed, changes.deleted, changes.sync_token
+        if changed is None:
+            resources = (
+                await provider.get(self.user.id, coll.id, None)
+                if wants_data
+                else await provider.resources(self.user.id, coll.id)
+            )
+        elif changed:
+            resources = (
+                await provider.get(self.user.id, coll.id, changed)
+                if wants_data
+                else [
+                    r
+                    for r in await provider.resources(self.user.id, coll.id)
+                    if r.name in set(changed)
+                ]
+            )
+        else:
+            resources = []
+        present = {r.name for r in resources}
+        for res in resources:
+            add(res)
+        for name in deleted:
+            if name not in present:
+                ms.response(self.res_href(segment, coll.id, name), {}, status=404)
+        ms.sync_token(f"{SYNC_PREFIX}{token}")
+        return self.multistatus(ms)
+
+    # -- resources ---------------------------------------------------------------------
+
+    async def resource_req(
+        self, method: str, segment: str, provider: DavProvider, coll_id: str, name: str
+    ) -> Response:
+        coll = await self.find(provider, coll_id)
+        if method in ("GET", "HEAD"):
+            got = await provider.get(self.user.id, coll.id, [name])
+            if not got:
+                raise DavError(404)
+            res = got[0]
+            content = (res.data or "").encode() if method == "GET" else b""
+            return Response(
+                status_code=200,
+                content=content,
+                media_type=provider.content_type,
+                headers={"ETag": f'"{res.etag}"'},
+            )
+        if method == "PROPFIND":
+            got = await provider.get(self.user.id, coll.id, [name])
+            if not got:
+                raise DavError(404)
+            return await self.respond_props(
+                [
+                    (
+                        self.res_href(segment, coll.id, name),
+                        self.resource_props(provider, got[0], True),
+                    )
+                ],
+                await self.body(),
+            )
+        if method == "PUT":
+            if coll.read_only:
+                raise DavError(403, condition=q(D, "need-privileges"))
+            data = (await self.request.body()).decode("utf-8", "replace")
+            etag, created = await provider.put(
+                self.user.id,
+                coll.id,
+                name,
+                data,
+                if_match=_etag_header(self.request.headers.get("if-match")),
+                if_none_match=(self.request.headers.get("if-none-match") or "").strip() == "*",
+            )
+            return Response(status_code=201 if created else 204, headers={"ETag": f'"{etag}"'})
+        if method == "DELETE":
+            if coll.read_only:
+                raise DavError(403)
+            await provider.delete(
+                self.user.id,
+                coll.id,
+                name,
+                if_match=_etag_header(self.request.headers.get("if-match")),
+            )
+            return Response(status_code=204)
+        return Response(status_code=405)
+
+
+def _last_segment(h: str) -> str:
+    path = urlsplit(h).path
+    return unquote(path.rstrip("/").rsplit("/", 1)[-1])
+
+
+def _token_number(token: str) -> int | None:
+    if not token:
+        return None
+    tail = token.rsplit("/", 1)[-1]
+    return int(tail) if tail.isdigit() else None
+
+
+def _etag_header(value: str | None) -> str | None:
+    if not value:
+        return None
+    value = value.strip()
+    if value == "*":
+        return "*"
+    if value.startswith("W/"):
+        value = value[2:]
+    return value.strip('"')
+
+
+__all__ = ["router", "uuid"]

backend/app/dav/xml.py

@@ -0,0 +1,236 @@
+"""WebDAV XML: reading request bodies, writing multistatus answers (RFC 4918,
+4791, 6352, 6578). Properties are ``{namespace}name`` strings, as ElementTree
+names them."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from defusedxml import ElementTree as SafeET
+
+D = "DAV:"
+C = "urn:ietf:params:xml:ns:caldav"
+CR = "urn:ietf:params:xml:ns:carddav"
+CS = "http://calendarserver.org/ns/"
+A = "http://apple.com/ns/ical/"
+
+for _prefix, _ns in (("d", D), ("cal", C), ("card", CR), ("cs", CS), ("ical", A)):
+    ET.register_namespace(_prefix, _ns)
+
+
+def q(ns: str, name: str) -> str:
+    return f"{{{ns}}}{name}"
+
+
+class BadRequest(Exception):
+    pass
+
+
+def parse(body: bytes) -> ET.Element | None:
+    if not body.strip():
+        return None
+    try:
+        root = SafeET.fromstring(body)
+    except Exception as err:  # noqa: BLE001 - any malformed XML is the client's error
+        raise BadRequest(str(err)) from err
+    return root  # type: ignore[no-any-return]
+
+
+@dataclass
+class PropRequest:
+    """PROPFIND: which properties (None = all)."""
+
+    names: list[str] | None = None
+    names_only: bool = False
+
+
+def propfind_request(root: ET.Element | None) -> PropRequest:
+    if root is None or root.find(q(D, "allprop")) is not None:
+        return PropRequest()
+    if root.find(q(D, "propname")) is not None:
+        return PropRequest(names_only=True)
+    prop = root.find(q(D, "prop"))
+    if prop is None:
+        return PropRequest()
+    return PropRequest(names=[child.tag for child in prop])
+
+
+def requested_props(root: ET.Element) -> list[str] | None:
+    prop = root.find(q(D, "prop"))
+    if prop is None:
+        return None
+    return [child.tag for child in prop]
+
+
+@dataclass
+class Report:
+    # calendar-query | calendar-multiget | addressbook-query | addressbook-multiget
+    # | sync-collection
+    kind: str
+    props: list[str] | None
+    hrefs: list[str] = field(default_factory=list)
+    start: datetime | None = None
+    end: datetime | None = None
+    sync_token: str = ""
+    limit: int | None = None
+
+
+def _ical_time(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        return datetime.strptime(value.strip(), "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
+    except ValueError:
+        return None
+
+
+def report_request(root: ET.Element) -> Report:
+    tag = root.tag
+    names = {
+        q(C, "calendar-query"): "calendar-query",
+        q(C, "calendar-multiget"): "calendar-multiget",
+        q(CR, "addressbook-query"): "addressbook-query",
+        q(CR, "addressbook-multiget"): "addressbook-multiget",
+        q(D, "sync-collection"): "sync-collection",
+    }
+    kind = names.get(tag)
+    if kind is None:
+        raise BadRequest(f"unsupported report {tag}")
+    rep = Report(kind=kind, props=requested_props(root))
+    rep.hrefs = [(h.text or "").strip() for h in root.findall(q(D, "href"))]
+    if kind == "calendar-query":
+        tr = root.find(f".//{q(C, 'time-range')}")
+        if tr is not None:
+            rep.start = _ical_time(tr.get("start"))
+            rep.end = _ical_time(tr.get("end"))
+    if kind == "sync-collection":
+        rep.sync_token = (root.findtext(q(D, "sync-token")) or "").strip()
+        nresults = root.find(f"{q(D, 'limit')}/{q(D, 'nresults')}")
+        if nresults is not None and (nresults.text or "").strip().isdigit():
+            rep.limit = int((nresults.text or "").strip())
+    return rep
+
+
+@dataclass
+class PropPatch:
+    set: dict[str, ET.Element] = field(default_factory=dict)
+    remove: list[str] = field(default_factory=list)
+
+
+def proppatch_request(root: ET.Element | None) -> PropPatch:
+    out = PropPatch()
+    if root is None:
+        return out
+    for action in root:
+        prop = action.find(q(D, "prop"))
+        if prop is None:
+            continue
+        for child in prop:
+            if action.tag == q(D, "set"):
+                out.set[child.tag] = child
+            elif action.tag == q(D, "remove"):
+                out.remove.append(child.tag)
+    return out
+
+
+def mkcol_props(root: ET.Element | None) -> dict[str, ET.Element]:
+    """MKCALENDAR / extended MKCOL: the properties to set on the new collection."""
+    if root is None:
+        return {}
+    out: dict[str, ET.Element] = {}
+    for sett in root.findall(q(D, "set")):
+        prop = sett.find(q(D, "prop"))
+        if prop is not None:
+            for child in prop:
+                out[child.tag] = child
+    return out
+
+
+# ---- answers --------------------------------------------------------------------------
+
+# A property's value: text, a list of child elements, or an element builder.
+PropValue = str | list[ET.Element] | None
+
+
+class Multistatus:
+    def __init__(self) -> None:
+        self.root = ET.Element(q(D, "multistatus"))
+
+    def response(
+        self,
+        href: str,
+        found: dict[str, PropValue],
+        missing: list[str] | None = None,
+        status: int | None = None,
+    ) -> None:
+        resp = ET.SubElement(self.root, q(D, "response"))
+        ET.SubElement(resp, q(D, "href")).text = href
+        if status is not None:
+            ET.SubElement(resp, q(D, "status")).text = _status_line(status)
+            return
+        if found:
+            ps = ET.SubElement(resp, q(D, "propstat"))
+            prop = ET.SubElement(ps, q(D, "prop"))
+            for name, value in found.items():
+                el = ET.SubElement(prop, name)
+                if isinstance(value, str):
+                    el.text = value
+                elif isinstance(value, list):
+                    el.extend(value)
+            ET.SubElement(ps, q(D, "status")).text = _status_line(200)
+        if missing:
+            ps = ET.SubElement(resp, q(D, "propstat"))
+            prop = ET.SubElement(ps, q(D, "prop"))
+            for name in missing:
+                ET.SubElement(prop, name)
+            ET.SubElement(ps, q(D, "status")).text = _status_line(404)
+
+    def sync_token(self, token: str) -> None:
+        ET.SubElement(self.root, q(D, "sync-token")).text = token
+
+    def to_bytes(self) -> bytes:
+        body: bytes = ET.tostring(self.root, encoding="utf-8")
+        return b'<?xml version="1.0" encoding="utf-8"?>\n' + body
+
+
+def el(
+    tag: str,
+    text: str | None = None,
+    children: list[ET.Element] | None = None,
+    attrs: dict[str, str] | None = None,
+) -> ET.Element:
+    e = ET.Element(tag, attrs or {})
+    if text is not None:
+        e.text = text
+    if children:
+        e.extend(children)
+    return e
+
+
+def href(path: str) -> ET.Element:
+    return el(q(D, "href"), path)
+
+
+def error_body(condition: str) -> bytes:
+    root = ET.Element(q(D, "error"))
+    ET.SubElement(root, condition)
+    return b'<?xml version="1.0" encoding="utf-8"?>\n' + bytes(ET.tostring(root, encoding="utf-8"))
+
+
+_REASONS = {
+    200: "OK",
+    201: "Created",
+    204: "No Content",
+    207: "Multi-Status",
+    403: "Forbidden",
+    404: "Not Found",
+    409: "Conflict",
+    412: "Precondition Failed",
+    507: "Insufficient Storage",
+}
+
+
+def _status_line(code: int) -> str:
+    return f"HTTP/1.1 {code} {_REASONS.get(code, '')}".rstrip()

backend/app/events.py

@@ -38,6 +38,12 @@ ALLOWED_EVENTS: frozenset[str] = frozenset(
         "account.state_changed",
         "draft.saved",
         "outbox.state_changed",
+        # Sync v2 §3: published on login and every WS connect; drives tier
+        # promotion in the sync workers (§5). Payload: {user_id}.
+        "user.active",
+        # The user's sessions were revoked by an administrator (blocked or
+        # deleted): open tabs drop to the sign-in screen. Payload: {user_id}.
+        "user.signed_out",
     }
 )
 

backend/app/hosted/__init__.py

@@ -0,0 +1,34 @@
+"""Hosted mailboxes (SPEC Appendix D): every user's own mailbox on the Stalwart
+mail server beside the app.
+
+* :mod:`app.hosted.admin` — the mail server's management API (JMAP);
+* :mod:`app.hosted.provision` — idempotent setup: its database and role, the
+  directory over the ``mail_directory`` view, the worker's sign-in;
+* :mod:`app.hosted.native` — the permanent account in the webmail, and how the
+  worker signs in to it without any stored password;
+* :mod:`app.hosted.rawcache` — raw messages kept once, on the mail server: the
+  webmail keeps only a bounded cache of them (§D.6 item 2).
+
+All of it is inert while ``HOSTED_MAILBOXES`` is off.
+"""
+
+from __future__ import annotations
+
+from app.config import get_settings
+
+# The webmail worker's sign-in on the mail server: ``user@domain%WORKER`` acts
+# for that user (Stalwart's master-user login, given the impersonate role only).
+WORKER_LOCALPART = "webmail-worker"
+
+
+def enabled() -> bool:
+    return get_settings().hosted_mailboxes
+
+
+def worker_address() -> str:
+    return f"{WORKER_LOCALPART}@{get_settings().app_domain}"
+
+
+def mail_hostname() -> str:
+    settings = get_settings()
+    return settings.mail_hostname or f"mail.{settings.app_domain}"

backend/app/hosted/admin.py

@@ -0,0 +1,177 @@
+"""The mail server's management API (Stalwart 0.16, JMAP ``x:`` methods).
+
+Signed in to as the recovery admin (``STALWART_RECOVERY_ADMIN`` in the mail
+server's environment, ``STALWART_ADMIN_SECRET`` here). Objects are Stalwart's
+registry: ``x:Account``, ``x:Domain``, ``x:Directory``, ``x:Role``,
+``x:AppPassword``… with the usual JMAP ``/get``, ``/query``, ``/set``.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Any
+
+import httpx
+
+from app.config import get_settings
+from app.hosted import worker_address
+
+ADMIN_USER = "admin"
+_USING = ["urn:ietf:params:jmap:core", "urn:stalwart:jmap"]
+_TIMEOUT = 15.0
+
+
+class MailServerError(Exception):
+    """The mail server refused a call, or could not be reached."""
+
+
+def _basic(user: str, secret: str) -> str:
+    return "Basic " + base64.b64encode(f"{user}:{secret}".encode()).decode()
+
+
+def _admin_auth() -> str:
+    return _basic(ADMIN_USER, get_settings().stalwart_admin_secret)
+
+
+async def call(
+    *method_calls: tuple[str, dict[str, Any]], auth: str | None = None
+) -> list[dict[str, Any]]:
+    """Run JMAP method calls in one request; return each call's arguments.
+
+    Raises :class:`MailServerError` on a method error or a refused ``set``."""
+    body = {
+        "using": _USING,
+        "methodCalls": [[name, args, str(i)] for i, (name, args) in enumerate(method_calls)],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.post(
+                f"{get_settings().stalwart_url}/jmap",
+                json=body,
+                headers={"Authorization": auth or _admin_auth()},
+            )
+    except httpx.HTTPError as exc:
+        raise MailServerError(f"mail server unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        raise MailServerError(f"mail server answered {resp.status_code}: {resp.text[:300]}")
+    out: list[dict[str, Any]] = []
+    for name, args, _ in resp.json()["methodResponses"]:
+        if name == "error":
+            raise MailServerError(f"{args.get('type')}: {args.get('description', '')}")
+        for kind in ("notCreated", "notUpdated", "notDestroyed"):
+            if args.get(kind):
+                raise MailServerError(f"{name} {kind}: {args[kind]}")
+        out.append(args)
+    return out
+
+
+async def get_all(obj: str, **args: Any) -> list[dict[str, Any]]:
+    """Every object of a type (registry objects are few)."""
+    (res,) = await call((f"x:{obj}/get", {"ids": None, **args}))
+    return list(res["list"])
+
+
+async def get_one(obj: str, object_id: str = "singleton") -> dict[str, Any]:
+    (res,) = await call((f"x:{obj}/get", {"ids": [object_id]}))
+    if not res["list"]:
+        raise MailServerError(f"{obj} {object_id} not found")
+    return dict(res["list"][0])
+
+
+async def create(obj: str, value: dict[str, Any], **args: Any) -> str:
+    (res,) = await call((f"x:{obj}/set", {"create": {"n": value}, **args}))
+    return str(res["created"]["n"]["id"])
+
+
+async def update(obj: str, object_id: str, patch: dict[str, Any], **args: Any) -> None:
+    await call((f"x:{obj}/set", {"update": {object_id: patch}, **args}))
+
+
+async def destroy(obj: str, ids: list[str], **args: Any) -> None:
+    if ids:
+        await call((f"x:{obj}/set", {"destroy": ids, **args}))
+
+
+# ---- accounts ----------------------------------------------------------------------
+
+
+async def account_id(email: str) -> str | None:
+    """The mail server's id of a mailbox, if it knows it yet.
+
+    Accounts from the directory reach the server's registry the first time
+    they are used (a sign-in, a delivery): see :func:`ensure_account`."""
+    local = email.split("@", 1)[0].lower()
+    (res,) = await call(("x:Account/query", {"filter": {"name": local}}))
+    ids = list(res.get("ids") or [])
+    if not ids:
+        return None
+    for acc in (await call(("x:Account/get", {"ids": ids, "properties": ["emailAddress"]})))[0][
+        "list"
+    ]:
+        if str(acc.get("emailAddress", "")).lower() == email.lower():
+            return str(acc["id"])
+    return None
+
+
+async def ensure_account(email: str) -> str:
+    """The mailbox's id, bringing it into the server's registry if new: a
+    sign-in as the worker acting for it does that (§D.3)."""
+    found = await account_id(email)
+    if found is not None:
+        return found
+    auth = _basic(f"{email}%{worker_address()}", get_settings().stalwart_master_secret)
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(
+                f"{get_settings().stalwart_url}/jmap/session", headers={"Authorization": auth}
+            )
+    except httpx.HTTPError as exc:
+        raise MailServerError(f"mail server unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        raise MailServerError(f"cannot open the mailbox {email}: {resp.status_code}")
+    found = await account_id(email)
+    if found is None:
+        raise MailServerError(f"mailbox {email} not found after sign-in")
+    return found
+
+
+async def set_blocked(email: str, blocked: bool) -> None:
+    """A blocked user's mailbox cannot be signed in to at all — app passwords
+    and the worker included. Mail to it is still accepted."""
+    acc = await account_id(email)
+    if acc is None:
+        return  # never used: the directory alone keeps it shut
+    permissions: dict[str, Any] = (
+        {"@type": "Merge", "enabledPermissions": {}, "disabledPermissions": {"authenticate": True}}
+        if blocked
+        else {"@type": "Inherit"}
+    )
+    await update("Account", acc, {"permissions": permissions})
+
+
+async def delete_account(email: str) -> None:
+    """Remove the mailbox and all its mail from the mail server."""
+    acc = await account_id(email)
+    if acc is not None:
+        await destroy("Account", [acc])
+
+
+# ---- app passwords (§D.5) -------------------------------------------------------------
+
+
+async def create_app_password(email: str, description: str) -> tuple[str, str]:
+    """A new app password for the mailbox: (server id, the password itself)."""
+    acc = await ensure_account(email)
+    (res,) = await call(
+        ("x:AppPassword/set", {"accountId": acc, "create": {"n": {"description": description}}})
+    )
+    created = res["created"]["n"]
+    return str(created["id"]), str(created["secret"])
+
+
+async def delete_app_passwords(email: str, server_ids: list[str]) -> None:
+    acc = await account_id(email)
+    if acc is not None and server_ids:
+        (res,) = await call(("x:AppPassword/get", {"accountId": acc, "ids": server_ids}))
+        present = [str(p["id"]) for p in res["list"]]
+        await destroy("AppPassword", present, accountId=acc)

backend/app/hosted/native.py

@@ -0,0 +1,154 @@
+"""The user's own mailbox as a webmail account (SPEC App. D §D.4).
+
+Every user has one ``kind = 'native'`` account: their handle's mailbox on the
+hosted mail server. It is created with the user (and, for users from before,
+by :func:`ensure_all`, run by provisioning), cannot be removed, and its server
+settings are not the user's to change.
+
+No password of the user's is stored for it: the worker signs in as itself,
+acting for the user (``user@domain%webmail-worker``, §D.3), over the Docker
+network (or loopback in dev) — TLS, with the certificate left unchecked since
+the traffic never leaves the host.
+"""
+
+from __future__ import annotations
+
+import logging
+import ssl
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.hosted import enabled, worker_address
+from app.models import Account, User
+from app.providers.imap_smtp import ProviderConfig
+from app.security.crypto import encrypt_secret
+
+logger = logging.getLogger(__name__)
+
+KIND = "native"
+
+
+def is_native(account: Account) -> bool:
+    return account.kind == KIND
+
+
+def _server(host: str, port: int) -> dict[str, object]:
+    return {"host": host, "port": port, "tls": "ssl", "auth": "password"}
+
+
+def internal_tls() -> ssl.SSLContext:
+    """TLS to the mail server on the internal network: its certificate is for
+    its public name (or self-signed), not the container's."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def native_provider_config(email: str) -> ProviderConfig:
+    """IMAP/SMTP to the user's mailbox, signed in as the worker acting for them.
+
+    Read from the settings, not the account row: the mail server's address is
+    the deployment's, and may change."""
+    settings = get_settings()
+    return ProviderConfig(
+        imap_host=settings.stalwart_imap_host,
+        imap_port=settings.stalwart_imap_port,
+        imap_tls="ssl",
+        smtp_host=settings.stalwart_smtp_host,
+        smtp_port=settings.stalwart_smtp_port,
+        smtp_tls="ssl",
+        username=f"{email}%{worker_address()}",
+        password=settings.stalwart_master_secret,
+        tls_context=internal_tls(),
+        internal=True,
+    )
+
+
+async def native_account(db: AsyncSession, user_id: uuid.UUID) -> Account | None:
+    account: Account | None = await db.scalar(
+        select(Account).where(Account.user_id == user_id, Account.kind == KIND)
+    )
+    return account
+
+
+async def ensure(db: AsyncSession, user: User) -> Account | None:
+    """The user's native account, created (and its first sync queued by the
+    caller, see :func:`ensure_and_sync`) if missing. None when hosted
+    mailboxes are off. Flushes, does not commit."""
+    if not enabled():
+        return None
+    existing = await native_account(db, user.id)
+    if existing is not None:
+        return existing
+    # An account for the same address added by hand: the same address at
+    # another provider (the mail before it moved here — e.g. while the domain
+    # still points elsewhere). It stays as it is; the mailbox account is added
+    # once it is removed (app/api/accounts.py).
+    same = await db.scalar(
+        select(Account.id).where(Account.user_id == user.id, Account.email == user.handle)
+    )
+    if same is not None:
+        logger.info("hosted: %s already has an account for its address; not added", user.handle)
+        return None
+    settings = get_settings()
+    imap = _server(settings.stalwart_imap_host, settings.stalwart_imap_port)
+    smtp = _server(settings.stalwart_smtp_host, settings.stalwart_smtp_port)
+    account = Account(
+        user_id=user.id,
+        email=user.handle,
+        kind=KIND,
+        imap_config=imap,
+        smtp_config=smtp,
+        secret_ct=encrypt_secret(b""),  # nothing: the worker signs in as itself
+        discovery_src="hosted",
+        state="backfilling",
+        sync_window="all",
+        # The mail server's own spam filter files into Junk: show it.
+        sync_spam=True,
+    )
+    db.add(account)
+    await db.flush()
+    return account
+
+
+async def ensure_and_sync(db: AsyncSession, user: User) -> Account | None:
+    """:func:`ensure`, committed, and the account's first sync queued if new."""
+    before = await native_account(db, user.id)
+    account = await ensure(db, user)
+    await db.commit()
+    if account is not None and before is None:
+        from app.queue import enqueue_job
+
+        await enqueue_job("sync_bootstrap_account", str(account.id))
+    return account
+
+
+async def ensure_all() -> int:
+    """Give every user without one their native account (users from before
+    hosted mailboxes). Returns how many were created."""
+    from app.db import session_scope
+
+    created = 0
+    async with session_scope() as db:
+        users = list(
+            await db.scalars(
+                select(User).where(
+                    ~select(Account.id)
+                    .where(Account.user_id == User.id, Account.kind == KIND)
+                    .exists()
+                )
+            )
+        )
+    for user in users:
+        async with session_scope() as db:
+            fresh = await db.get(User, user.id)
+            if fresh is None:
+                continue
+            await ensure_and_sync(db, fresh)
+            created += 1
+            logger.info("hosted: mailbox account for %s", fresh.handle)
+    return created

backend/app/hosted/provision.py

@@ -0,0 +1,323 @@
+"""Set up the mail server for hosted mailboxes (SPEC App. D §D.2/§D.3). Idempotent.
+
+Run after the migrations (the ``migrate`` service in production; in dev
+``python -m app.hosted.provision``). Two halves:
+
+1. **Postgres** (as the app's own, privileged database user): the mail
+   server's role ``stalwart`` (password ``STALWART_DB_PASSWORD``), which owns its
+   database and may read the ``mail_directory`` view of the app's database and
+   nothing else there; and the worker's sign-in in ``mail_service_accounts``.
+   The mail server cannot start before this has run once.
+2. **The mail server** (its management API, as the recovery admin): the mail
+   domain and host name, mail kept on disk (not in Postgres), the SQL directory
+   over ``mail_directory`` as the only source of accounts, the worker's
+   impersonate-only role, the submission port for mail apps.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import asyncpg  # type: ignore[import-untyped]
+
+from app.config import get_settings
+from app.hosted import admin, mail_hostname, worker_address
+from app.identity.passwords import hash_password, verify_password
+
+logger = logging.getLogger(__name__)
+
+DB_ROLE = "stalwart"
+DIRECTORY_NAME = "Ultimo webmail users"
+WORKER_ROLE_NAME = "Ultimo webmail worker"
+BLOB_PATH = "/var/lib/stalwart/blobs"
+
+# The directory's queries ($1: the address signed in with / mail is for). A
+# blocked user cannot sign in, but still receives mail.
+QUERY_LOGIN = (
+    "SELECT name, secret, type, description FROM mail_directory WHERE name = $1 AND active"
+)
+QUERY_RECIPIENT = (
+    "SELECT name, NULL::text AS secret, type, description FROM mail_directory WHERE name = $1"
+)
+
+
+def _pg_dsn() -> str:
+    return get_settings().database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+
+
+def _app_db_name() -> str:
+    return urlparse(_pg_dsn()).path.lstrip("/")
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+async def provision_database() -> None:
+    settings = get_settings()
+    if not settings.stalwart_db_password or not settings.stalwart_master_secret:
+        raise RuntimeError("STALWART_DB_PASSWORD and STALWART_MASTER_SECRET must be set")
+    conn = await asyncpg.connect(_pg_dsn())
+    try:
+        password = _quote_literal(settings.stalwart_db_password)
+        if await conn.fetchval("SELECT 1 FROM pg_roles WHERE rolname = $1", DB_ROLE):
+            await conn.execute(f"ALTER ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        else:
+            await conn.execute(f"CREATE ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        db = settings.stalwart_db_name
+        if not await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db):
+            await conn.execute(f'CREATE DATABASE "{db}" OWNER {DB_ROLE}')
+            logger.info("hosted: created database %s", db)
+        await conn.execute(f"GRANT SELECT ON mail_directory TO {DB_ROLE}")
+
+        # The worker's sign-in: re-hashed only when the secret changed.
+        name = worker_address()
+        current = await conn.fetchval(
+            "SELECT secret FROM mail_service_accounts WHERE name = $1", name
+        )
+        if current is None or not verify_password(current, settings.stalwart_master_secret):
+            await conn.execute(
+                """
+                INSERT INTO mail_service_accounts (name, secret, description)
+                VALUES ($1, $2, 'Webmail worker')
+                ON CONFLICT (name) DO UPDATE SET secret = EXCLUDED.secret
+                """,
+                name,
+                hash_password(settings.stalwart_master_secret),
+            )
+    finally:
+        await conn.close()
+
+
+# ---- the mail server ------------------------------------------------------------------
+
+
+async def _wait_for_server(timeout_s: float = 120.0) -> None:
+    deadline = asyncio.get_running_loop().time() + timeout_s
+    while True:
+        try:
+            await admin.get_one("SystemSettings")
+            return
+        except admin.MailServerError:
+            if asyncio.get_running_loop().time() > deadline:
+                raise
+            await asyncio.sleep(2)
+
+
+def _by(items: list[dict[str, Any]], key: str, value: str) -> dict[str, Any] | None:
+    return next((i for i in items if i.get(key) == value), None)
+
+
+async def _domain() -> str:
+    name = get_settings().app_domain.lower()
+    found = _by(await admin.get_all("Domain"), "name", name)
+    if found is not None:
+        return str(found["id"])
+    domain_id = await admin.create("Domain", {"name": name})
+    logger.info("hosted: created mail domain %s", name)
+    return domain_id
+
+
+async def _directory() -> str:
+    settings = get_settings()
+    dsn = urlparse(_pg_dsn())
+    value: dict[str, Any] = {
+        "@type": "Sql",
+        "description": DIRECTORY_NAME,
+        "columnEmail": "name",
+        "columnSecret": "secret",
+        "columnClass": "type",
+        "columnDescription": "description",
+        "queryLogin": QUERY_LOGIN,
+        "queryRecipient": QUERY_RECIPIENT,
+        "queryMemberOf": None,
+        "queryEmailAliases": None,
+        "store": {
+            "@type": "PostgreSql",
+            "host": settings.stalwart_pg_host,
+            "port": 5432 if settings.stalwart_pg_host == "postgres" else (dsn.port or 5432),
+            "database": _app_db_name(),
+            "authUsername": DB_ROLE,
+            "authSecret": {"@type": "EnvironmentVariable", "variableName": "STALWART_DB_PASSWORD"},
+            "useTls": False,
+            "allowInvalidCerts": False,
+            "poolMaxConnections": 5,
+            "poolRecyclingMethod": "fast",
+            "readReplicas": {},
+            "timeout": 15000,
+            "options": None,
+        },
+    }
+    found = _by(await admin.get_all("Directory"), "description", DIRECTORY_NAME)
+    if found is None:
+        directory_id = await admin.create("Directory", value)
+        logger.info("hosted: created the SQL directory")
+    else:
+        directory_id = str(found["id"])
+        await admin.update("Directory", directory_id, value)
+    await admin.update("Authentication", "singleton", {"directoryId": directory_id})
+    return directory_id
+
+
+async def _worker_role() -> str:
+    perms = {"impersonate": True, "authenticate": True}
+    found = _by(await admin.get_all("Role"), "description", WORKER_ROLE_NAME)
+    if found is not None:
+        role_id = str(found["id"])
+        await admin.update("Role", role_id, {"enabledPermissions": perms})
+        return role_id
+    return await admin.create(
+        "Role", {"description": WORKER_ROLE_NAME, "enabledPermissions": perms}
+    )
+
+
+async def _worker_account(role_id: str) -> None:
+    """The worker may act for any mailbox and do nothing as itself."""
+    # Signing in brings it from the directory into the server's registry. With
+    # its role already set that sign-in fails (it may not open a session as
+    # itself) — after the account exists, which is all this needs.
+    worker = worker_address()
+    acc = await admin.account_id(worker)
+    if acc is None:
+        import httpx
+
+        async with httpx.AsyncClient(timeout=15) as client:
+            await client.get(
+                f"{get_settings().stalwart_url}/jmap/session",
+                auth=(worker, get_settings().stalwart_master_secret),
+            )
+        acc = await admin.account_id(worker)
+    if acc is None:
+        raise admin.MailServerError("the worker account did not appear on the mail server")
+    await admin.update("Account", acc, {"roles": {"@type": "Custom", "roleIds": {role_id: True}}})
+
+
+async def _reload() -> None:
+    """Apply the settings now. Stalwart 0.16.19 ships a few spam-list rules its
+    own validator rejects (``bit_and`` in SURBL tags); they never load, but
+    they make every reload fail — so they are removed, which changes nothing."""
+    for _ in range(10):
+        try:
+            await admin.create("Action", {"@type": "ReloadSettings"})
+            return
+        except admin.MailServerError as exc:
+            match = re.search(r"'object': '(SpamDnsblServer)', 'id': '(\w+)'", str(exc))
+            if match is None:
+                raise
+            logger.info("hosted: removing the invalid built-in rule %s", match.group(2))
+            await admin.destroy(match.group(1), [match.group(2)])
+    raise admin.MailServerError("settings reload kept failing")
+
+
+async def _logging() -> None:
+    """Log to the console (``docker logs``), not to files inside the container."""
+    for tracer in await admin.get_all("Tracer"):
+        if tracer.get("@type") != "Stdout":
+            await admin.update(
+                "Tracer",
+                str(tracer["id"]),
+                {
+                    "@type": "Stdout",
+                    "ansi": False,
+                    "buffered": False,
+                    "enable": True,
+                    "level": "info",
+                    "events": {},
+                    "eventsPolicy": "exclude",
+                },
+            )
+
+
+async def _blob_store() -> bool:
+    """Mail on disk (``DATA_ROOT/stalwart``), not inside Postgres: database
+    backups stay small. Takes a restart; True when one is needed."""
+    current = await admin.get_one("BlobStore")
+    if current.get("@type") == "FileSystem" and current.get("path") == BLOB_PATH:
+        return False
+    await admin.update(
+        "BlobStore", "singleton", {"@type": "FileSystem", "path": BLOB_PATH, "depth": 2}
+    )
+    logger.info("hosted: mail is now kept on disk (takes effect after a restart)")
+    return True
+
+
+async def _listeners() -> None:
+    """Add submission with STARTTLS (587) beside the defaults: smtp 25, submissions
+    465, imaps 993 (and http 8080 for this API)."""
+    listeners = await admin.get_all("NetworkListener")
+    if _by(listeners, "name", "submission") is None:
+        await admin.create(
+            "NetworkListener",
+            {"name": "submission", "bind": {"[::]:587": True}, "protocol": "smtp", "useTls": True},
+        )
+
+
+# Folder names for new mailboxes: the plain ones mail apps expect, not
+# Stalwart's Outlook-style defaults ("Deleted Items", "Junk Mail", "Sent
+# Items"). Archive is made too: the webmail archives into it.
+FOLDERS = {
+    "trash": "Trash",
+    "junk": "Junk",
+    "drafts": "Drafts",
+    "sent": "Sent",
+    "archive": "Archive",
+}
+
+
+async def _folders() -> None:
+    folders = {
+        use: {"name": name, "create": True, "subscribe": True, "aliases": {}}
+        for use, name in FOLDERS.items()
+    }
+    await admin.update("Email", "singleton", {"defaultFolders": folders})
+
+
+async def provision_server() -> bool:
+    """Set the mail server up; True when it must be restarted to finish."""
+    await _wait_for_server()
+    await _logging()
+    restart = await _blob_store()
+    domain_id = await _domain()
+    await admin.update(
+        "SystemSettings",
+        "singleton",
+        {"defaultDomainId": domain_id, "defaultHostname": mail_hostname()},
+    )
+    await _listeners()
+    await _folders()
+    await _directory()
+    role_id = await _worker_role()
+    await _reload()
+    await _worker_account(role_id)
+    return restart
+
+
+async def provision() -> bool:
+    from app.hosted import native
+
+    await provision_database()
+    restart = await provision_server()
+    created = await native.ensure_all()
+    if created:
+        logger.info("hosted: added the mailbox account of %d existing user(s)", created)
+    return restart
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    if not get_settings().hosted_mailboxes:
+        logger.info("hosted mailboxes are off (HOSTED_MAILBOXES): nothing to set up")
+        return
+    if asyncio.run(provision()):
+        logger.warning("hosted mailboxes: RESTART the mail server (stalwart) to finish")
+    else:
+        logger.info("hosted mailboxes: mail server ready")
+
+
+if __name__ == "__main__":
+    main()

backend/app/identity/app_passwords.py

@@ -0,0 +1,140 @@
+"""App passwords: one generated password per mail app (SPEC App. D §D.5).
+
+A mail app (Thunderbird, a phone's Mail app) signs in to the hosted mailbox
+over IMAP/SMTP, and a calendar or contacts app to the DAV server — never with
+the account password (no 2FA code can be asked for there), always with one of
+these. Each gets a password of its own, shown once, that can be
+revoked without touching the main password or the other apps. The webmail
+itself never accepts one.
+
+With hosted mailboxes on, the mail server makes the password (``app_`` and 36
+more letters and digits: its format names the credential) and checks it for
+IMAP/SMTP itself; its argon2id hash is kept here too, for the DAV server, with
+the server's id to revoke it there. Without, it is 16 random letters, for DAV
+only. Either way the hash is of the bare password: apps send exactly that, so
+it is copied and entered without the spaces it is shown with.
+"""
+
+from __future__ import annotations
+
+import secrets
+import string
+import uuid
+
+from sqlalchemy import delete, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import hosted
+from app.identity.passwords import hash_password, verify_password
+from app.models import User, UserAppPassword
+
+LENGTH = 16  # 26^16 ≈ 2^75
+SERVER_LENGTH = 40  # the mail server's: "app_" + base32 of a 4-byte id and 18 random bytes
+MAX_PER_USER = 25
+NAME_MAX = 60
+_ALPHABET = string.ascii_lowercase
+
+
+def generation_key(user_id: uuid.UUID) -> str:
+    """Redis counter bumped whenever the user's app passwords change: DAV
+    sign-ins cached under the old value stop counting (app/dav/auth.py)."""
+    return f"apppw:gen:{user_id}"
+
+
+async def _changed(user_id: uuid.UUID) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().incr(generation_key(user_id))
+
+
+class TooMany(Exception):
+    """The user already has MAX_PER_USER app passwords."""
+
+
+def generate() -> str:
+    return "".join(secrets.choice(_ALPHABET) for _ in range(LENGTH))
+
+
+async def list_for(db: AsyncSession, user_id: uuid.UUID) -> list[UserAppPassword]:
+    rows = await db.scalars(
+        select(UserAppPassword)
+        .where(UserAppPassword.user_id == user_id)
+        .order_by(UserAppPassword.created_at.desc())
+    )
+    return list(rows)
+
+
+async def create(db: AsyncSession, user_id: uuid.UUID, name: str) -> tuple[UserAppPassword, str]:
+    """A new app password named ``name``; returns the row and the password
+    itself, which is never available again."""
+    count = await db.scalar(
+        select(func.count()).select_from(UserAppPassword).where(UserAppPassword.user_id == user_id)
+    )
+    if (count or 0) >= MAX_PER_USER:
+        raise TooMany
+    name = name.strip()[:NAME_MAX]
+    server_id: str | None = None
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        handle = await db.scalar(select(User.handle).where(User.id == user_id))
+        server_id, password = await mail_admin.create_app_password(str(handle), name)
+    else:
+        password = generate()
+    row = UserAppPassword(
+        user_id=user_id, name=name, password_hash=hash_password(password), server_id=server_id
+    )
+    db.add(row)
+    await db.commit()
+    return row, password
+
+
+async def _revoke_on_server(
+    db: AsyncSession, user_id: uuid.UUID, rows: list[UserAppPassword]
+) -> None:
+    """Revoke on the mail server too (first: if it cannot be reached, the
+    password keeps working there, so nothing is revoked here either)."""
+    server_ids = [r.server_id for r in rows if r.server_id]
+    if server_ids and hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        handle = await db.scalar(select(User.handle).where(User.id == user_id))
+        await mail_admin.delete_app_passwords(str(handle), server_ids)
+
+
+async def revoke(db: AsyncSession, user_id: uuid.UUID, app_password_id: uuid.UUID) -> bool:
+    rows = list(
+        await db.scalars(
+            select(UserAppPassword).where(
+                UserAppPassword.id == app_password_id, UserAppPassword.user_id == user_id
+            )
+        )
+    )
+    await _revoke_on_server(db, user_id, rows)
+    result = await db.execute(
+        delete(UserAppPassword).where(
+            UserAppPassword.id == app_password_id, UserAppPassword.user_id == user_id
+        )
+    )
+    await db.commit()
+    await _changed(user_id)
+    return bool(result.rowcount)  # type: ignore[attr-defined]
+
+
+async def revoke_all(db: AsyncSession, user_id: uuid.UUID) -> None:
+    """Every app signs in again (the main password was reset)."""
+    await _revoke_on_server(db, user_id, await list_for(db, user_id))
+    await db.execute(delete(UserAppPassword).where(UserAppPassword.user_id == user_id))
+    await db.commit()
+    await _changed(user_id)
+
+
+async def verify(db: AsyncSession, user_id: uuid.UUID, password: str) -> UserAppPassword | None:
+    """The app password ``password`` matches, if any — for the mail server's
+    sign-in and for tests; never for the webmail's own sign-in."""
+    if len(password) not in (LENGTH, SERVER_LENGTH):
+        return None
+    for row in await list_for(db, user_id):
+        if verify_password(row.password_hash, password):
+            return row
+    return None

backend/app/identity/deps.py

@@ -30,8 +30,9 @@ async def current_user(
 ) -> User:
     """Load the authenticated :class:`User` (I-7: scoped by the session's user_id)."""
     user = await db.scalar(select(User).where(User.id == session.user_id))
-    if user is None:
-        # Session references a deleted user; treat as unauthenticated.
+    if user is None or user.blocked_at is not None:
+        # Session references a deleted or blocked user; treat as unauthenticated.
+        # (Blocking also revokes the sessions; this covers any request in flight.)
         raise_api_error(401, "not_authenticated", "authentication required")
     return user
 
@@ -46,5 +47,13 @@ async def require_csrf(request: Request) -> None:
         raise_api_error(403, "csrf_failed", "missing or invalid CSRF token")
 
 
+async def current_admin(user: Annotated[User, Depends(current_user)]) -> User:
+    """Load the authenticated user and require the administrator flag."""
+    if not user.is_admin:
+        raise_api_error(403, "not_admin", "administrator privileges required")
+    return user
+
+
 CurrentUser = Annotated[User, Depends(current_user)]
+CurrentAdmin = Annotated[User, Depends(current_admin)]
 CurrentSession = Annotated[Session, Depends(current_session)]

backend/app/identity/invitations.py

@@ -0,0 +1,240 @@
+"""Invitations: a user sends a friend a single-use sign-up link.
+
+While the administrator has invitations switched on, every user may send
+``PER_USER`` of them. A link lets exactly one account be created — even with
+public registration closed — and works for ``VALID_FOR``; one that ran out
+unused is handed back to its sender. Only a hash of the link's token is kept.
+
+The email is built from the administrator's template (or the built-in one)
+with these placeholders, written ``{{name}}``:
+
+    invitee_name   the friend's name, as the sender typed it
+    inviter_name   the sender's name (their mail account's display name,
+                   else their address)
+    invite_link    the sign-up address itself
+    invite_button  a ready-made button linking to it
+    logo           the Ultimo Email logo
+
+Names are HTML-escaped where they go into the HTML: they come from users and
+end up in mail to strangers.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import hashlib
+import html as html_lib
+import re
+import secrets
+import uuid
+from dataclasses import dataclass
+
+import nh3
+from sqlalchemy import func, or_, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.identity.mailer import LOGO_HTML
+from app.models import Account, AppSettings, Invitation, User
+from app.outbox.inline_images import InlinePart, data_image
+
+PER_USER = 2
+VALID_FOR = dt.timedelta(days=14)
+MAX_TEMPLATE_BYTES = 500 * 1024
+
+PLACEHOLDERS = ("invitee_name", "inviter_name", "invite_link", "invite_button", "logo")
+_PLACEHOLDER = re.compile(r"\{\{\s*([a-z_]+)\s*\}\}")
+_DATA_SRC = re.compile(r"""(\bsrc\s*=\s*)(["'])(data:image/[^"']+)\2""", re.IGNORECASE)
+
+DEFAULT_SUBJECT = "{{inviter_name}} invites you to Ultimo Email"
+_BODY = "font-family:Arial,Helvetica,sans-serif;font-size:15px;color:#1f2937;line-height:1.5"
+_MUTED = "font-size:13px;color:#6b7280"
+_BUTTON = (
+    "display:inline-block;background:#2563eb;color:#ffffff;text-decoration:none;"
+    "padding:10px 18px;border-radius:8px;font-weight:bold"
+)
+DEFAULT_TEMPLATE = f"""<!doctype html>
+<html><body style="{_BODY}">
+{{{{logo}}}}
+<p>Hi {{{{invitee_name}}}},</p>
+<p><b>{{{{inviter_name}}}}</b> invites you to Ultimo Email — private email,
+calendar and contacts in one place.</p>
+<p>{{{{invite_button}}}}</p>
+<p style="{_MUTED}">Or paste this address into your browser:<br>{{{{invite_link}}}}</p>
+<p style="{_MUTED}">The invitation works once and expires in 14 days. If you
+weren't expecting it, you can ignore this email.</p>
+</body></html>
+"""
+
+
+class InvitationError(Exception):
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+@dataclass(frozen=True)
+class RenderedMail:
+    subject: str
+    text: str
+    html: str
+    inline: list[InlinePart]
+
+
+def hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def sender_address(row: AppSettings) -> str:
+    """The invitation's From. Empty: the system-mail From address, the one the
+    SMTP account is known to be allowed to send as."""
+    return row.invite_from.strip() or row.smtp_from
+
+
+def template_problem(html: str) -> str | None:
+    """Why ``html`` cannot be used as the invitation template, or None."""
+    if len(html.encode()) > MAX_TEMPLATE_BYTES:
+        return f"the template is larger than {MAX_TEMPLATE_BYTES // 1024} KB"
+    used = set(_PLACEHOLDER.findall(html))
+    unknown = sorted(used - set(PLACEHOLDERS))
+    if unknown:
+        return "unknown placeholder: " + ", ".join("{{" + u + "}}" for u in unknown)
+    if not used & {"invite_link", "invite_button"}:
+        return "the template must contain {{invite_link}} or {{invite_button}}"
+    return None
+
+
+def render(row: AppSettings, *, invitee_name: str, inviter_name: str, link: str) -> RenderedMail:
+    """The invitation email for one friend, from the configured template."""
+    template = row.invite_template or DEFAULT_TEMPLATE
+    subject_template = row.invite_subject or DEFAULT_SUBJECT
+    esc_link = html_lib.escape(link, quote=True)
+    html_values = {
+        "invitee_name": html_lib.escape(invitee_name),
+        "inviter_name": html_lib.escape(inviter_name),
+        "invite_link": f'<a href="{esc_link}">{esc_link}</a>',
+        "invite_button": f'<a href="{esc_link}" style="{_BUTTON}">Accept the invitation</a>',
+        "logo": LOGO_HTML,
+    }
+    text_values = {
+        "invitee_name": invitee_name,
+        "inviter_name": inviter_name,
+        "invite_link": link,
+        "invite_button": link,
+        "logo": "",
+    }
+
+    def fill(source: str, values: dict[str, str]) -> str:
+        return _PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), source)
+
+    # An image written into the template itself (data:) goes as an inline part:
+    # Gmail does not show data: images.
+    inline: list[InlinePart] = []
+
+    def to_cid(match: re.Match[str]) -> str:
+        part = data_image(match.group(3), get_settings().app_domain)
+        if part is None:
+            return match.group(0)
+        inline.append(part)
+        return f"{match.group(1)}{match.group(2)}cid:{part.cid}{match.group(2)}"
+
+    html = _DATA_SRC.sub(to_cid, fill(template, html_values))
+    text = fill(template, text_values)
+    # Keep the paragraphs and line breaks the HTML shows once its tags are gone.
+    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"</(?:p|div|h[1-6]|li|tr|table)\s*>", "\n\n", text, flags=re.IGNORECASE)
+    text = nh3.clean(text, tags=set(), attributes={}, strip_comments=True)
+    text = html_lib.unescape(re.sub(r"\n\s*\n\s*", "\n\n", text)).strip() + "\n"
+    subject = " ".join(fill(subject_template, text_values).split())
+    return RenderedMail(subject=subject, text=text, html=html, inline=inline)
+
+
+async def inviter_name(db: AsyncSession, user: User) -> str:
+    """How the friend is told who invited them."""
+    names = (
+        await db.execute(
+            select(Account.email, Account.display_name).where(Account.user_id == user.id)
+        )
+    ).all()
+    own = next((n for e, n in names if e.lower() == user.handle.lower() and n), None)
+    return own or next((n for _, n in names if n), None) or user.handle
+
+
+def _counts(now: dt.datetime):  # type: ignore[no-untyped-def]
+    # Used, or still valid: an invitation that expired unused is given back.
+    return or_(Invitation.used_at.isnot(None), Invitation.expires_at > now)
+
+
+async def remaining(db: AsyncSession, user_id: uuid.UUID) -> int:
+    now = dt.datetime.now(dt.UTC)
+    taken = await db.scalar(
+        select(func.count())
+        .select_from(Invitation)
+        .where(Invitation.inviter_id == user_id, _counts(now))
+    )
+    return max(0, PER_USER - int(taken or 0))
+
+
+async def sent(db: AsyncSession, user_id: uuid.UUID) -> list[Invitation]:
+    return list(
+        await db.scalars(
+            select(Invitation)
+            .where(Invitation.inviter_id == user_id)
+            .order_by(Invitation.created_at.desc())
+        )
+    )
+
+
+async def create(db: AsyncSession, user: User, *, name: str, email: str) -> tuple[Invitation, str]:
+    """A new invitation from ``user`` (not committed): the row and the token.
+    Raises :class:`InvitationError` when the user has none left."""
+    # One at a time per user, so two quick clicks cannot both take the last one.
+    await db.execute(select(User.id).where(User.id == user.id).with_for_update())
+    if await remaining(db, user.id) <= 0:
+        raise InvitationError("no_invitations_left", "you have no invitations left")
+    token = secrets.token_urlsafe(32)
+    invitation = Invitation(
+        inviter_id=user.id,
+        token_hash=hash_token(token),
+        invitee_name=name,
+        invitee_email=email,
+        expires_at=dt.datetime.now(dt.UTC) + VALID_FOR,
+    )
+    db.add(invitation)
+    await db.flush()
+    return invitation, token
+
+
+async def find_valid(db: AsyncSession, token: str) -> Invitation | None:
+    """The unused, unexpired invitation behind ``token``, without using it."""
+    invitation: Invitation | None = await db.scalar(
+        select(Invitation).where(
+            Invitation.token_hash == hash_token(token),
+            Invitation.used_at.is_(None),
+            Invitation.expires_at > dt.datetime.now(dt.UTC),
+        )
+    )
+    return invitation
+
+
+async def claim(db: AsyncSession, token: str) -> uuid.UUID | None:
+    """Use the invitation up (in the caller's transaction): its id, or None if
+    it is unknown, used or expired. Atomic — two sign-ups with one link cannot
+    both get it."""
+    return await db.scalar(
+        update(Invitation)
+        .where(
+            Invitation.token_hash == hash_token(token),
+            Invitation.used_at.is_(None),
+            Invitation.expires_at > dt.datetime.now(dt.UTC),
+        )
+        .values(used_at=func.now())
+        .returning(Invitation.id)
+    )
+
+
+async def record_user(db: AsyncSession, invitation_id: uuid.UUID, user_id: uuid.UUID) -> None:
+    await db.execute(
+        update(Invitation).where(Invitation.id == invitation_id).values(used_by=user_id)
+    )

backend/app/identity/mailer.py

@@ -0,0 +1,175 @@
+"""Outgoing system mail (SPEC §6.1): mail the server sends on its own behalf.
+
+Password-reset links, and invitations users send their friends
+(app/identity/invitations.py). It goes out through the SMTP
+account the administrator configured in the admin panel (``app_settings``),
+never through a user's own mail account — the whole point is that the user
+cannot sign in. The administrator's server is trusted, so unlike user mail
+accounts this connection is not held to the SSRF guard (it may well be a relay
+on the LAN).
+"""
+
+from __future__ import annotations
+
+import base64
+import html as html_lib
+import logging
+from collections.abc import Sequence
+from dataclasses import dataclass
+from email.message import EmailMessage
+from email.utils import formataddr, formatdate, make_msgid
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import aiosmtplib
+
+from app.config import smtp_helo_name
+from app.models import AppSettings
+from app.security.crypto import decrypt_secret
+
+if TYPE_CHECKING:
+    from app.outbox.inline_images import InlinePart
+
+logger = logging.getLogger(__name__)
+
+SENDER_NAME = "Ultimo Email"
+_TIMEOUT = 20.0
+
+# The logo travels inside the message (multipart/related, referenced by cid:),
+# not as a link: it shows even where remote images are blocked, and it is a PNG
+# because mail clients (Gmail first) do not display SVG. Rendered at twice the
+# size it is shown at, for sharp high-DPI screens.
+_LOGO_FILE = Path(__file__).parent / "assets" / "ultimo-logo.png"
+LOGO_CID = "ultimo-logo@ultimo.email"
+LOGO_HTML = (
+    f'<p style="text-align:center;margin:0 0 24px"><img src="cid:{LOGO_CID}" '
+    'width="180" height="47" alt="Ultimo Email" style="display:inline-block;border:0"></p>'
+)
+
+
+@cache
+def _logo() -> bytes:
+    return _LOGO_FILE.read_bytes()
+
+
+# For previews in the admin panel, which show the HTML without message parts.
+LOGO_DATA_URL = "data:image/png;base64," + base64.b64encode(_LOGO_FILE.read_bytes()).decode()
+
+
+class MailerError(Exception):
+    """System mail is not configured, or the SMTP server refused the message."""
+
+
+@dataclass(frozen=True)
+class MailerConfig:
+    host: str
+    port: int
+    security: str  # starttls | ssl | none
+    username: str
+    password: str
+    sender: str
+    public_url: str
+
+
+def mailer_config(row: AppSettings) -> MailerConfig | None:
+    """The usable configuration, or None while the admin has not set one up."""
+    if not (row.smtp_host and row.smtp_from and row.public_url):
+        return None
+    password = decrypt_secret(row.smtp_secret_ct).decode() if row.smtp_secret_ct else ""
+    return MailerConfig(
+        host=row.smtp_host,
+        port=row.smtp_port,
+        security=row.smtp_security,
+        username=row.smtp_username,
+        password=password,
+        sender=row.smtp_from,
+        public_url=row.public_url.rstrip("/"),
+    )
+
+
+async def send_mail(
+    cfg: MailerConfig,
+    to: str,
+    subject: str,
+    text: str,
+    html: str,
+    *,
+    sender: str | None = None,
+    reply_to: str | None = None,
+    inline: Sequence[InlinePart] = (),
+) -> None:
+    """Send one system message. ``sender`` replaces the configured From address
+    (the SMTP account must be allowed to send as it); ``inline`` are images the
+    HTML names by ``cid:``."""
+    msg = EmailMessage()
+    msg["From"] = formataddr((SENDER_NAME, sender or cfg.sender))
+    msg["To"] = to
+    if reply_to:
+        msg["Reply-To"] = reply_to
+    msg["Subject"] = subject
+    # RFC 5322 requires it, and mail without one scores as spam (Gmail).
+    msg["Date"] = formatdate(localtime=True)
+    msg["Message-ID"] = make_msgid(domain=cfg.sender.rsplit("@", 1)[-1])
+    msg["Auto-Submitted"] = "auto-generated"  # RFC 3834: do not auto-reply to this
+    msg.set_content(text)
+    msg.add_alternative(html, subtype="html")
+    if f"cid:{LOGO_CID}" in html:
+        html_part = msg.get_body(("html",))
+        if isinstance(html_part, EmailMessage):
+            html_part.add_related(_logo(), maintype="image", subtype="png", cid=f"<{LOGO_CID}>")
+    if inline:
+        html_part = msg.get_body(("html",))
+        assert isinstance(html_part, EmailMessage)
+        for part in inline:
+            maintype, _, subtype = part.mime_type.partition("/")
+            html_part.add_related(
+                part.data,
+                maintype=maintype,
+                subtype=subtype,
+                cid=f"<{part.cid}>",
+                filename=part.filename,
+                disposition="inline",
+            )
+    try:
+        await aiosmtplib.send(
+            msg,
+            hostname=cfg.host,
+            port=cfg.port,
+            username=cfg.username or None,
+            password=cfg.password or None,
+            use_tls=cfg.security == "ssl",
+            start_tls=cfg.security == "starttls",
+            timeout=_TIMEOUT,
+            local_hostname=smtp_helo_name(),
+        )
+    except (aiosmtplib.SMTPException, OSError) as exc:
+        raise MailerError(str(exc) or type(exc).__name__) from exc
+
+
+def reset_mail(link: str, handle: str) -> tuple[str, str, str]:
+    """(subject, text, html) of the password-reset message."""
+    subject = "Reset your Ultimo Email password"
+    text = (
+        f"Someone asked to change the password of the Ultimo Email account {handle}.\n\n"
+        f"To choose a new password, open this link within 30 minutes:\n\n{link}\n\n"
+        "The link works once. If you did not ask for this, ignore this email — "
+        "your password stays as it is.\n"
+    )
+    h, u = html_lib.escape(handle), html_lib.escape(link, quote=True)
+    muted = "font-size:13px;color:#6b7280"
+    button = (
+        "display:inline-block;background:#2563eb;color:#ffffff;text-decoration:none;"
+        "padding:10px 18px;border-radius:8px;font-weight:bold"
+    )
+    body = "font-family:Arial,Helvetica,sans-serif;font-size:15px;color:#1f2937;line-height:1.5"
+    html = (
+        f'<!doctype html>\n<html><body style="{body}">\n{LOGO_HTML}\n'
+        f"<p>Someone asked to change the password of the Ultimo Email account <b>{h}</b>.</p>\n"
+        "<p>To choose a new password, open this link within 30 minutes:</p>\n"
+        f'<p><a href="{u}" style="{button}">Change password</a></p>\n'
+        f'<p style="{muted}">Or paste this address into your browser:<br>{u}</p>\n'
+        f'<p style="{muted}">The link works once. If you did not ask for this, ignore this '
+        "email — your password stays as it is.</p>\n</body></html>\n"
+    )
+    return subject, text, html

backend/app/identity/password_policy.py

@@ -0,0 +1,145 @@
+"""What a new account password must not be (sign-up and reset).
+
+Length is bounded by the request schemas (8–128). On top of that a password is
+refused when it is one of the passwords every guessing attack tries first, is
+made of the user's own address, is one character repeated, or has turned up in
+a known data breach. The breach check asks Have I Been Pwned's range API with
+the first five hex digits of the password's SHA-1 only (k-anonymity: neither the
+password nor its hash leaves the server) and is skipped, not failed, when that
+service cannot be reached — a sign-up never hangs on someone else's outage.
+Set ``PASSWORD_BREACH_CHECK=false`` to never ask it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+
+import httpx
+
+from app.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+MIN_LENGTH = 8
+MAX_LENGTH = 128
+
+_RANGE_API = "https://api.pwnedpasswords.com/range/"
+_TIMEOUT = 3.0
+
+# The front of every guessing list (and of every breach), lower-cased.
+_COMMON = frozenset(
+    {
+        "password",
+        "password1",
+        "password12",
+        "password123",
+        "password1234",
+        "passw0rd",
+        "p@ssw0rd",
+        "p@ssword",
+        "12345678",
+        "123456789",
+        "1234567890",
+        "0123456789",
+        "87654321",
+        "11111111",
+        "00000000",
+        "12341234",
+        "qwertyui",
+        "qwerty123",
+        "qwerty1234",
+        "qwertyuiop",
+        "1q2w3e4r",
+        "1q2w3e4r5t",
+        "1qaz2wsx",
+        "zaq12wsx",
+        "asdfghjk",
+        "asdfasdf",
+        "iloveyou",
+        "iloveyou1",
+        "sunshine",
+        "princess",
+        "football",
+        "baseball",
+        "welcome1",
+        "welcome123",
+        "letmein1",
+        "trustno1",
+        "superman",
+        "batman123",
+        "dragon123",
+        "monkey123",
+        "abc12345",
+        "abcd1234",
+        "aa123456",
+        "qazwsxedc",
+        "changeme",
+        "changeme1",
+        "administrator",
+        "admin123",
+        "adminadmin",
+        "computer",
+        "internet",
+        "whatever",
+        "starwars",
+        "michelle",
+        "jennifer",
+        "charlie1",
+        "zaq1zaq1",
+        "haslo123",
+        "polska123",
+        "kochamcie",
+        "zaq1@wsx",
+        "qwerty12",
+        "1234qwer",
+        "q1w2e3r4",
+    }
+)
+
+
+class PasswordRejected(Exception):
+    """The password may not be used; ``code`` and ``message`` say why."""
+
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+async def _breached(password: str) -> bool:
+    digest = hashlib.sha1(password.encode("utf-8")).hexdigest().upper()  # noqa: S324 - HIBP's format
+    prefix, suffix = digest[:5], digest[5:]
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(_RANGE_API + prefix, headers={"Add-Padding": "true"})
+            resp.raise_for_status()
+    except httpx.HTTPError as exc:
+        logger.warning("breached-password check skipped: %s", exc)
+        return False
+    for line in resp.text.splitlines():
+        candidate, _, count = line.partition(":")
+        if candidate.strip() == suffix and count.strip() not in ("", "0"):  # padding rows say 0
+            return True
+    return False
+
+
+async def check_new_password(password: str, handle: str) -> None:
+    """Raise :class:`PasswordRejected` if ``password`` may not be used by ``handle``."""
+    lowered = password.lower()
+    if len(set(password)) <= 2:
+        raise PasswordRejected("weak_password", "this password is too easy to guess")
+    if lowered in _COMMON:
+        raise PasswordRejected("weak_password", "this is one of the most common passwords")
+    localpart = handle.split("@", 1)[0].lower()
+    if len(localpart) >= 3 and localpart in lowered:
+        # The name with a year or a few characters added ("jsmith2026") is the
+        # first thing anyone who knows the address will try.
+        rest = lowered.replace(localpart, "", 1)
+        if len(rest) < 4 or all(not c.isalpha() for c in rest):
+            raise PasswordRejected("weak_password", "the password must not be your address")
+    if get_settings().password_breach_check and await _breached(password):
+        raise PasswordRejected(
+            "breached_password",
+            "this password has appeared in a data breach — please choose another",
+        )

backend/app/identity/reset.py

@@ -1,43 +1,76 @@
 """Password reset tokens (SPEC §6.1).
 
-Single-use, 30-minute, Redis-backed tokens. Only issued when the account has a
-recovery email on file. Actual delivery of the token to the recovery address
-requires an app-level mailer (not part of M0/M1); until then the token is logged
-server-side. See docs/open-questions.md.
+Single-use, 30-minute, Redis-backed tokens, emailed as a link to the account's
+recovery address through the system mailer (:mod:`app.identity.mailer`).
+
+A token survives a wrong 2FA code (the page asks again) but not five: after
+that it is burned, so the link cannot be used to guess codes. Requests are
+capped per account so the reset form cannot be used to flood someone's inbox.
 """
 
 from __future__ import annotations
 
-import logging
 import secrets
 import uuid
 
 from app.redis_client import aw, get_redis
 
-logger = logging.getLogger(__name__)
-
 _PREFIX = "pwreset:"
+_FAILS = "pwreset-fail:"
+_RATE = "pwreset-rate:"
 _TTL = 30 * 60  # 30 minutes
+MAX_TOTP_FAILURES = 5
+MAX_REQUESTS_PER_HOUR = 3
 
 
-async def issue_token(user_id: uuid.UUID, recovery_email: str) -> str:
+async def issue_token(user_id: uuid.UUID) -> str:
     """Create a single-use reset token bound to ``user_id`` and return it."""
     token = secrets.token_urlsafe(32)
     await get_redis().set(_PREFIX + token, str(user_id), ex=_TTL)
-    # TODO(spec): deliver via the recovery address once an app mailer exists.
-    logger.info("password reset token issued for user=%s recovery=%s", user_id, recovery_email)
     return token
 
 
+async def request_allowed(user_id: uuid.UUID) -> bool:
+    """Count a reset request; False once the account has had its hourly share."""
+    redis = get_redis()
+    key = _RATE + str(user_id)
+    count = int(await redis.incr(key))
+    if count == 1:
+        await redis.expire(key, 3600)
+    return count <= MAX_REQUESTS_PER_HOUR
+
+
+def _user_id(raw: str | None) -> uuid.UUID | None:
+    if raw is None:
+        return None
+    try:
+        return uuid.UUID(raw)
+    except ValueError:
+        return None
+
+
+async def peek_token(token: str) -> uuid.UUID | None:
+    """The user a still-valid token belongs to, without using it up."""
+    if not token:
+        return None
+    return _user_id(await get_redis().get(_PREFIX + token))
+
+
 async def consume_token(token: str) -> uuid.UUID | None:
     """Atomically consume a reset token, returning the user_id (or None)."""
     if not token:
         return None
     redis = get_redis()
     raw = await aw(redis.getdel(_PREFIX + token))  # single-use: fetch and delete atomically
-    if raw is None:
-        return None
-    try:
-        return uuid.UUID(raw)
-    except ValueError:
-        return None
+    await redis.delete(_FAILS + token)
+    return _user_id(raw)
+
+
+async def record_totp_failure(token: str) -> None:
+    """Count a wrong 2FA code against the token; burn it at the limit."""
+    redis = get_redis()
+    fails = await redis.incr(_FAILS + token)
+    if fails == 1:
+        await redis.expire(_FAILS + token, _TTL)
+    if fails >= MAX_TOTP_FAILURES:
+        await redis.delete(_PREFIX + token, _FAILS + token)

backend/app/identity/routes.py

@@ -3,15 +3,22 @@ TOTP setup/enable, session list/revoke, password reset."""
 
 from __future__ import annotations
 
-from typing import Annotated
+import logging
+import re
+import uuid
+from typing import Annotated, NoReturn
 
-from fastapi import APIRouter, Depends, Request, Response
+from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import hosted
 from app.api.errors import raise_api_error
 from app.config import get_settings
 from app.db import get_session
+from app.events import get_event_bus
+from app.hosted.admin import MailServerError
+from app.identity import app_passwords, invitations, trusted_devices
 from app.identity import totp as totp_mod
 from app.identity.cookies import (
     clear_auth_cookies,
@@ -19,14 +26,31 @@ from app.identity.cookies import (
     set_auth_cookies,
 )
 from app.identity.deps import CurrentSession, CurrentUser, require_csrf
+from app.identity.mailer import MailerConfig, MailerError, mailer_config, reset_mail, send_mail
+from app.identity.password_policy import PasswordRejected, check_new_password
 from app.identity.passwords import hash_password, verify_password
-from app.identity.reset import consume_token, issue_token
+from app.identity.reset import (
+    consume_token,
+    issue_token,
+    peek_token,
+    record_totp_failure,
+    request_allowed,
+)
 from app.identity.schemas import (
+    AppPasswordCreatedOut,
+    AppPasswordIn,
+    AppPasswordOut,
+    InvitationCheckIn,
+    InvitationCheckOut,
     LoginRequest,
     MessageOut,
+    PasswordResetCheckIn,
+    PasswordResetCheckOut,
     PasswordResetIn,
     PasswordResetRequestIn,
+    RecoveryEmailIn,
     RegisterRequest,
+    RegistrationStatusOut,
     SessionOut,
     TotpDisableRequest,
     TotpEnableRequest,
@@ -34,12 +58,15 @@ from app.identity.schemas import (
     UserOut,
 )
 from app.identity.service import (
+    AccountBlocked,
     RateLimited,
     RegistrationError,
     TotpInvalid,
     TotpRequired,
+    app_settings,
     authenticate,
     register_user,
+    registration_status,
     resolve_handle,
 )
 from app.identity.sessions import (
@@ -50,9 +77,17 @@ from app.identity.sessions import (
     revoke_session,
 )
 from app.models import User
+from app.redis_client import get_redis
+from app.sync import leases
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/auth", tags=["auth"])
 
+# Deliberately loose: the recovery address only has to be something mail can be
+# sent to; the mail server is the real judge.
+_EMAIL_RE = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+")
+
 DbSession = Annotated[AsyncSession, Depends(get_session)]
 
 
@@ -74,6 +109,37 @@ def _user_out(user: User) -> UserOut:
         recovery_email=user.recovery_email,
         settings=user.settings,
         totp_enabled=user.totp_secret is not None,
+        is_admin=user.is_admin,
+        mail_host=hosted.mail_hostname() if hosted.enabled() else None,
+    )
+
+
+@router.get("/registration", response_model=RegistrationStatusOut)
+async def registration(db: DbSession) -> RegistrationStatusOut:
+    """Whether signup is open. Unauthenticated: the signup page needs it to render."""
+    status = await registration_status(db)
+    return RegistrationStatusOut(
+        open=status.open, bootstrap=status.bootstrap, domain=get_settings().app_domain
+    )
+
+
+_REGISTRATION_ERROR_STATUS = {"handle_taken": 409, "registration_closed": 403}
+
+
+@router.post("/invitation/check", response_model=InvitationCheckOut)
+async def invitation_check(body: InvitationCheckIn, db: DbSession) -> InvitationCheckOut:
+    """Whether an invitation link still works, and who it is from — for the
+    sign-up page. Does not use it up."""
+    if not (await app_settings(db)).invitations_enabled:
+        return InvitationCheckOut(valid=False)
+    invitation = await invitations.find_valid(db, body.token)
+    if invitation is None:
+        return InvitationCheckOut(valid=False)
+    inviter = await db.get(User, invitation.inviter_id)
+    return InvitationCheckOut(
+        valid=True,
+        invitee_name=invitation.invitee_name,
+        inviter_name=await invitations.inviter_name(db, inviter) if inviter else "",
     )
 
 
@@ -81,23 +147,48 @@ def _user_out(user: User) -> UserOut:
 async def register(
     body: RegisterRequest, request: Request, response: Response, db: DbSession
 ) -> UserOut:
+    try:
+        await check_new_password(body.password, resolve_handle(body.localpart))
+    except PasswordRejected as exc:
+        raise_api_error(400, exc.code, exc.message)
+    invitation_id: uuid.UUID | None = None
+    if body.invite:
+        # Used up in this same transaction: if the sign-up fails (the address
+        # is taken, …) the rollback hands the invitation back.
+        if (await app_settings(db)).invitations_enabled:
+            invitation_id = await invitations.claim(db, body.invite)
+        if invitation_id is None:
+            raise_api_error(
+                400, "invalid_invitation", "this invitation has already been used or has expired"
+            )
     try:
         user = await register_user(
             db,
             localpart=body.localpart,
             password=body.password,
             recovery_email=body.recovery_email,
+            invited=invitation_id is not None,
         )
     except RegistrationError as exc:
-        status = 409 if exc.code == "handle_taken" else 400
-        raise_api_error(status, exc.code, exc.message)
+        raise_api_error(_REGISTRATION_ERROR_STATUS.get(exc.code, 400), exc.code, exc.message)
+    if invitation_id is not None:
+        await invitations.record_user(db, invitation_id, user.id)
     await db.commit()
+    # The user's own mailbox (App. D §D.4): it exists on the mail server as soon
+    # as the user does (the directory reads the users table); this adds it to
+    # the webmail.
+    from app.hosted import native
+
+    await native.ensure_and_sync(db, user)
     await _establish_session(request, response, user)
     return _user_out(user)
 
 
 @router.post("/login", response_model=UserOut)
 async def login(body: LoginRequest, request: Request, response: Response, db: DbSession) -> UserOut:
+    async def trusted(user_id: uuid.UUID) -> bool:
+        return await trusted_devices.is_trusted(request, response, user_id)
+
     try:
         user = await authenticate(
             db,
@@ -105,6 +196,7 @@ async def login(body: LoginRequest, request: Request, response: Response, db: Db
             password=body.password,
             ip=_client_ip(request),
             totp=body.totp,
+            trusted_device=trusted,
         )
     except RateLimited as exc:
         response.headers["Retry-After"] = str(exc.retry_after)
@@ -113,9 +205,22 @@ async def login(body: LoginRequest, request: Request, response: Response, db: Db
         raise_api_error(401, "totp_required", "a 2FA code is required")
     except TotpInvalid:
         raise_api_error(401, "totp_invalid", "incorrect 2FA code")
+    except AccountBlocked:
+        raise_api_error(
+            403, "account_blocked", "this account has been blocked by the administrator"
+        )
     if user is None:
         raise_api_error(401, "invalid_credentials", "incorrect handle or password")
+    if body.remember_device and body.totp and user.totp_secret is not None:
+        # Only after a code was actually checked: a trusted sign-in cannot extend
+        # the trust to anything new.
+        await trusted_devices.trust(request, response, user.id)
     await _establish_session(request, response, user)
+    # Sync v2 §3/§5: a login marks the user active (session heat) and promotes
+    # their accounts' tier to hot.
+    settings = get_settings()
+    await leases.mark_user_active(get_redis(), user.id, settings.tier_session_grace_min * 60)
+    await get_event_bus().publish("user.active", {"user_id": str(user.id)})
     return _user_out(user)
 
 
@@ -164,6 +269,7 @@ async def totp_disable(body: TotpDisableRequest, user: CurrentUser, db: DbSessio
         raise_api_error(401, "invalid_credentials", "incorrect password")
     user.totp_secret = None
     await db.commit()
+    await trusted_devices.revoke_all(user.id)  # trust was in place of a code that no longer exists
     return MessageOut()
 
 
@@ -189,25 +295,167 @@ async def revoke(sid: str, user: CurrentUser) -> MessageOut:
 # --- Password reset (SPEC §6.1) ---------------------------------------------
 
 
+async def _send_reset_mail(cfg: MailerConfig, to: str, handle: str, token: str) -> None:
+    """Runs after the response has gone out (see password_reset_request)."""
+    # The token rides in the fragment, which browsers never send to a server, so
+    # it stays out of access logs and Referer headers.
+    subject, text, html = reset_mail(f"{cfg.public_url}/reset-password#token={token}", handle)
+    try:
+        await send_mail(cfg, to, subject, text, html)
+    except MailerError as exc:
+        logger.warning("password reset mail for %s could not be sent: %s", handle, exc)
+
+
 @router.post("/password/reset-request", response_model=MessageOut)
-async def password_reset_request(body: PasswordResetRequestIn, db: DbSession) -> MessageOut:
-    # Always return ok (do not leak whether the handle exists / has recovery mail).
+async def password_reset_request(
+    body: PasswordResetRequestIn, background: BackgroundTasks, db: DbSession
+) -> MessageOut:
+    """Email a reset link to the account's recovery address.
+
+    The answer is the same whether or not the account exists, has a recovery
+    address, or hit the hourly cap, and the mail is sent after the response —
+    so neither the reply nor its timing tells a stranger anything.
+    """
     handle = resolve_handle(body.handle)
     user = await db.scalar(select(User).where(User.handle == handle))
-    if user is not None and user.recovery_email:
-        await issue_token(user.id, user.recovery_email)
+    if user is None or not user.recovery_email:
         return MessageOut()
+    cfg = mailer_config(await app_settings(db))
+    if cfg is None:
+        logger.warning("password reset requested for %s, but system mail is not set up", handle)
+        return MessageOut()
+    if not await request_allowed(user.id):
+        return MessageOut()
+    token = await issue_token(user.id)
+    background.add_task(_send_reset_mail, cfg, user.recovery_email, user.handle, token)
+    return MessageOut()
+
+
+@router.post("/password/reset/check", response_model=PasswordResetCheckOut)
+async def password_reset_check(body: PasswordResetCheckIn, db: DbSession) -> PasswordResetCheckOut:
+    """Whether a reset link still works, so the page can say so before asking
+    for a new password. Does not use the link up."""
+    user_id = await peek_token(body.token)
+    user = await db.get(User, user_id) if user_id else None
+    if user is None:
+        return PasswordResetCheckOut(valid=False)
+    return PasswordResetCheckOut(valid=True, totp_required=user.totp_secret is not None)
 
 
 @router.post("/password/reset", response_model=MessageOut)
 async def password_reset(body: PasswordResetIn, db: DbSession) -> MessageOut:
-    user_id = await consume_token(body.token)
-    if user_id is None:
-        raise_api_error(400, "invalid_token", "reset token is invalid or expired")
-    user = await db.get(User, user_id)
+    """Set a new password with a reset link. With 2FA on, the code is checked
+    before anything changes; a missing or wrong code leaves the link usable
+    (up to MAX_TOTP_FAILURES wrong codes)."""
+    user_id = await peek_token(body.token)
+    user = await db.get(User, user_id) if user_id else None
     if user is None:
-        raise_api_error(400, "invalid_token", "reset token is invalid or expired")
+        raise_api_error(400, "invalid_token", "this link is invalid or has expired")
+    # Before the 2FA code and the link are spent: a refused password costs neither.
+    try:
+        await check_new_password(body.password, user.handle)
+    except PasswordRejected as exc:
+        raise_api_error(400, exc.code, exc.message)
+    if user.totp_secret is not None:
+        if not body.totp:
+            raise_api_error(401, "totp_required", "a 2FA code is required")
+        if not totp_mod.verify(totp_mod.decrypt_totp_secret(user.totp_secret), body.totp):
+            await record_totp_failure(body.token)
+            raise_api_error(401, "totp_invalid", "incorrect 2FA code")
+    # Only now is the link spent — and only if nobody spent it in the meantime.
+    if await consume_token(body.token) != user.id:
+        raise_api_error(400, "invalid_token", "this link is invalid or has expired")
     user.password_hash = hash_password(body.password)
     await db.commit()
-    await destroy_all_sessions(user_id)  # force re-login everywhere after reset
+    await destroy_all_sessions(user.id)  # force re-login everywhere after reset
+    # Mail apps sign in again too: the reset may be the owner taking the account back.
+    try:
+        await app_passwords.revoke_all(db, user.id)
+    except MailServerError:
+        # The new password is set; the app passwords stay listed (and working)
+        # until the mail server answers, so the user can still revoke them.
+        logger.exception("password reset: could not revoke app passwords on the mail server")
+    # Whoever reset the password may be the owner taking the account back: no
+    # device keeps skipping the code on the old trust.
+    await trusted_devices.revoke_all(user.id)
     return MessageOut()
+
+
+# --- App passwords (SPEC App. D §D.5) -----------------------------------------
+
+
+def _mail_server_down() -> NoReturn:
+    raise_api_error(
+        502, "mail_server_unavailable", "the mail server is not answering; try again shortly"
+    )
+
+
+def _app_password_out(row: object) -> AppPasswordOut:
+    from app.models import UserAppPassword
+
+    assert isinstance(row, UserAppPassword)
+    return AppPasswordOut(
+        id=str(row.id),
+        name=row.name,
+        created_at=row.created_at.isoformat(),
+        last_used_at=row.last_used_at.isoformat() if row.last_used_at else None,
+    )
+
+
+@router.get("/app-passwords", response_model=list[AppPasswordOut])
+async def list_app_passwords(user: CurrentUser, db: DbSession) -> list[AppPasswordOut]:
+    return [_app_password_out(r) for r in await app_passwords.list_for(db, user.id)]
+
+
+@router.post(
+    "/app-passwords", response_model=AppPasswordCreatedOut, dependencies=[Depends(require_csrf)]
+)
+async def create_app_password(
+    body: AppPasswordIn, user: CurrentUser, db: DbSession
+) -> AppPasswordCreatedOut:
+    """A new password for one mail app. Takes the account password: an app
+    password opens the mailbox without a 2FA code."""
+    if not verify_password(user.password_hash, body.password):
+        raise_api_error(403, "invalid_password", "incorrect password")
+    if not body.name.strip():
+        raise_api_error(400, "invalid_name", "give the app password a name")
+    try:
+        row, password = await app_passwords.create(db, user.id, body.name)
+    except app_passwords.TooMany:
+        raise_api_error(
+            409, "too_many", f"you can have up to {app_passwords.MAX_PER_USER} app passwords"
+        )
+    except MailServerError:
+        _mail_server_down()
+    return AppPasswordCreatedOut(**_app_password_out(row).model_dump(), app_password=password)
+
+
+@router.delete(
+    "/app-passwords/{app_password_id}",
+    response_model=MessageOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def revoke_app_password(
+    app_password_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> MessageOut:
+    try:
+        revoked = await app_passwords.revoke(db, user.id, app_password_id)
+    except MailServerError:
+        _mail_server_down()
+    if not revoked:
+        raise_api_error(404, "not_found", "app password not found")
+    return MessageOut()
+
+
+@router.put("/recovery-email", response_model=UserOut, dependencies=[Depends(require_csrf)])
+async def set_recovery_email(body: RecoveryEmailIn, user: CurrentUser, db: DbSession) -> UserOut:
+    """Add, change or remove the recovery address. Takes the password: whoever
+    controls this address can take the account over with a reset link."""
+    if not verify_password(user.password_hash, body.password):
+        raise_api_error(403, "invalid_password", "incorrect password")
+    email = (body.recovery_email or "").strip().lower() or None
+    if email is not None and not _EMAIL_RE.fullmatch(email):
+        raise_api_error(400, "invalid_email", "enter a valid email address")
+    user.recovery_email = email
+    await db.commit()
+    return _user_out(user)

backend/app/identity/schemas.py

@@ -3,14 +3,18 @@
 from __future__ import annotations
 
 import uuid
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
 
 class RegisterRequest(BaseModel):
     localpart: str = Field(min_length=3, max_length=30)
-    password: str = Field(min_length=8, max_length=1024)
+    password: str = Field(min_length=8, max_length=128)
     recovery_email: str | None = None
+    # A friend's invitation (app/identity/invitations.py): signs up even while
+    # public registration is closed, and is used up by it.
+    invite: str | None = Field(default=None, max_length=200)
 
 
 class LoginRequest(BaseModel):
@@ -18,6 +22,8 @@ class LoginRequest(BaseModel):
     handle: str = Field(min_length=1, max_length=320)
     password: str = Field(min_length=1, max_length=1024)
     totp: str | None = None  # required when the account has 2FA enabled
+    # With a correct code: don't ask this browser for one again (trusted device).
+    remember_device: bool = False
 
 
 class TotpSetupResponse(BaseModel):
@@ -46,7 +52,41 @@ class PasswordResetRequestIn(BaseModel):
 
 class PasswordResetIn(BaseModel):
     token: str = Field(min_length=1)
-    password: str = Field(min_length=8, max_length=1024)
+    password: str = Field(min_length=8, max_length=128)
+    totp: str | None = None  # required when the account has 2FA enabled
+
+
+class PasswordResetCheckIn(BaseModel):
+    token: str = Field(min_length=1)
+
+
+class PasswordResetCheckOut(BaseModel):
+    valid: bool
+    # Tells the page to expect the 2FA step; only the link's holder learns it.
+    totp_required: bool = False
+
+
+class AppPasswordIn(BaseModel):
+    name: str = Field(min_length=1, max_length=60)  # "Thunderbird on laptop"
+    password: str = Field(min_length=1, max_length=1024)  # the account password, to confirm
+
+
+class AppPasswordOut(BaseModel):
+    id: str
+    name: str
+    created_at: str
+    last_used_at: str | None = None
+
+
+class AppPasswordCreatedOut(AppPasswordOut):
+    # The generated password itself: in this answer only, never again.
+    app_password: str
+
+
+class RecoveryEmailIn(BaseModel):
+    password: str = Field(min_length=1, max_length=1024)
+    # None (or blank) removes the recovery address.
+    recovery_email: str | None = Field(default=None, max_length=320)
 
 
 class UserOut(BaseModel):
@@ -55,6 +95,127 @@ class UserOut(BaseModel):
     recovery_email: str | None
     settings: dict[str, object]
     totp_enabled: bool = False
+    is_admin: bool = False
+    # The hosted mail server's name for mail apps (IMAP 993, SMTP 465/587), or
+    # None when there are no hosted mailboxes (SPEC App. D).
+    mail_host: str | None = None
+
+
+class InvitationCheckIn(BaseModel):
+    token: str = Field(min_length=1, max_length=200)
+
+
+class InvitationCheckOut(BaseModel):
+    valid: bool
+    invitee_name: str = ""
+    inviter_name: str = ""
+
+
+class RegistrationStatusOut(BaseModel):
+    """Public, unauthenticated: what the signup page should show."""
+
+    open: bool
+    # True when the instance has no users yet, so signup creates the administrator.
+    bootstrap: bool
+    # The domain every handle lives under (localpart@domain), shown beside the
+    # handle field.
+    domain: str
+
+
+class AdminSettingsOut(BaseModel):
+    registration_open: bool
+    default_quota_bytes: int
+    # Users may send friends sign-up invitations (app/identity/invitations.py).
+    invitations_enabled: bool
+
+
+class AdminSettingsIn(BaseModel):
+    """Merge semantics: omitted fields are left as they are."""
+
+    registration_open: bool | None = None
+    invitations_enabled: bool | None = None
+    # Applies to users created after the change; existing mailboxes keep theirs.
+    default_quota_bytes: int | None = Field(default=None, gt=0)
+
+
+class AdminMailOut(BaseModel):
+    smtp_host: str
+    smtp_port: int
+    smtp_security: str
+    smtp_username: str
+    # The password itself never leaves the server.
+    has_password: bool
+    smtp_from: str
+    public_url: str
+    # Host, From address and public address are all set: reset mail can go out.
+    configured: bool
+
+
+class AdminMailIn(BaseModel):
+    """Merge semantics: omitted fields are left as they are."""
+
+    smtp_host: str | None = Field(default=None, max_length=255)
+    smtp_port: int | None = Field(default=None, ge=1, le=65535)
+    smtp_security: Literal["starttls", "ssl", "none"] | None = None
+    smtp_username: str | None = Field(default=None, max_length=320)
+    # Omitted keeps the stored password; an empty string removes it.
+    smtp_password: str | None = Field(default=None, max_length=1024)
+    smtp_from: str | None = Field(default=None, max_length=320)
+    public_url: str | None = Field(default=None, max_length=500)
+
+
+class AdminInviteTemplateOut(BaseModel):
+    # Empty strings: the built-in subject / template / sender is used.
+    subject: str
+    html: str
+    from_address: str
+    default_subject: str
+    default_html: str
+    default_from: str
+    placeholders: list[str]
+
+
+class AdminInviteTemplateIn(BaseModel):
+    """Merge semantics: omitted fields are left as they are; an empty string
+    goes back to the built-in one."""
+
+    subject: str | None = Field(default=None, max_length=300)
+    html: str | None = None
+    from_address: str | None = Field(default=None, max_length=320)
+
+
+class AdminInvitePreviewIn(BaseModel):
+    subject: str = Field(default="", max_length=300)
+    html: str = ""
+
+
+class AdminInvitePreviewOut(BaseModel):
+    subject: str
+    html: str
+
+
+class AdminMailTestIn(BaseModel):
+    to: str = Field(min_length=3, max_length=320)
+
+
+class AdminUserOut(BaseModel):
+    id: uuid.UUID
+    handle: str
+    is_admin: bool
+    # ISO timestamp while blocked; null otherwise.
+    blocked_at: str | None
+    quota_bytes: int
+    used_bytes: int
+    created_at: str
+    # The user's connected mail accounts, so the admin can tell who this is.
+    accounts: list[str]
+
+
+class AdminUserPatch(BaseModel):
+    """Merge semantics: omitted fields are left as they are."""
+
+    quota_bytes: int | None = Field(default=None, gt=0)
+    blocked: bool | None = None
 
 
 class MessageOut(BaseModel):

backend/app/identity/service.py

@@ -2,16 +2,19 @@
 
 from __future__ import annotations
 
+import uuid
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 
 from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import get_settings
 from app.identity.handles import InvalidHandle, full_handle, normalize_localpart
 from app.identity.passwords import hash_password, verify_password
-from app.models import User
+from app.models import AppSettings, User
 from app.redis_client import get_redis
 
 # SPEC §6.1: 10 attempts / 15 min per handle AND per IP.
@@ -28,6 +31,15 @@ class RegistrationError(Exception):
         self.message = message
 
 
+@dataclass(frozen=True)
+class RegistrationStatus:
+    """What the signup page may offer right now."""
+
+    open: bool
+    # True when the instance has no users: the next signup creates the administrator.
+    bootstrap: bool
+
+
 @dataclass(frozen=True)
 class RateLimited(Exception):
     """Raised when the login rate limit is exceeded."""
@@ -43,6 +55,10 @@ class TotpInvalid(Exception):
     """Password verified but the supplied TOTP code was wrong."""
 
 
+class AccountBlocked(Exception):
+    """Password verified but an administrator has blocked the account."""
+
+
 def resolve_handle(handle_input: str) -> str:
     """Normalize a login identifier to the full ``localpart@APP_DOMAIN`` form."""
     text = handle_input.strip().lower()
@@ -51,19 +67,69 @@ def resolve_handle(handle_input: str) -> str:
     return f"{normalize_localpart(text)}@{get_settings().app_domain}"
 
 
+async def app_settings(db: AsyncSession, *, for_update: bool = False) -> AppSettings:
+    """Load the singleton settings row, creating it if an old database lacks one.
+
+    ``for_update`` takes a row lock, which is what serialises concurrent signups.
+    A row conjured here defaults to *closed*: the safe answer when we cannot tell
+    what the operator intended.
+    """
+    stmt = select(AppSettings).where(AppSettings.id == 1)
+    row = await db.scalar(stmt.with_for_update() if for_update else stmt)
+    if row is None:
+        await db.execute(
+            pg_insert(AppSettings).values(id=1, registration_open=False).on_conflict_do_nothing()
+        )
+        row = await db.scalar(stmt.with_for_update() if for_update else stmt)
+        assert row is not None  # just inserted in this transaction
+    return row
+
+
+async def _has_users(db: AsyncSession) -> bool:
+    return await db.scalar(select(User.id).limit(1)) is not None
+
+
+async def registration_status(db: AsyncSession) -> RegistrationStatus:
+    """Whether signup is currently accepted, and whether it would bootstrap an admin."""
+    bootstrap = not await _has_users(db)
+    settings = await app_settings(db)
+    return RegistrationStatus(open=bootstrap or settings.registration_open, bootstrap=bootstrap)
+
+
 async def register_user(
-    db: AsyncSession, *, localpart: str, password: str, recovery_email: str | None
+    db: AsyncSession,
+    *,
+    localpart: str,
+    password: str,
+    recovery_email: str | None,
+    invited: bool = False,
 ) -> User:
-    """Create a new user. Raises :class:`RegistrationError` on validation/uniqueness."""
+    """Create a new user. Raises :class:`RegistrationError` on validation/uniqueness.
+
+    The first user on an empty instance is always allowed and becomes the
+    administrator, which closes registration behind them; after that it takes an
+    admin reopening it. The settings row is locked across the whole decision so
+    two simultaneous first signups cannot both see an empty table and both claim
+    administrator.
+    """
     try:
         handle = full_handle(localpart)
     except InvalidHandle as exc:
         raise RegistrationError(exc.code, exc.message) from exc
 
+    settings = await app_settings(db, for_update=True)
+    bootstrap = not await _has_users(db)
+    if not bootstrap and not settings.registration_open and not invited:
+        raise RegistrationError("registration_closed", "registration is closed on this server")
+
     user = User(
         handle=handle,
         password_hash=hash_password(password),
         recovery_email=(recovery_email.strip().lower() if recovery_email else None),
+        is_admin=bootstrap,
+        # Snapshotted at creation: later changes to the instance default leave
+        # this user's quota alone.
+        quota_bytes=settings.default_quota_bytes,
     )
     db.add(user)
     try:
@@ -71,6 +137,8 @@ async def register_user(
     except IntegrityError as exc:
         await db.rollback()
         raise RegistrationError("handle_taken", "that handle is already registered") from exc
+    if bootstrap:
+        settings.registration_open = False
     return user
 
 
@@ -91,12 +159,20 @@ async def _check_rate_limit(handle: str, ip: str) -> None:
 
 
 async def authenticate(
-    db: AsyncSession, *, handle_input: str, password: str, ip: str, totp: str | None = None
+    db: AsyncSession,
+    *,
+    handle_input: str,
+    password: str,
+    ip: str,
+    totp: str | None = None,
+    trusted_device: Callable[[uuid.UUID], Awaitable[bool]] | None = None,
 ) -> User | None:
     """Verify credentials (constant-time). Returns the user or None; may rate-limit.
 
     When the account has TOTP enabled, raises :class:`TotpRequired` (no code given)
-    or :class:`TotpInvalid` (wrong code) after the password check passes.
+    or :class:`TotpInvalid` (wrong code) after the password check passes — unless
+    ``trusted_device`` says this browser was trusted for the account at an earlier
+    2FA sign-in, in which case the password is enough.
     """
     handle = resolve_handle(handle_input)
     await _check_rate_limit(handle, ip)
@@ -107,11 +183,16 @@ async def authenticate(
     ok = verify_password(stored_hash, password)
     if user is None or not ok:
         return None
+    # Only after the password: a stranger must not learn that a handle is blocked.
+    if user.blocked_at is not None:
+        raise AccountBlocked
 
     if user.totp_secret is not None:
         from app.identity.totp import decrypt_totp_secret
         from app.identity.totp import verify as verify_totp
 
+        if not totp and trusted_device is not None and await trusted_device(user.id):
+            return user
         if not totp:
             raise TotpRequired
         if not verify_totp(decrypt_totp_secret(user.totp_secret), totp):

backend/app/identity/sessions.py

@@ -121,6 +121,13 @@ async def list_sessions(user_id: uuid.UUID, current_token: str) -> list[SessionI
     return out
 
 
+async def session_alive(user_id: uuid.UUID | str, sid: str) -> bool:
+    """Whether the session with public id ``sid`` is still signed in."""
+    redis = get_redis()
+    token = await aw(redis.hget(_index_key(user_id), sid))
+    return token is not None and bool(await redis.exists(_PREFIX + token))
+
+
 async def revoke_session(user_id: uuid.UUID, sid: str) -> bool:
     """Revoke a specific session by its public sid. Returns False if not found."""
     redis = get_redis()

backend/app/identity/trusted_devices.py

@@ -0,0 +1,91 @@
+"""Trusted devices: "don't ask for the 2FA code on this device" (SPEC §6.1).
+
+Ticking the box at a successful 2FA sign-in stores a random token in Redis
+(``trustdev:{token}`` -> user id) and hands it to the browser in its own
+HttpOnly cookie. A later sign-in to the *same* account from that browser may
+then skip the code — the password is still required. Other accounts signing in
+on the same browser are unaffected.
+
+The trust lasts TRUST_DAYS and is renewed each time it is used, so a device in
+regular use stays trusted while one left alone for that long has to show a code
+again. A per-user index (``trustdev-user:{user_id}``) lets everything be revoked
+at once: on a password reset, when 2FA is turned off, and when an administrator
+blocks or deletes the account. Signing out does not revoke it — that is the
+point of the box.
+"""
+
+from __future__ import annotations
+
+import secrets
+import uuid
+
+from fastapi import Request, Response
+
+from app.redis_client import aw, get_redis
+
+TRUST_DAYS = 90
+_TTL = TRUST_DAYS * 24 * 3600
+_PREFIX = "trustdev:"
+_INDEX = "trustdev-user:"
+
+COOKIE_SECURE = "__Host-trusted-device"
+COOKIE_INSECURE = "trusted_device"
+
+
+def _cookie_name(request: Request) -> str:
+    # Same split as the session cookie: the __Host- prefix needs HTTPS.
+    return COOKIE_SECURE if request.url.scheme == "https" else COOKIE_INSECURE
+
+
+def read_cookie(request: Request) -> str:
+    return request.cookies.get(COOKIE_SECURE) or request.cookies.get(COOKIE_INSECURE, "")
+
+
+async def trust(request: Request, response: Response, user_id: uuid.UUID) -> None:
+    """Remember this browser for ``user_id`` and set the cookie."""
+    token = secrets.token_urlsafe(32)
+    redis = get_redis()
+    await redis.set(_PREFIX + token, str(user_id), ex=_TTL)
+    await aw(redis.sadd(_INDEX + str(user_id), token))
+    await redis.expire(_INDEX + str(user_id), _TTL)
+    response.set_cookie(
+        _cookie_name(request),
+        token,
+        max_age=_TTL,
+        httponly=True,
+        secure=request.url.scheme == "https",
+        samesite="lax",
+        path="/",
+    )
+
+
+async def is_trusted(request: Request, response: Response, user_id: uuid.UUID) -> bool:
+    """Whether this browser is trusted for ``user_id``; renews the trust if so."""
+    token = read_cookie(request)
+    if not token:
+        return False
+    redis = get_redis()
+    if await redis.get(_PREFIX + token) != str(user_id):
+        return False
+    await redis.expire(_PREFIX + token, _TTL)
+    await redis.expire(_INDEX + str(user_id), _TTL)
+    response.set_cookie(
+        _cookie_name(request),
+        token,
+        max_age=_TTL,
+        httponly=True,
+        secure=request.url.scheme == "https",
+        samesite="lax",
+        path="/",
+    )
+    return True
+
+
+async def revoke_all(user_id: uuid.UUID) -> None:
+    """Forget every device trusted for ``user_id``."""
+    redis = get_redis()
+    index = _INDEX + str(user_id)
+    tokens: set[str] = await aw(redis.smembers(index))
+    if tokens:
+        await redis.delete(*(_PREFIX + t for t in tokens))
+    await redis.delete(index)

backend/app/identity/user_admin.py

@@ -0,0 +1,146 @@
+"""Administrator actions on user accounts (SPEC §6.1 admin panel).
+
+Search, per-user quota, blocking and deletion. Blocking keeps every byte of the
+user's data (and keeps their mail syncing) so unblocking is instant; it only
+refuses sign-in and revokes the sessions they hold. Deletion is total: mail
+accounts (through the same path the user's own "remove account" takes, so the
+sync runners stop), plugin rows, blobs on disk, the search index and the user
+row, whose foreign keys cascade to everything else core owns.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import shutil
+import uuid
+from pathlib import Path
+
+from sqlalchemy import func, or_, select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.accounts.service import delete_account
+from app.config import get_settings
+from app.events import get_event_bus
+from app.identity import trusted_devices
+from app.identity.sessions import destroy_all_sessions
+from app.models import Account, Message, User
+from app.plugins.db import plugin_session
+from app.search.factory import get_search_backend
+
+SEARCH_LIMIT = 25
+
+
+async def search_users(db: AsyncSession, q: str) -> list[tuple[User, int, list[str]]]:
+    """Users whose login, or one of whose mail accounts' address or display name,
+    contains ``q`` (case-insensitive). An empty ``q`` lists the first users by
+    login. Each row comes with its storage used and its mail account addresses."""
+    stmt = select(User).order_by(User.handle).limit(SEARCH_LIMIT)
+    q = q.strip()
+    if q:
+        # Typed % and _ are literal characters, not wildcards.
+        escaped = q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+        like = f"%{escaped}%"
+        by_account = select(Account.user_id).where(
+            or_(Account.email.ilike(like), Account.display_name.ilike(like))
+        )
+        stmt = stmt.where(or_(User.handle.ilike(like), User.id.in_(by_account)))
+    users = list(await db.scalars(stmt))
+    if not users:
+        return []
+    ids = [u.id for u in users]
+    used: dict[uuid.UUID, int] = {
+        user_id: int(total)
+        for user_id, total in await db.execute(
+            select(Message.user_id, func.coalesce(func.sum(Message.size_bytes), 0))
+            .where(Message.user_id.in_(ids))
+            .group_by(Message.user_id)
+        )
+    }
+    emails: dict[uuid.UUID, list[str]] = {}
+    for user_id, email in await db.execute(
+        select(Account.user_id, Account.email)
+        .where(Account.user_id.in_(ids))
+        .order_by(Account.email)
+    ):
+        emails.setdefault(user_id, []).append(email)
+    return [(u, int(used.get(u.id, 0)), emails.get(u.id, [])) for u in users]
+
+
+async def set_blocked(db: AsyncSession, user: User, blocked: bool) -> None:
+    """Block or unblock. Blocking also signs the user out everywhere, and shuts
+    their hosted mailbox to every sign-in (mail apps included); it still
+    receives mail. The mail server goes first: if it cannot be told, nothing
+    changes."""
+    from app import hosted
+
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        await mail_admin.set_blocked(user.handle, blocked)
+    if blocked and user.blocked_at is None:
+        user.blocked_at = dt.datetime.now(dt.UTC)
+        await db.flush()
+        await _sign_out_everywhere(user.id)
+    elif not blocked:
+        user.blocked_at = None
+
+
+async def _sign_out_everywhere(user_id: uuid.UUID) -> None:
+    """Revoke every session, and tell the user's open tabs so they leave at once
+    instead of showing already-loaded mail until their next request."""
+    await destroy_all_sessions(user_id)
+    await trusted_devices.revoke_all(user_id)
+    await get_event_bus().publish("user.signed_out", {"user_id": str(user_id)})
+
+
+async def _plugin_tables(db: AsyncSession) -> list[str]:
+    """Every plugin table holding per-user rows (``plg_*`` with a ``user_id``)."""
+    rows = await db.execute(
+        text(
+            "SELECT table_name FROM information_schema.columns "
+            "WHERE table_schema = current_schema() AND column_name = 'user_id' "
+            "AND table_name LIKE 'plg\\_%' ORDER BY table_name"
+        )
+    )
+    return [r[0] for r in rows]
+
+
+async def delete_user(db: AsyncSession, user: User) -> None:
+    """Delete ``user`` and everything they own. Commits.
+
+    Their hosted mailbox goes first, with all its mail: if the mail server
+    cannot be reached nothing is deleted — a mailbox left behind would be
+    handed, mail and all, to whoever registers the address next."""
+    from app import hosted
+
+    user_id = user.id
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        await mail_admin.delete_account(user.handle)
+
+    # Mail accounts first, one by one: this is what stops their sync runners and
+    # tidies threads, exactly as when the user removes an account themselves.
+    for account_id in list(await db.scalars(select(Account.id).where(Account.user_id == user_id))):
+        await delete_account(db, user_id=user_id, account_id=account_id)
+    await db.commit()
+
+    # Plugin tables carry no foreign key to users and sit behind row-level
+    # security, so they are cleared from inside the user's own RLS context.
+    tables = await _plugin_tables(db)
+    if tables:
+        async with plugin_session(user_id) as ps:
+            for table in tables:
+                await ps.execute(
+                    text(f'DELETE FROM "{table}" WHERE user_id = :uid'), {"uid": user_id}
+                )
+
+    await _sign_out_everywhere(user_id)
+    await get_search_backend().delete_user(user_id)
+
+    # The user row: its foreign keys cascade to the rest of core's tables.
+    await db.delete(user)
+    await db.commit()
+
+    # Whatever blobs are left (drafts' attachments and the like) go with the folder.
+    shutil.rmtree(Path(get_settings().blob_root) / str(user_id), ignore_errors=True)

backend/app/indexer/attachments.py

@@ -0,0 +1,244 @@
+"""Attachments, kept once: inside their message (SPEC App. D §D.6 item 1).
+
+An attachment is not stored apart from its message any more — it was, which
+kept most of a mailbox's bytes twice (once base64 in the raw message, once
+decoded on its own). It is taken out of the raw message when wanted:
+
+* :func:`load` — its bytes (downloads, zip, inline images, forwarding, thumbnails);
+* :func:`file` — a file holding it, for streaming with HTTP Range (videos):
+  taken out into ``{user}/cache/`` and kept a day after its last use;
+* :func:`dedupe_existing` — the one-time pass removing the separate copies
+  stored before, each only after checking it can be taken out of its message.
+
+Blocking (parsing, disk): call these in a thread — after
+:func:`app.indexer.rawstore.ensure` for the message, which fetches a hosted
+mailbox's message from its mail server when it is not cached.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import logging
+import time
+import uuid
+from collections import defaultdict
+from pathlib import Path
+
+from app.indexer import blobstore, rawstore
+from app.indexer.mime import load_message, part_payload
+
+logger = logging.getLogger(__name__)
+
+CACHE_KEEP_S = 24 * 3600  # a taken-out file is kept this long after its last use
+CACHE_MAX_BYTES = 1024**3  # per user; the least recently used go first beyond it
+# part_index of a row whose content is not in its message (kept as its own file).
+NOT_IN_MESSAGE = -1
+
+
+def _legacy(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The separate copy stored before attachments were kept once, if still there."""
+    path = blobstore.blob_path(user_id, sha)
+    return path if path.exists() else None
+
+
+def load(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bytes:
+    """An attachment's bytes. Raises FileNotFoundError when they cannot be had."""
+    legacy = _legacy(user_id, sha)
+    if legacy is not None:
+        return legacy.read_bytes()
+    raw = rawstore.locate(user_id, message_sha)
+    if part_index == NOT_IN_MESSAGE or raw is None:
+        raise FileNotFoundError(sha)
+    payload = part_payload(load_message(path=raw), part_index, sha)
+    if payload is None:
+        raise FileNotFoundError(sha)
+    return payload
+
+
+def materialize(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bool:
+    """Make sure the attachment exists as a blob of its own — for a compose
+    attachment (forward, a draft reopened), which is read by its hash. False
+    when its bytes cannot be had."""
+    if blobstore.exists(user_id, sha):
+        return True
+    try:
+        blobstore.store(user_id, load(user_id, message_sha, sha, part_index))
+    except FileNotFoundError:
+        return False
+    return True
+
+
+def _cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / "cache"
+
+
+def prune_cache(cache: Path) -> None:
+    """Drop streaming copies past their keep time, and the oldest beyond the cap."""
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file()]
+    now = time.time()
+    total = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > CACHE_KEEP_S or total > CACHE_MAX_BYTES:
+            path.unlink(missing_ok=True)
+
+
+def has_file(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether :func:`file` can answer without reading the message."""
+    return _legacy(user_id, sha) is not None or (_cache_dir(user_id) / sha).exists()
+
+
+def file(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> Path:
+    """A file holding the attachment, to stream from. Raises FileNotFoundError."""
+    legacy = _legacy(user_id, sha)
+    if legacy is not None:
+        return legacy
+    cache = _cache_dir(user_id)
+    path = cache / sha
+    if path.exists():
+        path.touch()  # used again: kept longer
+        return path
+    data = load(user_id, message_sha, sha, part_index)
+    cache.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+    tmp.write_bytes(data)
+    tmp.replace(path)
+    with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+        prune_cache(cache)
+    return path
+
+
+def drop_cached(user_id: uuid.UUID) -> None:
+    """Forget every taken-out file of a user (account removed)."""
+    cache = _cache_dir(user_id)
+    if cache.exists():
+        for path in cache.iterdir():
+            path.unlink(missing_ok=True)
+
+
+# ---- removing the copies stored before ------------------------------------------------
+
+# A file this recent may belong to work in progress (a sent message waiting
+# for its Sent copy is referenced only by its job): left for the next pass.
+_MIN_AGE_S = 3600
+
+
+def _part_indexes(raw_path: Path) -> dict[str, int]:
+    """Content hash → MIME part position, for every leaf part of a message."""
+    out: dict[str, int] = {}
+    for index, part in enumerate(load_message(path=raw_path).walk()):
+        if part.is_multipart():
+            continue
+        payload = part.get_payload(decode=True)
+        if isinstance(payload, bytes):
+            out.setdefault(hashlib.sha256(payload).hexdigest(), index)
+    return out
+
+
+async def dedupe_existing(batch: int = 200, *, user_id: uuid.UUID | None = None) -> dict[str, int]:
+    """Record where each older attachment sits in its message, and delete its
+    separate copy once it is known to be there. Idempotent; resumable."""
+    import asyncio
+
+    from sqlalchemy import select, update
+
+    from app.db import session_scope
+    from app.models import Attachment, ComposeAttachment, Message
+
+    stats = {"rows": 0, "found": 0, "not_found": 0, "files_removed": 0, "bytes_freed": 0}
+    while True:
+        async with session_scope() as db:
+            rows = (
+                await db.execute(
+                    select(
+                        Attachment.id,
+                        Attachment.user_id,
+                        Attachment.blob_sha256,
+                        Message.blob_sha256,
+                    )
+                    .join(Message, Message.id == Attachment.message_id)
+                    .where(
+                        Attachment.part_index.is_(None),
+                        *([Attachment.user_id == user_id] if user_id is not None else []),
+                    )
+                    .order_by(Attachment.message_id)
+                    .limit(batch)
+                )
+            ).all()
+        if not rows:
+            return stats
+        by_message: dict[tuple[uuid.UUID, str], list[tuple[uuid.UUID, str]]] = defaultdict(list)
+        for att_id, owner, sha, msg_sha in rows:
+            by_message[(owner, msg_sha)].append((att_id, sha))
+
+        located: dict[uuid.UUID, int] = {}
+        for (owner, msg_sha), atts in by_message.items():
+            raw = rawstore.locate(owner, msg_sha)
+            try:
+                indexes = await asyncio.to_thread(_part_indexes, raw) if raw is not None else {}
+            except Exception:  # noqa: BLE001 - unreadable message: keep the copies
+                logger.warning("attachments: cannot parse %s", raw, exc_info=True)
+                indexes = {}
+            for att_id, sha in atts:
+                located[att_id] = indexes.get(sha, NOT_IN_MESSAGE)
+
+        async with session_scope() as db:
+            for att_id, index in located.items():
+                await db.execute(
+                    update(Attachment).where(Attachment.id == att_id).values(part_index=index)
+                )
+        stats["rows"] += len(located)
+        stats["found"] += sum(1 for i in located.values() if i != NOT_IN_MESSAGE)
+        stats["not_found"] += sum(1 for i in located.values() if i == NOT_IN_MESSAGE)
+
+        # A separate copy goes only when every attachment row with that content
+        # is found in its message and nothing else uses the file.
+        candidates = {
+            (owner, sha)
+            for att_id, owner, sha, _ in rows
+            if located.get(att_id, NOT_IN_MESSAGE) != NOT_IN_MESSAGE
+        }
+        async with session_scope() as db:
+            for owner, sha in candidates:
+                path = blobstore.blob_path(owner, sha)
+                if not path.exists() or time.time() - path.stat().st_mtime < _MIN_AGE_S:
+                    continue
+                in_use = (
+                    await db.scalar(
+                        select(Message.id)
+                        .where(Message.user_id == owner, Message.blob_sha256 == sha)
+                        .limit(1)
+                    )
+                    or await db.scalar(
+                        select(ComposeAttachment.id)
+                        .where(
+                            ComposeAttachment.user_id == owner,
+                            ComposeAttachment.blob_sha256 == sha,
+                        )
+                        .limit(1)
+                    )
+                    or await db.scalar(
+                        select(Attachment.id)
+                        .where(
+                            Attachment.user_id == owner,
+                            Attachment.blob_sha256 == sha,
+                            (Attachment.part_index.is_(None))
+                            | (Attachment.part_index == NOT_IN_MESSAGE),
+                        )
+                        .limit(1)
+                    )
+                )
+                if in_use:
+                    continue
+                size = path.stat().st_size
+                path.unlink(missing_ok=True)
+                stats["files_removed"] += 1
+                stats["bytes_freed"] += size
+
+
+async def dedupe_existing_job(ctx: dict[str, object]) -> dict[str, int]:
+    """arq job (bulk queue): :func:`dedupe_existing`, run once after an update."""
+    stats = await dedupe_existing()
+    logger.info("attachments kept once: %s", stats)
+    return stats

backend/app/indexer/gc.py

@@ -0,0 +1,217 @@
+"""The nightly clean-up of stored files nothing uses any more.
+
+Messages leave in many ways — Delete forever, Empty Trash, the Trash/Spam
+retention purge, an expunge on the server, a folder re-synced — and none of
+them removes the message's file: files are stored by content, so one file can
+serve several rows (the same email in two folders, a forwarded file in a
+draft), and removing it is only safe once nothing refers to it. This pass does
+that for every user, whatever the way the rows went:
+
+* uploads (compose attachments) no draft has referred to for a week — left by
+  a sent message (its draft is gone) or a composer closed without saving;
+* message/attachment files no row refers to;
+* attachment previews (thumbnails) of attachments that are gone;
+* streaming copies past their keep time (app/indexer/attachments.py);
+* cached raw messages of a hosted mailbox beyond the cache's room
+  (app/indexer/rawstore.py).
+
+It also recounts threads whose unread/message counters disagree with their
+messages, and drops threads left empty (app/threading_/threader.py).
+
+A file younger than an hour is left alone: it may belong to work in progress
+(a message being stored, a sent message waiting for its Sent copy).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import delete, or_, select
+
+from app.config import get_settings
+from app.indexer import attachments as att_store
+from app.indexer import rawstore
+
+logger = logging.getLogger(__name__)
+
+MIN_AGE_S = 3600
+UPLOAD_KEEP = dt.timedelta(days=7)
+_SHA = re.compile(r"^[0-9a-f]{64}$")
+
+
+# ---- files a job still needs ---------------------------------------------------------
+#
+# A sent message whose Sent copy could not be filed yet is referenced only by
+# its retrying job (app/outbox/jobs.py), for up to a day: held here meanwhile.
+# Entries older than HOLD_MAX are ignored, in case a job died without letting go.
+
+HOLD_MAX_S = 3 * 24 * 3600
+
+
+def _hold_key(user_id: uuid.UUID | str) -> str:
+    return f"blob:held:{user_id}"
+
+
+async def hold(user_id: uuid.UUID | str, sha: str) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().zadd(_hold_key(user_id), {sha: time.time()})
+
+
+async def release(user_id: uuid.UUID | str, sha: str) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().zrem(_hold_key(user_id), sha)
+
+
+async def _held(user_id: uuid.UUID) -> set[str]:
+    from app.redis_client import get_redis
+
+    redis = get_redis()
+    oldest = time.time() - HOLD_MAX_S
+    await redis.zremrangebyscore(_hold_key(user_id), "-inf", oldest)
+    return {
+        m.decode() if isinstance(m, bytes) else str(m)
+        for m in await redis.zrangebyscore(_hold_key(user_id), oldest, "+inf")
+    }
+
+
+def _draft_upload_ids(payloads: list[dict[str, Any]]) -> set[uuid.UUID]:
+    out: set[uuid.UUID] = set()
+    for payload in payloads:
+        for value in payload.get("attachment_ids") or []:
+            try:
+                out.add(uuid.UUID(str(value)))
+            except ValueError:
+                continue
+    return out
+
+
+def _old(path: Path, now: float) -> bool:
+    return now - path.stat().st_mtime >= MIN_AGE_S
+
+
+async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
+    """Clean one user's stored files; returns what was removed."""
+    from app.db import session_scope
+    from app.models import Account, Attachment, ComposeAttachment, Draft, Message
+
+    stats = {"uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
+    async with session_scope() as db:
+        # Uploads no draft refers to, once a week old.
+        used = _draft_upload_ids(
+            list(await db.scalars(select(Draft.payload).where(Draft.user_id == user_id)))
+        )
+        stale = select(ComposeAttachment.id).where(
+            ComposeAttachment.user_id == user_id,
+            ComposeAttachment.created_at < dt.datetime.now(dt.UTC) - UPLOAD_KEEP,
+        )
+        if used:
+            stale = stale.where(ComposeAttachment.id.not_in(used))
+        gone = list(await db.scalars(stale))
+        if gone:
+            await db.execute(delete(ComposeAttachment).where(ComposeAttachment.id.in_(gone)))
+        stats["uploads"] = len(gone)
+
+    async with session_scope() as db:
+        # Every file still referred to. Attachments count only where their file
+        # is their only copy (not found in the message, or not yet looked for).
+        # A hosted mailbox's messages are kept by its mail server: their raw
+        # file here (left by a Sent copy filed late) is not needed.
+        referenced = set(
+            await db.scalars(
+                select(Message.blob_sha256)
+                .join(Account, Account.id == Message.account_id)
+                .where(Message.user_id == user_id, Account.kind != "native")
+            )
+        )
+        referenced |= set(
+            await db.scalars(
+                select(ComposeAttachment.blob_sha256).where(ComposeAttachment.user_id == user_id)
+            )
+        )
+        referenced |= set(
+            await db.scalars(
+                select(Attachment.blob_sha256).where(
+                    Attachment.user_id == user_id,
+                    or_(
+                        Attachment.part_index.is_(None),
+                        Attachment.part_index == att_store.NOT_IN_MESSAGE,
+                    ),
+                )
+            )
+        )
+        referenced |= await _held(user_id)
+        attachment_shas = set(
+            await db.scalars(select(Attachment.blob_sha256).where(Attachment.user_id == user_id))
+        )
+
+    # Counters out of step with their messages (see repair_threads).
+    from app.threading_.threader import repair_threads
+
+    async with session_scope() as db:
+        stats["threads_repaired"] = await repair_threads(db, user_id)
+
+    root = Path(get_settings().blob_root) / str(user_id)
+    if not root.is_dir():
+        return stats
+    now = time.time()
+    for path in root.glob("??/*"):
+        if not path.is_file() or not _SHA.match(path.name) or path.name in referenced:
+            continue
+        if not _old(path, now):
+            continue
+        size = path.stat().st_size
+        path.unlink(missing_ok=True)
+        stats["files"] += 1
+        stats["bytes"] += size
+    thumbs = root / "thumbs"
+    if thumbs.is_dir():
+        for path in thumbs.iterdir():
+            sha = path.name.split("-", 1)[0]
+            if (
+                path.is_file()
+                and _SHA.match(sha)
+                and sha not in attachment_shas
+                and _old(path, now)
+            ):
+                path.unlink(missing_ok=True)
+                stats["thumbs"] += 1
+    cache = root / "cache"
+    if cache.is_dir():
+        att_store.prune_cache(cache)
+    rawstore.prune(user_id)
+    return stats
+
+
+async def sweep(user_id: uuid.UUID | None = None) -> dict[str, int]:
+    """:func:`sweep_user` for one user, or every user."""
+    from app.db import session_scope
+    from app.models import User
+
+    async with session_scope() as db:
+        users = [user_id] if user_id is not None else list(await db.scalars(select(User.id)))
+    total = {"users": 0, "uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
+    for uid in users:
+        try:
+            stats = await sweep_user(uid)
+        except Exception:  # noqa: BLE001 - one user's trouble must not stop the rest
+            logger.warning("stored-file clean-up failed for user %s", uid, exc_info=True)
+            continue
+        total["users"] += 1
+        for key, value in stats.items():
+            total[key] += value
+    return total
+
+
+async def sweep_job(ctx: dict[str, Any]) -> dict[str, int]:
+    """arq cron job: the nightly :func:`sweep`."""
+    stats = await sweep()
+    logger.info("stored-file clean-up: %s", stats)
+    return stats

backend/app/indexer/indexer.py

@@ -17,12 +17,15 @@ from sqlalchemy import select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.indexer import blobstore
-from app.indexer.mime import ParsedEmail, parse_message
+from app.concurrency import loop_semaphore
+from app.config import get_settings
+from app.indexer import blobstore, rawstore
+from app.indexer.mime import ParsedEmail, parse_message, parse_message_file
 from app.indexer.sanitize import sanitize_html
-from app.models import Attachment, BodyPart, KnownRecipient, Message
+from app.models import Account, Attachment, BodyPart, KnownRecipient, Message
 from app.providers.base import RemoteMessage
 from app.search.base import SearchDoc
+from app.subscriptions import read_list_headers
 from app.threading_.normalize import synthesize_message_id
 from app.threading_.threader import thread_message
 
@@ -34,6 +37,7 @@ class PersistResult:
     message: Message
     search_doc: SearchDoc
     has_remote_images: bool
+    gmail_labels: tuple[str, ...] = ()
 
 
 def _epoch(value: datetime) -> int:
@@ -52,7 +56,7 @@ def _build_search_doc(message: Message, parsed: ParsedEmail, special_use: str |
         cc=parsed.cc_addr,
         subject=parsed.subject,
         body_text=parsed.text_body,
-        attachment_names=[a.filename for a in parsed.attachments],
+        attachment_names=[a.filename for a in parsed.attachments if a.content_id is None],
         has_attachments=parsed.has_attachments,
         is_unread="\\Seen" not in message.flags,
         is_starred="\\Flagged" in message.flags,
@@ -115,6 +119,29 @@ async def _resolve_message_id(
     return parsed_id
 
 
+async def _keeps_raw(session: AsyncSession, account_id: uuid.UUID) -> bool:
+    """Whether the raw message is kept for good: not for a hosted mailbox, whose
+    mail server holds it (it is only cached, App. D §D.6 item 2)."""
+    kind = await session.scalar(select(Account.kind).where(Account.id == account_id))
+    return kind != "native"
+
+
+async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmail:
+    """Parse under the PARSE_CONCURRENCY cap; stream large messages from disk (§7).
+
+    Peak RAM stays ~PARSE_CONCURRENCY x STREAM_PARSE_MB: only a few parses run at
+    once, and a message above the threshold is parsed from its (already written)
+    blob file instead of being held as an in-memory tree beside its raw bytes.
+    """
+    settings = get_settings()
+    threshold = settings.stream_parse_mb * 1024 * 1024
+    async with loop_semaphore("mime_parse", "parse", settings.parse_concurrency):
+        path = rawstore.locate(user_id, sha) if len(raw) > threshold else None
+        if path is not None:
+            return parse_message_file(path)
+        return parse_message(raw)
+
+
 async def persist_message(
     session: AsyncSession,
     *,
@@ -123,20 +150,41 @@ async def persist_message(
     mailbox_id: uuid.UUID,
     mailbox_special_use: str | None,
     remote: RemoteMessage,
+    dedupe_account_scope: bool = False,
 ) -> PersistResult | None:
-    """Persist one message. Returns None if it already exists (idempotent)."""
+    """Persist one message. Returns None if it already exists (idempotent).
+
+    ``dedupe_account_scope`` skips a message already stored under a *different*
+    mailbox of the same account (same Message-ID) — used for Gmail's All Mail pass so
+    a message already synced from INBOX/Sent isn't duplicated (SPEC: Gmail sync)."""
     existing = await session.scalar(
         select(Message.id).where(Message.mailbox_id == mailbox_id, Message.uid == remote.uid)
     )
     if existing is not None:
         return None
 
+    if await _keeps_raw(session, account_id):
         sha = blobstore.store(user_id, remote.raw)
-    parsed = parse_message(remote.raw)
+    else:
+        sha = rawstore.put_cached(user_id, remote.raw)
+    parsed = await _parse_bounded(user_id, sha, remote.raw)
+    list_headers = read_list_headers(remote.raw)  # for the Subscriptions page
 
     message_id_hdr = await _resolve_message_id(
         session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw
     )
+    if dedupe_account_scope and message_id_hdr is not None:
+        dup = await session.scalar(
+            select(Message.id)
+            .where(
+                Message.account_id == account_id,
+                Message.message_id_hdr == message_id_hdr,
+                Message.mailbox_id != mailbox_id,
+            )
+            .limit(1)
+        )
+        if dup is not None:
+            return None
     message = Message(
         id=uuid.uuid4(),
         user_id=user_id,
@@ -144,6 +192,9 @@ async def persist_message(
         mailbox_id=mailbox_id,
         uid=remote.uid,
         message_id_hdr=message_id_hdr,
+        # Keep the real delivered Message-ID (may differ from the stored, possibly
+        # synthesized, message_id_hdr) so cross-account copies collapse in threads.
+        delivery_id=parsed.message_id,
         in_reply_to=parsed.in_reply_to,
         references_hdr=parsed.references,
         subject=parsed.subject,
@@ -158,6 +209,9 @@ async def persist_message(
         has_attachments=parsed.has_attachments,
         snippet=parsed.snippet,
         blob_sha256=sha,
+        list_id=list_headers.list_id,
+        list_unsubscribe=list_headers.unsubscribe,
+        list_unsubscribe_post=list_headers.one_click,
     )
     session.add(message)
     await session.flush()
@@ -175,11 +229,13 @@ async def persist_message(
                 kind="html_sanitized",
                 content=result.html,
                 sanitizer_v=result.sanitizer_v,
+                trackers_blocked=result.trackers_blocked,
             )
         )
 
+    # Attachments are not stored apart: they stay inside the raw blob, and are
+    # taken out of it when wanted (app/indexer/attachments.py).
     for att in parsed.attachments:
-        att_sha = blobstore.store(user_id, att.payload)
         session.add(
             Attachment(
                 message_id=message.id,
@@ -188,7 +244,8 @@ async def persist_message(
                 mime_type=att.mime_type,
                 size_bytes=len(att.payload),
                 content_id=att.content_id,
-                blob_sha256=att_sha,
+                blob_sha256=blobstore.sha256_hex(att.payload),
+                part_index=att.part_index,
             )
         )
 
@@ -203,4 +260,9 @@ async def persist_message(
 
     await session.flush()
     doc = _build_search_doc(message, parsed, mailbox_special_use)
-    return PersistResult(message=message, search_doc=doc, has_remote_images=has_remote_images)
+    return PersistResult(
+        message=message,
+        search_doc=doc,
+        has_remote_images=has_remote_images,
+        gmail_labels=remote.gmail_labels,
+    )

backend/app/indexer/mime.py

@@ -7,6 +7,8 @@ Header RFC2047 decoding is automatic under ``policy.default``.
 from __future__ import annotations
 
 import email
+import os
+import re
 from dataclasses import dataclass, field
 from datetime import datetime
 from email.headerregistry import AddressHeader
@@ -19,6 +21,19 @@ from app.threading_.normalize import normalize_message_id, normalize_reference_l
 
 _SNIPPET_LEN = 160
 
+# Undecodable header/body bytes come out of the stdlib parser as lone UTF-16
+# surrogates (U+D800–U+DFFF). Those are invalid UTF-8, so Postgres rejects them in
+# text/JSONB columns — scrub them to U+FFFD before anything is stored.
+_SURROGATE_RE = re.compile("[\ud800-\udfff]")
+
+
+def _scrub(value: str) -> str:
+    return _SURROGATE_RE.sub("�", value) if value else value
+
+
+def _scrub_addrs(addrs: list[dict[str, str]]) -> list[dict[str, str]]:
+    return [{k: _scrub(v) for k, v in addr.items()} for addr in addrs]
+
 
 @dataclass(frozen=True)
 class ParsedAttachment:
@@ -26,6 +41,9 @@ class ParsedAttachment:
     mime_type: str
     content_id: str | None
     payload: bytes
+    # The part's position in ``msg.walk()``: where to find it again in the raw
+    # message, which is the only copy kept (attachments are not stored apart).
+    part_index: int = -1
 
 
 @dataclass
@@ -45,7 +63,9 @@ class ParsedEmail:
 
     @property
     def has_attachments(self) -> bool:
-        return len(self.attachments) > 0
+        # Inline resources (cid: images) carry a content_id and render in-body, not
+        # as attachment chips — only real attachments flip the paperclip.
+        return any(a.content_id is None for a in self.attachments)
 
     @property
     def snippet(self) -> str:
@@ -66,7 +86,14 @@ def _addresses(msg: EmailMessage, header: str) -> list[dict[str, str]]:
 
 def _html_to_text(html: str) -> str:
     """Derive plain text from HTML by stripping all tags (SPEC §6.4 nh3 strip)."""
-    return nh3.clean(html, tags=set(), attributes={}, strip_comments=True)
+    return nh3.clean(
+        html,
+        tags=set(),
+        attributes={},
+        strip_comments=True,
+        # The document <title> is not body text; unwrapped it would lead the snippet.
+        clean_content_tags={"script", "style", "title"},
+    )
 
 
 def _get_body_content(msg: EmailMessage, subtype: str) -> str | None:
@@ -84,30 +111,76 @@ def _get_body_content(msg: EmailMessage, subtype: str) -> str | None:
 
 
 def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
+    # Walk the whole tree, not iter_attachments(): the latter is shallow and treats
+    # multipart/related as a body, so an inline image nested in mixed>related>alt is
+    # never yielded (its cid: reference then renders broken). A part is an attachment
+    # or inline resource if it has a filename, a Content-ID, or attachment disposition;
+    # the body text/plain and text/html parts have none of these and are skipped.
     out: list[ParsedAttachment] = []
-    for part in msg.iter_attachments():
-        if not isinstance(part, EmailMessage):
+    for index, part in enumerate(msg.walk()):
+        if part.is_multipart() or not isinstance(part, EmailMessage):
+            continue
+        cid = part.get("Content-ID")
+        filename = part.get_filename()
+        disposition = part.get_content_disposition()
+        if not (filename or cid or disposition == "attachment"):
             continue
         payload = part.get_payload(decode=True)
         if not isinstance(payload, bytes):
             continue
-        cid = part.get("Content-ID")
+        # A part is an INLINE resource (cid: image rendered in-body, hidden from the
+        # attachment list) only when it carries a Content-ID AND is not explicitly an
+        # attachment. A part marked `Content-Disposition: attachment` is a real
+        # attachment even when it also has a Content-ID (e.g. Fastmail's invite.ics),
+        # so we drop the cid for it and it shows as a downloadable chip.
+        inline = cid is not None and disposition != "attachment"
         out.append(
             ParsedAttachment(
-                filename=part.get_filename() or "attachment",
+                filename=_scrub(filename or "attachment"),
                 mime_type=part.get_content_type(),
-                content_id=cid.strip("<>") if cid else None,
+                content_id=cid.strip("<>") if (inline and cid is not None) else None,
                 payload=payload,
+                part_index=index,
             )
         )
+    # An invitation sent only as the text/calendar alternative (Outlook, often) has
+    # no file to show: keep it as invite.ics, so the event can be seen, answered
+    # and downloaded like a Gmail one. Not when the message attaches its own .ics.
+    if not any(a.filename.lower().endswith(".ics") for a in out):
+        for index, part in enumerate(msg.walk()):
+            if part.get_content_type() != "text/calendar" or part.get_filename():
+                continue
+            payload = part.get_payload(decode=True)
+            if isinstance(payload, bytes) and payload.strip():
+                out.append(
+                    ParsedAttachment(
+                        filename="invite.ics",
+                        mime_type="text/calendar",
+                        content_id=None,
+                        payload=payload,
+                        part_index=index,
+                    )
+                )
+                break
     return out
 
 
 def parse_message(raw: bytes) -> ParsedEmail:
-    """Parse a raw RFC822 message into a :class:`ParsedEmail`."""
+    """Parse a raw RFC822 message (in memory) into a :class:`ParsedEmail`."""
     msg = email.message_from_bytes(raw, policy=default_policy)
     assert isinstance(msg, EmailMessage)
+    return _parse_email(msg)
 
+
+def parse_message_file(path: str | os.PathLike[str]) -> ParsedEmail:
+    """Parse a message straight from a blob file handle (large-message path, §7)."""
+    with open(path, "rb") as handle:
+        msg = email.message_from_binary_file(handle, policy=default_policy)
+    assert isinstance(msg, EmailMessage)
+    return _parse_email(msg)
+
+
+def _parse_email(msg: EmailMessage) -> ParsedEmail:
     message_id = normalize_message_id(msg.get("Message-ID"))
     in_reply_to_ids = normalize_reference_list(msg.get("In-Reply-To"))
     in_reply_to = in_reply_to_ids[0] if in_reply_to_ids else None
@@ -137,13 +210,46 @@ def parse_message(raw: bytes) -> ParsedEmail:
         message_id=message_id,
         in_reply_to=in_reply_to,
         references=references,
-        subject=subject,
-        from_addr=_addresses(msg, "From"),
-        to_addr=_addresses(msg, "To"),
-        cc_addr=_addresses(msg, "Cc"),
-        bcc_addr=_addresses(msg, "Bcc"),
+        subject=_scrub(subject),
+        from_addr=_scrub_addrs(_addresses(msg, "From")),
+        to_addr=_scrub_addrs(_addresses(msg, "To")),
+        cc_addr=_scrub_addrs(_addresses(msg, "Cc")),
+        bcc_addr=_scrub_addrs(_addresses(msg, "Bcc")),
         date=date,
-        text_body=text_body or "",
-        html_body=html_body,
+        text_body=_scrub(text_body or ""),
+        html_body=_scrub(html_body) if html_body is not None else None,
         attachments=_extract_attachments(msg),
     )
+
+
+def load_message(
+    raw: bytes | None = None, path: str | os.PathLike[str] | None = None
+) -> EmailMessage:
+    """The parsed tree of a raw message, from memory or straight from its file."""
+    if path is not None:
+        with open(path, "rb") as handle:
+            msg = email.message_from_binary_file(handle, policy=default_policy)
+    else:
+        msg = email.message_from_bytes(raw or b"", policy=default_policy)
+    assert isinstance(msg, EmailMessage)
+    return msg
+
+
+def part_payload(msg: EmailMessage, part_index: int | None, sha256: str) -> bytes | None:
+    """The decoded bytes of the attachment whose content hashes to ``sha256``:
+    the part at ``part_index`` when that is it, else whichever part is (rows
+    stored before the index was kept). None when the message has no such part."""
+    import hashlib
+
+    parts = list(msg.walk())
+    if part_index is not None and 0 <= part_index < len(parts):
+        payload = parts[part_index].get_payload(decode=True)
+        if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
+            return payload
+    for part in parts:
+        if part.is_multipart():
+            continue
+        payload = part.get_payload(decode=True)
+        if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
+            return payload
+    return None

backend/app/indexer/rawstore.py

@@ -0,0 +1,170 @@
+"""Where a message's raw source is (SPEC App. D §D.6 item 2).
+
+An external account's messages are kept whole in the blob store, as always.
+A hosted mailbox's are not: the mail server beside the app holds them, so the
+webmail keeps only what it reads them for (headers, the sanitized body, the
+snippet, the search index) and a **cache** of raw messages — the newest ones
+on arrival, the ones opened since — bounded per user (``RAW_CACHE_MB``,
+least recently used out first). Anything else is fetched from the mail server
+when needed: downloading an attachment, "show original", forwarding.
+
+Layout: ``${BLOB_ROOT}/{user_id}/rawcache/{sha256}``, the same content hash
+``messages.blob_sha256`` names.
+
+* :func:`locate` — the file, if it is here (blob store or cache); cheap, sync;
+* :func:`ensure` — the file, fetched from the mail server if need be (async):
+  call it before handing the work to a thread that reads the file.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import time
+import uuid
+from pathlib import Path
+
+from app.config import get_settings
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+CACHE_DIR = "rawcache"
+KEEP_S = 30 * 24 * 3600  # unused this long: dropped whatever the room
+PRUNE_EVERY = 50
+_writes: dict[uuid.UUID, int] = {}
+
+
+def cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / CACHE_DIR
+
+
+def cache_path(user_id: uuid.UUID, sha: str) -> Path:
+    return cache_dir(user_id) / sha
+
+
+def locate(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The message's raw file, if it is kept or cached here."""
+    kept = blobstore.blob_path(user_id, sha)
+    if kept.exists():
+        return kept
+    cached = cache_path(user_id, sha)
+    if cached.exists():
+        with contextlib.suppress(OSError):
+            cached.touch()  # used again: kept longer
+        return cached
+    return None
+
+
+def load(user_id: uuid.UUID, sha: str) -> bytes:
+    """The raw bytes if they are here. Raises FileNotFoundError."""
+    path = locate(user_id, sha)
+    if path is None:
+        raise FileNotFoundError(sha)
+    return path.read_bytes()
+
+
+def put_cached(user_id: uuid.UUID, data: bytes, sha: str | None = None) -> str:
+    """Cache a raw message (``sha`` names it; its content hash by default)."""
+    sha = sha or blobstore.sha256_hex(data)
+    if blobstore.exists(user_id, sha):
+        return sha  # kept for good already (another account has the same message)
+    path = cache_path(user_id, sha)
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+        tmp.write_bytes(data)
+        tmp.replace(path)
+    else:
+        path.touch()
+    # Pruning lists the whole cache: done every PRUNE_EVERY writes, not each
+    # (a backfill writes thousands); meanwhile it overshoots by that much at most.
+    writes = _writes.get(user_id, 0) + 1
+    _writes[user_id] = writes % PRUNE_EVERY
+    if writes >= PRUNE_EVERY:
+        with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+            prune(user_id)
+    return sha
+
+
+def prune(user_id: uuid.UUID, *, max_bytes: int | None = None) -> int:
+    """Drop cached messages unused for KEEP_S, and the least recently used
+    beyond the cap. Returns bytes freed."""
+    cache = cache_dir(user_id)
+    if not cache.is_dir():
+        return 0
+    cap = max_bytes if max_bytes is not None else get_settings().raw_cache_mb * 1024 * 1024
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file() and not p.name.endswith(".tmp")]
+    now = time.time()
+    total = freed = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > KEEP_S or total > cap:
+            path.unlink(missing_ok=True)
+            freed += st.st_size
+    return freed
+
+
+def drop(user_id: uuid.UUID, sha: str) -> None:
+    cache_path(user_id, sha).unlink(missing_ok=True)
+
+
+async def ensure(message: object) -> Path:
+    """The raw file of ``message`` (a :class:`~app.models.Message`), fetched
+    from the hosted mail server into the cache when it is not here. Raises
+    FileNotFoundError when it cannot be had (an external account's message
+    missing from the blob store; a native one gone from the server)."""
+    from app.models import Message
+
+    assert isinstance(message, Message)
+    found = locate(message.user_id, message.blob_sha256)
+    if found is not None:
+        return found
+    raw = await _fetch_native(message)
+    if raw is None:
+        raise FileNotFoundError(message.blob_sha256)
+    got = blobstore.sha256_hex(raw)
+    if got != message.blob_sha256:
+        # The server's copy differs from what was indexed (it should not):
+        # cached under the indexed name, so the message can still be read.
+        logger.warning("raw of message %s changed on the mail server", message.id)
+    put_cached(message.user_id, raw, message.blob_sha256)
+    return cache_path(message.user_id, message.blob_sha256)
+
+
+async def ensure_bytes(message: object) -> bytes:
+    path = await ensure(message)
+    import asyncio
+
+    return await asyncio.to_thread(path.read_bytes)
+
+
+async def _fetch_native(message: object) -> bytes | None:
+    from sqlalchemy import select
+
+    from app.db import session_scope
+    from app.models import Account, Mailbox, Message
+
+    assert isinstance(message, Message)
+    async with session_scope() as db:
+        row = (
+            await db.execute(
+                select(Account.kind, Account.email, Mailbox.path)
+                .join(Mailbox, Mailbox.account_id == Account.id)
+                .where(Account.id == message.account_id, Mailbox.id == message.mailbox_id)
+            )
+        ).first()
+    if row is None or row.kind != "native":
+        return None
+    from app.hosted.native import native_provider_config
+    from app.providers.base import ProviderError
+    from app.providers.imap_smtp import ImapSmtpProvider
+
+    provider = ImapSmtpProvider(native_provider_config(row.email))
+    try:
+        return await provider.fetch_raw(row.path, message.uid)
+    except ProviderError as exc:
+        logger.warning("cannot fetch message %s from the mail server: %s", message.id, exc)
+        return None
+    finally:
+        await provider.close()

backend/app/indexer/sanitize.py

@@ -17,7 +17,16 @@ import nh3
 
 # Bump when the sanitization policy changes; body_parts.sanitizer_v records it so a
 # render re-sanitizes stale content from the raw blob (SPEC §10.4).
-SANITIZER_VERSION = 2
+# v5: this server's own inline-image URLs quoted in a message load from here.
+# v6: links keep only http(s) and mailto (no data: pages).
+# v7: tracking pixels are removed (and counted), even when images are allowed.
+# v8: CSS sizes in other units (in, em, %, pt) no longer read as pixels — v7
+#     took 0.31in signature logos for trackers.
+SANITIZER_VERSION = 8
+
+# This server's inline-image URL, however it ended up written: "/api/…",
+# "https://host/api/…", or mangled to "http:///api/…" by an earlier copy.
+INTERNAL_CID = re.compile(r"/api/messages/([0-9a-fA-F-]{36})/cid/([^\"'?#\s<>]+)")
 
 _ALLOWED_TAGS: set[str] = {
     # structure
@@ -76,6 +85,7 @@ _ALLOWED_TAGS: set[str] = {
     "span",
     "strike",
     "strong",
+    "style",
     "sub",
     "summary",
     "sup",
@@ -97,6 +107,7 @@ _ALLOWED_TAGS: set[str] = {
 # Attributes allowed on any tag (presentational email formatting).
 _GENERIC_ATTRS: set[str] = {
     "style",
+    "class",
     "align",
     "valign",
     "dir",
@@ -119,6 +130,7 @@ _TAG_SPECIFIC: dict[str, set[str]] = {
     "th": {"colspan", "rowspan", "scope"},
     "table": {"cellpadding", "cellspacing", "border"},
     "font": {"face", "color", "size"},
+    "style": {"type", "media"},
 }
 
 # nh3 has no "generic attributes" option, so expand the generic set onto every tag.
@@ -132,6 +144,12 @@ _STYLE_DANGEROUS = re.compile(
     r"(expression\s*\(|javascript:|vbscript:|@import|-moz-binding|behavior\s*:)", re.IGNORECASE
 )
 _STYLE_URL = re.compile(r"url\s*\([^)]*\)", re.IGNORECASE)
+# Script-like CSS that voids a whole <style> block (unlike @import, which we can strip).
+_STYLE_BLOCK_DANGEROUS = re.compile(
+    r"(expression\s*\(|javascript:|vbscript:|-moz-binding|behavior\s*:)", re.IGNORECASE
+)
+_STYLE_IMPORT = re.compile(r"@import[^;]*;?", re.IGNORECASE)
+_STYLE_ELEMENT = re.compile(r"(<style\b[^>]*>)(.*?)(</style>)", re.IGNORECASE | re.DOTALL)
 
 
 @dataclass(frozen=True)
@@ -139,6 +157,94 @@ class SanitizeResult:
     html: str
     has_remote_images: bool
     sanitizer_v: int
+    trackers_blocked: int = 0
+
+
+# --- Tracking pixels ---------------------------------------------------------------
+# A remote image that is invisible (1×1, or hidden by CSS) has no use but to tell
+# the sender the mail was opened, and by whom. A few well-known open-tracking URL
+# shapes are caught whatever their size. Matching runs on the raw HTML before nh3:
+# a miss only leaves an image in, and nh3 still cleans whatever remains.
+_IMG_TAG = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
+_ATTR = re.compile(r"""([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("[^"]*"|'[^']*'|[^\s"'>]+)""")
+_CSS_HIDDEN = re.compile(
+    r"display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?![.\d]*[1-9])",
+    re.IGNORECASE,
+)
+_TRACKER_URL = re.compile(
+    r"//[^/]*mailtrack\.io/"
+    r"|/track/open\b"
+    r"|/wf/open\b"
+    r"|/open\.(?:php|aspx|gif)\b"
+    r"|[?&]tracking[_-]?pixel=",
+    re.IGNORECASE,
+)
+
+
+_PX = re.compile(r"\s*(\d+(?:\.\d*)?|\.\d+)\s*(px)?\s*$", re.IGNORECASE)
+
+
+def _px(value: str | None) -> float | None:
+    """A size in pixels ("1", "1px"), or None for anything else: other units
+    (0.31in is a 30-pixel signature logo), auto, %, or malformed (".", "1.2.3")."""
+    if value is None:
+        return None
+    match = _PX.match(value)
+    return float(match.group(1)) if match else None
+
+
+def _tiny(value: str | None) -> bool:
+    px = _px(value)
+    return px is not None and px <= 2
+
+
+def _css(style: str) -> dict[str, str]:
+    """An inline style's declarations, lower-cased, without !important."""
+    out: dict[str, str] = {}
+    for decl in style.split(";"):
+        name, _, value = decl.partition(":")
+        value = value.lower().replace("!important", "").strip()
+        if name.strip() and value:
+            out[name.strip().lower()] = value
+    return out
+
+
+def is_tracking_pixel(tag: str) -> bool:
+    """Whether an ``<img …>`` tag is a remote open-tracking pixel."""
+    attrs = {name.lower(): value.strip("\"'") for name, value in _ATTR.findall(tag)}
+    src = attrs.get("src", "").strip()
+    if not src.lower().startswith(("http:", "https:", "//")):
+        return False  # inline (cid:/data:) images can't report anything
+    if _TRACKER_URL.search(src):
+        return True
+    style = attrs.get("style", "")
+    if _CSS_HIDDEN.search(style):
+        return True
+    # The size the browser draws: a style width/height overrides the attribute
+    # (and "auto", % or another unit is no tiny size); a max-width/max-height
+    # of a pixel or two caps it; height "auto" follows the width.
+    css = _css(style)
+    wide = _tiny(css["width"]) if "width" in css else _tiny(attrs.get("width"))
+    tall = _tiny(css["height"]) if "height" in css else _tiny(attrs.get("height"))
+    wide = wide or _tiny(css.get("max-width"))
+    tall = tall or _tiny(css.get("max-height"))
+    if css.get("height") == "auto":
+        tall = wide
+    return wide and tall
+
+
+def strip_tracking_pixels(html: str) -> tuple[str, int]:
+    """Remove tracking pixels from raw HTML; returns the HTML and how many."""
+    count = 0
+
+    def repl(match: re.Match[str]) -> str:
+        nonlocal count
+        if is_tracking_pixel(match.group(0)):
+            count += 1
+            return ""
+        return match.group(0)
+
+    return _IMG_TAG.sub(repl, html), count
 
 
 def _sanitize_style(value: str, allow_remote: bool) -> str | None:
@@ -150,27 +256,70 @@ def _sanitize_style(value: str, allow_remote: bool) -> str | None:
     return value
 
 
+def _sanitize_style_elements(html: str, allow_remote: bool, state: dict[str, bool]) -> str:
+    """Sanitize CSS inside ``<style>`` blocks (nh3 keeps the tag but not its text).
+
+    Many mailers (e.g. Fastmail's calendar invites) put all layout in a ``<style>``
+    block of class rules — dropping it loses tables/buttons entirely. We keep the
+    block but neutralize script-like CSS and, unless remote images are approved,
+    strip ``@import``/``url()`` so it can't fetch remote/tracking resources.
+    """
+
+    def repl(match: re.Match[str]) -> str:
+        css = match.group(2)
+        if _STYLE_BLOCK_DANGEROUS.search(css):
+            return ""  # nuke the block entirely
+        css = _STYLE_IMPORT.sub("", css)
+        if not allow_remote and _STYLE_URL.search(css):
+            state["has_remote"] = True
+            css = _STYLE_URL.sub("none", css)
+        return f"{match.group(1)}{css}{match.group(3)}"
+
+    return _STYLE_ELEMENT.sub(repl, html)
+
+
 def sanitize_html(
-    html: str, message_id: uuid.UUID, *, allow_remote_images: bool = False
+    html: str,
+    message_id: uuid.UUID,
+    *,
+    allow_remote_images: bool = False,
+    block_trackers: bool = True,
 ) -> SanitizeResult:
     """Sanitize ``html`` for the message ``message_id``.
 
     Returns the safe HTML, whether any remote images were blocked, and the
     sanitizer version. Pass ``allow_remote_images=True`` after the user approves
     a sender to render remote images and CSS backgrounds (SPEC §9.4).
+    ``block_trackers=False`` (Settings → Privacy) leaves tracking pixels in; they
+    are remote images like any other then.
     """
     state = {"has_remote": False}
+    trackers = 0
+    if block_trackers:
+        html, trackers = strip_tracking_pixels(html)
 
     def attribute_filter(tag: str, attr: str, value: str) -> str | None:
         if attr == "style":
             return _sanitize_style(value, allow_remote_images)
         if tag == "a" and attr == "target":
             return "_blank"
+        if attr == "href":
+            # A link goes to the web or to a mail address: data: (a page made
+            # up by the sender) and cid: are kept for inline images only.
+            scheme = value.strip().lower().split(":", 1)[0] if ":" in value else ""
+            return value if scheme in ("", "http", "https", "mailto") else None
         if tag == "img" and attr == "src":
             lowered = value.strip().lower()
             if lowered.startswith("cid:"):
                 content_id = value.strip()[4:]
                 return f"/api/messages/{message_id}/cid/{content_id}"
+            # This server's own inline-image URL, quoted from another message
+            # (by an earlier version, or another of the user's own mails) —
+            # possibly mangled to "http:///api/…", which a browser takes for a
+            # host named "api": load it from here, never look that host up.
+            internal = INTERNAL_CID.search(value)
+            if internal is not None and not lowered.startswith("data:"):
+                return f"/api/messages/{internal.group(1)}/cid/{internal.group(2)}"
             if lowered.startswith(("http:", "https:")):
                 state["has_remote"] = True
                 return value if allow_remote_images else None  # block by default (FR-23)
@@ -186,10 +335,17 @@ def sanitize_html(
         url_schemes=_URL_SCHEMES,
         link_rel="noopener noreferrer nofollow",
         strip_comments=True,
+        # nh3 strips <script>/<style> *content* by default; we allow <style> (above)
+        # and sanitize its CSS ourselves, so its content stays. <title> is dropped
+        # with its text: as a disallowed tag nh3 would unwrap it, and the document
+        # title ("mailer", the subject) would render as a line above the body.
+        clean_content_tags={"script", "title"},
         attribute_filter=attribute_filter,
     )
+    cleaned = _sanitize_style_elements(cleaned, allow_remote_images, state)
     return SanitizeResult(
         html=cleaned,
         has_remote_images=state["has_remote"],
         sanitizer_v=SANITIZER_VERSION,
+        trackers_blocked=trackers,
     )

backend/app/indexer/thumbs.py

@@ -0,0 +1,132 @@
+"""Attachment thumbnails for the reading pane: images and a PDF's first page.
+
+A thumbnail fits within THUMB_MAX × THUMB_MAX with its aspect ratio kept (the
+browser decides how to frame it), as WebP. Made once and kept beside the blob
+store — ``{BLOB_ROOT}/{user}/thumbs/{sha}-{THUMB_MAX}.webp`` — with a ``.none``
+marker for files that cannot have one (a broken image, an encrypted PDF, HEIC),
+so neither is attempted twice.
+
+Attachments are hostile input: decoding runs in a worker thread, oversized
+files and decompression bombs are refused.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+THUMB_MAX = 480  # px: sharp on a 2× screen for a tile of up to 230 px
+_MAX_INPUT_BYTES = 40 * 1024 * 1024
+_MAX_PIXELS = 64_000_000  # a larger image is refused, not decoded
+_IMAGE_MIME = frozenset(
+    {"image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff"}
+)
+
+
+_EXT_MIME = {
+    "jpg": "image/jpeg",
+    "jpeg": "image/jpeg",
+    "png": "image/png",
+    "gif": "image/gif",
+    "webp": "image/webp",
+    "bmp": "image/bmp",
+    "tif": "image/tiff",
+    "tiff": "image/tiff",
+    "pdf": "application/pdf",
+}
+
+
+def thumb_type(mime_type: str, filename: str) -> str | None:
+    """The type to render a thumbnail as, or None. Senders often label files
+    application/octet-stream, so the extension stands in for a missing type."""
+    mime = mime_type.lower()
+    if mime in _IMAGE_MIME or mime == "application/pdf":
+        return mime
+    return _EXT_MIME.get(filename.rsplit(".", 1)[-1].lower()) if "." in filename else None
+
+
+def can_thumbnail(mime_type: str) -> bool:
+    return mime_type in _IMAGE_MIME or mime_type == "application/pdf"
+
+
+def _path(user_id: uuid.UUID, sha: str, suffix: str) -> Path:
+    return blobstore.blob_path(user_id, sha).parent.parent / "thumbs" / f"{sha}-{THUMB_MAX}{suffix}"
+
+
+def _render(data: bytes, mime_type: str) -> bytes | None:
+    from PIL import Image, ImageOps
+
+    Image.MAX_IMAGE_PIXELS = _MAX_PIXELS  # beyond: DecompressionBombError
+    if mime_type == "application/pdf":
+        import pypdfium2 as pdfium  # type: ignore[import-untyped]
+
+        doc = pdfium.PdfDocument(data)
+        try:
+            page = doc[0]
+            width, height = page.get_size()
+            scale = THUMB_MAX / max(width, height, 1)
+            img = page.render(scale=scale, draw_annots=True).to_pil()
+        finally:
+            doc.close()
+    else:
+        img = Image.open(io.BytesIO(data))
+        img.seek(0)  # an animation: its first frame
+        img = ImageOps.exif_transpose(img)  # phone photos: upright
+        img.thumbnail((THUMB_MAX, THUMB_MAX))
+    if img.mode not in ("RGB", "RGBA"):
+        img = img.convert("RGBA" if "A" in img.getbands() or img.mode == "P" else "RGB")
+    out = io.BytesIO()
+    img.save(out, "WEBP", quality=80, method=4)
+    return out.getvalue()
+
+
+def made(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether the attachment's thumbnail (or its absence) is settled already."""
+    return _path(user_id, sha, ".webp").exists() or _path(user_id, sha, ".none").exists()
+
+
+def thumbnail(
+    user_id: uuid.UUID,
+    sha: str,
+    mime_type: str,
+    size: int,
+    load: Callable[[], bytes],
+) -> bytes | None:
+    """The thumbnail of an attachment (content hash ``sha``, ``size`` bytes,
+    read with ``load`` only when it is not made yet), or None if it cannot
+    have one.
+
+    Blocking (decoding is CPU work): call it in a thread.
+    """
+    done = _path(user_id, sha, ".webp")
+    if done.exists():
+        return done.read_bytes()
+    none = _path(user_id, sha, ".none")
+    if none.exists() or not can_thumbnail(mime_type):
+        return None
+    if size > _MAX_INPUT_BYTES:
+        return None
+    try:
+        data = load()
+    except FileNotFoundError:
+        return None
+    try:
+        thumb = _render(data, mime_type)
+    except Exception as exc:  # noqa: BLE001 - any undecodable file: no thumbnail
+        logger.info("no thumbnail for blob %s (%s): %s", sha, mime_type, exc)
+        thumb = None
+    done.parent.mkdir(parents=True, exist_ok=True)
+    if thumb is None:
+        none.touch()
+        return None
+    tmp = done.with_suffix(".tmp")
+    tmp.write_bytes(thumb)
+    tmp.replace(done)
+    return thumb

backend/app/main.py

@@ -18,6 +18,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """Warm shared resources on startup, dispose them on shutdown."""
     # Touch settings early so misconfiguration fails fast.
     get_settings()
+    # Subscribe plugin hooks so events published in the API process reach them
+    # (worker-published events are handled by the worker's own registration).
+    from app.events import get_event_bus
+    from app.plugins import loader
+
+    loader.register_hooks(get_event_bus())
     yield
     await close_redis()
     await dispose_engine()
@@ -31,6 +37,10 @@ def create_app() -> FastAPI:
 
     register_error_handlers(app)
 
+    from app.security.cross_site import CrossSiteGuard
+
+    app.add_middleware(CrossSiteGuard)
+
     @app.get("/api/healthz")
     async def healthz() -> dict[str, str]:
         """Liveness/readiness probe: checks DB and Redis (dev-setup §6)."""
@@ -39,6 +49,11 @@ def create_app() -> FastAPI:
         await get_redis().ping()
         return {"status": "ok"}
 
+    @app.get("/api/version")
+    async def version() -> dict[str, str]:
+        """Report the running release, for deploy/rollback verification."""
+        return {"version": get_settings().app_version}
+
     from app.api import register_routers
 
     register_routers(app)

backend/app/models.py

@@ -8,9 +8,11 @@ from __future__ import annotations
 
 import datetime as dt
 import uuid
+from typing import Any
 
 from sqlalchemy import (
     BigInteger,
+    CheckConstraint,
     ForeignKey,
     Index,
     Integer,
@@ -44,6 +46,57 @@ def _now() -> Mapped[dt.datetime]:
     return mapped_column(TIMESTAMP(timezone=True), nullable=False, server_default=text("now()"))
 
 
+class AppSettings(Base):
+    """Instance-wide settings, exactly one row (id = 1).
+
+    Distinct from :func:`app.config.get_settings`, which is deployment environment
+    read at boot; these are options an administrator changes at runtime from the
+    admin panel. Locking this row serialises the registration decision below.
+    """
+
+    __tablename__ = "app_settings"
+    __table_args__ = (
+        CheckConstraint("id = 1", name="ck_app_settings_singleton"),
+        CheckConstraint("default_quota_bytes > 0", name="ck_app_settings_quota_positive"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, server_default=text("1"))
+    # Whether POST /auth/register accepts new users. Closed automatically once the
+    # first account (the administrator) is created; an admin reopens it later.
+    registration_open: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    # Quota handed to the NEXT user created. Existing users keep whatever
+    # users.quota_bytes already says — changing this never resizes a mailbox.
+    default_quota_bytes: Mapped[int] = mapped_column(
+        BigInteger, nullable=False, server_default=text("5368709120")
+    )
+    # Outgoing system mail (password-reset links): the SMTP account the server
+    # sends from. The password is vault-encrypted (like Account.secret_ct).
+    smtp_host: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    smtp_port: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("587"))
+    smtp_security: Mapped[str] = mapped_column(
+        Text, nullable=False, server_default=text("'starttls'")
+    )  # starttls | ssl | none
+    smtp_username: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    smtp_secret_ct: Mapped[bytes | None] = mapped_column(BYTEA)
+    smtp_from: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    # Where users reach the app, e.g. https://mail.example.com — reset links are
+    # built from this, never from the request's Host header.
+    public_url: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    # The key pair push notifications are signed with (VAPID, RFC 8292): made on
+    # first use; the private half vault-encrypted, the public half as browsers
+    # take it (base64url, uncompressed P-256 point).
+    vapid_public: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    vapid_private_ct: Mapped[bytes | None] = mapped_column(BYTEA)
+    # Users may invite friends (app/identity/invitations.py); the invitation
+    # email's From address (empty: postmaster@APP_DOMAIN), subject and HTML
+    # template (empty: the built-in ones).
+    invitations_enabled: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    invite_from: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    invite_subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    invite_template: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    updated_at: Mapped[dt.datetime] = _now()
+
+
 class User(Base):
     __tablename__ = "users"
 
@@ -60,6 +113,12 @@ class User(Base):
     quota_bytes: Mapped[int] = mapped_column(
         BigInteger, nullable=False, server_default=text("5368709120")
     )
+    # The first account created on a fresh install bootstraps as administrator;
+    # the admin panel can grant it to others later.
+    is_admin: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    # Set while an administrator has blocked the account (no sign-in, sessions
+    # refused); NULL when not blocked.
+    blocked_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
     created_at: Mapped[dt.datetime] = _now()
 
 
@@ -79,10 +138,24 @@ class Account(Base):
     imap_config: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     smtp_config: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     secret_ct: Mapped[bytes] = mapped_column(BYTEA, nullable=False)
+    # external: any IMAP/SMTP server (the user's password in secret_ct);
+    # native: the user's own mailbox on the hosted mail server — permanent, no
+    # stored password (SPEC App. D §D.4; app/hosted/).
+    kind: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'external'"))
     discovery_src: Mapped[str] = mapped_column(Text, nullable=False)
     state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'backfilling'"))
     state_detail: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    # How much history to import (SPEC §12 M2): all | 3months | new. Drives the
+    # initial backfill; backfill_total/backfill_done feed the progress UX.
+    sync_window: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'all'"))
+    # Whether the Spam/Junk folder is synced (opt-in; SPEC §6.3 folder selection).
+    sync_spam: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    # The server has not answered for a while ("unreachable"); cleared on success.
+    sync_error: Mapped[str | None] = mapped_column(Text)
+    sync_error_since: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
     created_at: Mapped[dt.datetime] = _now()
 
 
@@ -99,9 +172,10 @@ class Mailbox(Base):
     )
     path: Mapped[str] = mapped_column(Text, nullable=False)
     special_use: Mapped[str | None] = mapped_column(Text)
-    uidvalidity: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-    highestmodseq: Mapped[int | None] = mapped_column(Integer)
-    last_seen_uid: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # IMAP ids are unsigned 32-bit (MODSEQ 63-bit): bigint, not integer.
+    uidvalidity: Mapped[int] = mapped_column(BigInteger, nullable=False, server_default=text("0"))
+    highestmodseq: Mapped[int | None] = mapped_column(BigInteger)
+    last_seen_uid: Mapped[int] = mapped_column(BigInteger, nullable=False, server_default=text("0"))
 
 
 class Thread(Base):
@@ -125,6 +199,12 @@ class Thread(Base):
     has_attachments: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     last_message_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    # Snooze: out of the Inbox until this time (NULL when not snoozed). A message
+    # that arrives after ``snoozed_at`` wakes it early; ``unsnoozed_at`` (when it
+    # last woke) puts it back at the top of the Inbox.
+    snoozed_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    snoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    unsnoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
 
 class Message(Base):
@@ -147,11 +227,20 @@ class Message(Base):
     mailbox_id: Mapped[uuid.UUID] = mapped_column(
         UUID(as_uuid=True), ForeignKey("mailboxes.id", ondelete="CASCADE"), nullable=False
     )
-    uid: Mapped[int] = mapped_column(Integer, nullable=False)
+    # The folder this message was in before being moved to Archive/Trash/Junk, so
+    # Restore can return it there (e.g. Sent) instead of the Inbox. NULL otherwise.
+    restore_mailbox_id: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("mailboxes.id", ondelete="SET NULL")
+    )
+    uid: Mapped[int] = mapped_column(BigInteger, nullable=False)
     thread_id: Mapped[uuid.UUID | None] = mapped_column(
         UUID(as_uuid=True), ForeignKey("threads.id")
     )
     message_id_hdr: Mapped[str | None] = mapped_column(Text)
+    # The real header Message-ID as delivered (kept even when message_id_hdr is
+    # synthesized for a cross-account duplicate, §7.1). Identifies one delivery, so
+    # the same email fanned out to several accounts collapses to one row in a thread.
+    delivery_id: Mapped[str | None] = mapped_column(Text)
     in_reply_to: Mapped[str | None] = mapped_column(Text)
     references_hdr: Mapped[list[str]] = mapped_column(
         ARRAY(Text), nullable=False, server_default=text("'{}'")
@@ -180,6 +269,17 @@ class Message(Base):
     snippet: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
     body_indexed: Mapped[str | None] = mapped_column(TSVECTOR)
+    # Mailing-list headers, for the Subscriptions page. ``list_unsubscribe`` is
+    # NULL until read (a backfill job fills older mail), '' when there is none.
+    list_id: Mapped[str | None] = mapped_column(Text)
+    list_unsubscribe: Mapped[str | None] = mapped_column(Text)
+    # RFC 8058: the sender takes a one-click unsubscribe POST.
+    list_unsubscribe_post: Mapped[bool] = mapped_column(
+        nullable=False, server_default=text("false")
+    )
+    # The language of its text (app/translation.py), found the first time it is
+    # shown: NULL not looked at yet, '' could not tell.
+    content_language: Mapped[str | None] = mapped_column(Text)
 
 
 class BodyPart(Base):
@@ -193,6 +293,8 @@ class BodyPart(Base):
     content: Mapped[str] = mapped_column(Text, nullable=False)
     # SPEC §10.4: re-sanitize on render if the sanitizer version changed.
     sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # Tracking pixels taken out of this HTML (shown as "N trackers blocked").
+    trackers_blocked: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
 
 
 class Attachment(Base):
@@ -207,7 +309,11 @@ class Attachment(Base):
     mime_type: Mapped[str] = mapped_column(Text, nullable=False)
     size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     content_id: Mapped[str | None] = mapped_column(Text)
+    # The content's hash (names its thumbnail, checks what is taken out). The
+    # bytes themselves live only inside the message's raw blob, at MIME part
+    # ``part_index`` (NULL for rows stored before that was recorded).
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
+    part_index: Mapped[int | None] = mapped_column(Integer)
 
 
 class ComposeAttachment(Base):
@@ -246,7 +352,7 @@ class Draft(Base):
     payload: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     # UID of the mirrored copy in the account's IMAP Drafts folder (so the draft
     # is visible/continuable on other mail clients). Replaced on each edit.
-    imap_uid: Mapped[int | None] = mapped_column(Integer)
+    imap_uid: Mapped[int | None] = mapped_column(BigInteger)
     updated_at: Mapped[dt.datetime] = _now()
 
 
@@ -259,6 +365,10 @@ class Outbox(Base):
     state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'undo_window'"))
     send_after: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
     error: Mapped[str | None] = mapped_column(Text)
+    # Retries after a send the server could not take (0021): state "retrying".
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    next_attempt_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    retry_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
 
 class SavedSearch(Base):
@@ -274,6 +384,155 @@ class SavedSearch(Base):
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("true"))
 
 
+class UserAppPassword(Base):
+    """A password for one mail app (IMAP/SMTP), revocable on its own (§D.5)."""
+
+    __tablename__ = "user_app_passwords"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)  # "Thunderbird on laptop"
+    password_hash: Mapped[str] = mapped_column(Text, nullable=False)  # argon2id
+    created_at: Mapped[dt.datetime] = _now()
+    # Set by the mail server's sign-in, once there is one.
+    last_used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    # The hosted mail server's id of the same password (§D.5), when it has one.
+    server_id: Mapped[str | None] = mapped_column(Text)
+
+
+class Invitation(Base):
+    """A single-use link that lets one person sign up, sent by a user."""
+
+    __tablename__ = "invitations"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    inviter_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    # sha256 of the token in the link; the token itself is only in the email.
+    token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    invitee_name: Mapped[str] = mapped_column(Text, nullable=False)
+    invitee_email: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+    expires_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    used_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL")
+    )
+
+
+class SubscriptionState(Base):
+    """What a user did about a bulk sender on the Subscriptions page."""
+
+    __tablename__ = "subscription_states"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    # The sender group: "list:<List-Id>" or "from:<address>".
+    key: Mapped[str] = mapped_column(Text, primary_key=True)
+    unsubscribed_at: Mapped[dt.datetime] = _now()
+    method: Mapped[str] = mapped_column(Text, nullable=False)  # one_click | mailto | web
+
+
+class MailTemplate(Base):
+    """A saved reply, inserted from the compose window."""
+
+    __tablename__ = "mail_templates"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)
+    subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    body_html: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    created_at: Mapped[dt.datetime] = _now()
+    updated_at: Mapped[dt.datetime] = _now()
+
+
+class MessageTranslation(Base):
+    """A message's HTML translated into a language (app/translation.py), kept so
+    it is translated once. ``images``: with remote images shown or not."""
+
+    __tablename__ = "message_translations"
+
+    message_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
+    )
+    target: Mapped[str] = mapped_column(Text, primary_key=True)
+    images: Mapped[bool] = mapped_column(primary_key=True)
+    source: Mapped[str] = mapped_column(Text, nullable=False)
+    html: Mapped[str] = mapped_column(Text, nullable=False)
+    sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+
+
+class CompletionPhrase(Base):
+    """Sentence completion (app/completion.py): after these words, the user
+    usually writes this — seen ``count`` times in their sent mail."""
+
+    __tablename__ = "completion_phrases"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    context: Mapped[str] = mapped_column(Text, primary_key=True)  # normalized words
+    continuation: Mapped[str] = mapped_column(Text, primary_key=True)  # as written
+    count: Mapped[int] = mapped_column(Integer, nullable=False)
+
+
+class CompletionIndexState(Base):
+    """When a user's completion index was last built."""
+
+    __tablename__ = "completion_index_state"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    built_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+
+
+class PushSubscription(Base):
+    """A browser or installed app to tell about new mail (Web Push)."""
+
+    __tablename__ = "push_subscriptions"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    # The push service URL the browser handed out; posting to it reaches the device.
+    endpoint: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    # The browser's keys the message is encrypted to (RFC 8291), base64url.
+    p256dh: Mapped[str] = mapped_column(Text, nullable=False)
+    auth: Mapped[str] = mapped_column(Text, nullable=False)
+    # The sign-in it was made under: nothing is sent once that session is gone.
+    session_sid: Mapped[str] = mapped_column(Text, nullable=False)
+    user_agent: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    created_at: Mapped[dt.datetime] = _now()
+    last_used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+
+
+class PluginKv(Base):
+    """Namespaced key/value store for plugins (SPEC §11 ``plugin_kv``).
+
+    Core-owned infrastructure (not a ``plg_``-prefixed plugin table): every plugin
+    gets a slice keyed by (plugin, user_id, key). ``value`` is arbitrary JSON.
+    Invariant I-7 (user-data rows carry ``user_id``) holds here too."""
+
+    __tablename__ = "plugin_kv"
+
+    plugin: Mapped[str] = mapped_column(Text, primary_key=True)
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    key: Mapped[str] = mapped_column(Text, primary_key=True)
+    value: Mapped[Any] = mapped_column(JSONB, nullable=False)
+
+
 class KnownRecipient(Base):
     __tablename__ = "known_recipients"
     __table_args__ = (

backend/app/outbox/draft_sync.py

@@ -17,6 +17,7 @@ from sqlalchemy import select
 
 from app.db import session_scope
 from app.models import Account, Draft, Mailbox, Message
+from app.outbox import inline_images
 from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
 from app.sync.provider_factory import build_provider
@@ -68,7 +69,15 @@ async def push_draft_to_imap_job(ctx: dict[str, Any], draft_id: str) -> None:
             else None
         )
         attachments = await load_attachments(session, user_id, draft)
-        _, sent_raw, _, _ = build_mime(account, draft, reply_to, attachments)
+        html, inline = await inline_images.embed(
+            session,
+            user_id,
+            str(draft.payload.get("html", "") or ""),
+            account.email.rsplit("@", 1)[-1],
+        )
+        _, sent_raw, _, _ = build_mime(
+            account, draft, reply_to, attachments, html=html, inline=inline
+        )
         drafts_mb = await session.scalar(
             select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )

backend/app/outbox/inline_images.py

@@ -0,0 +1,153 @@
+"""Inline images in an outgoing message's HTML.
+
+A reply quotes the original as the reading pane shows it, so an image embedded
+in the original (``cid:`` — a signature logo, a pasted screenshot) appears in
+the quote as this server's own URL, ``/api/messages/{id}/cid/{content-id}``.
+Sent as is, that URL means nothing to the recipient (and to anyone else,
+including ourselves once the original is gone). Before sending, each such image
+is put back into the message as an inline part with a new Content-ID — as any
+mail program does — and one that can no longer be had is left out.
+
+An image written into the HTML itself (``data:image/…;base64,…`` — a signature
+logo pasted into the editor) goes the same way: Gmail and other mail programs
+do not show ``data:`` images in mail they receive, so each becomes an inline
+part too (one per distinct image).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import binascii
+import hashlib
+import re
+import uuid
+from dataclasses import dataclass
+from email.utils import make_msgid
+from urllib.parse import unquote
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.indexer import attachments as att_store
+from app.indexer import rawstore
+from app.indexer.sanitize import INTERNAL_CID
+from app.models import Attachment, Message
+
+_DATA_IMG = re.compile(r"^data:(image/[a-z0-9.+-]+);base64,(.*)$", re.IGNORECASE | re.DOTALL)
+_EXT = {"jpeg": "jpg", "svg+xml": "svg"}
+
+_IMG = re.compile(r"<img\b[^>]*?\bsrc\s*=\s*([\"'])(.*?)\1[^>]*>", re.IGNORECASE | re.DOTALL)
+
+
+@dataclass(frozen=True)
+class InlinePart:
+    cid: str  # without angle brackets
+    mime_type: str
+    filename: str
+    data: bytes
+
+
+def data_image(src: str, domain: str) -> InlinePart | None:
+    """The inline part for a ``data:image/…;base64`` URL, or None if it is not one."""
+    m = _DATA_IMG.match(src.strip())
+    if m is None:
+        return None
+    try:
+        data = base64.b64decode(re.sub(r"\s+", "", m.group(2)), validate=True)
+    except (binascii.Error, ValueError):
+        return None
+    if not data:
+        return None
+    mime_type = m.group(1).lower()
+    subtype = mime_type.split("/", 1)[1]
+    name = f"image.{_EXT.get(subtype, subtype)}"
+    return InlinePart(make_msgid(domain=domain)[1:-1], mime_type, name, data)
+
+
+def internal_cid(src: str) -> tuple[uuid.UUID, str] | None:
+    """The (message id, content id) an image URL of this server points at."""
+    value = src.strip()
+    if value.lower().startswith(("cid:", "data:")):
+        return None
+    m = INTERNAL_CID.search(value)
+    if m is None:
+        return None
+    try:
+        return uuid.UUID(m.group(1)), unquote(m.group(2))
+    except ValueError:
+        return None
+
+
+async def embed(
+    session: AsyncSession, user_id: uuid.UUID, html: str, domain: str
+) -> tuple[str, list[InlinePart]]:
+    """``html`` with this server's image URLs turned into ``cid:`` references,
+    and the parts to send them in."""
+    parts: dict[tuple[uuid.UUID, str], InlinePart | None] = {}
+    # Images written into the HTML, by content: the same logo twice is one part.
+    data_parts: dict[str, InlinePart | None] = {}
+
+    def data_key(src: str) -> str:
+        return hashlib.sha256(src.strip().encode()).hexdigest()
+
+    for match in _IMG.finditer(html):
+        src = match.group(2)
+        if src.strip().lower().startswith("data:"):
+            key = data_key(src)
+            if key not in data_parts:
+                data_parts[key] = data_image(src, domain)
+            continue
+        target = internal_cid(src)
+        if target is None or target in parts:
+            continue
+        message_id, content_id = target
+        found = (
+            await session.execute(
+                select(Attachment, Message)
+                .join(Message, Message.id == Attachment.message_id)
+                .where(
+                    Attachment.message_id == message_id,
+                    Attachment.content_id == content_id,
+                    Message.user_id == user_id,
+                )
+                .limit(1)
+            )
+        ).first()
+        part: InlinePart | None = None
+        if found is not None:
+            att, message = found
+            try:
+                await rawstore.ensure(message)  # a hosted mailbox's: fetched if need be
+                data = await asyncio.to_thread(
+                    att_store.load, user_id, message.blob_sha256, att.blob_sha256, att.part_index
+                )
+                cid = make_msgid(domain=domain)[1:-1]
+                part = InlinePart(cid, att.mime_type, att.filename, data)
+            except FileNotFoundError:
+                part = None
+        parts[target] = part
+
+    def replace(match: re.Match[str]) -> str:
+        quote = match.group(1)
+        src = match.group(2)
+        if src.strip().lower().startswith("data:"):
+            data_part = data_parts.get(data_key(src))
+            if data_part is None:
+                return match.group(0)  # not an image we can carry: left as it was
+            return match.group(0).replace(
+                f"src={quote}{src}{quote}", f"src={quote}cid:{data_part.cid}{quote}", 1
+            )
+        target = internal_cid(match.group(2))
+        if target is None:
+            return match.group(0)
+        part = parts.get(target)
+        if part is None:
+            return ""  # gone: no broken image for the recipient
+        return match.group(0).replace(
+            f"src={quote}{match.group(2)}{quote}", f"src={quote}cid:{part.cid}{quote}", 1
+        )
+
+    out = _IMG.sub(replace, html) if parts or data_parts else html
+    quoted = [p for p in parts.values() if p is not None]
+    return out, quoted + [p for p in data_parts.values() if p is not None]

backend/app/outbox/jobs.py

@@ -8,24 +8,31 @@ to Sent (skipped for Gmail, which self-appends), harvest recipients into
 
 from __future__ import annotations
 
-import contextlib
 import logging
 import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
+from email.utils import getaddresses
 from typing import Any
 
-from sqlalchemy import select
+from arq import Retry
+from sqlalchemy import select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import session_scope
 from app.events import get_event_bus
+from app.indexer import blobstore, gc
 from app.indexer.indexer import persist_message
 from app.models import Account, Draft, KnownRecipient, Mailbox, Message, Outbox
+from app.outbox import inline_images
 from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
+from app.outbox.service import delete_mirrored_draft_row
 from app.providers.base import RemoteMessage
+from app.queue import enqueue_job
 from app.search.factory import get_search_backend
+from app.sync.actions import _retry_later
+from app.sync.health import is_temporary
 from app.sync.provider_factory import build_provider
 
 logger = logging.getLogger(__name__)
@@ -36,6 +43,17 @@ def _is_gmail(account: Account) -> bool:
     return "gmail.com" in host or "googlemail.com" in host
 
 
+def _addr_dicts(values: object) -> list[dict[str, str]]:
+    """Parse a draft payload's ``["Name <email>", …]`` list into name+email dicts."""
+    if not isinstance(values, list):
+        return []
+    return [
+        {"name": name, "email": email}
+        for name, email in getaddresses([str(v) for v in values])
+        if email
+    ]
+
+
 async def _harvest_recipients(session: AsyncSession, user_id: uuid.UUID, emails: list[str]) -> None:
     now = datetime.now(UTC)
     for email in {e.lower() for e in emails if e}:
@@ -98,14 +116,125 @@ async def _persist_sent_copy(user_id: uuid.UUID, account: Account, raw: bytes, u
         await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
 
 
+# A send the mail server could not take (unreachable, a temporary 4xx) is tried
+# again after these gaps, then hourly, until RETRY_WINDOW after the first
+# failure. Meanwhile the message waits in the Scheduled folder.
+_RETRY_GAPS_S = (60, 120, 300, 600, 1800)
+_RETRY_HOURLY_S = 3600
+RETRY_WINDOW = timedelta(hours=24)
+
+
+def _retry_gap(attempts: int) -> int:
+    return _RETRY_GAPS_S[attempts - 1] if attempts <= len(_RETRY_GAPS_S) else _RETRY_HOURLY_S
+
+
+async def _send_failed(
+    oid: uuid.UUID, user_id: uuid.UUID, subject: str, exc: BaseException
+) -> None:
+    """Record a failed send: "retrying" if it may work later, else "failed"."""
+    now = datetime.now(UTC)
+    async with session_scope() as session:
+        ob = await session.get(Outbox, oid)
+        if ob is None:
+            return
+        ob.attempts += 1
+        ob.error = (getattr(exc, "detail", None) or str(exc) or type(exc).__name__)[:500]
+        ob.retry_until = ob.retry_until or now + RETRY_WINDOW
+        if is_temporary(exc) and now < ob.retry_until:
+            ob.state = "retrying"
+            ob.next_attempt_at = now + timedelta(seconds=_retry_gap(ob.attempts))
+        else:
+            ob.state = "failed"
+            ob.next_attempt_at = None
+        state, attempts, next_at, error = ob.state, ob.attempts, ob.next_attempt_at, ob.error
+    if state == "retrying" and next_at is not None:
+        await enqueue_job(
+            "send_job", str(oid), _defer_until=next_at, _job_id=f"send:{oid}:{attempts}"
+        )
+    await get_event_bus().publish(
+        "outbox.state_changed",
+        {
+            "user_id": str(user_id),
+            "outbox_id": str(oid),
+            "state": state,
+            "attempts": attempts,
+            "subject": subject,
+            "error": error,
+            "next_attempt_at": next_at.isoformat() if next_at else None,
+        },
+    )
+    if state == "failed":
+        logger.warning("send failed for outbox %s: %s", oid, error)
+    else:
+        logger.info(
+            "send of outbox %s failed (try %d), retrying at %s: %s", oid, attempts, next_at, error
+        )
+
+
+async def file_sent_copy_job(ctx: dict[str, Any], account_id: str, user_id: str, sha: str) -> None:
+    """File a sent message's copy in Sent, after the send itself succeeded but
+    the IMAP server could not be reached. Retried (arq Retry) for ~24 hours.
+
+    Until then the copy's file is referenced only by this job: it is held from
+    the nightly clean-up (app/indexer/gc.py) and let go once filed or given up."""
+    try:
+        await _file_sent_copy(ctx, account_id, user_id, sha)
+    except Retry:
+        raise  # tried again later: the file is still needed
+    except BaseException:
+        await gc.release(user_id, sha)
+        raise
+    await gc.release(user_id, sha)
+
+
+async def _file_sent_copy(ctx: dict[str, Any], account_id: str, user_id: str, sha: str) -> None:
+    uid = uuid.UUID(user_id)
+    async with session_scope() as session:
+        account = await session.get(Account, uuid.UUID(account_id))
+        if account is None:
+            return
+        sent_path = await session.scalar(
+            select(Mailbox.path).where(
+                Mailbox.account_id == account.id, Mailbox.special_use == "sent"
+            )
+        )
+    if not sent_path:
+        return
+    raw = blobstore.load(uid, sha)
+    provider = build_provider(account)
+    try:
+        appended_uid = await provider.append(sent_path, raw, frozenset({"\\Seen"}))
+    except Exception as exc:
+        retry = _retry_later(ctx, exc)
+        if retry is not None:
+            raise retry from exc
+        logger.error("could not file a sent message in Sent (blob %s): %s", sha, exc)
+        return
+    finally:
+        await provider.close()
+    await _persist_sent_copy(uid, account, raw, appended_uid)
+    logger.info("filed a sent message in Sent for account %s", account_id)
+
+
 async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
-    """Send a queued message unless it was cancelled during the undo window."""
+    """Send a queued message unless it was cancelled during the undo window.
+
+    Also runs each retry of a send that failed for a temporary reason. The row is
+    claimed with one conditional UPDATE, so two jobs for the same send (a planned
+    retry and "Retry now") can never both send it.
+    """
     oid = uuid.UUID(outbox_id)
     async with session_scope() as session:
-        outbox = await session.get(Outbox, oid)
-        if outbox is None or outbox.state != "undo_window":
+        claimed = await session.scalar(
+            update(Outbox)
+            .where(Outbox.id == oid, Outbox.state.in_(("undo_window", "retrying")))
+            .values(state="sending")
+            .returning(Outbox.id)
+        )
+        if claimed is None:
             return  # cancelled, or already handled -> no SMTP traffic
-        outbox.state = "sending"
+        outbox = await session.get(Outbox, oid)
+        assert outbox is not None
         draft = await session.get(Draft, outbox.draft_id)
         if draft is None:
             outbox.state = "failed"
@@ -123,8 +252,16 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
             return
         user_id = outbox.user_id
         attachments = await load_attachments(session, user_id, draft)
+        # Images quoted from the original travel inside the message, not as
+        # links to this server (app/outbox/inline_images.py).
+        html, inline = await inline_images.embed(
+            session,
+            user_id,
+            str(draft.payload.get("html", "") or ""),
+            account.email.rsplit("@", 1)[-1],
+        )
         transmit_raw, sent_raw, envelope_from, envelope_to = build_mime(
-            account, draft, reply_to, attachments
+            account, draft, reply_to, attachments, html=html, inline=inline
         )
         sent_path = await session.scalar(
             select(Mailbox.path).where(
@@ -140,28 +277,32 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
         gmail = _is_gmail(account)
         account_obj = account
 
+    subject = str(draft.payload.get("subject") or "")
     provider = build_provider(account_obj)
     appended_uid = 0
+    try:
         try:
             await provider.send(transmit_raw, envelope_from, envelope_to)
+        except Exception as exc:  # noqa: BLE001 - retry or record; the draft is kept
+            await _send_failed(oid, user_id, subject, exc)
+            return
+        # Delivered. If filing the copy in Sent fails (the IMAP server is not
+        # there right now), the send still succeeded — and the copy must not be
+        # lost: it is kept and filed later by file_sent_copy_job.
         if sent_path and not gmail:
+            try:
                 appended_uid = await provider.append(sent_path, sent_raw, frozenset({"\\Seen"}))
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("outbox %s sent; filing it in Sent failed, will retry: %s", oid, exc)
+                sha = blobstore.store(user_id, sent_raw)
+                await gc.hold(user_id, sha)  # until filed: only the job refers to it
+                await enqueue_job("file_sent_copy_job", str(account_obj.id), str(user_id), sha)
         # Remove the mirrored Drafts-folder copy now that it has been sent.
         if drafts_path and draft_imap_uid:
-            with contextlib.suppress(Exception):
+            try:
                 await provider.delete_messages(drafts_path, [draft_imap_uid])
-    except Exception as exc:  # noqa: BLE001 - record and notify, keep the draft
-        async with session_scope() as session:
-            ob = await session.get(Outbox, oid)
-            if ob is not None:
-                ob.state = "failed"
-                ob.error = str(exc)[:500]
-        await get_event_bus().publish(
-            "outbox.state_changed",
-            {"user_id": str(user_id), "outbox_id": str(oid), "state": "failed"},
-        )
-        logger.exception("send failed for outbox %s", oid)
-        return
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("outbox %s sent; removing its Drafts copy failed: %s", oid, exc)
     finally:
         await provider.close()
 
@@ -178,12 +319,32 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
         draft_row = await session.get(Draft, draft.id)
         if draft_row is not None:
             await session.delete(draft_row)
+        # Also drop the mirrored Drafts-folder copy's DB row (we expunged it on the
+        # server above) so it doesn't resurface as a "remote" draft.
+        if draft_imap_uid:
+            await delete_mirrored_draft_row(
+                session, account_id=account_obj.id, imap_uid=draft_imap_uid
+            )
         await _harvest_recipients(session, user_id, envelope_to)
 
     bus = get_event_bus()
     await bus.publish(
         "message.sent",
-        {"user_id": str(user_id), "outbox_id": str(oid), "thread_id": str(draft.thread_id)},
+        {
+            "user_id": str(user_id),
+            "outbox_id": str(oid),
+            "thread_id": str(draft.thread_id),
+            # Recipients (name+email) so consumers (e.g. contacts autosave) don't
+            # need to read the outbox row. payload to/cc are "Name <email>" strings.
+            "to": _addr_dicts(draft.payload.get("to")),
+            "cc": _addr_dicts(draft.payload.get("cc")),
+            # What it answers: "reply" / "reply_all" / "forward" / "new", and the
+            # message (a reply says the user wanted that mail — antispam learns it).
+            "kind": draft.kind,
+            "reply_to_message_id": (
+                str(draft.reply_to_message_id) if draft.reply_to_message_id else None
+            ),
+        },
     )
     await bus.publish(
         "outbox.state_changed",

backend/app/outbox/mime_build.py

@@ -14,6 +14,7 @@ from email.utils import format_datetime, make_msgid
 import nh3
 
 from app.models import Account, Draft, Message
+from app.outbox.inline_images import InlinePart
 
 
 def _html_to_text(html: str) -> str:
@@ -49,6 +50,9 @@ def build_mime(
     draft: Draft,
     reply_to: Message | None,
     attachments: list[tuple[str, str, bytes]] | None = None,
+    *,
+    html: str | None = None,
+    inline: list[InlinePart] | None = None,
 ) -> tuple[bytes, bytes, str, list[str]]:
     """Return ``(transmit_raw, sent_raw, envelope_from, envelope_to)`` for ``draft``.
 
@@ -59,13 +63,17 @@ def build_mime(
 
     ``attachments`` is a list of ``(filename, mime_type, data)`` uploaded via the
     compose Attach action; each is added as a file attachment part.
+
+    ``html`` replaces the draft's HTML when given (images already turned into
+    ``cid:`` references by app/outbox/inline_images.py), and ``inline`` are the
+    image parts those references name, sent alongside the HTML.
     """
     payload = draft.payload
     to = _addr_list(payload.get("to"))
     cc = _addr_list(payload.get("cc"))
     bcc = _addr_list(payload.get("bcc"))
     subject = str(payload.get("subject", ""))
-    html = str(payload.get("html", "") or "")
+    html = html if html is not None else str(payload.get("html", "") or "")
     text = str(payload.get("text", "") or "")
 
     msg = EmailMessage()
@@ -91,6 +99,19 @@ def build_mime(
     if html:
         msg.set_content(text or _html_to_text(html))
         msg.add_alternative(html, subtype="html")
+        if inline:
+            html_part = next(p for p in msg.walk() if p.get_content_type() == "text/html")
+            assert isinstance(html_part, EmailMessage)
+            for part in inline:
+                maintype, _, subtype = part.mime_type.partition("/")
+                html_part.add_related(
+                    part.data,
+                    maintype=maintype or "image",
+                    subtype=subtype or "png",
+                    cid=f"<{part.cid}>",
+                    filename=part.filename,
+                    disposition="inline",
+                )
     else:
         msg.set_content(text)
 

backend/app/outbox/schemas.py

@@ -56,8 +56,28 @@ class SendRequest(BaseModel):
 class SendResponse(BaseModel):
     outbox_id: uuid.UUID
     undo_until: str
+    # The same deadline as seconds from now: the browser counts down from this,
+    # since its clock may be seconds off the server's.
+    undo_seconds: float
 
 
 class OutboxStateOut(BaseModel):
     outbox_id: uuid.UUID
     state: str
+
+
+class OutboxItemOut(BaseModel):
+    """A send waiting in the Scheduled folder: being retried, or failed for good."""
+
+    outbox_id: uuid.UUID
+    draft_id: uuid.UUID
+    account_id: uuid.UUID
+    state: str  # "retrying" | "failed"
+    attempts: int
+    error: str | None
+    next_attempt_at: str | None
+    retry_until: str | None
+    sent_at: str  # when Send was pressed (the end of the undo window)
+    to: list[str]
+    subject: str
+    preview: str

backend/app/outbox/service.py

@@ -5,11 +5,11 @@ from __future__ import annotations
 import uuid
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import select
+from sqlalchemy import ColumnElement, exists, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import get_settings
-from app.models import Account, Draft, Outbox
+from app.models import Account, Draft, Mailbox, Message, Outbox
 from app.outbox.schemas import DraftIn
 
 
@@ -17,6 +17,29 @@ class DraftError(Exception):
     """Draft could not be created/updated (e.g. account not owned)."""
 
 
+async def delete_mirrored_draft_row(
+    session: AsyncSession, *, account_id: uuid.UUID, imap_uid: int
+) -> None:
+    """Delete the DB row for a draft that was mirrored into the IMAP Drafts folder.
+
+    The server copy is expunged separately (send_job / remove_imap_draft_job).
+    Without also dropping this row the draft resurfaces in ``list_remote_drafts``
+    as an "other client" draft once its local ``drafts`` row is gone — until the
+    next Drafts poll would catch the expunge. Used on both send and discard.
+    """
+    mirror = await session.scalar(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.account_id == account_id,
+            Message.uid == imap_uid,
+            Mailbox.special_use == "drafts",
+        )
+    )
+    if mirror is not None:
+        await session.delete(mirror)
+
+
 async def upsert_draft(
     session: AsyncSession, *, user_id: uuid.UUID, body: DraftIn, draft_id: uuid.UUID | None
 ) -> Draft:
@@ -55,6 +78,16 @@ async def upsert_draft(
     return draft
 
 
+# A draft with one of these sends is in flight — or waiting in Scheduled for a
+# retry — so it is not listed (or counted) in Drafts.
+OUTBOX_ACTIVE = ("undo_window", "sending", "retrying", "failed")
+
+
+def not_being_sent() -> ColumnElement[bool]:
+    """SQL condition: this Draft row has no send in flight or waiting."""
+    return ~exists().where(Outbox.draft_id == Draft.id, Outbox.state.in_(OUTBOX_ACTIVE))
+
+
 async def create_outbox(session: AsyncSession, *, user_id: uuid.UUID, draft: Draft) -> Outbox:
     """Create an outbox row in the undo window for a draft."""
     send_after = datetime.now(UTC) + timedelta(seconds=get_settings().undo_seconds)

backend/app/plugins/__init__.py

@@ -0,0 +1,6 @@
+"""Plugin platform (SPEC §11).
+
+Public surface for plugin authors is :mod:`app.plugins.sdk`; the loader and
+core wiring live here but plugins must not import them directly (SPEC §11:
+"plugins may not import from ``app.*`` internals other than ``app.plugins.sdk``").
+"""

backend/app/plugins/actions.py

@@ -0,0 +1,335 @@
+"""Message read + flag helpers exposed to plugins via the SDK (SPEC §11).
+
+Plugins like ``filters`` need to inspect a newly-arrived message and flag it.
+Both operations are user-scoped: a plugin can only read/act on the given user's
+own message (the same app-level ``user_id`` scoping core uses — core tables have
+no RLS). Flagging mirrors the core action path: update the DB, enqueue the IMAP
+push, and publish ``flags.changed`` so the UI updates.
+
+Actions exposed: flags, trash (whole thread), and ``move_to_special`` for a
+single message — the last added for the antispam plugin, which must file one
+spam message into Junk without dragging its thread along.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from collections.abc import Iterable
+from email import message_from_bytes
+from typing import Any
+
+from sqlalchemy import select
+
+from app.db import session_scope
+from app.events import get_event_bus
+from app.indexer import rawstore
+from app.models import Account, Mailbox, Message, Thread, User
+
+
+class PluginMessages:
+    """Read-only, user-scoped view of a message's matchable fields."""
+
+    async def header(self, user_id: uuid.UUID, message_id: uuid.UUID, name: str) -> str | None:
+        """Return a raw header (e.g. ``List-Unsubscribe``) from the user's message,
+        or None if the message/header/blob is missing. Parses the stored RFC822."""
+        async with session_scope() as session:
+            msg = await session.scalar(
+                select(Message).where(Message.id == message_id, Message.user_id == user_id)
+            )
+            if msg is None or not msg.blob_sha256:
+                return None
+        try:
+            raw = await rawstore.ensure_bytes(msg)
+        except FileNotFoundError:
+            return None
+        return message_from_bytes(raw).get(name)
+
+    async def raw(self, user_id: uuid.UUID, message_id: uuid.UUID) -> bytes | None:
+        """Return the stored RFC822 bytes of the user's message, or None.
+
+        For engines that need the whole message rather than one header — spam
+        scoring wants headers *and* body. None when the message isn't the user's,
+        has no blob, or the blob is unreadable; callers must treat that as "cannot
+        classify", never as an error.
+        """
+        async with session_scope() as session:
+            msg = await session.scalar(
+                select(Message).where(Message.id == message_id, Message.user_id == user_id)
+            )
+            if msg is None or not msg.blob_sha256:
+                return None
+        try:
+            return await rawstore.ensure_bytes(msg)
+        except FileNotFoundError:
+            return None
+
+    async def sample(
+        self,
+        user_id: uuid.UUID,
+        *,
+        account_id: uuid.UUID,
+        special_use: str,
+        limit: int = 500,
+        older_than_days: int | None = None,
+    ) -> list[uuid.UUID]:
+        """Message ids from one account folder, newest first, capped at ``limit``.
+
+        For building a training corpus out of mail the user already has — the
+        provider's own Junk folder is a labelled spam set, and aged Inbox mail
+        that was never moved is a reasonable ham set. ``older_than_days`` skips
+        recent arrivals, which are the ones a user has not triaged yet.
+
+        Ids only: the caller fetches bodies through :meth:`raw` one at a time,
+        so a 500-message sample never materialises 500 message bodies at once.
+        """
+        stmt = (
+            select(Message.id)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(
+                Message.user_id == user_id,
+                Message.account_id == account_id,
+                Mailbox.special_use == special_use,
+            )
+            .order_by(Message.internal_date.desc())
+            .limit(max(0, min(limit, 5000)))
+        )
+        if older_than_days is not None:
+            cutoff = dt.datetime.now(dt.UTC) - dt.timedelta(days=older_than_days)
+            stmt = stmt.where(Message.internal_date < cutoff)
+        async with session_scope() as session:
+            return list(await session.scalars(stmt))
+
+    async def threads_of(
+        self, user_id: uuid.UUID, message_ids: Iterable[str]
+    ) -> dict[str, str | None]:
+        """Map still-existing message ids to their current thread id (None if
+        unthreaded). Deleted messages are omitted, so a plugin can prune stale
+        per-message references and re-home labels onto the live thread after a
+        JWZ thread merge."""
+        ids: list[uuid.UUID] = []
+        for mid in message_ids:
+            try:
+                ids.append(uuid.UUID(str(mid)))
+            except (ValueError, AttributeError, TypeError):
+                continue
+        if not ids:
+            return {}
+        async with session_scope() as session:
+            rows = await session.execute(
+                select(Message.id, Message.thread_id).where(
+                    Message.user_id == user_id, Message.id.in_(ids)
+                )
+            )
+            return {str(mid): (str(tid) if tid else None) for mid, tid in rows}
+
+    async def envelope(self, user_id: uuid.UUID, message_id: uuid.UUID) -> dict[str, Any] | None:
+        async with session_scope() as session:
+            row = (
+                await session.execute(
+                    select(Message, Mailbox.special_use, Account.state)
+                    .join(Mailbox, Mailbox.id == Message.mailbox_id, isouter=True)
+                    .join(Account, Account.id == Message.account_id, isouter=True)
+                    .where(Message.id == message_id, Message.user_id == user_id)
+                )
+            ).first()
+            if row is None:
+                return None
+            msg, special_use, account_state = row
+            sender = msg.from_addr[0] if msg.from_addr else {}
+            return {
+                "message_id": str(msg.id),
+                "account_id": str(msg.account_id),
+                # Where it landed: "inbox", "junk", "sent", … or None for a folder
+                # with no SPECIAL-USE. A plugin that should only act on newly
+                # arrived mail checks this — `message.new` itself does not say.
+                "mailbox_special_use": special_use,
+                # "backfilling" | "live" | … — `message.new` fires for imported
+                # history too, so a plugin that should only react to genuinely
+                # new arrivals has to be able to tell the difference.
+                "account_state": account_state,
+                "subject": msg.subject or "",
+                "from_email": (sender.get("email") or "").lower(),
+                "from_name": sender.get("name") or "",
+                # Full recipient list (multi-account setups match on "to").
+                "to": [
+                    {"email": (a.get("email") or "").lower(), "name": a.get("name") or ""}
+                    for a in msg.to_addr
+                ],
+                "flags": list(msg.flags),
+                # Whether files came with it — e.g. an invitation's .ics.
+                "has_attachments": bool(msg.has_attachments),
+            }
+
+
+class PluginAccounts:
+    """Account queries: ``live()`` is cross-user, for scheduled jobs (a sweep runs
+    on behalf of the instance and returns only ids and state); ``of_user`` and
+    ``settings_of`` read one user's own accounts and settings, for a handler
+    already acting as that user."""
+
+    async def live(self) -> list[tuple[uuid.UUID, uuid.UUID]]:
+        """``(user_id, account_id)`` for every account finished with its initial
+        import. Accounts still backfilling are excluded: acting on one means
+        acting on a partial mailbox."""
+        async with session_scope() as session:
+            rows = await session.execute(
+                select(Account.user_id, Account.id).where(Account.state == "live")
+            )
+            return [(u, a) for u, a in rows]
+
+    async def of_user(self, user_id: uuid.UUID) -> list[tuple[uuid.UUID, str, str]]:
+        """``(account_id, email, display_name)`` of the user's own mail accounts —
+        e.g. to tell which attendee of an emailed invitation is the user."""
+        async with session_scope() as session:
+            rows = await session.execute(
+                select(Account.id, Account.email, Account.display_name)
+                .where(Account.user_id == user_id)
+                .order_by(Account.created_at)
+            )
+            return [(a, str(e), str(n or "")) for a, e, n in rows]
+
+    async def settings_of(self, user_id: uuid.UUID) -> dict[str, object]:
+        """The user's settings (time zone, default calendar, …), read-only."""
+        async with session_scope() as session:
+            settings = await session.scalar(select(User.settings).where(User.id == user_id))
+            return dict(settings or {})
+
+
+class PluginThreads:
+    """Read-only, user-scoped thread queries (e.g. self-healing stale references)."""
+
+    async def existing(self, user_id: uuid.UUID, thread_ids: Iterable[str]) -> set[str]:
+        """Return the subset of ``thread_ids`` that still exist for the user.
+
+        Lets a plugin prune references to threads deleted out from under it (e.g. an
+        account removal drops its threads, orphaning label aggregates keyed by id)."""
+        ids: list[uuid.UUID] = []
+        for tid in thread_ids:
+            try:
+                ids.append(uuid.UUID(str(tid)))
+            except (ValueError, AttributeError, TypeError):
+                continue
+        if not ids:
+            return set()
+        async with session_scope() as session:
+            rows = await session.scalars(
+                select(Thread.id).where(Thread.user_id == user_id, Thread.id.in_(ids))
+            )
+            return {str(r) for r in rows}
+
+
+class PluginActions:
+    """User-scoped actions a plugin can take on a message."""
+
+    async def set_flags(
+        self,
+        user_id: uuid.UUID,
+        message_id: uuid.UUID,
+        *,
+        add: Iterable[str] = (),
+        remove: Iterable[str] = (),
+    ) -> bool:
+        """Add/remove IMAP flags on the user's message (mirrors the core path).
+
+        Only user-settable flags survive (\\Seen/\\Flagged/\\Answered). Returns
+        False if nothing owned matches or nothing would change."""
+        from app.queue import enqueue_job
+        from app.sync.actions import apply_flags_to_messages, sanitize_flags
+
+        add_f, remove_f = sanitize_flags(add), sanitize_flags(remove)
+        if not add_f and not remove_f:
+            return False
+        async with session_scope() as session:
+            msg = await session.scalar(
+                select(Message).where(Message.id == message_id, Message.user_id == user_id)
+            )
+            if msg is None:
+                return False
+            await apply_flags_to_messages(session, [msg], add_f, remove_f)
+            flags = list(msg.flags)
+        await enqueue_job("push_flags_job", str(message_id), sorted(add_f), sorted(remove_f))
+        await get_event_bus().publish(
+            "flags.changed",
+            {"user_id": str(user_id), "message_id": str(message_id), "flags": flags},
+        )
+        return True
+
+    async def move_to_special(
+        self, user_id: uuid.UUID, message_id: uuid.UUID, special_use: str
+    ) -> bool:
+        """Move ONE message into the account's folder with this ``special_use``
+        (``junk``, ``inbox``, …), on the same path as the UI's move: optimistic DB
+        update, then ``move_job`` performs the IMAP MOVE and reconciles the uid.
+
+        Per-message on purpose. ``trash`` moves a whole thread, which is right for
+        Delete and wrong for a spam verdict — one spam reply must not drag a
+        legitimate conversation into Junk.
+
+        Returns False when the message isn't the user's, the account has no such
+        folder, or it is already there. That is "could not act", not an error:
+        an account whose server has no Junk folder must still receive its mail.
+        """
+        from app.queue import enqueue_job
+
+        async with session_scope() as session:
+            msg = await session.scalar(
+                select(Message).where(Message.id == message_id, Message.user_id == user_id)
+            )
+            if msg is None:
+                return False
+            dst = await session.scalar(
+                select(Mailbox).where(
+                    Mailbox.user_id == user_id,
+                    Mailbox.account_id == msg.account_id,  # moves stay within an account
+                    Mailbox.special_use == special_use,
+                )
+            )
+            if dst is None or dst.id == msg.mailbox_id:
+                return False
+            dst_id, src_id = dst.id, msg.mailbox_id
+            # The SOURCE folder's special-use, not the destination parameter:
+            # consumers of message.changed read the pair as a transition, and
+            # reporting them equal would erase the move.
+            src_special = await session.scalar(
+                select(Mailbox.special_use).where(Mailbox.id == src_id)
+            )
+            thread_id, dst_special = msg.thread_id, dst.special_use
+            msg.mailbox_id = dst_id  # optimistic, as the core endpoint does
+            await session.commit()
+
+        await enqueue_job("move_job", str(message_id), str(dst_id), str(src_id))
+        # Announce the optimistic state now rather than waiting for the IMAP move
+        # to finish. The core's own move endpoint gets this for free — its caller
+        # updates the UI from the response — but a plugin acting on the user's
+        # behalf has no client-side handle, so without this the message lingers on
+        # screen until the job lands. move_job publishes again when it does.
+        await get_event_bus().publish(
+            "message.changed",
+            {
+                "user_id": str(user_id),
+                "message_id": str(message_id),
+                "thread_id": str(thread_id) if thread_id else None,
+                "mailbox_id": str(dst_id),
+                "previous_mailbox_id": str(src_id),
+                "special_use": dst_special,
+                "previous_special_use": src_special,
+            },
+        )
+        return True
+
+    async def trash(self, user_id: uuid.UUID, message_id: uuid.UUID) -> bool:
+        """Move the message's thread to Trash — the same path as the UI Delete
+        (move_thread_to_folder_job handles the IMAP move, restore state, folder
+        creation and events). Returns False if the message isn't the user's."""
+        from app.queue import enqueue_job
+
+        async with session_scope() as session:
+            msg = await session.scalar(
+                select(Message).where(Message.id == message_id, Message.user_id == user_id)
+            )
+            if msg is None or msg.thread_id is None:
+                return False
+            thread_id = msg.thread_id
+        await enqueue_job("move_thread_to_folder_job", str(thread_id), "trash")
+        return True

backend/app/plugins/alembic/env.py

@@ -0,0 +1,39 @@
+"""Shared Alembic environment for plugin migration branches (SPEC §11.1).
+
+Every plugin reuses this env; the runner (:mod:`app.plugins.migrate`) points
+``version_locations`` at the plugin's own revision directory and gives it a
+dedicated ``version_table`` (``alembic_version_plg_{name}``) so each plugin's
+migration history is independent of core and of every other plugin.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from alembic import context
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+
+config = context.config
+config.set_main_option("sqlalchemy.url", get_settings().database_url)
+_version_table = config.get_main_option("version_table") or "alembic_version"
+
+
+def _do_run_migrations(connection: object) -> None:
+    context.configure(connection=connection, version_table=_version_table)  # type: ignore[arg-type]
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def _run_online() -> None:
+    engine = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}), prefix="sqlalchemy.", poolclass=NullPool
+    )
+    async with engine.connect() as connection:
+        await connection.run_sync(_do_run_migrations)
+    await engine.dispose()
+
+
+asyncio.run(_run_online())

backend/app/plugins/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
+depends_on: str | Sequence[str] | None = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

backend/app/plugins/cli.py

@@ -0,0 +1,58 @@
+"""Plugin management CLI (SPEC §11.1).
+
+    python -m app.plugins.cli list
+    python -m app.plugins.cli install <name>
+    python -m app.plugins.cli uninstall <name> [--drop-data]
+
+Builtin (core) plugins refuse uninstall.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from app.plugins import loader, migrate
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m app.plugins.cli", description="Manage webmail plugins"
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("list", help="list discovered plugins")
+    p_install = sub.add_parser("install", help="apply a plugin's migrations")
+    p_install.add_argument("name")
+    p_uninstall = sub.add_parser("uninstall", help="remove a plugin (refuses builtin)")
+    p_uninstall.add_argument("name")
+    p_uninstall.add_argument(
+        "--drop-data", action="store_true", help="also drop the plugin's tables"
+    )
+    args = parser.parse_args(argv)
+
+    if args.cmd == "list":
+        for plugin in loader.get_plugins():
+            kind = "builtin" if plugin.builtin else "removable"
+            migr = "migrations" if plugin.migrations else "no-migrations"
+            print(f"{plugin.name}\t{plugin.version}\t{kind}\t{migr}")
+        return 0
+
+    if args.cmd == "install":
+        migrate.install(args.name)
+        print(f"installed {args.name}")
+        return 0
+
+    if args.cmd == "uninstall":
+        try:
+            migrate.uninstall(args.name, drop_data=args.drop_data)
+        except loader.PluginRemovalError as exc:
+            print(f"error: {exc}")
+            return 1
+        suffix = " (data dropped)" if args.drop_data else " (data kept)"
+        print(f"uninstalled {args.name}{suffix}")
+        return 0
+
+    return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

backend/app/plugins/db.py

@@ -0,0 +1,44 @@
+"""RLS-enforced database access for plugins (SPEC §11.1 + privacy hardening).
+
+``plugin_session(user_id)`` yields a normal SQLAlchemy session whose transaction
+has been dropped to the least-privilege :data:`~app.plugins.rls.PLUGIN_ROLE` and
+bound to the per-user RLS context. Both are set with transaction-local scope
+(``SET LOCAL`` / ``set_config(..., is_local=true)``), so they auto-revert on
+commit/rollback and cannot leak across pooled connections.
+
+Because the role is non-superuser / non-owner / NOBYPASSRLS, the row-level policy
+from :mod:`app.plugins.rls` is actually enforced: a plugin query that forgets to
+filter ``user_id`` still returns only the current user's rows, and it cannot write
+rows for anyone else (policy ``WITH CHECK``).
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import get_sessionmaker
+from app.plugins.rls import PLUGIN_ROLE
+
+
+@asynccontextmanager
+async def plugin_session(user_id: uuid.UUID) -> AsyncIterator[AsyncSession]:
+    """A transactional session scoped to ``user_id`` under RLS (SPEC §11.1)."""
+    async with get_sessionmaker()() as session:
+        try:
+            # Bind the RLS user context first (while still the privileged role), then
+            # drop into the restricted role for the rest of the transaction.
+            await session.execute(
+                text("SELECT set_config('app.current_user_id', :uid, true)"),
+                {"uid": str(user_id)},
+            )
+            await session.execute(text(f"SET LOCAL ROLE {PLUGIN_ROLE}"))
+            yield session
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise

backend/app/plugins/http.py

@@ -0,0 +1,39 @@
+"""SSRF-guarded, multi-method HTTP client exposed to plugins (SPEC §10.6).
+
+``sdk.http_get`` covers simple GETs; heavy plugins that speak CalDAV/CardDAV need
+other verbs (PROPFIND/REPORT/PUT/DELETE) with request bodies and basic auth. This
+facade routes all of them through the same SSRF policy (HTTPS-only, public host,
+no off-domain redirects, size cap) and returns status + headers + body.
+"""
+
+from __future__ import annotations
+
+from app.accounts.ssrf import GuardedResponse, guarded_request
+
+
+class PluginHttp:
+    async def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        content: bytes | str | None = None,
+        auth: tuple[str, str] | None = None,
+    ) -> GuardedResponse:
+        return await guarded_request(method, url, headers=headers, content=content, auth=auth)
+
+    async def get(self, url: str, **kw: object) -> GuardedResponse:
+        return await self.request("GET", url, **kw)  # type: ignore[arg-type]
+
+    async def put(self, url: str, **kw: object) -> GuardedResponse:
+        return await self.request("PUT", url, **kw)  # type: ignore[arg-type]
+
+    async def delete(self, url: str, **kw: object) -> GuardedResponse:
+        return await self.request("DELETE", url, **kw)  # type: ignore[arg-type]
+
+    async def propfind(self, url: str, **kw: object) -> GuardedResponse:
+        return await self.request("PROPFIND", url, **kw)  # type: ignore[arg-type]
+
+    async def report(self, url: str, **kw: object) -> GuardedResponse:
+        return await self.request("REPORT", url, **kw)  # type: ignore[arg-type]

backend/app/plugins/jobs.py

@@ -0,0 +1,130 @@
+"""Plugin background jobs on the shared arq worker (SPEC §11.2).
+
+Each plugin ``JobSpec`` is registered as an arq function under a namespaced name
+``plg.{plugin}.{func}`` (enqueue on demand from hooks/routes via ``sdk.enqueue``),
+and, when it has a ``schedule``, also as an arq cron job. ``singleton=True`` wraps
+the job in a Redis NX lock keyed by (plugin, func, user) so at most one instance
+runs per user (or globally for scheduleds with no user). Jobs share the worker
+pool + retry semantics with core jobs and log under the ``plg`` logger.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from arq import cron
+from arq.cron import CronJob
+from arq.worker import Function, func
+
+from app.plugins import loader
+from app.plugins.types import JobSpec, Plugin
+from app.redis_client import get_redis
+
+logger = logging.getLogger("plg")
+
+# Safety expiry on the singleton lock so a crashed job can't wedge a user forever.
+# A job that runs longer than this could overlap; acceptable for M3.
+_LOCK_TTL = 3600
+
+# Standard 5-field cron order and each field's inclusive value bounds.
+_FIELD_ORDER = ("minute", "hour", "day", "month", "weekday")
+_FIELD_BOUNDS = {
+    "minute": (0, 59),
+    "hour": (0, 23),
+    "day": (1, 31),
+    "month": (1, 12),
+    "weekday": (0, 6),
+}
+
+
+def job_name(plugin_name: str, spec: JobSpec) -> str:
+    """The arq function name a plugin job is registered under."""
+    return f"plg.{plugin_name}.{spec.func.__name__}"
+
+
+def _lock_key(plugin_name: str, func_name: str, user: object) -> str:
+    return f"plglock:{plugin_name}:{func_name}:{user}"
+
+
+def _make_runner(plugin: Plugin, spec: JobSpec) -> tuple[Any, str]:
+    """Wrap a job func with the singleton lock (when requested) + namespaced name."""
+    func_name = spec.func.__name__
+    name = job_name(plugin.name, spec)
+
+    async def runner(ctx: dict[str, Any], *args: Any, **kwargs: Any) -> Any:
+        if not spec.singleton:
+            return await spec.func(ctx, *args, **kwargs)
+        user = kwargs.get("user_id", "global")
+        key = _lock_key(plugin.name, func_name, user)
+        acquired = await get_redis().set(key, b"1", nx=True, ex=_LOCK_TTL)
+        if not acquired:
+            logger.info("%s skipped: singleton already running (user=%s)", name, user)
+            return None
+        try:
+            return await spec.func(ctx, *args, **kwargs)
+        finally:
+            await get_redis().delete(key)
+
+    return runner, name
+
+
+def plugin_arq_functions() -> list[Function]:
+    """Every plugin job as an arq function (enqueue-on-demand), namespaced."""
+    funcs: list[Function] = []
+    for plugin in loader.get_plugins():
+        for spec in plugin.jobs:
+            runner, name = _make_runner(plugin, spec)
+            funcs.append(func(runner, name=name))
+    return funcs
+
+
+def _parse_field(token: str, lo: int, hi: int) -> set[int] | None:
+    """Parse one cron field to a value set (``*`` -> None = every)."""
+    if token == "*":
+        return None
+    values: set[int] = set()
+    for part in token.split(","):
+        if part.startswith("*/"):
+            values.update(range(lo, hi + 1, int(part[2:])))
+        elif "-" in part:
+            start, end = part.split("-", 1)
+            values.update(range(int(start), int(end) + 1))
+        else:
+            values.add(int(part))
+    return values
+
+
+def parse_cron(expr: str) -> dict[str, set[int] | None]:
+    """Parse a standard 5-field cron string into arq ``cron`` field kwargs."""
+    parts = expr.split()
+    if len(parts) != 5:
+        raise ValueError(f"cron expression must have 5 fields (min hour dom mon dow): {expr!r}")
+    return {
+        name: _parse_field(token, *_FIELD_BOUNDS[name])
+        for name, token in zip(_FIELD_ORDER, parts, strict=True)
+    }
+
+
+def plugin_cron_jobs() -> list[CronJob]:
+    """Cron-scheduled plugin jobs as arq ``CronJob``s (run on the shared worker)."""
+    jobs: list[CronJob] = []
+    for plugin in loader.get_plugins():
+        for spec in plugin.jobs:
+            if spec.schedule is None:
+                continue
+            runner, name = _make_runner(plugin, spec)
+            f = parse_cron(spec.schedule)
+            jobs.append(
+                cron(
+                    runner,
+                    name=f"{name}.cron",
+                    second=0,
+                    minute=f["minute"],
+                    hour=f["hour"],
+                    day=f["day"],
+                    month=f["month"],
+                    weekday=f["weekday"],
+                )
+            )
+    return jobs

backend/app/plugins/kv.py

@@ -0,0 +1,63 @@
+"""Namespaced key/value store handed to plugins (SPEC §11 ``plugin_kv``).
+
+A :class:`PluginKv` is bound to one (plugin, user_id) slice of the core
+``plugin_kv`` table; values are arbitrary JSON. This is the light-state store
+every plugin gets — heavy plugins use their own ``plg_``-prefixed tables (§11.1).
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy import delete, select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from app.db import session_scope
+from app.models import PluginKv
+
+
+class PluginKvStore:
+    """CRUD over one plugin's per-user KV slice. Instantiate via ``sdk.kv(user_id)``."""
+
+    def __init__(self, plugin: str, user_id: uuid.UUID) -> None:
+        self._plugin = plugin
+        self._user_id = user_id
+
+    async def get(self, key: str, default: Any = None) -> Any:
+        async with session_scope() as session:
+            row = await session.get(PluginKv, (self._plugin, self._user_id, key))
+            return row.value if row is not None else default
+
+    async def set(self, key: str, value: Any) -> None:
+        """Upsert ``key`` -> ``value`` (JSON-serialisable)."""
+        stmt = (
+            pg_insert(PluginKv)
+            .values(plugin=self._plugin, user_id=self._user_id, key=key, value=value)
+            .on_conflict_do_update(
+                index_elements=[PluginKv.plugin, PluginKv.user_id, PluginKv.key],
+                set_={"value": value},
+            )
+        )
+        async with session_scope() as session:
+            await session.execute(stmt)
+
+    async def delete(self, key: str) -> None:
+        async with session_scope() as session:
+            await session.execute(
+                delete(PluginKv).where(
+                    PluginKv.plugin == self._plugin,
+                    PluginKv.user_id == self._user_id,
+                    PluginKv.key == key,
+                )
+            )
+
+    async def items(self) -> dict[str, Any]:
+        """Return every key/value in this plugin+user slice."""
+        async with session_scope() as session:
+            rows = await session.execute(
+                select(PluginKv.key, PluginKv.value).where(
+                    PluginKv.plugin == self._plugin, PluginKv.user_id == self._user_id
+                )
+            )
+            return dict(rows.all())  # type: ignore[arg-type]

backend/app/plugins/loader.py

@@ -0,0 +1,149 @@
+"""Plugin discovery + core wiring (SPEC §11).
+
+Discovers plugins via the ``webmail.plugins`` entry-point group, then:
+- mounts each plugin router at ``/api/plugins/{name}`` (API process),
+- forwards §8.3 events to plugin hooks (both API and worker processes),
+- aggregates UI manifests for ``GET /api/plugins/ui-manifest``.
+
+Plugin background jobs (§11.2) and migrations (§11.1) are wired in later M3 phases.
+"""
+
+from __future__ import annotations
+
+import logging
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING, Any
+
+from app.events import EventBus
+from app.plugins.types import Page, Plugin, Slot, UiManifest
+
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+logger = logging.getLogger(__name__)
+
+ENTRYPOINT_GROUP = "webmail.plugins"
+
+_plugins: list[Plugin] | None = None
+# Plugin names whose hooks are already subscribed on a given bus, so repeated
+# create_app()/worker starts in one process don't double-register (tests, reload).
+_hooks_registered: set[tuple[int, str]] = set()
+
+
+def _coerce_plugin(obj: Any) -> Plugin:
+    """An entry point may resolve to a ``Plugin`` or a module exposing ``plugin``."""
+    if isinstance(obj, Plugin):
+        return obj
+    candidate = getattr(obj, "plugin", None)
+    if isinstance(candidate, Plugin):
+        return candidate
+    raise TypeError(f"entry point {obj!r} does not expose a Plugin (SPEC §11)")
+
+
+def discover_plugins() -> list[Plugin]:
+    """Load every plugin registered under the ``webmail.plugins`` group."""
+    found: list[Plugin] = []
+    seen: set[str] = set()
+    for ep in entry_points(group=ENTRYPOINT_GROUP):
+        try:
+            plugin = _coerce_plugin(ep.load())
+        except Exception:  # noqa: BLE001 - one bad plugin must not break the app
+            logger.exception("failed to load plugin entry point %s", ep.name)
+            continue
+        if plugin.name in seen:
+            logger.error("duplicate plugin name %r; skipping", plugin.name)
+            continue
+        seen.add(plugin.name)
+        found.append(plugin)
+        logger.info("loaded plugin %s v%s", plugin.name, plugin.version)
+    return found
+
+
+def get_plugins() -> list[Plugin]:
+    """Return the process-wide discovered plugin list (cached)."""
+    global _plugins
+    if _plugins is None:
+        _plugins = discover_plugins()
+    return _plugins
+
+
+def set_plugins_for_test(plugins: list[Plugin] | None) -> None:
+    """Override (or reset with ``None``) the plugin registry — tests only."""
+    global _plugins
+    _plugins = plugins
+    _hooks_registered.clear()
+
+
+def register_hooks(bus: EventBus) -> None:
+    """Subscribe each plugin's hooks to the bus (idempotent per process/bus).
+
+    Called in both the API and the worker so hooks fire wherever the event is
+    published (e.g. ``message.new`` originates in the worker)."""
+    for plugin in get_plugins():
+        marker = (id(bus), plugin.name)
+        if marker in _hooks_registered:
+            continue
+        for event, handler in plugin.hooks.items():
+            bus.subscribe(event, handler)
+        _hooks_registered.add(marker)
+
+
+def mount_routers(app: FastAPI) -> None:
+    """Mount each plugin's router at ``/api/plugins/{name}`` (SPEC §11)."""
+    for plugin in get_plugins():
+        if plugin.router is not None:
+            app.include_router(plugin.router, prefix=f"/api/plugins/{plugin.name}")
+
+
+def _slot_dict(slot: Slot) -> dict[str, str]:
+    d = {"name": slot.name, "bundle": slot.bundle}
+    if slot.title:
+        d["title"] = slot.title
+    return d
+
+
+def _page_dict(page: Page) -> dict[str, Any]:
+    return {
+        "route": page.route,
+        "bundle": page.bundle,
+        "nav": {"icon": page.nav.icon, "label": page.nav.label, "order": page.nav.order},
+    }
+
+
+def _manifest_dict(plugin: Plugin, manifest: UiManifest) -> dict[str, Any]:
+    return {
+        "name": plugin.name,
+        "version": plugin.version,
+        "builtin": plugin.builtin,  # core plugins the UI must not offer to remove
+        "slots": [_slot_dict(s) for s in manifest.slots],
+        "pages": [_page_dict(p) for p in manifest.pages],
+        "capabilities": dict(manifest.capabilities),
+    }
+
+
+def build_ui_manifest() -> dict[str, Any]:
+    """Aggregate every plugin's UI manifest for the frontend loader (SPEC §11.3)."""
+    plugins = [_manifest_dict(p, p.ui_manifest) for p in get_plugins() if p.ui_manifest is not None]
+    return {"plugins": plugins}
+
+
+def get_plugin(name: str) -> Plugin | None:
+    """Return the loaded plugin named ``name``, or None."""
+    return next((p for p in get_plugins() if p.name == name), None)
+
+
+class PluginRemovalError(Exception):
+    """Raised when trying to uninstall a builtin (core) plugin (SPEC §11 / §12)."""
+
+
+def assert_removable(name: str) -> Plugin:
+    """Return the plugin, or raise if it's unknown or builtin (non-removable).
+
+    Core plugins (calendar, contacts) ship ``builtin=True`` and may never be
+    uninstalled by a user; the uninstall path calls this before touching data."""
+    plugin = get_plugin(name)
+    if plugin is None:
+        raise PluginRemovalError(f"unknown plugin {name!r}")
+    if plugin.builtin:
+        raise PluginRemovalError(f"plugin {name!r} is builtin and cannot be removed")
+    return plugin

backend/app/plugins/mail.py

@@ -0,0 +1,113 @@
+"""Send an email *as the user*, from one of their own accounts (SPEC §11 SDK).
+
+Heavy plugins (calendar iMIP invites, later contacts) need to send a message
+through the organizer's real mailbox — reusing the same SMTP path as the compose
+outbox, but with a plugin-built body (e.g. a ``text/calendar; method=REQUEST``
+part). This helper looks up the user's account, assembles the MIME, and sends via
+the account's provider. It is exposed as ``sdk(name).mail`` so plugins never touch
+``app.providers`` / ``app.sync`` directly.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import format_datetime, make_msgid
+
+from sqlalchemy import select
+
+from app.db import session_scope
+from app.models import Account
+from app.sync.provider_factory import build_provider
+
+
+class PluginMailError(Exception):
+    """No usable account, or the send failed."""
+
+
+class PluginMail:
+    """Send outbound mail from one of a user's own accounts."""
+
+    async def send(
+        self,
+        user_id: uuid.UUID,
+        *,
+        to: list[str],
+        subject: str,
+        html: str,
+        text: str,
+        from_email: str | None = None,
+        ics: str | None = None,
+        ics_method: str = "REQUEST",
+        ics_filename: str = "invite.ics",
+    ) -> None:
+        """Send an email to ``to`` from the user's account.
+
+        ``from_email`` selects which of the user's accounts to send from (falls
+        back to their first account). When ``ics`` is given it is added both as a
+        ``text/calendar; method=<ics_method>`` alternative (so the recipient's
+        client shows its native RSVP UI) and as a downloadable attachment.
+        """
+        recipients = [r for r in (to or []) if r]
+        if not recipients:
+            return
+        async with session_scope() as session:
+            account = None
+            if from_email:
+                account = await session.scalar(
+                    select(Account).where(
+                        Account.user_id == user_id,
+                        Account.email == from_email,
+                    )
+                )
+            if account is None:
+                account = await session.scalar(
+                    select(Account).where(Account.user_id == user_id).order_by(Account.created_at)
+                )
+            if account is None:
+                raise PluginMailError("no account to send from")
+            raw = _build_message(
+                account, recipients, subject, html, text, ics, ics_method, ics_filename
+            )
+            envelope_from = account.email
+            provider = build_provider(account)
+        await provider.send(raw, envelope_from, recipients)
+
+
+def _build_message(
+    account: Account,
+    to: list[str],
+    subject: str,
+    html: str,
+    text: str,
+    ics: str | None,
+    ics_method: str,
+    ics_filename: str,
+) -> bytes:
+    root = MIMEMultipart("mixed")
+    from_name = account.display_name or ""
+    root["From"] = f"{from_name} <{account.email}>" if from_name else account.email
+    root["To"] = ", ".join(to)
+    root["Subject"] = subject
+    root["Date"] = format_datetime(datetime.now(UTC))
+    root["Message-ID"] = make_msgid(domain=account.email.rsplit("@", 1)[-1])
+
+    alt = MIMEMultipart("alternative")
+    alt.attach(MIMEText(text, "plain", "utf-8"))
+    alt.attach(MIMEText(html, "html", "utf-8"))
+    if ics is not None:
+        cal = MIMEText(ics, "calendar", "utf-8")
+        cal.set_param("method", ics_method)
+        cal.set_param("component", "VEVENT")
+        alt.attach(cal)
+    root.attach(alt)
+
+    if ics is not None:
+        att = MIMEApplication(ics.encode("utf-8"), _subtype="ics")
+        att.add_header("Content-Disposition", "attachment", filename=ics_filename)
+        root.attach(att)
+
+    return root.as_bytes()

backend/app/plugins/migrate.py

@@ -0,0 +1,98 @@
+"""Per-plugin Alembic migration runner + install/uninstall lifecycle (SPEC §11.1).
+
+Each plugin ships its own revision directory (``Plugin.migrations``) and runs
+against a dedicated version table, so its schema history is independent of core
+and of other plugins. ``install`` upgrades to head; ``uninstall --drop-data``
+runs the plugin's downgrade chain and removes the version table, leaving zero
+residue (the §12 M3 acceptance round-trip). Builtin (core) plugins refuse removal.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+from app.plugins import loader
+from app.plugins.types import Plugin
+
+logger = logging.getLogger(__name__)
+
+_SHARED_ENV = str(Path(__file__).parent / "alembic")
+
+
+class PluginMigrationError(Exception):
+    """Raised when a plugin's migrations can't be run."""
+
+
+def _version_table(plugin: Plugin) -> str:
+    # plugin.name is validated [a-z][a-z0-9_]* at construction, so this is safe to
+    # interpolate into DDL.
+    return f"alembic_version_plg_{plugin.name}"
+
+
+def _config(plugin: Plugin) -> Config:
+    if not plugin.migrations:
+        raise PluginMigrationError(f"plugin {plugin.name!r} declares no migrations")
+    cfg = Config()
+    cfg.set_main_option("script_location", _SHARED_ENV)
+    cfg.set_main_option("version_locations", plugin.migrations)
+    cfg.set_main_option("path_separator", "os")  # split version_locations on os.pathsep only
+    cfg.set_main_option("version_table", _version_table(plugin))
+    return cfg
+
+
+def upgrade(plugin: Plugin, revision: str = "head") -> None:
+    """Run the plugin's migrations up to ``revision`` (default: head)."""
+    command.upgrade(_config(plugin), revision)
+
+
+def downgrade(plugin: Plugin, revision: str = "base") -> None:
+    """Run the plugin's downgrades to ``revision`` (default: base = drop all)."""
+    command.downgrade(_config(plugin), revision)
+
+
+async def _drop_version_table(plugin: Plugin) -> None:
+    # Fresh engine (not the shared global one) so this is safe to call from any loop.
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    try:
+        async with engine.begin() as conn:
+            await conn.execute(text(f"DROP TABLE IF EXISTS {_version_table(plugin)}"))
+    finally:
+        await engine.dispose()
+
+
+def install(name: str) -> None:
+    """Apply a plugin's migrations (create its ``plg_*`` tables). No-op if none."""
+    plugin = loader.get_plugin(name)
+    if plugin is None:
+        raise PluginMigrationError(f"unknown plugin {name!r}")
+    if plugin.migrations is None:
+        logger.info("plugin %s has no migrations; nothing to install", name)
+        return
+    logger.info("installing plugin %s: upgrade -> head", name)
+    upgrade(plugin, "head")
+
+
+def uninstall(name: str, *, drop_data: bool = False) -> None:
+    """Uninstall a plugin. Refuses builtin plugins.
+
+    Without ``drop_data`` the schema/data are kept (a reversible disable). With
+    ``drop_data`` the downgrade chain runs and the version table is removed, so
+    nothing remains in the database (SPEC §12 M3 acceptance)."""
+    plugin = loader.assert_removable(name)  # raises PluginRemovalError for builtin/unknown
+    if not drop_data:
+        logger.info("uninstalling plugin %s (data kept)", name)
+        return
+    if plugin.migrations is None:
+        return
+    logger.info("uninstalling plugin %s --drop-data: downgrade -> base", name)
+    downgrade(plugin, "base")
+    asyncio.run(_drop_version_table(plugin))

backend/app/plugins/rls.py

@@ -0,0 +1,46 @@
+"""Row-Level Security for plugin-owned tables (SPEC §11.1 + privacy hardening).
+
+Per-user isolation for ``plg_*`` tables is enforced by Postgres, not by remembering
+a ``WHERE user_id = ?`` in every query. Each user-data table gets RLS enabled +
+forced, with a policy comparing ``user_id`` to the transaction-local GUC
+``app.current_user_id``. Plugin code reaches these tables only through
+``sdk.session(user_id)`` (see :mod:`app.plugins.db`), which drops to the
+least-privilege :data:`PLUGIN_ROLE` and binds that GUC — so even a query that
+forgets to filter cannot see or write another user's rows.
+
+Plugin migrations call :func:`enable_rls` right after creating a user-data table.
+"""
+
+from __future__ import annotations
+
+# NOLOGIN, non-superuser, NOBYPASSRLS role plugin sessions SET LOCAL ROLE into
+# (created by core migration 0014). RLS is skipped for superusers / table owners /
+# BYPASSRLS roles, so plugin access must run as this restricted role to be enforced.
+PLUGIN_ROLE = "webmail_plugin"
+
+_USER_CTX = "current_setting('app.current_user_id', true)::uuid"
+
+
+def rls_ddl(table: str, *, user_column: str = "user_id") -> list[str]:
+    """SQL statements that lock ``table`` to the current user (SPEC §11.1).
+
+    Pure/testable: returns the DDL rather than executing it. ``FORCE`` makes even
+    the table owner subject to the policy; ``WITH CHECK`` blocks inserting/updating
+    rows for a different user."""
+    policy = f"{table}_user_isolation"
+    return [
+        f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY",
+        f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY",
+        f"CREATE POLICY {policy} ON {table} "
+        f"USING ({user_column} = {_USER_CTX}) "
+        f"WITH CHECK ({user_column} = {_USER_CTX})",
+        f"GRANT SELECT, INSERT, UPDATE, DELETE ON {table} TO {PLUGIN_ROLE}",
+    ]
+
+
+def enable_rls(table: str, *, user_column: str = "user_id") -> None:
+    """Apply :func:`rls_ddl` via Alembic ``op`` — call from a plugin migration."""
+    from alembic import op
+
+    for statement in rls_ddl(table, user_column=user_column):
+        op.execute(statement)

backend/app/plugins/sdk.py

@@ -0,0 +1,231 @@
+"""The plugin SDK — the ONLY ``app.*`` surface a plugin may import (SPEC §11).
+
+Everything a plugin needs is reached through :func:`sdk`, which returns a
+:class:`PluginSdk` bound to the plugin's name: a plugin-scoped DB session, the
+event bus (publishing under ``plugin.{name}.*``), the blob store, outbox
+draft/send helpers, the settings accessor, an SSRF-guarded HTTP client
+(§10.6), and the namespaced KV store (§11).
+
+The frozen contract types (:class:`Plugin`, :class:`UiManifest`, :class:`JobSpec`,
+…) are re-exported here so plugin packages import only from ``app.plugins.sdk``.
+"""
+
+from __future__ import annotations
+
+import uuid
+from contextlib import AbstractAsyncContextManager
+from dataclasses import dataclass
+from typing import Annotated, Any
+
+from fastapi import Depends
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.accounts.ssrf import guarded_get
+from app.config import Settings, get_settings
+from app.db import session_scope
+from app.events import get_event_bus
+from app.identity.deps import current_user, require_csrf
+from app.indexer import blobstore
+from app.models import User
+from app.outbox import service as outbox_service
+from app.plugins.actions import (
+    PluginAccounts,
+    PluginActions,
+    PluginMessages,
+    PluginThreads,
+)
+from app.plugins.db import plugin_session
+from app.plugins.http import PluginHttp
+from app.plugins.kv import PluginKvStore
+from app.plugins.mail import PluginMail
+from app.plugins.types import (
+    Hook,
+    JobFunc,
+    JobSpec,
+    Page,
+    PageNav,
+    Plugin,
+    Slot,
+    UiManifest,
+)
+from app.plugins.vault import PluginVault
+from app.queue import enqueue_job
+
+__all__ = [
+    "CurrentUserId",
+    "Hook",
+    "JobFunc",
+    "JobSpec",
+    "Page",
+    "PageNav",
+    "Plugin",
+    "PluginSdk",
+    "Slot",
+    "UiManifest",
+    "blobstore",
+    "require_csrf",
+    "sdk",
+]
+
+
+async def _current_user_id(user: Annotated[User, Depends(current_user)]) -> uuid.UUID:
+    return user.id
+
+
+# Auth for plugin routers (plugins must not import app.identity directly, SPEC §11):
+# `CurrentUserId` resolves the authenticated user's id; `require_csrf` guards mutating
+# routes. Use with the RLS-scoped `sdk(name).session(user_id)`.
+CurrentUserId = Annotated[uuid.UUID, Depends(_current_user_id)]
+
+
+@dataclass(frozen=True)
+class PluginSdk:
+    """A lightweight facade over shared services, scoped to one plugin.
+
+    Cheap to construct (wraps process-wide singletons + the plugin name), so
+    handlers/routes obtain it on demand via ``sdk("<name>")``."""
+
+    name: str
+
+    # --- config -------------------------------------------------------------
+    @property
+    def settings(self) -> Settings:
+        return get_settings()
+
+    # --- database -----------------------------------------------------------
+    def session(self, user_id: uuid.UUID) -> AbstractAsyncContextManager[AsyncSession]:
+        """An RLS-enforced transactional session scoped to ``user_id`` (SPEC §11.1).
+
+        Row-level security limits every query to this user's rows on ``plg_*``
+        tables — even a query that forgets ``WHERE user_id`` cannot leak or write
+        across users (see :mod:`app.plugins.db`)."""
+        return plugin_session(user_id)
+
+    def public_session(self) -> AbstractAsyncContextManager[AsyncSession]:
+        """A privileged, NON-RLS session for PUBLIC, unauthenticated endpoints that
+        resolve their own opaque token and therefore must look up rows before any
+        user is known (e.g. a public ``.ics`` feed keyed by an unguessable slug).
+
+        There is no row-level safety net here: EVERY query MUST filter explicitly by
+        the token (or an id derived from it). Never use this for authenticated
+        handlers — use :meth:`session` so RLS scopes queries to the user."""
+        return session_scope()
+
+    def kv(self, user_id: uuid.UUID) -> PluginKvStore:
+        """The namespaced KV store for this plugin and ``user_id`` (SPEC §11)."""
+        return PluginKvStore(self.name, user_id)
+
+    # --- events -------------------------------------------------------------
+    async def publish(self, event: str, payload: dict[str, object]) -> None:
+        """Publish ``plugin.{name}.{event}`` on the bus (payload needs ``user_id``)."""
+        await get_event_bus().publish(f"plugin.{self.name}.{event}", payload)
+
+    # --- background jobs ----------------------------------------------------
+    async def enqueue(self, job_func: JobFunc, **kwargs: Any) -> None:
+        """Enqueue one of this plugin's jobs on the shared worker (SPEC §11.2).
+
+        ``job_func`` must be a job declared in this plugin's ``jobs=[...]``. Pass
+        ``user_id=`` for singleton jobs so the per-user lock applies."""
+        await enqueue_job(f"plg.{self.name}.{job_func.__name__}", **kwargs)
+
+    # --- blob store ---------------------------------------------------------
+    @property
+    def blob(self) -> object:
+        """The blob store (``store``/``load``/``exists``/``remove``), keyed by user."""
+        return blobstore
+
+    # --- outbox -------------------------------------------------------------
+    @property
+    def outbox(self) -> object:
+        """Draft/send helpers (``upsert_draft``, ``create_outbox``, …)."""
+        return outbox_service
+
+    # --- messages -----------------------------------------------------------
+    @property
+    def mail(self) -> PluginMail:
+        """Send an email from one of the user's own accounts (iMIP invites, …)."""
+        return PluginMail()
+
+    @property
+    def messages(self) -> PluginMessages:
+        """Read a user's message: envelope (subject/sender/flags), one header, or
+        the whole RFC822 blob for engines that need it."""
+        return PluginMessages()
+
+    @property
+    def actions(self) -> PluginActions:
+        """User-scoped message actions: ``set_flags``, ``trash`` (thread), and
+        ``move_to_special`` (one message into e.g. Junk)."""
+        return PluginActions()
+
+    @property
+    def accounts(self) -> PluginAccounts:
+        """Cross-user account listing for scheduled jobs (`live()`)."""
+        return PluginAccounts()
+
+    @property
+    def threads(self) -> PluginThreads:
+        """Read-only thread queries (e.g. pruning stale references)."""
+        return PluginThreads()
+
+    # --- outbound HTTP ------------------------------------------------------
+    async def http_get(self, url: str) -> str | None:
+        """GET ``url`` through the §10.6 SSRF guard (blocks private IPs/oversize)."""
+        return await guarded_get(url)
+
+    @property
+    def http(self) -> PluginHttp:
+        """Multi-method SSRF-guarded HTTP (GET/PUT/DELETE/PROPFIND/REPORT) for
+        CalDAV/CardDAV/ICS. Returns status + headers + body."""
+        return PluginHttp()
+
+    # --- notifications ------------------------------------------------------
+    @property
+    def push(self) -> PluginPush:
+        """Notifications on the user's devices (Web Push) — calendar reminders."""
+        return PluginPush(self.name)
+
+    # --- secret vault -------------------------------------------------------
+    @property
+    def vault(self) -> PluginVault:
+        """Envelope encrypt/decrypt for a plugin's own ``secret_ct`` columns
+        (CalDAV/CardDAV credentials, tokens) — §10.2 at-rest protection."""
+        return PluginVault()
+
+
+class PluginPush:
+    """``sdk.push``: show a notification on every device where the user turned
+    notifications on (Settings → Accounts → Notifications)."""
+
+    def __init__(self, plugin: str) -> None:
+        self._plugin = plugin
+
+    async def notify(
+        self,
+        user_id: uuid.UUID,
+        *,
+        title: str,
+        body: str = "",
+        url: str = "/mail",
+        tag: str | None = None,
+        sticky: bool = False,
+    ) -> int:
+        """Send one notification; returns how many devices it went to. ``tag``
+        replaces an earlier notification with the same tag (it is prefixed with
+        the plugin's name); ``sticky`` keeps it on screen until dismissed."""
+        from app.push.service import deliver
+
+        payload: dict[str, object] = {
+            "type": f"plugin.{self._plugin}",
+            "title": title[:200],
+            "body": body[:500],
+            "url": url if url.startswith("/") else "/mail",
+            "tag": f"{self._plugin}-{tag}" if tag else self._plugin,
+            "sticky": sticky,
+        }
+        return await deliver(user_id, [payload])
+
+
+def sdk(name: str) -> PluginSdk:
+    """Return the SDK facade bound to plugin ``name`` (SPEC §11)."""
+    return PluginSdk(name)

backend/app/plugins/types.py

@@ -0,0 +1,123 @@
+# INTERFACE VERSION: 1
+"""Frozen plugin contract types (SPEC §11).
+
+A plugin is a Python package ``webmail_plugin_<name>`` exposing a module-level
+``plugin = Plugin(...)``, discovered via the ``webmail.plugins`` entry-point group.
+
+These dataclasses are the frozen surface (SPEC §11 "freeze in M3"): fields may be
+added with defaults, but existing names/meanings may not change without bumping
+the INTERFACE VERSION above.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from fastapi import APIRouter
+
+# A hook handler receives an event payload (always carrying ``user_id``, SPEC §5.3).
+Hook = Callable[[dict[str, object]], Awaitable[None]]
+# A background job body: receives the arq context dict plus job-specific kwargs.
+JobFunc = Callable[..., Awaitable[Any]]
+
+# The frozen slot catalog (SPEC §11.3). Bundles are injected into these UI points.
+SLOT_NAMES: frozenset[str] = frozenset(
+    {
+        "thread.toolbar",
+        "message.actions",
+        # Inside the per-message Actions dropdown, for plugin verbs that belong
+        # with Reply/Print/Delete rather than sitting on the header as a button.
+        # The host closes the menu after a click, so a bundle here just acts.
+        "message.actions.menu",
+        "message.banner",
+        "sidebar.section",
+        "settings.page",
+    }
+)
+
+_NAME_RE = re.compile(r"[a-z][a-z0-9_]*")
+
+
+@dataclass(frozen=True)
+class Slot:
+    """A lazy-loaded ESM bundle injected into a named mail-UI slot (SPEC §11.3)."""
+
+    name: str
+    bundle: str  # path served by the plugin router, resolved by the frontend loader
+    # Human label for slots the host renders as a named area of its own — today
+    # `settings.page`, which becomes a section in Settings rather than being
+    # stacked under someone else's. Falls back to the plugin's name.
+    title: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.name not in SLOT_NAMES:
+            raise ValueError(f"unknown slot {self.name!r} (SPEC §11.3 slot catalog)")
+
+
+@dataclass(frozen=True)
+class PageNav:
+    """Nav entry for a page-plugin (SPEC §11.3)."""
+
+    icon: str
+    label: str
+    order: int = 100
+
+
+@dataclass(frozen=True)
+class Page:
+    """A full-content-area view mounted at ``route`` with a nav entry (SPEC §11.3)."""
+
+    route: str
+    nav: PageNav
+    bundle: str
+
+
+@dataclass(frozen=True)
+class UiManifest:
+    """A plugin's declared frontend surfaces (SPEC §11.3)."""
+
+    slots: list[Slot] = field(default_factory=list)
+    pages: list[Page] = field(default_factory=list)
+    capabilities: dict[str, str] = field(default_factory=dict)  # name -> endpoint path
+
+
+@dataclass(frozen=True)
+class JobSpec:
+    """A background task on the shared arq worker (SPEC §11.2).
+
+    ``schedule`` is a cron string for periodic jobs (None = enqueue on demand);
+    ``singleton`` caps it to one running instance per (plugin, user) via a Redis
+    lock. Registration onto the worker is done by the loader (M3 §11.2)."""
+
+    func: JobFunc
+    schedule: str | None = None
+    singleton: bool = False
+
+
+@dataclass(frozen=True)
+class Plugin:
+    """The object a plugin package exposes as module-level ``plugin`` (SPEC §11)."""
+
+    name: str
+    version: str
+    hooks: dict[str, Hook] = field(default_factory=dict)
+    router: APIRouter | None = None
+    ui_manifest: UiManifest | None = None
+    migrations: str | None = None  # path to the plugin's Alembic version branch (11.1)
+    jobs: list[JobSpec] = field(default_factory=list)
+    # Core plugins (calendar, contacts) ship builtin=True: always installed, never
+    # offered for removal in the UI, and the uninstall path refuses --drop-data.
+    builtin: bool = False
+    # Serves its collections to CalDAV / CardDAV clients (app/dav/provider.py
+    # DavProvider; SPEC Appendix E).
+    dav: Any = None
+
+    def __post_init__(self) -> None:
+        if not _NAME_RE.fullmatch(self.name):
+            raise ValueError(
+                f"invalid plugin name {self.name!r}: must match [a-z][a-z0-9_]* (SPEC §11)"
+            )

backend/app/plugins/vault.py

@@ -0,0 +1,21 @@
+"""Secret encryption exposed to plugins (SPEC §10.2 vault).
+
+Heavy plugins (calendar/contacts) store third-party credentials — CalDAV/CardDAV
+passwords, ICS tokens — in their own ``secret_ct bytea`` columns. They encrypt via
+this facade so those secrets are protected at rest with the same envelope scheme as
+core account credentials, without importing ``app.security`` directly.
+"""
+
+from __future__ import annotations
+
+from app.security.crypto import decrypt_secret, encrypt_secret
+
+
+class PluginVault:
+    def encrypt(self, plaintext: bytes) -> bytes:
+        """Envelope-encrypt ``plaintext`` for storage in a ``secret_ct`` column."""
+        return encrypt_secret(plaintext)
+
+    def decrypt(self, ciphertext: bytes) -> bytes:
+        """Reverse :meth:`encrypt`. Raises on a wrong/rotated master key."""
+        return decrypt_secret(ciphertext)

backend/app/providers/base.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import date, datetime
 from typing import Protocol
 
 
@@ -29,6 +29,11 @@ class ProviderConnectError(ProviderError):
     """Could not connect / TLS / greeting failure."""
 
 
+class ProviderRejectedError(ProviderError):
+    """The server refused for good (a permanent 5xx: recipient unknown, message
+    rejected). Trying again would only be refused again."""
+
+
 class ProviderNotSupported(ProviderError):
     """A capability (e.g. IDLE) is not supported by the server."""
 
@@ -46,6 +51,7 @@ class RemoteMessage:
     raw: bytes  # full RFC822
     flags: frozenset[str]
     internal_date: datetime
+    gmail_labels: tuple[str, ...] = ()  # custom Gmail labels (Gmail accounts only)
 
 
 @dataclass(frozen=True)
@@ -71,9 +77,25 @@ class MailProvider(Protocol):
     async def list_mailboxes(self) -> list[RemoteMailbox]: ...
 
     def backfill(
-        self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
+        self,
+        mailbox: str,
+        *,
+        newest_first: bool = True,
+        batch_size: int = 200,
+        since: date | None = None,
+        before: date | None = None,
     ) -> AsyncIterator[list[RemoteMessage]]:
-        """Yields batches until the mailbox is exhausted. Headers+body in one pass."""
+        """Yields batches until the mailbox is exhausted. Headers+body in one pass.
+        ``since``/``before`` bound the import by internal date (SPEC §12 M2)."""
+        ...
+
+    async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
+        """Number of messages to import (progress denominator); whole mailbox or
+        the ``since`` window."""
+        ...
+
+    async def newest_uid(self, mailbox: str) -> int:
+        """Current high-water UID; used by the ``new`` window to skip history."""
         ...
 
     async def poll(
@@ -83,9 +105,13 @@ class MailProvider(Protocol):
         otherwise UID SEARCH + FETCH FLAGS diff."""
         ...
 
+    async def list_uids(self, mailbox: str) -> set[int]:
+        """Full set of UIDs currently present in the mailbox (expunge reconciliation)."""
+        ...
+
     async def idle(self, mailbox: str) -> None:
         """Blocks until the server signals change or 25 min elapse (re-issue IDLE). May raise
-        ProviderNotSupported => sync loop falls back to poll() every POLL_INTERVAL (default 120 s).
+        ProviderNotSupported => sync loop falls back to poll() every POLL_WARM (default 180 s).
         """
         ...
 

backend/app/providers/imap_smtp/parsing.py

@@ -12,6 +12,7 @@ from dataclasses import dataclass
 from datetime import datetime
 
 from imapclient.imap_utf7 import decode as imap_utf7_decode
+from imapclient.imap_utf7 import encode as imap_utf7_encode
 
 # --- SPECIAL-USE mapping (SPEC §4 special_use values) -----------------------
 
@@ -27,6 +28,7 @@ _SPECIAL_USE_FLAGS = {
 _LIST_RE = re.compile(rb'^\(([^)]*)\)\s+(?:"([^"]*)"|NIL)\s+(.+)$')
 _UID_RE = re.compile(rb"UID\s+(\d+)")
 _FLAGS_RE = re.compile(rb"FLAGS\s+\(([^)]*)\)")
+_GM_LABELS_RE = re.compile(rb"X-GM-LABELS\s+\(([^)]*)\)")
 _INTERNALDATE_RE = re.compile(rb'INTERNALDATE\s+"([^"]+)"')
 _LITERAL_RE = re.compile(rb"\{(\d+)\}\s*$")
 _FETCH_HEADER_RE = re.compile(rb"^\d+\s+FETCH\s+\(", re.IGNORECASE)
@@ -44,6 +46,7 @@ class ParsedFetch:
     flags: frozenset[str]
     internal_date: datetime | None
     raw: bytes | None  # None when the fetch did not request the body
+    gmail_labels: tuple[str, ...] = ()  # custom Gmail labels (X-GM-LABELS), system ones dropped
 
 
 def _decode_mailbox_name(raw: bytes) -> str:
@@ -54,12 +57,29 @@ def _decode_mailbox_name(raw: bytes) -> str:
     return str(imap_utf7_decode(bytes(name)))
 
 
+def encode_mailbox_name(name: str) -> str:
+    """Modified UTF-7 wire form of a mailbox name (inverse of ``_decode_mailbox_name``).
+
+    Mailbox names are stored/displayed as Unicode but IMAP commands
+    (SELECT/STATUS/APPEND) require them mUTF-7 encoded. ASCII names are unchanged;
+    non-ASCII names — e.g. Gmail's localized ``[Gmail]/Wysłane`` — get encoded so the
+    server can find them."""
+    return str(imap_utf7_encode(name).decode("ascii"))
+
+
 def parse_list_line(line: bytes) -> ParsedMailbox | None:
-    """Parse one LIST response line into a :class:`ParsedMailbox` (or None)."""
+    """Parse one LIST response line into a :class:`ParsedMailbox` (or None).
+
+    Non-selectable mailboxes are skipped (return None): ``\\Noselect`` /
+    ``\\NonExistent`` names are pure hierarchy containers that hold no messages and
+    cannot be SELECTed — e.g. Gmail's ``[Gmail]`` root or a parent label like
+    ``Serial numbers`` that only groups child labels."""
     m = _LIST_RE.match(line.strip())
     if not m:
         return None
     flags = m.group(1).lower().split()
+    if b"\\noselect" in flags or b"\\nonexistent" in flags:
+        return None
     name = _decode_mailbox_name(m.group(3))
     special: str | None = None
     for flag in flags:
@@ -100,6 +120,57 @@ def parse_fetch_header(line: bytes) -> tuple[int | None, frozenset[str], datetim
     return uid, flags, internal
 
 
+def _tokenize_gm_labels(blob: bytes) -> list[bytes]:
+    """Split an ``X-GM-LABELS (...)`` body into raw tokens, keeping quoted strings
+    (which may contain spaces) whole. Escaped chars inside quotes are unescaped."""
+    tokens: list[bytes] = []
+    i, n = 0, len(blob)
+    while i < n:
+        ch = blob[i : i + 1]
+        if ch in (b" ", b"\t"):
+            i += 1
+            continue
+        if ch == b'"':
+            j = i + 1
+            buf = bytearray()
+            while j < n and blob[j : j + 1] != b'"':
+                if blob[j : j + 1] == b"\\" and j + 1 < n:
+                    buf += blob[j + 1 : j + 2]
+                    j += 2
+                    continue
+                buf += blob[j : j + 1]
+                j += 1
+            tokens.append(bytes(buf))
+            i = j + 1
+        else:
+            j = i
+            while j < n and blob[j : j + 1] not in (b" ", b"\t"):
+                j += 1
+            tokens.append(blob[i:j])
+            i = j
+    return tokens
+
+
+def parse_gm_labels(header: bytes) -> tuple[str, ...]:
+    """Custom Gmail labels from a FETCH header's ``X-GM-LABELS`` (SPEC: Gmail sync).
+
+    System labels (``\\Inbox``, ``\\Sent``, ``\\Important``, ``\\Starred``, …) are
+    dropped — only user labels become chips. Names are modified-UTF-7 decoded, like
+    mailbox names, so e.g. ``Zg&AAI...`` → the real Unicode label."""
+    m = _GM_LABELS_RE.search(header)
+    if not m:
+        return ()
+    out: list[str] = []
+    for tok in _tokenize_gm_labels(m.group(1)):
+        if not tok or tok.startswith(b"\\"):  # system label
+            continue
+        try:
+            out.append(str(imap_utf7_decode(tok)))
+        except Exception:  # noqa: BLE001 - fall back to a best-effort ASCII decode
+            out.append(tok.decode("utf-8", "ignore"))
+    return tuple(out)
+
+
 def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
     """Walk a full FETCH response, returning one :class:`ParsedFetch` per message.
 
@@ -131,10 +202,53 @@ def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
         else:
             i += 1
         if uid is not None:
-            results.append(ParsedFetch(uid=uid, flags=flags, internal_date=internal, raw=raw))
+            results.append(
+                ParsedFetch(
+                    uid=uid,
+                    flags=flags,
+                    internal_date=internal,
+                    raw=raw,
+                    gmail_labels=parse_gm_labels(header),
+                )
+            )
     return results
 
 
+_VANISHED_RE = re.compile(rb"^VANISHED\s+(?:\(EARLIER\)\s+)?(\S+)", re.IGNORECASE)
+
+
+def _expand_uid_set(token: bytes) -> list[int]:
+    """Expand an IMAP uid-set (``1,3:5,9``) to a flat list of ints (``*`` skipped)."""
+    out: list[int] = []
+    for part in token.split(b","):
+        part = part.strip()
+        lo_b, sep, hi_b = part.partition(b":")
+        if sep:
+            if lo_b.isdigit() and hi_b.isdigit():
+                lo, hi = int(lo_b), int(hi_b)
+                out.extend(range(min(lo, hi), max(lo, hi) + 1))
+        elif part.isdigit():
+            out.append(int(part))
+    return out
+
+
+def parse_vanished(lines: list[bytes | bytearray]) -> list[int]:
+    """UIDs from QRESYNC ``VANISHED [(EARLIER)] <uid-set>`` responses (RFC 7162).
+
+    aioimaplib strips the leading ``* `` before buffering the line, so we match on
+    the bare ``VANISHED`` token. Both the SELECT-time ``(EARLIER)`` form and the
+    live-expunge form are accepted.
+    """
+    out: list[int] = []
+    for line in lines:
+        if not isinstance(line, (bytes, bytearray)):
+            continue
+        m = _VANISHED_RE.match(bytes(line).strip())
+        if m:
+            out.extend(_expand_uid_set(m.group(1)))
+    return out
+
+
 def parse_search_uids(lines: list[bytes | bytearray]) -> list[int]:
     """Parse a UID SEARCH response's first line into a list of ints."""
     if not lines:

backend/app/providers/imap_smtp/provider.py

@@ -11,34 +11,125 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import re
+import ssl
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
+from datetime import date
 
 import aiosmtplib
 from aioimaplib import aioimaplib
 
+from app.concurrency import loop_semaphore
+from app.config import get_settings, smtp_helo_name
 from app.providers.base import (
     FlagChange,
     MailboxDelta,
     MailProvider,
     ProviderAuthError,
     ProviderConnectError,
+    ProviderError,
     ProviderNotSupported,
+    ProviderRejectedError,
     RemoteMailbox,
     RemoteMessage,
 )
 from app.providers.imap_smtp.parsing import (
+    encode_mailbox_name,
     parse_fetch_lines,
     parse_list_line,
     parse_search_uids,
     parse_select_untagged,
+    parse_vanished,
 )
 
 _FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
-_IDLE_TIMEOUT = 25 * 60  # SPEC §5.1: re-issue IDLE at least every 25 min
+# Gmail (X-GM-EXT-1): also pull each message's labels so custom labels can map to
+# chips without visiting every label folder (SPEC: Gmail sync / no-duplication).
+_FETCH_ITEMS_GM = "(UID FLAGS INTERNALDATE RFC822.SIZE X-GM-LABELS BODY.PEEK[])"
+# SPEC §5.1 asks for IDLE to be re-issued at least every 25 min; 10 also means a
+# connection that died silently (no FIN, e.g. the network dropped) is noticed
+# within ~11 min — the re-issue finds it — instead of ~29.
+_IDLE_TIMEOUT = 10 * 60
 _CONNECT_TIMEOUT = 20
+_PER_ACCOUNT_CONN_CAP = 4  # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
 _APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
 _COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")
+_IMAP_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
+
+
+class _KeepConnectTask:
+    """aioimaplib starts the TCP connection as a task it never awaits: a refused
+    or unreachable server then surfaces only as a 20 s greeting timeout, plus a
+    "Task exception was never retrieved" traceback in the log. Keep the task, so
+    ``_connect`` can await it — failing at once, with the real reason."""
+
+    connect_task: asyncio.Task[object]
+
+    def create_client(  # type: ignore[no-untyped-def]
+        self, host, port, loop, conn_lost_cb=None, ssl_context=None
+    ) -> None:
+        # This replaces IMAP4_SSL.create_client too (it comes first in the MRO),
+        # so do its one job here: without a context the connection is plain
+        # TCP to port 993, and the server just waits for a TLS handshake.
+        if ssl_context is None and isinstance(self, aioimaplib.IMAP4_SSL):
+            ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+        local_loop = loop if loop is not None else asyncio.get_running_loop()
+        self.protocol = aioimaplib.IMAP4ClientProtocol(local_loop, conn_lost_cb)
+        self.connect_task = local_loop.create_task(
+            local_loop.create_connection(lambda: self.protocol, host, port, ssl=ssl_context)
+        )
+
+
+class _IMAP4(_KeepConnectTask, aioimaplib.IMAP4):  # type: ignore[misc]
+    pass
+
+
+class _IMAP4_SSL(_KeepConnectTask, aioimaplib.IMAP4_SSL):  # type: ignore[misc]  # noqa: N801
+    pass
+
+
+def _smtp_error(exc: aiosmtplib.SMTPException) -> ProviderError:
+    """An SMTP failure as the provider error that says whether to try again.
+
+    A 5xx reply is the server's final word (recipient unknown, message refused):
+    ``ProviderRejectedError``, or ``ProviderAuthError`` for a refused login. Any
+    4xx reply, a timeout or a dropped connection is temporary:
+    ``ProviderConnectError``, and the send is retried later.
+    """
+    if isinstance(exc, aiosmtplib.SMTPRecipientsRefused):
+        codes = [r.code for r in exc.recipients]
+        if codes and all(c >= 500 for c in codes):
+            return ProviderRejectedError(f"recipients refused: {exc}")
+        return ProviderConnectError(f"SMTP send failed: {exc}")
+    if isinstance(exc, aiosmtplib.SMTPResponseException) and exc.code >= 500:
+        if isinstance(exc, aiosmtplib.SMTPAuthenticationError):
+            return ProviderAuthError(f"SMTP auth failed: {exc}")
+        return ProviderRejectedError(f"SMTP refused: {exc}")
+    return ProviderConnectError(f"SMTP send failed: {exc}")
+
+
+def _imap_date(d: date) -> str:
+    """RFC 3501 search date, e.g. ``5-Jul-2026`` (locale-independent)."""
+    return f"{d.day}-{_IMAP_MONTHS[d.month - 1]}-{d.year}"
+
+
+def _date_criteria(since: date | None, before: date | None) -> str:
+    parts: list[str] = []
+    if since is not None:
+        parts.append(f"SINCE {_imap_date(since)}")
+    if before is not None:
+        parts.append(f"BEFORE {_imap_date(before)}")
+    return " ".join(parts) if parts else "ALL"
+
+
+async def _require_public(host: str) -> None:
+    """Refuse a user-named server that now resolves to a non-public address."""
+    from app.accounts.ssrf import SsrfError, assert_host_is_public
+
+    try:
+        await assert_host_is_public(host)
+    except SsrfError as exc:
+        raise ProviderConnectError(f"refusing to connect to {host}: {exc}") from exc
 
 
 @dataclass(frozen=True)
@@ -51,6 +142,16 @@ class ProviderConfig:
     smtp_tls: str  # "ssl" | "starttls" | "plain"
     username: str
     password: str  # plaintext; caller decrypts per §10.2
+    # TLS settings other than the default (certificate checked): the hosted
+    # mail server on the internal network (app/hosted/native.py).
+    tls_context: ssl.SSLContext | None = None
+    # The hosted mail server: every user's mailbox is on that one host, so the
+    # per-host connection cap (which spares outside providers) does not apply.
+    internal: bool = False
+    # A server the user named (every external account): its name is checked
+    # again before each connection, so DNS changed after the account was added
+    # cannot turn the worker on the internal network (DNS rebinding).
+    public_only: bool = False
 
 
 class ImapSmtpProvider(MailProvider):
@@ -59,40 +160,122 @@ class ImapSmtpProvider(MailProvider):
         self._imap: aioimaplib.IMAP4 | None = None
         self._selected: str | None = None
         self._uidvalidity_seen: dict[str, int] = {}
+        self._conn_slots: list[asyncio.Semaphore] = []
+        self._qresync_enabled = False
+        self._gm_ext = False
+
+    @property
+    def gmail_labels_supported(self) -> bool:
+        """True when the server advertises ``X-GM-EXT-1`` (Gmail), so FETCH can pull
+        ``X-GM-LABELS`` and the engine can use the no-duplication Gmail sync path."""
+        return self._gm_ext
+
+    @property
+    def qresync_enabled(self) -> bool:
+        """True once ``ENABLE QRESYNC`` succeeded on this connection (RFC 7162).
+
+        When set, :meth:`poll` reports server-side expunges via VANISHED; otherwise
+        deletion detection falls back to the engine's full-UID reconcile."""
+        return self._qresync_enabled
 
     # --- connection lifecycle ------------------------------------------------
 
+    async def _acquire_conn_slots(self) -> None:
+        """Bound simultaneous open sockets: ≤4 per account and ≤cap per host (§4).
+
+        Held for the connection's whole lifetime and released in :meth:`close`.
+        """
+        cfg = self._cfg
+        account = loop_semaphore(
+            "imap_account", f"{cfg.username}@{cfg.imap_host}", _PER_ACCOUNT_CONN_CAP
+        )
+        await account.acquire()  # consistent order (account, then host) avoids deadlock
+        if cfg.internal:
+            self._conn_slots = [account]
+            return
+        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
+        try:
+            await host.acquire()
+        except BaseException:
+            account.release()
+            raise
+        self._conn_slots = [host, account]
+
+    def _release_conn_slots(self) -> None:
+        for sem in self._conn_slots:
+            sem.release()
+        self._conn_slots = []
+
     async def _connect(self) -> aioimaplib.IMAP4:
         if self._imap is not None:
             return self._imap
         cfg = self._cfg
+        if cfg.public_only:
+            await _require_public(cfg.imap_host)
+        await self._acquire_conn_slots()
         try:
+            try:
+                imap: aioimaplib.IMAP4
                 if cfg.imap_tls == "ssl":
-                imap = aioimaplib.IMAP4_SSL(
-                    host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
+                    imap = _IMAP4_SSL(
+                        host=cfg.imap_host,
+                        port=cfg.imap_port,
+                        timeout=_CONNECT_TIMEOUT,
+                        ssl_context=cfg.tls_context,
                     )
                 else:
-                imap = aioimaplib.IMAP4(
-                    host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
-                )
+                    imap = _IMAP4(host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT)
+                # Refused / unknown host: fails here at once. No answer at all:
+                # gives up after the timeout (the connect attempt is cancelled).
+                await asyncio.wait_for(imap.connect_task, _CONNECT_TIMEOUT)
                 await imap.wait_hello_from_server()
                 if cfg.imap_tls == "starttls":
                     await imap.starttls()
             except (OSError, TimeoutError) as exc:
-            raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc
+                why = str(exc) or f"no answer within {_CONNECT_TIMEOUT} s"
+                raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {why}") from exc
 
             resp = await imap.login(cfg.username, cfg.password)
             if resp.result != "OK":
                 detail = (
-                b" ".join(resp.lines).decode("utf-8", "ignore") if resp.lines else "login failed"
+                    b" ".join(resp.lines).decode("utf-8", "ignore")
+                    if resp.lines
+                    else "login failed"
                 )
                 raise ProviderAuthError(detail)
+        except BaseException:
+            self._release_conn_slots()  # never hold a slot without an open socket
+            raise
         self._imap = imap
+        caps = {c.upper() for c in (imap.protocol.capabilities or set())}
+        self._gm_ext = "X-GM-EXT-1" in caps
+        await self._maybe_enable_qresync(imap)
         return imap
 
+    async def _maybe_enable_qresync(self, imap: aioimaplib.IMAP4) -> None:
+        """Enable QRESYNC when advertised so SELECT reports VANISHED (RFC 7162).
+
+        Best-effort: servers without QRESYNC (e.g. Gmail, which is CONDSTORE-only)
+        simply keep ``_qresync_enabled`` False and use the engine's reconcile path."""
+        caps = {c.upper() for c in (imap.protocol.capabilities or set())}
+        if "QRESYNC" not in caps:
+            return
+        try:
+            resp = await imap.enable("QRESYNC")
+            self._qresync_enabled = resp.result == "OK"
+        except Exception:  # noqa: BLE001 - degrade gracefully to the reconcile path
+            self._qresync_enabled = False
+
+    @staticmethod
+    def _wire(mailbox: str) -> str:
+        """Server wire form of a mailbox name: mUTF-7 encoded and quoted. Handles
+        non-ASCII names (Gmail's localized ``[Gmail]/Wysłane``) and names with
+        spaces (``[Gmail]/All Mail``); ASCII names are unchanged bar the quotes."""
+        return str(aioimaplib.quoted(encode_mailbox_name(mailbox)))
+
     async def _select(self, mailbox: str) -> dict[str, int]:
         imap = await self._connect()
-        resp = await imap.select(mailbox)
+        resp = await imap.select(self._wire(mailbox))
         if resp.result != "OK":
             raise ProviderConnectError(f"cannot select {mailbox!r}")
         self._selected = mailbox
@@ -104,6 +287,7 @@ class ImapSmtpProvider(MailProvider):
                 await self._imap.logout()
             self._imap = None
             self._selected = None
+            self._release_conn_slots()  # free the per-account / per-host slots
 
     # --- MailProvider protocol ----------------------------------------------
 
@@ -136,7 +320,7 @@ class ImapSmtpProvider(MailProvider):
     async def _mailbox_uidvalidity(self, path: str) -> int:
         imap = await self._connect()
         try:
-            resp = await imap.status(path, "(UIDVALIDITY)")
+            resp = await imap.status(self._wire(path), "(UIDVALIDITY)")
         except Exception:
             return 0
         if resp.result != "OK":
@@ -149,12 +333,21 @@ class ImapSmtpProvider(MailProvider):
         return 0
 
     async def backfill(
-        self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
+        self,
+        mailbox: str,
+        *,
+        newest_first: bool = True,
+        batch_size: int = 200,
+        since: date | None = None,
+        before: date | None = None,
     ) -> AsyncIterator[list[RemoteMessage]]:
+        """Yield message batches. ``since``/``before`` bound the import by IMAP
+        internal date (``SINCE`` inclusive, ``BEFORE`` exclusive) so the caller
+        can sync a recent window and/or the older remainder (SPEC §12 M2)."""
         info = await self._select(mailbox)
         self._uidvalidity_seen[mailbox] = info.get("uidvalidity", 0)
         imap = await self._connect()
-        search = await imap.uid_search("ALL")
+        search = await imap.uid_search(_date_criteria(since, before))
         uids = parse_search_uids(search.lines)
         uids.sort(reverse=newest_first)
         for start in range(0, len(uids), batch_size):
@@ -163,11 +356,49 @@ class ImapSmtpProvider(MailProvider):
             if messages:
                 yield messages
 
+    async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
+        """Number of messages to import for the progress denominator. Uses the
+        cheap ``EXISTS`` from SELECT for the whole mailbox, or a dated SEARCH."""
+        info = await self._select(mailbox)
+        if since is None:
+            return int(info.get("exists", 0))
+        imap = await self._connect()
+        search = await imap.uid_search(_date_criteria(since, None))
+        return len(parse_search_uids(search.lines))
+
+    async def newest_uid(self, mailbox: str) -> int:
+        """Current high-water UID (``UIDNEXT - 1``); 0 for an empty mailbox. Used
+        by the ``new`` sync window to skip history and only pull future mail."""
+        info = await self._select(mailbox)
+        return max(0, int(info.get("uidnext", 1)) - 1)
+
+    async def list_uids(self, mailbox: str) -> set[int]:
+        """Full set of UIDs currently present in the mailbox (expunge reconciliation).
+
+        Cheap — a single ``UID SEARCH ALL``. The CONDSTORE poll only reports new and
+        changed messages, not vanished ones (that needs QRESYNC/VANISHED); callers use
+        this to detect server-side expunges for small folders like Drafts.
+        """
+        await self._select(mailbox)
+        imap = await self._connect()
+        resp = await imap.uid_search("ALL")
+        return set(parse_search_uids(resp.lines))
+
+    async def fetch_raw(self, mailbox: str, uid: int) -> bytes | None:
+        """One message's raw bytes, or None if it is not (or no longer) there."""
+        await self._select(mailbox)
+        imap = await self._connect()
+        for message in await self._fetch_messages(imap, [uid]):
+            if message.uid == uid:
+                return message.raw
+        return None
+
     async def _fetch_messages(self, imap: aioimaplib.IMAP4, uids: list[int]) -> list[RemoteMessage]:
         if not uids:
             return []
         uid_set = ",".join(str(u) for u in uids)
-        resp = await imap.uid("fetch", uid_set, _FETCH_ITEMS)
+        items = _FETCH_ITEMS_GM if self._gm_ext else _FETCH_ITEMS
+        resp = await imap.uid("fetch", uid_set, items)
         if resp.result != "OK":
             raise ProviderConnectError("FETCH failed")
         out: list[RemoteMessage] = []
@@ -176,7 +407,11 @@ class ImapSmtpProvider(MailProvider):
                 continue
             out.append(
                 RemoteMessage(
-                    uid=pf.uid, raw=pf.raw, flags=pf.flags, internal_date=pf.internal_date
+                    uid=pf.uid,
+                    raw=pf.raw,
+                    flags=pf.flags,
+                    internal_date=pf.internal_date,
+                    gmail_labels=pf.gmail_labels,
                 )
             )
         return out
@@ -199,26 +434,53 @@ class ImapSmtpProvider(MailProvider):
         search = await imap.uid_search(f"UID {last_seen_uid + 1}:*")
         candidate_uids = [u for u in parse_search_uids(search.lines) if u > last_seen_uid]
         new_messages = await self._fetch_messages(imap, sorted(candidate_uids))
+        new_uid_set = {m.uid for m in new_messages}
 
-        # Flag changes via CONDSTORE when we have a modseq baseline.
         flag_changes: list[FlagChange] = []
-        if highestmodseq is not None:
+        expunged_uids: list[int] = []
+        if highestmodseq is not None and self._qresync_enabled:
+            # QRESYNC fast resync (RFC 7162): a re-SELECT with (QRESYNC (uidvalidity
+            # modseq)) makes the server report, in one pass, the FLAGS changed and the
+            # UIDs VANISHED since ``modseq``. VANISHED is only surfaced on a *sync*
+            # command, so we harvest both from the SELECT response here.
+            qr = await self._qresync_select(mailbox, current_uidvalidity, highestmodseq)
+            expunged_uids = [u for u in parse_vanished(qr.lines) if u not in new_uid_set]
+            for pf in parse_fetch_lines(qr.lines):
+                if pf.uid not in new_uid_set:
+                    flag_changes.append(FlagChange(uid=pf.uid, flags=pf.flags))
+        elif highestmodseq is not None:
+            # CONDSTORE-only server (e.g. Gmail): flag delta via CHANGEDSINCE. There
+            # is no VANISHED here — the engine reconciles expunges by full-UID diff.
             fresp = await imap.uid("fetch", "1:*", f"(UID FLAGS) (CHANGEDSINCE {highestmodseq})")
             if fresp.result == "OK":
-                new_uid_set = {m.uid for m in new_messages}
                 for pf in parse_fetch_lines(fresp.lines):
                     if pf.uid not in new_uid_set:
                         flag_changes.append(FlagChange(uid=pf.uid, flags=pf.flags))
 
-        # TODO(spec): expunge detection needs QRESYNC/VANISHED (M2 hardening).
         return MailboxDelta(
             new=new_messages,
             flag_changes=flag_changes,
-            expunged_uids=[],
+            expunged_uids=expunged_uids,
             new_uidvalidity=None,
             new_highestmodseq=info.get("highestmodseq"),
         )
 
+    async def _qresync_select(
+        self, mailbox: str, uidvalidity: int, modseq: int
+    ) -> aioimaplib.Response:
+        """Raw ``SELECT <mailbox> (QRESYNC (uidvalidity modseq))`` (RFC 7162).
+
+        Issued as a low-level command because aioimaplib's high-level ``select``
+        takes no parameters, and because SELECT is a *sync* command — the only kind
+        whose untagged responses (VANISHED, changed FETCH) aioimaplib buffers."""
+        imap = await self._connect()
+        proto = imap.protocol
+        param = f"(QRESYNC ({uidvalidity} {modseq}))"
+        cmd = aioimaplib.Command(
+            "SELECT", proto.new_tag(), self._wire(mailbox), param, loop=proto.loop
+        )
+        return await proto.execute(cmd)
+
     async def idle(self, mailbox: str) -> None:
         imap = await self._connect()
         if "IDLE" not in (imap.protocol.capabilities or set()):
@@ -226,7 +488,10 @@ class ImapSmtpProvider(MailProvider):
         await self._select(mailbox)
         idle_task = await imap.idle_start(timeout=_IDLE_TIMEOUT)
         try:
-            await imap.wait_server_push()
+            # No word from the server even after IDLE's own end: just return, and
+            # the caller's poll shows whether the connection is still there.
+            with contextlib.suppress(TimeoutError):
+                await imap.wait_server_push(timeout=_IDLE_TIMEOUT + 60)
         finally:
             imap.idle_done()
             with contextlib.suppress(Exception):
@@ -246,12 +511,12 @@ class ImapSmtpProvider(MailProvider):
         """Create a mailbox on the server (idempotent — ignores 'already exists')."""
         imap = await self._connect()
         with contextlib.suppress(Exception):
-            await imap.create(path)
+            await imap.create(self._wire(path))
 
     async def move(self, uid: int, src: str, dst: str) -> int:
         await self._select(src)
         imap = await self._connect()
-        resp = await imap.uid("move", str(uid), dst)
+        resp = await imap.uid("move", str(uid), self._wire(dst))
         if resp.result != "OK":
             raise ProviderConnectError(f"MOVE {uid} -> {dst} failed")
         for line in resp.lines:
@@ -279,7 +544,7 @@ class ImapSmtpProvider(MailProvider):
     async def append(self, mailbox: str, raw: bytes, flags: frozenset[str]) -> int:
         imap = await self._connect()
         flag_str = f"({' '.join(sorted(flags))})" if flags else None
-        resp = await imap.append(raw, mailbox=mailbox, flags=flag_str)
+        resp = await imap.append(raw, mailbox=self._wire(mailbox), flags=flag_str)
         if resp.result != "OK":
             raise ProviderConnectError(f"APPEND to {mailbox} failed")
         for line in resp.lines:
@@ -291,9 +556,17 @@ class ImapSmtpProvider(MailProvider):
 
     async def send(self, raw: bytes, envelope_from: str, envelope_to: list[str]) -> None:
         cfg = self._cfg
+        if cfg.public_only:
+            await _require_public(cfg.smtp_host)
         use_tls = cfg.smtp_tls == "ssl"
         start_tls = cfg.smtp_tls == "starttls"
+        host_slot = (
+            contextlib.nullcontext()
+            if cfg.internal
+            else loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
+        )
         try:
+            async with host_slot:  # bound simultaneous sockets to the SMTP host (§4)
                 await aiosmtplib.send(
                     raw,
                     sender=envelope_from,
@@ -304,9 +577,10 @@ class ImapSmtpProvider(MailProvider):
                     password=cfg.password,
                     use_tls=use_tls,
                     start_tls=start_tls,
+                    tls_context=cfg.tls_context,
                     timeout=_CONNECT_TIMEOUT,
+                    # Our own name, not the container's random host name.
+                    local_hostname=smtp_helo_name(),
                 )
-        except aiosmtplib.SMTPAuthenticationError as exc:
-            raise ProviderAuthError(f"SMTP auth failed: {exc}") from exc
         except aiosmtplib.SMTPException as exc:
-            raise ProviderConnectError(f"SMTP send failed: {exc}") from exc
+            raise _smtp_error(exc) from exc

backend/app/push/__init__.py

@@ -0,0 +1 @@
+"""Push notifications about new mail (Web Push)."""

backend/app/push/service.py

@@ -0,0 +1,295 @@
+"""Push notifications about new mail: the server's keys, the devices, and the job.
+
+New Inbox mail found by the live sync enqueues :func:`push_new_mail_job` a few
+seconds later, so filters and the spam check have moved what they move by then;
+the job looks again at each message and tells the user's devices only about
+mail that is still unread in an Inbox, recent, and not from the user.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Any
+
+import httpx
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.identity.sessions import session_alive
+from app.models import Account, AppSettings, Mailbox, Message, PushSubscription
+from app.push import webpush
+from app.security.crypto import decrypt_secret, encrypt_secret
+
+logger = logging.getLogger(__name__)
+
+# How long after new Inbox mail is found the job runs (filters, spam check).
+NOTIFY_DELAY_S = 5
+# Mail older than this is not news: a message moved back into the Inbox, or a
+# backlog after the server was unreachable for a while.
+MAX_AGE = dt.timedelta(hours=2)
+# More new messages than this at once become a single "N new messages".
+MAX_SINGLE = 3
+SNIPPET_CHARS = 140
+
+
+# ---- the server's key pair ------------------------------------------------------------
+
+
+async def vapid(db: AsyncSession) -> webpush.Vapid:
+    """The server's VAPID key pair, made (once) on first use."""
+    row = await db.get(AppSettings, 1)
+    if row is not None and row.vapid_private_ct:
+        return webpush.Vapid.from_pem(decrypt_secret(row.vapid_private_ct))
+    row = await db.scalar(select(AppSettings).where(AppSettings.id == 1).with_for_update())
+    if row is None:
+        row = AppSettings(id=1)
+        db.add(row)
+    elif row.vapid_private_ct:  # made by another request meanwhile
+        return webpush.Vapid.from_pem(decrypt_secret(row.vapid_private_ct))
+    key = webpush.Vapid.generate()
+    row.vapid_private_ct = encrypt_secret(key.pem())
+    row.vapid_public = key.public_b64
+    await db.commit()
+    return key
+
+
+def _subject(row: AppSettings | None) -> str:
+    """Who to contact about our pushes (the VAPID ``sub``): the app's address
+    when it is set and https, else the postmaster of the app's domain."""
+    url = row.public_url if row is not None else ""
+    if url.startswith("https://"):
+        return url
+    return f"mailto:postmaster@{get_settings().app_domain}"
+
+
+# ---- devices ------------------------------------------------------------------------
+
+
+async def subscribe(
+    db: AsyncSession,
+    *,
+    user_id: uuid.UUID,
+    session_sid: str,
+    endpoint: str,
+    p256dh: str,
+    auth: str,
+    user_agent: str,
+) -> None:
+    """Remember a device (or refresh it). One row per endpoint: a browser that
+    is now signed in as someone else is theirs from now on."""
+    row = await db.scalar(select(PushSubscription).where(PushSubscription.endpoint == endpoint))
+    if row is None:
+        row = PushSubscription(endpoint=endpoint)
+        db.add(row)
+    row.user_id = user_id
+    row.session_sid = session_sid
+    row.p256dh = p256dh
+    row.auth = auth
+    row.user_agent = user_agent[:256]
+    await db.commit()
+
+
+async def unsubscribe(db: AsyncSession, *, user_id: uuid.UUID, endpoint: str) -> None:
+    await db.execute(
+        delete(PushSubscription).where(
+            PushSubscription.user_id == user_id, PushSubscription.endpoint == endpoint
+        )
+    )
+    await db.commit()
+
+
+# ---- what to say --------------------------------------------------------------------
+
+
+def _sender(msg: Message) -> str:
+    first = (msg.from_addr or [{}])[0]
+    return (first.get("name") or first.get("email") or "Unknown sender").strip()
+
+
+def _payloads(messages: list[Message], unread: int) -> list[dict[str, object]]:
+    """One notification per message, or one summary when there are many."""
+    if len(messages) > MAX_SINGLE:
+        senders: list[str] = []
+        for m in messages:
+            name = _sender(m)
+            if name not in senders:
+                senders.append(name)
+        shown = ", ".join(senders[:3]) + ("…" if len(senders) > 3 else "")
+        return [
+            {
+                "type": "mail",
+                "title": f"{len(messages)} new messages",
+                "body": shown,
+                "tag": "new-mail",
+                "url": "/mail",
+                "unread": unread,
+            }
+        ]
+    out: list[dict[str, object]] = []
+    for m in messages:
+        subject = m.subject.strip() or "(no subject)"
+        snippet = " ".join(m.snippet.split())[:SNIPPET_CHARS]
+        out.append(
+            {
+                "type": "mail",
+                "title": _sender(m),
+                "body": f"{subject}\n{snippet}" if snippet else subject,
+                "tag": f"thread-{m.thread_id}",
+                "url": f"/mail/t/{m.thread_id}" if m.thread_id else "/mail",
+                "thread_id": str(m.thread_id) if m.thread_id else None,
+                "unread": unread,
+            }
+        )
+    return out
+
+
+async def _news(db: AsyncSession, user_id: uuid.UUID, ids: list[uuid.UUID]) -> list[Message]:
+    """The messages still worth a notification, oldest first."""
+    own = {
+        e.lower() for e in await db.scalars(select(Account.email).where(Account.user_id == user_id))
+    }
+    rows = await db.scalars(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.user_id == user_id,
+            Message.id.in_(ids),
+            Mailbox.special_use == "inbox",
+            ~Message.flags.contains(["\\Seen"]),
+            Message.internal_date >= dt.datetime.now(dt.UTC) - MAX_AGE,
+        )
+        .order_by(Message.internal_date)
+    )
+    out: list[Message] = []
+    seen: set[str] = set()
+    for m in rows:
+        sender = ((m.from_addr or [{}])[0].get("email") or "").lower()
+        if sender in own:
+            continue  # sent to yourself, or a copy of your own message
+        key = m.delivery_id or str(m.id)  # one email that reached two accounts
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(m)
+    return out
+
+
+async def _unread(db: AsyncSession, user_id: uuid.UUID) -> int:
+    """Unread Inbox messages, for the app icon's badge."""
+    count = await db.scalar(
+        select(func.count())
+        .select_from(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.user_id == user_id,
+            Mailbox.special_use == "inbox",
+            ~Message.flags.contains(["\\Seen"]),
+        )
+    )
+    return int(count or 0)
+
+
+# ---- the job ------------------------------------------------------------------------
+
+
+async def notify(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> int:
+    """Tell the user's signed-in devices about ``message_ids``; returns how many
+    pushes were delivered to push services."""
+    from app.db import session_scope
+
+    async with session_scope() as db:
+        if not await db.scalar(
+            select(PushSubscription.id).where(PushSubscription.user_id == user_id).limit(1)
+        ):
+            return 0
+        messages = await _news(db, user_id, message_ids)
+        if not messages:
+            return 0
+        payloads = _payloads(messages, await _unread(db, user_id))
+    return await deliver(user_id, payloads)
+
+
+async def deliver(user_id: uuid.UUID, payloads: list[dict[str, object]]) -> int:
+    """Push ``payloads`` to every device of the user that is still signed in
+    (for new mail, and for plugins: calendar reminders); returns how many were
+    handed to push services. Devices that are gone are forgotten."""
+    from app.db import session_scope
+
+    if not payloads:
+        return 0
+    async with session_scope() as db:
+        # Plain values: the rows expire when the key pair is first saved below.
+        subs = [
+            (row.id, row.endpoint, row.p256dh, row.auth, row.session_sid)
+            for row in await db.scalars(
+                select(PushSubscription).where(PushSubscription.user_id == user_id)
+            )
+        ]
+        if not subs:
+            return 0
+        key = await vapid(db)
+        subject = _subject(await db.get(AppSettings, 1))
+
+    sent = 0
+    gone: list[uuid.UUID] = []
+    used: list[uuid.UUID] = []
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        for sub_id, endpoint, p256dh, auth, sid in subs:
+            if not await session_alive(user_id, sid):
+                gone.append(sub_id)  # signed out: this device hears nothing more
+                continue
+            try:
+                for payload in payloads:
+                    await webpush.send(
+                        client,
+                        endpoint=endpoint,
+                        p256dh=p256dh,
+                        auth=auth,
+                        payload=payload,
+                        vapid=key,
+                        subject=subject,
+                    )
+                    sent += 1
+                used.append(sub_id)
+            except webpush.Gone:
+                gone.append(sub_id)
+            except webpush.Failed as err:
+                logger.warning("push to %s failed: %s", endpoint.split("/")[2], err)
+
+    if gone or used:
+        async with session_scope() as db:
+            if gone:
+                await db.execute(delete(PushSubscription).where(PushSubscription.id.in_(gone)))
+            if used:
+                await db.execute(
+                    update(PushSubscription)
+                    .where(PushSubscription.id.in_(used))
+                    .values(last_used_at=func.now())
+                )
+    return sent
+
+
+async def push_new_mail_job(ctx: dict[str, Any], user_id: str, message_ids: list[str]) -> int:
+    """arq job: notify about new Inbox mail (enqueued by the live sync)."""
+    return await notify(uuid.UUID(user_id), [uuid.UUID(m) for m in message_ids])
+
+
+async def enqueue_new_mail(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> None:
+    """Called by the live sync with new Inbox mail. Never raises: a missing
+    notification must not stop the sync."""
+    if not message_ids:
+        return
+    from app.queue import enqueue_job
+
+    try:
+        await enqueue_job(
+            "push_new_mail_job",
+            str(user_id),
+            [str(m) for m in message_ids],
+            _defer_by=NOTIFY_DELAY_S,
+        )
+    except Exception:  # noqa: BLE001 - best effort
+        logger.warning("could not enqueue new-mail push for user %s", user_id, exc_info=True)

backend/app/push/webpush.py

@@ -0,0 +1,193 @@
+"""Web Push, the protocol (RFC 8030 delivery, RFC 8291 encryption, RFC 8292 VAPID).
+
+A browser that wants notifications gives us a push service URL (Google's,
+Mozilla's, Apple's, Microsoft's) and two keys. A message is encrypted to those
+keys, so the push service only carries it, and signed with our own key pair
+(VAPID), so the push service knows it comes from the server the browser
+subscribed to. Written here rather than taken from pywebpush, which needs a
+newer ``cryptography`` than the rest of the backend pins; it is small, and
+``tests/push`` checks it against the RFC 8291 worked example.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import time
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+import httpx
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+RECORD_SIZE = 4096
+# Push services only: the endpoint comes from the browser, i.e. from the user,
+# and the server posts to it — anything else would let a user aim the server at
+# an internal address (SSRF). Every browser in use delivers through one of these.
+PUSH_HOSTS = (
+    "fcm.googleapis.com",  # Chrome, Edge on Android, Opera, Samsung Internet
+    "android.googleapis.com",
+    ".google.com",  # Chromium builds subscribe at jmt17.google.com and the like
+    ".push.services.mozilla.com",  # Firefox
+    ".push.apple.com",  # Safari, and installed apps on iPhone/iPad
+    ".notify.windows.com",  # Edge on Windows
+)
+
+
+def b64u_decode(value: str) -> bytes:
+    return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
+
+
+def b64u_encode(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
+
+
+def allowed_endpoint(endpoint: str) -> bool:
+    """Whether ``endpoint`` is an https URL of a known push service."""
+    parts = urlsplit(endpoint)
+    host = (parts.hostname or "").lower()
+    if parts.scheme != "https" or not host or parts.port not in (None, 443):
+        return False
+    return any(host == h or (h.startswith(".") and host.endswith(h)) for h in PUSH_HOSTS)
+
+
+def _public_bytes(key: ec.EllipticCurvePrivateKey) -> bytes:
+    return key.public_key().public_bytes(
+        serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint
+    )
+
+
+def _hkdf(salt: bytes, ikm: bytes, info: bytes, length: int) -> bytes:
+    return HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
+
+
+def encrypt(
+    payload: bytes,
+    p256dh: str,
+    auth: str,
+    *,
+    salt: bytes | None = None,
+    sender: ec.EllipticCurvePrivateKey | None = None,
+) -> bytes:
+    """``payload`` encrypted for one browser (RFC 8291, ``aes128gcm``), as the
+    request body: header (salt, record size, our one-off public key) + one
+    record. ``salt`` and ``sender`` are for the RFC test vector only."""
+    ua_public = b64u_decode(p256dh)
+    auth_secret = b64u_decode(auth)
+    sender = sender or ec.generate_private_key(ec.SECP256R1())
+    as_public = _public_bytes(sender)
+    ua_key = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), ua_public)
+    ecdh_secret = sender.exchange(ec.ECDH(), ua_key)
+    ikm = _hkdf(auth_secret, ecdh_secret, b"WebPush: info\x00" + ua_public + as_public, 32)
+    salt = salt or os.urandom(16)
+    cek = _hkdf(salt, ikm, b"Content-Encoding: aes128gcm\x00", 16)
+    nonce = _hkdf(salt, ikm, b"Content-Encoding: nonce\x00", 12)
+    # One record, no padding: 0x02 marks the last (only) record.
+    record = AESGCM(cek).encrypt(nonce, payload + b"\x02", None)
+    header = salt + RECORD_SIZE.to_bytes(4, "big") + bytes([len(as_public)]) + as_public
+    return header + record
+
+
+# ---- VAPID (RFC 8292) ---------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class Vapid:
+    """The server's signing key pair."""
+
+    private: ec.EllipticCurvePrivateKey
+
+    @classmethod
+    def generate(cls) -> Vapid:
+        return cls(ec.generate_private_key(ec.SECP256R1()))
+
+    @classmethod
+    def from_pem(cls, pem: bytes) -> Vapid:
+        key = serialization.load_pem_private_key(pem, password=None)
+        if not isinstance(key, ec.EllipticCurvePrivateKey):
+            raise ValueError("VAPID key is not an EC key")
+        return cls(key)
+
+    def pem(self) -> bytes:
+        return self.private.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.PKCS8,
+            serialization.NoEncryption(),
+        )
+
+    @property
+    def public_b64(self) -> str:
+        """The public key as browsers take it (``applicationServerKey``)."""
+        return b64u_encode(_public_bytes(self.private))
+
+    def authorization(self, endpoint: str, subject: str, *, ttl: int = 12 * 3600) -> str:
+        """The ``Authorization`` header for a push to ``endpoint``: a JWT (ES256)
+        naming the push service's origin, with our public key."""
+        parts = urlsplit(endpoint)
+        claims = {
+            "aud": f"{parts.scheme}://{parts.netloc}",
+            "exp": int(time.time()) + ttl,
+            "sub": subject,
+        }
+        header = {"typ": "JWT", "alg": "ES256"}
+        signing_input = (
+            b64u_encode(json.dumps(header, separators=(",", ":")).encode())
+            + "."
+            + b64u_encode(json.dumps(claims, separators=(",", ":")).encode())
+        )
+        der = self.private.sign(signing_input.encode(), ec.ECDSA(hashes.SHA256()))
+        r, s = decode_dss_signature(der)
+        signature = r.to_bytes(32, "big") + s.to_bytes(32, "big")
+        return f"vapid t={signing_input}.{b64u_encode(signature)}, k={self.public_b64}"
+
+
+# ---- delivery (RFC 8030) ------------------------------------------------------------
+
+
+class Gone(Exception):
+    """The subscription no longer exists (404/410): forget it."""
+
+
+class Failed(Exception):
+    """The push service refused or could not be reached; worth trying later."""
+
+
+async def send(
+    client: httpx.AsyncClient,
+    *,
+    endpoint: str,
+    p256dh: str,
+    auth: str,
+    payload: dict[str, object],
+    vapid: Vapid,
+    subject: str,
+    ttl: int = 24 * 3600,
+    urgency: str = "normal",
+    topic: str | None = None,
+) -> None:
+    """Deliver one encrypted message; raise :class:`Gone` or :class:`Failed`."""
+    if not allowed_endpoint(endpoint):
+        raise Gone("not a push service")
+    body = encrypt(json.dumps(payload, separators=(",", ":")).encode(), p256dh, auth)
+    headers = {
+        "Authorization": vapid.authorization(endpoint, subject),
+        "Content-Encoding": "aes128gcm",
+        "Content-Type": "application/octet-stream",
+        "TTL": str(ttl),
+        "Urgency": urgency,
+    }
+    if topic:
+        headers["Topic"] = topic  # replaces an undelivered message with the same topic
+    try:
+        resp = await client.post(endpoint, content=body, headers=headers)
+    except httpx.HTTPError as err:
+        raise Failed(str(err)) from err
+    if resp.status_code in (404, 410):
+        raise Gone(f"HTTP {resp.status_code}")
+    if resp.status_code >= 400:
+        raise Failed(f"HTTP {resp.status_code}: {resp.text[:200]}")

backend/app/queue.py

@@ -21,7 +21,14 @@ QUEUE_INTERACTIVE = "interactive"
 QUEUE_BULK = "bulk"
 
 # Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
-_BULK_JOBS = frozenset({"sync_bootstrap_account"})
+_BULK_JOBS = frozenset(
+    {
+        "sync_bootstrap_account",
+        "dedupe_existing_job",
+        "backfill_list_headers_job",
+        "build_index_job",
+    }
+)
 
 
 def queue_for(job_name: str) -> str:

backend/app/redis_client.py

@@ -20,6 +20,7 @@ def aw(result: Any) -> Awaitable[Any]:
 
 
 _client: aioredis.Redis | None = None
+_bytes_client: aioredis.Redis | None = None
 
 
 def get_redis() -> aioredis.Redis:
@@ -34,9 +35,25 @@ def get_redis() -> aioredis.Redis:
     return _client
 
 
+def get_redis_bytes() -> aioredis.Redis:
+    """Return the process-wide Redis client for binary values.
+
+    ``get_redis()`` decodes every reply as UTF-8 text, which fails on (or
+    silently changes the type of) raw bytes such as a cached image. Anything that
+    stores bytes goes through this one instead.
+    """
+    global _bytes_client
+    if _bytes_client is None:
+        _bytes_client = aioredis.Redis.from_url(get_settings().redis_url, decode_responses=False)
+    return _bytes_client
+
+
 async def close_redis() -> None:
-    """Close the Redis client (used on app shutdown)."""
-    global _client
+    """Close the Redis clients (used on app shutdown)."""
+    global _client, _bytes_client
     if _client is not None:
         await _client.aclose()
         _client = None
+    if _bytes_client is not None:
+        await _bytes_client.aclose()
+        _bytes_client = None

backend/app/search/factory.py

@@ -10,12 +10,14 @@ _backend: SearchBackend | None = None
 
 
 def get_search_backend() -> SearchBackend:
-    """Return the process-wide SearchBackend (``pg`` in M0-M1, ``meili`` in M2)."""
+    """Return the process-wide SearchBackend (``pg`` or ``meili`` per SEARCH_BACKEND)."""
     global _backend
     if _backend is None:
         which = get_settings().search_backend
         if which == "meili":
-            # TODO(spec): Meilisearch backend lands in M2.
-            raise NotImplementedError("meili backend is implemented in M2")
+            from app.search.meili import MeiliSearchBackend
+
+            _backend = MeiliSearchBackend()
+        else:
             _backend = PgSearchBackend()
     return _backend