Loading splash: animated logo over a dark overlay on desktop

index.html carries the overlay markup and styles inline so it paints before
any script; public/splash.js (external, for the CSP) plays the logo
animation from icons/ultimo-splash.html, holds, fades out after ~3.25 s and
removes itself. Skipped at phone sizes (same query as useIsMobile) and in
pop-out compose windows; a click skips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.env.example

@@ -18,3 +18,8 @@ LOG_LEVEL=DEBUG
 RSPAMD_URL=http://127.0.0.1:11333
 RSPAMD_CONTROLLER_URL=http://127.0.0.1:11334
 RSPAMD_CONTROLLER_PASSWORD=devpass
+
+# Spelling/grammar check in the compose window (LanguageTool). Empty = off.
+LANGUAGETOOL_URL=http://127.0.0.1:8010
+# Email translation (the translator service, translator/). Empty = off.
+TRANSLATOR_URL=http://127.0.0.1:5005

Makefile

@@ -1,5 +1,5 @@
 up:        ## start services
-	docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch
+	docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool translator
 down:      ## stop services (data kept)
 	docker compose -f docker/docker-compose.yml down
 reset-db:  ## DESTRUCTIVE: wipe db + search + blobs, re-migrate

backend/app/accounts/discovery.py

@@ -125,9 +125,7 @@ def parse_autoconfig_xml(xml_text: str) -> CandidateConfig | None:
     smtp = _server("outgoingServer", "smtp")
     if imap is None or smtp is None:
         return None
-    return CandidateConfig(
-        imap, smtp, "autoconfig", auth_strategy=_strategy_from_auth(imap.auth)
-    )
+    return CandidateConfig(imap, smtp, "autoconfig", auth_strategy=_strategy_from_auth(imap.auth))
 
 
 # --- DNS SRV (RFC 6186) ------------------------------------------------------

backend/app/accounts/schemas.py

@@ -102,3 +102,4 @@ class AccountDetailOut(BaseModel):
     backfill_done: int
     imap_config: dict[str, object]
     smtp_config: dict[str, object]
+    kind: str = "external"

backend/app/accounts/service.py

@@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.accounts.schemas import CreateAccountRequest
 from app.accounts.types import ServerConfig
 from app.indexer import blobstore
-from app.models import Account, Attachment, Message, Thread
+from app.models import Account, Attachment, ComposeAttachment, Message, Thread
 from app.security.crypto import encrypt_secret
 
 
@@ -127,6 +127,13 @@ async def delete_account(
                 .where(Attachment.user_id == user_id, Attachment.blob_sha256 == sha)
                 .limit(1)
             )
+        if still_used is None:
+            # A forwarded file waiting in a draft shares the hash: keep it for that.
+            still_used = await session.scalar(
+                select(ComposeAttachment.id)
+                .where(ComposeAttachment.user_id == user_id, ComposeAttachment.blob_sha256 == sha)
+                .limit(1)
+            )
         if still_used is None:
             blobstore.remove(user_id, sha)
     return True

backend/app/accounts/ssrf.py

@@ -2,9 +2,10 @@
 
 All autodiscovery HTTP is HTTPS-only, 5 s timeout, response capped at 256 KB, no
 off-domain redirects, and the target host must resolve exclusively to public IPs
-(reject private/loopback/link-local/reserved ranges). A residual TOCTOU gap
-(DNS rebinding between our resolve and httpx's own resolve) is accepted for M1;
-a pinned-IP transport can close it later.
+(reject private/loopback/link-local/reserved ranges). The request then goes to
+the very address that was checked — the name is not looked up a second time —
+so a DNS answer that changes in between (DNS rebinding) cannot point it inward.
+TLS is still verified against the host name.
 """
 
 from __future__ import annotations
@@ -26,15 +27,12 @@ class SsrfError(Exception):
 
 
 def _is_public_ip(ip: str) -> bool:
-    addr = ipaddress.ip_address(ip)
-    return not (
-        addr.is_private
-        or addr.is_loopback
-        or addr.is_link_local
-        or addr.is_multicast
-        or addr.is_reserved
-        or addr.is_unspecified
-    )
+    addr = ipaddress.ip_address(ip.split("%", 1)[0])  # drop an IPv6 zone index
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
+        addr = addr.ipv4_mapped  # ::ffff:127.0.0.1 is 127.0.0.1
+    # is_global: not private, loopback, link-local, reserved, unspecified, nor
+    # shared/carrier-grade NAT (100.64.0.0/10, which is_private misses).
+    return addr.is_global and not addr.is_multicast
 
 
 async def _resolve_all(host: str) -> list[str]:
@@ -43,8 +41,10 @@ async def _resolve_all(host: str) -> list[str]:
     return [info[4][0] for info in infos]
 
 
-async def assert_host_is_public(host: str) -> None:
-    """Resolve ``host`` and reject if any address is non-public (SPEC §10.6)."""
+async def assert_host_is_public(host: str) -> list[str]:
+    """Resolve ``host`` and reject if any address is non-public (SPEC §10.6).
+
+    Returns the addresses checked, for the caller to connect to."""
     try:
         addresses = await _resolve_all(host)
     except socket.gaierror as exc:
@@ -54,6 +54,31 @@ async def assert_host_is_public(host: str) -> None:
     for ip in addresses:
         if not _is_public_ip(ip):
             raise SsrfError(f"{host} resolves to non-public address {ip}")
+    return addresses
+
+
+async def _pinned(url: str) -> tuple[str, dict[str, str], dict[str, str]]:
+    """``url`` aimed at the address just checked: (url, headers, extensions).
+
+    The URL names the IP, the Host header keeps the name (virtual hosting), and
+    TLS gets the name for SNI and for checking the certificate."""
+    parts = urlsplit(url)
+    if parts.scheme != "https":
+        raise SsrfError("only https is permitted")
+    hostname = parts.hostname
+    if not hostname:
+        raise SsrfError("missing host")
+    addresses = await assert_host_is_public(hostname)
+    # IPv4 first: containers often have no IPv6 route, though DNS returns AAAA.
+    ip = next((a for a in addresses if ":" not in a), addresses[0])
+    literal = f"[{ip}]" if ":" in ip else ip
+    netloc = f"{literal}:{parts.port}" if parts.port else literal
+    host_header = f"{hostname}:{parts.port}" if parts.port else hostname
+    return (
+        parts._replace(netloc=netloc).geturl(),
+        {"Host": host_header},
+        {"sni_hostname": hostname},
+    )
 
 
 async def guarded_get(url: str) -> str | None:
@@ -63,12 +88,7 @@ async def guarded_get(url: str) -> str | None:
     (any non-200 response). Raises :class:`SsrfError` on a policy violation
     (bad scheme, private IP, oversize); lets httpx network errors propagate.
     """
-    parts = urlsplit(url)
-    if parts.scheme != "https":
-        raise SsrfError("only https is permitted")
-    if not parts.hostname:
-        raise SsrfError("missing host")
-    await assert_host_is_public(parts.hostname)
+    target, pin_headers, extensions = await _pinned(url)
 
     async with (
         httpx.AsyncClient(
@@ -76,7 +96,7 @@ async def guarded_get(url: str) -> str | None:
             timeout=HTTP_TIMEOUT,
             headers={"User-Agent": "webmail-autodiscovery/1.0"},
         ) as client,
-        client.stream("GET", url) as resp,
+        client.stream("GET", target, headers=pin_headers, extensions=extensions) as resp,
     ):
         if resp.status_code != 200:
             return None
@@ -114,16 +134,17 @@ async def guarded_request(
     fetch ICS feeds. HTTPS-only, public host required, no off-domain redirects, body
     capped at ``max_bytes``. ``auth`` is an (user, password) basic-auth pair. Returns
     status + headers + body (all statuses; the caller decides what's an error)."""
-    parts = urlsplit(url)
-    if parts.scheme != "https":
-        raise SsrfError("only https is permitted")
-    if not parts.hostname:
-        raise SsrfError("missing host")
-    await assert_host_is_public(parts.hostname)
+    target, pin_headers, extensions = await _pinned(url)
 
     async with (
         httpx.AsyncClient(follow_redirects=False, timeout=timeout, auth=auth) as client,
-        client.stream(method, url, headers=headers, content=content) as resp,
+        client.stream(
+            method,
+            target,
+            headers={**(headers or {}), **pin_headers},
+            content=content,
+            extensions=extensions,
+        ) as resp,
     ):
         chunks: list[bytes] = []
         total = 0

backend/app/accounts/testing.py

@@ -14,6 +14,7 @@ import aiosmtplib
 
 from app.accounts.ssrf import SsrfError, assert_host_is_public
 from app.accounts.types import ServerConfig
+from app.config import smtp_helo_name
 from app.providers.base import ProviderAuthError, ProviderConnectError
 from app.providers.imap_smtp import ImapSmtpProvider, ProviderConfig
 
@@ -53,6 +54,7 @@ async def _test_smtp(config: ServerConfig, username: str, password: str) -> None
         use_tls=config.tls == "ssl",
         start_tls=config.tls == "starttls",
         timeout=_SMTP_TIMEOUT,
+        local_hostname=smtp_helo_name(),
     )
     try:
         await smtp.connect()
@@ -89,6 +91,7 @@ async def test_account(
             smtp_tls=smtp.tls,
             username=username,
             password=password,
+            public_only=True,
         )
     )
     try:

backend/app/api/__init__.py

@@ -15,15 +15,23 @@ def register_routers(app: FastAPI) -> None:
     from app.api.actions import router as actions_router
     from app.api.admin import router as admin_router
     from app.api.avatar import router as avatar_router
+    from app.api.completion import router as completion_router
     from app.api.compose import router as compose_router
+    from app.api.invitations import router as invitations_router
     from app.api.mailboxes import router as mailboxes_router
     from app.api.messages import router as messages_router
     from app.api.plugins import router as plugins_router
+    from app.api.push import router as push_router
     from app.api.recipients import router as recipients_router
     from app.api.saved_searches import router as saved_searches_router
     from app.api.search import router as search_router
     from app.api.settings import router as settings_router
+    from app.api.snooze import router as snooze_router
+    from app.api.spellcheck import router as spellcheck_router
+    from app.api.subscriptions import router as subscriptions_router
+    from app.api.templates import router as templates_router
     from app.api.threads import router as threads_router
+    from app.api.translation import router as translation_router
     from app.identity.routes import router as auth_router
     from app.ws import router as ws_router
 
@@ -42,9 +50,21 @@ def register_routers(app: FastAPI) -> None:
         avatar_router,
         plugins_router,
         admin_router,
+        spellcheck_router,
+        push_router,
+        invitations_router,
+        templates_router,
+        snooze_router,
+        subscriptions_router,
+        completion_router,
+        translation_router,
     ):
         app.include_router(router, prefix="/api")
     app.include_router(ws_router, prefix="/api")
+    # CalDAV / CardDAV (SPEC Appendix E): at the root, where clients look for it.
+    from app.dav.routes import router as dav_router
+
+    app.include_router(dav_router)
 
     # Mount discovered plugin routers at /api/plugins/{name} (SPEC §11). Hooks are
     # subscribed in the lifespan startup so they register once per process.

backend/app/api/accounts.py

@@ -101,6 +101,10 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
             sync_window=a.sync_window,
             backfill_total=a.backfill_total,
             backfill_done=a.backfill_done,
+            sync_error=a.sync_error,
+            sync_error_since=a.sync_error_since.isoformat() if a.sync_error_since else None,
+            state_detail=a.state_detail,
+            kind=a.kind,
         )
         for a in accounts
     ]
@@ -222,6 +226,8 @@ async def update_account(
     # re-encrypt the secret and re-sync. The password must be supplied (§10.2: the
     # stored secret is only decrypted in the worker, never here).
     conn_change = body.imap is not None or body.smtp is not None or body.password is not None
+    if conn_change and account.kind == "native":
+        raise_api_error(400, "native_account", "the settings of your own mailbox cannot be changed")
     if conn_change:
         if not body.password:
             raise_api_error(
@@ -264,10 +270,20 @@ async def update_account(
 
 @router.delete("/{account_id}", dependencies=[Depends(require_csrf)])
 async def remove_account(account_id: uuid.UUID, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    kind = await db.scalar(
+        select(Account.kind).where(Account.id == account_id, Account.user_id == user.id)
+    )
+    if kind == "native":
+        raise_api_error(409, "native_account", "your own mailbox cannot be removed")
     ok = await delete_account(db, user_id=user.id, account_id=account_id)
     if not ok:
         raise_api_error(404, "not_found", "account not found")
     await db.commit()
+    # Removing an account for the user's own address (the mail at its old
+    # provider) makes room for their hosted mailbox's (App. D §D.4).
+    from app.hosted import native
+
+    await native.ensure_and_sync(db, user)
     return {"status": "deleted"}
 
 
@@ -285,4 +301,5 @@ def _detail(account: Account) -> AccountDetailOut:
         backfill_done=account.backfill_done,
         imap_config=account.imap_config,
         smtp_config=account.smtp_config,
+        kind=account.kind,
     )

backend/app/api/actions.py

@@ -125,9 +125,7 @@ async def mark_all_read(
         return {"changed": 0}
     messages = list(
         await db.scalars(
-            select(Message).where(
-                Message.thread_id.in_(thread_ids), Message.user_id == user.id
-            )
+            select(Message).where(Message.thread_id.in_(thread_ids), Message.user_id == user.id)
         )
     )
     add, remove = sanitize_flags(["\\Seen"]), sanitize_flags([])
@@ -137,9 +135,7 @@ async def mark_all_read(
     await _push_and_publish(user.id, changed, add, remove)
     bus = get_event_bus()
     for tid in thread_ids:
-        await bus.publish(
-            "thread.updated", {"user_id": str(user.id), "thread_id": str(tid)}
-        )
+        await bus.publish("thread.updated", {"user_id": str(user.id), "thread_id": str(tid)})
     return {"changed": len(changed)}
 
 

backend/app/api/admin.py

@@ -1,24 +1,59 @@
 """Instance administration (SPEC §6.1).
 
-The backend seam for the admin panel. Today it carries only the registration
-gate — the first account on a fresh install becomes the administrator and closes
-signup behind itself, and this is how that gets reopened. Further instance-wide
-options land here as the panel grows.
+The backend seam for the admin panel: the registration gate — the first account
+on a fresh install becomes the administrator and closes signup behind itself,
+and this is how that gets reopened — the default quota, and managing users
+(search, per-user quota, block, delete). An administrator can change their own
+quota but cannot block or delete their own account, so the instance can never
+lock out its last admin that way.
 """
 
 from __future__ import annotations
 
+import base64
 import datetime as dt
+import re
+import uuid
 from typing import Annotated
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Path, Query
+from pydantic import BaseModel
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import translation
+from app.api.errors import raise_api_error
+from app.config import get_settings
 from app.db import get_session
+from app.hosted.admin import MailServerError
+from app.identity import invitations
 from app.identity.deps import CurrentAdmin, require_csrf
-from app.identity.schemas import AdminSettingsIn, AdminSettingsOut
+from app.identity.mailer import (
+    LOGO_CID,
+    LOGO_DATA_URL,
+    LOGO_HTML,
+    MailerError,
+    mailer_config,
+    send_mail,
+)
+from app.identity.schemas import (
+    AdminInvitePreviewIn,
+    AdminInvitePreviewOut,
+    AdminInviteTemplateIn,
+    AdminInviteTemplateOut,
+    AdminMailIn,
+    AdminMailOut,
+    AdminMailTestIn,
+    AdminSettingsIn,
+    AdminSettingsOut,
+    AdminUserOut,
+    AdminUserPatch,
+    MessageOut,
+)
 from app.identity.service import app_settings
-from app.models import AppSettings
+from app.identity.user_admin import delete_user, search_users, set_blocked
+from app.models import Account, AppSettings, Message, User
+from app.security.crypto import encrypt_secret
 
 router = APIRouter(prefix="/admin", tags=["admin"])
 
@@ -29,6 +64,7 @@ def _out(row: AppSettings) -> AdminSettingsOut:
     return AdminSettingsOut(
         registration_open=row.registration_open,
         default_quota_bytes=row.default_quota_bytes,
+        invitations_enabled=row.invitations_enabled,
     )
 
 
@@ -51,6 +87,314 @@ async def write_settings(
         row.registration_open = body.registration_open
     if body.default_quota_bytes is not None:
         row.default_quota_bytes = body.default_quota_bytes
+    if body.invitations_enabled is not None:
+        row.invitations_enabled = body.invitations_enabled
     row.updated_at = dt.datetime.now(dt.UTC)
     await db.commit()
     return _out(row)
+
+
+# --- Outgoing system mail -------------------------------------------------------
+
+
+def _mail_out(row: AppSettings) -> AdminMailOut:
+    return AdminMailOut(
+        smtp_host=row.smtp_host,
+        smtp_port=row.smtp_port,
+        smtp_security=row.smtp_security,
+        smtp_username=row.smtp_username,
+        has_password=row.smtp_secret_ct is not None,
+        smtp_from=row.smtp_from,
+        public_url=row.public_url,
+        configured=bool(row.smtp_host and row.smtp_from and row.public_url),
+    )
+
+
+@router.get("/mail", response_model=AdminMailOut)
+async def read_mail(admin: CurrentAdmin, db: DbSession) -> AdminMailOut:
+    return _mail_out(await app_settings(db))
+
+
+@router.put("/mail", response_model=AdminMailOut, dependencies=[Depends(require_csrf)])
+async def write_mail(body: AdminMailIn, admin: CurrentAdmin, db: DbSession) -> AdminMailOut:
+    """Merge the supplied SMTP fields. The password is stored vault-encrypted."""
+    row = await app_settings(db, for_update=True)
+    for name in ("smtp_host", "smtp_username", "smtp_from"):
+        value = getattr(body, name)
+        if value is not None:
+            setattr(row, name, value.strip())
+    if body.smtp_port is not None:
+        row.smtp_port = body.smtp_port
+    if body.smtp_security is not None:
+        row.smtp_security = body.smtp_security
+    if body.smtp_password is not None:
+        row.smtp_secret_ct = (
+            encrypt_secret(body.smtp_password.encode()) if body.smtp_password else None
+        )
+    if body.public_url is not None:
+        url = body.public_url.strip().rstrip("/")
+        if url and not url.startswith(("http://", "https://")):
+            raise_api_error(400, "invalid_url", "the address must start with http:// or https://")
+        row.public_url = url
+    row.updated_at = dt.datetime.now(dt.UTC)
+    await db.commit()
+    return _mail_out(row)
+
+
+@router.post("/mail/test", response_model=MessageOut, dependencies=[Depends(require_csrf)])
+async def test_mail(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -> MessageOut:
+    """Send a test message with the saved settings, reporting the server's error."""
+    cfg = mailer_config(await app_settings(db))
+    if cfg is None:
+        raise_api_error(
+            409, "mail_not_configured", "set the SMTP server, From address and public address first"
+        )
+    text = (
+        "This is a test message from Ultimo Email.\n\n"
+        "Outgoing system mail works: password-reset links will be sent from this account.\n"
+    )
+    # Same logo header as the reset mail, so the test shows what users will get.
+    html = LOGO_HTML + f"<p>{text.replace(chr(10) * 2, '</p><p>').strip()}</p>"
+    try:
+        await send_mail(cfg, body.to.strip(), "Ultimo Email test message", text, html)
+    except MailerError as exc:
+        raise_api_error(502, "mail_failed", f"sending failed: {exc}")
+    return MessageOut()
+
+
+# --- Invitation email ------------------------------------------------------------
+
+_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+")
+_SAMPLE = {"invitee_name": "Anna Nowak", "inviter_name": "Jan Kowalski"}
+
+
+def _template_out(row: AppSettings) -> AdminInviteTemplateOut:
+    return AdminInviteTemplateOut(
+        subject=row.invite_subject,
+        html=row.invite_template,
+        from_address=row.invite_from,
+        default_subject=invitations.DEFAULT_SUBJECT,
+        default_html=invitations.DEFAULT_TEMPLATE,
+        default_from=row.smtp_from,
+        placeholders=list(invitations.PLACEHOLDERS),
+    )
+
+
+@router.get("/invitations/template", response_model=AdminInviteTemplateOut)
+async def read_invite_template(admin: CurrentAdmin, db: DbSession) -> AdminInviteTemplateOut:
+    return _template_out(await app_settings(db))
+
+
+@router.put(
+    "/invitations/template",
+    response_model=AdminInviteTemplateOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def write_invite_template(
+    body: AdminInviteTemplateIn, admin: CurrentAdmin, db: DbSession
+) -> AdminInviteTemplateOut:
+    """Upload the invitation email's HTML template, subject and sender."""
+    row = await app_settings(db, for_update=True)
+    if body.html is not None:
+        html = body.html.strip()
+        if html and (problem := invitations.template_problem(html)):
+            raise_api_error(400, "invalid_template", problem)
+        row.invite_template = html
+    if body.subject is not None:
+        row.invite_subject = " ".join(body.subject.split())
+    if body.from_address is not None:
+        sender = body.from_address.strip()
+        if sender and not _EMAIL_RE.fullmatch(sender):
+            raise_api_error(400, "invalid_email", "the sender must be an email address")
+        row.invite_from = sender
+    row.updated_at = dt.datetime.now(dt.UTC)
+    await db.commit()
+    return _template_out(row)
+
+
+def _sample(row: AppSettings, subject: str, html: str) -> invitations.RenderedMail:
+    """The invitation as a friend would get it, with sample names."""
+    if html.strip() and (problem := invitations.template_problem(html)):
+        raise_api_error(400, "invalid_template", problem)
+    draft = AppSettings(invite_subject=subject, invite_template=html.strip())
+    base = row.public_url.rstrip("/") or f"https://{get_settings().app_domain}"
+    return invitations.render(draft, link=f"{base}/register#invite=sample", **_SAMPLE)
+
+
+@router.post(
+    "/invitations/preview",
+    response_model=AdminInvitePreviewOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def preview_invite(
+    body: AdminInvitePreviewIn, admin: CurrentAdmin, db: DbSession
+) -> AdminInvitePreviewOut:
+    """Render a template (saved or not) with sample names, for the panel to show."""
+    mail = _sample(await app_settings(db), body.subject, body.html)
+    html = mail.html
+    # The preview frame has no message parts: show the images it would carry.
+    for part in mail.inline:
+        data = base64.b64encode(part.data).decode()
+        html = html.replace(f"cid:{part.cid}", f"data:{part.mime_type};base64,{data}")
+    html = html.replace(f"cid:{LOGO_CID}", LOGO_DATA_URL)
+    return AdminInvitePreviewOut(subject=mail.subject, html=html)
+
+
+@router.post("/invitations/test", response_model=MessageOut, dependencies=[Depends(require_csrf)])
+async def test_invite(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -> MessageOut:
+    """Send the saved invitation email, with sample names and a dummy link."""
+    row = await app_settings(db)
+    cfg = mailer_config(row)
+    if cfg is None:
+        raise_api_error(
+            409, "mail_not_configured", "set the SMTP server, From address and public address first"
+        )
+    mail = _sample(row, row.invite_subject, row.invite_template)
+    try:
+        await send_mail(
+            cfg,
+            body.to.strip(),
+            mail.subject,
+            mail.text,
+            mail.html,
+            sender=invitations.sender_address(row),
+            inline=mail.inline,
+        )
+    except MailerError as exc:
+        raise_api_error(502, "mail_failed", f"sending failed: {exc}")
+    return MessageOut()
+
+
+# --- Users --------------------------------------------------------------------
+
+
+def _user_out(user: User, used_bytes: int, accounts: list[str]) -> AdminUserOut:
+    return AdminUserOut(
+        id=user.id,
+        handle=user.handle,
+        is_admin=user.is_admin,
+        blocked_at=user.blocked_at.isoformat() if user.blocked_at else None,
+        quota_bytes=user.quota_bytes,
+        used_bytes=used_bytes,
+        created_at=user.created_at.isoformat(),
+        accounts=accounts,
+    )
+
+
+async def _target(
+    db: AsyncSession, admin: User, user_id: uuid.UUID, *, allow_self: bool = False
+) -> User:
+    """The user an admin action is aimed at. Blocking and deleting are never
+    allowed on the admin themselves; a quota change is (``allow_self``)."""
+    user = await db.scalar(select(User).where(User.id == user_id))
+    if user is None:
+        raise_api_error(404, "not_found", "user not found")
+    if user.id == admin.id and not allow_self:
+        raise_api_error(409, "cannot_modify_self", "you cannot do this to your own account")
+    return user
+
+
+@router.get("/users", response_model=list[AdminUserOut])
+async def list_users(
+    admin: CurrentAdmin, db: DbSession, q: str = Query("", max_length=200)
+) -> list[AdminUserOut]:
+    """Search users by login, or by a mail account's address or display name."""
+    return [_user_out(u, used, emails) for u, used, emails in await search_users(db, q)]
+
+
+@router.patch("/users/{user_id}", response_model=AdminUserOut, dependencies=[Depends(require_csrf)])
+async def patch_user(
+    user_id: uuid.UUID, body: AdminUserPatch, admin: CurrentAdmin, db: DbSession
+) -> AdminUserOut:
+    """Change a user's quota and/or block or unblock them."""
+    # Your own quota is fine to change; blocking yourself is not.
+    user = await _target(db, admin, user_id, allow_self=body.blocked is None)
+    if body.quota_bytes is not None:
+        user.quota_bytes = body.quota_bytes
+    if body.blocked is not None:
+        try:
+            await set_blocked(db, user, body.blocked)
+        except MailServerError as exc:
+            raise_api_error(
+                502, "mail_server_unavailable", f"the mail server did not answer: {exc}"
+            )
+    await db.commit()
+    used = await db.scalar(
+        select(func.coalesce(func.sum(Message.size_bytes), 0)).where(Message.user_id == user.id)
+    )
+    emails = list(
+        await db.scalars(
+            select(Account.email).where(Account.user_id == user.id).order_by(Account.email)
+        )
+    )
+    return _user_out(user, int(used or 0), emails)
+
+
+@router.delete("/users/{user_id}", dependencies=[Depends(require_csrf)])
+async def remove_user(user_id: uuid.UUID, admin: CurrentAdmin, db: DbSession) -> dict[str, str]:
+    """Delete a user and all of their data. Cannot be undone."""
+    user = await _target(db, admin, user_id)
+    try:
+        await delete_user(db, user)
+    except MailServerError as exc:
+        raise_api_error(502, "mail_server_unavailable", f"the mail server did not answer: {exc}")
+    return {"status": "deleted"}
+
+
+# --- email translation languages (translator service) ------------------------------
+
+
+class InstallProgressOut(BaseModel):
+    done: int
+    total: int
+
+
+class AdminTranslationOut(BaseModel):
+    enabled: bool  # TRANSLATOR_URL is set
+    reachable: bool
+    installed: list[str] = []
+    installing: dict[str, InstallProgressOut] = {}
+    pending: list[str] = []  # waiting to be tried again (no network last time)
+    available: list[str] = []  # what Mozilla has models for (empty: offline)
+    errors: dict[str, str] = {}  # why a language's last install failed
+
+
+_LANG = Path(pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$")
+
+
+@router.get("/translation", response_model=AdminTranslationOut)
+async def translation_status(admin: CurrentAdmin) -> AdminTranslationOut:
+    if not translation.enabled():
+        return AdminTranslationOut(enabled=False, reachable=False)
+    try:
+        data = await translation.status()
+    except translation.TranslationError:
+        return AdminTranslationOut(enabled=True, reachable=False)
+    return AdminTranslationOut(enabled=True, reachable=True, **data)
+
+
+@router.post(
+    "/translation/languages/{code}",
+    response_model=AdminTranslationOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def install_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
+    """Download a language's models (to and from English) in the background."""
+    try:
+        await translation.install(code)
+    except translation.TranslationError as exc:
+        raise_api_error(502, "translator_failed", str(exc))
+    return await translation_status(admin)
+
+
+@router.delete(
+    "/translation/languages/{code}",
+    response_model=AdminTranslationOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def remove_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
+    try:
+        await translation.remove(code)
+    except translation.TranslationError as exc:
+        raise_api_error(502, "translator_failed", str(exc))
+    return await translation_status(admin)

backend/app/api/avatar.py

@@ -3,8 +3,9 @@
 The browser only ever calls ``GET /api/avatar`` (same origin), so a sender can never
 see the user's IP or read activity: the server does the outbound fetch under the
 §10.6 SSRF guard and caches the result (both hits and misses) in Redis. A miss
-returns 404, which the UI renders as its initials avatar. Disable with
-``SENDER_AVATARS_ENABLED=false``.
+returns 204 No Content: the <img> fails to decode it and the UI falls back to its
+initials, without the browser logging an error for every sender who has no avatar
+(a 404 did, filling the console). Disable with ``SENDER_AVATARS_ENABLED=false``.
 """
 
 from __future__ import annotations
@@ -17,7 +18,7 @@ from fastapi import APIRouter, Query, Response
 from app.accounts.ssrf import SsrfError, guarded_request
 from app.config import get_settings
 from app.identity.deps import CurrentUser
-from app.redis_client import get_redis
+from app.redis_client import get_redis_bytes
 
 router = APIRouter(prefix="/avatar", tags=["mail"])
 
@@ -25,6 +26,12 @@ _CACHE_TTL_HIT = 7 * 24 * 3600  # remember a found avatar for a week
 _CACHE_TTL_MISS = 24 * 3600  # re-check "no avatar" daily
 _MAX_BYTES = 200 * 1024
 _NONE = b"none"
+# The browser keeps an answer as long as we would, so it stops re-asking per page.
+_BROWSER_CACHE = {"Cache-Control": "private, max-age=86400"}
+
+
+def _no_avatar() -> Response:
+    return Response(status_code=204, headers=_BROWSER_CACHE)
 
 
 def _email_md5(email: str) -> str:
@@ -64,28 +71,27 @@ async def _resolve(email: str, md5: str) -> tuple[str, bytes] | None:
 async def sender_avatar(_user: CurrentUser, email: str = Query(...)) -> Response:
     email = email.strip().lower()
     if not get_settings().sender_avatars_enabled or "@" not in email:
-        return Response(status_code=404)
+        return _no_avatar()
 
-    redis = get_redis()
+    # Binary-safe client: the cache holds raw image bytes (and a b"none" marker).
+    redis = get_redis_bytes()
     md5 = _email_md5(email)
     key = f"avatar:{md5}"
-    headers = {"Cache-Control": "private, max-age=86400"}
-
     cached: bytes | None = await redis.get(key)
     if cached == _NONE:
-        return Response(status_code=404)
+        return _no_avatar()
     if cached:
         cached_ctype, _, cached_body = cached.partition(b"\n")
         return Response(
             content=cached_body,
             media_type=cached_ctype.decode("ascii", "ignore"),
-            headers=headers,
+            headers=_BROWSER_CACHE,
         )
 
     result = await _resolve(email, md5)
     if result is None:
         await redis.set(key, _NONE, ex=_CACHE_TTL_MISS)
-        return Response(status_code=404)
+        return _no_avatar()
     ctype, body = result
     await redis.set(key, ctype.encode("ascii", "ignore") + b"\n" + body, ex=_CACHE_TTL_HIT)
-    return Response(content=body, media_type=ctype, headers=headers)
+    return Response(content=body, media_type=ctype, headers=_BROWSER_CACHE)

backend/app/api/completion.py

@@ -0,0 +1,44 @@
+"""Sentence completion for the compose window (app/completion.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import completion
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import CompletionIndexState
+from app.queue import enqueue_job
+
+router = APIRouter(prefix="/completion", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class CompletionIn(BaseModel):
+    # The text before the cursor (its last line is all that matters). A POST,
+    # not a query string: what the user is typing stays out of access logs.
+    before: str = Field(max_length=2000)
+
+
+class CompletionOut(BaseModel):
+    suggestion: str | None
+
+
+@router.post("", response_model=CompletionOut)
+async def suggest(body: CompletionIn, user: CurrentUser, db: DbSession) -> CompletionOut:
+    before = body.before[-300:]
+    built = await db.scalar(
+        select(CompletionIndexState.built_at).where(CompletionIndexState.user_id == user.id)
+    )
+    if built is None or built < dt.datetime.now(dt.UTC) - completion.REBUILD_AFTER:
+        # In the background; until it is built, the starter phrases still answer.
+        await enqueue_job("build_index_job", str(user.id), _job_id=f"complete:{user.id}")
+    personal = await completion.personal_phrases(db, user.id, completion.keys_for(before))
+    return CompletionOut(suggestion=completion.complete(personal, before))

backend/app/api/compose.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import uuid
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, File, Query, UploadFile
@@ -15,11 +17,14 @@ from app.api.schemas import AttachmentOut
 from app.config import get_settings
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
-from app.indexer import blobstore
-from app.models import Attachment, BodyPart, ComposeAttachment, Draft, Mailbox, Message
+from app.indexer import attachments as att_store
+from app.indexer import blobstore, rawstore
+from app.models import Attachment, BodyPart, ComposeAttachment, Draft, Mailbox, Message, Outbox
+from app.outbox.jobs import RETRY_WINDOW
 from app.outbox.schemas import (
     DraftIn,
     DraftOut,
+    OutboxItemOut,
     RemoteDraftOut,
     SendRequest,
     SendResponse,
@@ -29,6 +34,7 @@ from app.outbox.service import (
     cancel_outbox,
     create_outbox,
     delete_mirrored_draft_row,
+    not_being_sent,
     upsert_draft,
 )
 from app.queue import enqueue_job
@@ -109,7 +115,9 @@ def _draft_out(draft: Draft) -> DraftOut:
 @router.get("/drafts", response_model=list[DraftOut])
 async def list_drafts(user: CurrentUser, db: DbSession) -> list[DraftOut]:
     drafts = await db.scalars(
-        select(Draft).where(Draft.user_id == user.id).order_by(Draft.updated_at.desc())
+        select(Draft)
+        .where(Draft.user_id == user.id, not_being_sent())
+        .order_by(Draft.updated_at.desc())
     )
     return [_draft_out(d) for d in drafts]
 
@@ -201,7 +209,15 @@ async def adopt_remote_draft(message_id: uuid.UUID, user: CurrentUser, db: DbSes
         )
     )
     attachment_ids: list[str] = []
+    if att_rows:
+        with contextlib.suppress(FileNotFoundError):  # fetched if a hosted mailbox's
+            await rawstore.ensure(msg)
     for a in att_rows:
+        # Kept only inside the message: give the compose attachment a blob of its own.
+        if not await asyncio.to_thread(
+            att_store.materialize, user.id, msg.blob_sha256, a.blob_sha256, a.part_index
+        ):
+            continue
         ca = ComposeAttachment(
             user_id=user.id,
             filename=a.filename,
@@ -300,7 +316,10 @@ async def send(body: SendRequest, user: CurrentUser, db: DbSession) -> SendRespo
         str(outbox.id),
         _defer_by=timedelta(seconds=get_settings().undo_seconds),
     )
-    return SendResponse(outbox_id=outbox.id, undo_until=outbox.send_after.isoformat())
+    left = max(0.0, (outbox.send_after - datetime.now(UTC)).total_seconds())
+    return SendResponse(
+        outbox_id=outbox.id, undo_until=outbox.send_after.isoformat(), undo_seconds=left
+    )
 
 
 @router.post("/send/{outbox_id}/undo")
@@ -310,3 +329,72 @@ async def undo_send(outbox_id: uuid.UUID, user: CurrentUser, db: DbSession) -> d
     if not ok:
         raise_api_error(409, "undo_too_late", "the message has already left the undo window")
     return {"status": "cancelled"}
+
+
+# --- the Scheduled folder's server side: sends being retried, or failed ------------
+
+
+@router.get("/outbox", response_model=list[OutboxItemOut])
+async def list_outbox(user: CurrentUser, db: DbSession) -> list[OutboxItemOut]:
+    """Sends that did not go out: retrying (the server could not take them yet)
+    or failed for good. They wait in Scheduled, not in Drafts."""
+    rows = await db.execute(
+        select(Outbox, Draft)
+        .join(Draft, Draft.id == Outbox.draft_id)
+        .where(Outbox.user_id == user.id, Outbox.state.in_(("retrying", "failed")))
+        .order_by(Outbox.send_after.desc())
+    )
+    out: list[OutboxItemOut] = []
+    for ob, draft in rows.tuples():
+        payload = draft.payload or {}
+        to = payload.get("to")
+        out.append(
+            OutboxItemOut(
+                outbox_id=ob.id,
+                draft_id=draft.id,
+                account_id=draft.account_id,
+                state=ob.state,
+                attempts=ob.attempts,
+                error=ob.error,
+                next_attempt_at=ob.next_attempt_at.isoformat() if ob.next_attempt_at else None,
+                retry_until=ob.retry_until.isoformat() if ob.retry_until else None,
+                sent_at=ob.send_after.isoformat(),
+                to=[str(t) for t in to] if isinstance(to, list) else [],
+                subject=str(payload.get("subject") or ""),
+                preview=str(payload.get("text") or "")[:200],
+            )
+        )
+    return out
+
+
+async def _waiting_send(db: AsyncSession, user_id: uuid.UUID, outbox_id: uuid.UUID) -> Outbox:
+    ob = await db.scalar(select(Outbox).where(Outbox.id == outbox_id, Outbox.user_id == user_id))
+    if ob is None or ob.state not in ("retrying", "failed"):
+        raise_api_error(409, "not_waiting", "this message is not waiting to be sent")
+    return ob
+
+
+@router.post("/outbox/{outbox_id}/retry")
+async def retry_outbox(outbox_id: uuid.UUID, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    """Retry now. A send that had failed for good gets a fresh 24-hour window."""
+    ob = await _waiting_send(db, user.id, outbox_id)
+    now = datetime.now(UTC)
+    if ob.state == "failed" or ob.retry_until is None or ob.retry_until <= now:
+        ob.retry_until = now + RETRY_WINDOW
+    ob.state = "retrying"
+    ob.next_attempt_at = now
+    await db.commit()
+    await enqueue_job("send_job", str(ob.id), _job_id=f"send:{ob.id}:now:{ob.attempts}")
+    return {"status": "retrying"}
+
+
+@router.post("/outbox/{outbox_id}/cancel")
+async def cancel_waiting_send(
+    outbox_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    """Stop trying: the message goes back to Drafts (Edit, or before Delete)."""
+    ob = await _waiting_send(db, user.id, outbox_id)
+    ob.state = "cancelled"
+    ob.next_attempt_at = None
+    await db.commit()
+    return {"status": "cancelled", "draft_id": str(ob.draft_id)}

backend/app/api/errors.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from typing import NoReturn
 
 from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
@@ -29,7 +30,23 @@ async def _handler(request: Request, exc: StarletteHTTPException) -> JSONRespons
     return JSONResponse(status_code=exc.status_code, content=body, headers=headers)
 
 
+async def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+    """A malformed request, in the usual error shape. Names the first field at
+    fault and what is wrong with it — enough to show the user — but not the
+    whole request model the framework's own answer would spell out."""
+    errors = exc.errors()
+    first = errors[0] if errors else {}
+    loc = [str(p) for p in first.get("loc", ()) if p not in ("body", "query", "path")]
+    field = loc[-1] if loc else ""
+    reason = str(first.get("msg", "invalid value"))
+    message = f"{field}: {reason}" if field else reason
+    return JSONResponse(
+        status_code=422, content={"error": {"code": "validation_error", "message": message}}
+    )
+
+
 def register_error_handlers(app: FastAPI) -> None:
     """Install the error-shape handler for all HTTP exceptions."""
     app.add_exception_handler(StarletteHTTPException, _handler)  # type: ignore[arg-type]
     app.add_exception_handler(HTTPException, _handler)  # type: ignore[arg-type]
+    app.add_exception_handler(RequestValidationError, _validation_handler)  # type: ignore[arg-type]

backend/app/api/invitations.py

@@ -0,0 +1,123 @@
+"""Invite a friend (app/identity/invitations.py): what the sidebar entry and its
+dialog need — how many invitations are left, and sending one."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity import invitations
+from app.identity.deps import CurrentUser, require_csrf
+from app.identity.mailer import MailerError, mailer_config, send_mail
+from app.identity.service import app_settings
+from app.models import Invitation
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/invitations", tags=["invitations"])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+")
+
+
+class SentInvitationOut(BaseModel):
+    invitee_name: str
+    invitee_email: str
+    created_at: str
+    used: bool
+    expired: bool
+
+
+class InvitationsOut(BaseModel):
+    # Invitations are switched on and system mail can send them.
+    available: bool
+    remaining: int
+    per_user: int
+    sent: list[SentInvitationOut]
+
+
+class InvitationIn(BaseModel):
+    name: str = Field(min_length=1, max_length=100)
+    email: str = Field(min_length=3, max_length=320)
+
+
+@router.get("", response_model=InvitationsOut)
+async def read(user: CurrentUser, db: DbSession) -> InvitationsOut:
+    row = await app_settings(db)
+    now = dt.datetime.now(dt.UTC)
+    return InvitationsOut(
+        available=row.invitations_enabled and mailer_config(row) is not None,
+        remaining=await invitations.remaining(db, user.id),
+        per_user=invitations.PER_USER,
+        sent=[
+            SentInvitationOut(
+                invitee_name=i.invitee_name,
+                invitee_email=i.invitee_email,
+                created_at=i.created_at.isoformat(),
+                used=i.used_at is not None,
+                expired=i.used_at is None and i.expires_at <= now,
+            )
+            for i in await invitations.sent(db, user.id)
+        ],
+    )
+
+
+@router.post("", response_model=InvitationsOut, dependencies=[Depends(require_csrf)])
+async def send(body: InvitationIn, user: CurrentUser, db: DbSession) -> InvitationsOut:
+    """Email a friend a single-use sign-up link. Sent before answering, so the
+    user hears at once if it could not go out (and keeps the invitation)."""
+    name = " ".join(body.name.split())
+    email = body.email.strip()
+    if not name:
+        raise_api_error(400, "invalid_name", "enter your friend's name")
+    if not _EMAIL_RE.fullmatch(email):
+        raise_api_error(400, "invalid_email", "enter a valid email address")
+    row = await app_settings(db)
+    if not row.invitations_enabled:
+        raise_api_error(403, "invitations_disabled", "invitations are switched off")
+    cfg = mailer_config(row)
+    if cfg is None:
+        raise_api_error(503, "mail_not_configured", "the server cannot send email yet")
+    try:
+        invitation, token = await invitations.create(db, user, name=name, email=email)
+    except invitations.InvitationError as exc:
+        raise_api_error(409, exc.code, exc.message)
+    await db.commit()
+
+    # The token rides in the fragment: never sent to a server, so it stays out
+    # of access logs and Referer headers (as with password-reset links).
+    link = f"{cfg.public_url}/register#invite={token}"
+    mail = invitations.render(
+        row,
+        invitee_name=name,
+        inviter_name=await invitations.inviter_name(db, user),
+        link=link,
+    )
+    try:
+        await send_mail(
+            cfg,
+            email,
+            mail.subject,
+            mail.text,
+            mail.html,
+            sender=invitations.sender_address(row),
+            reply_to=user.handle,  # an answer reaches the friend who invited
+            inline=mail.inline,
+        )
+    except MailerError as exc:
+        logger.warning("invitation from %s could not be sent: %s", user.handle, exc)
+        # Not sent: the user keeps the invitation.
+        await db.execute(delete(Invitation).where(Invitation.id == invitation.id))
+        await db.commit()
+        raise_api_error(502, "mail_failed", "the invitation could not be sent — try again later")
+    return await read(user, db)

backend/app/api/mailboxes.py

@@ -12,7 +12,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Draft, Mailbox, Message
+from app.models import Draft, Mailbox, Message, Thread
+from app.outbox.service import not_being_sent
 
 router = APIRouter(prefix="/mailboxes", tags=["mail"])
 
@@ -29,6 +30,7 @@ class MailboxOut(BaseModel):
 class FolderCounts(BaseModel):
     inbox_unread: int  # unread messages sitting in Inbox mailboxes
     drafts: int  # drafts shown in the Drafts folder (local + un-adopted remote)
+    snoozed: int = 0  # conversations snoozed out of the Inbox
 
 
 @router.get("", response_model=list[MailboxOut])
@@ -53,12 +55,23 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
             Message.user_id == user.id,
             Mailbox.special_use == "inbox",
             ~Message.flags.contains(["\\Seen"]),
+            # A snoozed conversation is out of the Inbox until it wakes.
+            ~Message.thread_id.in_(
+                select(Thread.id).where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+            ),
         )
     )
+    snoozed = await db.scalar(
+        select(func.count())
+        .select_from(Thread)
+        .where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+    )
 
     # Drafts folder = local drafts + IMAP-Drafts messages we haven't adopted (mirror
     # UIDs excluded), matching what list_drafts + list_remote_drafts return.
-    local = await db.scalar(select(func.count()).select_from(Draft).where(Draft.user_id == user.id))
+    local = await db.scalar(
+        select(func.count()).select_from(Draft).where(Draft.user_id == user.id, not_being_sent())
+    )
     owned = {
         (r.account_id, r.imap_uid)
         for r in await db.execute(
@@ -74,4 +87,8 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
     )
     remote = sum(1 for r in remote_rows if (r.account_id, r.uid) not in owned)
 
-    return FolderCounts(inbox_unread=int(inbox_unread or 0), drafts=int(local or 0) + remote)
+    return FolderCounts(
+        inbox_unread=int(inbox_unread or 0),
+        drafts=int(local or 0) + remote,
+        snoozed=int(snoozed or 0),
+    )

backend/app/api/messages.py

@@ -2,27 +2,50 @@
 
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import html as html_lib
 import io
+import logging
 import uuid
 import zipfile
 from typing import Annotated
 from urllib.parse import quote
 
 from fastapi import APIRouter, Depends, Query
-from fastapi.responses import HTMLResponse, Response
+from fastapi.responses import FileResponse, HTMLResponse, Response
 from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import translation
 from app.api.errors import raise_api_error
 from app.api.schemas import AttachmentOut
-from app.api.settings import add_allowed_image_sender, allowed_image_senders
+from app.api.settings import (
+    add_allowed_image_sender,
+    allowed_image_senders,
+    blocks_trackers,
+)
+from app.completion import html_to_text
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
-from app.indexer import blobstore
+from app.indexer import attachments as att_store
+from app.indexer import rawstore, thumbs
 from app.indexer.mime import parse_message
 from app.indexer.sanitize import SANITIZER_VERSION, sanitize_html
-from app.models import Attachment, BodyPart, Message
+from app.indexer.thumbs import thumb_type
+from app.models import (
+    Account,
+    Attachment,
+    BodyPart,
+    ComposeAttachment,
+    Message,
+    MessageTranslation,
+    User,
+)
+from app.translation_prep import find_names_in_signature
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/messages", tags=["mail"])
 
@@ -31,6 +54,26 @@ DbSession = Annotated[AsyncSession, Depends(get_session)]
 # Content types safe to serve inline (SPEC §10.8); everything else is forced to
 # an attachment download with a neutral type.
 _INLINE_SAFE = {"image/png", "image/jpeg", "image/gif", "image/webp", "application/pdf"}
+# Videos the browser can play, by extension — senders often label them
+# application/octet-stream. A QuickTime .mov is usually H.264 in an MP4-style
+# container: offered as video/mp4, browsers play it (an HEVC one stays unplayable,
+# and the preview offers a download instead). Media cannot run script: safe inline.
+_VIDEO_EXT_MIME = {
+    "mp4": "video/mp4",
+    "m4v": "video/mp4",
+    "mov": "video/mp4",
+    "webm": "video/webm",
+    "ogv": "video/ogg",
+}
+
+
+def _inline_type(mime_type: str, filename: str) -> str | None:
+    """The type to serve an attachment as for in-app viewing, or None."""
+    if mime_type in _INLINE_SAFE:
+        return mime_type
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    return _VIDEO_EXT_MIME.get(ext)
+
 
 # Base styles injected around the sanitized fragment so email content renders on a
 # readable light canvas with images that fit the pane (the iframe is sandboxed).
@@ -64,6 +107,31 @@ def _wrap(fragment: str) -> str:
     return _DOC_TEMPLATE.format(body=fragment)
 
 
+# The sanitized message as a document of its own. The app shows it in a
+# sandboxed frame (frontend MessageBody), but this URL can also be opened
+# directly in a tab, on our own origin: there, this policy is what keeps any
+# script the sanitizer might ever miss from running — no scripts, no plugins,
+# no forms, a sandbox of its own origin, and nothing framing it elsewhere.
+_MESSAGE_CSP = (
+    "default-src 'none'; script-src 'none'; object-src 'none'; base-uri 'none'; "
+    "form-action 'none'; frame-ancestors 'self'; img-src 'self' data: https: http:; "
+    "style-src 'self' 'unsafe-inline'; font-src 'self' data: https:; media-src 'self'; "
+    "sandbox allow-popups allow-popups-to-escape-sandbox"
+)
+
+
+def _html(document: str, trackers: int = 0, extra: dict[str, str] | None = None) -> HTMLResponse:
+    # The page reads what it needs from headers: the body is the message.
+    return HTMLResponse(
+        content=document,
+        headers={
+            "Content-Security-Policy": _MESSAGE_CSP,
+            "X-Trackers-Blocked": str(trackers),
+            **(extra or {}),
+        },
+    )
+
+
 async def _owned_message(db: AsyncSession, user_id: uuid.UUID, message_id: uuid.UUID) -> Message:
     msg = await db.scalar(
         select(Message).where(Message.id == message_id, Message.user_id == user_id)
@@ -77,54 +145,167 @@ def _sender_email(message: Message) -> str:
     return message.from_addr[0].get("email", "").lower() if message.from_addr else ""
 
 
+async def _render_fragment(
+    db: AsyncSession, message: Message, allow_remote: bool, block: bool
+) -> tuple[str, int]:
+    """The sanitized body the user sees, and how many trackers came out of it.
+
+    Re-sanitizes from the raw blob when remote images are allowed, the stored
+    copy predates the current sanitizer (SPEC §10.4), or trackers are to stay
+    in; otherwise the stored copy is served as is.
+    """
+    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message.id)))
+    html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
+
+    stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
+    # The stored copy has its trackers taken out; with blocking switched off, a
+    # message that had some is rendered afresh with them left in.
+    trackers_kept = not block and html_part is not None and html_part.trackers_blocked > 0
+    if html_part is not None and not allow_remote and not stale and not trackers_kept:
+        return html_part.content, html_part.trackers_blocked if block else 0
+
+    raw = None
+    if html_part is not None or allow_remote:  # a plain-text message has nothing to redo
+        try:
+            raw = await rawstore.ensure_bytes(message)
+        except FileNotFoundError:
+            raw = None
+    parsed = parse_message(raw) if raw is not None else None
+    if parsed is not None and parsed.html_body is not None:
+        if stale and html_part is not None:
+            # The stored copy is always the plain one: no remote images, trackers out.
+            upgraded = sanitize_html(parsed.html_body, message.id)
+            html_part.content = upgraded.html
+            html_part.sanitizer_v = upgraded.sanitizer_v
+            html_part.trackers_blocked = upgraded.trackers_blocked
+            await db.commit()
+            if block and not allow_remote:
+                return upgraded.html, upgraded.trackers_blocked
+        result = sanitize_html(
+            parsed.html_body, message.id, allow_remote_images=allow_remote, block_trackers=block
+        )
+        return result.html, result.trackers_blocked
+
+    if html_part is not None:
+        return html_part.content, html_part.trackers_blocked
+    text_part = next((p for p in parts if p.kind == "text"), None)
+    body = html_lib.escape(text_part.content) if text_part else ""
+    return f'<div class="wm-plaintext">{body}</div>', 0
+
+
+async def _content_language(db: AsyncSession, message: Message) -> str:
+    """The message's language, found (and kept) the first time it is shown."""
+    if message.content_language is not None or not translation.enabled():
+        return message.content_language or ""
+    text = await db.scalar(
+        select(BodyPart.content).where(BodyPart.message_id == message.id, BodyPart.kind == "text")
+    )
+    try:
+        found = await translation.detect(text or message.snippet or "")
+    except translation.TranslationError as exc:
+        logger.info("language detection unavailable: %s", exc)
+        return ""  # not kept: tried again once the service is back
+    message.content_language = found
+    await db.commit()
+    return found
+
+
 @router.get("/{message_id}/html", response_class=HTMLResponse)
 async def message_html(
     message_id: uuid.UUID,
     user: CurrentUser,
     db: DbSession,
     images: bool = Query(False),
+    translate: str | None = Query(None, pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$"),
 ) -> HTMLResponse:
     """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
 
-    Re-sanitizes from the raw blob when the sender is image-approved, or when the
-    stored copy predates the current sanitizer version (SPEC §10.4). ``images=1``
-    is a one-time "Load images" — render remote images for this request only,
-    without persisting the sender as trusted.
+    ``images=1`` is a one-time "Load images" — render remote images for this
+    request only, without persisting the sender as trusted. ``translate=pl``
+    returns it translated (app/translation.py). Headers tell the page what it
+    got: X-Trackers-Blocked, X-Content-Language (the message's own language,
+    when known) and, translated, X-Translated-From / X-Translated-To.
     """
     message = await _owned_message(db, user.id, message_id)
     allow_remote = images or _sender_email(message) in allowed_image_senders(user)
+    block = blocks_trackers(user)
+    fragment, trackers = await _render_fragment(db, message, allow_remote, block)
 
-    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
-    html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
+    headers: dict[str, str] = {}
+    language = await _content_language(db, message)
+    if language:
+        headers["X-Content-Language"] = language
+    if translate:
+        fragment, source = await _translated(
+            db, user, message, fragment, language, translate, allow_remote, block
+        )
+        headers["X-Translated-From"] = source
+        headers["X-Translated-To"] = translate
+    return _html(_wrap(fragment), trackers, headers)
 
-    stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
-    if html_part is not None and not allow_remote and not stale:
-        return HTMLResponse(content=_wrap(html_part.content))
 
-    # Re-sanitize the original HTML (image-approved sender, or a stale stored copy).
-    # If the raw blob is unavailable, fall back to whatever is already stored.
-    if allow_remote or stale:
-        try:
-            raw = blobstore.load(user.id, message.blob_sha256)
-        except FileNotFoundError:
-            raw = None
-        if raw is not None:
-            parsed = parse_message(raw)
-            if parsed.html_body is not None:
-                result = sanitize_html(
-                    parsed.html_body, message.id, allow_remote_images=allow_remote
-                )
-                if html_part is not None and not allow_remote:
-                    html_part.content = result.html  # persist the upgraded sanitization
-                    html_part.sanitizer_v = result.sanitizer_v
-                    await db.commit()
-                return HTMLResponse(content=_wrap(result.html))
+async def _names(db: AsyncSession, user: User, message: Message) -> list[str]:
+    """Names a translation leaves as they are if the model would change them:
+    the people on the message (and the sender's single names, as signed:
+    "Milan"), the user's own names (accounts, signatures)."""
+    names: list[str] = []
+    for people in (message.from_addr, message.to_addr, message.cc_addr):
+        names += [p.get("name", "") for p in people if p.get("name")]
+    for p in message.from_addr:
+        names += [w for w in str(p.get("name", "")).split() if len(w) >= 3 and w[0].isupper()]
+    rows = await db.execute(
+        select(Account.display_name, Account.signature).where(Account.user_id == user.id)
+    )
+    for display_name, signature in rows:
+        if display_name:
+            names.append(display_name)
+        if signature:
+            names += find_names_in_signature(html_to_text(signature))
+    return names
 
-    if html_part is not None:
-        return HTMLResponse(content=_wrap(html_part.content))
-    text_part = next((p for p in parts if p.kind == "text"), None)
-    body = html_lib.escape(text_part.content) if text_part else ""
-    return HTMLResponse(content=_wrap(f'<div class="wm-plaintext">{body}</div>'))
+
+async def _translated(
+    db: AsyncSession,
+    user: User,
+    message: Message,
+    fragment: str,
+    language: str,
+    target: str,
+    allow_remote: bool,
+    block: bool,
+) -> tuple[str, str]:
+    """The fragment in ``target`` — from the cache, or translated and kept."""
+    version = SANITIZER_VERSION * 1000 + translation.PIPELINE_VERSION
+    cached = await db.get(MessageTranslation, (message.id, target, allow_remote))
+    if cached is not None and cached.sanitizer_v == version:
+        return cached.html, cached.source
+    try:
+        out = await translation.translate_html(
+            fragment, language or "auto", target, names=await _names(db, user, message)
+        )
+    except translation.TranslationError as exc:
+        raise_api_error(503, "translation_failed", str(exc))
+    # The translation comes back as HTML: cleaned like any HTML we show.
+    clean = sanitize_html(
+        out, message.id, allow_remote_images=allow_remote, block_trackers=block
+    ).html
+    source = language or "auto"
+    stmt = insert(MessageTranslation).values(
+        message_id=message.id,
+        target=target,
+        images=allow_remote,
+        source=source,
+        html=clean,
+        sanitizer_v=version,
+    )
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["message_id", "target", "images"],
+            set_={"html": clean, "source": source, "sanitizer_v": version},
+        )
+    )
+    await db.commit()
+    return clean, source
 
 
 @router.post("/{message_id}/allow-remote-images", dependencies=[Depends(require_csrf)])
@@ -154,7 +335,7 @@ async def message_cid(
     )
     if att is None:
         raise_api_error(404, "not_found", "inline part not found")
-    data = blobstore.load(msg.user_id, att.blob_sha256)
+    data = await _attachment_bytes(msg, att)
     media_type = att.mime_type if att.mime_type in _INLINE_SAFE else "application/octet-stream"
     return Response(content=data, media_type=media_type)
 
@@ -168,6 +349,24 @@ def _content_disposition(disposition: str, filename: str) -> str:
     return f"{disposition}; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename)}"
 
 
+async def _raw_ready(msg: Message) -> None:
+    """Have the message's raw source here (a hosted mailbox's is fetched from
+    the mail server when not cached); a miss is left to the reader to report."""
+    with contextlib.suppress(FileNotFoundError):
+        await rawstore.ensure(msg)
+
+
+async def _attachment_bytes(msg: Message, att: Attachment) -> bytes:
+    """An attachment's bytes, taken out of its message (404 if they cannot be)."""
+    await _raw_ready(msg)
+    try:
+        return await asyncio.to_thread(
+            att_store.load, msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index
+        )
+    except FileNotFoundError:
+        raise_api_error(404, "not_found", "attachment content missing")
+
+
 async def _real_attachments(db: AsyncSession, message_id: uuid.UUID) -> list[Attachment]:
     """Downloadable (non-inline) attachments for a message, in a stable order."""
     rows = await db.scalars(
@@ -190,6 +389,41 @@ async def list_attachments(
     ]
 
 
+@router.post(
+    "/{message_id}/forward-attachments",
+    response_model=list[AttachmentOut],
+    dependencies=[Depends(require_csrf)],
+)
+async def forward_attachments(
+    message_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> list[AttachmentOut]:
+    """Attach a message's files to a forward of it: one compose attachment per
+    real (non-inline) attachment. Each is taken out of the message into a blob
+    of its own (shared, by hash, with any other use of the same file).
+    Returns them as the composer lists its attachments."""
+    msg = await _owned_message(db, user.id, message_id)
+    await _raw_ready(msg)
+    rows = [
+        ComposeAttachment(
+            user_id=user.id,
+            filename=a.filename,
+            mime_type=a.mime_type,
+            size_bytes=a.size_bytes,
+            blob_sha256=a.blob_sha256,
+        )
+        for a in await _real_attachments(db, message_id)
+        if await asyncio.to_thread(
+            att_store.materialize, user.id, msg.blob_sha256, a.blob_sha256, a.part_index
+        )
+    ]
+    db.add_all(rows)
+    await db.commit()
+    return [
+        AttachmentOut(id=r.id, filename=r.filename, mime_type=r.mime_type, size_bytes=r.size_bytes)
+        for r in rows
+    ]
+
+
 @router.get("/{message_id}/attachments.zip")
 async def download_attachments_zip(
     message_id: uuid.UUID,
@@ -223,7 +457,7 @@ async def download_attachments_zip(
                 name = f"{stem}-{seen[name]}{dot}{ext}" if dot else f"{name}-{seen[name]}"
             else:
                 seen[name] = 1
-            zf.writestr(name, blobstore.load(msg.user_id, att.blob_sha256))
+            zf.writestr(name, await _attachment_bytes(msg, att))
 
     return Response(
         content=buffer.getvalue(),
@@ -232,6 +466,42 @@ async def download_attachments_zip(
     )
 
 
+@router.get("/{message_id}/attachments/{attachment_id}/thumb")
+async def attachment_thumbnail(
+    message_id: uuid.UUID, attachment_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> Response:
+    """A small preview of an image or a PDF's first page (WebP, at most
+    480 px, aspect kept). 404 when the file cannot have one: the reading pane
+    then shows its type icon instead."""
+    msg = await _owned_message(db, user.id, message_id)
+    att = await db.scalar(
+        select(Attachment).where(
+            Attachment.id == attachment_id, Attachment.message_id == message_id
+        )
+    )
+    kind = thumb_type(att.mime_type, att.filename) if att is not None else None
+    if att is None or kind is None:
+        raise_api_error(404, "not_found", "no thumbnail")
+    if not thumbs.made(msg.user_id, att.blob_sha256):
+        await _raw_ready(msg)
+    data = await asyncio.to_thread(
+        thumbs.thumbnail,
+        msg.user_id,
+        att.blob_sha256,
+        kind,
+        att.size_bytes,
+        lambda: att_store.load(msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index),
+    )
+    if data is None:
+        raise_api_error(404, "not_found", "no thumbnail")
+    # An attachment never changes under its id: the browser may keep this.
+    return Response(
+        content=data,
+        media_type="image/webp",
+        headers={"Cache-Control": "private, max-age=31536000, immutable"},
+    )
+
+
 @router.get("/{message_id}/attachments/{attachment_id}")
 async def message_attachment(
     message_id: uuid.UUID,
@@ -249,12 +519,22 @@ async def message_attachment(
     )
     if att is None:
         raise_api_error(404, "not_found", "attachment not found")
-    data = blobstore.load(msg.user_id, att.blob_sha256)
-    inline = att.mime_type in _INLINE_SAFE and not download
-    media_type = att.mime_type if inline else "application/octet-stream"
-    disposition = "inline" if inline else "attachment"
-    return Response(
-        content=data,
-        media_type=media_type,
+    if not att_store.has_file(msg.user_id, att.blob_sha256):
+        await _raw_ready(msg)
+    try:
+        # Taken out of the message into a short-lived file (kept a day after
+        # its last use), so it can be streamed with HTTP Range like before.
+        path = await asyncio.to_thread(
+            att_store.file, msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index
+        )
+    except FileNotFoundError:
+        raise_api_error(404, "not_found", "attachment content missing")
+    inline_type = None if download else _inline_type(att.mime_type, att.filename)
+    disposition = "inline" if inline_type else "attachment"
+    # Streamed from disk, with HTTP Range: a video can be played and seeked
+    # without being sent whole first (Safari will not play one otherwise).
+    return FileResponse(
+        path,
+        media_type=inline_type or "application/octet-stream",
         headers={"Content-Disposition": _content_disposition(disposition, att.filename)},
     )

backend/app/api/push.py

@@ -0,0 +1,80 @@
+"""Push notifications about new mail: the server's public key, and the devices
+that want them (Settings → Accounts → Notifications)."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel, Field
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentSession, CurrentUser, require_csrf
+from app.push import service, webpush
+
+router = APIRouter(prefix="/push", tags=["push"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class PushKeyOut(BaseModel):
+    public_key: str
+
+
+class PushKeys(BaseModel):
+    p256dh: str = Field(min_length=1, max_length=200)
+    auth: str = Field(min_length=1, max_length=100)
+
+
+class SubscriptionIn(BaseModel):
+    """The browser's ``PushSubscription.toJSON()``."""
+
+    endpoint: str = Field(min_length=1, max_length=2000)
+    keys: PushKeys
+
+
+class EndpointIn(BaseModel):
+    endpoint: str = Field(min_length=1, max_length=2000)
+
+
+@router.get("/key", response_model=PushKeyOut)
+async def push_key(user: CurrentUser, db: DbSession) -> PushKeyOut:
+    """The key a browser subscribes with (``applicationServerKey``)."""
+    return PushKeyOut(public_key=(await service.vapid(db)).public_b64)
+
+
+@router.post("/subscriptions")
+async def add_subscription(
+    body: SubscriptionIn,
+    request: Request,
+    user: CurrentUser,
+    session: CurrentSession,
+    db: DbSession,
+) -> dict[str, str]:
+    if not webpush.allowed_endpoint(body.endpoint):
+        raise_api_error(400, "push_service", "not a known push service")
+    try:
+        p256dh = webpush.b64u_decode(body.keys.p256dh)
+        auth = webpush.b64u_decode(body.keys.auth)
+        if len(p256dh) != 65 or len(auth) != 16:
+            raise ValueError
+    except ValueError:
+        raise_api_error(400, "push_keys", "invalid subscription keys")
+    await service.subscribe(
+        db,
+        user_id=user.id,
+        session_sid=session.sid,
+        endpoint=body.endpoint,
+        p256dh=body.keys.p256dh,
+        auth=body.keys.auth,
+        user_agent=request.headers.get("user-agent", ""),
+    )
+    return {"status": "ok"}
+
+
+@router.post("/subscriptions/remove")
+async def remove_subscription(body: EndpointIn, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    await service.unsubscribe(db, user_id=user.id, endpoint=body.endpoint)
+    return {"status": "ok"}

backend/app/api/schemas.py

@@ -27,6 +27,8 @@ class ThreadSummary(BaseModel):
     has_attachments: bool
     pinned: bool
     last_message_at: dt.datetime
+    # Set while the thread is snoozed: when it comes back to the Inbox.
+    snoozed_until: dt.datetime | None = None
 
 
 class ThreadListOut(BaseModel):
@@ -64,6 +66,13 @@ class MessageEnvelope(BaseModel):
     snippet: str
     # Other accounts this same delivery also landed in (collapsed duplicates).
     also_in_account_ids: list[uuid.UUID] = []
+    # Bulk mail: the Subscriptions group it belongs to, how it can be left
+    # (one_click | mailto | web), and whether the user already did.
+    subscription_key: str | None = None
+    unsubscribe_via: str | None = None
+    unsubscribed: bool = False
+    # Its language, once found (the first time it is shown): "" unknown.
+    content_language: str = ""
 
 
 class ThreadDetail(BaseModel):
@@ -72,6 +81,7 @@ class ThreadDetail(BaseModel):
     order: str  # "asc" | "desc"
     pinned: bool
     messages: list[MessageEnvelope]  # always ASC (SPEC §9.4)
+    snoozed_until: dt.datetime | None = None
 
 
 class FlagChangeRequest(BaseModel):
@@ -97,6 +107,13 @@ class AccountOut(BaseModel):
     sync_window: str = "all"
     backfill_total: int = 0
     backfill_done: int = 0
+    # "unreachable" while the mail server has not answered for a while.
+    sync_error: str | None = None
+    sync_error_since: str | None = None
+    # Why the state is what it is (e.g. the server's reply to a refused sign-in).
+    state_detail: str = ""
+    # "native": the user's own hosted mailbox — cannot be removed (App. D §D.4).
+    kind: str = "external"
 
 
 class SearchThreadHit(BaseModel):
@@ -107,3 +124,5 @@ class SearchThreadHit(BaseModel):
 class SearchOut(BaseModel):
     threads: list[SearchThreadHit]
     cursor: str | None = None
+    # Conversations matching in all (first page only; None where not counted).
+    total: int | None = None

backend/app/api/search.py

@@ -29,13 +29,22 @@ async def search(
     q: str = Query(""),
     cursor: str | None = None,
     limit: int = Query(50, ge=1, le=200),
+    filter: str | None = Query(None, pattern="^(unread|pinned)$"),
 ) -> SearchOut:
+    """One page of matching messages, grouped into conversations. ``filter``
+    keeps conversations with unread mail / pinned ones (the list's Unread and
+    Pinned tabs). The first page also says how many conversations match in
+    all, where the search engine can count them."""
     if not q.strip():
         return SearchOut(threads=[], cursor=None)
 
     backend = get_search_backend()
     query = to_search_query(q, user.id)
     hits, next_cursor = await backend.query(query, limit=limit, cursor=cursor)
+    total: int | None = None
+    count = getattr(backend, "count_threads", None)
+    if cursor is None and count is not None:
+        total = await count(query, thread_filter=filter)
 
     # Group hits by thread, preserving order and keeping the first highlight.
     order: list[uuid.UUID] = []
@@ -48,7 +57,7 @@ async def search(
             highlights[hit.thread_id] = hit.highlights
 
     if not order:
-        return SearchOut(threads=[], cursor=next_cursor)
+        return SearchOut(threads=[], cursor=next_cursor, total=total)
 
     threads = {
         t.id: t
@@ -64,5 +73,9 @@ async def search(
         )
         for tid in order
         if tid in threads
+        and (
+            filter is None
+            or (threads[tid].unread_count > 0 if filter == "unread" else threads[tid].pinned)
+        )
     ]
-    return SearchOut(threads=results, cursor=next_cursor)
+    return SearchOut(threads=results, cursor=next_cursor, total=total)

backend/app/api/settings.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import re
 from typing import Annotated
 
 from fastapi import APIRouter, Depends
@@ -24,6 +25,11 @@ _ALLOWED_INVITE_POLICY = {"add_update", "update_only", "none"}
 _ALLOWED_INVITE_FROM = {"contacts", "anyone"}
 
 
+# "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de".
+_LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*")
+_UI_LANGUAGE = re.compile(r"[a-z]{2,3}(-[A-Za-z]{2,4})?")
+
+
 class ReminderPref(BaseModel):
     method: str = "alert"  # alert = in-app notification, email = email me
     minutes: int = 0  # minutes before the event start (0 = at time of event)
@@ -37,11 +43,26 @@ class SettingsOut(BaseModel):
     default_calendar: str = ""  # calendar id new events default to (blank => first)
     default_reminders_timed: list[ReminderPref] = []  # defaults for timed events
     default_reminders_allday: list[ReminderPref] = []  # defaults for all-day events
-    invitation_policy: str = "add_update"  # how emailed events update the calendar
+    # How emailed events update the calendar: none | update_only | add_update.
+    invitation_policy: str = "none"
     invitation_from: str = "contacts"  # whose emailed events auto-add (contacts|anyone)
     invitation_calendar: str = ""  # calendar id emailed events land in (blank => default)
     invitation_file: bool = False  # also mark the invite message read + filed
     allowed_image_senders: list[str] = []
+    # Spelling and grammar check in the compose window, and its language
+    # ("auto" detects it per message).
+    spellcheck_enabled: bool = True
+    spellcheck_language: str = "auto"
+    # Gmail-style keyboard shortcuts in the mail view (Settings → Keyboard shortcuts).
+    shortcuts_enabled: bool = True
+    # Remove tracking pixels from mail (Settings → Privacy).
+    block_trackers: bool = True
+    # Grey sentence endings in the compose window, taken with Tab.
+    completion_enabled: bool = True
+    # Interface language ("en", "pl", …); "" until chosen — the web app then
+    # picks one from the browser on first sign-in and saves it here.
+    ui_language: str = ""
+    # Names and words email translation leaves as they are.
 
 
 class SettingsPatch(BaseModel):
@@ -56,6 +77,12 @@ class SettingsPatch(BaseModel):
     invitation_from: str | None = None
     invitation_calendar: str | None = None
     invitation_file: bool | None = None
+    spellcheck_enabled: bool | None = None
+    spellcheck_language: str | None = None
+    shortcuts_enabled: bool | None = None
+    block_trackers: bool | None = None
+    completion_enabled: bool | None = None
+    ui_language: str | None = None
 
 
 class QuotaOut(BaseModel):
@@ -70,6 +97,11 @@ def allowed_image_senders(user: User) -> set[str]:
     return set()
 
 
+def blocks_trackers(user: User) -> bool:
+    """Whether tracking pixels are taken out of this user's mail (on unless switched off)."""
+    return bool(user.settings.get("block_trackers", True))
+
+
 def add_allowed_image_sender(user: User, email: str) -> None:
     current = sorted(allowed_image_senders(user))
     if email.lower() not in current:
@@ -108,11 +140,17 @@ def _settings_out(user: User) -> SettingsOut:
         default_calendar=str(s.get("default_calendar", "")),
         default_reminders_timed=_reminder_list(s.get("default_reminders_timed")),
         default_reminders_allday=_reminder_list(s.get("default_reminders_allday")),
-        invitation_policy=str(s.get("invitation_policy", "add_update")),
+        invitation_policy=str(s.get("invitation_policy", "none")),
         invitation_from=str(s.get("invitation_from", "contacts")),
         invitation_calendar=str(s.get("invitation_calendar", "")),
         invitation_file=bool(s.get("invitation_file", False)),
         allowed_image_senders=sorted(allowed_image_senders(user)),
+        spellcheck_enabled=bool(s.get("spellcheck_enabled", True)),
+        spellcheck_language=str(s.get("spellcheck_language", "auto")),
+        shortcuts_enabled=bool(s.get("shortcuts_enabled", True)),
+        block_trackers=blocks_trackers(user),
+        completion_enabled=bool(s.get("completion_enabled", True)),
+        ui_language=str(s.get("ui_language", "")),
     )
 
 
@@ -144,9 +182,7 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession)
     if body.default_calendar is not None:
         updates["default_calendar"] = body.default_calendar
     if body.default_reminders_timed is not None:
-        updates["default_reminders_timed"] = [
-            r.model_dump() for r in body.default_reminders_timed
-        ]
+        updates["default_reminders_timed"] = [r.model_dump() for r in body.default_reminders_timed]
     if body.default_reminders_allday is not None:
         updates["default_reminders_allday"] = [
             r.model_dump() for r in body.default_reminders_allday
@@ -159,6 +195,18 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession)
         updates["invitation_calendar"] = body.invitation_calendar
     if body.invitation_file is not None:
         updates["invitation_file"] = body.invitation_file
+    if body.spellcheck_enabled is not None:
+        updates["spellcheck_enabled"] = body.spellcheck_enabled
+    if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language):
+        updates["spellcheck_language"] = body.spellcheck_language
+    if body.shortcuts_enabled is not None:
+        updates["shortcuts_enabled"] = body.shortcuts_enabled
+    if body.block_trackers is not None:
+        updates["block_trackers"] = body.block_trackers
+    if body.completion_enabled is not None:
+        updates["completion_enabled"] = body.completion_enabled
+    if body.ui_language is not None and _UI_LANGUAGE.fullmatch(body.ui_language):
+        updates["ui_language"] = body.ui_language
     user.settings = updates
     await db.commit()
     return _settings_out(user)

backend/app/api/snooze.py

@@ -0,0 +1,76 @@
+"""Snooze endpoints: hide a conversation from the Inbox until a set time."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import snooze
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.events import get_event_bus
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Thread
+
+router = APIRouter(prefix="/threads", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class SnoozeIn(BaseModel):
+    until: dt.datetime
+
+
+class SnoozeOut(BaseModel):
+    thread_id: uuid.UUID
+    snoozed_until: dt.datetime | None
+
+
+async def _thread(db: AsyncSession, user_id: uuid.UUID, thread_id: uuid.UUID) -> Thread:
+    thread = await db.scalar(
+        select(Thread).where(Thread.id == thread_id, Thread.user_id == user_id)
+    )
+    if thread is None:
+        raise_api_error(404, "not_found", "thread not found")
+    return thread
+
+
+async def _announce(user_id: uuid.UUID, thread_id: uuid.UUID) -> None:
+    await get_event_bus().publish(
+        "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}
+    )
+
+
+@router.post("/{thread_id}/snooze", response_model=SnoozeOut)
+async def snooze_thread(
+    thread_id: uuid.UUID, body: SnoozeIn, user: CurrentUser, db: DbSession
+) -> SnoozeOut:
+    until = body.until
+    if until.tzinfo is None:
+        raise_api_error(400, "invalid_time", "the time needs a time zone")
+    now = dt.datetime.now(dt.UTC)
+    if until <= now:
+        raise_api_error(400, "invalid_time", "pick a time in the future")
+    if until > now + snooze.MAX_SNOOZE:
+        raise_api_error(400, "invalid_time", "snooze for at most a year")
+    thread = await _thread(db, user.id, thread_id)
+    snooze.snooze(thread, until, now)
+    await db.commit()
+    await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=thread.snoozed_until)
+
+
+@router.post("/{thread_id}/unsnooze", response_model=SnoozeOut)
+async def unsnooze_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -> SnoozeOut:
+    thread = await _thread(db, user.id, thread_id)
+    if thread.snoozed_until is not None:
+        snooze.wake(thread, dt.datetime.now(dt.UTC))
+        await db.commit()
+        await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=None)

backend/app/api/spellcheck.py

@@ -0,0 +1,187 @@
+"""Spelling and grammar check for the compose window, backed by LanguageTool.
+
+The browser sends the text it wants checked here, never to LanguageTool
+itself: the engine stays on the internal network, only signed-in users reach
+it, and nothing leaves the server. When no engine is configured (or it is
+down), the check answers 503 and the compose window simply goes without.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections import Counter
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.identity.deps import CurrentUser, require_csrf
+
+router = APIRouter(prefix="/spellcheck", tags=["compose"])
+
+_TIMEOUT = 10.0
+# A message body is checked as a whole; past this it is a document, not mail.
+MAX_TEXT = 20_000
+_MAX_SUGGESTIONS = 6
+# With "auto", LanguageTool settles on ONE language per request, so a message
+# mixing Polish and English paragraphs would have every English word flagged as
+# a Polish typo. Auto-detected text is therefore checked in chunks, each in its
+# own language. A chunk is at least this many characters of consecutive lines:
+# a greeting alone ("Witam,") is too short to detect reliably, so it rides with
+# the lines after it.
+_MIN_CHUNK = 60
+_PARALLEL = 4
+
+
+class CheckIn(BaseModel):
+    text: str = Field(max_length=MAX_TEXT)
+    # "auto" or a LanguageTool code ("pl-PL", "en-GB", "de-DE" ...).
+    language: str = Field(default="auto", max_length=16)
+
+
+class Issue(BaseModel):
+    offset: int
+    length: int
+    message: str
+    short_message: str
+    replacements: list[str]
+    # spelling | grammar | style: what the underline colour follows.
+    kind: str
+    rule_id: str
+
+
+class CheckOut(BaseModel):
+    language: str  # the language checked (detected, with "auto")
+    language_name: str
+    issues: list[Issue]
+
+
+class LanguageOut(BaseModel):
+    code: str
+    name: str
+
+
+def _base() -> str:
+    url = get_settings().languagetool_url.rstrip("/")
+    if not url:
+        raise_api_error(503, "spellcheck_unavailable", "spell checking is not set up")
+    return url
+
+
+def _kind(issue_type: str) -> str:
+    if issue_type == "misspelling":
+        return "spelling"
+    if issue_type in ("style", "typographical", "whitespace", "register", "locale-violation"):
+        return "style"
+    return "grammar"
+
+
+def chunks(text: str) -> list[tuple[int, str]]:
+    """Split ``text`` into (offset, chunk) runs of whole lines, each at least
+    _MIN_CHUNK characters where the text allows; a blank line always ends one."""
+    out: list[tuple[int, str]] = []
+    start = pos = 0
+    for line in text.split("\n"):
+        end = pos + len(line)
+        if not line.strip():
+            if text[start:pos].strip():
+                # Up to the line break before the blank line, not including it.
+                out.append((start, text[start:pos].rstrip("\n")))
+            start = end + 1
+        elif end - start >= _MIN_CHUNK:
+            out.append((start, text[start:end]))
+            start = end + 1
+        pos = end + 1
+    if text[start:].strip():
+        out.append((start, text[start:]))
+    return out
+
+
+async def _check_one(
+    client: httpx.AsyncClient, base: str, text: str, language: str
+) -> dict[str, Any]:
+    resp = await client.post(f"{base}/v2/check", data={"text": text, "language": language})
+    resp.raise_for_status()
+    data: dict[str, Any] = resp.json()
+    return data
+
+
+def _issues(data: dict[str, Any], shift: int) -> list[Issue]:
+    return [
+        Issue(
+            offset=int(m["offset"]) + shift,
+            length=int(m["length"]),
+            message=str(m.get("message", "")),
+            short_message=str(m.get("shortMessage", "")),
+            replacements=[str(r["value"]) for r in (m.get("replacements") or [])][
+                :_MAX_SUGGESTIONS
+            ],
+            kind=_kind(str((m.get("rule") or {}).get("issueType", ""))),
+            rule_id=str((m.get("rule") or {}).get("id", "")),
+        )
+        for m in data.get("matches") or []
+    ]
+
+
+def _detected(data: dict[str, Any]) -> tuple[str, str]:
+    lang = data.get("language") or {}
+    detected = lang.get("detectedLanguage") or lang
+    return str(detected.get("code", "")), str(detected.get("name", ""))
+
+
+@router.post("", response_model=CheckOut, dependencies=[Depends(require_csrf)])
+async def check(body: CheckIn, user: CurrentUser) -> CheckOut:
+    base = _base()
+    if not body.text.strip():
+        return CheckOut(language=body.language, language_name="", issues=[])
+    parts = chunks(body.text) if body.language == "auto" else [(0, body.text)]
+    gate = asyncio.Semaphore(_PARALLEL)
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+
+            async def one(text: str) -> dict[str, Any]:
+                async with gate:
+                    return await _check_one(client, base, text, body.language)
+
+            results = await asyncio.gather(*(one(text) for _, text in parts))
+    except (httpx.HTTPError, ValueError):
+        raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering")
+
+    issues = [
+        i for (shift, _), data in zip(parts, results, strict=True) for i in _issues(data, shift)
+    ]
+    # The language reported is the one most of the text was written in.
+    weight: Counter[tuple[str, str]] = Counter()
+    for (_, text), data in zip(parts, results, strict=True):
+        weight[_detected(data)] += len(text)
+    (code, name), _ = weight.most_common(1)[0]
+    return CheckOut(language=code or body.language, language_name=name, issues=issues)
+
+
+_languages_cache: tuple[float, list[LanguageOut]] | None = None
+
+
+@router.get("/languages", response_model=list[LanguageOut])
+async def languages(user: CurrentUser) -> list[LanguageOut]:
+    """What the engine can check, for the language picker in Settings."""
+    global _languages_cache
+    if _languages_cache and time.monotonic() - _languages_cache[0] < 3600:
+        return _languages_cache[1]
+    base = _base()
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(f"{base}/v2/languages")
+            resp.raise_for_status()
+            rows: list[dict[str, Any]] = resp.json()
+    except (httpx.HTTPError, ValueError):
+        raise_api_error(503, "spellcheck_unavailable", "the spell checker is not answering")
+    seen: dict[str, str] = {}
+    for row in rows:
+        seen.setdefault(str(row["longCode"]), str(row["name"]))
+    out = sorted((LanguageOut(code=c, name=n) for c, n in seen.items()), key=lambda x: x.name)
+    _languages_cache = (time.monotonic(), out)
+    return out

backend/app/api/subscriptions.py

@@ -0,0 +1,372 @@
+"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import case, func, literal, or_, select
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.elements import ColumnElement
+
+from app import subscriptions as subs
+from app.accounts.ssrf import guarded_request
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Mailbox, Message, SubscriptionState
+from app.outbox.schemas import DraftIn, DraftPayload
+from app.outbox.service import DraftError, create_outbox, upsert_draft
+from app.queue import enqueue_job
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+# RFC 8058 senders should stop within two days of a one-click request.
+GRACE = dt.timedelta(days=2)
+# Not counted: mail the user sent, and mail already thrown away (Trash, Spam) —
+# a sender whose mail was all deleted drops off the list.
+_LEFT_OUT = ("sent", "drafts", "trash", "junk")
+
+
+class SubscriptionOut(BaseModel):
+    key: str
+    name: str
+    email: str
+    list_id: str | None
+    total: int
+    unread: int
+    last_30_days: int
+    per_month: float
+    first_at: dt.datetime
+    last_at: dt.datetime
+    last_subject: str
+    one_click: bool
+    has_mailto: bool
+    has_web: bool
+    unsubscribed_at: dt.datetime | None
+    method: str | None
+    still_sending: bool
+
+
+class SubscriptionListOut(BaseModel):
+    subscriptions: list[SubscriptionOut]  # one page
+    total: int  # senders in all
+    pending: int  # messages whose headers are still being read
+
+
+class KeyIn(BaseModel):
+    key: str
+
+
+class UnsubscribeIn(KeyIn):
+    # From the reading pane: this message's own header, wherever it is filed
+    # (Spam included, which the page's groups leave out).
+    message_id: uuid.UUID | None = None
+
+
+class UnsubscribeOut(BaseModel):
+    status: str  # done | open
+    method: str  # one_click | mailto | web
+    url: str | None = None
+
+
+def _sender() -> ColumnElement[Any]:
+    return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, ""))
+
+
+def _key() -> ColumnElement[Any]:
+    return case(
+        (Message.list_id.isnot(None), literal("list:") + Message.list_id),
+        else_=literal("from:") + _sender(),
+    )
+
+
+def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]:
+    return [
+        Message.user_id == user_id,
+        Message.list_unsubscribe.isnot(None),
+        Message.list_unsubscribe != "",
+        or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)),
+    ]
+
+
+async def _start_backfill(user_id: uuid.UUID) -> None:
+    # One job per user at a time: arq refuses a second while the first exists.
+    await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}")
+
+
+@router.get("", response_model=SubscriptionListOut)
+async def list_subscriptions(
+    user: CurrentUser,
+    db: DbSession,
+    offset: int = Query(0, ge=0),
+    limit: int = Query(50, ge=1, le=200),
+    order: str = Query("desc"),  # desc: heard from most recently first | asc: longest ago
+) -> SubscriptionListOut:
+    """One page of senders, by when their latest mail came."""
+    pending = await subs.pending_count(db, user.id)
+    if pending:
+        await _start_backfill(user.id)
+
+    now = dt.datetime.now(dt.UTC)
+    key = _key().label("key")
+    seen = Message.flags.contains(["\\Seen"])
+    groups = (
+        select(
+            key,
+            func.count().label("total"),
+            func.count().filter(~seen).label("unread"),
+            func.count()
+            .filter(Message.internal_date > now - dt.timedelta(days=30))
+            .label("recent"),
+            func.min(Message.internal_date).label("first_at"),
+            func.max(Message.internal_date).label("last_at"),
+        )
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id))
+        .group_by(key)
+    )
+    total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0
+    agg_rows = (
+        await db.execute(
+            groups.order_by(
+                func.max(Message.internal_date).asc()
+                if order == "asc"
+                else func.max(Message.internal_date).desc(),
+                key,
+            )
+            .offset(offset)
+            .limit(limit)
+        )
+    ).all()
+    page_keys = [r.key for r in agg_rows]
+    if not page_keys:
+        return SubscriptionListOut(subscriptions=[], total=total, pending=pending)
+
+    in_page = _key().in_(page_keys)
+    latest_rows = (
+        await db.execute(
+            select(
+                key,
+                Message.from_addr,
+                Message.subject,
+                Message.list_id,
+                Message.list_unsubscribe,
+                Message.list_unsubscribe_post,
+            )
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(*_bulk(user.id), in_page)
+            .distinct(key)
+            .order_by(key, Message.internal_date.desc())
+        )
+    ).all()
+    latest = {r.key: r for r in latest_rows}
+    states = {
+        s.key: s
+        for s in await db.scalars(
+            select(SubscriptionState).where(
+                SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys)
+            )
+        )
+    }
+    # Senders still mailing after the grace period, for the whole page at once.
+    still_sending = set(
+        await db.scalars(
+            select(key)
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .join(
+                SubscriptionState,
+                (SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()),
+            )
+            .where(
+                *_bulk(user.id),
+                in_page,
+                Message.internal_date > SubscriptionState.unsubscribed_at + GRACE,
+            )
+            .distinct()
+        )
+    )
+
+    out: list[SubscriptionOut] = []
+    for row in agg_rows:
+        last = latest.get(row.key)
+        if last is None:
+            continue
+        sender = last.from_addr[0] if last.from_addr else {}
+        targets = subs.unsubscribe_targets(last.list_unsubscribe or "")
+        state = states.get(row.key)
+        months = max(1.0, (row.last_at - row.first_at).days / 30.44)
+        out.append(
+            SubscriptionOut(
+                key=row.key,
+                name=sender.get("name") or sender.get("email", ""),
+                email=sender.get("email", ""),
+                list_id=last.list_id,
+                total=row.total,
+                unread=row.unread,
+                last_30_days=row.recent,
+                per_month=round(row.total / months, 1),
+                first_at=row.first_at,
+                last_at=row.last_at,
+                last_subject=last.subject,
+                one_click=bool(last.list_unsubscribe_post and targets.https),
+                has_mailto=targets.mailto is not None,
+                has_web=targets.https is not None,
+                unsubscribed_at=state.unsubscribed_at if state else None,
+                method=state.method if state else None,
+                still_sending=row.key in still_sending,
+            )
+        )
+    return SubscriptionListOut(subscriptions=out, total=total, pending=pending)
+
+
+@router.get("/threads", response_model=list[uuid.UUID])
+async def subscription_threads(
+    user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200)
+) -> list[uuid.UUID]:
+    """The group's conversations, newest first (for the reading pane)."""
+    rows = await db.execute(
+        select(Message.thread_id, func.max(Message.internal_date).label("at"))
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None))
+        .group_by(Message.thread_id)
+        .order_by(func.max(Message.internal_date).desc())
+        .limit(limit)
+    )
+    return [r.thread_id for r in rows if r.thread_id is not None]
+
+
+async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message:
+    message = await db.scalar(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user_id), _key() == key)
+        .order_by(Message.internal_date.desc())
+        .limit(1)
+    )
+    if message is None:
+        raise_api_error(404, "not_found", "no mail from this sender")
+    return message
+
+
+async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None:
+    stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method)
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["user_id", "key"],
+            set_={"method": method, "unsubscribed_at": func.now()},
+        )
+    )
+    await db.commit()
+
+
+async def _one_click(url: str) -> bool:
+    try:
+        resp = await guarded_request(
+            "POST",
+            url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            content="List-Unsubscribe=One-Click",
+            max_bytes=256 * 1024,
+        )
+    except Exception as exc:  # SSRF refusal, network, TLS: any failure tries the next way
+        logger.info("one-click unsubscribe failed: %s", exc)
+        return False
+    return 200 <= resp.status < 400
+
+
+async def _send_mailto(
+    db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets
+) -> bool:
+    assert targets.mailto is not None
+    body = DraftIn(
+        account_id=message.account_id,
+        kind="new",
+        payload=DraftPayload(
+            to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body
+        ),
+    )
+    try:
+        draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None)
+    except DraftError:
+        return False
+    outbox = await create_outbox(db, user_id=user_id, draft=draft)
+    await db.commit()
+    await enqueue_job(
+        "send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds)
+    )
+    return True
+
+
+@router.post("/unsubscribe", response_model=UnsubscribeOut)
+async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut:
+    """Unsubscribe from a sender: one-click if offered, else by mail, else the web page."""
+    if body.message_id is not None:
+        found = await db.scalar(
+            select(Message).where(
+                Message.id == body.message_id,
+                Message.user_id == user.id,
+                Message.list_unsubscribe.isnot(None),
+                Message.list_unsubscribe != "",
+            )
+        )
+        if found is None:
+            raise_api_error(404, "not_found", "this message offers no way to unsubscribe")
+        message = found
+    else:
+        message = await _latest_with_header(db, user.id, body.key)
+    targets = subs.unsubscribe_targets(message.list_unsubscribe or "")
+    if message.list_unsubscribe_post and targets.https and await _one_click(targets.https):
+        await _record(db, user.id, body.key, "one_click")
+        return UnsubscribeOut(status="done", method="one_click")
+    if targets.mailto and await _send_mailto(db, user.id, message, targets):
+        await _record(db, user.id, body.key, "mailto")
+        return UnsubscribeOut(status="done", method="mailto")
+    if targets.https:
+        # The sender wants it done on their page: the user opens it.
+        await _record(db, user.id, body.key, "web")
+        return UnsubscribeOut(status="open", method="web", url=targets.https)
+    raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe")
+
+
+@router.post("/trash")
+async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]:
+    """Move every conversation from this sender to Trash.
+
+    Only conversations made up entirely of this sender's mail: one where the
+    user replied, or someone else wrote, stays where it is.
+    """
+    rows = (
+        await db.execute(
+            select(Message.thread_id, _key().label("key"), Mailbox.special_use)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(
+                Message.user_id == user.id,
+                Message.thread_id.in_(
+                    select(Message.thread_id)
+                    .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                    .where(*_bulk(user.id), _key() == body.key)
+                ),
+            )
+        )
+    ).all()
+    own: dict[uuid.UUID, bool] = {}
+    for thread_id, key, special in rows:
+        if thread_id is None or special == "trash":
+            continue
+        own[thread_id] = own.get(thread_id, True) and key == body.key
+    moved = [tid for tid, only_theirs in own.items() if only_theirs]
+    for tid in moved:
+        await enqueue_job("move_thread_to_folder_job", str(tid), "trash")
+    return {"moved": len(moved)}

backend/app/api/templates.py

@@ -0,0 +1,129 @@
+"""Templates: saved replies, inserted from the compose window's More menu."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+import nh3
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import MailTemplate
+
+router = APIRouter(prefix="/templates", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+MAX_TEMPLATES = 200
+MAX_BODY_BYTES = 256 * 1024
+
+# What the compose editor writes. A template goes back into the editor as HTML,
+# so it is cleaned like any HTML the app renders: no scripts, no handlers.
+_TAGS = {
+    "a", "b", "blockquote", "br", "code", "div", "em", "font", "h1", "h2", "h3",
+    "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strike", "strong",
+    "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "u", "ul",
+}  # fmt: skip
+_GENERIC = {"style", "class", "dir", "align"}
+_SPECIFIC = {
+    "a": {"href", "title"},
+    "img": {"src", "alt", "width", "height"},
+    "font": {"color", "face", "size"},
+    "td": {"colspan", "rowspan"},
+    "th": {"colspan", "rowspan"},
+}
+# nh3 takes no wildcard: the generic attributes go onto every tag.
+_ATTRS = {tag: _GENERIC | _SPECIFIC.get(tag, set()) for tag in _TAGS}
+
+
+class TemplateOut(BaseModel):
+    id: uuid.UUID
+    name: str
+    subject: str
+    body_html: str
+    updated_at: dt.datetime
+
+
+class TemplateIn(BaseModel):
+    name: str = Field(min_length=1, max_length=80)
+    subject: str = Field(default="", max_length=500)
+    body_html: str = ""
+
+
+def _out(t: MailTemplate) -> TemplateOut:
+    return TemplateOut(
+        id=t.id, name=t.name, subject=t.subject, body_html=t.body_html, updated_at=t.updated_at
+    )
+
+
+def clean_body(html: str) -> str:
+    return nh3.clean(
+        html,
+        tags=_TAGS,
+        attributes=_ATTRS,
+        url_schemes={"http", "https", "mailto", "data"},
+        link_rel="noopener noreferrer",
+    )
+
+
+@router.get("", response_model=list[TemplateOut])
+async def list_templates(user: CurrentUser, db: DbSession) -> list[TemplateOut]:
+    rows = await db.scalars(
+        select(MailTemplate)
+        .where(MailTemplate.user_id == user.id)
+        .order_by(func.lower(MailTemplate.name))
+    )
+    return [_out(t) for t in rows]
+
+
+@router.post("", response_model=TemplateOut)
+async def save_template(body: TemplateIn, user: CurrentUser, db: DbSession) -> TemplateOut:
+    """Save a template. Saving under a name that exists replaces that template."""
+    name = " ".join(body.name.split())
+    if not name:
+        raise_api_error(400, "invalid", "a template needs a name")
+    if len(body.body_html.encode()) > MAX_BODY_BYTES:
+        raise_api_error(400, "too_large", "the template is too large")
+    html = clean_body(body.body_html)
+    subject = " ".join(body.subject.split())
+    existing = await db.scalar(
+        select(MailTemplate).where(
+            MailTemplate.user_id == user.id, func.lower(MailTemplate.name) == name.lower()
+        )
+    )
+    if existing is not None:
+        existing.name, existing.subject, existing.body_html = name, subject, html
+        existing.updated_at = dt.datetime.now(dt.UTC)
+        saved = existing
+    else:
+        count = await db.scalar(
+            select(func.count()).select_from(MailTemplate).where(MailTemplate.user_id == user.id)
+        )
+        if (count or 0) >= MAX_TEMPLATES:
+            raise_api_error(409, "too_many", f"at most {MAX_TEMPLATES} templates")
+        saved = MailTemplate(user_id=user.id, name=name, subject=subject, body_html=html)
+        db.add(saved)
+    await db.commit()
+    await db.refresh(saved)
+    return _out(saved)
+
+
+@router.delete("/{template_id}")
+async def delete_template(
+    template_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    row = await db.scalar(
+        select(MailTemplate).where(MailTemplate.id == template_id, MailTemplate.user_id == user.id)
+    )
+    if row is None:
+        raise_api_error(404, "not_found", "template not found")
+    await db.delete(row)
+    await db.commit()
+    return {"status": "deleted"}

backend/app/api/threads.py

@@ -4,13 +4,14 @@ from __future__ import annotations
 
 import uuid
 from collections.abc import Collection
-from typing import Annotated
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Query
 from sqlalchemy import Text, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.sql.elements import ColumnElement
 
+from app import subscriptions as subs
 from app.api.errors import raise_api_error
 from app.api.schemas import (
     Address,
@@ -19,10 +20,10 @@ from app.api.schemas import (
     ThreadListOut,
     ThreadSummary,
 )
-from app.api.settings import allowed_image_senders
+from app.api.settings import allowed_image_senders, blocks_trackers
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Mailbox, Message, Thread, User
+from app.models import BodyPart, Mailbox, Message, SubscriptionState, Thread, User
 
 router = APIRouter(prefix="/threads", tags=["mail"])
 
@@ -50,6 +51,7 @@ def _summary(
         has_attachments=thread.has_attachments,
         pinned=thread.pinned,
         last_message_at=thread.last_message_at,
+        snoozed_until=thread.snoozed_until,
     )
 
 
@@ -74,6 +76,7 @@ def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnv
         images_allowed=bool(sender) and sender in allowed_senders,
         size_bytes=msg.size_bytes,
         snippet=msg.snippet,
+        content_language=msg.content_language or "",
     )
 
 
@@ -130,6 +133,10 @@ async def list_threads(
             .where(Message.user_id == user.id, Mailbox.special_use == special)
         )
         conditions.append(Thread.id.in_(in_folder))
+        if special == "inbox":
+            conditions.append(Thread.snoozed_until.is_(None))  # back when it wakes
+    elif view == "snoozed":
+        conditions.append(Thread.snoozed_until.isnot(None))
     elif view != "unified":
         raise_api_error(400, "bad_view", "unsupported view")
 
@@ -144,11 +151,20 @@ async def list_threads(
         )
     ) or 0
 
-    order_by = (
-        (Thread.last_message_at.asc(), Thread.id.asc())
-        if order == "asc"
-        else (Thread.last_message_at.desc(), Thread.id.desc())
+    # A thread woken from snooze sorts as if it had just arrived.
+    sort_at = func.greatest(
+        Thread.last_message_at, func.coalesce(Thread.unsnoozed_at, Thread.last_message_at)
     )
+    if view == "snoozed":
+        # Soonest back first, whatever the list order.
+        order_by: tuple[ColumnElement[Any], ...] = (
+            Thread.snoozed_until.asc(),
+            Thread.id.asc(),
+        )
+    elif order == "asc":
+        order_by = (sort_at.asc(), Thread.id.asc())
+    else:
+        order_by = (sort_at.desc(), Thread.id.desc())
     stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit)
     threads = list(await db.scalars(stmt))
 
@@ -177,6 +193,41 @@ async def list_threads(
     )
 
 
+async def _mark_subscriptions(
+    db: AsyncSession,
+    user_id: uuid.UUID,
+    messages: list[Message],
+    envelopes: list[MessageEnvelope],
+) -> None:
+    """Tell the reading pane which messages can be unsubscribed from, and how."""
+    keys: dict[uuid.UUID, str] = {}
+    for m, env in zip(messages, envelopes, strict=True):
+        if not m.list_unsubscribe:
+            continue
+        targets = subs.unsubscribe_targets(m.list_unsubscribe)
+        if m.list_unsubscribe_post and targets.https:
+            env.unsubscribe_via = "one_click"
+        elif targets.mailto:
+            env.unsubscribe_via = "mailto"
+        elif targets.https:
+            env.unsubscribe_via = "web"
+        else:
+            continue
+        sender = m.from_addr[0].get("email", "") if m.from_addr else ""
+        env.subscription_key = keys[m.id] = subs.group_key(m.list_id, sender)
+    if keys:
+        done = set(
+            await db.scalars(
+                select(SubscriptionState.key).where(
+                    SubscriptionState.user_id == user_id,
+                    SubscriptionState.key.in_(set(keys.values())),
+                )
+            )
+        )
+        for env in envelopes:
+            env.unsubscribed = env.subscription_key in done
+
+
 def _visible_in(current_special: str | None) -> ColumnElement[bool]:
     """Which of a thread's messages a view shows (see ``_REMOVAL_FOLDERS``): Trash/Spam
     show only their own messages; every other view hides them."""
@@ -281,6 +332,20 @@ async def get_thread(
         env = _envelope(m, allowed_senders)
         env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"]
         envelopes.append(env)
+    await _mark_subscriptions(db, user.id, ordered, envelopes)
+    if not blocks_trackers(user) and ordered:
+        # Trackers left in are remote images: offer "Load images" for them too.
+        with_trackers = set(
+            await db.scalars(
+                select(BodyPart.message_id).where(
+                    BodyPart.message_id.in_([m.id for m in ordered]),
+                    BodyPart.kind == "html_sanitized",
+                    BodyPart.trackers_blocked > 0,
+                )
+            )
+        )
+        for env in envelopes:
+            env.has_remote_images = env.has_remote_images or env.id in with_trackers
 
     order = "desc" if sort == "desc" else "asc"
     return ThreadDetail(
@@ -289,4 +354,5 @@ async def get_thread(
         order=order,
         pinned=thread.pinned,
         messages=envelopes,
+        snoozed_until=thread.snoozed_until,
     )

backend/app/api/translation.py

@@ -0,0 +1,30 @@
+"""What email translation offers (app/translation.py), for the reading pane."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+from app import translation
+from app.identity.deps import CurrentUser
+
+router = APIRouter(prefix="/translation", tags=["mail"])
+
+
+class LanguageOut(BaseModel):
+    code: str
+    name: str
+
+
+class TranslationInfoOut(BaseModel):
+    available: bool
+    languages: list[LanguageOut]
+
+
+@router.get("/languages", response_model=TranslationInfoOut)
+async def translation_languages(user: CurrentUser) -> TranslationInfoOut:
+    found = await translation.languages()
+    return TranslationInfoOut(
+        available=len(found) > 1,  # English alone translates nothing
+        languages=[LanguageOut(code=x.code, name=x.name) for x in found],
+    )

backend/app/completion.py

@@ -0,0 +1,391 @@
+"""Sentence completion in the compose window: phrases the user writes, finished.
+
+No AI: a per-user index of the word sequences the user has written at least
+twice, built from their own sent mail (quoted replies and signatures cut off),
+plus a few common email phrases per language to start from. Typing "I look
+forward" finds what usually follows those words — "to hearing from you." — and
+the compose window shows it in grey until Tab takes it.
+
+It works in any language the user writes in: matching is on the words just
+typed, so a Polish sentence only ever meets Polish phrases.
+
+The index lives in ``completion_phrases`` (context → continuation, with how
+often it was seen) and is rebuilt in the background at most once a day, when
+the compose window asks for suggestions (:func:`build_index_job`).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import html as html_lib
+import logging
+import re
+import uuid
+from collections import Counter
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import delete, select
+from sqlalchemy.dialects.postgresql import insert
+
+from app.db import session_scope
+from app.models import (
+    Account,
+    BodyPart,
+    CompletionIndexState,
+    CompletionPhrase,
+    Mailbox,
+    Message,
+)
+
+logger = logging.getLogger(__name__)
+
+START = "^"  # a sentence's start, as a word: "^ pozdr" completes the first word
+MAX_CONTEXT = 3  # words before the cursor matched on
+MAX_CONTINUATION = 6  # words suggested at most
+MIN_COUNT = 2  # seen this often to be offered; one-offs are noise (and private)
+EXTEND_SHARE = 0.6  # a longer suggestion only while it is what usually follows
+CORPUS_MESSAGES = 500  # most recent sent messages read
+CORPUS_CHARS = 3000  # of each
+REBUILD_AFTER = dt.timedelta(days=1)
+
+# Where the user's own words end and quoted mail or a signature begins: a reply
+# header ("On … wrote:", Polish Gmail's "… napisał(a):", …), a forwarded or
+# original-message marker, or an Outlook-style header block.
+_QUOTE_START = re.compile(
+    r"^\s*(?:"
+    r"-{2,}\s*(?:original message|forwarded message|wiadomość oryginalna"
+    r"|wiadomość przekazana dalej|ursprüngliche nachricht|weitergeleitete nachricht)"
+    r"|(?:from|od|von|de):\s.+"
+    r"|sent from my\s"
+    r")",
+    re.IGNORECASE,
+)
+_QUOTE_HEADER_END = re.compile(
+    # German names the writer after the verb: "Am … schrieb Anna:".
+    r"(?:wrote|napisał\(a\)|napisała?|pisze|schrieb(?:\s+.{0,100})?|a écrit|escribió)\s*:\s*$",
+    re.IGNORECASE,
+)
+_SIGNATURE = re.compile(r"^--\s*$")
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+|\n+")
+_WORD = re.compile(r"\S+")
+_EDGE_PUNCT = re.compile(r"^[\"'“”„«»(\[–—-]+|[\"'“”„«»)\],.;:!?…–—-]+$")
+# Words a phrase is never learned across: numbers (phones, dates, prices),
+# addresses and links — useless to complete, and often private.
+_NOT_A_WORD = re.compile(r"\d|@|www\.|https?:", re.IGNORECASE)
+
+
+def html_to_text(html: str) -> str:
+    """An HTML signature as text lines (block tags and <br> break lines)."""
+    text = re.sub(r"(?i)<br\s*/?>|</(?:p|div|li|tr|h[1-6])\s*>", "\n", html)
+    text = re.sub(r"<[^>]+>", "", text)
+    return html_lib.unescape(text)
+
+
+def own_text(text: str, signatures: Iterable[str] = ()) -> str:
+    """The part of a sent message the user wrote: no quoted mail, no signature
+    (the ``--`` kind, or one of ``signatures`` — the accounts' own, as text)."""
+    sig_lines = {norm_line(x) for sig in signatures for x in sig.splitlines() if norm_line(x)}
+    kept: list[str] = []
+    for line in text.splitlines():
+        if _SIGNATURE.match(line) or _QUOTE_START.match(line) or _QUOTE_HEADER_END.search(line):
+            break
+        if norm_line(line) in sig_lines:
+            continue
+        if line.lstrip().startswith(">"):
+            continue  # an inline-quoted line
+        kept.append(line)
+    return "\n".join(kept)
+
+
+def norm_line(line: str) -> str:
+    return " ".join(line.lower().split())
+
+
+def norm(word: str) -> str:
+    """A word as matched: lower case, outer punctuation off ("Hello," -> "hello")."""
+    return _EDGE_PUNCT.sub("", word.lower())
+
+
+def sentences(text: str) -> Iterable[list[str]]:
+    for chunk in _SENTENCE_END.split(text):
+        words = _WORD.findall(chunk)
+        if words:
+            yield words
+
+
+@dataclass(frozen=True)
+class Phrase:
+    context: str  # normalized words before the cursor, space-joined ("^ i look")
+    continuation: str  # the words that follow, as written ("forward to hearing")
+    count: int
+
+
+def build_phrases(texts: Iterable[str], weight: int = 1) -> dict[tuple[str, str], int]:
+    """(context, continuation) -> count, for every sequence seen MIN_COUNT times.
+
+    Counts each sentence's word sequences (with the start marker), keeps the
+    frequent ones, and splits each into "what was typed" and "what follows".
+    """
+    counts: Counter[tuple[str, ...]] = Counter()
+    surface: dict[tuple[str, ...], tuple[str, ...]] = {}
+    longest = MAX_CONTEXT + MAX_CONTINUATION
+    for text in texts:
+        for words in sentences(text):
+            seq = [START, *words]
+            keys = [START, *("" if _NOT_A_WORD.search(w) else norm(w) for w in words)]
+            for i in range(len(seq)):
+                for n in range(2, min(longest, len(seq) - i) + 1):
+                    gram = tuple(keys[i : i + n])
+                    if "" in gram:
+                        break  # a lone punctuation mark: no phrase across it
+                    counts[gram] += weight
+                    surface.setdefault(gram, tuple(seq[i : i + n]))
+    out: dict[tuple[str, str], int] = {}
+    for gram, count in counts.items():
+        if count < MIN_COUNT:
+            continue
+        written = surface[gram]
+        for k in range(1, min(MAX_CONTEXT, len(gram) - 1) + 1):
+            rest = len(gram) - k
+            if rest > MAX_CONTINUATION:
+                continue
+            key = (" ".join(gram[:k]), " ".join(written[k:]))
+            out[key] = max(out.get(key, 0), count)
+    return out
+
+
+# A few common phrases per language, for users with little sent mail yet.
+STARTER_PHRASES: dict[str, list[str]] = {
+    "en": [
+        "I look forward to hearing from you.",
+        "I'm looking forward to hearing from you.",
+        "Looking forward to hearing from you.",
+        "Looking forward to working with you.",
+        "Thank you for your email.",
+        "Thank you for your help.",
+        "Thank you for your quick reply.",
+        "Thanks in advance.",
+        "Please let me know if you have any questions.",
+        "Let me know if you have any questions.",
+        "Let me know what you think.",
+        "Please find attached the",
+        "I hope this email finds you well.",
+        "I hope you're doing well.",
+        "Sorry for the late reply.",
+        "I would be happy to help.",
+        "Feel free to contact me if you need anything else.",
+        "Could you please send me the",
+        "Please confirm at your earliest convenience.",
+        "Have a great day!",
+        "Have a nice weekend!",
+        "Best regards,",
+        "Kind regards,",
+    ],
+    "pl": [
+        "Z poważaniem,",
+        "Pozdrawiam serdecznie,",
+        "Dziękuję za wiadomość.",
+        "Dziękuję za szybką odpowiedź.",
+        "Dziękuję za pomoc.",
+        "Z góry dziękuję.",
+        "W załączniku przesyłam",
+        "Proszę o informację zwrotną.",
+        "W razie pytań proszę o kontakt.",
+        "Czekam na odpowiedź.",
+        "Przepraszam za opóźnienie w odpowiedzi.",
+        "Będę wdzięczny za odpowiedź.",
+        "Będę wdzięczna za odpowiedź.",
+        "Zgodnie z naszą rozmową",
+        "Proszę o potwierdzenie.",
+        "Daj znać, co myślisz.",
+        "Miłego dnia!",
+        "Miłego weekendu!",
+    ],
+    "de": [
+        "Mit freundlichen Grüßen",
+        "Viele Grüße",
+        "Beste Grüße",
+        "Vielen Dank für Ihre E-Mail.",
+        "Vielen Dank für Ihre schnelle Antwort.",
+        "Vielen Dank im Voraus.",
+        "Ich freue mich auf Ihre Antwort.",
+        "Bei Fragen stehe ich Ihnen gerne zur Verfügung.",
+        "Im Anhang finden Sie",
+        "Entschuldigen Sie die späte Antwort.",
+        "Ich hoffe, es geht Ihnen gut.",
+        "Schönen Tag noch!",
+        "Schönes Wochenende!",
+    ],
+}
+
+# Built once: every starter phrase counts as seen MIN_COUNT times.
+_STARTERS: dict[str, Counter[str]] = {}
+for (_ctx, _cont), _n in build_phrases(
+    (p for phrases in STARTER_PHRASES.values() for p in phrases), weight=MIN_COUNT
+).items():
+    _STARTERS.setdefault(_ctx, Counter())[_cont] = _n
+
+_LEADING_PUNCT = re.compile(r"^[\"'“”„«»(\[]+")
+
+
+def _candidates(phrases: dict[str, Counter[str]], context: str, partial: str) -> Counter[str]:
+    """Continuations of ``context`` whose first word starts with ``partial``."""
+    out: Counter[str] = Counter()
+    for continuation, count in phrases.get(context, Counter()).items():
+        first = continuation.split(" ", 1)[0]
+        if partial and not first.lower().startswith(partial.lower()):
+            continue
+        if partial and len(first) <= len(partial):
+            continue  # that word is typed in full already
+        out[continuation] = count
+    return out
+
+
+def choose(candidates: Counter[str]) -> str | None:
+    """The suggestion: the most frequent next word, extended word by word only
+    while the longer phrase is what usually follows (EXTEND_SHARE)."""
+    if not candidates:
+        return None
+    by_first: Counter[str] = Counter()
+    for continuation, count in candidates.items():
+        first = continuation.split(" ", 1)[0]
+        by_first[first] = max(by_first[first], count)
+    chosen, chosen_count = by_first.most_common(1)[0]
+    while True:
+        depth = chosen.count(" ") + 1
+        longer = [
+            (c, n)
+            for c, n in candidates.items()
+            if c.startswith(chosen + " ") and c.count(" ") == depth
+        ]
+        if not longer:
+            return chosen
+        best, best_count = max(longer, key=lambda x: x[1])
+        if best_count < max(MIN_COUNT, EXTEND_SHARE * chosen_count):
+            return chosen
+        chosen, chosen_count = best, best_count
+
+
+def contexts(before: str) -> tuple[list[str], str]:
+    """For the text before the cursor: the contexts to try, longest first, and
+    the partly typed word ('' after a space). No contexts: nothing to suggest
+    (an empty sentence, or the cursor right after a punctuation mark)."""
+    if not before or not before.strip():
+        return [], ""
+    last = before[-1]
+    if not last.isspace() and not last.isalnum() and last not in "'’-":
+        return [], ""  # "you," or "end." — not mid-word, and no space typed yet
+    tail = _SENTENCE_END.split(before)[-1]
+    words = _WORD.findall(tail)
+    partial = ""
+    if words and not last.isspace():
+        partial = _LEADING_PUNCT.sub("", words.pop())
+    if not words and not partial:
+        return [], ""  # a new sentence with nothing in it yet
+    keys = [START, *(norm(w) for w in words)]
+    if "" in keys:  # a lone punctuation mark: the phrase starts after it
+        keys = keys[len(keys) - keys[::-1].index("") :]
+    if not keys:
+        return [], ""
+    return [" ".join(keys[-k:]) for k in range(min(MAX_CONTEXT, len(keys)), 0, -1)], partial
+
+
+def suggest(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """What to show after ``before`` (the text up to the cursor), or None.
+
+    The returned text starts where the cursor is: the rest of a partly typed
+    word, or (after a space) the next words. The user's own phrases and the
+    starter phrases count together.
+    """
+    tried, partial = contexts(before)
+    for context in tried:
+        merged = _candidates(personal, context, partial)
+        for cont, count in _candidates(_STARTERS, context, partial).items():
+            merged[cont] = merged.get(cont, 0) + count
+        if " " not in context and not partial:
+            # A single word before a space is a weak hint: only a strong habit.
+            merged = Counter({c: n for c, n in merged.items() if n >= 2 * MIN_COUNT})
+        best = choose(merged)
+        if best:
+            return best[len(partial) :] if partial else best
+    return None
+
+
+def complete(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """:func:`suggest`, and for a word typed in full but no space yet ("I look
+    forward"), what follows it — with the space ("␣to hearing from you.")."""
+    found = suggest(personal, before)
+    if found is None and before[-1:].isalnum():
+        after_space = suggest(personal, before + " ")
+        if after_space:
+            return " " + after_space
+    return found
+
+
+def keys_for(before: str) -> list[str]:
+    """The contexts :func:`complete` may look up, to fetch them from the index."""
+    keys = contexts(before)[0]
+    if before[-1:].isalnum():
+        keys += contexts(before + " ")[0]
+    return keys
+
+
+async def personal_phrases(
+    session: Any, user_id: uuid.UUID, keys: list[str]
+) -> dict[str, Counter[str]]:
+    rows = await session.execute(
+        select(
+            CompletionPhrase.context, CompletionPhrase.continuation, CompletionPhrase.count
+        ).where(CompletionPhrase.user_id == user_id, CompletionPhrase.context.in_(keys))
+    )
+    out: dict[str, Counter[str]] = {}
+    for context, continuation, count in rows:
+        out.setdefault(context, Counter())[continuation] = count
+    return out
+
+
+async def build_index_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Rebuild a user's phrase index from their recent sent mail. Returns its size."""
+    uid = uuid.UUID(user_id)
+    async with session_scope() as session:
+        texts = list(
+            await session.scalars(
+                select(BodyPart.content)
+                .join(Message, Message.id == BodyPart.message_id)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.user_id == uid,
+                    Mailbox.special_use == "sent",
+                    BodyPart.kind == "text",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(CORPUS_MESSAGES)
+            )
+        )
+        signatures = [
+            html_to_text(sig)
+            for sig in await session.scalars(
+                select(Account.signature).where(Account.user_id == uid, Account.signature != "")
+            )
+        ]
+        phrases = build_phrases(own_text(t, signatures)[:CORPUS_CHARS] for t in texts)
+        await session.execute(delete(CompletionPhrase).where(CompletionPhrase.user_id == uid))
+        rows = [
+            {"user_id": uid, "context": c, "continuation": cont, "count": n}
+            for (c, cont), n in phrases.items()
+        ]
+        for start in range(0, len(rows), 1000):
+            await session.execute(insert(CompletionPhrase), rows[start : start + 1000])
+        stmt = insert(CompletionIndexState).values(user_id=uid, built_at=dt.datetime.now(dt.UTC))
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=["user_id"], set_={"built_at": stmt.excluded.built_at}
+            )
+        )
+        await session.commit()
+    logger.info(
+        "completion index for %s: %d phrases from %d messages", user_id, len(rows), len(texts)
+    )
+    return len(rows)

backend/app/config.py

@@ -68,11 +68,56 @@ class Settings(BaseSettings):
     # Server-side sender avatars (Gravatar + domain favicon). Fetched by the server
     # (never the browser) so senders can't see the user's IP or read activity.
     sender_avatars_enabled: bool = Field(default=True, alias="SENDER_AVATARS_ENABLED")
+    # Spelling and grammar checking in the compose window (LanguageTool server,
+    # e.g. http://languagetool:8010). Empty: the check is off for everyone.
+    languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL")
+    # Email translation: the translator service (translator/, app/translation.py),
+    # e.g. http://translator:5000. Empty: translation is off.
+    translator_url: str = Field(default="", alias="TRANSLATOR_URL")
+    # New passwords are checked against known data breaches (Have I Been Pwned's
+    # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server).
+    password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK")
+    # The name the webmail gives itself when it connects to a mail server to
+    # send (SMTP EHLO). Empty: APP_DOMAIN — not the container's random name.
+    smtp_helo_name: str = Field(default="", alias="SMTP_HELO_NAME")
     # Auto-purge Trash/Spam messages older than this many days (0 disables).
     trash_retention_days: int = Field(default=30, alias="TRASH_RETENTION_DAYS")
 
+    # --- Hosted mailboxes (SPEC Appendix D, app/hosted/) -----------------------
+    # Off: no mail server, byte-identical to before (every account external).
+    hosted_mailboxes: bool = Field(default=False, alias="HOSTED_MAILBOXES")
+    # The mail server's management API (JMAP), signed in to as its recovery admin.
+    stalwart_url: str = Field(default="http://stalwart:8080", alias="STALWART_URL")
+    stalwart_admin_secret: str = Field(default="", alias="STALWART_ADMIN_SECRET")
+    # The webmail worker's own sign-in, which may act for any mailbox
+    # (``user%worker``): no user's password is ever stored (§D.3).
+    stalwart_master_secret: str = Field(default="", alias="STALWART_MASTER_SECRET")
+    # The mail server's role on Postgres (its database, and the directory view).
+    stalwart_db_password: str = Field(default="", alias="STALWART_DB_PASSWORD")
+    stalwart_db_name: str = Field(default="stalwart", alias="STALWART_DB_NAME")
+    # Where the mail server reaches Postgres (its own network view of it).
+    stalwart_pg_host: str = Field(default="postgres", alias="STALWART_PG_HOST")
+    # IMAP/SMTP as the webmail reaches them (TLS, certificate not checked: the
+    # traffic stays on the Docker network or loopback).
+    stalwart_imap_host: str = Field(default="stalwart", alias="STALWART_IMAP_HOST")
+    stalwart_imap_port: int = Field(default=993, alias="STALWART_IMAP_PORT")
+    stalwart_smtp_host: str = Field(default="stalwart", alias="STALWART_SMTP_HOST")
+    stalwart_smtp_port: int = Field(default=465, alias="STALWART_SMTP_PORT")
+    # The mail server's public name (SMTP greeting, reverse DNS, what mail apps
+    # are told to connect to). Default: mail.APP_DOMAIN.
+    mail_hostname: str = Field(default="", alias="MAIL_HOSTNAME")
+    # Raw messages of hosted mailboxes are kept only in a cache (§D.6 item 2);
+    # beyond this per user, the least recently used go first.
+    raw_cache_mb: int = Field(default=256, alias="RAW_CACHE_MB")
+
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
     """Return the process-wide settings singleton."""
     return Settings()  # values are sourced from the environment
+
+
+def smtp_helo_name() -> str:
+    """The name to introduce ourselves with to SMTP servers (EHLO)."""
+    settings = get_settings()
+    return settings.smtp_helo_name.strip() or settings.app_domain

backend/app/dav/__init__.py

@@ -0,0 +1 @@
+"""CalDAV / CardDAV server (SPEC Appendix E)."""

backend/app/dav/auth.py

@@ -0,0 +1,93 @@
+"""Signing in a DAV client: HTTP Basic with an app password (SPEC App. E.2).
+
+Only app passwords: the account password (with or without 2FA) never works
+here, so a lost phone is one revocation away and 2FA cannot be bypassed. A
+sync makes many requests, each with the credentials; a successful check is
+remembered for a few minutes under a hash of the password, and forgotten as
+soon as the user's app passwords change (``app_passwords`` bumps a counter).
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import datetime as dt
+import hashlib
+import uuid
+
+from sqlalchemy import select, update
+
+from app.db import session_scope
+from app.identity import app_passwords
+from app.identity.service import resolve_handle
+from app.models import User, UserAppPassword
+from app.redis_client import get_redis
+
+CACHE_S = 300
+FAIL_WINDOW_S = 900
+MAX_FAILS_PER_ADDRESS = 10
+MAX_FAILS_PER_IP = 30
+_LAST_USED_EVERY = dt.timedelta(minutes=5)
+
+
+class Throttled(Exception):
+    """Too many failed sign-ins recently."""
+
+
+def _credentials(header: str | None) -> tuple[str, str] | None:
+    if not header or not header.lower().startswith("basic "):
+        return None
+    try:
+        decoded = base64.b64decode(header[6:].strip(), validate=True).decode("utf-8")
+    except (binascii.Error, UnicodeDecodeError):
+        return None
+    login, sep, password = decoded.partition(":")
+    return (login.strip(), password) if sep and login.strip() else None
+
+
+async def authenticate(header: str | None, ip: str) -> User | None:
+    """The user these Basic credentials sign in, or None. Raises Throttled."""
+    creds = _credentials(header)
+    if creds is None:
+        return None
+    login, password = creds
+    redis = get_redis()
+    handle = resolve_handle(login)
+    fail_keys = (f"dav:fail:addr:{handle}", f"dav:fail:ip:{ip}")
+    fails = [int(await redis.get(k) or 0) for k in fail_keys]
+    if fails[0] >= MAX_FAILS_PER_ADDRESS or fails[1] >= MAX_FAILS_PER_IP:
+        raise Throttled
+
+    async with session_scope() as db:
+        user = await db.scalar(select(User).where(User.handle == handle))
+        if user is None or user.blocked_at is not None:
+            await _failed(fail_keys)
+            return None
+        gen = int(await redis.get(app_passwords.generation_key(user.id)) or 0)
+        digest = hashlib.sha256(f"{user.id}:{password}".encode()).hexdigest()
+        cache_key = f"dav:auth:{user.id}:{gen}:{digest}"
+        if await redis.get(cache_key):
+            return user
+        row = await app_passwords.verify(db, user.id, password)
+        if row is None:
+            await _failed(fail_keys)
+            return None
+        await redis.set(cache_key, str(row.id), ex=CACHE_S)
+        now = dt.datetime.now(dt.UTC)
+        if row.last_used_at is None or now - row.last_used_at > _LAST_USED_EVERY:
+            await db.execute(
+                update(UserAppPassword).where(UserAppPassword.id == row.id).values(last_used_at=now)
+            )
+        db.expunge(user)
+        return user
+
+
+async def _failed(keys: tuple[str, ...]) -> None:
+    redis = get_redis()
+    for key in keys:
+        await redis.incr(key)
+        await redis.expire(key, FAIL_WINDOW_S)
+
+
+def user_id_of(user: User) -> uuid.UUID:
+    return user.id

backend/app/dav/provider.py

@@ -0,0 +1,118 @@
+"""What a plugin implements to serve its collections over DAV (SPEC App. E.1).
+
+The core (app/dav/routes.py) speaks the protocol; a provider only knows its
+own data: collections (calendars, address books) and the resources in them
+(one iCalendar object / one vCard each), named, with an ETag. Every method
+gets the signed-in user's id; providers open the user's RLS session
+(``sdk(name).session(user_id)``) themselves.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Literal, Protocol
+
+
+class DavError(Exception):
+    """A request the provider refuses: ``status`` is the HTTP answer
+    (403 read-only, 404 missing, 409 conflict, 412 precondition failed, 415
+    unsupported data); ``condition`` an optional DAV precondition element
+    (e.g. ``{urn:ietf:params:xml:ns:caldav}valid-calendar-data``)."""
+
+    def __init__(self, status: int, message: str = "", condition: str | None = None) -> None:
+        super().__init__(message or str(status))
+        self.status = status
+        self.condition = condition
+
+
+@dataclass
+class Collection:
+    id: str
+    displayname: str
+    sync_token: int  # the latest change number; also the CTag
+    description: str = ""
+    color: str | None = None  # "#rrggbb"
+    read_only: bool = False
+    components: list[str] = field(default_factory=lambda: ["VEVENT"])  # calendars only
+
+
+@dataclass
+class Resource:
+    name: str  # last path segment, e.g. "abc.ics"
+    etag: str  # without quotes
+    data: str | None = None  # the iCalendar / vCard text, when asked for
+
+
+@dataclass
+class Changes:
+    sync_token: int
+    changed: list[str]  # resource names changed or added since the old token
+    deleted: list[str]  # resource names gone since the old token
+
+
+class DavProvider(Protocol):
+    kind: Literal["calendar", "addressbook"]
+    content_type: str  # "text/calendar; charset=utf-8" / "text/vcard; charset=utf-8"
+    extension: str  # ".ics" / ".vcf"
+
+    async def collections(self, user_id: uuid.UUID) -> list[Collection]: ...
+
+    async def resources(self, user_id: uuid.UUID, collection_id: str) -> list[Resource]: ...
+
+    async def get(
+        self, user_id: uuid.UUID, collection_id: str, names: list[str] | None
+    ) -> list[Resource]: ...
+
+    async def put(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        name: str,
+        data: str,
+        *,
+        if_match: str | None,
+        if_none_match: bool,
+    ) -> tuple[str, bool]:
+        """Store ``data`` as ``name``; returns (new ETag, created)."""
+        ...
+
+    async def delete(
+        self, user_id: uuid.UUID, collection_id: str, name: str, *, if_match: str | None
+    ) -> None: ...
+
+    async def changes(self, user_id: uuid.UUID, collection_id: str, since: int) -> Changes | None:
+        """What changed after change number ``since``; None when that token is
+        too old to answer (the client then resyncs from scratch)."""
+        ...
+
+    async def create_collection(
+        self, user_id: uuid.UUID, collection_id: str, displayname: str, color: str | None
+    ) -> None:
+        """Create a collection at the id the client chose (MKCALENDAR / MKCOL);
+        DavError(403) when that id cannot be used."""
+        ...
+
+    async def update_collection(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        *,
+        displayname: str | None = None,
+        color: str | None = None,
+        description: str | None = None,
+    ) -> None: ...
+
+    async def delete_collection(self, user_id: uuid.UUID, collection_id: str) -> None: ...
+
+    async def query(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        start: datetime | None,
+        end: datetime | None,
+    ) -> list[str] | None:
+        """Calendars: names of resources with an occurrence overlapping the
+        range; None when the provider cannot filter (the core then returns all)."""
+        ...

backend/app/dav/routes.py

@@ -0,0 +1,595 @@
+"""The CalDAV / CardDAV endpoints (SPEC Appendix E).
+
+    /.well-known/caldav, /.well-known/carddav   → 301 /dav/
+    /dav/                                        root: points at the principal
+    /dav/principals/{user}/                      the signed-in user
+    /dav/calendars/{user}/[{calendar}/[{name}]]  calendar home, calendars, events
+    /dav/addressbooks/{user}/[{book}/[{name}]]   address-book home, books, contacts
+
+The data comes from the plugins' providers (app/dav/provider.py): the calendar
+plugin's for ``calendars``, the contacts plugin's for ``addressbooks``. Signed
+in with HTTP Basic and an app password only (app/dav/auth.py); session cookies
+are never looked at, so a web page cannot drive these endpoints.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+import xml.etree.ElementTree as ET
+from typing import Any
+from urllib.parse import quote, unquote, urlsplit
+
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import RedirectResponse
+from sqlalchemy import select
+
+from app.dav import auth
+from app.dav.provider import Collection, DavError, DavProvider, Resource
+from app.dav.xml import (
+    CR,
+    CS,
+    A,
+    BadRequest,
+    C,
+    D,
+    Multistatus,
+    PropValue,
+    el,
+    error_body,
+    href,
+    mkcol_props,
+    parse,
+    propfind_request,
+    proppatch_request,
+    q,
+    report_request,
+)
+from app.db import session_scope
+from app.models import Account, User
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+METHODS = [
+    "OPTIONS",
+    "PROPFIND",
+    "PROPPATCH",
+    "REPORT",
+    "GET",
+    "HEAD",
+    "PUT",
+    "DELETE",
+    "MKCALENDAR",
+    "MKCOL",
+]
+DAV_HEADER = "1, 3, calendar-access, addressbook, extended-mkcol"
+XML_TYPE = "application/xml; charset=utf-8"
+HOMES = {"calendars": "calendar", "addressbooks": "addressbook"}
+SYNC_PREFIX = "http://ultimo.email/ns/sync/"
+
+
+def _providers() -> dict[str, DavProvider]:
+    from app.plugins import loader
+
+    out: dict[str, DavProvider] = {}
+    for plugin in loader.get_plugins():
+        if plugin.dav is not None:
+            out[plugin.dav.kind] = plugin.dav
+    return out
+
+
+# ---- entry points ------------------------------------------------------------------
+
+
+@router.api_route("/.well-known/caldav", methods=["GET", "HEAD", "PROPFIND", "OPTIONS"])
+@router.api_route("/.well-known/carddav", methods=["GET", "HEAD", "PROPFIND", "OPTIONS"])
+async def well_known() -> Response:
+    return RedirectResponse("/dav/", status_code=301)
+
+
+@router.api_route("/dav", methods=METHODS)
+@router.api_route("/dav/{path:path}", methods=METHODS)
+async def dav(request: Request, path: str = "") -> Response:
+    method = request.method.upper()
+    if method == "OPTIONS":
+        return Response(
+            status_code=200,
+            headers={"DAV": DAV_HEADER, "Allow": ", ".join(METHODS), "Content-Length": "0"},
+        )
+    ip = request.client.host if request.client else ""
+    try:
+        user = await auth.authenticate(request.headers.get("authorization"), ip)
+    except auth.Throttled:
+        return Response(status_code=429, headers={"Retry-After": "900"})
+    if user is None:
+        return Response(
+            status_code=401,
+            headers={"WWW-Authenticate": 'Basic realm="Ultimo", charset="UTF-8"'},
+        )
+    parts = [unquote(p) for p in path.split("/") if p]
+    ctx = _Ctx(request, user, parts)
+    try:
+        return await ctx.handle(method)
+    except BadRequest as err:
+        return Response(status_code=400, content=str(err))
+    except DavError as err:
+        body = error_body(err.condition) if err.condition else b""
+        return Response(
+            status_code=err.status,
+            content=body,
+            media_type=XML_TYPE if body else None,
+        )
+
+
+class _Ctx:
+    def __init__(self, request: Request, user: User, parts: list[str]) -> None:
+        self.request = request
+        self.user = user
+        self.uid = str(user.id)
+        self.parts = parts
+        self.providers = _providers()
+
+    # -- paths -----------------------------------------------------------------------
+
+    def principal(self) -> str:
+        return f"/dav/principals/{self.uid}/"
+
+    def home(self, segment: str) -> str:
+        return f"/dav/{segment}/{self.uid}/"
+
+    def coll_href(self, segment: str, coll_id: str) -> str:
+        return f"{self.home(segment)}{quote(coll_id, safe='')}/"
+
+    def res_href(self, segment: str, coll_id: str, name: str) -> str:
+        return f"{self.coll_href(segment, coll_id)}{quote(name, safe='')}"
+
+    # -- dispatch --------------------------------------------------------------------
+
+    async def handle(self, method: str) -> Response:
+        p = self.parts
+        if not p:
+            return await self.root(method)
+        if p[0] == "principals":
+            if len(p) != 2 or p[1] != self.uid:
+                raise DavError(404)
+            return await self.principal_req(method)
+        if p[0] in HOMES and len(p) >= 2:
+            if p[1] != self.uid:
+                raise DavError(403)
+            provider = self.providers.get(HOMES[p[0]])
+            if provider is None:
+                raise DavError(404)
+            if len(p) == 2:
+                return await self.home_req(method, p[0], provider)
+            if len(p) == 3:
+                return await self.collection_req(method, p[0], provider, p[2])
+            if len(p) == 4:
+                return await self.resource_req(method, p[0], provider, p[2], p[3])
+        raise DavError(404)
+
+    async def body(self) -> ET.Element | None:
+        return parse(await self.request.body())
+
+    def depth(self) -> int:
+        value = (self.request.headers.get("depth") or "1").strip().lower()
+        return 0 if value == "0" else 1
+
+    @staticmethod
+    def multistatus(ms: Multistatus) -> Response:
+        return Response(status_code=207, content=ms.to_bytes(), media_type=XML_TYPE)
+
+    async def respond_props(
+        self, items: list[tuple[str, dict[str, PropValue]]], method_root: ET.Element | None
+    ) -> Response:
+        req = propfind_request(method_root)
+        ms = Multistatus()
+        for path, props in items:
+            if req.names_only:
+                ms.response(path, dict.fromkeys(props))
+            elif req.names is None:
+                ms.response(path, props)
+            else:
+                found = {n: props[n] for n in req.names if n in props}
+                missing = [n for n in req.names if n not in props]
+                ms.response(path, found, missing)
+        return self.multistatus(ms)
+
+    # -- root and principal ----------------------------------------------------------
+
+    def common(self) -> dict[str, PropValue]:
+        return {
+            q(D, "current-user-principal"): [href(self.principal())],
+            q(D, "principal-URL"): [href(self.principal())],
+            q(D, "owner"): [href(self.principal())],
+        }
+
+    async def root(self, method: str) -> Response:
+        if method != "PROPFIND":
+            return Response(status_code=200 if method in ("GET", "HEAD") else 405)
+        props = {**self.common(), q(D, "resourcetype"): [el(q(D, "collection"))]}
+        return await self.respond_props([("/dav/", props)], await self.body())
+
+    async def addresses(self) -> list[str]:
+        async with session_scope() as db:
+            emails = list(
+                await db.scalars(select(Account.email).where(Account.user_id == self.user.id))
+            )
+        out = [self.user.handle, *emails]
+        return list(dict.fromkeys(e.lower() for e in out))
+
+    async def principal_props(self) -> dict[str, PropValue]:
+        props = {
+            **self.common(),
+            q(D, "resourcetype"): [el(q(D, "principal")), el(q(D, "collection"))],
+            q(D, "displayname"): self.user.handle,
+            q(C, "calendar-user-address-set"): [
+                href(f"mailto:{a}") for a in await self.addresses()
+            ],
+            q(C, "calendar-user-type"): "INDIVIDUAL",
+            q(D, "principal-collection-set"): [href("/dav/principals/")],
+            q(D, "supported-report-set"): [],
+        }
+        if "calendar" in self.providers:
+            props[q(C, "calendar-home-set")] = [href(self.home("calendars"))]
+        if "addressbook" in self.providers:
+            props[q(CR, "addressbook-home-set")] = [href(self.home("addressbooks"))]
+        return props
+
+    async def principal_req(self, method: str) -> Response:
+        if method == "PROPFIND":
+            return await self.respond_props(
+                [(self.principal(), await self.principal_props())], await self.body()
+            )
+        if method == "REPORT":  # principal-property-search, expand-property: nothing to add
+            return self.multistatus(Multistatus())
+        return Response(status_code=405)
+
+    # -- homes -------------------------------------------------------------------------
+
+    def privileges(self, read_only: bool) -> list[ET.Element]:
+        names = ["read", "read-current-user-privilege-set"]
+        if not read_only:
+            names += ["write", "write-content", "write-properties", "bind", "unbind"]
+        return [el(q(D, "privilege"), children=[el(q(D, n))]) for n in names]
+
+    def reports(self, kind: str) -> list[ET.Element]:
+        ns, reports = (
+            (C, ["calendar-query", "calendar-multiget"])
+            if kind == "calendar"
+            else (CR, ["addressbook-query", "addressbook-multiget"])
+        )
+        items = [(ns, r) for r in reports] + [(D, "sync-collection")]
+        return [
+            el(q(D, "supported-report"), children=[el(q(D, "report"), children=[el(q(n, r))])])
+            for n, r in items
+        ]
+
+    async def home_req(self, method: str, segment: str, provider: DavProvider) -> Response:
+        if method == "PROPFIND":
+            home_props: dict[str, PropValue] = {
+                **self.common(),
+                q(D, "resourcetype"): [el(q(D, "collection"))],
+                q(D, "displayname"): "Calendars" if provider.kind == "calendar" else "Contacts",
+                q(D, "current-user-privilege-set"): self.privileges(False),
+            }
+            items = [(self.home(segment), home_props)]
+            if self.depth():
+                for coll in await provider.collections(self.user.id):
+                    items.append(
+                        (self.coll_href(segment, coll.id), self.collection_props(provider, coll))
+                    )
+            return await self.respond_props(items, await self.body())
+        if method == "REPORT":
+            return self.multistatus(Multistatus())
+        if method in ("GET", "HEAD"):
+            return Response(status_code=200)
+        return Response(status_code=405)
+
+    # -- collections -------------------------------------------------------------------
+
+    def collection_props(self, provider: DavProvider, coll: Collection) -> dict[str, PropValue]:
+        token = f"{SYNC_PREFIX}{coll.sync_token}"
+        props: dict[str, PropValue] = {
+            **self.common(),
+            q(D, "displayname"): coll.displayname,
+            q(D, "current-user-privilege-set"): self.privileges(coll.read_only),
+            q(D, "supported-report-set"): self.reports(provider.kind),
+            q(D, "sync-token"): token,
+            q(CS, "getctag"): str(coll.sync_token),
+        }
+        if provider.kind == "calendar":
+            props[q(D, "resourcetype")] = [el(q(D, "collection")), el(q(C, "calendar"))]
+            props[q(C, "supported-calendar-component-set")] = [
+                el(q(C, "comp"), attrs={"name": comp}) for comp in coll.components
+            ]
+            props[q(C, "supported-calendar-data")] = [
+                el(q(C, "calendar-data"), attrs={"content-type": "text/calendar", "version": "2.0"})
+            ]
+            props[q(C, "calendar-description")] = coll.description
+            if coll.color:
+                props[q(A, "calendar-color")] = coll.color
+        else:
+            props[q(D, "resourcetype")] = [el(q(D, "collection")), el(q(CR, "addressbook"))]
+            props[q(CR, "supported-address-data")] = [
+                el(
+                    q(CR, "address-data-type"),
+                    attrs={"content-type": "text/vcard", "version": "3.0"},
+                ),
+                el(
+                    q(CR, "address-data-type"),
+                    attrs={"content-type": "text/vcard", "version": "4.0"},
+                ),
+            ]
+            props[q(CR, "addressbook-description")] = coll.description
+        return props
+
+    async def find(self, provider: DavProvider, coll_id: str) -> Collection:
+        for coll in await provider.collections(self.user.id):
+            if coll.id == coll_id:
+                return coll
+        raise DavError(404)
+
+    def resource_props(
+        self, provider: DavProvider, res: Resource, with_data: bool
+    ) -> dict[str, PropValue]:
+        props: dict[str, PropValue] = {
+            q(D, "getetag"): f'"{res.etag}"',
+            q(D, "getcontenttype"): provider.content_type,
+            q(D, "resourcetype"): [],
+        }
+        if with_data and res.data is not None:
+            data_prop = (
+                q(C, "calendar-data") if provider.kind == "calendar" else q(CR, "address-data")
+            )
+            props[data_prop] = res.data
+        return props
+
+    async def collection_req(
+        self, method: str, segment: str, provider: DavProvider, coll_id: str
+    ) -> Response:
+        if method == "MKCALENDAR" or method == "MKCOL":
+            return await self.make_collection(segment, provider, coll_id)
+        coll = await self.find(provider, coll_id)
+        if method == "PROPFIND":
+            items = [(self.coll_href(segment, coll.id), self.collection_props(provider, coll))]
+            if self.depth():
+                for res in await provider.resources(self.user.id, coll.id):
+                    items.append(
+                        (
+                            self.res_href(segment, coll.id, res.name),
+                            self.resource_props(provider, res, False),
+                        )
+                    )
+            return await self.respond_props(items, await self.body())
+        if method == "REPORT":
+            return await self.report(segment, provider, coll)
+        if method == "PROPPATCH":
+            return await self.proppatch(segment, provider, coll)
+        if method == "DELETE":
+            if coll.read_only:
+                raise DavError(403)
+            await provider.delete_collection(self.user.id, coll.id)
+            return Response(status_code=204)
+        if method in ("GET", "HEAD"):
+            return Response(status_code=200)
+        return Response(status_code=405)
+
+    async def make_collection(self, segment: str, provider: DavProvider, coll_id: str) -> Response:
+        if any(c.id == coll_id for c in await provider.collections(self.user.id)):
+            raise DavError(405)  # exists already
+        root = await self.body()
+        props = mkcol_props(root)
+        rtype = props.get(q(D, "resourcetype"))
+        if self.request.method.upper() == "MKCOL" and (
+            rtype is None
+            or rtype.find(q(CR, "addressbook")) is None
+            or provider.kind != "addressbook"
+        ):
+            raise DavError(403)
+        if self.request.method.upper() == "MKCALENDAR" and provider.kind != "calendar":
+            raise DavError(403)
+        name_el = props.get(q(D, "displayname"))
+        color_el = props.get(q(A, "calendar-color"))
+        await provider.create_collection(
+            self.user.id,
+            coll_id,
+            (name_el.text or "").strip() if name_el is not None else "",
+            (color_el.text or "").strip() if color_el is not None else None,
+        )
+        return Response(status_code=201)
+
+    async def proppatch(self, segment: str, provider: DavProvider, coll: Collection) -> Response:
+        patch = proppatch_request(await self.body())
+        if coll.read_only:
+            raise DavError(403)
+        values: dict[str, Any] = {}
+        for name, value in patch.set.items():
+            text = (value.text or "").strip()
+            if name == q(D, "displayname"):
+                values["displayname"] = text
+            elif name == q(A, "calendar-color"):
+                values["color"] = text
+            elif name in (q(C, "calendar-description"), q(CR, "addressbook-description")):
+                values["description"] = text
+        if values:
+            await provider.update_collection(self.user.id, coll.id, **values)
+        # Properties not kept (calendar order, …) are accepted and ignored:
+        # clients treat a refusal as a failed sync.
+        ms = Multistatus()
+        ms.response(self.coll_href(segment, coll.id), dict.fromkeys([*patch.set, *patch.remove]))
+        return self.multistatus(ms)
+
+    async def report(self, segment: str, provider: DavProvider, coll: Collection) -> Response:
+        root = await self.body()
+        if root is None:
+            raise BadRequest("empty report")
+        rep = report_request(root)
+        wants_data = rep.props is None or any(
+            p in (q(C, "calendar-data"), q(CR, "address-data")) for p in rep.props
+        )
+        ms = Multistatus()
+
+        def add(res: Resource) -> None:
+            props = self.resource_props(provider, res, wants_data)
+            if rep.props is None:
+                ms.response(self.res_href(segment, coll.id, res.name), props)
+            else:
+                found = {n: props[n] for n in rep.props if n in props}
+                missing = [n for n in rep.props if n not in props]
+                ms.response(self.res_href(segment, coll.id, res.name), found, missing)
+
+        if rep.kind in ("calendar-multiget", "addressbook-multiget"):
+            names = [_last_segment(h) for h in rep.hrefs]
+            got = {r.name: r for r in await provider.get(self.user.id, coll.id, names)}
+            for h, name in zip(rep.hrefs, names, strict=True):
+                if name in got:
+                    add(got[name])
+                else:
+                    ms.response(h, {}, status=404)
+            return self.multistatus(ms)
+
+        if rep.kind in ("calendar-query", "addressbook-query"):
+            names_q = (
+                await provider.query(self.user.id, coll.id, rep.start, rep.end)
+                if rep.kind == "calendar-query"
+                else None
+            )
+            for res in (
+                await provider.get(self.user.id, coll.id, names_q)
+                if wants_data
+                else (
+                    await provider.resources(self.user.id, coll.id)
+                    if names_q is None
+                    else [
+                        r
+                        for r in await provider.resources(self.user.id, coll.id)
+                        if r.name in set(names_q)
+                    ]
+                )
+            ):
+                add(res)
+            return self.multistatus(ms)
+
+        # sync-collection (RFC 6578)
+        since = _token_number(rep.sync_token)
+        if rep.sync_token and since is None:
+            raise DavError(403, condition=q(D, "valid-sync-token"))
+        if since is None:
+            changed = None
+            deleted: list[str] = []
+            token = coll.sync_token
+        else:
+            changes = await provider.changes(self.user.id, coll.id, since)
+            if changes is None:
+                raise DavError(403, condition=q(D, "valid-sync-token"))
+            changed, deleted, token = changes.changed, changes.deleted, changes.sync_token
+        if changed is None:
+            resources = (
+                await provider.get(self.user.id, coll.id, None)
+                if wants_data
+                else await provider.resources(self.user.id, coll.id)
+            )
+        elif changed:
+            resources = (
+                await provider.get(self.user.id, coll.id, changed)
+                if wants_data
+                else [
+                    r
+                    for r in await provider.resources(self.user.id, coll.id)
+                    if r.name in set(changed)
+                ]
+            )
+        else:
+            resources = []
+        present = {r.name for r in resources}
+        for res in resources:
+            add(res)
+        for name in deleted:
+            if name not in present:
+                ms.response(self.res_href(segment, coll.id, name), {}, status=404)
+        ms.sync_token(f"{SYNC_PREFIX}{token}")
+        return self.multistatus(ms)
+
+    # -- resources ---------------------------------------------------------------------
+
+    async def resource_req(
+        self, method: str, segment: str, provider: DavProvider, coll_id: str, name: str
+    ) -> Response:
+        coll = await self.find(provider, coll_id)
+        if method in ("GET", "HEAD"):
+            got = await provider.get(self.user.id, coll.id, [name])
+            if not got:
+                raise DavError(404)
+            res = got[0]
+            content = (res.data or "").encode() if method == "GET" else b""
+            return Response(
+                status_code=200,
+                content=content,
+                media_type=provider.content_type,
+                headers={"ETag": f'"{res.etag}"'},
+            )
+        if method == "PROPFIND":
+            got = await provider.get(self.user.id, coll.id, [name])
+            if not got:
+                raise DavError(404)
+            return await self.respond_props(
+                [
+                    (
+                        self.res_href(segment, coll.id, name),
+                        self.resource_props(provider, got[0], True),
+                    )
+                ],
+                await self.body(),
+            )
+        if method == "PUT":
+            if coll.read_only:
+                raise DavError(403, condition=q(D, "need-privileges"))
+            data = (await self.request.body()).decode("utf-8", "replace")
+            etag, created = await provider.put(
+                self.user.id,
+                coll.id,
+                name,
+                data,
+                if_match=_etag_header(self.request.headers.get("if-match")),
+                if_none_match=(self.request.headers.get("if-none-match") or "").strip() == "*",
+            )
+            return Response(status_code=201 if created else 204, headers={"ETag": f'"{etag}"'})
+        if method == "DELETE":
+            if coll.read_only:
+                raise DavError(403)
+            await provider.delete(
+                self.user.id,
+                coll.id,
+                name,
+                if_match=_etag_header(self.request.headers.get("if-match")),
+            )
+            return Response(status_code=204)
+        return Response(status_code=405)
+
+
+def _last_segment(h: str) -> str:
+    path = urlsplit(h).path
+    return unquote(path.rstrip("/").rsplit("/", 1)[-1])
+
+
+def _token_number(token: str) -> int | None:
+    if not token:
+        return None
+    tail = token.rsplit("/", 1)[-1]
+    return int(tail) if tail.isdigit() else None
+
+
+def _etag_header(value: str | None) -> str | None:
+    if not value:
+        return None
+    value = value.strip()
+    if value == "*":
+        return "*"
+    if value.startswith("W/"):
+        value = value[2:]
+    return value.strip('"')
+
+
+__all__ = ["router", "uuid"]

backend/app/dav/xml.py

@@ -0,0 +1,236 @@
+"""WebDAV XML: reading request bodies, writing multistatus answers (RFC 4918,
+4791, 6352, 6578). Properties are ``{namespace}name`` strings, as ElementTree
+names them."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from defusedxml import ElementTree as SafeET
+
+D = "DAV:"
+C = "urn:ietf:params:xml:ns:caldav"
+CR = "urn:ietf:params:xml:ns:carddav"
+CS = "http://calendarserver.org/ns/"
+A = "http://apple.com/ns/ical/"
+
+for _prefix, _ns in (("d", D), ("cal", C), ("card", CR), ("cs", CS), ("ical", A)):
+    ET.register_namespace(_prefix, _ns)
+
+
+def q(ns: str, name: str) -> str:
+    return f"{{{ns}}}{name}"
+
+
+class BadRequest(Exception):
+    pass
+
+
+def parse(body: bytes) -> ET.Element | None:
+    if not body.strip():
+        return None
+    try:
+        root = SafeET.fromstring(body)
+    except Exception as err:  # noqa: BLE001 - any malformed XML is the client's error
+        raise BadRequest(str(err)) from err
+    return root  # type: ignore[no-any-return]
+
+
+@dataclass
+class PropRequest:
+    """PROPFIND: which properties (None = all)."""
+
+    names: list[str] | None = None
+    names_only: bool = False
+
+
+def propfind_request(root: ET.Element | None) -> PropRequest:
+    if root is None or root.find(q(D, "allprop")) is not None:
+        return PropRequest()
+    if root.find(q(D, "propname")) is not None:
+        return PropRequest(names_only=True)
+    prop = root.find(q(D, "prop"))
+    if prop is None:
+        return PropRequest()
+    return PropRequest(names=[child.tag for child in prop])
+
+
+def requested_props(root: ET.Element) -> list[str] | None:
+    prop = root.find(q(D, "prop"))
+    if prop is None:
+        return None
+    return [child.tag for child in prop]
+
+
+@dataclass
+class Report:
+    # calendar-query | calendar-multiget | addressbook-query | addressbook-multiget
+    # | sync-collection
+    kind: str
+    props: list[str] | None
+    hrefs: list[str] = field(default_factory=list)
+    start: datetime | None = None
+    end: datetime | None = None
+    sync_token: str = ""
+    limit: int | None = None
+
+
+def _ical_time(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        return datetime.strptime(value.strip(), "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
+    except ValueError:
+        return None
+
+
+def report_request(root: ET.Element) -> Report:
+    tag = root.tag
+    names = {
+        q(C, "calendar-query"): "calendar-query",
+        q(C, "calendar-multiget"): "calendar-multiget",
+        q(CR, "addressbook-query"): "addressbook-query",
+        q(CR, "addressbook-multiget"): "addressbook-multiget",
+        q(D, "sync-collection"): "sync-collection",
+    }
+    kind = names.get(tag)
+    if kind is None:
+        raise BadRequest(f"unsupported report {tag}")
+    rep = Report(kind=kind, props=requested_props(root))
+    rep.hrefs = [(h.text or "").strip() for h in root.findall(q(D, "href"))]
+    if kind == "calendar-query":
+        tr = root.find(f".//{q(C, 'time-range')}")
+        if tr is not None:
+            rep.start = _ical_time(tr.get("start"))
+            rep.end = _ical_time(tr.get("end"))
+    if kind == "sync-collection":
+        rep.sync_token = (root.findtext(q(D, "sync-token")) or "").strip()
+        nresults = root.find(f"{q(D, 'limit')}/{q(D, 'nresults')}")
+        if nresults is not None and (nresults.text or "").strip().isdigit():
+            rep.limit = int((nresults.text or "").strip())
+    return rep
+
+
+@dataclass
+class PropPatch:
+    set: dict[str, ET.Element] = field(default_factory=dict)
+    remove: list[str] = field(default_factory=list)
+
+
+def proppatch_request(root: ET.Element | None) -> PropPatch:
+    out = PropPatch()
+    if root is None:
+        return out
+    for action in root:
+        prop = action.find(q(D, "prop"))
+        if prop is None:
+            continue
+        for child in prop:
+            if action.tag == q(D, "set"):
+                out.set[child.tag] = child
+            elif action.tag == q(D, "remove"):
+                out.remove.append(child.tag)
+    return out
+
+
+def mkcol_props(root: ET.Element | None) -> dict[str, ET.Element]:
+    """MKCALENDAR / extended MKCOL: the properties to set on the new collection."""
+    if root is None:
+        return {}
+    out: dict[str, ET.Element] = {}
+    for sett in root.findall(q(D, "set")):
+        prop = sett.find(q(D, "prop"))
+        if prop is not None:
+            for child in prop:
+                out[child.tag] = child
+    return out
+
+
+# ---- answers --------------------------------------------------------------------------
+
+# A property's value: text, a list of child elements, or an element builder.
+PropValue = str | list[ET.Element] | None
+
+
+class Multistatus:
+    def __init__(self) -> None:
+        self.root = ET.Element(q(D, "multistatus"))
+
+    def response(
+        self,
+        href: str,
+        found: dict[str, PropValue],
+        missing: list[str] | None = None,
+        status: int | None = None,
+    ) -> None:
+        resp = ET.SubElement(self.root, q(D, "response"))
+        ET.SubElement(resp, q(D, "href")).text = href
+        if status is not None:
+            ET.SubElement(resp, q(D, "status")).text = _status_line(status)
+            return
+        if found:
+            ps = ET.SubElement(resp, q(D, "propstat"))
+            prop = ET.SubElement(ps, q(D, "prop"))
+            for name, value in found.items():
+                el = ET.SubElement(prop, name)
+                if isinstance(value, str):
+                    el.text = value
+                elif isinstance(value, list):
+                    el.extend(value)
+            ET.SubElement(ps, q(D, "status")).text = _status_line(200)
+        if missing:
+            ps = ET.SubElement(resp, q(D, "propstat"))
+            prop = ET.SubElement(ps, q(D, "prop"))
+            for name in missing:
+                ET.SubElement(prop, name)
+            ET.SubElement(ps, q(D, "status")).text = _status_line(404)
+
+    def sync_token(self, token: str) -> None:
+        ET.SubElement(self.root, q(D, "sync-token")).text = token
+
+    def to_bytes(self) -> bytes:
+        body: bytes = ET.tostring(self.root, encoding="utf-8")
+        return b'<?xml version="1.0" encoding="utf-8"?>\n' + body
+
+
+def el(
+    tag: str,
+    text: str | None = None,
+    children: list[ET.Element] | None = None,
+    attrs: dict[str, str] | None = None,
+) -> ET.Element:
+    e = ET.Element(tag, attrs or {})
+    if text is not None:
+        e.text = text
+    if children:
+        e.extend(children)
+    return e
+
+
+def href(path: str) -> ET.Element:
+    return el(q(D, "href"), path)
+
+
+def error_body(condition: str) -> bytes:
+    root = ET.Element(q(D, "error"))
+    ET.SubElement(root, condition)
+    return b'<?xml version="1.0" encoding="utf-8"?>\n' + bytes(ET.tostring(root, encoding="utf-8"))
+
+
+_REASONS = {
+    200: "OK",
+    201: "Created",
+    204: "No Content",
+    207: "Multi-Status",
+    403: "Forbidden",
+    404: "Not Found",
+    409: "Conflict",
+    412: "Precondition Failed",
+    507: "Insufficient Storage",
+}
+
+
+def _status_line(code: int) -> str:
+    return f"HTTP/1.1 {code} {_REASONS.get(code, '')}".rstrip()

backend/app/events.py

@@ -41,6 +41,9 @@ ALLOWED_EVENTS: frozenset[str] = frozenset(
         # Sync v2 §3: published on login and every WS connect; drives tier
         # promotion in the sync workers (§5). Payload: {user_id}.
         "user.active",
+        # The user's sessions were revoked by an administrator (blocked or
+        # deleted): open tabs drop to the sign-in screen. Payload: {user_id}.
+        "user.signed_out",
     }
 )
 

backend/app/hosted/__init__.py

@@ -0,0 +1,34 @@
+"""Hosted mailboxes (SPEC Appendix D): every user's own mailbox on the Stalwart
+mail server beside the app.
+
+* :mod:`app.hosted.admin` — the mail server's management API (JMAP);
+* :mod:`app.hosted.provision` — idempotent setup: its database and role, the
+  directory over the ``mail_directory`` view, the worker's sign-in;
+* :mod:`app.hosted.native` — the permanent account in the webmail, and how the
+  worker signs in to it without any stored password;
+* :mod:`app.hosted.rawcache` — raw messages kept once, on the mail server: the
+  webmail keeps only a bounded cache of them (§D.6 item 2).
+
+All of it is inert while ``HOSTED_MAILBOXES`` is off.
+"""
+
+from __future__ import annotations
+
+from app.config import get_settings
+
+# The webmail worker's sign-in on the mail server: ``user@domain%WORKER`` acts
+# for that user (Stalwart's master-user login, given the impersonate role only).
+WORKER_LOCALPART = "webmail-worker"
+
+
+def enabled() -> bool:
+    return get_settings().hosted_mailboxes
+
+
+def worker_address() -> str:
+    return f"{WORKER_LOCALPART}@{get_settings().app_domain}"
+
+
+def mail_hostname() -> str:
+    settings = get_settings()
+    return settings.mail_hostname or f"mail.{settings.app_domain}"

backend/app/hosted/admin.py

@@ -0,0 +1,177 @@
+"""The mail server's management API (Stalwart 0.16, JMAP ``x:`` methods).
+
+Signed in to as the recovery admin (``STALWART_RECOVERY_ADMIN`` in the mail
+server's environment, ``STALWART_ADMIN_SECRET`` here). Objects are Stalwart's
+registry: ``x:Account``, ``x:Domain``, ``x:Directory``, ``x:Role``,
+``x:AppPassword``… with the usual JMAP ``/get``, ``/query``, ``/set``.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Any
+
+import httpx
+
+from app.config import get_settings
+from app.hosted import worker_address
+
+ADMIN_USER = "admin"
+_USING = ["urn:ietf:params:jmap:core", "urn:stalwart:jmap"]
+_TIMEOUT = 15.0
+
+
+class MailServerError(Exception):
+    """The mail server refused a call, or could not be reached."""
+
+
+def _basic(user: str, secret: str) -> str:
+    return "Basic " + base64.b64encode(f"{user}:{secret}".encode()).decode()
+
+
+def _admin_auth() -> str:
+    return _basic(ADMIN_USER, get_settings().stalwart_admin_secret)
+
+
+async def call(
+    *method_calls: tuple[str, dict[str, Any]], auth: str | None = None
+) -> list[dict[str, Any]]:
+    """Run JMAP method calls in one request; return each call's arguments.
+
+    Raises :class:`MailServerError` on a method error or a refused ``set``."""
+    body = {
+        "using": _USING,
+        "methodCalls": [[name, args, str(i)] for i, (name, args) in enumerate(method_calls)],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.post(
+                f"{get_settings().stalwart_url}/jmap",
+                json=body,
+                headers={"Authorization": auth or _admin_auth()},
+            )
+    except httpx.HTTPError as exc:
+        raise MailServerError(f"mail server unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        raise MailServerError(f"mail server answered {resp.status_code}: {resp.text[:300]}")
+    out: list[dict[str, Any]] = []
+    for name, args, _ in resp.json()["methodResponses"]:
+        if name == "error":
+            raise MailServerError(f"{args.get('type')}: {args.get('description', '')}")
+        for kind in ("notCreated", "notUpdated", "notDestroyed"):
+            if args.get(kind):
+                raise MailServerError(f"{name} {kind}: {args[kind]}")
+        out.append(args)
+    return out
+
+
+async def get_all(obj: str, **args: Any) -> list[dict[str, Any]]:
+    """Every object of a type (registry objects are few)."""
+    (res,) = await call((f"x:{obj}/get", {"ids": None, **args}))
+    return list(res["list"])
+
+
+async def get_one(obj: str, object_id: str = "singleton") -> dict[str, Any]:
+    (res,) = await call((f"x:{obj}/get", {"ids": [object_id]}))
+    if not res["list"]:
+        raise MailServerError(f"{obj} {object_id} not found")
+    return dict(res["list"][0])
+
+
+async def create(obj: str, value: dict[str, Any], **args: Any) -> str:
+    (res,) = await call((f"x:{obj}/set", {"create": {"n": value}, **args}))
+    return str(res["created"]["n"]["id"])
+
+
+async def update(obj: str, object_id: str, patch: dict[str, Any], **args: Any) -> None:
+    await call((f"x:{obj}/set", {"update": {object_id: patch}, **args}))
+
+
+async def destroy(obj: str, ids: list[str], **args: Any) -> None:
+    if ids:
+        await call((f"x:{obj}/set", {"destroy": ids, **args}))
+
+
+# ---- accounts ----------------------------------------------------------------------
+
+
+async def account_id(email: str) -> str | None:
+    """The mail server's id of a mailbox, if it knows it yet.
+
+    Accounts from the directory reach the server's registry the first time
+    they are used (a sign-in, a delivery): see :func:`ensure_account`."""
+    local = email.split("@", 1)[0].lower()
+    (res,) = await call(("x:Account/query", {"filter": {"name": local}}))
+    ids = list(res.get("ids") or [])
+    if not ids:
+        return None
+    for acc in (await call(("x:Account/get", {"ids": ids, "properties": ["emailAddress"]})))[0][
+        "list"
+    ]:
+        if str(acc.get("emailAddress", "")).lower() == email.lower():
+            return str(acc["id"])
+    return None
+
+
+async def ensure_account(email: str) -> str:
+    """The mailbox's id, bringing it into the server's registry if new: a
+    sign-in as the worker acting for it does that (§D.3)."""
+    found = await account_id(email)
+    if found is not None:
+        return found
+    auth = _basic(f"{email}%{worker_address()}", get_settings().stalwart_master_secret)
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(
+                f"{get_settings().stalwart_url}/jmap/session", headers={"Authorization": auth}
+            )
+    except httpx.HTTPError as exc:
+        raise MailServerError(f"mail server unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        raise MailServerError(f"cannot open the mailbox {email}: {resp.status_code}")
+    found = await account_id(email)
+    if found is None:
+        raise MailServerError(f"mailbox {email} not found after sign-in")
+    return found
+
+
+async def set_blocked(email: str, blocked: bool) -> None:
+    """A blocked user's mailbox cannot be signed in to at all — app passwords
+    and the worker included. Mail to it is still accepted."""
+    acc = await account_id(email)
+    if acc is None:
+        return  # never used: the directory alone keeps it shut
+    permissions: dict[str, Any] = (
+        {"@type": "Merge", "enabledPermissions": {}, "disabledPermissions": {"authenticate": True}}
+        if blocked
+        else {"@type": "Inherit"}
+    )
+    await update("Account", acc, {"permissions": permissions})
+
+
+async def delete_account(email: str) -> None:
+    """Remove the mailbox and all its mail from the mail server."""
+    acc = await account_id(email)
+    if acc is not None:
+        await destroy("Account", [acc])
+
+
+# ---- app passwords (§D.5) -------------------------------------------------------------
+
+
+async def create_app_password(email: str, description: str) -> tuple[str, str]:
+    """A new app password for the mailbox: (server id, the password itself)."""
+    acc = await ensure_account(email)
+    (res,) = await call(
+        ("x:AppPassword/set", {"accountId": acc, "create": {"n": {"description": description}}})
+    )
+    created = res["created"]["n"]
+    return str(created["id"]), str(created["secret"])
+
+
+async def delete_app_passwords(email: str, server_ids: list[str]) -> None:
+    acc = await account_id(email)
+    if acc is not None and server_ids:
+        (res,) = await call(("x:AppPassword/get", {"accountId": acc, "ids": server_ids}))
+        present = [str(p["id"]) for p in res["list"]]
+        await destroy("AppPassword", present, accountId=acc)

backend/app/hosted/native.py

@@ -0,0 +1,154 @@
+"""The user's own mailbox as a webmail account (SPEC App. D §D.4).
+
+Every user has one ``kind = 'native'`` account: their handle's mailbox on the
+hosted mail server. It is created with the user (and, for users from before,
+by :func:`ensure_all`, run by provisioning), cannot be removed, and its server
+settings are not the user's to change.
+
+No password of the user's is stored for it: the worker signs in as itself,
+acting for the user (``user@domain%webmail-worker``, §D.3), over the Docker
+network (or loopback in dev) — TLS, with the certificate left unchecked since
+the traffic never leaves the host.
+"""
+
+from __future__ import annotations
+
+import logging
+import ssl
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.hosted import enabled, worker_address
+from app.models import Account, User
+from app.providers.imap_smtp import ProviderConfig
+from app.security.crypto import encrypt_secret
+
+logger = logging.getLogger(__name__)
+
+KIND = "native"
+
+
+def is_native(account: Account) -> bool:
+    return account.kind == KIND
+
+
+def _server(host: str, port: int) -> dict[str, object]:
+    return {"host": host, "port": port, "tls": "ssl", "auth": "password"}
+
+
+def internal_tls() -> ssl.SSLContext:
+    """TLS to the mail server on the internal network: its certificate is for
+    its public name (or self-signed), not the container's."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def native_provider_config(email: str) -> ProviderConfig:
+    """IMAP/SMTP to the user's mailbox, signed in as the worker acting for them.
+
+    Read from the settings, not the account row: the mail server's address is
+    the deployment's, and may change."""
+    settings = get_settings()
+    return ProviderConfig(
+        imap_host=settings.stalwart_imap_host,
+        imap_port=settings.stalwart_imap_port,
+        imap_tls="ssl",
+        smtp_host=settings.stalwart_smtp_host,
+        smtp_port=settings.stalwart_smtp_port,
+        smtp_tls="ssl",
+        username=f"{email}%{worker_address()}",
+        password=settings.stalwart_master_secret,
+        tls_context=internal_tls(),
+        internal=True,
+    )
+
+
+async def native_account(db: AsyncSession, user_id: uuid.UUID) -> Account | None:
+    account: Account | None = await db.scalar(
+        select(Account).where(Account.user_id == user_id, Account.kind == KIND)
+    )
+    return account
+
+
+async def ensure(db: AsyncSession, user: User) -> Account | None:
+    """The user's native account, created (and its first sync queued by the
+    caller, see :func:`ensure_and_sync`) if missing. None when hosted
+    mailboxes are off. Flushes, does not commit."""
+    if not enabled():
+        return None
+    existing = await native_account(db, user.id)
+    if existing is not None:
+        return existing
+    # An account for the same address added by hand: the same address at
+    # another provider (the mail before it moved here — e.g. while the domain
+    # still points elsewhere). It stays as it is; the mailbox account is added
+    # once it is removed (app/api/accounts.py).
+    same = await db.scalar(
+        select(Account.id).where(Account.user_id == user.id, Account.email == user.handle)
+    )
+    if same is not None:
+        logger.info("hosted: %s already has an account for its address; not added", user.handle)
+        return None
+    settings = get_settings()
+    imap = _server(settings.stalwart_imap_host, settings.stalwart_imap_port)
+    smtp = _server(settings.stalwart_smtp_host, settings.stalwart_smtp_port)
+    account = Account(
+        user_id=user.id,
+        email=user.handle,
+        kind=KIND,
+        imap_config=imap,
+        smtp_config=smtp,
+        secret_ct=encrypt_secret(b""),  # nothing: the worker signs in as itself
+        discovery_src="hosted",
+        state="backfilling",
+        sync_window="all",
+        # The mail server's own spam filter files into Junk: show it.
+        sync_spam=True,
+    )
+    db.add(account)
+    await db.flush()
+    return account
+
+
+async def ensure_and_sync(db: AsyncSession, user: User) -> Account | None:
+    """:func:`ensure`, committed, and the account's first sync queued if new."""
+    before = await native_account(db, user.id)
+    account = await ensure(db, user)
+    await db.commit()
+    if account is not None and before is None:
+        from app.queue import enqueue_job
+
+        await enqueue_job("sync_bootstrap_account", str(account.id))
+    return account
+
+
+async def ensure_all() -> int:
+    """Give every user without one their native account (users from before
+    hosted mailboxes). Returns how many were created."""
+    from app.db import session_scope
+
+    created = 0
+    async with session_scope() as db:
+        users = list(
+            await db.scalars(
+                select(User).where(
+                    ~select(Account.id)
+                    .where(Account.user_id == User.id, Account.kind == KIND)
+                    .exists()
+                )
+            )
+        )
+    for user in users:
+        async with session_scope() as db:
+            fresh = await db.get(User, user.id)
+            if fresh is None:
+                continue
+            await ensure_and_sync(db, fresh)
+            created += 1
+            logger.info("hosted: mailbox account for %s", fresh.handle)
+    return created

backend/app/hosted/provision.py

@@ -0,0 +1,323 @@
+"""Set up the mail server for hosted mailboxes (SPEC App. D §D.2/§D.3). Idempotent.
+
+Run after the migrations (the ``migrate`` service in production; in dev
+``python -m app.hosted.provision``). Two halves:
+
+1. **Postgres** (as the app's own, privileged database user): the mail
+   server's role ``stalwart`` (password ``STALWART_DB_PASSWORD``), which owns its
+   database and may read the ``mail_directory`` view of the app's database and
+   nothing else there; and the worker's sign-in in ``mail_service_accounts``.
+   The mail server cannot start before this has run once.
+2. **The mail server** (its management API, as the recovery admin): the mail
+   domain and host name, mail kept on disk (not in Postgres), the SQL directory
+   over ``mail_directory`` as the only source of accounts, the worker's
+   impersonate-only role, the submission port for mail apps.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import asyncpg  # type: ignore[import-untyped]
+
+from app.config import get_settings
+from app.hosted import admin, mail_hostname, worker_address
+from app.identity.passwords import hash_password, verify_password
+
+logger = logging.getLogger(__name__)
+
+DB_ROLE = "stalwart"
+DIRECTORY_NAME = "Ultimo webmail users"
+WORKER_ROLE_NAME = "Ultimo webmail worker"
+BLOB_PATH = "/var/lib/stalwart/blobs"
+
+# The directory's queries ($1: the address signed in with / mail is for). A
+# blocked user cannot sign in, but still receives mail.
+QUERY_LOGIN = (
+    "SELECT name, secret, type, description FROM mail_directory WHERE name = $1 AND active"
+)
+QUERY_RECIPIENT = (
+    "SELECT name, NULL::text AS secret, type, description FROM mail_directory WHERE name = $1"
+)
+
+
+def _pg_dsn() -> str:
+    return get_settings().database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+
+
+def _app_db_name() -> str:
+    return urlparse(_pg_dsn()).path.lstrip("/")
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+async def provision_database() -> None:
+    settings = get_settings()
+    if not settings.stalwart_db_password or not settings.stalwart_master_secret:
+        raise RuntimeError("STALWART_DB_PASSWORD and STALWART_MASTER_SECRET must be set")
+    conn = await asyncpg.connect(_pg_dsn())
+    try:
+        password = _quote_literal(settings.stalwart_db_password)
+        if await conn.fetchval("SELECT 1 FROM pg_roles WHERE rolname = $1", DB_ROLE):
+            await conn.execute(f"ALTER ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        else:
+            await conn.execute(f"CREATE ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        db = settings.stalwart_db_name
+        if not await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db):
+            await conn.execute(f'CREATE DATABASE "{db}" OWNER {DB_ROLE}')
+            logger.info("hosted: created database %s", db)
+        await conn.execute(f"GRANT SELECT ON mail_directory TO {DB_ROLE}")
+
+        # The worker's sign-in: re-hashed only when the secret changed.
+        name = worker_address()
+        current = await conn.fetchval(
+            "SELECT secret FROM mail_service_accounts WHERE name = $1", name
+        )
+        if current is None or not verify_password(current, settings.stalwart_master_secret):
+            await conn.execute(
+                """
+                INSERT INTO mail_service_accounts (name, secret, description)
+                VALUES ($1, $2, 'Webmail worker')
+                ON CONFLICT (name) DO UPDATE SET secret = EXCLUDED.secret
+                """,
+                name,
+                hash_password(settings.stalwart_master_secret),
+            )
+    finally:
+        await conn.close()
+
+
+# ---- the mail server ------------------------------------------------------------------
+
+
+async def _wait_for_server(timeout_s: float = 120.0) -> None:
+    deadline = asyncio.get_running_loop().time() + timeout_s
+    while True:
+        try:
+            await admin.get_one("SystemSettings")
+            return
+        except admin.MailServerError:
+            if asyncio.get_running_loop().time() > deadline:
+                raise
+            await asyncio.sleep(2)
+
+
+def _by(items: list[dict[str, Any]], key: str, value: str) -> dict[str, Any] | None:
+    return next((i for i in items if i.get(key) == value), None)
+
+
+async def _domain() -> str:
+    name = get_settings().app_domain.lower()
+    found = _by(await admin.get_all("Domain"), "name", name)
+    if found is not None:
+        return str(found["id"])
+    domain_id = await admin.create("Domain", {"name": name})
+    logger.info("hosted: created mail domain %s", name)
+    return domain_id
+
+
+async def _directory() -> str:
+    settings = get_settings()
+    dsn = urlparse(_pg_dsn())
+    value: dict[str, Any] = {
+        "@type": "Sql",
+        "description": DIRECTORY_NAME,
+        "columnEmail": "name",
+        "columnSecret": "secret",
+        "columnClass": "type",
+        "columnDescription": "description",
+        "queryLogin": QUERY_LOGIN,
+        "queryRecipient": QUERY_RECIPIENT,
+        "queryMemberOf": None,
+        "queryEmailAliases": None,
+        "store": {
+            "@type": "PostgreSql",
+            "host": settings.stalwart_pg_host,
+            "port": 5432 if settings.stalwart_pg_host == "postgres" else (dsn.port or 5432),
+            "database": _app_db_name(),
+            "authUsername": DB_ROLE,
+            "authSecret": {"@type": "EnvironmentVariable", "variableName": "STALWART_DB_PASSWORD"},
+            "useTls": False,
+            "allowInvalidCerts": False,
+            "poolMaxConnections": 5,
+            "poolRecyclingMethod": "fast",
+            "readReplicas": {},
+            "timeout": 15000,
+            "options": None,
+        },
+    }
+    found = _by(await admin.get_all("Directory"), "description", DIRECTORY_NAME)
+    if found is None:
+        directory_id = await admin.create("Directory", value)
+        logger.info("hosted: created the SQL directory")
+    else:
+        directory_id = str(found["id"])
+        await admin.update("Directory", directory_id, value)
+    await admin.update("Authentication", "singleton", {"directoryId": directory_id})
+    return directory_id
+
+
+async def _worker_role() -> str:
+    perms = {"impersonate": True, "authenticate": True}
+    found = _by(await admin.get_all("Role"), "description", WORKER_ROLE_NAME)
+    if found is not None:
+        role_id = str(found["id"])
+        await admin.update("Role", role_id, {"enabledPermissions": perms})
+        return role_id
+    return await admin.create(
+        "Role", {"description": WORKER_ROLE_NAME, "enabledPermissions": perms}
+    )
+
+
+async def _worker_account(role_id: str) -> None:
+    """The worker may act for any mailbox and do nothing as itself."""
+    # Signing in brings it from the directory into the server's registry. With
+    # its role already set that sign-in fails (it may not open a session as
+    # itself) — after the account exists, which is all this needs.
+    worker = worker_address()
+    acc = await admin.account_id(worker)
+    if acc is None:
+        import httpx
+
+        async with httpx.AsyncClient(timeout=15) as client:
+            await client.get(
+                f"{get_settings().stalwart_url}/jmap/session",
+                auth=(worker, get_settings().stalwart_master_secret),
+            )
+        acc = await admin.account_id(worker)
+    if acc is None:
+        raise admin.MailServerError("the worker account did not appear on the mail server")
+    await admin.update("Account", acc, {"roles": {"@type": "Custom", "roleIds": {role_id: True}}})
+
+
+async def _reload() -> None:
+    """Apply the settings now. Stalwart 0.16.19 ships a few spam-list rules its
+    own validator rejects (``bit_and`` in SURBL tags); they never load, but
+    they make every reload fail — so they are removed, which changes nothing."""
+    for _ in range(10):
+        try:
+            await admin.create("Action", {"@type": "ReloadSettings"})
+            return
+        except admin.MailServerError as exc:
+            match = re.search(r"'object': '(SpamDnsblServer)', 'id': '(\w+)'", str(exc))
+            if match is None:
+                raise
+            logger.info("hosted: removing the invalid built-in rule %s", match.group(2))
+            await admin.destroy(match.group(1), [match.group(2)])
+    raise admin.MailServerError("settings reload kept failing")
+
+
+async def _logging() -> None:
+    """Log to the console (``docker logs``), not to files inside the container."""
+    for tracer in await admin.get_all("Tracer"):
+        if tracer.get("@type") != "Stdout":
+            await admin.update(
+                "Tracer",
+                str(tracer["id"]),
+                {
+                    "@type": "Stdout",
+                    "ansi": False,
+                    "buffered": False,
+                    "enable": True,
+                    "level": "info",
+                    "events": {},
+                    "eventsPolicy": "exclude",
+                },
+            )
+
+
+async def _blob_store() -> bool:
+    """Mail on disk (``DATA_ROOT/stalwart``), not inside Postgres: database
+    backups stay small. Takes a restart; True when one is needed."""
+    current = await admin.get_one("BlobStore")
+    if current.get("@type") == "FileSystem" and current.get("path") == BLOB_PATH:
+        return False
+    await admin.update(
+        "BlobStore", "singleton", {"@type": "FileSystem", "path": BLOB_PATH, "depth": 2}
+    )
+    logger.info("hosted: mail is now kept on disk (takes effect after a restart)")
+    return True
+
+
+async def _listeners() -> None:
+    """Add submission with STARTTLS (587) beside the defaults: smtp 25, submissions
+    465, imaps 993 (and http 8080 for this API)."""
+    listeners = await admin.get_all("NetworkListener")
+    if _by(listeners, "name", "submission") is None:
+        await admin.create(
+            "NetworkListener",
+            {"name": "submission", "bind": {"[::]:587": True}, "protocol": "smtp", "useTls": True},
+        )
+
+
+# Folder names for new mailboxes: the plain ones mail apps expect, not
+# Stalwart's Outlook-style defaults ("Deleted Items", "Junk Mail", "Sent
+# Items"). Archive is made too: the webmail archives into it.
+FOLDERS = {
+    "trash": "Trash",
+    "junk": "Junk",
+    "drafts": "Drafts",
+    "sent": "Sent",
+    "archive": "Archive",
+}
+
+
+async def _folders() -> None:
+    folders = {
+        use: {"name": name, "create": True, "subscribe": True, "aliases": {}}
+        for use, name in FOLDERS.items()
+    }
+    await admin.update("Email", "singleton", {"defaultFolders": folders})
+
+
+async def provision_server() -> bool:
+    """Set the mail server up; True when it must be restarted to finish."""
+    await _wait_for_server()
+    await _logging()
+    restart = await _blob_store()
+    domain_id = await _domain()
+    await admin.update(
+        "SystemSettings",
+        "singleton",
+        {"defaultDomainId": domain_id, "defaultHostname": mail_hostname()},
+    )
+    await _listeners()
+    await _folders()
+    await _directory()
+    role_id = await _worker_role()
+    await _reload()
+    await _worker_account(role_id)
+    return restart
+
+
+async def provision() -> bool:
+    from app.hosted import native
+
+    await provision_database()
+    restart = await provision_server()
+    created = await native.ensure_all()
+    if created:
+        logger.info("hosted: added the mailbox account of %d existing user(s)", created)
+    return restart
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    if not get_settings().hosted_mailboxes:
+        logger.info("hosted mailboxes are off (HOSTED_MAILBOXES): nothing to set up")
+        return
+    if asyncio.run(provision()):
+        logger.warning("hosted mailboxes: RESTART the mail server (stalwart) to finish")
+    else:
+        logger.info("hosted mailboxes: mail server ready")
+
+
+if __name__ == "__main__":
+    main()

backend/app/identity/app_passwords.py

@@ -0,0 +1,140 @@
+"""App passwords: one generated password per mail app (SPEC App. D §D.5).
+
+A mail app (Thunderbird, a phone's Mail app) signs in to the hosted mailbox
+over IMAP/SMTP, and a calendar or contacts app to the DAV server — never with
+the account password (no 2FA code can be asked for there), always with one of
+these. Each gets a password of its own, shown once, that can be
+revoked without touching the main password or the other apps. The webmail
+itself never accepts one.
+
+With hosted mailboxes on, the mail server makes the password (``app_`` and 36
+more letters and digits: its format names the credential) and checks it for
+IMAP/SMTP itself; its argon2id hash is kept here too, for the DAV server, with
+the server's id to revoke it there. Without, it is 16 random letters, for DAV
+only. Either way the hash is of the bare password: apps send exactly that, so
+it is copied and entered without the spaces it is shown with.
+"""
+
+from __future__ import annotations
+
+import secrets
+import string
+import uuid
+
+from sqlalchemy import delete, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import hosted
+from app.identity.passwords import hash_password, verify_password
+from app.models import User, UserAppPassword
+
+LENGTH = 16  # 26^16 ≈ 2^75
+SERVER_LENGTH = 40  # the mail server's: "app_" + base32 of a 4-byte id and 18 random bytes
+MAX_PER_USER = 25
+NAME_MAX = 60
+_ALPHABET = string.ascii_lowercase
+
+
+def generation_key(user_id: uuid.UUID) -> str:
+    """Redis counter bumped whenever the user's app passwords change: DAV
+    sign-ins cached under the old value stop counting (app/dav/auth.py)."""
+    return f"apppw:gen:{user_id}"
+
+
+async def _changed(user_id: uuid.UUID) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().incr(generation_key(user_id))
+
+
+class TooMany(Exception):
+    """The user already has MAX_PER_USER app passwords."""
+
+
+def generate() -> str:
+    return "".join(secrets.choice(_ALPHABET) for _ in range(LENGTH))
+
+
+async def list_for(db: AsyncSession, user_id: uuid.UUID) -> list[UserAppPassword]:
+    rows = await db.scalars(
+        select(UserAppPassword)
+        .where(UserAppPassword.user_id == user_id)
+        .order_by(UserAppPassword.created_at.desc())
+    )
+    return list(rows)
+
+
+async def create(db: AsyncSession, user_id: uuid.UUID, name: str) -> tuple[UserAppPassword, str]:
+    """A new app password named ``name``; returns the row and the password
+    itself, which is never available again."""
+    count = await db.scalar(
+        select(func.count()).select_from(UserAppPassword).where(UserAppPassword.user_id == user_id)
+    )
+    if (count or 0) >= MAX_PER_USER:
+        raise TooMany
+    name = name.strip()[:NAME_MAX]
+    server_id: str | None = None
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        handle = await db.scalar(select(User.handle).where(User.id == user_id))
+        server_id, password = await mail_admin.create_app_password(str(handle), name)
+    else:
+        password = generate()
+    row = UserAppPassword(
+        user_id=user_id, name=name, password_hash=hash_password(password), server_id=server_id
+    )
+    db.add(row)
+    await db.commit()
+    return row, password
+
+
+async def _revoke_on_server(
+    db: AsyncSession, user_id: uuid.UUID, rows: list[UserAppPassword]
+) -> None:
+    """Revoke on the mail server too (first: if it cannot be reached, the
+    password keeps working there, so nothing is revoked here either)."""
+    server_ids = [r.server_id for r in rows if r.server_id]
+    if server_ids and hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        handle = await db.scalar(select(User.handle).where(User.id == user_id))
+        await mail_admin.delete_app_passwords(str(handle), server_ids)
+
+
+async def revoke(db: AsyncSession, user_id: uuid.UUID, app_password_id: uuid.UUID) -> bool:
+    rows = list(
+        await db.scalars(
+            select(UserAppPassword).where(
+                UserAppPassword.id == app_password_id, UserAppPassword.user_id == user_id
+            )
+        )
+    )
+    await _revoke_on_server(db, user_id, rows)
+    result = await db.execute(
+        delete(UserAppPassword).where(
+            UserAppPassword.id == app_password_id, UserAppPassword.user_id == user_id
+        )
+    )
+    await db.commit()
+    await _changed(user_id)
+    return bool(result.rowcount)  # type: ignore[attr-defined]
+
+
+async def revoke_all(db: AsyncSession, user_id: uuid.UUID) -> None:
+    """Every app signs in again (the main password was reset)."""
+    await _revoke_on_server(db, user_id, await list_for(db, user_id))
+    await db.execute(delete(UserAppPassword).where(UserAppPassword.user_id == user_id))
+    await db.commit()
+    await _changed(user_id)
+
+
+async def verify(db: AsyncSession, user_id: uuid.UUID, password: str) -> UserAppPassword | None:
+    """The app password ``password`` matches, if any — for the mail server's
+    sign-in and for tests; never for the webmail's own sign-in."""
+    if len(password) not in (LENGTH, SERVER_LENGTH):
+        return None
+    for row in await list_for(db, user_id):
+        if verify_password(row.password_hash, password):
+            return row
+    return None

backend/app/identity/deps.py

@@ -30,8 +30,9 @@ async def current_user(
 ) -> User:
     """Load the authenticated :class:`User` (I-7: scoped by the session's user_id)."""
     user = await db.scalar(select(User).where(User.id == session.user_id))
-    if user is None:
-        # Session references a deleted user; treat as unauthenticated.
+    if user is None or user.blocked_at is not None:
+        # Session references a deleted or blocked user; treat as unauthenticated.
+        # (Blocking also revokes the sessions; this covers any request in flight.)
         raise_api_error(401, "not_authenticated", "authentication required")
     return user
 

backend/app/identity/invitations.py

@@ -0,0 +1,240 @@
+"""Invitations: a user sends a friend a single-use sign-up link.
+
+While the administrator has invitations switched on, every user may send
+``PER_USER`` of them. A link lets exactly one account be created — even with
+public registration closed — and works for ``VALID_FOR``; one that ran out
+unused is handed back to its sender. Only a hash of the link's token is kept.
+
+The email is built from the administrator's template (or the built-in one)
+with these placeholders, written ``{{name}}``:
+
+    invitee_name   the friend's name, as the sender typed it
+    inviter_name   the sender's name (their mail account's display name,
+                   else their address)
+    invite_link    the sign-up address itself
+    invite_button  a ready-made button linking to it
+    logo           the Ultimo Email logo
+
+Names are HTML-escaped where they go into the HTML: they come from users and
+end up in mail to strangers.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import hashlib
+import html as html_lib
+import re
+import secrets
+import uuid
+from dataclasses import dataclass
+
+import nh3
+from sqlalchemy import func, or_, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.identity.mailer import LOGO_HTML
+from app.models import Account, AppSettings, Invitation, User
+from app.outbox.inline_images import InlinePart, data_image
+
+PER_USER = 2
+VALID_FOR = dt.timedelta(days=14)
+MAX_TEMPLATE_BYTES = 500 * 1024
+
+PLACEHOLDERS = ("invitee_name", "inviter_name", "invite_link", "invite_button", "logo")
+_PLACEHOLDER = re.compile(r"\{\{\s*([a-z_]+)\s*\}\}")
+_DATA_SRC = re.compile(r"""(\bsrc\s*=\s*)(["'])(data:image/[^"']+)\2""", re.IGNORECASE)
+
+DEFAULT_SUBJECT = "{{inviter_name}} invites you to Ultimo Email"
+_BODY = "font-family:Arial,Helvetica,sans-serif;font-size:15px;color:#1f2937;line-height:1.5"
+_MUTED = "font-size:13px;color:#6b7280"
+_BUTTON = (
+    "display:inline-block;background:#2563eb;color:#ffffff;text-decoration:none;"
+    "padding:10px 18px;border-radius:8px;font-weight:bold"
+)
+DEFAULT_TEMPLATE = f"""<!doctype html>
+<html><body style="{_BODY}">
+{{{{logo}}}}
+<p>Hi {{{{invitee_name}}}},</p>
+<p><b>{{{{inviter_name}}}}</b> invites you to Ultimo Email — private email,
+calendar and contacts in one place.</p>
+<p>{{{{invite_button}}}}</p>
+<p style="{_MUTED}">Or paste this address into your browser:<br>{{{{invite_link}}}}</p>
+<p style="{_MUTED}">The invitation works once and expires in 14 days. If you
+weren't expecting it, you can ignore this email.</p>
+</body></html>
+"""
+
+
+class InvitationError(Exception):
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+@dataclass(frozen=True)
+class RenderedMail:
+    subject: str
+    text: str
+    html: str
+    inline: list[InlinePart]
+
+
+def hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def sender_address(row: AppSettings) -> str:
+    """The invitation's From. Empty: the system-mail From address, the one the
+    SMTP account is known to be allowed to send as."""
+    return row.invite_from.strip() or row.smtp_from
+
+
+def template_problem(html: str) -> str | None:
+    """Why ``html`` cannot be used as the invitation template, or None."""
+    if len(html.encode()) > MAX_TEMPLATE_BYTES:
+        return f"the template is larger than {MAX_TEMPLATE_BYTES // 1024} KB"
+    used = set(_PLACEHOLDER.findall(html))
+    unknown = sorted(used - set(PLACEHOLDERS))
+    if unknown:
+        return "unknown placeholder: " + ", ".join("{{" + u + "}}" for u in unknown)
+    if not used & {"invite_link", "invite_button"}:
+        return "the template must contain {{invite_link}} or {{invite_button}}"
+    return None
+
+
+def render(row: AppSettings, *, invitee_name: str, inviter_name: str, link: str) -> RenderedMail:
+    """The invitation email for one friend, from the configured template."""
+    template = row.invite_template or DEFAULT_TEMPLATE
+    subject_template = row.invite_subject or DEFAULT_SUBJECT
+    esc_link = html_lib.escape(link, quote=True)
+    html_values = {
+        "invitee_name": html_lib.escape(invitee_name),
+        "inviter_name": html_lib.escape(inviter_name),
+        "invite_link": f'<a href="{esc_link}">{esc_link}</a>',
+        "invite_button": f'<a href="{esc_link}" style="{_BUTTON}">Accept the invitation</a>',
+        "logo": LOGO_HTML,
+    }
+    text_values = {
+        "invitee_name": invitee_name,
+        "inviter_name": inviter_name,
+        "invite_link": link,
+        "invite_button": link,
+        "logo": "",
+    }
+
+    def fill(source: str, values: dict[str, str]) -> str:
+        return _PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), source)
+
+    # An image written into the template itself (data:) goes as an inline part:
+    # Gmail does not show data: images.
+    inline: list[InlinePart] = []
+
+    def to_cid(match: re.Match[str]) -> str:
+        part = data_image(match.group(3), get_settings().app_domain)
+        if part is None:
+            return match.group(0)
+        inline.append(part)
+        return f"{match.group(1)}{match.group(2)}cid:{part.cid}{match.group(2)}"
+
+    html = _DATA_SRC.sub(to_cid, fill(template, html_values))
+    text = fill(template, text_values)
+    # Keep the paragraphs and line breaks the HTML shows once its tags are gone.
+    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"</(?:p|div|h[1-6]|li|tr|table)\s*>", "\n\n", text, flags=re.IGNORECASE)
+    text = nh3.clean(text, tags=set(), attributes={}, strip_comments=True)
+    text = html_lib.unescape(re.sub(r"\n\s*\n\s*", "\n\n", text)).strip() + "\n"
+    subject = " ".join(fill(subject_template, text_values).split())
+    return RenderedMail(subject=subject, text=text, html=html, inline=inline)
+
+
+async def inviter_name(db: AsyncSession, user: User) -> str:
+    """How the friend is told who invited them."""
+    names = (
+        await db.execute(
+            select(Account.email, Account.display_name).where(Account.user_id == user.id)
+        )
+    ).all()
+    own = next((n for e, n in names if e.lower() == user.handle.lower() and n), None)
+    return own or next((n for _, n in names if n), None) or user.handle
+
+
+def _counts(now: dt.datetime):  # type: ignore[no-untyped-def]
+    # Used, or still valid: an invitation that expired unused is given back.
+    return or_(Invitation.used_at.isnot(None), Invitation.expires_at > now)
+
+
+async def remaining(db: AsyncSession, user_id: uuid.UUID) -> int:
+    now = dt.datetime.now(dt.UTC)
+    taken = await db.scalar(
+        select(func.count())
+        .select_from(Invitation)
+        .where(Invitation.inviter_id == user_id, _counts(now))
+    )
+    return max(0, PER_USER - int(taken or 0))
+
+
+async def sent(db: AsyncSession, user_id: uuid.UUID) -> list[Invitation]:
+    return list(
+        await db.scalars(
+            select(Invitation)
+            .where(Invitation.inviter_id == user_id)
+            .order_by(Invitation.created_at.desc())
+        )
+    )
+
+
+async def create(db: AsyncSession, user: User, *, name: str, email: str) -> tuple[Invitation, str]:
+    """A new invitation from ``user`` (not committed): the row and the token.
+    Raises :class:`InvitationError` when the user has none left."""
+    # One at a time per user, so two quick clicks cannot both take the last one.
+    await db.execute(select(User.id).where(User.id == user.id).with_for_update())
+    if await remaining(db, user.id) <= 0:
+        raise InvitationError("no_invitations_left", "you have no invitations left")
+    token = secrets.token_urlsafe(32)
+    invitation = Invitation(
+        inviter_id=user.id,
+        token_hash=hash_token(token),
+        invitee_name=name,
+        invitee_email=email,
+        expires_at=dt.datetime.now(dt.UTC) + VALID_FOR,
+    )
+    db.add(invitation)
+    await db.flush()
+    return invitation, token
+
+
+async def find_valid(db: AsyncSession, token: str) -> Invitation | None:
+    """The unused, unexpired invitation behind ``token``, without using it."""
+    invitation: Invitation | None = await db.scalar(
+        select(Invitation).where(
+            Invitation.token_hash == hash_token(token),
+            Invitation.used_at.is_(None),
+            Invitation.expires_at > dt.datetime.now(dt.UTC),
+        )
+    )
+    return invitation
+
+
+async def claim(db: AsyncSession, token: str) -> uuid.UUID | None:
+    """Use the invitation up (in the caller's transaction): its id, or None if
+    it is unknown, used or expired. Atomic — two sign-ups with one link cannot
+    both get it."""
+    return await db.scalar(
+        update(Invitation)
+        .where(
+            Invitation.token_hash == hash_token(token),
+            Invitation.used_at.is_(None),
+            Invitation.expires_at > dt.datetime.now(dt.UTC),
+        )
+        .values(used_at=func.now())
+        .returning(Invitation.id)
+    )
+
+
+async def record_user(db: AsyncSession, invitation_id: uuid.UUID, user_id: uuid.UUID) -> None:
+    await db.execute(
+        update(Invitation).where(Invitation.id == invitation_id).values(used_by=user_id)
+    )

backend/app/identity/mailer.py

@@ -0,0 +1,175 @@
+"""Outgoing system mail (SPEC §6.1): mail the server sends on its own behalf.
+
+Password-reset links, and invitations users send their friends
+(app/identity/invitations.py). It goes out through the SMTP
+account the administrator configured in the admin panel (``app_settings``),
+never through a user's own mail account — the whole point is that the user
+cannot sign in. The administrator's server is trusted, so unlike user mail
+accounts this connection is not held to the SSRF guard (it may well be a relay
+on the LAN).
+"""
+
+from __future__ import annotations
+
+import base64
+import html as html_lib
+import logging
+from collections.abc import Sequence
+from dataclasses import dataclass
+from email.message import EmailMessage
+from email.utils import formataddr, formatdate, make_msgid
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import aiosmtplib
+
+from app.config import smtp_helo_name
+from app.models import AppSettings
+from app.security.crypto import decrypt_secret
+
+if TYPE_CHECKING:
+    from app.outbox.inline_images import InlinePart
+
+logger = logging.getLogger(__name__)
+
+SENDER_NAME = "Ultimo Email"
+_TIMEOUT = 20.0
+
+# The logo travels inside the message (multipart/related, referenced by cid:),
+# not as a link: it shows even where remote images are blocked, and it is a PNG
+# because mail clients (Gmail first) do not display SVG. Rendered at twice the
+# size it is shown at, for sharp high-DPI screens.
+_LOGO_FILE = Path(__file__).parent / "assets" / "ultimo-logo.png"
+LOGO_CID = "ultimo-logo@ultimo.email"
+LOGO_HTML = (
+    f'<p style="text-align:center;margin:0 0 24px"><img src="cid:{LOGO_CID}" '
+    'width="180" height="47" alt="Ultimo Email" style="display:inline-block;border:0"></p>'
+)
+
+
+@cache
+def _logo() -> bytes:
+    return _LOGO_FILE.read_bytes()
+
+
+# For previews in the admin panel, which show the HTML without message parts.
+LOGO_DATA_URL = "data:image/png;base64," + base64.b64encode(_LOGO_FILE.read_bytes()).decode()
+
+
+class MailerError(Exception):
+    """System mail is not configured, or the SMTP server refused the message."""
+
+
+@dataclass(frozen=True)
+class MailerConfig:
+    host: str
+    port: int
+    security: str  # starttls | ssl | none
+    username: str
+    password: str
+    sender: str
+    public_url: str
+
+
+def mailer_config(row: AppSettings) -> MailerConfig | None:
+    """The usable configuration, or None while the admin has not set one up."""
+    if not (row.smtp_host and row.smtp_from and row.public_url):
+        return None
+    password = decrypt_secret(row.smtp_secret_ct).decode() if row.smtp_secret_ct else ""
+    return MailerConfig(
+        host=row.smtp_host,
+        port=row.smtp_port,
+        security=row.smtp_security,
+        username=row.smtp_username,
+        password=password,
+        sender=row.smtp_from,
+        public_url=row.public_url.rstrip("/"),
+    )
+
+
+async def send_mail(
+    cfg: MailerConfig,
+    to: str,
+    subject: str,
+    text: str,
+    html: str,
+    *,
+    sender: str | None = None,
+    reply_to: str | None = None,
+    inline: Sequence[InlinePart] = (),
+) -> None:
+    """Send one system message. ``sender`` replaces the configured From address
+    (the SMTP account must be allowed to send as it); ``inline`` are images the
+    HTML names by ``cid:``."""
+    msg = EmailMessage()
+    msg["From"] = formataddr((SENDER_NAME, sender or cfg.sender))
+    msg["To"] = to
+    if reply_to:
+        msg["Reply-To"] = reply_to
+    msg["Subject"] = subject
+    # RFC 5322 requires it, and mail without one scores as spam (Gmail).
+    msg["Date"] = formatdate(localtime=True)
+    msg["Message-ID"] = make_msgid(domain=cfg.sender.rsplit("@", 1)[-1])
+    msg["Auto-Submitted"] = "auto-generated"  # RFC 3834: do not auto-reply to this
+    msg.set_content(text)
+    msg.add_alternative(html, subtype="html")
+    if f"cid:{LOGO_CID}" in html:
+        html_part = msg.get_body(("html",))
+        if isinstance(html_part, EmailMessage):
+            html_part.add_related(_logo(), maintype="image", subtype="png", cid=f"<{LOGO_CID}>")
+    if inline:
+        html_part = msg.get_body(("html",))
+        assert isinstance(html_part, EmailMessage)
+        for part in inline:
+            maintype, _, subtype = part.mime_type.partition("/")
+            html_part.add_related(
+                part.data,
+                maintype=maintype,
+                subtype=subtype,
+                cid=f"<{part.cid}>",
+                filename=part.filename,
+                disposition="inline",
+            )
+    try:
+        await aiosmtplib.send(
+            msg,
+            hostname=cfg.host,
+            port=cfg.port,
+            username=cfg.username or None,
+            password=cfg.password or None,
+            use_tls=cfg.security == "ssl",
+            start_tls=cfg.security == "starttls",
+            timeout=_TIMEOUT,
+            local_hostname=smtp_helo_name(),
+        )
+    except (aiosmtplib.SMTPException, OSError) as exc:
+        raise MailerError(str(exc) or type(exc).__name__) from exc
+
+
+def reset_mail(link: str, handle: str) -> tuple[str, str, str]:
+    """(subject, text, html) of the password-reset message."""
+    subject = "Reset your Ultimo Email password"
+    text = (
+        f"Someone asked to change the password of the Ultimo Email account {handle}.\n\n"
+        f"To choose a new password, open this link within 30 minutes:\n\n{link}\n\n"
+        "The link works once. If you did not ask for this, ignore this email — "
+        "your password stays as it is.\n"
+    )
+    h, u = html_lib.escape(handle), html_lib.escape(link, quote=True)
+    muted = "font-size:13px;color:#6b7280"
+    button = (
+        "display:inline-block;background:#2563eb;color:#ffffff;text-decoration:none;"
+        "padding:10px 18px;border-radius:8px;font-weight:bold"
+    )
+    body = "font-family:Arial,Helvetica,sans-serif;font-size:15px;color:#1f2937;line-height:1.5"
+    html = (
+        f'<!doctype html>\n<html><body style="{body}">\n{LOGO_HTML}\n'
+        f"<p>Someone asked to change the password of the Ultimo Email account <b>{h}</b>.</p>\n"
+        "<p>To choose a new password, open this link within 30 minutes:</p>\n"
+        f'<p><a href="{u}" style="{button}">Change password</a></p>\n'
+        f'<p style="{muted}">Or paste this address into your browser:<br>{u}</p>\n'
+        f'<p style="{muted}">The link works once. If you did not ask for this, ignore this '
+        "email — your password stays as it is.</p>\n</body></html>\n"
+    )
+    return subject, text, html

backend/app/identity/password_policy.py

@@ -0,0 +1,145 @@
+"""What a new account password must not be (sign-up and reset).
+
+Length is bounded by the request schemas (8–128). On top of that a password is
+refused when it is one of the passwords every guessing attack tries first, is
+made of the user's own address, is one character repeated, or has turned up in
+a known data breach. The breach check asks Have I Been Pwned's range API with
+the first five hex digits of the password's SHA-1 only (k-anonymity: neither the
+password nor its hash leaves the server) and is skipped, not failed, when that
+service cannot be reached — a sign-up never hangs on someone else's outage.
+Set ``PASSWORD_BREACH_CHECK=false`` to never ask it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+
+import httpx
+
+from app.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+MIN_LENGTH = 8
+MAX_LENGTH = 128
+
+_RANGE_API = "https://api.pwnedpasswords.com/range/"
+_TIMEOUT = 3.0
+
+# The front of every guessing list (and of every breach), lower-cased.
+_COMMON = frozenset(
+    {
+        "password",
+        "password1",
+        "password12",
+        "password123",
+        "password1234",
+        "passw0rd",
+        "p@ssw0rd",
+        "p@ssword",
+        "12345678",
+        "123456789",
+        "1234567890",
+        "0123456789",
+        "87654321",
+        "11111111",
+        "00000000",
+        "12341234",
+        "qwertyui",
+        "qwerty123",
+        "qwerty1234",
+        "qwertyuiop",
+        "1q2w3e4r",
+        "1q2w3e4r5t",
+        "1qaz2wsx",
+        "zaq12wsx",
+        "asdfghjk",
+        "asdfasdf",
+        "iloveyou",
+        "iloveyou1",
+        "sunshine",
+        "princess",
+        "football",
+        "baseball",
+        "welcome1",
+        "welcome123",
+        "letmein1",
+        "trustno1",
+        "superman",
+        "batman123",
+        "dragon123",
+        "monkey123",
+        "abc12345",
+        "abcd1234",
+        "aa123456",
+        "qazwsxedc",
+        "changeme",
+        "changeme1",
+        "administrator",
+        "admin123",
+        "adminadmin",
+        "computer",
+        "internet",
+        "whatever",
+        "starwars",
+        "michelle",
+        "jennifer",
+        "charlie1",
+        "zaq1zaq1",
+        "haslo123",
+        "polska123",
+        "kochamcie",
+        "zaq1@wsx",
+        "qwerty12",
+        "1234qwer",
+        "q1w2e3r4",
+    }
+)
+
+
+class PasswordRejected(Exception):
+    """The password may not be used; ``code`` and ``message`` say why."""
+
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+async def _breached(password: str) -> bool:
+    digest = hashlib.sha1(password.encode("utf-8")).hexdigest().upper()  # noqa: S324 - HIBP's format
+    prefix, suffix = digest[:5], digest[5:]
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(_RANGE_API + prefix, headers={"Add-Padding": "true"})
+            resp.raise_for_status()
+    except httpx.HTTPError as exc:
+        logger.warning("breached-password check skipped: %s", exc)
+        return False
+    for line in resp.text.splitlines():
+        candidate, _, count = line.partition(":")
+        if candidate.strip() == suffix and count.strip() not in ("", "0"):  # padding rows say 0
+            return True
+    return False
+
+
+async def check_new_password(password: str, handle: str) -> None:
+    """Raise :class:`PasswordRejected` if ``password`` may not be used by ``handle``."""
+    lowered = password.lower()
+    if len(set(password)) <= 2:
+        raise PasswordRejected("weak_password", "this password is too easy to guess")
+    if lowered in _COMMON:
+        raise PasswordRejected("weak_password", "this is one of the most common passwords")
+    localpart = handle.split("@", 1)[0].lower()
+    if len(localpart) >= 3 and localpart in lowered:
+        # The name with a year or a few characters added ("jsmith2026") is the
+        # first thing anyone who knows the address will try.
+        rest = lowered.replace(localpart, "", 1)
+        if len(rest) < 4 or all(not c.isalpha() for c in rest):
+            raise PasswordRejected("weak_password", "the password must not be your address")
+    if get_settings().password_breach_check and await _breached(password):
+        raise PasswordRejected(
+            "breached_password",
+            "this password has appeared in a data breach — please choose another",
+        )

backend/app/identity/reset.py

@@ -1,43 +1,76 @@
 """Password reset tokens (SPEC §6.1).
 
-Single-use, 30-minute, Redis-backed tokens. Only issued when the account has a
-recovery email on file. Actual delivery of the token to the recovery address
-requires an app-level mailer (not part of M0/M1); until then the token is logged
-server-side. See docs/open-questions.md.
+Single-use, 30-minute, Redis-backed tokens, emailed as a link to the account's
+recovery address through the system mailer (:mod:`app.identity.mailer`).
+
+A token survives a wrong 2FA code (the page asks again) but not five: after
+that it is burned, so the link cannot be used to guess codes. Requests are
+capped per account so the reset form cannot be used to flood someone's inbox.
 """
 
 from __future__ import annotations
 
-import logging
 import secrets
 import uuid
 
 from app.redis_client import aw, get_redis
 
-logger = logging.getLogger(__name__)
-
 _PREFIX = "pwreset:"
+_FAILS = "pwreset-fail:"
+_RATE = "pwreset-rate:"
 _TTL = 30 * 60  # 30 minutes
+MAX_TOTP_FAILURES = 5
+MAX_REQUESTS_PER_HOUR = 3
 
 
-async def issue_token(user_id: uuid.UUID, recovery_email: str) -> str:
+async def issue_token(user_id: uuid.UUID) -> str:
     """Create a single-use reset token bound to ``user_id`` and return it."""
     token = secrets.token_urlsafe(32)
     await get_redis().set(_PREFIX + token, str(user_id), ex=_TTL)
-    # TODO(spec): deliver via the recovery address once an app mailer exists.
-    logger.info("password reset token issued for user=%s recovery=%s", user_id, recovery_email)
     return token
 
 
+async def request_allowed(user_id: uuid.UUID) -> bool:
+    """Count a reset request; False once the account has had its hourly share."""
+    redis = get_redis()
+    key = _RATE + str(user_id)
+    count = int(await redis.incr(key))
+    if count == 1:
+        await redis.expire(key, 3600)
+    return count <= MAX_REQUESTS_PER_HOUR
+
+
+def _user_id(raw: str | None) -> uuid.UUID | None:
+    if raw is None:
+        return None
+    try:
+        return uuid.UUID(raw)
+    except ValueError:
+        return None
+
+
+async def peek_token(token: str) -> uuid.UUID | None:
+    """The user a still-valid token belongs to, without using it up."""
+    if not token:
+        return None
+    return _user_id(await get_redis().get(_PREFIX + token))
+
+
 async def consume_token(token: str) -> uuid.UUID | None:
     """Atomically consume a reset token, returning the user_id (or None)."""
     if not token:
         return None
     redis = get_redis()
     raw = await aw(redis.getdel(_PREFIX + token))  # single-use: fetch and delete atomically
-    if raw is None:
-        return None
-    try:
-        return uuid.UUID(raw)
-    except ValueError:
-        return None
+    await redis.delete(_FAILS + token)
+    return _user_id(raw)
+
+
+async def record_totp_failure(token: str) -> None:
+    """Count a wrong 2FA code against the token; burn it at the limit."""
+    redis = get_redis()
+    fails = await redis.incr(_FAILS + token)
+    if fails == 1:
+        await redis.expire(_FAILS + token, _TTL)
+    if fails >= MAX_TOTP_FAILURES:
+        await redis.delete(_PREFIX + token, _FAILS + token)

backend/app/identity/routes.py

@@ -3,16 +3,22 @@ TOTP setup/enable, session list/revoke, password reset."""
 
 from __future__ import annotations
 
-from typing import Annotated
+import logging
+import re
+import uuid
+from typing import Annotated, NoReturn
 
-from fastapi import APIRouter, Depends, Request, Response
+from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import hosted
 from app.api.errors import raise_api_error
 from app.config import get_settings
 from app.db import get_session
 from app.events import get_event_bus
+from app.hosted.admin import MailServerError
+from app.identity import app_passwords, invitations, trusted_devices
 from app.identity import totp as totp_mod
 from app.identity.cookies import (
     clear_auth_cookies,
@@ -20,13 +26,29 @@ from app.identity.cookies import (
     set_auth_cookies,
 )
 from app.identity.deps import CurrentSession, CurrentUser, require_csrf
+from app.identity.mailer import MailerConfig, MailerError, mailer_config, reset_mail, send_mail
+from app.identity.password_policy import PasswordRejected, check_new_password
 from app.identity.passwords import hash_password, verify_password
-from app.identity.reset import consume_token, issue_token
+from app.identity.reset import (
+    consume_token,
+    issue_token,
+    peek_token,
+    record_totp_failure,
+    request_allowed,
+)
 from app.identity.schemas import (
+    AppPasswordCreatedOut,
+    AppPasswordIn,
+    AppPasswordOut,
+    InvitationCheckIn,
+    InvitationCheckOut,
     LoginRequest,
     MessageOut,
+    PasswordResetCheckIn,
+    PasswordResetCheckOut,
     PasswordResetIn,
     PasswordResetRequestIn,
+    RecoveryEmailIn,
     RegisterRequest,
     RegistrationStatusOut,
     SessionOut,
@@ -36,10 +58,12 @@ from app.identity.schemas import (
     UserOut,
 )
 from app.identity.service import (
+    AccountBlocked,
     RateLimited,
     RegistrationError,
     TotpInvalid,
     TotpRequired,
+    app_settings,
     authenticate,
     register_user,
     registration_status,
@@ -56,8 +80,14 @@ from app.models import User
 from app.redis_client import get_redis
 from app.sync import leases
 
+logger = logging.getLogger(__name__)
+
 router = APIRouter(prefix="/auth", tags=["auth"])
 
+# Deliberately loose: the recovery address only has to be something mail can be
+# sent to; the mail server is the real judge.
+_EMAIL_RE = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+")
+
 DbSession = Annotated[AsyncSession, Depends(get_session)]
 
 
@@ -80,6 +110,7 @@ def _user_out(user: User) -> UserOut:
         settings=user.settings,
         totp_enabled=user.totp_secret is not None,
         is_admin=user.is_admin,
+        mail_host=hosted.mail_hostname() if hosted.enabled() else None,
     )
 
 
@@ -87,32 +118,77 @@ def _user_out(user: User) -> UserOut:
 async def registration(db: DbSession) -> RegistrationStatusOut:
     """Whether signup is open. Unauthenticated: the signup page needs it to render."""
     status = await registration_status(db)
-    return RegistrationStatusOut(open=status.open, bootstrap=status.bootstrap)
+    return RegistrationStatusOut(
+        open=status.open, bootstrap=status.bootstrap, domain=get_settings().app_domain
+    )
 
 
 _REGISTRATION_ERROR_STATUS = {"handle_taken": 409, "registration_closed": 403}
 
 
+@router.post("/invitation/check", response_model=InvitationCheckOut)
+async def invitation_check(body: InvitationCheckIn, db: DbSession) -> InvitationCheckOut:
+    """Whether an invitation link still works, and who it is from — for the
+    sign-up page. Does not use it up."""
+    if not (await app_settings(db)).invitations_enabled:
+        return InvitationCheckOut(valid=False)
+    invitation = await invitations.find_valid(db, body.token)
+    if invitation is None:
+        return InvitationCheckOut(valid=False)
+    inviter = await db.get(User, invitation.inviter_id)
+    return InvitationCheckOut(
+        valid=True,
+        invitee_name=invitation.invitee_name,
+        inviter_name=await invitations.inviter_name(db, inviter) if inviter else "",
+    )
+
+
 @router.post("/register", response_model=UserOut)
 async def register(
     body: RegisterRequest, request: Request, response: Response, db: DbSession
 ) -> UserOut:
+    try:
+        await check_new_password(body.password, resolve_handle(body.localpart))
+    except PasswordRejected as exc:
+        raise_api_error(400, exc.code, exc.message)
+    invitation_id: uuid.UUID | None = None
+    if body.invite:
+        # Used up in this same transaction: if the sign-up fails (the address
+        # is taken, …) the rollback hands the invitation back.
+        if (await app_settings(db)).invitations_enabled:
+            invitation_id = await invitations.claim(db, body.invite)
+        if invitation_id is None:
+            raise_api_error(
+                400, "invalid_invitation", "this invitation has already been used or has expired"
+            )
     try:
         user = await register_user(
             db,
             localpart=body.localpart,
             password=body.password,
             recovery_email=body.recovery_email,
+            invited=invitation_id is not None,
         )
     except RegistrationError as exc:
         raise_api_error(_REGISTRATION_ERROR_STATUS.get(exc.code, 400), exc.code, exc.message)
+    if invitation_id is not None:
+        await invitations.record_user(db, invitation_id, user.id)
     await db.commit()
+    # The user's own mailbox (App. D §D.4): it exists on the mail server as soon
+    # as the user does (the directory reads the users table); this adds it to
+    # the webmail.
+    from app.hosted import native
+
+    await native.ensure_and_sync(db, user)
     await _establish_session(request, response, user)
     return _user_out(user)
 
 
 @router.post("/login", response_model=UserOut)
 async def login(body: LoginRequest, request: Request, response: Response, db: DbSession) -> UserOut:
+    async def trusted(user_id: uuid.UUID) -> bool:
+        return await trusted_devices.is_trusted(request, response, user_id)
+
     try:
         user = await authenticate(
             db,
@@ -120,6 +196,7 @@ async def login(body: LoginRequest, request: Request, response: Response, db: Db
             password=body.password,
             ip=_client_ip(request),
             totp=body.totp,
+            trusted_device=trusted,
         )
     except RateLimited as exc:
         response.headers["Retry-After"] = str(exc.retry_after)
@@ -128,8 +205,16 @@ async def login(body: LoginRequest, request: Request, response: Response, db: Db
         raise_api_error(401, "totp_required", "a 2FA code is required")
     except TotpInvalid:
         raise_api_error(401, "totp_invalid", "incorrect 2FA code")
+    except AccountBlocked:
+        raise_api_error(
+            403, "account_blocked", "this account has been blocked by the administrator"
+        )
     if user is None:
         raise_api_error(401, "invalid_credentials", "incorrect handle or password")
+    if body.remember_device and body.totp and user.totp_secret is not None:
+        # Only after a code was actually checked: a trusted sign-in cannot extend
+        # the trust to anything new.
+        await trusted_devices.trust(request, response, user.id)
     await _establish_session(request, response, user)
     # Sync v2 §3/§5: a login marks the user active (session heat) and promotes
     # their accounts' tier to hot.
@@ -184,6 +269,7 @@ async def totp_disable(body: TotpDisableRequest, user: CurrentUser, db: DbSessio
         raise_api_error(401, "invalid_credentials", "incorrect password")
     user.totp_secret = None
     await db.commit()
+    await trusted_devices.revoke_all(user.id)  # trust was in place of a code that no longer exists
     return MessageOut()
 
 
@@ -209,25 +295,167 @@ async def revoke(sid: str, user: CurrentUser) -> MessageOut:
 # --- Password reset (SPEC §6.1) ---------------------------------------------
 
 
+async def _send_reset_mail(cfg: MailerConfig, to: str, handle: str, token: str) -> None:
+    """Runs after the response has gone out (see password_reset_request)."""
+    # The token rides in the fragment, which browsers never send to a server, so
+    # it stays out of access logs and Referer headers.
+    subject, text, html = reset_mail(f"{cfg.public_url}/reset-password#token={token}", handle)
+    try:
+        await send_mail(cfg, to, subject, text, html)
+    except MailerError as exc:
+        logger.warning("password reset mail for %s could not be sent: %s", handle, exc)
+
+
 @router.post("/password/reset-request", response_model=MessageOut)
-async def password_reset_request(body: PasswordResetRequestIn, db: DbSession) -> MessageOut:
-    # Always return ok (do not leak whether the handle exists / has recovery mail).
+async def password_reset_request(
+    body: PasswordResetRequestIn, background: BackgroundTasks, db: DbSession
+) -> MessageOut:
+    """Email a reset link to the account's recovery address.
+
+    The answer is the same whether or not the account exists, has a recovery
+    address, or hit the hourly cap, and the mail is sent after the response —
+    so neither the reply nor its timing tells a stranger anything.
+    """
     handle = resolve_handle(body.handle)
     user = await db.scalar(select(User).where(User.handle == handle))
-    if user is not None and user.recovery_email:
-        await issue_token(user.id, user.recovery_email)
+    if user is None or not user.recovery_email:
+        return MessageOut()
+    cfg = mailer_config(await app_settings(db))
+    if cfg is None:
+        logger.warning("password reset requested for %s, but system mail is not set up", handle)
+        return MessageOut()
+    if not await request_allowed(user.id):
+        return MessageOut()
+    token = await issue_token(user.id)
+    background.add_task(_send_reset_mail, cfg, user.recovery_email, user.handle, token)
     return MessageOut()
 
 
+@router.post("/password/reset/check", response_model=PasswordResetCheckOut)
+async def password_reset_check(body: PasswordResetCheckIn, db: DbSession) -> PasswordResetCheckOut:
+    """Whether a reset link still works, so the page can say so before asking
+    for a new password. Does not use the link up."""
+    user_id = await peek_token(body.token)
+    user = await db.get(User, user_id) if user_id else None
+    if user is None:
+        return PasswordResetCheckOut(valid=False)
+    return PasswordResetCheckOut(valid=True, totp_required=user.totp_secret is not None)
+
+
 @router.post("/password/reset", response_model=MessageOut)
 async def password_reset(body: PasswordResetIn, db: DbSession) -> MessageOut:
-    user_id = await consume_token(body.token)
-    if user_id is None:
-        raise_api_error(400, "invalid_token", "reset token is invalid or expired")
-    user = await db.get(User, user_id)
+    """Set a new password with a reset link. With 2FA on, the code is checked
+    before anything changes; a missing or wrong code leaves the link usable
+    (up to MAX_TOTP_FAILURES wrong codes)."""
+    user_id = await peek_token(body.token)
+    user = await db.get(User, user_id) if user_id else None
     if user is None:
-        raise_api_error(400, "invalid_token", "reset token is invalid or expired")
+        raise_api_error(400, "invalid_token", "this link is invalid or has expired")
+    # Before the 2FA code and the link are spent: a refused password costs neither.
+    try:
+        await check_new_password(body.password, user.handle)
+    except PasswordRejected as exc:
+        raise_api_error(400, exc.code, exc.message)
+    if user.totp_secret is not None:
+        if not body.totp:
+            raise_api_error(401, "totp_required", "a 2FA code is required")
+        if not totp_mod.verify(totp_mod.decrypt_totp_secret(user.totp_secret), body.totp):
+            await record_totp_failure(body.token)
+            raise_api_error(401, "totp_invalid", "incorrect 2FA code")
+    # Only now is the link spent — and only if nobody spent it in the meantime.
+    if await consume_token(body.token) != user.id:
+        raise_api_error(400, "invalid_token", "this link is invalid or has expired")
     user.password_hash = hash_password(body.password)
     await db.commit()
-    await destroy_all_sessions(user_id)  # force re-login everywhere after reset
+    await destroy_all_sessions(user.id)  # force re-login everywhere after reset
+    # Mail apps sign in again too: the reset may be the owner taking the account back.
+    try:
+        await app_passwords.revoke_all(db, user.id)
+    except MailServerError:
+        # The new password is set; the app passwords stay listed (and working)
+        # until the mail server answers, so the user can still revoke them.
+        logger.exception("password reset: could not revoke app passwords on the mail server")
+    # Whoever reset the password may be the owner taking the account back: no
+    # device keeps skipping the code on the old trust.
+    await trusted_devices.revoke_all(user.id)
     return MessageOut()
+
+
+# --- App passwords (SPEC App. D §D.5) -----------------------------------------
+
+
+def _mail_server_down() -> NoReturn:
+    raise_api_error(
+        502, "mail_server_unavailable", "the mail server is not answering; try again shortly"
+    )
+
+
+def _app_password_out(row: object) -> AppPasswordOut:
+    from app.models import UserAppPassword
+
+    assert isinstance(row, UserAppPassword)
+    return AppPasswordOut(
+        id=str(row.id),
+        name=row.name,
+        created_at=row.created_at.isoformat(),
+        last_used_at=row.last_used_at.isoformat() if row.last_used_at else None,
+    )
+
+
+@router.get("/app-passwords", response_model=list[AppPasswordOut])
+async def list_app_passwords(user: CurrentUser, db: DbSession) -> list[AppPasswordOut]:
+    return [_app_password_out(r) for r in await app_passwords.list_for(db, user.id)]
+
+
+@router.post(
+    "/app-passwords", response_model=AppPasswordCreatedOut, dependencies=[Depends(require_csrf)]
+)
+async def create_app_password(
+    body: AppPasswordIn, user: CurrentUser, db: DbSession
+) -> AppPasswordCreatedOut:
+    """A new password for one mail app. Takes the account password: an app
+    password opens the mailbox without a 2FA code."""
+    if not verify_password(user.password_hash, body.password):
+        raise_api_error(403, "invalid_password", "incorrect password")
+    if not body.name.strip():
+        raise_api_error(400, "invalid_name", "give the app password a name")
+    try:
+        row, password = await app_passwords.create(db, user.id, body.name)
+    except app_passwords.TooMany:
+        raise_api_error(
+            409, "too_many", f"you can have up to {app_passwords.MAX_PER_USER} app passwords"
+        )
+    except MailServerError:
+        _mail_server_down()
+    return AppPasswordCreatedOut(**_app_password_out(row).model_dump(), app_password=password)
+
+
+@router.delete(
+    "/app-passwords/{app_password_id}",
+    response_model=MessageOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def revoke_app_password(
+    app_password_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> MessageOut:
+    try:
+        revoked = await app_passwords.revoke(db, user.id, app_password_id)
+    except MailServerError:
+        _mail_server_down()
+    if not revoked:
+        raise_api_error(404, "not_found", "app password not found")
+    return MessageOut()
+
+
+@router.put("/recovery-email", response_model=UserOut, dependencies=[Depends(require_csrf)])
+async def set_recovery_email(body: RecoveryEmailIn, user: CurrentUser, db: DbSession) -> UserOut:
+    """Add, change or remove the recovery address. Takes the password: whoever
+    controls this address can take the account over with a reset link."""
+    if not verify_password(user.password_hash, body.password):
+        raise_api_error(403, "invalid_password", "incorrect password")
+    email = (body.recovery_email or "").strip().lower() or None
+    if email is not None and not _EMAIL_RE.fullmatch(email):
+        raise_api_error(400, "invalid_email", "enter a valid email address")
+    user.recovery_email = email
+    await db.commit()
+    return _user_out(user)

backend/app/identity/schemas.py

@@ -3,14 +3,18 @@
 from __future__ import annotations
 
 import uuid
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
 
 class RegisterRequest(BaseModel):
     localpart: str = Field(min_length=3, max_length=30)
-    password: str = Field(min_length=8, max_length=1024)
+    password: str = Field(min_length=8, max_length=128)
     recovery_email: str | None = None
+    # A friend's invitation (app/identity/invitations.py): signs up even while
+    # public registration is closed, and is used up by it.
+    invite: str | None = Field(default=None, max_length=200)
 
 
 class LoginRequest(BaseModel):
@@ -18,6 +22,8 @@ class LoginRequest(BaseModel):
     handle: str = Field(min_length=1, max_length=320)
     password: str = Field(min_length=1, max_length=1024)
     totp: str | None = None  # required when the account has 2FA enabled
+    # With a correct code: don't ask this browser for one again (trusted device).
+    remember_device: bool = False
 
 
 class TotpSetupResponse(BaseModel):
@@ -46,7 +52,41 @@ class PasswordResetRequestIn(BaseModel):
 
 class PasswordResetIn(BaseModel):
     token: str = Field(min_length=1)
-    password: str = Field(min_length=8, max_length=1024)
+    password: str = Field(min_length=8, max_length=128)
+    totp: str | None = None  # required when the account has 2FA enabled
+
+
+class PasswordResetCheckIn(BaseModel):
+    token: str = Field(min_length=1)
+
+
+class PasswordResetCheckOut(BaseModel):
+    valid: bool
+    # Tells the page to expect the 2FA step; only the link's holder learns it.
+    totp_required: bool = False
+
+
+class AppPasswordIn(BaseModel):
+    name: str = Field(min_length=1, max_length=60)  # "Thunderbird on laptop"
+    password: str = Field(min_length=1, max_length=1024)  # the account password, to confirm
+
+
+class AppPasswordOut(BaseModel):
+    id: str
+    name: str
+    created_at: str
+    last_used_at: str | None = None
+
+
+class AppPasswordCreatedOut(AppPasswordOut):
+    # The generated password itself: in this answer only, never again.
+    app_password: str
+
+
+class RecoveryEmailIn(BaseModel):
+    password: str = Field(min_length=1, max_length=1024)
+    # None (or blank) removes the recovery address.
+    recovery_email: str | None = Field(default=None, max_length=320)
 
 
 class UserOut(BaseModel):
@@ -56,6 +96,19 @@ class UserOut(BaseModel):
     settings: dict[str, object]
     totp_enabled: bool = False
     is_admin: bool = False
+    # The hosted mail server's name for mail apps (IMAP 993, SMTP 465/587), or
+    # None when there are no hosted mailboxes (SPEC App. D).
+    mail_host: str | None = None
+
+
+class InvitationCheckIn(BaseModel):
+    token: str = Field(min_length=1, max_length=200)
+
+
+class InvitationCheckOut(BaseModel):
+    valid: bool
+    invitee_name: str = ""
+    inviter_name: str = ""
 
 
 class RegistrationStatusOut(BaseModel):
@@ -64,20 +117,106 @@ class RegistrationStatusOut(BaseModel):
     open: bool
     # True when the instance has no users yet, so signup creates the administrator.
     bootstrap: bool
+    # The domain every handle lives under (localpart@domain), shown beside the
+    # handle field.
+    domain: str
 
 
 class AdminSettingsOut(BaseModel):
     registration_open: bool
     default_quota_bytes: int
+    # Users may send friends sign-up invitations (app/identity/invitations.py).
+    invitations_enabled: bool
 
 
 class AdminSettingsIn(BaseModel):
     """Merge semantics: omitted fields are left as they are."""
 
     registration_open: bool | None = None
+    invitations_enabled: bool | None = None
     # Applies to users created after the change; existing mailboxes keep theirs.
     default_quota_bytes: int | None = Field(default=None, gt=0)
 
 
+class AdminMailOut(BaseModel):
+    smtp_host: str
+    smtp_port: int
+    smtp_security: str
+    smtp_username: str
+    # The password itself never leaves the server.
+    has_password: bool
+    smtp_from: str
+    public_url: str
+    # Host, From address and public address are all set: reset mail can go out.
+    configured: bool
+
+
+class AdminMailIn(BaseModel):
+    """Merge semantics: omitted fields are left as they are."""
+
+    smtp_host: str | None = Field(default=None, max_length=255)
+    smtp_port: int | None = Field(default=None, ge=1, le=65535)
+    smtp_security: Literal["starttls", "ssl", "none"] | None = None
+    smtp_username: str | None = Field(default=None, max_length=320)
+    # Omitted keeps the stored password; an empty string removes it.
+    smtp_password: str | None = Field(default=None, max_length=1024)
+    smtp_from: str | None = Field(default=None, max_length=320)
+    public_url: str | None = Field(default=None, max_length=500)
+
+
+class AdminInviteTemplateOut(BaseModel):
+    # Empty strings: the built-in subject / template / sender is used.
+    subject: str
+    html: str
+    from_address: str
+    default_subject: str
+    default_html: str
+    default_from: str
+    placeholders: list[str]
+
+
+class AdminInviteTemplateIn(BaseModel):
+    """Merge semantics: omitted fields are left as they are; an empty string
+    goes back to the built-in one."""
+
+    subject: str | None = Field(default=None, max_length=300)
+    html: str | None = None
+    from_address: str | None = Field(default=None, max_length=320)
+
+
+class AdminInvitePreviewIn(BaseModel):
+    subject: str = Field(default="", max_length=300)
+    html: str = ""
+
+
+class AdminInvitePreviewOut(BaseModel):
+    subject: str
+    html: str
+
+
+class AdminMailTestIn(BaseModel):
+    to: str = Field(min_length=3, max_length=320)
+
+
+class AdminUserOut(BaseModel):
+    id: uuid.UUID
+    handle: str
+    is_admin: bool
+    # ISO timestamp while blocked; null otherwise.
+    blocked_at: str | None
+    quota_bytes: int
+    used_bytes: int
+    created_at: str
+    # The user's connected mail accounts, so the admin can tell who this is.
+    accounts: list[str]
+
+
+class AdminUserPatch(BaseModel):
+    """Merge semantics: omitted fields are left as they are."""
+
+    quota_bytes: int | None = Field(default=None, gt=0)
+    blocked: bool | None = None
+
+
 class MessageOut(BaseModel):
     status: str = "ok"

backend/app/identity/service.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import uuid
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 
 from sqlalchemy import select
@@ -53,6 +55,10 @@ class TotpInvalid(Exception):
     """Password verified but the supplied TOTP code was wrong."""
 
 
+class AccountBlocked(Exception):
+    """Password verified but an administrator has blocked the account."""
+
+
 def resolve_handle(handle_input: str) -> str:
     """Normalize a login identifier to the full ``localpart@APP_DOMAIN`` form."""
     text = handle_input.strip().lower()
@@ -91,7 +97,12 @@ async def registration_status(db: AsyncSession) -> RegistrationStatus:
 
 
 async def register_user(
-    db: AsyncSession, *, localpart: str, password: str, recovery_email: str | None
+    db: AsyncSession,
+    *,
+    localpart: str,
+    password: str,
+    recovery_email: str | None,
+    invited: bool = False,
 ) -> User:
     """Create a new user. Raises :class:`RegistrationError` on validation/uniqueness.
 
@@ -108,7 +119,7 @@ async def register_user(
 
     settings = await app_settings(db, for_update=True)
     bootstrap = not await _has_users(db)
-    if not bootstrap and not settings.registration_open:
+    if not bootstrap and not settings.registration_open and not invited:
         raise RegistrationError("registration_closed", "registration is closed on this server")
 
     user = User(
@@ -148,12 +159,20 @@ async def _check_rate_limit(handle: str, ip: str) -> None:
 
 
 async def authenticate(
-    db: AsyncSession, *, handle_input: str, password: str, ip: str, totp: str | None = None
+    db: AsyncSession,
+    *,
+    handle_input: str,
+    password: str,
+    ip: str,
+    totp: str | None = None,
+    trusted_device: Callable[[uuid.UUID], Awaitable[bool]] | None = None,
 ) -> User | None:
     """Verify credentials (constant-time). Returns the user or None; may rate-limit.
 
     When the account has TOTP enabled, raises :class:`TotpRequired` (no code given)
-    or :class:`TotpInvalid` (wrong code) after the password check passes.
+    or :class:`TotpInvalid` (wrong code) after the password check passes — unless
+    ``trusted_device`` says this browser was trusted for the account at an earlier
+    2FA sign-in, in which case the password is enough.
     """
     handle = resolve_handle(handle_input)
     await _check_rate_limit(handle, ip)
@@ -164,11 +183,16 @@ async def authenticate(
     ok = verify_password(stored_hash, password)
     if user is None or not ok:
         return None
+    # Only after the password: a stranger must not learn that a handle is blocked.
+    if user.blocked_at is not None:
+        raise AccountBlocked
 
     if user.totp_secret is not None:
         from app.identity.totp import decrypt_totp_secret
         from app.identity.totp import verify as verify_totp
 
+        if not totp and trusted_device is not None and await trusted_device(user.id):
+            return user
         if not totp:
             raise TotpRequired
         if not verify_totp(decrypt_totp_secret(user.totp_secret), totp):

backend/app/identity/sessions.py

@@ -121,6 +121,13 @@ async def list_sessions(user_id: uuid.UUID, current_token: str) -> list[SessionI
     return out
 
 
+async def session_alive(user_id: uuid.UUID | str, sid: str) -> bool:
+    """Whether the session with public id ``sid`` is still signed in."""
+    redis = get_redis()
+    token = await aw(redis.hget(_index_key(user_id), sid))
+    return token is not None and bool(await redis.exists(_PREFIX + token))
+
+
 async def revoke_session(user_id: uuid.UUID, sid: str) -> bool:
     """Revoke a specific session by its public sid. Returns False if not found."""
     redis = get_redis()

backend/app/identity/trusted_devices.py

@@ -0,0 +1,91 @@
+"""Trusted devices: "don't ask for the 2FA code on this device" (SPEC §6.1).
+
+Ticking the box at a successful 2FA sign-in stores a random token in Redis
+(``trustdev:{token}`` -> user id) and hands it to the browser in its own
+HttpOnly cookie. A later sign-in to the *same* account from that browser may
+then skip the code — the password is still required. Other accounts signing in
+on the same browser are unaffected.
+
+The trust lasts TRUST_DAYS and is renewed each time it is used, so a device in
+regular use stays trusted while one left alone for that long has to show a code
+again. A per-user index (``trustdev-user:{user_id}``) lets everything be revoked
+at once: on a password reset, when 2FA is turned off, and when an administrator
+blocks or deletes the account. Signing out does not revoke it — that is the
+point of the box.
+"""
+
+from __future__ import annotations
+
+import secrets
+import uuid
+
+from fastapi import Request, Response
+
+from app.redis_client import aw, get_redis
+
+TRUST_DAYS = 90
+_TTL = TRUST_DAYS * 24 * 3600
+_PREFIX = "trustdev:"
+_INDEX = "trustdev-user:"
+
+COOKIE_SECURE = "__Host-trusted-device"
+COOKIE_INSECURE = "trusted_device"
+
+
+def _cookie_name(request: Request) -> str:
+    # Same split as the session cookie: the __Host- prefix needs HTTPS.
+    return COOKIE_SECURE if request.url.scheme == "https" else COOKIE_INSECURE
+
+
+def read_cookie(request: Request) -> str:
+    return request.cookies.get(COOKIE_SECURE) or request.cookies.get(COOKIE_INSECURE, "")
+
+
+async def trust(request: Request, response: Response, user_id: uuid.UUID) -> None:
+    """Remember this browser for ``user_id`` and set the cookie."""
+    token = secrets.token_urlsafe(32)
+    redis = get_redis()
+    await redis.set(_PREFIX + token, str(user_id), ex=_TTL)
+    await aw(redis.sadd(_INDEX + str(user_id), token))
+    await redis.expire(_INDEX + str(user_id), _TTL)
+    response.set_cookie(
+        _cookie_name(request),
+        token,
+        max_age=_TTL,
+        httponly=True,
+        secure=request.url.scheme == "https",
+        samesite="lax",
+        path="/",
+    )
+
+
+async def is_trusted(request: Request, response: Response, user_id: uuid.UUID) -> bool:
+    """Whether this browser is trusted for ``user_id``; renews the trust if so."""
+    token = read_cookie(request)
+    if not token:
+        return False
+    redis = get_redis()
+    if await redis.get(_PREFIX + token) != str(user_id):
+        return False
+    await redis.expire(_PREFIX + token, _TTL)
+    await redis.expire(_INDEX + str(user_id), _TTL)
+    response.set_cookie(
+        _cookie_name(request),
+        token,
+        max_age=_TTL,
+        httponly=True,
+        secure=request.url.scheme == "https",
+        samesite="lax",
+        path="/",
+    )
+    return True
+
+
+async def revoke_all(user_id: uuid.UUID) -> None:
+    """Forget every device trusted for ``user_id``."""
+    redis = get_redis()
+    index = _INDEX + str(user_id)
+    tokens: set[str] = await aw(redis.smembers(index))
+    if tokens:
+        await redis.delete(*(_PREFIX + t for t in tokens))
+    await redis.delete(index)

backend/app/identity/user_admin.py

@@ -0,0 +1,146 @@
+"""Administrator actions on user accounts (SPEC §6.1 admin panel).
+
+Search, per-user quota, blocking and deletion. Blocking keeps every byte of the
+user's data (and keeps their mail syncing) so unblocking is instant; it only
+refuses sign-in and revokes the sessions they hold. Deletion is total: mail
+accounts (through the same path the user's own "remove account" takes, so the
+sync runners stop), plugin rows, blobs on disk, the search index and the user
+row, whose foreign keys cascade to everything else core owns.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import shutil
+import uuid
+from pathlib import Path
+
+from sqlalchemy import func, or_, select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.accounts.service import delete_account
+from app.config import get_settings
+from app.events import get_event_bus
+from app.identity import trusted_devices
+from app.identity.sessions import destroy_all_sessions
+from app.models import Account, Message, User
+from app.plugins.db import plugin_session
+from app.search.factory import get_search_backend
+
+SEARCH_LIMIT = 25
+
+
+async def search_users(db: AsyncSession, q: str) -> list[tuple[User, int, list[str]]]:
+    """Users whose login, or one of whose mail accounts' address or display name,
+    contains ``q`` (case-insensitive). An empty ``q`` lists the first users by
+    login. Each row comes with its storage used and its mail account addresses."""
+    stmt = select(User).order_by(User.handle).limit(SEARCH_LIMIT)
+    q = q.strip()
+    if q:
+        # Typed % and _ are literal characters, not wildcards.
+        escaped = q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+        like = f"%{escaped}%"
+        by_account = select(Account.user_id).where(
+            or_(Account.email.ilike(like), Account.display_name.ilike(like))
+        )
+        stmt = stmt.where(or_(User.handle.ilike(like), User.id.in_(by_account)))
+    users = list(await db.scalars(stmt))
+    if not users:
+        return []
+    ids = [u.id for u in users]
+    used: dict[uuid.UUID, int] = {
+        user_id: int(total)
+        for user_id, total in await db.execute(
+            select(Message.user_id, func.coalesce(func.sum(Message.size_bytes), 0))
+            .where(Message.user_id.in_(ids))
+            .group_by(Message.user_id)
+        )
+    }
+    emails: dict[uuid.UUID, list[str]] = {}
+    for user_id, email in await db.execute(
+        select(Account.user_id, Account.email)
+        .where(Account.user_id.in_(ids))
+        .order_by(Account.email)
+    ):
+        emails.setdefault(user_id, []).append(email)
+    return [(u, int(used.get(u.id, 0)), emails.get(u.id, [])) for u in users]
+
+
+async def set_blocked(db: AsyncSession, user: User, blocked: bool) -> None:
+    """Block or unblock. Blocking also signs the user out everywhere, and shuts
+    their hosted mailbox to every sign-in (mail apps included); it still
+    receives mail. The mail server goes first: if it cannot be told, nothing
+    changes."""
+    from app import hosted
+
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        await mail_admin.set_blocked(user.handle, blocked)
+    if blocked and user.blocked_at is None:
+        user.blocked_at = dt.datetime.now(dt.UTC)
+        await db.flush()
+        await _sign_out_everywhere(user.id)
+    elif not blocked:
+        user.blocked_at = None
+
+
+async def _sign_out_everywhere(user_id: uuid.UUID) -> None:
+    """Revoke every session, and tell the user's open tabs so they leave at once
+    instead of showing already-loaded mail until their next request."""
+    await destroy_all_sessions(user_id)
+    await trusted_devices.revoke_all(user_id)
+    await get_event_bus().publish("user.signed_out", {"user_id": str(user_id)})
+
+
+async def _plugin_tables(db: AsyncSession) -> list[str]:
+    """Every plugin table holding per-user rows (``plg_*`` with a ``user_id``)."""
+    rows = await db.execute(
+        text(
+            "SELECT table_name FROM information_schema.columns "
+            "WHERE table_schema = current_schema() AND column_name = 'user_id' "
+            "AND table_name LIKE 'plg\\_%' ORDER BY table_name"
+        )
+    )
+    return [r[0] for r in rows]
+
+
+async def delete_user(db: AsyncSession, user: User) -> None:
+    """Delete ``user`` and everything they own. Commits.
+
+    Their hosted mailbox goes first, with all its mail: if the mail server
+    cannot be reached nothing is deleted — a mailbox left behind would be
+    handed, mail and all, to whoever registers the address next."""
+    from app import hosted
+
+    user_id = user.id
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        await mail_admin.delete_account(user.handle)
+
+    # Mail accounts first, one by one: this is what stops their sync runners and
+    # tidies threads, exactly as when the user removes an account themselves.
+    for account_id in list(await db.scalars(select(Account.id).where(Account.user_id == user_id))):
+        await delete_account(db, user_id=user_id, account_id=account_id)
+    await db.commit()
+
+    # Plugin tables carry no foreign key to users and sit behind row-level
+    # security, so they are cleared from inside the user's own RLS context.
+    tables = await _plugin_tables(db)
+    if tables:
+        async with plugin_session(user_id) as ps:
+            for table in tables:
+                await ps.execute(
+                    text(f'DELETE FROM "{table}" WHERE user_id = :uid'), {"uid": user_id}
+                )
+
+    await _sign_out_everywhere(user_id)
+    await get_search_backend().delete_user(user_id)
+
+    # The user row: its foreign keys cascade to the rest of core's tables.
+    await db.delete(user)
+    await db.commit()
+
+    # Whatever blobs are left (drafts' attachments and the like) go with the folder.
+    shutil.rmtree(Path(get_settings().blob_root) / str(user_id), ignore_errors=True)

backend/app/indexer/attachments.py

@@ -0,0 +1,244 @@
+"""Attachments, kept once: inside their message (SPEC App. D §D.6 item 1).
+
+An attachment is not stored apart from its message any more — it was, which
+kept most of a mailbox's bytes twice (once base64 in the raw message, once
+decoded on its own). It is taken out of the raw message when wanted:
+
+* :func:`load` — its bytes (downloads, zip, inline images, forwarding, thumbnails);
+* :func:`file` — a file holding it, for streaming with HTTP Range (videos):
+  taken out into ``{user}/cache/`` and kept a day after its last use;
+* :func:`dedupe_existing` — the one-time pass removing the separate copies
+  stored before, each only after checking it can be taken out of its message.
+
+Blocking (parsing, disk): call these in a thread — after
+:func:`app.indexer.rawstore.ensure` for the message, which fetches a hosted
+mailbox's message from its mail server when it is not cached.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import logging
+import time
+import uuid
+from collections import defaultdict
+from pathlib import Path
+
+from app.indexer import blobstore, rawstore
+from app.indexer.mime import load_message, part_payload
+
+logger = logging.getLogger(__name__)
+
+CACHE_KEEP_S = 24 * 3600  # a taken-out file is kept this long after its last use
+CACHE_MAX_BYTES = 1024**3  # per user; the least recently used go first beyond it
+# part_index of a row whose content is not in its message (kept as its own file).
+NOT_IN_MESSAGE = -1
+
+
+def _legacy(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The separate copy stored before attachments were kept once, if still there."""
+    path = blobstore.blob_path(user_id, sha)
+    return path if path.exists() else None
+
+
+def load(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bytes:
+    """An attachment's bytes. Raises FileNotFoundError when they cannot be had."""
+    legacy = _legacy(user_id, sha)
+    if legacy is not None:
+        return legacy.read_bytes()
+    raw = rawstore.locate(user_id, message_sha)
+    if part_index == NOT_IN_MESSAGE or raw is None:
+        raise FileNotFoundError(sha)
+    payload = part_payload(load_message(path=raw), part_index, sha)
+    if payload is None:
+        raise FileNotFoundError(sha)
+    return payload
+
+
+def materialize(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bool:
+    """Make sure the attachment exists as a blob of its own — for a compose
+    attachment (forward, a draft reopened), which is read by its hash. False
+    when its bytes cannot be had."""
+    if blobstore.exists(user_id, sha):
+        return True
+    try:
+        blobstore.store(user_id, load(user_id, message_sha, sha, part_index))
+    except FileNotFoundError:
+        return False
+    return True
+
+
+def _cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / "cache"
+
+
+def prune_cache(cache: Path) -> None:
+    """Drop streaming copies past their keep time, and the oldest beyond the cap."""
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file()]
+    now = time.time()
+    total = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > CACHE_KEEP_S or total > CACHE_MAX_BYTES:
+            path.unlink(missing_ok=True)
+
+
+def has_file(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether :func:`file` can answer without reading the message."""
+    return _legacy(user_id, sha) is not None or (_cache_dir(user_id) / sha).exists()
+
+
+def file(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> Path:
+    """A file holding the attachment, to stream from. Raises FileNotFoundError."""
+    legacy = _legacy(user_id, sha)
+    if legacy is not None:
+        return legacy
+    cache = _cache_dir(user_id)
+    path = cache / sha
+    if path.exists():
+        path.touch()  # used again: kept longer
+        return path
+    data = load(user_id, message_sha, sha, part_index)
+    cache.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+    tmp.write_bytes(data)
+    tmp.replace(path)
+    with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+        prune_cache(cache)
+    return path
+
+
+def drop_cached(user_id: uuid.UUID) -> None:
+    """Forget every taken-out file of a user (account removed)."""
+    cache = _cache_dir(user_id)
+    if cache.exists():
+        for path in cache.iterdir():
+            path.unlink(missing_ok=True)
+
+
+# ---- removing the copies stored before ------------------------------------------------
+
+# A file this recent may belong to work in progress (a sent message waiting
+# for its Sent copy is referenced only by its job): left for the next pass.
+_MIN_AGE_S = 3600
+
+
+def _part_indexes(raw_path: Path) -> dict[str, int]:
+    """Content hash → MIME part position, for every leaf part of a message."""
+    out: dict[str, int] = {}
+    for index, part in enumerate(load_message(path=raw_path).walk()):
+        if part.is_multipart():
+            continue
+        payload = part.get_payload(decode=True)
+        if isinstance(payload, bytes):
+            out.setdefault(hashlib.sha256(payload).hexdigest(), index)
+    return out
+
+
+async def dedupe_existing(batch: int = 200, *, user_id: uuid.UUID | None = None) -> dict[str, int]:
+    """Record where each older attachment sits in its message, and delete its
+    separate copy once it is known to be there. Idempotent; resumable."""
+    import asyncio
+
+    from sqlalchemy import select, update
+
+    from app.db import session_scope
+    from app.models import Attachment, ComposeAttachment, Message
+
+    stats = {"rows": 0, "found": 0, "not_found": 0, "files_removed": 0, "bytes_freed": 0}
+    while True:
+        async with session_scope() as db:
+            rows = (
+                await db.execute(
+                    select(
+                        Attachment.id,
+                        Attachment.user_id,
+                        Attachment.blob_sha256,
+                        Message.blob_sha256,
+                    )
+                    .join(Message, Message.id == Attachment.message_id)
+                    .where(
+                        Attachment.part_index.is_(None),
+                        *([Attachment.user_id == user_id] if user_id is not None else []),
+                    )
+                    .order_by(Attachment.message_id)
+                    .limit(batch)
+                )
+            ).all()
+        if not rows:
+            return stats
+        by_message: dict[tuple[uuid.UUID, str], list[tuple[uuid.UUID, str]]] = defaultdict(list)
+        for att_id, owner, sha, msg_sha in rows:
+            by_message[(owner, msg_sha)].append((att_id, sha))
+
+        located: dict[uuid.UUID, int] = {}
+        for (owner, msg_sha), atts in by_message.items():
+            raw = rawstore.locate(owner, msg_sha)
+            try:
+                indexes = await asyncio.to_thread(_part_indexes, raw) if raw is not None else {}
+            except Exception:  # noqa: BLE001 - unreadable message: keep the copies
+                logger.warning("attachments: cannot parse %s", raw, exc_info=True)
+                indexes = {}
+            for att_id, sha in atts:
+                located[att_id] = indexes.get(sha, NOT_IN_MESSAGE)
+
+        async with session_scope() as db:
+            for att_id, index in located.items():
+                await db.execute(
+                    update(Attachment).where(Attachment.id == att_id).values(part_index=index)
+                )
+        stats["rows"] += len(located)
+        stats["found"] += sum(1 for i in located.values() if i != NOT_IN_MESSAGE)
+        stats["not_found"] += sum(1 for i in located.values() if i == NOT_IN_MESSAGE)
+
+        # A separate copy goes only when every attachment row with that content
+        # is found in its message and nothing else uses the file.
+        candidates = {
+            (owner, sha)
+            for att_id, owner, sha, _ in rows
+            if located.get(att_id, NOT_IN_MESSAGE) != NOT_IN_MESSAGE
+        }
+        async with session_scope() as db:
+            for owner, sha in candidates:
+                path = blobstore.blob_path(owner, sha)
+                if not path.exists() or time.time() - path.stat().st_mtime < _MIN_AGE_S:
+                    continue
+                in_use = (
+                    await db.scalar(
+                        select(Message.id)
+                        .where(Message.user_id == owner, Message.blob_sha256 == sha)
+                        .limit(1)
+                    )
+                    or await db.scalar(
+                        select(ComposeAttachment.id)
+                        .where(
+                            ComposeAttachment.user_id == owner,
+                            ComposeAttachment.blob_sha256 == sha,
+                        )
+                        .limit(1)
+                    )
+                    or await db.scalar(
+                        select(Attachment.id)
+                        .where(
+                            Attachment.user_id == owner,
+                            Attachment.blob_sha256 == sha,
+                            (Attachment.part_index.is_(None))
+                            | (Attachment.part_index == NOT_IN_MESSAGE),
+                        )
+                        .limit(1)
+                    )
+                )
+                if in_use:
+                    continue
+                size = path.stat().st_size
+                path.unlink(missing_ok=True)
+                stats["files_removed"] += 1
+                stats["bytes_freed"] += size
+
+
+async def dedupe_existing_job(ctx: dict[str, object]) -> dict[str, int]:
+    """arq job (bulk queue): :func:`dedupe_existing`, run once after an update."""
+    stats = await dedupe_existing()
+    logger.info("attachments kept once: %s", stats)
+    return stats

backend/app/indexer/gc.py

@@ -0,0 +1,217 @@
+"""The nightly clean-up of stored files nothing uses any more.
+
+Messages leave in many ways — Delete forever, Empty Trash, the Trash/Spam
+retention purge, an expunge on the server, a folder re-synced — and none of
+them removes the message's file: files are stored by content, so one file can
+serve several rows (the same email in two folders, a forwarded file in a
+draft), and removing it is only safe once nothing refers to it. This pass does
+that for every user, whatever the way the rows went:
+
+* uploads (compose attachments) no draft has referred to for a week — left by
+  a sent message (its draft is gone) or a composer closed without saving;
+* message/attachment files no row refers to;
+* attachment previews (thumbnails) of attachments that are gone;
+* streaming copies past their keep time (app/indexer/attachments.py);
+* cached raw messages of a hosted mailbox beyond the cache's room
+  (app/indexer/rawstore.py).
+
+It also recounts threads whose unread/message counters disagree with their
+messages, and drops threads left empty (app/threading_/threader.py).
+
+A file younger than an hour is left alone: it may belong to work in progress
+(a message being stored, a sent message waiting for its Sent copy).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import delete, or_, select
+
+from app.config import get_settings
+from app.indexer import attachments as att_store
+from app.indexer import rawstore
+
+logger = logging.getLogger(__name__)
+
+MIN_AGE_S = 3600
+UPLOAD_KEEP = dt.timedelta(days=7)
+_SHA = re.compile(r"^[0-9a-f]{64}$")
+
+
+# ---- files a job still needs ---------------------------------------------------------
+#
+# A sent message whose Sent copy could not be filed yet is referenced only by
+# its retrying job (app/outbox/jobs.py), for up to a day: held here meanwhile.
+# Entries older than HOLD_MAX are ignored, in case a job died without letting go.
+
+HOLD_MAX_S = 3 * 24 * 3600
+
+
+def _hold_key(user_id: uuid.UUID | str) -> str:
+    return f"blob:held:{user_id}"
+
+
+async def hold(user_id: uuid.UUID | str, sha: str) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().zadd(_hold_key(user_id), {sha: time.time()})
+
+
+async def release(user_id: uuid.UUID | str, sha: str) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().zrem(_hold_key(user_id), sha)
+
+
+async def _held(user_id: uuid.UUID) -> set[str]:
+    from app.redis_client import get_redis
+
+    redis = get_redis()
+    oldest = time.time() - HOLD_MAX_S
+    await redis.zremrangebyscore(_hold_key(user_id), "-inf", oldest)
+    return {
+        m.decode() if isinstance(m, bytes) else str(m)
+        for m in await redis.zrangebyscore(_hold_key(user_id), oldest, "+inf")
+    }
+
+
+def _draft_upload_ids(payloads: list[dict[str, Any]]) -> set[uuid.UUID]:
+    out: set[uuid.UUID] = set()
+    for payload in payloads:
+        for value in payload.get("attachment_ids") or []:
+            try:
+                out.add(uuid.UUID(str(value)))
+            except ValueError:
+                continue
+    return out
+
+
+def _old(path: Path, now: float) -> bool:
+    return now - path.stat().st_mtime >= MIN_AGE_S
+
+
+async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
+    """Clean one user's stored files; returns what was removed."""
+    from app.db import session_scope
+    from app.models import Account, Attachment, ComposeAttachment, Draft, Message
+
+    stats = {"uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
+    async with session_scope() as db:
+        # Uploads no draft refers to, once a week old.
+        used = _draft_upload_ids(
+            list(await db.scalars(select(Draft.payload).where(Draft.user_id == user_id)))
+        )
+        stale = select(ComposeAttachment.id).where(
+            ComposeAttachment.user_id == user_id,
+            ComposeAttachment.created_at < dt.datetime.now(dt.UTC) - UPLOAD_KEEP,
+        )
+        if used:
+            stale = stale.where(ComposeAttachment.id.not_in(used))
+        gone = list(await db.scalars(stale))
+        if gone:
+            await db.execute(delete(ComposeAttachment).where(ComposeAttachment.id.in_(gone)))
+        stats["uploads"] = len(gone)
+
+    async with session_scope() as db:
+        # Every file still referred to. Attachments count only where their file
+        # is their only copy (not found in the message, or not yet looked for).
+        # A hosted mailbox's messages are kept by its mail server: their raw
+        # file here (left by a Sent copy filed late) is not needed.
+        referenced = set(
+            await db.scalars(
+                select(Message.blob_sha256)
+                .join(Account, Account.id == Message.account_id)
+                .where(Message.user_id == user_id, Account.kind != "native")
+            )
+        )
+        referenced |= set(
+            await db.scalars(
+                select(ComposeAttachment.blob_sha256).where(ComposeAttachment.user_id == user_id)
+            )
+        )
+        referenced |= set(
+            await db.scalars(
+                select(Attachment.blob_sha256).where(
+                    Attachment.user_id == user_id,
+                    or_(
+                        Attachment.part_index.is_(None),
+                        Attachment.part_index == att_store.NOT_IN_MESSAGE,
+                    ),
+                )
+            )
+        )
+        referenced |= await _held(user_id)
+        attachment_shas = set(
+            await db.scalars(select(Attachment.blob_sha256).where(Attachment.user_id == user_id))
+        )
+
+    # Counters out of step with their messages (see repair_threads).
+    from app.threading_.threader import repair_threads
+
+    async with session_scope() as db:
+        stats["threads_repaired"] = await repair_threads(db, user_id)
+
+    root = Path(get_settings().blob_root) / str(user_id)
+    if not root.is_dir():
+        return stats
+    now = time.time()
+    for path in root.glob("??/*"):
+        if not path.is_file() or not _SHA.match(path.name) or path.name in referenced:
+            continue
+        if not _old(path, now):
+            continue
+        size = path.stat().st_size
+        path.unlink(missing_ok=True)
+        stats["files"] += 1
+        stats["bytes"] += size
+    thumbs = root / "thumbs"
+    if thumbs.is_dir():
+        for path in thumbs.iterdir():
+            sha = path.name.split("-", 1)[0]
+            if (
+                path.is_file()
+                and _SHA.match(sha)
+                and sha not in attachment_shas
+                and _old(path, now)
+            ):
+                path.unlink(missing_ok=True)
+                stats["thumbs"] += 1
+    cache = root / "cache"
+    if cache.is_dir():
+        att_store.prune_cache(cache)
+    rawstore.prune(user_id)
+    return stats
+
+
+async def sweep(user_id: uuid.UUID | None = None) -> dict[str, int]:
+    """:func:`sweep_user` for one user, or every user."""
+    from app.db import session_scope
+    from app.models import User
+
+    async with session_scope() as db:
+        users = [user_id] if user_id is not None else list(await db.scalars(select(User.id)))
+    total = {"users": 0, "uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
+    for uid in users:
+        try:
+            stats = await sweep_user(uid)
+        except Exception:  # noqa: BLE001 - one user's trouble must not stop the rest
+            logger.warning("stored-file clean-up failed for user %s", uid, exc_info=True)
+            continue
+        total["users"] += 1
+        for key, value in stats.items():
+            total[key] += value
+    return total
+
+
+async def sweep_job(ctx: dict[str, Any]) -> dict[str, int]:
+    """arq cron job: the nightly :func:`sweep`."""
+    stats = await sweep()
+    logger.info("stored-file clean-up: %s", stats)
+    return stats

backend/app/indexer/indexer.py

@@ -19,12 +19,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.concurrency import loop_semaphore
 from app.config import get_settings
-from app.indexer import blobstore
+from app.indexer import blobstore, rawstore
 from app.indexer.mime import ParsedEmail, parse_message, parse_message_file
 from app.indexer.sanitize import sanitize_html
-from app.models import Attachment, BodyPart, KnownRecipient, Message
+from app.models import Account, Attachment, BodyPart, KnownRecipient, Message
 from app.providers.base import RemoteMessage
 from app.search.base import SearchDoc
+from app.subscriptions import read_list_headers
 from app.threading_.normalize import synthesize_message_id
 from app.threading_.threader import thread_message
 
@@ -118,6 +119,13 @@ async def _resolve_message_id(
     return parsed_id
 
 
+async def _keeps_raw(session: AsyncSession, account_id: uuid.UUID) -> bool:
+    """Whether the raw message is kept for good: not for a hosted mailbox, whose
+    mail server holds it (it is only cached, App. D §D.6 item 2)."""
+    kind = await session.scalar(select(Account.kind).where(Account.id == account_id))
+    return kind != "native"
+
+
 async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmail:
     """Parse under the PARSE_CONCURRENCY cap; stream large messages from disk (§7).
 
@@ -128,8 +136,9 @@ async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmai
     settings = get_settings()
     threshold = settings.stream_parse_mb * 1024 * 1024
     async with loop_semaphore("mime_parse", "parse", settings.parse_concurrency):
-        if len(raw) > threshold:
-            return parse_message_file(blobstore.blob_path(user_id, sha))
+        path = rawstore.locate(user_id, sha) if len(raw) > threshold else None
+        if path is not None:
+            return parse_message_file(path)
         return parse_message(raw)
 
 
@@ -154,8 +163,12 @@ async def persist_message(
     if existing is not None:
         return None
 
-    sha = blobstore.store(user_id, remote.raw)
+    if await _keeps_raw(session, account_id):
+        sha = blobstore.store(user_id, remote.raw)
+    else:
+        sha = rawstore.put_cached(user_id, remote.raw)
     parsed = await _parse_bounded(user_id, sha, remote.raw)
+    list_headers = read_list_headers(remote.raw)  # for the Subscriptions page
 
     message_id_hdr = await _resolve_message_id(
         session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw
@@ -196,6 +209,9 @@ async def persist_message(
         has_attachments=parsed.has_attachments,
         snippet=parsed.snippet,
         blob_sha256=sha,
+        list_id=list_headers.list_id,
+        list_unsubscribe=list_headers.unsubscribe,
+        list_unsubscribe_post=list_headers.one_click,
     )
     session.add(message)
     await session.flush()
@@ -213,11 +229,13 @@ async def persist_message(
                 kind="html_sanitized",
                 content=result.html,
                 sanitizer_v=result.sanitizer_v,
+                trackers_blocked=result.trackers_blocked,
             )
         )
 
+    # Attachments are not stored apart: they stay inside the raw blob, and are
+    # taken out of it when wanted (app/indexer/attachments.py).
     for att in parsed.attachments:
-        att_sha = blobstore.store(user_id, att.payload)
         session.add(
             Attachment(
                 message_id=message.id,
@@ -226,7 +244,8 @@ async def persist_message(
                 mime_type=att.mime_type,
                 size_bytes=len(att.payload),
                 content_id=att.content_id,
-                blob_sha256=att_sha,
+                blob_sha256=blobstore.sha256_hex(att.payload),
+                part_index=att.part_index,
             )
         )
 

backend/app/indexer/mime.py

@@ -41,6 +41,9 @@ class ParsedAttachment:
     mime_type: str
     content_id: str | None
     payload: bytes
+    # The part's position in ``msg.walk()``: where to find it again in the raw
+    # message, which is the only copy kept (attachments are not stored apart).
+    part_index: int = -1
 
 
 @dataclass
@@ -114,7 +117,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
     # or inline resource if it has a filename, a Content-ID, or attachment disposition;
     # the body text/plain and text/html parts have none of these and are skipped.
     out: list[ParsedAttachment] = []
-    for part in msg.walk():
+    for index, part in enumerate(msg.walk()):
         if part.is_multipart() or not isinstance(part, EmailMessage):
             continue
         cid = part.get("Content-ID")
@@ -137,8 +140,28 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
                 mime_type=part.get_content_type(),
                 content_id=cid.strip("<>") if (inline and cid is not None) else None,
                 payload=payload,
+                part_index=index,
             )
         )
+    # An invitation sent only as the text/calendar alternative (Outlook, often) has
+    # no file to show: keep it as invite.ics, so the event can be seen, answered
+    # and downloaded like a Gmail one. Not when the message attaches its own .ics.
+    if not any(a.filename.lower().endswith(".ics") for a in out):
+        for index, part in enumerate(msg.walk()):
+            if part.get_content_type() != "text/calendar" or part.get_filename():
+                continue
+            payload = part.get_payload(decode=True)
+            if isinstance(payload, bytes) and payload.strip():
+                out.append(
+                    ParsedAttachment(
+                        filename="invite.ics",
+                        mime_type="text/calendar",
+                        content_id=None,
+                        payload=payload,
+                        part_index=index,
+                    )
+                )
+                break
     return out
 
 
@@ -197,3 +220,36 @@ def _parse_email(msg: EmailMessage) -> ParsedEmail:
         html_body=_scrub(html_body) if html_body is not None else None,
         attachments=_extract_attachments(msg),
     )
+
+
+def load_message(
+    raw: bytes | None = None, path: str | os.PathLike[str] | None = None
+) -> EmailMessage:
+    """The parsed tree of a raw message, from memory or straight from its file."""
+    if path is not None:
+        with open(path, "rb") as handle:
+            msg = email.message_from_binary_file(handle, policy=default_policy)
+    else:
+        msg = email.message_from_bytes(raw or b"", policy=default_policy)
+    assert isinstance(msg, EmailMessage)
+    return msg
+
+
+def part_payload(msg: EmailMessage, part_index: int | None, sha256: str) -> bytes | None:
+    """The decoded bytes of the attachment whose content hashes to ``sha256``:
+    the part at ``part_index`` when that is it, else whichever part is (rows
+    stored before the index was kept). None when the message has no such part."""
+    import hashlib
+
+    parts = list(msg.walk())
+    if part_index is not None and 0 <= part_index < len(parts):
+        payload = parts[part_index].get_payload(decode=True)
+        if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
+            return payload
+    for part in parts:
+        if part.is_multipart():
+            continue
+        payload = part.get_payload(decode=True)
+        if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
+            return payload
+    return None

backend/app/indexer/rawstore.py

@@ -0,0 +1,170 @@
+"""Where a message's raw source is (SPEC App. D §D.6 item 2).
+
+An external account's messages are kept whole in the blob store, as always.
+A hosted mailbox's are not: the mail server beside the app holds them, so the
+webmail keeps only what it reads them for (headers, the sanitized body, the
+snippet, the search index) and a **cache** of raw messages — the newest ones
+on arrival, the ones opened since — bounded per user (``RAW_CACHE_MB``,
+least recently used out first). Anything else is fetched from the mail server
+when needed: downloading an attachment, "show original", forwarding.
+
+Layout: ``${BLOB_ROOT}/{user_id}/rawcache/{sha256}``, the same content hash
+``messages.blob_sha256`` names.
+
+* :func:`locate` — the file, if it is here (blob store or cache); cheap, sync;
+* :func:`ensure` — the file, fetched from the mail server if need be (async):
+  call it before handing the work to a thread that reads the file.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import time
+import uuid
+from pathlib import Path
+
+from app.config import get_settings
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+CACHE_DIR = "rawcache"
+KEEP_S = 30 * 24 * 3600  # unused this long: dropped whatever the room
+PRUNE_EVERY = 50
+_writes: dict[uuid.UUID, int] = {}
+
+
+def cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / CACHE_DIR
+
+
+def cache_path(user_id: uuid.UUID, sha: str) -> Path:
+    return cache_dir(user_id) / sha
+
+
+def locate(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The message's raw file, if it is kept or cached here."""
+    kept = blobstore.blob_path(user_id, sha)
+    if kept.exists():
+        return kept
+    cached = cache_path(user_id, sha)
+    if cached.exists():
+        with contextlib.suppress(OSError):
+            cached.touch()  # used again: kept longer
+        return cached
+    return None
+
+
+def load(user_id: uuid.UUID, sha: str) -> bytes:
+    """The raw bytes if they are here. Raises FileNotFoundError."""
+    path = locate(user_id, sha)
+    if path is None:
+        raise FileNotFoundError(sha)
+    return path.read_bytes()
+
+
+def put_cached(user_id: uuid.UUID, data: bytes, sha: str | None = None) -> str:
+    """Cache a raw message (``sha`` names it; its content hash by default)."""
+    sha = sha or blobstore.sha256_hex(data)
+    if blobstore.exists(user_id, sha):
+        return sha  # kept for good already (another account has the same message)
+    path = cache_path(user_id, sha)
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+        tmp.write_bytes(data)
+        tmp.replace(path)
+    else:
+        path.touch()
+    # Pruning lists the whole cache: done every PRUNE_EVERY writes, not each
+    # (a backfill writes thousands); meanwhile it overshoots by that much at most.
+    writes = _writes.get(user_id, 0) + 1
+    _writes[user_id] = writes % PRUNE_EVERY
+    if writes >= PRUNE_EVERY:
+        with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+            prune(user_id)
+    return sha
+
+
+def prune(user_id: uuid.UUID, *, max_bytes: int | None = None) -> int:
+    """Drop cached messages unused for KEEP_S, and the least recently used
+    beyond the cap. Returns bytes freed."""
+    cache = cache_dir(user_id)
+    if not cache.is_dir():
+        return 0
+    cap = max_bytes if max_bytes is not None else get_settings().raw_cache_mb * 1024 * 1024
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file() and not p.name.endswith(".tmp")]
+    now = time.time()
+    total = freed = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > KEEP_S or total > cap:
+            path.unlink(missing_ok=True)
+            freed += st.st_size
+    return freed
+
+
+def drop(user_id: uuid.UUID, sha: str) -> None:
+    cache_path(user_id, sha).unlink(missing_ok=True)
+
+
+async def ensure(message: object) -> Path:
+    """The raw file of ``message`` (a :class:`~app.models.Message`), fetched
+    from the hosted mail server into the cache when it is not here. Raises
+    FileNotFoundError when it cannot be had (an external account's message
+    missing from the blob store; a native one gone from the server)."""
+    from app.models import Message
+
+    assert isinstance(message, Message)
+    found = locate(message.user_id, message.blob_sha256)
+    if found is not None:
+        return found
+    raw = await _fetch_native(message)
+    if raw is None:
+        raise FileNotFoundError(message.blob_sha256)
+    got = blobstore.sha256_hex(raw)
+    if got != message.blob_sha256:
+        # The server's copy differs from what was indexed (it should not):
+        # cached under the indexed name, so the message can still be read.
+        logger.warning("raw of message %s changed on the mail server", message.id)
+    put_cached(message.user_id, raw, message.blob_sha256)
+    return cache_path(message.user_id, message.blob_sha256)
+
+
+async def ensure_bytes(message: object) -> bytes:
+    path = await ensure(message)
+    import asyncio
+
+    return await asyncio.to_thread(path.read_bytes)
+
+
+async def _fetch_native(message: object) -> bytes | None:
+    from sqlalchemy import select
+
+    from app.db import session_scope
+    from app.models import Account, Mailbox, Message
+
+    assert isinstance(message, Message)
+    async with session_scope() as db:
+        row = (
+            await db.execute(
+                select(Account.kind, Account.email, Mailbox.path)
+                .join(Mailbox, Mailbox.account_id == Account.id)
+                .where(Account.id == message.account_id, Mailbox.id == message.mailbox_id)
+            )
+        ).first()
+    if row is None or row.kind != "native":
+        return None
+    from app.hosted.native import native_provider_config
+    from app.providers.base import ProviderError
+    from app.providers.imap_smtp import ImapSmtpProvider
+
+    provider = ImapSmtpProvider(native_provider_config(row.email))
+    try:
+        return await provider.fetch_raw(row.path, message.uid)
+    except ProviderError as exc:
+        logger.warning("cannot fetch message %s from the mail server: %s", message.id, exc)
+        return None
+    finally:
+        await provider.close()

backend/app/indexer/sanitize.py

@@ -17,7 +17,16 @@ import nh3
 
 # Bump when the sanitization policy changes; body_parts.sanitizer_v records it so a
 # render re-sanitizes stale content from the raw blob (SPEC §10.4).
-SANITIZER_VERSION = 4
+# v5: this server's own inline-image URLs quoted in a message load from here.
+# v6: links keep only http(s) and mailto (no data: pages).
+# v7: tracking pixels are removed (and counted), even when images are allowed.
+# v8: CSS sizes in other units (in, em, %, pt) no longer read as pixels — v7
+#     took 0.31in signature logos for trackers.
+SANITIZER_VERSION = 8
+
+# This server's inline-image URL, however it ended up written: "/api/…",
+# "https://host/api/…", or mangled to "http:///api/…" by an earlier copy.
+INTERNAL_CID = re.compile(r"/api/messages/([0-9a-fA-F-]{36})/cid/([^\"'?#\s<>]+)")
 
 _ALLOWED_TAGS: set[str] = {
     # structure
@@ -148,6 +157,94 @@ class SanitizeResult:
     html: str
     has_remote_images: bool
     sanitizer_v: int
+    trackers_blocked: int = 0
+
+
+# --- Tracking pixels ---------------------------------------------------------------
+# A remote image that is invisible (1×1, or hidden by CSS) has no use but to tell
+# the sender the mail was opened, and by whom. A few well-known open-tracking URL
+# shapes are caught whatever their size. Matching runs on the raw HTML before nh3:
+# a miss only leaves an image in, and nh3 still cleans whatever remains.
+_IMG_TAG = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
+_ATTR = re.compile(r"""([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("[^"]*"|'[^']*'|[^\s"'>]+)""")
+_CSS_HIDDEN = re.compile(
+    r"display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?![.\d]*[1-9])",
+    re.IGNORECASE,
+)
+_TRACKER_URL = re.compile(
+    r"//[^/]*mailtrack\.io/"
+    r"|/track/open\b"
+    r"|/wf/open\b"
+    r"|/open\.(?:php|aspx|gif)\b"
+    r"|[?&]tracking[_-]?pixel=",
+    re.IGNORECASE,
+)
+
+
+_PX = re.compile(r"\s*(\d+(?:\.\d*)?|\.\d+)\s*(px)?\s*$", re.IGNORECASE)
+
+
+def _px(value: str | None) -> float | None:
+    """A size in pixels ("1", "1px"), or None for anything else: other units
+    (0.31in is a 30-pixel signature logo), auto, %, or malformed (".", "1.2.3")."""
+    if value is None:
+        return None
+    match = _PX.match(value)
+    return float(match.group(1)) if match else None
+
+
+def _tiny(value: str | None) -> bool:
+    px = _px(value)
+    return px is not None and px <= 2
+
+
+def _css(style: str) -> dict[str, str]:
+    """An inline style's declarations, lower-cased, without !important."""
+    out: dict[str, str] = {}
+    for decl in style.split(";"):
+        name, _, value = decl.partition(":")
+        value = value.lower().replace("!important", "").strip()
+        if name.strip() and value:
+            out[name.strip().lower()] = value
+    return out
+
+
+def is_tracking_pixel(tag: str) -> bool:
+    """Whether an ``<img …>`` tag is a remote open-tracking pixel."""
+    attrs = {name.lower(): value.strip("\"'") for name, value in _ATTR.findall(tag)}
+    src = attrs.get("src", "").strip()
+    if not src.lower().startswith(("http:", "https:", "//")):
+        return False  # inline (cid:/data:) images can't report anything
+    if _TRACKER_URL.search(src):
+        return True
+    style = attrs.get("style", "")
+    if _CSS_HIDDEN.search(style):
+        return True
+    # The size the browser draws: a style width/height overrides the attribute
+    # (and "auto", % or another unit is no tiny size); a max-width/max-height
+    # of a pixel or two caps it; height "auto" follows the width.
+    css = _css(style)
+    wide = _tiny(css["width"]) if "width" in css else _tiny(attrs.get("width"))
+    tall = _tiny(css["height"]) if "height" in css else _tiny(attrs.get("height"))
+    wide = wide or _tiny(css.get("max-width"))
+    tall = tall or _tiny(css.get("max-height"))
+    if css.get("height") == "auto":
+        tall = wide
+    return wide and tall
+
+
+def strip_tracking_pixels(html: str) -> tuple[str, int]:
+    """Remove tracking pixels from raw HTML; returns the HTML and how many."""
+    count = 0
+
+    def repl(match: re.Match[str]) -> str:
+        nonlocal count
+        if is_tracking_pixel(match.group(0)):
+            count += 1
+            return ""
+        return match.group(0)
+
+    return _IMG_TAG.sub(repl, html), count
 
 
 def _sanitize_style(value: str, allow_remote: bool) -> str | None:
@@ -182,26 +279,47 @@ def _sanitize_style_elements(html: str, allow_remote: bool, state: dict[str, boo
 
 
 def sanitize_html(
-    html: str, message_id: uuid.UUID, *, allow_remote_images: bool = False
+    html: str,
+    message_id: uuid.UUID,
+    *,
+    allow_remote_images: bool = False,
+    block_trackers: bool = True,
 ) -> SanitizeResult:
     """Sanitize ``html`` for the message ``message_id``.
 
     Returns the safe HTML, whether any remote images were blocked, and the
     sanitizer version. Pass ``allow_remote_images=True`` after the user approves
     a sender to render remote images and CSS backgrounds (SPEC §9.4).
+    ``block_trackers=False`` (Settings → Privacy) leaves tracking pixels in; they
+    are remote images like any other then.
     """
     state = {"has_remote": False}
+    trackers = 0
+    if block_trackers:
+        html, trackers = strip_tracking_pixels(html)
 
     def attribute_filter(tag: str, attr: str, value: str) -> str | None:
         if attr == "style":
             return _sanitize_style(value, allow_remote_images)
         if tag == "a" and attr == "target":
             return "_blank"
+        if attr == "href":
+            # A link goes to the web or to a mail address: data: (a page made
+            # up by the sender) and cid: are kept for inline images only.
+            scheme = value.strip().lower().split(":", 1)[0] if ":" in value else ""
+            return value if scheme in ("", "http", "https", "mailto") else None
         if tag == "img" and attr == "src":
             lowered = value.strip().lower()
             if lowered.startswith("cid:"):
                 content_id = value.strip()[4:]
                 return f"/api/messages/{message_id}/cid/{content_id}"
+            # This server's own inline-image URL, quoted from another message
+            # (by an earlier version, or another of the user's own mails) —
+            # possibly mangled to "http:///api/…", which a browser takes for a
+            # host named "api": load it from here, never look that host up.
+            internal = INTERNAL_CID.search(value)
+            if internal is not None and not lowered.startswith("data:"):
+                return f"/api/messages/{internal.group(1)}/cid/{internal.group(2)}"
             if lowered.startswith(("http:", "https:")):
                 state["has_remote"] = True
                 return value if allow_remote_images else None  # block by default (FR-23)
@@ -229,4 +347,5 @@ def sanitize_html(
         html=cleaned,
         has_remote_images=state["has_remote"],
         sanitizer_v=SANITIZER_VERSION,
+        trackers_blocked=trackers,
     )

backend/app/indexer/thumbs.py

@@ -0,0 +1,132 @@
+"""Attachment thumbnails for the reading pane: images and a PDF's first page.
+
+A thumbnail fits within THUMB_MAX × THUMB_MAX with its aspect ratio kept (the
+browser decides how to frame it), as WebP. Made once and kept beside the blob
+store — ``{BLOB_ROOT}/{user}/thumbs/{sha}-{THUMB_MAX}.webp`` — with a ``.none``
+marker for files that cannot have one (a broken image, an encrypted PDF, HEIC),
+so neither is attempted twice.
+
+Attachments are hostile input: decoding runs in a worker thread, oversized
+files and decompression bombs are refused.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+THUMB_MAX = 480  # px: sharp on a 2× screen for a tile of up to 230 px
+_MAX_INPUT_BYTES = 40 * 1024 * 1024
+_MAX_PIXELS = 64_000_000  # a larger image is refused, not decoded
+_IMAGE_MIME = frozenset(
+    {"image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff"}
+)
+
+
+_EXT_MIME = {
+    "jpg": "image/jpeg",
+    "jpeg": "image/jpeg",
+    "png": "image/png",
+    "gif": "image/gif",
+    "webp": "image/webp",
+    "bmp": "image/bmp",
+    "tif": "image/tiff",
+    "tiff": "image/tiff",
+    "pdf": "application/pdf",
+}
+
+
+def thumb_type(mime_type: str, filename: str) -> str | None:
+    """The type to render a thumbnail as, or None. Senders often label files
+    application/octet-stream, so the extension stands in for a missing type."""
+    mime = mime_type.lower()
+    if mime in _IMAGE_MIME or mime == "application/pdf":
+        return mime
+    return _EXT_MIME.get(filename.rsplit(".", 1)[-1].lower()) if "." in filename else None
+
+
+def can_thumbnail(mime_type: str) -> bool:
+    return mime_type in _IMAGE_MIME or mime_type == "application/pdf"
+
+
+def _path(user_id: uuid.UUID, sha: str, suffix: str) -> Path:
+    return blobstore.blob_path(user_id, sha).parent.parent / "thumbs" / f"{sha}-{THUMB_MAX}{suffix}"
+
+
+def _render(data: bytes, mime_type: str) -> bytes | None:
+    from PIL import Image, ImageOps
+
+    Image.MAX_IMAGE_PIXELS = _MAX_PIXELS  # beyond: DecompressionBombError
+    if mime_type == "application/pdf":
+        import pypdfium2 as pdfium  # type: ignore[import-untyped]
+
+        doc = pdfium.PdfDocument(data)
+        try:
+            page = doc[0]
+            width, height = page.get_size()
+            scale = THUMB_MAX / max(width, height, 1)
+            img = page.render(scale=scale, draw_annots=True).to_pil()
+        finally:
+            doc.close()
+    else:
+        img = Image.open(io.BytesIO(data))
+        img.seek(0)  # an animation: its first frame
+        img = ImageOps.exif_transpose(img)  # phone photos: upright
+        img.thumbnail((THUMB_MAX, THUMB_MAX))
+    if img.mode not in ("RGB", "RGBA"):
+        img = img.convert("RGBA" if "A" in img.getbands() or img.mode == "P" else "RGB")
+    out = io.BytesIO()
+    img.save(out, "WEBP", quality=80, method=4)
+    return out.getvalue()
+
+
+def made(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether the attachment's thumbnail (or its absence) is settled already."""
+    return _path(user_id, sha, ".webp").exists() or _path(user_id, sha, ".none").exists()
+
+
+def thumbnail(
+    user_id: uuid.UUID,
+    sha: str,
+    mime_type: str,
+    size: int,
+    load: Callable[[], bytes],
+) -> bytes | None:
+    """The thumbnail of an attachment (content hash ``sha``, ``size`` bytes,
+    read with ``load`` only when it is not made yet), or None if it cannot
+    have one.
+
+    Blocking (decoding is CPU work): call it in a thread.
+    """
+    done = _path(user_id, sha, ".webp")
+    if done.exists():
+        return done.read_bytes()
+    none = _path(user_id, sha, ".none")
+    if none.exists() or not can_thumbnail(mime_type):
+        return None
+    if size > _MAX_INPUT_BYTES:
+        return None
+    try:
+        data = load()
+    except FileNotFoundError:
+        return None
+    try:
+        thumb = _render(data, mime_type)
+    except Exception as exc:  # noqa: BLE001 - any undecodable file: no thumbnail
+        logger.info("no thumbnail for blob %s (%s): %s", sha, mime_type, exc)
+        thumb = None
+    done.parent.mkdir(parents=True, exist_ok=True)
+    if thumb is None:
+        none.touch()
+        return None
+    tmp = done.with_suffix(".tmp")
+    tmp.write_bytes(thumb)
+    tmp.replace(done)
+    return thumb

backend/app/main.py

@@ -37,6 +37,10 @@ def create_app() -> FastAPI:
 
     register_error_handlers(app)
 
+    from app.security.cross_site import CrossSiteGuard
+
+    app.add_middleware(CrossSiteGuard)
+
     @app.get("/api/healthz")
     async def healthz() -> dict[str, str]:
         """Liveness/readiness probe: checks DB and Redis (dev-setup §6)."""

backend/app/models.py

@@ -69,6 +69,31 @@ class AppSettings(Base):
     default_quota_bytes: Mapped[int] = mapped_column(
         BigInteger, nullable=False, server_default=text("5368709120")
     )
+    # Outgoing system mail (password-reset links): the SMTP account the server
+    # sends from. The password is vault-encrypted (like Account.secret_ct).
+    smtp_host: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    smtp_port: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("587"))
+    smtp_security: Mapped[str] = mapped_column(
+        Text, nullable=False, server_default=text("'starttls'")
+    )  # starttls | ssl | none
+    smtp_username: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    smtp_secret_ct: Mapped[bytes | None] = mapped_column(BYTEA)
+    smtp_from: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    # Where users reach the app, e.g. https://mail.example.com — reset links are
+    # built from this, never from the request's Host header.
+    public_url: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    # The key pair push notifications are signed with (VAPID, RFC 8292): made on
+    # first use; the private half vault-encrypted, the public half as browsers
+    # take it (base64url, uncompressed P-256 point).
+    vapid_public: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    vapid_private_ct: Mapped[bytes | None] = mapped_column(BYTEA)
+    # Users may invite friends (app/identity/invitations.py); the invitation
+    # email's From address (empty: postmaster@APP_DOMAIN), subject and HTML
+    # template (empty: the built-in ones).
+    invitations_enabled: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    invite_from: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    invite_subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    invite_template: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     updated_at: Mapped[dt.datetime] = _now()
 
 
@@ -91,6 +116,9 @@ class User(Base):
     # The first account created on a fresh install bootstraps as administrator;
     # the admin panel can grant it to others later.
     is_admin: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    # Set while an administrator has blocked the account (no sign-in, sessions
+    # refused); NULL when not blocked.
+    blocked_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
     created_at: Mapped[dt.datetime] = _now()
 
 
@@ -110,6 +138,10 @@ class Account(Base):
     imap_config: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     smtp_config: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     secret_ct: Mapped[bytes] = mapped_column(BYTEA, nullable=False)
+    # external: any IMAP/SMTP server (the user's password in secret_ct);
+    # native: the user's own mailbox on the hosted mail server — permanent, no
+    # stored password (SPEC App. D §D.4; app/hosted/).
+    kind: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'external'"))
     discovery_src: Mapped[str] = mapped_column(Text, nullable=False)
     state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'backfilling'"))
     state_detail: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
@@ -121,6 +153,9 @@ class Account(Base):
     backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    # The server has not answered for a while ("unreachable"); cleared on success.
+    sync_error: Mapped[str | None] = mapped_column(Text)
+    sync_error_since: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
     created_at: Mapped[dt.datetime] = _now()
 
 
@@ -137,9 +172,10 @@ class Mailbox(Base):
     )
     path: Mapped[str] = mapped_column(Text, nullable=False)
     special_use: Mapped[str | None] = mapped_column(Text)
-    uidvalidity: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-    highestmodseq: Mapped[int | None] = mapped_column(Integer)
-    last_seen_uid: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # IMAP ids are unsigned 32-bit (MODSEQ 63-bit): bigint, not integer.
+    uidvalidity: Mapped[int] = mapped_column(BigInteger, nullable=False, server_default=text("0"))
+    highestmodseq: Mapped[int | None] = mapped_column(BigInteger)
+    last_seen_uid: Mapped[int] = mapped_column(BigInteger, nullable=False, server_default=text("0"))
 
 
 class Thread(Base):
@@ -163,6 +199,12 @@ class Thread(Base):
     has_attachments: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     last_message_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    # Snooze: out of the Inbox until this time (NULL when not snoozed). A message
+    # that arrives after ``snoozed_at`` wakes it early; ``unsnoozed_at`` (when it
+    # last woke) puts it back at the top of the Inbox.
+    snoozed_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    snoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    unsnoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
 
 class Message(Base):
@@ -190,7 +232,7 @@ class Message(Base):
     restore_mailbox_id: Mapped[uuid.UUID | None] = mapped_column(
         UUID(as_uuid=True), ForeignKey("mailboxes.id", ondelete="SET NULL")
     )
-    uid: Mapped[int] = mapped_column(Integer, nullable=False)
+    uid: Mapped[int] = mapped_column(BigInteger, nullable=False)
     thread_id: Mapped[uuid.UUID | None] = mapped_column(
         UUID(as_uuid=True), ForeignKey("threads.id")
     )
@@ -227,6 +269,17 @@ class Message(Base):
     snippet: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
     body_indexed: Mapped[str | None] = mapped_column(TSVECTOR)
+    # Mailing-list headers, for the Subscriptions page. ``list_unsubscribe`` is
+    # NULL until read (a backfill job fills older mail), '' when there is none.
+    list_id: Mapped[str | None] = mapped_column(Text)
+    list_unsubscribe: Mapped[str | None] = mapped_column(Text)
+    # RFC 8058: the sender takes a one-click unsubscribe POST.
+    list_unsubscribe_post: Mapped[bool] = mapped_column(
+        nullable=False, server_default=text("false")
+    )
+    # The language of its text (app/translation.py), found the first time it is
+    # shown: NULL not looked at yet, '' could not tell.
+    content_language: Mapped[str | None] = mapped_column(Text)
 
 
 class BodyPart(Base):
@@ -240,6 +293,8 @@ class BodyPart(Base):
     content: Mapped[str] = mapped_column(Text, nullable=False)
     # SPEC §10.4: re-sanitize on render if the sanitizer version changed.
     sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # Tracking pixels taken out of this HTML (shown as "N trackers blocked").
+    trackers_blocked: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
 
 
 class Attachment(Base):
@@ -254,7 +309,11 @@ class Attachment(Base):
     mime_type: Mapped[str] = mapped_column(Text, nullable=False)
     size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     content_id: Mapped[str | None] = mapped_column(Text)
+    # The content's hash (names its thumbnail, checks what is taken out). The
+    # bytes themselves live only inside the message's raw blob, at MIME part
+    # ``part_index`` (NULL for rows stored before that was recorded).
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
+    part_index: Mapped[int | None] = mapped_column(Integer)
 
 
 class ComposeAttachment(Base):
@@ -293,7 +352,7 @@ class Draft(Base):
     payload: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     # UID of the mirrored copy in the account's IMAP Drafts folder (so the draft
     # is visible/continuable on other mail clients). Replaced on each edit.
-    imap_uid: Mapped[int | None] = mapped_column(Integer)
+    imap_uid: Mapped[int | None] = mapped_column(BigInteger)
     updated_at: Mapped[dt.datetime] = _now()
 
 
@@ -306,6 +365,10 @@ class Outbox(Base):
     state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'undo_window'"))
     send_after: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
     error: Mapped[str | None] = mapped_column(Text)
+    # Retries after a send the server could not take (0021): state "retrying".
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    next_attempt_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    retry_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
 
 class SavedSearch(Base):
@@ -321,6 +384,138 @@ class SavedSearch(Base):
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("true"))
 
 
+class UserAppPassword(Base):
+    """A password for one mail app (IMAP/SMTP), revocable on its own (§D.5)."""
+
+    __tablename__ = "user_app_passwords"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)  # "Thunderbird on laptop"
+    password_hash: Mapped[str] = mapped_column(Text, nullable=False)  # argon2id
+    created_at: Mapped[dt.datetime] = _now()
+    # Set by the mail server's sign-in, once there is one.
+    last_used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    # The hosted mail server's id of the same password (§D.5), when it has one.
+    server_id: Mapped[str | None] = mapped_column(Text)
+
+
+class Invitation(Base):
+    """A single-use link that lets one person sign up, sent by a user."""
+
+    __tablename__ = "invitations"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    inviter_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    # sha256 of the token in the link; the token itself is only in the email.
+    token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    invitee_name: Mapped[str] = mapped_column(Text, nullable=False)
+    invitee_email: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+    expires_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    used_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL")
+    )
+
+
+class SubscriptionState(Base):
+    """What a user did about a bulk sender on the Subscriptions page."""
+
+    __tablename__ = "subscription_states"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    # The sender group: "list:<List-Id>" or "from:<address>".
+    key: Mapped[str] = mapped_column(Text, primary_key=True)
+    unsubscribed_at: Mapped[dt.datetime] = _now()
+    method: Mapped[str] = mapped_column(Text, nullable=False)  # one_click | mailto | web
+
+
+class MailTemplate(Base):
+    """A saved reply, inserted from the compose window."""
+
+    __tablename__ = "mail_templates"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)
+    subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    body_html: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    created_at: Mapped[dt.datetime] = _now()
+    updated_at: Mapped[dt.datetime] = _now()
+
+
+class MessageTranslation(Base):
+    """A message's HTML translated into a language (app/translation.py), kept so
+    it is translated once. ``images``: with remote images shown or not."""
+
+    __tablename__ = "message_translations"
+
+    message_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
+    )
+    target: Mapped[str] = mapped_column(Text, primary_key=True)
+    images: Mapped[bool] = mapped_column(primary_key=True)
+    source: Mapped[str] = mapped_column(Text, nullable=False)
+    html: Mapped[str] = mapped_column(Text, nullable=False)
+    sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+
+
+class CompletionPhrase(Base):
+    """Sentence completion (app/completion.py): after these words, the user
+    usually writes this — seen ``count`` times in their sent mail."""
+
+    __tablename__ = "completion_phrases"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    context: Mapped[str] = mapped_column(Text, primary_key=True)  # normalized words
+    continuation: Mapped[str] = mapped_column(Text, primary_key=True)  # as written
+    count: Mapped[int] = mapped_column(Integer, nullable=False)
+
+
+class CompletionIndexState(Base):
+    """When a user's completion index was last built."""
+
+    __tablename__ = "completion_index_state"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    built_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+
+
+class PushSubscription(Base):
+    """A browser or installed app to tell about new mail (Web Push)."""
+
+    __tablename__ = "push_subscriptions"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    # The push service URL the browser handed out; posting to it reaches the device.
+    endpoint: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    # The browser's keys the message is encrypted to (RFC 8291), base64url.
+    p256dh: Mapped[str] = mapped_column(Text, nullable=False)
+    auth: Mapped[str] = mapped_column(Text, nullable=False)
+    # The sign-in it was made under: nothing is sent once that session is gone.
+    session_sid: Mapped[str] = mapped_column(Text, nullable=False)
+    user_agent: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    created_at: Mapped[dt.datetime] = _now()
+    last_used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+
+
 class PluginKv(Base):
     """Namespaced key/value store for plugins (SPEC §11 ``plugin_kv``).
 

backend/app/outbox/draft_sync.py

@@ -17,6 +17,7 @@ from sqlalchemy import select
 
 from app.db import session_scope
 from app.models import Account, Draft, Mailbox, Message
+from app.outbox import inline_images
 from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
 from app.sync.provider_factory import build_provider
@@ -68,7 +69,15 @@ async def push_draft_to_imap_job(ctx: dict[str, Any], draft_id: str) -> None:
             else None
         )
         attachments = await load_attachments(session, user_id, draft)
-        _, sent_raw, _, _ = build_mime(account, draft, reply_to, attachments)
+        html, inline = await inline_images.embed(
+            session,
+            user_id,
+            str(draft.payload.get("html", "") or ""),
+            account.email.rsplit("@", 1)[-1],
+        )
+        _, sent_raw, _, _ = build_mime(
+            account, draft, reply_to, attachments, html=html, inline=inline
+        )
         drafts_mb = await session.scalar(
             select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )

backend/app/outbox/inline_images.py

@@ -0,0 +1,153 @@
+"""Inline images in an outgoing message's HTML.
+
+A reply quotes the original as the reading pane shows it, so an image embedded
+in the original (``cid:`` — a signature logo, a pasted screenshot) appears in
+the quote as this server's own URL, ``/api/messages/{id}/cid/{content-id}``.
+Sent as is, that URL means nothing to the recipient (and to anyone else,
+including ourselves once the original is gone). Before sending, each such image
+is put back into the message as an inline part with a new Content-ID — as any
+mail program does — and one that can no longer be had is left out.
+
+An image written into the HTML itself (``data:image/…;base64,…`` — a signature
+logo pasted into the editor) goes the same way: Gmail and other mail programs
+do not show ``data:`` images in mail they receive, so each becomes an inline
+part too (one per distinct image).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import binascii
+import hashlib
+import re
+import uuid
+from dataclasses import dataclass
+from email.utils import make_msgid
+from urllib.parse import unquote
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.indexer import attachments as att_store
+from app.indexer import rawstore
+from app.indexer.sanitize import INTERNAL_CID
+from app.models import Attachment, Message
+
+_DATA_IMG = re.compile(r"^data:(image/[a-z0-9.+-]+);base64,(.*)$", re.IGNORECASE | re.DOTALL)
+_EXT = {"jpeg": "jpg", "svg+xml": "svg"}
+
+_IMG = re.compile(r"<img\b[^>]*?\bsrc\s*=\s*([\"'])(.*?)\1[^>]*>", re.IGNORECASE | re.DOTALL)
+
+
+@dataclass(frozen=True)
+class InlinePart:
+    cid: str  # without angle brackets
+    mime_type: str
+    filename: str
+    data: bytes
+
+
+def data_image(src: str, domain: str) -> InlinePart | None:
+    """The inline part for a ``data:image/…;base64`` URL, or None if it is not one."""
+    m = _DATA_IMG.match(src.strip())
+    if m is None:
+        return None
+    try:
+        data = base64.b64decode(re.sub(r"\s+", "", m.group(2)), validate=True)
+    except (binascii.Error, ValueError):
+        return None
+    if not data:
+        return None
+    mime_type = m.group(1).lower()
+    subtype = mime_type.split("/", 1)[1]
+    name = f"image.{_EXT.get(subtype, subtype)}"
+    return InlinePart(make_msgid(domain=domain)[1:-1], mime_type, name, data)
+
+
+def internal_cid(src: str) -> tuple[uuid.UUID, str] | None:
+    """The (message id, content id) an image URL of this server points at."""
+    value = src.strip()
+    if value.lower().startswith(("cid:", "data:")):
+        return None
+    m = INTERNAL_CID.search(value)
+    if m is None:
+        return None
+    try:
+        return uuid.UUID(m.group(1)), unquote(m.group(2))
+    except ValueError:
+        return None
+
+
+async def embed(
+    session: AsyncSession, user_id: uuid.UUID, html: str, domain: str
+) -> tuple[str, list[InlinePart]]:
+    """``html`` with this server's image URLs turned into ``cid:`` references,
+    and the parts to send them in."""
+    parts: dict[tuple[uuid.UUID, str], InlinePart | None] = {}
+    # Images written into the HTML, by content: the same logo twice is one part.
+    data_parts: dict[str, InlinePart | None] = {}
+
+    def data_key(src: str) -> str:
+        return hashlib.sha256(src.strip().encode()).hexdigest()
+
+    for match in _IMG.finditer(html):
+        src = match.group(2)
+        if src.strip().lower().startswith("data:"):
+            key = data_key(src)
+            if key not in data_parts:
+                data_parts[key] = data_image(src, domain)
+            continue
+        target = internal_cid(src)
+        if target is None or target in parts:
+            continue
+        message_id, content_id = target
+        found = (
+            await session.execute(
+                select(Attachment, Message)
+                .join(Message, Message.id == Attachment.message_id)
+                .where(
+                    Attachment.message_id == message_id,
+                    Attachment.content_id == content_id,
+                    Message.user_id == user_id,
+                )
+                .limit(1)
+            )
+        ).first()
+        part: InlinePart | None = None
+        if found is not None:
+            att, message = found
+            try:
+                await rawstore.ensure(message)  # a hosted mailbox's: fetched if need be
+                data = await asyncio.to_thread(
+                    att_store.load, user_id, message.blob_sha256, att.blob_sha256, att.part_index
+                )
+                cid = make_msgid(domain=domain)[1:-1]
+                part = InlinePart(cid, att.mime_type, att.filename, data)
+            except FileNotFoundError:
+                part = None
+        parts[target] = part
+
+    def replace(match: re.Match[str]) -> str:
+        quote = match.group(1)
+        src = match.group(2)
+        if src.strip().lower().startswith("data:"):
+            data_part = data_parts.get(data_key(src))
+            if data_part is None:
+                return match.group(0)  # not an image we can carry: left as it was
+            return match.group(0).replace(
+                f"src={quote}{src}{quote}", f"src={quote}cid:{data_part.cid}{quote}", 1
+            )
+        target = internal_cid(match.group(2))
+        if target is None:
+            return match.group(0)
+        part = parts.get(target)
+        if part is None:
+            return ""  # gone: no broken image for the recipient
+        return match.group(0).replace(
+            f"src={quote}{match.group(2)}{quote}", f"src={quote}cid:{part.cid}{quote}", 1
+        )
+
+    out = _IMG.sub(replace, html) if parts or data_parts else html
+    quoted = [p for p in parts.values() if p is not None]
+    return out, quoted + [p for p in data_parts.values() if p is not None]

backend/app/outbox/jobs.py

@@ -8,26 +8,31 @@ to Sent (skipped for Gmail, which self-appends), harvest recipients into
 
 from __future__ import annotations
 
-import contextlib
 import logging
 import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from email.utils import getaddresses
 from typing import Any
 
-from sqlalchemy import select
+from arq import Retry
+from sqlalchemy import select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import session_scope
 from app.events import get_event_bus
+from app.indexer import blobstore, gc
 from app.indexer.indexer import persist_message
 from app.models import Account, Draft, KnownRecipient, Mailbox, Message, Outbox
+from app.outbox import inline_images
 from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
 from app.outbox.service import delete_mirrored_draft_row
 from app.providers.base import RemoteMessage
+from app.queue import enqueue_job
 from app.search.factory import get_search_backend
+from app.sync.actions import _retry_later
+from app.sync.health import is_temporary
 from app.sync.provider_factory import build_provider
 
 logger = logging.getLogger(__name__)
@@ -111,14 +116,125 @@ async def _persist_sent_copy(user_id: uuid.UUID, account: Account, raw: bytes, u
         await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
 
 
+# A send the mail server could not take (unreachable, a temporary 4xx) is tried
+# again after these gaps, then hourly, until RETRY_WINDOW after the first
+# failure. Meanwhile the message waits in the Scheduled folder.
+_RETRY_GAPS_S = (60, 120, 300, 600, 1800)
+_RETRY_HOURLY_S = 3600
+RETRY_WINDOW = timedelta(hours=24)
+
+
+def _retry_gap(attempts: int) -> int:
+    return _RETRY_GAPS_S[attempts - 1] if attempts <= len(_RETRY_GAPS_S) else _RETRY_HOURLY_S
+
+
+async def _send_failed(
+    oid: uuid.UUID, user_id: uuid.UUID, subject: str, exc: BaseException
+) -> None:
+    """Record a failed send: "retrying" if it may work later, else "failed"."""
+    now = datetime.now(UTC)
+    async with session_scope() as session:
+        ob = await session.get(Outbox, oid)
+        if ob is None:
+            return
+        ob.attempts += 1
+        ob.error = (getattr(exc, "detail", None) or str(exc) or type(exc).__name__)[:500]
+        ob.retry_until = ob.retry_until or now + RETRY_WINDOW
+        if is_temporary(exc) and now < ob.retry_until:
+            ob.state = "retrying"
+            ob.next_attempt_at = now + timedelta(seconds=_retry_gap(ob.attempts))
+        else:
+            ob.state = "failed"
+            ob.next_attempt_at = None
+        state, attempts, next_at, error = ob.state, ob.attempts, ob.next_attempt_at, ob.error
+    if state == "retrying" and next_at is not None:
+        await enqueue_job(
+            "send_job", str(oid), _defer_until=next_at, _job_id=f"send:{oid}:{attempts}"
+        )
+    await get_event_bus().publish(
+        "outbox.state_changed",
+        {
+            "user_id": str(user_id),
+            "outbox_id": str(oid),
+            "state": state,
+            "attempts": attempts,
+            "subject": subject,
+            "error": error,
+            "next_attempt_at": next_at.isoformat() if next_at else None,
+        },
+    )
+    if state == "failed":
+        logger.warning("send failed for outbox %s: %s", oid, error)
+    else:
+        logger.info(
+            "send of outbox %s failed (try %d), retrying at %s: %s", oid, attempts, next_at, error
+        )
+
+
+async def file_sent_copy_job(ctx: dict[str, Any], account_id: str, user_id: str, sha: str) -> None:
+    """File a sent message's copy in Sent, after the send itself succeeded but
+    the IMAP server could not be reached. Retried (arq Retry) for ~24 hours.
+
+    Until then the copy's file is referenced only by this job: it is held from
+    the nightly clean-up (app/indexer/gc.py) and let go once filed or given up."""
+    try:
+        await _file_sent_copy(ctx, account_id, user_id, sha)
+    except Retry:
+        raise  # tried again later: the file is still needed
+    except BaseException:
+        await gc.release(user_id, sha)
+        raise
+    await gc.release(user_id, sha)
+
+
+async def _file_sent_copy(ctx: dict[str, Any], account_id: str, user_id: str, sha: str) -> None:
+    uid = uuid.UUID(user_id)
+    async with session_scope() as session:
+        account = await session.get(Account, uuid.UUID(account_id))
+        if account is None:
+            return
+        sent_path = await session.scalar(
+            select(Mailbox.path).where(
+                Mailbox.account_id == account.id, Mailbox.special_use == "sent"
+            )
+        )
+    if not sent_path:
+        return
+    raw = blobstore.load(uid, sha)
+    provider = build_provider(account)
+    try:
+        appended_uid = await provider.append(sent_path, raw, frozenset({"\\Seen"}))
+    except Exception as exc:
+        retry = _retry_later(ctx, exc)
+        if retry is not None:
+            raise retry from exc
+        logger.error("could not file a sent message in Sent (blob %s): %s", sha, exc)
+        return
+    finally:
+        await provider.close()
+    await _persist_sent_copy(uid, account, raw, appended_uid)
+    logger.info("filed a sent message in Sent for account %s", account_id)
+
+
 async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
-    """Send a queued message unless it was cancelled during the undo window."""
+    """Send a queued message unless it was cancelled during the undo window.
+
+    Also runs each retry of a send that failed for a temporary reason. The row is
+    claimed with one conditional UPDATE, so two jobs for the same send (a planned
+    retry and "Retry now") can never both send it.
+    """
     oid = uuid.UUID(outbox_id)
     async with session_scope() as session:
-        outbox = await session.get(Outbox, oid)
-        if outbox is None or outbox.state != "undo_window":
+        claimed = await session.scalar(
+            update(Outbox)
+            .where(Outbox.id == oid, Outbox.state.in_(("undo_window", "retrying")))
+            .values(state="sending")
+            .returning(Outbox.id)
+        )
+        if claimed is None:
             return  # cancelled, or already handled -> no SMTP traffic
-        outbox.state = "sending"
+        outbox = await session.get(Outbox, oid)
+        assert outbox is not None
         draft = await session.get(Draft, outbox.draft_id)
         if draft is None:
             outbox.state = "failed"
@@ -136,8 +252,16 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
             return
         user_id = outbox.user_id
         attachments = await load_attachments(session, user_id, draft)
+        # Images quoted from the original travel inside the message, not as
+        # links to this server (app/outbox/inline_images.py).
+        html, inline = await inline_images.embed(
+            session,
+            user_id,
+            str(draft.payload.get("html", "") or ""),
+            account.email.rsplit("@", 1)[-1],
+        )
         transmit_raw, sent_raw, envelope_from, envelope_to = build_mime(
-            account, draft, reply_to, attachments
+            account, draft, reply_to, attachments, html=html, inline=inline
         )
         sent_path = await session.scalar(
             select(Mailbox.path).where(
@@ -153,28 +277,32 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
         gmail = _is_gmail(account)
         account_obj = account
 
+    subject = str(draft.payload.get("subject") or "")
     provider = build_provider(account_obj)
     appended_uid = 0
     try:
-        await provider.send(transmit_raw, envelope_from, envelope_to)
+        try:
+            await provider.send(transmit_raw, envelope_from, envelope_to)
+        except Exception as exc:  # noqa: BLE001 - retry or record; the draft is kept
+            await _send_failed(oid, user_id, subject, exc)
+            return
+        # Delivered. If filing the copy in Sent fails (the IMAP server is not
+        # there right now), the send still succeeded — and the copy must not be
+        # lost: it is kept and filed later by file_sent_copy_job.
         if sent_path and not gmail:
-            appended_uid = await provider.append(sent_path, sent_raw, frozenset({"\\Seen"}))
+            try:
+                appended_uid = await provider.append(sent_path, sent_raw, frozenset({"\\Seen"}))
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("outbox %s sent; filing it in Sent failed, will retry: %s", oid, exc)
+                sha = blobstore.store(user_id, sent_raw)
+                await gc.hold(user_id, sha)  # until filed: only the job refers to it
+                await enqueue_job("file_sent_copy_job", str(account_obj.id), str(user_id), sha)
         # Remove the mirrored Drafts-folder copy now that it has been sent.
         if drafts_path and draft_imap_uid:
-            with contextlib.suppress(Exception):
+            try:
                 await provider.delete_messages(drafts_path, [draft_imap_uid])
-    except Exception as exc:  # noqa: BLE001 - record and notify, keep the draft
-        async with session_scope() as session:
-            ob = await session.get(Outbox, oid)
-            if ob is not None:
-                ob.state = "failed"
-                ob.error = str(exc)[:500]
-        await get_event_bus().publish(
-            "outbox.state_changed",
-            {"user_id": str(user_id), "outbox_id": str(oid), "state": "failed"},
-        )
-        logger.exception("send failed for outbox %s", oid)
-        return
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("outbox %s sent; removing its Drafts copy failed: %s", oid, exc)
     finally:
         await provider.close()
 
@@ -210,6 +338,12 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
             # need to read the outbox row. payload to/cc are "Name <email>" strings.
             "to": _addr_dicts(draft.payload.get("to")),
             "cc": _addr_dicts(draft.payload.get("cc")),
+            # What it answers: "reply" / "reply_all" / "forward" / "new", and the
+            # message (a reply says the user wanted that mail — antispam learns it).
+            "kind": draft.kind,
+            "reply_to_message_id": (
+                str(draft.reply_to_message_id) if draft.reply_to_message_id else None
+            ),
         },
     )
     await bus.publish(

backend/app/outbox/mime_build.py

@@ -14,6 +14,7 @@ from email.utils import format_datetime, make_msgid
 import nh3
 
 from app.models import Account, Draft, Message
+from app.outbox.inline_images import InlinePart
 
 
 def _html_to_text(html: str) -> str:
@@ -49,6 +50,9 @@ def build_mime(
     draft: Draft,
     reply_to: Message | None,
     attachments: list[tuple[str, str, bytes]] | None = None,
+    *,
+    html: str | None = None,
+    inline: list[InlinePart] | None = None,
 ) -> tuple[bytes, bytes, str, list[str]]:
     """Return ``(transmit_raw, sent_raw, envelope_from, envelope_to)`` for ``draft``.
 
@@ -59,13 +63,17 @@ def build_mime(
 
     ``attachments`` is a list of ``(filename, mime_type, data)`` uploaded via the
     compose Attach action; each is added as a file attachment part.
+
+    ``html`` replaces the draft's HTML when given (images already turned into
+    ``cid:`` references by app/outbox/inline_images.py), and ``inline`` are the
+    image parts those references name, sent alongside the HTML.
     """
     payload = draft.payload
     to = _addr_list(payload.get("to"))
     cc = _addr_list(payload.get("cc"))
     bcc = _addr_list(payload.get("bcc"))
     subject = str(payload.get("subject", ""))
-    html = str(payload.get("html", "") or "")
+    html = html if html is not None else str(payload.get("html", "") or "")
     text = str(payload.get("text", "") or "")
 
     msg = EmailMessage()
@@ -91,6 +99,19 @@ def build_mime(
     if html:
         msg.set_content(text or _html_to_text(html))
         msg.add_alternative(html, subtype="html")
+        if inline:
+            html_part = next(p for p in msg.walk() if p.get_content_type() == "text/html")
+            assert isinstance(html_part, EmailMessage)
+            for part in inline:
+                maintype, _, subtype = part.mime_type.partition("/")
+                html_part.add_related(
+                    part.data,
+                    maintype=maintype or "image",
+                    subtype=subtype or "png",
+                    cid=f"<{part.cid}>",
+                    filename=part.filename,
+                    disposition="inline",
+                )
     else:
         msg.set_content(text)
 

backend/app/outbox/schemas.py

@@ -56,8 +56,28 @@ class SendRequest(BaseModel):
 class SendResponse(BaseModel):
     outbox_id: uuid.UUID
     undo_until: str
+    # The same deadline as seconds from now: the browser counts down from this,
+    # since its clock may be seconds off the server's.
+    undo_seconds: float
 
 
 class OutboxStateOut(BaseModel):
     outbox_id: uuid.UUID
     state: str
+
+
+class OutboxItemOut(BaseModel):
+    """A send waiting in the Scheduled folder: being retried, or failed for good."""
+
+    outbox_id: uuid.UUID
+    draft_id: uuid.UUID
+    account_id: uuid.UUID
+    state: str  # "retrying" | "failed"
+    attempts: int
+    error: str | None
+    next_attempt_at: str | None
+    retry_until: str | None
+    sent_at: str  # when Send was pressed (the end of the undo window)
+    to: list[str]
+    subject: str
+    preview: str

backend/app/outbox/service.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import uuid
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import select
+from sqlalchemy import ColumnElement, exists, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import get_settings
@@ -78,6 +78,16 @@ async def upsert_draft(
     return draft
 
 
+# A draft with one of these sends is in flight — or waiting in Scheduled for a
+# retry — so it is not listed (or counted) in Drafts.
+OUTBOX_ACTIVE = ("undo_window", "sending", "retrying", "failed")
+
+
+def not_being_sent() -> ColumnElement[bool]:
+    """SQL condition: this Draft row has no send in flight or waiting."""
+    return ~exists().where(Outbox.draft_id == Draft.id, Outbox.state.in_(OUTBOX_ACTIVE))
+
+
 async def create_outbox(session: AsyncSession, *, user_id: uuid.UUID, draft: Draft) -> Outbox:
     """Create an outbox row in the undo window for a draft."""
     send_after = datetime.now(UTC) + timedelta(seconds=get_settings().undo_seconds)

backend/app/plugins/actions.py

@@ -23,8 +23,8 @@ from sqlalchemy import select
 
 from app.db import session_scope
 from app.events import get_event_bus
-from app.indexer import blobstore
-from app.models import Account, Mailbox, Message, Thread
+from app.indexer import rawstore
+from app.models import Account, Mailbox, Message, Thread, User
 
 
 class PluginMessages:
@@ -39,9 +39,8 @@ class PluginMessages:
             )
             if msg is None or not msg.blob_sha256:
                 return None
-            sha = msg.blob_sha256
         try:
-            raw = blobstore.load(user_id, sha)
+            raw = await rawstore.ensure_bytes(msg)
         except FileNotFoundError:
             return None
         return message_from_bytes(raw).get(name)
@@ -60,9 +59,8 @@ class PluginMessages:
             )
             if msg is None or not msg.blob_sha256:
                 return None
-            sha = msg.blob_sha256
         try:
-            return blobstore.load(user_id, sha)
+            return await rawstore.ensure_bytes(msg)
         except FileNotFoundError:
             return None
 
@@ -159,17 +157,16 @@ class PluginMessages:
                     for a in msg.to_addr
                 ],
                 "flags": list(msg.flags),
+                # Whether files came with it — e.g. an invitation's .ics.
+                "has_attachments": bool(msg.has_attachments),
             }
 
 
 class PluginAccounts:
-    """Cross-user account queries, for scheduled jobs.
-
-    Deliberately not user-scoped: a sweep runs on behalf of the instance, not a
-    request, and needs to find work across every user. Nothing here returns
-    content — only ids and state — so a plugin still has to go through the
-    user-scoped accessors to read anything.
-    """
+    """Account queries: ``live()`` is cross-user, for scheduled jobs (a sweep runs
+    on behalf of the instance and returns only ids and state); ``of_user`` and
+    ``settings_of`` read one user's own accounts and settings, for a handler
+    already acting as that user."""
 
     async def live(self) -> list[tuple[uuid.UUID, uuid.UUID]]:
         """``(user_id, account_id)`` for every account finished with its initial
@@ -181,6 +178,23 @@ class PluginAccounts:
             )
             return [(u, a) for u, a in rows]
 
+    async def of_user(self, user_id: uuid.UUID) -> list[tuple[uuid.UUID, str, str]]:
+        """``(account_id, email, display_name)`` of the user's own mail accounts —
+        e.g. to tell which attendee of an emailed invitation is the user."""
+        async with session_scope() as session:
+            rows = await session.execute(
+                select(Account.id, Account.email, Account.display_name)
+                .where(Account.user_id == user_id)
+                .order_by(Account.created_at)
+            )
+            return [(a, str(e), str(n or "")) for a, e, n in rows]
+
+    async def settings_of(self, user_id: uuid.UUID) -> dict[str, object]:
+        """The user's settings (time zone, default calendar, …), read-only."""
+        async with session_scope() as session:
+            settings = await session.scalar(select(User.settings).where(User.id == user_id))
+            return dict(settings or {})
+
 
 class PluginThreads:
     """Read-only, user-scoped thread queries (e.g. self-healing stale references)."""

backend/app/plugins/http.py

@@ -21,9 +21,7 @@ class PluginHttp:
         content: bytes | str | None = None,
         auth: tuple[str, str] | None = None,
     ) -> GuardedResponse:
-        return await guarded_request(
-            method, url, headers=headers, content=content, auth=auth
-        )
+        return await guarded_request(method, url, headers=headers, content=content, auth=auth)
 
     async def get(self, url: str, **kw: object) -> GuardedResponse:
         return await self.request("GET", url, **kw)  # type: ignore[arg-type]

backend/app/plugins/sdk.py

@@ -179,6 +179,12 @@ class PluginSdk:
         CalDAV/CardDAV/ICS. Returns status + headers + body."""
         return PluginHttp()
 
+    # --- notifications ------------------------------------------------------
+    @property
+    def push(self) -> PluginPush:
+        """Notifications on the user's devices (Web Push) — calendar reminders."""
+        return PluginPush(self.name)
+
     # --- secret vault -------------------------------------------------------
     @property
     def vault(self) -> PluginVault:
@@ -187,6 +193,39 @@ class PluginSdk:
         return PluginVault()
 
 
+class PluginPush:
+    """``sdk.push``: show a notification on every device where the user turned
+    notifications on (Settings → Accounts → Notifications)."""
+
+    def __init__(self, plugin: str) -> None:
+        self._plugin = plugin
+
+    async def notify(
+        self,
+        user_id: uuid.UUID,
+        *,
+        title: str,
+        body: str = "",
+        url: str = "/mail",
+        tag: str | None = None,
+        sticky: bool = False,
+    ) -> int:
+        """Send one notification; returns how many devices it went to. ``tag``
+        replaces an earlier notification with the same tag (it is prefixed with
+        the plugin's name); ``sticky`` keeps it on screen until dismissed."""
+        from app.push.service import deliver
+
+        payload: dict[str, object] = {
+            "type": f"plugin.{self._plugin}",
+            "title": title[:200],
+            "body": body[:500],
+            "url": url if url.startswith("/") else "/mail",
+            "tag": f"{self._plugin}-{tag}" if tag else self._plugin,
+            "sticky": sticky,
+        }
+        return await deliver(user_id, [payload])
+
+
 def sdk(name: str) -> PluginSdk:
     """Return the SDK facade bound to plugin ``name`` (SPEC §11)."""
     return PluginSdk(name)

backend/app/plugins/types.py

@@ -112,6 +112,9 @@ class Plugin:
     # Core plugins (calendar, contacts) ship builtin=True: always installed, never
     # offered for removal in the UI, and the uninstall path refuses --drop-data.
     builtin: bool = False
+    # Serves its collections to CalDAV / CardDAV clients (app/dav/provider.py
+    # DavProvider; SPEC Appendix E).
+    dav: Any = None
 
     def __post_init__(self) -> None:
         if not _NAME_RE.fullmatch(self.name):

backend/app/providers/base.py

@@ -29,6 +29,11 @@ class ProviderConnectError(ProviderError):
     """Could not connect / TLS / greeting failure."""
 
 
+class ProviderRejectedError(ProviderError):
+    """The server refused for good (a permanent 5xx: recipient unknown, message
+    rejected). Trying again would only be refused again."""
+
+
 class ProviderNotSupported(ProviderError):
     """A capability (e.g. IDLE) is not supported by the server."""
 

backend/app/providers/imap_smtp/provider.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import re
+import ssl
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
 from datetime import date
@@ -19,14 +20,16 @@ import aiosmtplib
 from aioimaplib import aioimaplib
 
 from app.concurrency import loop_semaphore
-from app.config import get_settings
+from app.config import get_settings, smtp_helo_name
 from app.providers.base import (
     FlagChange,
     MailboxDelta,
     MailProvider,
     ProviderAuthError,
     ProviderConnectError,
+    ProviderError,
     ProviderNotSupported,
+    ProviderRejectedError,
     RemoteMailbox,
     RemoteMessage,
 )
@@ -43,7 +46,10 @@ _FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
 # Gmail (X-GM-EXT-1): also pull each message's labels so custom labels can map to
 # chips without visiting every label folder (SPEC: Gmail sync / no-duplication).
 _FETCH_ITEMS_GM = "(UID FLAGS INTERNALDATE RFC822.SIZE X-GM-LABELS BODY.PEEK[])"
-_IDLE_TIMEOUT = 25 * 60  # SPEC §5.1: re-issue IDLE at least every 25 min
+# SPEC §5.1 asks for IDLE to be re-issued at least every 25 min; 10 also means a
+# connection that died silently (no FIN, e.g. the network dropped) is noticed
+# within ~11 min — the re-issue finds it — instead of ~29.
+_IDLE_TIMEOUT = 10 * 60
 _CONNECT_TIMEOUT = 20
 _PER_ACCOUNT_CONN_CAP = 4  # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
 _APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
@@ -51,6 +57,57 @@ _COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")
 _IMAP_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
 
 
+class _KeepConnectTask:
+    """aioimaplib starts the TCP connection as a task it never awaits: a refused
+    or unreachable server then surfaces only as a 20 s greeting timeout, plus a
+    "Task exception was never retrieved" traceback in the log. Keep the task, so
+    ``_connect`` can await it — failing at once, with the real reason."""
+
+    connect_task: asyncio.Task[object]
+
+    def create_client(  # type: ignore[no-untyped-def]
+        self, host, port, loop, conn_lost_cb=None, ssl_context=None
+    ) -> None:
+        # This replaces IMAP4_SSL.create_client too (it comes first in the MRO),
+        # so do its one job here: without a context the connection is plain
+        # TCP to port 993, and the server just waits for a TLS handshake.
+        if ssl_context is None and isinstance(self, aioimaplib.IMAP4_SSL):
+            ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+        local_loop = loop if loop is not None else asyncio.get_running_loop()
+        self.protocol = aioimaplib.IMAP4ClientProtocol(local_loop, conn_lost_cb)
+        self.connect_task = local_loop.create_task(
+            local_loop.create_connection(lambda: self.protocol, host, port, ssl=ssl_context)
+        )
+
+
+class _IMAP4(_KeepConnectTask, aioimaplib.IMAP4):  # type: ignore[misc]
+    pass
+
+
+class _IMAP4_SSL(_KeepConnectTask, aioimaplib.IMAP4_SSL):  # type: ignore[misc]  # noqa: N801
+    pass
+
+
+def _smtp_error(exc: aiosmtplib.SMTPException) -> ProviderError:
+    """An SMTP failure as the provider error that says whether to try again.
+
+    A 5xx reply is the server's final word (recipient unknown, message refused):
+    ``ProviderRejectedError``, or ``ProviderAuthError`` for a refused login. Any
+    4xx reply, a timeout or a dropped connection is temporary:
+    ``ProviderConnectError``, and the send is retried later.
+    """
+    if isinstance(exc, aiosmtplib.SMTPRecipientsRefused):
+        codes = [r.code for r in exc.recipients]
+        if codes and all(c >= 500 for c in codes):
+            return ProviderRejectedError(f"recipients refused: {exc}")
+        return ProviderConnectError(f"SMTP send failed: {exc}")
+    if isinstance(exc, aiosmtplib.SMTPResponseException) and exc.code >= 500:
+        if isinstance(exc, aiosmtplib.SMTPAuthenticationError):
+            return ProviderAuthError(f"SMTP auth failed: {exc}")
+        return ProviderRejectedError(f"SMTP refused: {exc}")
+    return ProviderConnectError(f"SMTP send failed: {exc}")
+
+
 def _imap_date(d: date) -> str:
     """RFC 3501 search date, e.g. ``5-Jul-2026`` (locale-independent)."""
     return f"{d.day}-{_IMAP_MONTHS[d.month - 1]}-{d.year}"
@@ -65,6 +122,16 @@ def _date_criteria(since: date | None, before: date | None) -> str:
     return " ".join(parts) if parts else "ALL"
 
 
+async def _require_public(host: str) -> None:
+    """Refuse a user-named server that now resolves to a non-public address."""
+    from app.accounts.ssrf import SsrfError, assert_host_is_public
+
+    try:
+        await assert_host_is_public(host)
+    except SsrfError as exc:
+        raise ProviderConnectError(f"refusing to connect to {host}: {exc}") from exc
+
+
 @dataclass(frozen=True)
 class ProviderConfig:
     imap_host: str
@@ -75,6 +142,16 @@ class ProviderConfig:
     smtp_tls: str  # "ssl" | "starttls" | "plain"
     username: str
     password: str  # plaintext; caller decrypts per §10.2
+    # TLS settings other than the default (certificate checked): the hosted
+    # mail server on the internal network (app/hosted/native.py).
+    tls_context: ssl.SSLContext | None = None
+    # The hosted mail server: every user's mailbox is on that one host, so the
+    # per-host connection cap (which spares outside providers) does not apply.
+    internal: bool = False
+    # A server the user named (every external account): its name is checked
+    # again before each connection, so DNS changed after the account was added
+    # cannot turn the worker on the internal network (DNS rebinding).
+    public_only: bool = False
 
 
 class ImapSmtpProvider(MailProvider):
@@ -112,8 +189,11 @@ class ImapSmtpProvider(MailProvider):
         account = loop_semaphore(
             "imap_account", f"{cfg.username}@{cfg.imap_host}", _PER_ACCOUNT_CONN_CAP
         )
-        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
         await account.acquire()  # consistent order (account, then host) avoids deadlock
+        if cfg.internal:
+            self._conn_slots = [account]
+            return
+        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
         try:
             await host.acquire()
         except BaseException:
@@ -130,22 +210,30 @@ class ImapSmtpProvider(MailProvider):
         if self._imap is not None:
             return self._imap
         cfg = self._cfg
+        if cfg.public_only:
+            await _require_public(cfg.imap_host)
         await self._acquire_conn_slots()
         try:
             try:
+                imap: aioimaplib.IMAP4
                 if cfg.imap_tls == "ssl":
-                    imap = aioimaplib.IMAP4_SSL(
-                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
+                    imap = _IMAP4_SSL(
+                        host=cfg.imap_host,
+                        port=cfg.imap_port,
+                        timeout=_CONNECT_TIMEOUT,
+                        ssl_context=cfg.tls_context,
                     )
                 else:
-                    imap = aioimaplib.IMAP4(
-                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
-                    )
+                    imap = _IMAP4(host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT)
+                # Refused / unknown host: fails here at once. No answer at all:
+                # gives up after the timeout (the connect attempt is cancelled).
+                await asyncio.wait_for(imap.connect_task, _CONNECT_TIMEOUT)
                 await imap.wait_hello_from_server()
                 if cfg.imap_tls == "starttls":
                     await imap.starttls()
             except (OSError, TimeoutError) as exc:
-                raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc
+                why = str(exc) or f"no answer within {_CONNECT_TIMEOUT} s"
+                raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {why}") from exc
 
             resp = await imap.login(cfg.username, cfg.password)
             if resp.result != "OK":
@@ -296,6 +384,15 @@ class ImapSmtpProvider(MailProvider):
         resp = await imap.uid_search("ALL")
         return set(parse_search_uids(resp.lines))
 
+    async def fetch_raw(self, mailbox: str, uid: int) -> bytes | None:
+        """One message's raw bytes, or None if it is not (or no longer) there."""
+        await self._select(mailbox)
+        imap = await self._connect()
+        for message in await self._fetch_messages(imap, [uid]):
+            if message.uid == uid:
+                return message.raw
+        return None
+
     async def _fetch_messages(self, imap: aioimaplib.IMAP4, uids: list[int]) -> list[RemoteMessage]:
         if not uids:
             return []
@@ -391,7 +488,10 @@ class ImapSmtpProvider(MailProvider):
         await self._select(mailbox)
         idle_task = await imap.idle_start(timeout=_IDLE_TIMEOUT)
         try:
-            await imap.wait_server_push()
+            # No word from the server even after IDLE's own end: just return, and
+            # the caller's poll shows whether the connection is still there.
+            with contextlib.suppress(TimeoutError):
+                await imap.wait_server_push(timeout=_IDLE_TIMEOUT + 60)
         finally:
             imap.idle_done()
             with contextlib.suppress(Exception):
@@ -411,12 +511,12 @@ class ImapSmtpProvider(MailProvider):
         """Create a mailbox on the server (idempotent — ignores 'already exists')."""
         imap = await self._connect()
         with contextlib.suppress(Exception):
-            await imap.create(path)
+            await imap.create(self._wire(path))
 
     async def move(self, uid: int, src: str, dst: str) -> int:
         await self._select(src)
         imap = await self._connect()
-        resp = await imap.uid("move", str(uid), dst)
+        resp = await imap.uid("move", str(uid), self._wire(dst))
         if resp.result != "OK":
             raise ProviderConnectError(f"MOVE {uid} -> {dst} failed")
         for line in resp.lines:
@@ -456,9 +556,15 @@ class ImapSmtpProvider(MailProvider):
 
     async def send(self, raw: bytes, envelope_from: str, envelope_to: list[str]) -> None:
         cfg = self._cfg
+        if cfg.public_only:
+            await _require_public(cfg.smtp_host)
         use_tls = cfg.smtp_tls == "ssl"
         start_tls = cfg.smtp_tls == "starttls"
-        host_slot = loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
+        host_slot = (
+            contextlib.nullcontext()
+            if cfg.internal
+            else loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
+        )
         try:
             async with host_slot:  # bound simultaneous sockets to the SMTP host (§4)
                 await aiosmtplib.send(
@@ -471,9 +577,10 @@ class ImapSmtpProvider(MailProvider):
                     password=cfg.password,
                     use_tls=use_tls,
                     start_tls=start_tls,
+                    tls_context=cfg.tls_context,
                     timeout=_CONNECT_TIMEOUT,
+                    # Our own name, not the container's random host name.
+                    local_hostname=smtp_helo_name(),
                 )
-        except aiosmtplib.SMTPAuthenticationError as exc:
-            raise ProviderAuthError(f"SMTP auth failed: {exc}") from exc
         except aiosmtplib.SMTPException as exc:
-            raise ProviderConnectError(f"SMTP send failed: {exc}") from exc
+            raise _smtp_error(exc) from exc

backend/app/push/__init__.py

@@ -0,0 +1 @@
+"""Push notifications about new mail (Web Push)."""

backend/app/push/service.py

@@ -0,0 +1,295 @@
+"""Push notifications about new mail: the server's keys, the devices, and the job.
+
+New Inbox mail found by the live sync enqueues :func:`push_new_mail_job` a few
+seconds later, so filters and the spam check have moved what they move by then;
+the job looks again at each message and tells the user's devices only about
+mail that is still unread in an Inbox, recent, and not from the user.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Any
+
+import httpx
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.identity.sessions import session_alive
+from app.models import Account, AppSettings, Mailbox, Message, PushSubscription
+from app.push import webpush
+from app.security.crypto import decrypt_secret, encrypt_secret
+
+logger = logging.getLogger(__name__)
+
+# How long after new Inbox mail is found the job runs (filters, spam check).
+NOTIFY_DELAY_S = 5
+# Mail older than this is not news: a message moved back into the Inbox, or a
+# backlog after the server was unreachable for a while.
+MAX_AGE = dt.timedelta(hours=2)
+# More new messages than this at once become a single "N new messages".
+MAX_SINGLE = 3
+SNIPPET_CHARS = 140
+
+
+# ---- the server's key pair ------------------------------------------------------------
+
+
+async def vapid(db: AsyncSession) -> webpush.Vapid:
+    """The server's VAPID key pair, made (once) on first use."""
+    row = await db.get(AppSettings, 1)
+    if row is not None and row.vapid_private_ct:
+        return webpush.Vapid.from_pem(decrypt_secret(row.vapid_private_ct))
+    row = await db.scalar(select(AppSettings).where(AppSettings.id == 1).with_for_update())
+    if row is None:
+        row = AppSettings(id=1)
+        db.add(row)
+    elif row.vapid_private_ct:  # made by another request meanwhile
+        return webpush.Vapid.from_pem(decrypt_secret(row.vapid_private_ct))
+    key = webpush.Vapid.generate()
+    row.vapid_private_ct = encrypt_secret(key.pem())
+    row.vapid_public = key.public_b64
+    await db.commit()
+    return key
+
+
+def _subject(row: AppSettings | None) -> str:
+    """Who to contact about our pushes (the VAPID ``sub``): the app's address
+    when it is set and https, else the postmaster of the app's domain."""
+    url = row.public_url if row is not None else ""
+    if url.startswith("https://"):
+        return url
+    return f"mailto:postmaster@{get_settings().app_domain}"
+
+
+# ---- devices ------------------------------------------------------------------------
+
+
+async def subscribe(
+    db: AsyncSession,
+    *,
+    user_id: uuid.UUID,
+    session_sid: str,
+    endpoint: str,
+    p256dh: str,
+    auth: str,
+    user_agent: str,
+) -> None:
+    """Remember a device (or refresh it). One row per endpoint: a browser that
+    is now signed in as someone else is theirs from now on."""
+    row = await db.scalar(select(PushSubscription).where(PushSubscription.endpoint == endpoint))
+    if row is None:
+        row = PushSubscription(endpoint=endpoint)
+        db.add(row)
+    row.user_id = user_id
+    row.session_sid = session_sid
+    row.p256dh = p256dh
+    row.auth = auth
+    row.user_agent = user_agent[:256]
+    await db.commit()
+
+
+async def unsubscribe(db: AsyncSession, *, user_id: uuid.UUID, endpoint: str) -> None:
+    await db.execute(
+        delete(PushSubscription).where(
+            PushSubscription.user_id == user_id, PushSubscription.endpoint == endpoint
+        )
+    )
+    await db.commit()
+
+
+# ---- what to say --------------------------------------------------------------------
+
+
+def _sender(msg: Message) -> str:
+    first = (msg.from_addr or [{}])[0]
+    return (first.get("name") or first.get("email") or "Unknown sender").strip()
+
+
+def _payloads(messages: list[Message], unread: int) -> list[dict[str, object]]:
+    """One notification per message, or one summary when there are many."""
+    if len(messages) > MAX_SINGLE:
+        senders: list[str] = []
+        for m in messages:
+            name = _sender(m)
+            if name not in senders:
+                senders.append(name)
+        shown = ", ".join(senders[:3]) + ("…" if len(senders) > 3 else "")
+        return [
+            {
+                "type": "mail",
+                "title": f"{len(messages)} new messages",
+                "body": shown,
+                "tag": "new-mail",
+                "url": "/mail",
+                "unread": unread,
+            }
+        ]
+    out: list[dict[str, object]] = []
+    for m in messages:
+        subject = m.subject.strip() or "(no subject)"
+        snippet = " ".join(m.snippet.split())[:SNIPPET_CHARS]
+        out.append(
+            {
+                "type": "mail",
+                "title": _sender(m),
+                "body": f"{subject}\n{snippet}" if snippet else subject,
+                "tag": f"thread-{m.thread_id}",
+                "url": f"/mail/t/{m.thread_id}" if m.thread_id else "/mail",
+                "thread_id": str(m.thread_id) if m.thread_id else None,
+                "unread": unread,
+            }
+        )
+    return out
+
+
+async def _news(db: AsyncSession, user_id: uuid.UUID, ids: list[uuid.UUID]) -> list[Message]:
+    """The messages still worth a notification, oldest first."""
+    own = {
+        e.lower() for e in await db.scalars(select(Account.email).where(Account.user_id == user_id))
+    }
+    rows = await db.scalars(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.user_id == user_id,
+            Message.id.in_(ids),
+            Mailbox.special_use == "inbox",
+            ~Message.flags.contains(["\\Seen"]),
+            Message.internal_date >= dt.datetime.now(dt.UTC) - MAX_AGE,
+        )
+        .order_by(Message.internal_date)
+    )
+    out: list[Message] = []
+    seen: set[str] = set()
+    for m in rows:
+        sender = ((m.from_addr or [{}])[0].get("email") or "").lower()
+        if sender in own:
+            continue  # sent to yourself, or a copy of your own message
+        key = m.delivery_id or str(m.id)  # one email that reached two accounts
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(m)
+    return out
+
+
+async def _unread(db: AsyncSession, user_id: uuid.UUID) -> int:
+    """Unread Inbox messages, for the app icon's badge."""
+    count = await db.scalar(
+        select(func.count())
+        .select_from(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(
+            Message.user_id == user_id,
+            Mailbox.special_use == "inbox",
+            ~Message.flags.contains(["\\Seen"]),
+        )
+    )
+    return int(count or 0)
+
+
+# ---- the job ------------------------------------------------------------------------
+
+
+async def notify(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> int:
+    """Tell the user's signed-in devices about ``message_ids``; returns how many
+    pushes were delivered to push services."""
+    from app.db import session_scope
+
+    async with session_scope() as db:
+        if not await db.scalar(
+            select(PushSubscription.id).where(PushSubscription.user_id == user_id).limit(1)
+        ):
+            return 0
+        messages = await _news(db, user_id, message_ids)
+        if not messages:
+            return 0
+        payloads = _payloads(messages, await _unread(db, user_id))
+    return await deliver(user_id, payloads)
+
+
+async def deliver(user_id: uuid.UUID, payloads: list[dict[str, object]]) -> int:
+    """Push ``payloads`` to every device of the user that is still signed in
+    (for new mail, and for plugins: calendar reminders); returns how many were
+    handed to push services. Devices that are gone are forgotten."""
+    from app.db import session_scope
+
+    if not payloads:
+        return 0
+    async with session_scope() as db:
+        # Plain values: the rows expire when the key pair is first saved below.
+        subs = [
+            (row.id, row.endpoint, row.p256dh, row.auth, row.session_sid)
+            for row in await db.scalars(
+                select(PushSubscription).where(PushSubscription.user_id == user_id)
+            )
+        ]
+        if not subs:
+            return 0
+        key = await vapid(db)
+        subject = _subject(await db.get(AppSettings, 1))
+
+    sent = 0
+    gone: list[uuid.UUID] = []
+    used: list[uuid.UUID] = []
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        for sub_id, endpoint, p256dh, auth, sid in subs:
+            if not await session_alive(user_id, sid):
+                gone.append(sub_id)  # signed out: this device hears nothing more
+                continue
+            try:
+                for payload in payloads:
+                    await webpush.send(
+                        client,
+                        endpoint=endpoint,
+                        p256dh=p256dh,
+                        auth=auth,
+                        payload=payload,
+                        vapid=key,
+                        subject=subject,
+                    )
+                    sent += 1
+                used.append(sub_id)
+            except webpush.Gone:
+                gone.append(sub_id)
+            except webpush.Failed as err:
+                logger.warning("push to %s failed: %s", endpoint.split("/")[2], err)
+
+    if gone or used:
+        async with session_scope() as db:
+            if gone:
+                await db.execute(delete(PushSubscription).where(PushSubscription.id.in_(gone)))
+            if used:
+                await db.execute(
+                    update(PushSubscription)
+                    .where(PushSubscription.id.in_(used))
+                    .values(last_used_at=func.now())
+                )
+    return sent
+
+
+async def push_new_mail_job(ctx: dict[str, Any], user_id: str, message_ids: list[str]) -> int:
+    """arq job: notify about new Inbox mail (enqueued by the live sync)."""
+    return await notify(uuid.UUID(user_id), [uuid.UUID(m) for m in message_ids])
+
+
+async def enqueue_new_mail(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> None:
+    """Called by the live sync with new Inbox mail. Never raises: a missing
+    notification must not stop the sync."""
+    if not message_ids:
+        return
+    from app.queue import enqueue_job
+
+    try:
+        await enqueue_job(
+            "push_new_mail_job",
+            str(user_id),
+            [str(m) for m in message_ids],
+            _defer_by=NOTIFY_DELAY_S,
+        )
+    except Exception:  # noqa: BLE001 - best effort
+        logger.warning("could not enqueue new-mail push for user %s", user_id, exc_info=True)

backend/app/push/webpush.py

@@ -0,0 +1,193 @@
+"""Web Push, the protocol (RFC 8030 delivery, RFC 8291 encryption, RFC 8292 VAPID).
+
+A browser that wants notifications gives us a push service URL (Google's,
+Mozilla's, Apple's, Microsoft's) and two keys. A message is encrypted to those
+keys, so the push service only carries it, and signed with our own key pair
+(VAPID), so the push service knows it comes from the server the browser
+subscribed to. Written here rather than taken from pywebpush, which needs a
+newer ``cryptography`` than the rest of the backend pins; it is small, and
+``tests/push`` checks it against the RFC 8291 worked example.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import time
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+import httpx
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+RECORD_SIZE = 4096
+# Push services only: the endpoint comes from the browser, i.e. from the user,
+# and the server posts to it — anything else would let a user aim the server at
+# an internal address (SSRF). Every browser in use delivers through one of these.
+PUSH_HOSTS = (
+    "fcm.googleapis.com",  # Chrome, Edge on Android, Opera, Samsung Internet
+    "android.googleapis.com",
+    ".google.com",  # Chromium builds subscribe at jmt17.google.com and the like
+    ".push.services.mozilla.com",  # Firefox
+    ".push.apple.com",  # Safari, and installed apps on iPhone/iPad
+    ".notify.windows.com",  # Edge on Windows
+)
+
+
+def b64u_decode(value: str) -> bytes:
+    return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
+
+
+def b64u_encode(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
+
+
+def allowed_endpoint(endpoint: str) -> bool:
+    """Whether ``endpoint`` is an https URL of a known push service."""
+    parts = urlsplit(endpoint)
+    host = (parts.hostname or "").lower()
+    if parts.scheme != "https" or not host or parts.port not in (None, 443):
+        return False
+    return any(host == h or (h.startswith(".") and host.endswith(h)) for h in PUSH_HOSTS)
+
+
+def _public_bytes(key: ec.EllipticCurvePrivateKey) -> bytes:
+    return key.public_key().public_bytes(
+        serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint
+    )
+
+
+def _hkdf(salt: bytes, ikm: bytes, info: bytes, length: int) -> bytes:
+    return HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
+
+
+def encrypt(
+    payload: bytes,
+    p256dh: str,
+    auth: str,
+    *,
+    salt: bytes | None = None,
+    sender: ec.EllipticCurvePrivateKey | None = None,
+) -> bytes:
+    """``payload`` encrypted for one browser (RFC 8291, ``aes128gcm``), as the
+    request body: header (salt, record size, our one-off public key) + one
+    record. ``salt`` and ``sender`` are for the RFC test vector only."""
+    ua_public = b64u_decode(p256dh)
+    auth_secret = b64u_decode(auth)
+    sender = sender or ec.generate_private_key(ec.SECP256R1())
+    as_public = _public_bytes(sender)
+    ua_key = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), ua_public)
+    ecdh_secret = sender.exchange(ec.ECDH(), ua_key)
+    ikm = _hkdf(auth_secret, ecdh_secret, b"WebPush: info\x00" + ua_public + as_public, 32)
+    salt = salt or os.urandom(16)
+    cek = _hkdf(salt, ikm, b"Content-Encoding: aes128gcm\x00", 16)
+    nonce = _hkdf(salt, ikm, b"Content-Encoding: nonce\x00", 12)
+    # One record, no padding: 0x02 marks the last (only) record.
+    record = AESGCM(cek).encrypt(nonce, payload + b"\x02", None)
+    header = salt + RECORD_SIZE.to_bytes(4, "big") + bytes([len(as_public)]) + as_public
+    return header + record
+
+
+# ---- VAPID (RFC 8292) ---------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class Vapid:
+    """The server's signing key pair."""
+
+    private: ec.EllipticCurvePrivateKey
+
+    @classmethod
+    def generate(cls) -> Vapid:
+        return cls(ec.generate_private_key(ec.SECP256R1()))
+
+    @classmethod
+    def from_pem(cls, pem: bytes) -> Vapid:
+        key = serialization.load_pem_private_key(pem, password=None)
+        if not isinstance(key, ec.EllipticCurvePrivateKey):
+            raise ValueError("VAPID key is not an EC key")
+        return cls(key)
+
+    def pem(self) -> bytes:
+        return self.private.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.PKCS8,
+            serialization.NoEncryption(),
+        )
+
+    @property
+    def public_b64(self) -> str:
+        """The public key as browsers take it (``applicationServerKey``)."""
+        return b64u_encode(_public_bytes(self.private))
+
+    def authorization(self, endpoint: str, subject: str, *, ttl: int = 12 * 3600) -> str:
+        """The ``Authorization`` header for a push to ``endpoint``: a JWT (ES256)
+        naming the push service's origin, with our public key."""
+        parts = urlsplit(endpoint)
+        claims = {
+            "aud": f"{parts.scheme}://{parts.netloc}",
+            "exp": int(time.time()) + ttl,
+            "sub": subject,
+        }
+        header = {"typ": "JWT", "alg": "ES256"}
+        signing_input = (
+            b64u_encode(json.dumps(header, separators=(",", ":")).encode())
+            + "."
+            + b64u_encode(json.dumps(claims, separators=(",", ":")).encode())
+        )
+        der = self.private.sign(signing_input.encode(), ec.ECDSA(hashes.SHA256()))
+        r, s = decode_dss_signature(der)
+        signature = r.to_bytes(32, "big") + s.to_bytes(32, "big")
+        return f"vapid t={signing_input}.{b64u_encode(signature)}, k={self.public_b64}"
+
+
+# ---- delivery (RFC 8030) ------------------------------------------------------------
+
+
+class Gone(Exception):
+    """The subscription no longer exists (404/410): forget it."""
+
+
+class Failed(Exception):
+    """The push service refused or could not be reached; worth trying later."""
+
+
+async def send(
+    client: httpx.AsyncClient,
+    *,
+    endpoint: str,
+    p256dh: str,
+    auth: str,
+    payload: dict[str, object],
+    vapid: Vapid,
+    subject: str,
+    ttl: int = 24 * 3600,
+    urgency: str = "normal",
+    topic: str | None = None,
+) -> None:
+    """Deliver one encrypted message; raise :class:`Gone` or :class:`Failed`."""
+    if not allowed_endpoint(endpoint):
+        raise Gone("not a push service")
+    body = encrypt(json.dumps(payload, separators=(",", ":")).encode(), p256dh, auth)
+    headers = {
+        "Authorization": vapid.authorization(endpoint, subject),
+        "Content-Encoding": "aes128gcm",
+        "Content-Type": "application/octet-stream",
+        "TTL": str(ttl),
+        "Urgency": urgency,
+    }
+    if topic:
+        headers["Topic"] = topic  # replaces an undelivered message with the same topic
+    try:
+        resp = await client.post(endpoint, content=body, headers=headers)
+    except httpx.HTTPError as err:
+        raise Failed(str(err)) from err
+    if resp.status_code in (404, 410):
+        raise Gone(f"HTTP {resp.status_code}")
+    if resp.status_code >= 400:
+        raise Failed(f"HTTP {resp.status_code}: {resp.text[:200]}")

backend/app/queue.py

@@ -21,7 +21,14 @@ QUEUE_INTERACTIVE = "interactive"
 QUEUE_BULK = "bulk"
 
 # Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
-_BULK_JOBS = frozenset({"sync_bootstrap_account"})
+_BULK_JOBS = frozenset(
+    {
+        "sync_bootstrap_account",
+        "dedupe_existing_job",
+        "backfill_list_headers_job",
+        "build_index_job",
+    }
+)
 
 
 def queue_for(job_name: str) -> str:

backend/app/redis_client.py

@@ -20,6 +20,7 @@ def aw(result: Any) -> Awaitable[Any]:
 
 
 _client: aioredis.Redis | None = None
+_bytes_client: aioredis.Redis | None = None
 
 
 def get_redis() -> aioredis.Redis:
@@ -34,9 +35,25 @@ def get_redis() -> aioredis.Redis:
     return _client
 
 
+def get_redis_bytes() -> aioredis.Redis:
+    """Return the process-wide Redis client for binary values.
+
+    ``get_redis()`` decodes every reply as UTF-8 text, which fails on (or
+    silently changes the type of) raw bytes such as a cached image. Anything that
+    stores bytes goes through this one instead.
+    """
+    global _bytes_client
+    if _bytes_client is None:
+        _bytes_client = aioredis.Redis.from_url(get_settings().redis_url, decode_responses=False)
+    return _bytes_client
+
+
 async def close_redis() -> None:
-    """Close the Redis client (used on app shutdown)."""
-    global _client
+    """Close the Redis clients (used on app shutdown)."""
+    global _client, _bytes_client
     if _client is not None:
         await _client.aclose()
         _client = None
+    if _bytes_client is not None:
+        await _bytes_client.aclose()
+        _bytes_client = None

backend/app/search/pg.py

@@ -13,6 +13,7 @@ import base64
 import datetime as dt
 import html
 import re
+from typing import Any
 from uuid import UUID
 
 from sqlalchemy import (
@@ -20,6 +21,7 @@ from sqlalchemy import (
     any_,
     case,
     cast,
+    distinct,
     exists,
     func,
     literal,
@@ -32,7 +34,7 @@ from sqlalchemy import (
 from sqlalchemy.sql.elements import ColumnElement
 
 from app.db import get_sessionmaker
-from app.models import Attachment, Mailbox, Message
+from app.models import Attachment, Mailbox, Message, Thread
 from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
 
 _TS_CONFIG = "simple"  # language-neutral; avoids wrong-language stemming (M0)
@@ -52,6 +54,8 @@ def _prefix_tsquery(value: str) -> ColumnElement[object]:
     lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text)
     prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g")
     return func.to_tsquery(_TS_CONFIG, prefixed)
+
+
 _RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
 
 
@@ -254,20 +258,9 @@ class PgSearchBackend(SearchBackend):
             .join(Mailbox, Mailbox.id == Message.mailbox_id)
             .where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
         )
-        for cond in conditions:
+        for cond in self._scope(groups, conditions):
             stmt = stmt.where(cond)
 
-        # Exclude Trash/Junk from search by default (Gmail-style), so deleting a
-        # message from search results removes it. An explicit `in:trash`/`in:junk`
-        # overrides this to search there on purpose.
-        if not _targets_removal_folder(groups):
-            stmt = stmt.where(
-                or_(
-                    Mailbox.special_use.is_(None),
-                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
-                )
-            )
-
         decoded = _decode_cursor(cursor)
         if decoded is not None:
             cur_hit, cur_date, cur_id = decoded
@@ -301,6 +294,41 @@ class PgSearchBackend(SearchBackend):
         )
         return hits, next_cursor
 
+    @staticmethod
+    def _scope(groups: list[list[dict[str, object]]], conditions: list[Any]) -> list[Any]:
+        """The query's conditions, plus leaving out Trash/Junk by default
+        (Gmail-style), so deleting a message from search results removes it. An
+        explicit `in:trash`/`in:junk` overrides this to search there on purpose."""
+        out = list(conditions)
+        if not _targets_removal_folder(groups):
+            out.append(
+                or_(
+                    Mailbox.special_use.is_(None),
+                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
+                )
+            )
+        return out
+
+    async def count_threads(self, q: SearchQuery, *, thread_filter: str | None = None) -> int:
+        """How many conversations match (``thread_filter``: "unread" / "pinned"
+        keeps only conversations with unread mail / pinned ones)."""
+        groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
+        assert isinstance(groups, list)
+        stmt = (
+            select(func.count(distinct(Message.thread_id)))
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
+        )
+        for cond in self._scope(groups, self._build_conditions(groups)):
+            stmt = stmt.where(cond)
+        if thread_filter in ("unread", "pinned"):
+            stmt = stmt.join(Thread, Thread.id == Message.thread_id).where(
+                Thread.unread_count > 0 if thread_filter == "unread" else Thread.pinned.is_(True)
+            )
+        async with get_sessionmaker()() as session:
+            return int(await session.scalar(stmt) or 0)
+
 
 _REMOVAL_FOLDERS = ("trash", "junk")
 

backend/app/security/cross_site.py

@@ -0,0 +1,91 @@
+"""Refuse state-changing API requests that another web site makes a browser send.
+
+The double-submit CSRF token (app/identity/deps.py ``require_csrf``) guards
+everything a signed-in user can change. The requests made before anyone is
+signed in — sign-in itself, sign-up, a password reset — have no token yet, so a
+hostile page could make a visitor's browser sign in to an account of the
+attacker's choosing (login CSRF): whatever the victim then writes, files or
+uploads lands in the attacker's account.
+
+Browsers say where a request comes from: ``Sec-Fetch-Site`` in every current
+browser, ``Origin`` on any cross-origin POST in older ones. A request another
+site started is refused here, before any route runs. Mail apps, calendar apps
+and servers send neither header and are not affected (DAV lives outside /api
+anyway).
+"""
+
+from __future__ import annotations
+
+import json
+from urllib.parse import urlsplit
+
+from starlette.datastructures import Headers
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+from app.config import get_settings
+
+_SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+_BODY = json.dumps(
+    {"error": {"code": "cross_site", "message": "request refused: sent from another site"}}
+).encode()
+
+
+def _hostname(value: str) -> str:
+    """``host[:port]`` -> ``host``, lower-cased (IPv6 brackets kept)."""
+    value = value.strip().lower()
+    if value.startswith("["):
+        return value.split("]", 1)[0] + "]"
+    return value.rsplit(":", 1)[0] if value.count(":") == 1 else value
+
+
+def is_cross_site(headers: Headers) -> bool:
+    """Whether a browser reports this request as started by another site."""
+    fetch_site = headers.get("sec-fetch-site")
+    if fetch_site is not None:
+        # "same-origin", "same-site" (our own subdomains), "none" (typed or
+        # bookmarked): all fine. Only another site is refused.
+        return fetch_site.strip().lower() == "cross-site"
+    origin = headers.get("origin")
+    if origin is None:
+        return False  # not a browser, or a same-origin request from an old one
+    if origin.strip().lower() == "null":
+        return True  # sandboxed frame, data: URL, file: page
+    origin_host = (urlsplit(origin.strip()).hostname or "").lower()
+    # Our own domain, whatever a reverse proxy did to the Host header.
+    domain = get_settings().app_domain.lower()
+    if origin_host == domain or origin_host.endswith("." + domain):
+        return False
+    # Otherwise the name the browser used: Host, or the one a reverse proxy saw.
+    ours = {_hostname(headers.get("host", ""))}
+    forwarded = headers.get("x-forwarded-host")
+    if forwarded:
+        ours.add(_hostname(forwarded.split(",")[0]))
+    return origin_host not in ours
+
+
+class CrossSiteGuard:
+    """ASGI middleware: 403 on cross-site POST/PUT/PATCH/DELETE under /api/."""
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if (
+            scope["type"] == "http"
+            and scope["method"] not in _SAFE_METHODS
+            and scope["path"].startswith("/api/")
+            and is_cross_site(Headers(scope=scope))
+        ):
+            await send(
+                {
+                    "type": "http.response.start",
+                    "status": 403,
+                    "headers": [
+                        (b"content-type", b"application/json"),
+                        (b"content-length", str(len(_BODY)).encode()),
+                    ],
+                }
+            )
+            await send({"type": "http.response.body", "body": _BODY})
+            return
+        await self.app(scope, receive, send)

backend/app/snooze.py

@@ -0,0 +1,92 @@
+"""Snooze: a conversation leaves the Inbox until a chosen time, then comes back.
+
+Kept in this database only (the mail server knows nothing of it): a snoozed
+thread is hidden from the Inbox view and listed under Snoozed. The wake job,
+run every minute, brings back threads whose time has come — or that got a new
+message meanwhile, as Gmail does — at the top of the Inbox, the newest message
+marked unread.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import or_, select
+
+from app.db import session_scope
+from app.events import get_event_bus
+from app.models import Mailbox, Message, Thread
+from app.queue import enqueue_job
+from app.sync.actions import apply_flags_to_messages
+
+logger = logging.getLogger(__name__)
+
+MAX_SNOOZE = dt.timedelta(days=366)
+_SEEN = frozenset({"\\Seen"})
+
+
+def snooze(thread: Thread, until: dt.datetime, now: dt.datetime) -> None:
+    thread.snoozed_until = until
+    thread.snoozed_at = now
+
+
+def wake(thread: Thread, now: dt.datetime) -> None:
+    thread.snoozed_until = None
+    thread.snoozed_at = None
+    thread.unsnoozed_at = now
+
+
+async def wake_due_job(ctx: dict[str, Any]) -> int:
+    """Cron (every minute): wake snoozed threads that are due. Returns how many."""
+    now = dt.datetime.now(dt.UTC)
+    woken: list[tuple[uuid.UUID, uuid.UUID]] = []
+    unread: list[tuple[uuid.UUID, Message]] = []
+    async with session_scope() as session:
+        due = list(
+            await session.scalars(
+                select(Thread)
+                .where(
+                    Thread.snoozed_until.isnot(None),
+                    or_(Thread.snoozed_until <= now, Thread.last_message_at > Thread.snoozed_at),
+                )
+                .with_for_update(skip_locked=True)
+            )
+        )
+        for thread in due:
+            on_time = thread.snoozed_until is not None and thread.snoozed_until <= now
+            wake(thread, now)
+            woken.append((thread.user_id, thread.id))
+            if not on_time:
+                continue  # a new message woke it: that message is unread already
+            # Back as new: the newest Inbox message becomes unread again.
+            newest = await session.scalar(
+                select(Message)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.thread_id == thread.id,
+                    Message.user_id == thread.user_id,
+                    Mailbox.special_use == "inbox",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(1)
+            )
+            if newest is not None and "\\Seen" in newest.flags:
+                await apply_flags_to_messages(session, [newest], frozenset(), _SEEN)
+                unread.append((thread.user_id, newest))
+        await session.commit()
+
+    bus = get_event_bus()
+    for user_id, message in unread:
+        await enqueue_job("push_flags_job", str(message.id), [], ["\\Seen"])
+        await bus.publish(
+            "flags.changed",
+            {"user_id": str(user_id), "message_id": str(message.id), "flags": list(message.flags)},
+        )
+    for user_id, thread_id in woken:
+        await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
+    if woken:
+        logger.info("woke %d snoozed thread(s)", len(woken))
+    return len(woken)

backend/app/subscriptions.py

@@ -0,0 +1,151 @@
+"""Subscriptions: the bulk senders mailing a user, and unsubscribing from them.
+
+No service records what a user signed up for, so the list is worked out from
+received mail, as Gmail does: a message is a subscription's when it carries a
+``List-Unsubscribe`` header (required of bulk senders by Gmail and Yahoo since
+2024). Messages are grouped by ``List-Id`` when there is one (a company can run
+several lists), otherwise by sender address.
+
+The headers are read when a message is stored (indexer.persist_message); older
+mail is read once by :func:`backfill_list_headers_job`. Unsubscribing uses the
+sender's one-click POST (RFC 8058) when offered, else mails the ``mailto:``
+address, else hands the web page back for the user to open.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from dataclasses import dataclass
+from email.parser import BytesHeaderParser
+from email.policy import default as default_policy
+from typing import Any
+from urllib.parse import parse_qs, unquote, urlsplit
+
+from sqlalchemy import select, update
+
+from app.db import session_scope
+from app.indexer import rawstore
+from app.models import Message
+
+logger = logging.getLogger(__name__)
+
+_MAX_HEADER = 2000
+_ANGLE = re.compile(r"<([^<>]+)>")
+_BATCH = 200
+
+
+@dataclass(frozen=True)
+class ListHeaders:
+    list_id: str | None
+    unsubscribe: str  # '' when the message has none
+    one_click: bool
+
+
+def read_list_headers(raw: bytes) -> ListHeaders:
+    """The mailing-list headers of a raw message (headers only are parsed)."""
+    end = raw.find(b"\r\n\r\n")
+    if end < 0:
+        end = raw.find(b"\n\n")
+    head = raw if end < 0 else raw[: end + 2]
+    try:
+        msg = BytesHeaderParser(policy=default_policy).parsebytes(head)
+        list_id_raw = str(msg.get("List-Id") or "")
+        unsubscribe = str(msg.get("List-Unsubscribe") or "")
+        post = str(msg.get("List-Unsubscribe-Post") or "")
+    except Exception:  # a malformed header must not stop the message being stored
+        return ListHeaders(None, "", False)
+    list_id = None
+    if list_id_raw:
+        match = _ANGLE.search(list_id_raw)
+        list_id = (match.group(1) if match else list_id_raw).strip().lower()[:300] or None
+    unsubscribe = " ".join(unsubscribe.split())[:_MAX_HEADER]
+    one_click = "list-unsubscribe=one-click" in post.replace(" ", "").lower()
+    return ListHeaders(list_id, unsubscribe, one_click and bool(unsubscribe))
+
+
+@dataclass(frozen=True)
+class UnsubscribeTargets:
+    https: str | None
+    mailto: str | None  # the address
+    mailto_subject: str
+    mailto_body: str
+
+
+def unsubscribe_targets(header: str) -> UnsubscribeTargets:
+    """The https and mailto targets listed in a ``List-Unsubscribe`` header."""
+    https = mailto = None
+    subject, body = "unsubscribe", "unsubscribe"
+    for target in _ANGLE.findall(header):
+        target = target.strip()
+        lowered = target.lower()
+        if https is None and lowered.startswith("https://"):
+            https = target
+        elif mailto is None and lowered.startswith("mailto:"):
+            parts = urlsplit(target)
+            address = unquote(parts.path).strip()
+            if "@" in address and "\n" not in address and "\r" not in address:
+                mailto = address
+                query = parse_qs(parts.query)
+                subject = (query.get("subject") or [subject])[0][:200]
+                body = (query.get("body") or [body])[0][:2000]
+    return UnsubscribeTargets(https, mailto, subject, body)
+
+
+def group_key(list_id: str | None, sender: str) -> str:
+    return f"list:{list_id}" if list_id else f"from:{sender.lower()}"
+
+
+async def pending_count(session: Any, user_id: uuid.UUID) -> int:
+    from sqlalchemy import func
+
+    return int(
+        await session.scalar(
+            select(func.count())
+            .select_from(Message)
+            .where(Message.user_id == user_id, Message.list_unsubscribe.is_(None))
+        )
+        or 0
+    )
+
+
+async def backfill_list_headers_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Read the list headers of a user's older mail (stored before this feature).
+
+    Works in batches; a message whose raw copy can't be had is marked as having
+    none, so it is not tried again. Returns how many messages were read.
+    """
+    uid = uuid.UUID(user_id)
+    done = 0
+    while True:
+        async with session_scope() as session:
+            batch = list(
+                await session.scalars(
+                    select(Message)
+                    .where(Message.user_id == uid, Message.list_unsubscribe.is_(None))
+                    .limit(_BATCH)
+                )
+            )
+            if not batch:
+                break
+            for message in batch:
+                try:
+                    raw = await rawstore.ensure_bytes(message)
+                    headers = read_list_headers(raw)
+                except Exception:  # missing blob, server unreachable: give up on it
+                    headers = ListHeaders(None, "", False)
+                await session.execute(
+                    update(Message)
+                    .where(Message.id == message.id)
+                    .values(
+                        list_id=headers.list_id,
+                        list_unsubscribe=headers.unsubscribe,
+                        list_unsubscribe_post=headers.one_click,
+                    )
+                )
+            await session.commit()
+            done += len(batch)
+    if done:
+        logger.info("read list headers of %d message(s) for user %s", done, user_id)
+    return done

backend/app/sync/actions.py

@@ -1,9 +1,11 @@
 """Outbound user actions: flag changes and moves (SPEC §6.3).
 
 The API writes the change to the DB first, then enqueues a job here to push it to
-the mail server. On provider failure the job retries (arq, ×5 exponential) and, on
-final failure, reverts the DB change and emits ``flags.changed`` with
-``reverted=true`` so the UI can roll back.
+the mail server. While the server cannot be reached the job is retried (arq
+``Retry``) with growing gaps for about 24 hours (``ACTION_MAX_TRIES``); a flag
+change that still cannot be pushed is then reverted in the DB and announced with
+``flags.changed`` / ``reverted=true`` so the UI rolls back. A plain exception does
+NOT make arq retry — only ``Retry`` does.
 """
 
 from __future__ import annotations
@@ -15,6 +17,7 @@ from collections.abc import Iterable
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
+from arq import Retry
 from sqlalchemy import delete, func, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -22,6 +25,7 @@ from app.config import get_settings
 from app.db import session_scope
 from app.events import get_event_bus
 from app.models import Account, Draft, Mailbox, Message, Thread
+from app.sync.health import is_temporary
 from app.sync.provider_factory import build_provider
 from app.threading_.threader import denormalize_thread
 
@@ -30,7 +34,26 @@ logger = logging.getLogger(__name__)
 # User-settable IMAP flags (SPEC §8.2 is:read/starred/answered). \Deleted/\Draft
 # are managed by move/expunge and drafts, not this surface.
 ALLOWED_FLAGS = frozenset({"\\Seen", "\\Flagged", "\\Answered"})
-_MAX_TRIES = 5
+# Gaps between tries while the server is unreachable: ~24 hours in all.
+_RETRY_DEFERS_S = (30, 60, 120, 300, 600, 1800, 3600, 7200, 14400, 28800, 28800)
+ACTION_MAX_TRIES = len(_RETRY_DEFERS_S) + 1
+
+
+def _retry_later(ctx: dict[str, Any], exc: BaseException) -> Retry | None:
+    """A ``Retry`` for a temporary failure while tries remain, else None (give up)."""
+    if not is_temporary(exc):
+        return None
+    attempt = int(ctx.get("job_try", 1))
+    if attempt >= ACTION_MAX_TRIES:
+        return None
+    logger.info(
+        "server unreachable (try %d), retrying in %ds: %s",
+        attempt,
+        _RETRY_DEFERS_S[attempt - 1],
+        exc,
+    )
+    return Retry(defer=_RETRY_DEFERS_S[attempt - 1])
+
 
 # Folder names to create when a target special-use folder is missing on the server.
 _DEFAULT_FOLDER_NAME = {"archive": "Archive", "trash": "Trash", "junk": "Spam"}
@@ -54,7 +77,7 @@ async def apply_flags_to_messages(
         if message.thread_id is not None:
             thread_ids.add(message.thread_id)
     await session.flush()
-    for tid in thread_ids:
+    for tid in sorted(thread_ids):  # one lock order across transactions
         await denormalize_thread(session, tid)
     return thread_ids
 
@@ -71,6 +94,8 @@ async def push_flags_job(
         message = await session.get(Message, mid)
         if message is None:
             return
+        if _superseded(message.flags, add, remove):
+            return  # changed again since: that later change's own job pushes it
         account = await session.get(Account, message.account_id)
         mailbox = await session.get(Mailbox, message.mailbox_id)
         if account is None or mailbox is None:
@@ -80,21 +105,29 @@ async def push_flags_job(
     provider = build_provider(account)
     try:
         await provider.push_flags(path, uid, frozenset(add), frozenset(remove))
-    except Exception:
-        if ctx.get("job_try", 1) >= _MAX_TRIES:
-            await _revert_flags(mid, user_id, add, remove)
-            return
-        raise  # arq retries with backoff
+    except Exception as exc:
+        retry = _retry_later(ctx, exc)
+        if retry is not None:
+            raise retry from exc
+        logger.warning("flag change for message %s could not be pushed: %s", mid, exc)
+        await _revert_flags(mid, user_id, add, remove)
     finally:
         await provider.close()
 
 
+def _superseded(flags: list[str], add: list[str], remove: list[str]) -> bool:
+    """The message's flags no longer show this change: a later one replaced it
+    (e.g. read, then unread again while the server was unreachable)."""
+    now = set(flags)
+    return not set(add) <= now or bool(set(remove) & now)
+
+
 async def _revert_flags(
     message_id: uuid.UUID, user_id: uuid.UUID, add: list[str], remove: list[str]
 ) -> None:
     async with session_scope() as session:
         message = await session.get(Message, message_id)
-        if message is None:
+        if message is None or _superseded(message.flags, add, remove):
             return
         # Undo the change: remove what we added, add back what we removed.
         message.flags = sorted((set(message.flags) - set(add)) | set(remove))
@@ -138,11 +171,12 @@ async def move_job(
     provider = build_provider(account)
     try:
         new_uid = await provider.move(uid, src_path, dst_path)
-    except Exception:
-        if ctx.get("job_try", 1) >= _MAX_TRIES:
-            logger.warning("move failed for message %s; leaving optimistic state", mid)
-            return
-        raise
+    except Exception as exc:
+        retry = _retry_later(ctx, exc)
+        if retry is not None:
+            raise retry from exc
+        logger.warning("move failed for message %s; leaving optimistic state: %s", mid, exc)
+        return
     finally:
         await provider.close()
 
@@ -193,28 +227,39 @@ async def move_thread_to_folder_job(ctx: dict[str, Any], thread_id: str, special
     # (message_id, account_id, target_path, new_uid) applied after the network moves.
     moves: list[tuple[uuid.UUID, uuid.UUID, str, int]] = []
     to_create: list[tuple[uuid.UUID, str, int]] = []  # account_id, path, uidvalidity
+    unreachable: BaseException | None = None  # set: retry the moves that did not happen
 
     for account_id in account_ids:
         provider = build_provider(accounts[account_id])
         try:
-            if account_id in target_id_path:
-                target_id, target_path = target_id_path[account_id]
-            else:
-                target_id, target_path = None, _DEFAULT_FOLDER_NAME.get(special_use, special_use)
-                await provider.create_mailbox(target_path)
-                remote = await provider.list_mailboxes()
-                uidv = next((r.uidvalidity for r in remote if r.path == target_path), 0)
-                to_create.append((account_id, target_path, uidv))
-            for mid, m_account, m_mailbox, m_uid in snapshot:
-                if m_account != account_id or m_mailbox == target_id:
-                    continue
-                src_path = path_by_id.get(m_mailbox, "INBOX")
-                try:
-                    new_uid = await provider.move(m_uid, src_path, target_path)
-                except Exception:
-                    logger.exception("move failed for message %s -> %s", mid, target_path)
-                    continue
-                moves.append((mid, account_id, target_path, new_uid))
+            try:
+                if account_id in target_id_path:
+                    target_id, target_path = target_id_path[account_id]
+                else:
+                    target_id, target_path = (
+                        None,
+                        _DEFAULT_FOLDER_NAME.get(special_use, special_use),
+                    )
+                    await provider.create_mailbox(target_path)
+                    remote = await provider.list_mailboxes()
+                    uidv = next((r.uidvalidity for r in remote if r.path == target_path), 0)
+                    to_create.append((account_id, target_path, uidv))
+                for mid, m_account, m_mailbox, m_uid in snapshot:
+                    if m_account != account_id or m_mailbox == target_id:
+                        continue
+                    src_path = path_by_id.get(m_mailbox, "INBOX")
+                    try:
+                        new_uid = await provider.move(m_uid, src_path, target_path)
+                    except Exception as exc:
+                        if is_temporary(exc):
+                            raise  # the server is gone: stop this account, retry it later
+                        logger.exception("move failed for message %s -> %s", mid, target_path)
+                        continue
+                    moves.append((mid, account_id, target_path, new_uid))
+            except Exception as exc:
+                if not is_temporary(exc):
+                    raise
+                unreachable = exc
         finally:
             await provider.close()
 
@@ -246,6 +291,10 @@ async def move_thread_to_folder_job(ctx: dict[str, Any], thread_id: str, special
     await get_event_bus().publish(
         "thread.updated", {"user_id": str(user_id), "thread_id": str(tid)}
     )
+    # What moved is recorded; the rest goes again once the server is back (the
+    # re-run skips messages already in the target folder).
+    if unreachable is not None and (retry := _retry_later(ctx, unreachable)) is not None:
+        raise retry from unreachable
 
 
 async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
@@ -253,7 +302,8 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
 
     Each message returns to ``restore_mailbox_id`` (the folder it was in before it
     was archived/trashed) — e.g. a deleted Sent message goes back to Sent. Messages
-    with no recorded origin fall back to the account's Inbox.
+    with no recorded origin fall back to the account's Inbox. Only messages in
+    Archive, Trash or Spam move: the thread's others (your reply in Sent) stay.
     """
     tid = uuid.UUID(thread_id)
     async with session_scope() as session:
@@ -270,12 +320,13 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
             await session.scalars(select(Mailbox).where(Mailbox.account_id.in_(account_ids)))
         )
         path_by_id = {mb.id: mb.path for mb in mailboxes}
-        inbox_by_account = {
-            mb.account_id: mb.id for mb in mailboxes if mb.special_use == "inbox"
-        }
+        removal_ids = {mb.id for mb in mailboxes if mb.special_use in _REMOVAL_FOLDERS}
+        inbox_by_account = {mb.account_id: mb.id for mb in mailboxes if mb.special_use == "inbox"}
         # (message_id, account_id, source_path, target_id) — resolve the target now.
         plan: list[tuple[uuid.UUID, uuid.UUID, str, uuid.UUID, int]] = []
         for m in messages:
+            if m.mailbox_id not in removal_ids:
+                continue  # never removed: nothing to restore
             target_id = m.restore_mailbox_id
             if target_id is None or target_id not in path_by_id:
                 target_id = inbox_by_account.get(m.account_id)
@@ -287,6 +338,7 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
 
     # (message_id, target_id, new_uid) applied after the network moves.
     moves: list[tuple[uuid.UUID, uuid.UUID, int]] = []
+    unreachable: BaseException | None = None
     for account_id in account_ids:
         provider = build_provider(accounts[account_id])
         try:
@@ -296,7 +348,10 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
                 target_path = path_by_id[target_id]
                 try:
                     new_uid = await provider.move(uid, src_path, target_path)
-                except Exception:
+                except Exception as exc:
+                    if is_temporary(exc):
+                        unreachable = exc  # the server is gone: stop this account
+                        break
                     logger.exception("restore move failed for message %s -> %s", mid, target_path)
                     continue
                 moves.append((mid, target_id, new_uid))
@@ -314,6 +369,8 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
     await get_event_bus().publish(
         "thread.updated", {"user_id": str(user_id), "thread_id": str(tid)}
     )
+    if unreachable is not None and (retry := _retry_later(ctx, unreachable)) is not None:
+        raise retry from unreachable
 
 
 async def delete_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
@@ -352,8 +409,12 @@ async def delete_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
                     continue
                 try:
                     await provider.delete_messages(path, uids)
-                except Exception:
-                    logger.exception("expunge failed for %s in %s", uids, path)
+                except Exception as exc:
+                    # Keep the rows while the server is away: deleted here but
+                    # not there, the next sync would bring the messages back.
+                    if (retry := _retry_later(ctx, exc)) is not None:
+                        raise retry from exc
+                    logger.warning("expunge failed for %s in %s: %s", uids, path, exc)
         finally:
             await provider.close()
 
@@ -389,9 +450,7 @@ async def empty_folder_job(ctx: dict[str, Any], user_id: str, special_use: str)
     async with session_scope() as session:
         mailboxes = list(
             await session.scalars(
-                select(Mailbox).where(
-                    Mailbox.user_id == uid, Mailbox.special_use == special_use
-                )
+                select(Mailbox).where(Mailbox.user_id == uid, Mailbox.special_use == special_use)
             )
         )
         if not mailboxes:
@@ -423,8 +482,11 @@ async def empty_folder_job(ctx: dict[str, Any], user_id: str, special_use: str)
                     continue
                 try:
                     await provider.delete_messages(path, uids)
-                except Exception:
-                    logger.exception("empty folder: expunge failed for %s in %s", uids, path)
+                except Exception as exc:
+                    # As for a thread: keep the rows until the server has them gone.
+                    if (retry := _retry_later(ctx, exc)) is not None:
+                        raise retry from exc
+                    logger.warning("empty folder: expunge failed for %s in %s: %s", uids, path, exc)
         finally:
             await provider.close()
 
@@ -446,7 +508,7 @@ async def empty_folder_job(ctx: dict[str, Any], user_id: str, special_use: str)
         await session.flush()
         # Threads with no messages left get dropped (drafts un-linked first); the
         # rest are re-denormalized so their counts/snippet reflect the removal.
-        for tid in thread_ids:
+        for tid in sorted(thread_ids):
             remaining = await session.scalar(
                 select(func.count()).select_from(Message).where(Message.thread_id == tid)
             )
@@ -520,13 +582,13 @@ async def autopurge_trash_spam(ctx: dict[str, Any]) -> None:
     user_ids = {r.user_id for r in stale}
     async with session_scope() as session:
         await session.execute(
-            update(Draft).where(Draft.reply_to_message_id.in_(msg_ids)).values(
-                reply_to_message_id=None
-            )
+            update(Draft)
+            .where(Draft.reply_to_message_id.in_(msg_ids))
+            .values(reply_to_message_id=None)
         )
         await session.execute(delete(Message).where(Message.id.in_(msg_ids)))
         await session.flush()
-        for tid in thread_ids:
+        for tid in sorted(thread_ids):
             remaining = await session.scalar(
                 select(func.count()).select_from(Message).where(Message.thread_id == tid)
             )
@@ -539,6 +601,4 @@ async def autopurge_trash_spam(ctx: dict[str, Any]) -> None:
     bus = get_event_bus()
     for user_id in user_ids:
         await bus.publish("message.deleted", {"user_id": str(user_id)})
-    logger.info(
-        "autopurge: removed %d Trash/Spam message(s) older than %dd", len(msg_ids), days
-    )
+    logger.info("autopurge: removed %d Trash/Spam message(s) older than %dd", len(msg_ids), days)

backend/app/sync/engine.py

@@ -18,10 +18,11 @@ from app.config import get_settings
 from app.events import get_event_bus
 from app.indexer.indexer import PersistResult, persist_message
 from app.models import Account, Draft, Mailbox, Message
-from app.providers.base import MailboxDelta, ProviderNotSupported
+from app.providers.base import MailboxDelta, ProviderAuthError, ProviderNotSupported
 from app.providers.imap_smtp import ImapSmtpProvider
 from app.search.factory import get_search_backend
 from app.sync.folder_names import infer_special_use
+from app.sync.health import AccountHealth, is_temporary
 from app.sync.provider_factory import build_provider
 from app.threading_.threader import denormalize_thread
 
@@ -58,6 +59,12 @@ async def _set_state(account_id: uuid.UUID, state: str, detail: str = "") -> Non
         await session.execute(
             update(Account).where(Account.id == account_id).values(state=state, state_detail=detail)
         )
+        if state == "live":  # (re)connected and synced: no sync problem any more
+            await session.execute(
+                update(Account)
+                .where(Account.id == account_id)
+                .values(sync_error=None, sync_error_since=None)
+            )
     # Keep the sync:live registry in step with the account's state so the sync
     # coordinator claims it once live and drops it otherwise (refactor §2).
     from app.redis_client import get_redis
@@ -165,9 +172,7 @@ class _BackfillProgress:
         self._mailboxes.add(mailbox_path)
         async with session_scope() as session:
             await session.execute(
-                update(Account)
-                .where(Account.id == self.account.id)
-                .values(backfill_done=self.done)
+                update(Account).where(Account.id == self.account.id).values(backfill_done=self.done)
             )
         await get_event_bus().publish(
             "sync.progress",
@@ -314,16 +319,28 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
             dedupe = gmail and mailbox.special_use == "archive"
             if since is not None:  # 3months: single windowed pass
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, since=since,
+                    provider,
+                    account_obj,
+                    mailbox,
+                    progress,
+                    since=since,
                     dedupe_account_scope=dedupe,
                 )
             else:  # all: recent window first (usable fast), then the remainder
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, since=recent_cutoff,
+                    provider,
+                    account_obj,
+                    mailbox,
+                    progress,
+                    since=recent_cutoff,
                     dedupe_account_scope=dedupe,
                 )
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, before=recent_cutoff,
+                    provider,
+                    account_obj,
+                    mailbox,
+                    progress,
+                    before=recent_cutoff,
                     dedupe_account_scope=dedupe,
                 )
         await _set_state(account_id, "live")
@@ -347,7 +364,18 @@ async def apply_delta(
     if delta.new_uidvalidity is not None:
         # UIDVALIDITY changed: wipe this mailbox's rows and re-backfill (SPEC §6.3).
         async with session_scope() as session:
+            # Their threads are recounted: a message not brought back by the
+            # re-import (gone meanwhile) must not stay counted.
+            wiped_threads: set[uuid.UUID] = {
+                tid
+                for tid in await session.scalars(
+                    select(Message.thread_id).where(Message.mailbox_id == mailbox.id)
+                )
+                if tid is not None
+            }
             await session.execute(delete(Message).where(Message.mailbox_id == mailbox.id))
+            for tid in sorted(wiped_threads):
+                await denormalize_thread(session, tid)
             await session.execute(
                 update(Mailbox)
                 .where(Mailbox.id == mailbox.id)
@@ -421,7 +449,7 @@ async def apply_delta(
             )
         # After the deletes, so a thread left with no messages is removed rather
         # than recounted (denormalize_thread handles that case).
-        for tid in touched_threads:
+        for tid in sorted(touched_threads):  # one lock order (denormalize_thread)
             await denormalize_thread(session, tid)
 
         updates = {"last_seen_uid": max_uid}
@@ -430,6 +458,12 @@ async def apply_delta(
         await session.execute(update(Mailbox).where(Mailbox.id == mailbox.id).values(**updates))
 
     await _emit_batch(results, account, mailbox.path, len(results))
+    if results and mailbox.special_use == "inbox":
+        # New mail while live (backfills never come through here): tell the
+        # user's devices, a few seconds later (app/push/service.py).
+        from app.push.service import enqueue_new_mail
+
+        await enqueue_new_mail(account.user_id, [r.message.id for r in results])
 
     if gone_ids:
         # Drop expunged messages from the search index (PG keeps its tsvector on the
@@ -467,9 +501,7 @@ async def apply_delta(
         await _reconcile_drafts(provider, account, mailbox)
 
 
-async def _reconcile_drafts(
-    provider: ImapSmtpProvider, account: Account, mailbox: Mailbox
-) -> None:
+async def _reconcile_drafts(provider: ImapSmtpProvider, account: Account, mailbox: Mailbox) -> None:
     """Drop DB rows for drafts that no longer exist on the server (were expunged)."""
     from app.db import session_scope
 
@@ -504,7 +536,10 @@ async def _reconcile_drafts(
 
 
 async def live_loop(
-    account_id: uuid.UUID, mailbox_path: str = "INBOX", stop: asyncio.Event | None = None
+    account_id: uuid.UUID,
+    mailbox_path: str = "INBOX",
+    stop: asyncio.Event | None = None,
+    health: AccountHealth | None = None,
 ) -> None:
     """Maintain a live IDLE/poll loop for one mailbox (SPEC §6.3)."""
     from app.db import session_scope
@@ -518,13 +553,16 @@ async def live_loop(
 
     provider = build_provider(account_obj)
     try:
+        await provider.verify()  # signed in: the account is reachable
+        if health is not None:
+            await health.ok()
+        first = True
         while not stop.is_set():
-            try:
-                await provider.idle(mailbox_path)
-            except ProviderNotSupported:
-                await asyncio.sleep(idle_fallback)
-            if stop.is_set():
-                break
+            # Catch up before each IDLE, not only after it: IDLE reports what
+            # arrives from now on, so mail that came while this loop was not
+            # running (a new account's first minutes, a reconnect, a worker
+            # restart) would otherwise wait for the next new message or the
+            # IDLE timeout (10 min) to show.
             async with session_scope() as session:
                 mailbox = await session.scalar(
                     select(Mailbox).where(
@@ -532,19 +570,30 @@ async def live_loop(
                     )
                 )
                 mailbox_obj = mailbox
-            if mailbox_obj is None:
+            if mailbox_obj is not None:
+                delta = await provider.poll(
+                    mailbox_path,
+                    last_seen_uid=mailbox_obj.last_seen_uid,
+                    highestmodseq=mailbox_obj.highestmodseq,
+                )
+                await apply_delta(provider, account_obj, mailbox_obj, delta)
+            elif not first:
+                break  # the folder is gone
+            # else: the account's first sync has not recorded the folder yet.
+            first = False
+            if stop.is_set():
                 break
-            delta = await provider.poll(
-                mailbox_path,
-                last_seen_uid=mailbox_obj.last_seen_uid,
-                highestmodseq=mailbox_obj.highestmodseq,
-            )
-            await apply_delta(provider, account_obj, mailbox_obj, delta)
+            try:
+                await provider.idle(mailbox_path)
+            except ProviderNotSupported:
+                await asyncio.sleep(idle_fallback)
     finally:
         await provider.close()
 
 
-async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> None:
+async def poll_folders_once(
+    account_id: uuid.UUID, *, include_inbox: bool, health: AccountHealth | None = None
+) -> None:
     """One CONDSTORE poll pass over an account's special-use folders.
 
     ``include_inbox`` also polls INBOX (used by the manual Refresh); the periodic
@@ -586,14 +635,20 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No
                     highestmodseq=mailbox_obj.highestmodseq,
                 )
                 await apply_delta(provider, account_obj, mailbox_obj, delta)
-            except Exception:
+            except Exception as exc:
+                # Server gone, or the sign-in refused: the other folders would only
+                # fail the same way (20 s each). End the pass; the caller decides.
+                if is_temporary(exc) or isinstance(exc, ProviderAuthError):
+                    raise
                 logger.exception("poll failed: account=%s mailbox=%s", account_id, mailbox_obj.path)
+        if health is not None:
+            await health.ok()
     finally:
         await provider.close()
 
 
-async def _poll_secondary_once(account_id: uuid.UUID) -> None:
-    await poll_folders_once(account_id, include_inbox=False)
+async def _poll_secondary_once(account_id: uuid.UUID, health: AccountHealth | None = None) -> None:
+    await poll_folders_once(account_id, include_inbox=False, health=health)
 
 
 async def refresh_account_job(ctx: dict[str, Any], account_id: str) -> None:
@@ -602,7 +657,10 @@ async def refresh_account_job(ctx: dict[str, Any], account_id: str) -> None:
 
 
 async def secondary_poll_loop(
-    account_id: uuid.UUID, stop: asyncio.Event | None = None, interval: float | None = None
+    account_id: uuid.UUID,
+    stop: asyncio.Event | None = None,
+    interval: float | None = None,
+    health: AccountHealth | None = None,
 ) -> None:
     """Periodically CONDSTORE-poll non-INBOX folders (Sent/Drafts/Archive/Trash/Junk).
 
@@ -624,11 +682,14 @@ async def secondary_poll_loop(
             await asyncio.wait_for(stop.wait(), timeout=wait)
         if stop.is_set():
             break
-        await _poll_secondary_once(account_id)
+        await _poll_secondary_once(account_id, health)
 
 
 async def poll_all_loop(
-    account_id: uuid.UUID, interval: float, stop: asyncio.Event | None = None
+    account_id: uuid.UUID,
+    interval: float,
+    stop: asyncio.Event | None = None,
+    health: AccountHealth | None = None,
 ) -> None:
     """Transient CONDSTORE poll of INBOX + special folders every ``interval`` seconds.
 
@@ -646,4 +707,4 @@ async def poll_all_loop(
             await asyncio.wait_for(stop.wait(), timeout=wait)
         if stop.is_set():
             break
-        await poll_folders_once(account_id, include_inbox=True)
+        await poll_folders_once(account_id, include_inbox=True, health=health)

backend/app/sync/folder_names.py

@@ -28,24 +28,52 @@ def _compact(name: str) -> str:
 # Localized folder names by special_use. Written readably; compacted at load.
 _NAMES: dict[str, list[str]] = {
     "sent": [
-        "sent", "sent items", "sent mail", "sent messages",  # English / Outlook / Gmail
-        "wysłane", "elementy wysłane", "wiadomości wysłane",  # Polish
-        "gesendet", "gesendete objekte", "gesendete elemente",  # German
-        "envoyés", "éléments envoyés", "messages envoyés", "courrier envoyé",  # French
-        "enviados", "elementos enviados", "correo enviado",  # Spanish
-        "inviata", "posta inviata", "messaggi inviati",  # Italian
-        "verzonden", "verzonden items", "verzonden berichten",  # Dutch
-        "enviadas", "itens enviados", "mensagens enviadas",  # Portuguese
-        "отправленные", "исходящие",  # Russian
-        "skickat", "skickade", "skickade objekt",  # Swedish
-        "sendt", "sendte", "sendte elementer",  # Norwegian / Danish
+        "sent",
+        "sent items",
+        "sent mail",
+        "sent messages",  # English / Outlook / Gmail
+        "wysłane",
+        "elementy wysłane",
+        "wiadomości wysłane",  # Polish
+        "gesendet",
+        "gesendete objekte",
+        "gesendete elemente",  # German
+        "envoyés",
+        "éléments envoyés",
+        "messages envoyés",
+        "courrier envoyé",  # French
+        "enviados",
+        "elementos enviados",
+        "correo enviado",  # Spanish
+        "inviata",
+        "posta inviata",
+        "messaggi inviati",  # Italian
+        "verzonden",
+        "verzonden items",
+        "verzonden berichten",  # Dutch
+        "enviadas",
+        "itens enviados",
+        "mensagens enviadas",  # Portuguese
+        "отправленные",
+        "исходящие",  # Russian
+        "skickat",
+        "skickade",
+        "skickade objekt",  # Swedish
+        "sendt",
+        "sendte",
+        "sendte elementer",  # Norwegian / Danish
         "lähetetyt",  # Finnish
-        "odeslané", "odeslaná pošta",  # Czech
+        "odeslané",
+        "odeslaná pošta",  # Czech
     ],
     "drafts": [
-        "drafts", "draft",
-        "robocze", "kopie robocze", "wersje robocze",
-        "entwürfe", "entwurf",
+        "drafts",
+        "draft",
+        "robocze",
+        "kopie robocze",
+        "wersje robocze",
+        "entwürfe",
+        "entwurf",
         "brouillons",
         "borradores",
         "bozze",
@@ -58,44 +86,80 @@ _NAMES: dict[str, list[str]] = {
         "koncepty",  # Czech
     ],
     "trash": [
-        "trash", "deleted", "deleted items", "deleted messages", "bin", "recycle bin",
+        "trash",
+        "deleted",
+        "deleted items",
+        "deleted messages",
+        "bin",
+        "recycle bin",
         "kosz",
-        "papierkorb", "gelöschte objekte", "gelöschte elemente",
-        "corbeille", "éléments supprimés",
-        "papelera", "elementos eliminados",
-        "cestino", "posta eliminata",
-        "prullenbak", "verwijderde items",
-        "lixeira", "lixo", "itens excluídos",
-        "корзина", "удаленные",
-        "papperskorg", "borttaget",
-        "papirkurv", "slettet",
-        "roskakori", "poistetut",
+        "papierkorb",
+        "gelöschte objekte",
+        "gelöschte elemente",
+        "corbeille",
+        "éléments supprimés",
+        "papelera",
+        "elementos eliminados",
+        "cestino",
+        "posta eliminata",
+        "prullenbak",
+        "verwijderde items",
+        "lixeira",
+        "lixo",
+        "itens excluídos",
+        "корзина",
+        "удаленные",
+        "papperskorg",
+        "borttaget",
+        "papirkurv",
+        "slettet",
+        "roskakori",
+        "poistetut",
         "koš",  # Czech
     ],
     "junk": [
-        "junk", "junk email", "junk e-mail", "spam", "bulk mail", "bulk",
-        "wiadomości-śmieci", "niechciane",
-        "unerwünscht", "werbung",
-        "courrier indésirable", "indésirables", "pourriel",
-        "correo no deseado", "no deseado", "correo basura",
-        "posta indesiderata", "indesiderata",
-        "ongewenst", "ongewenste e-mail",
+        "junk",
+        "junk email",
+        "junk e-mail",
+        "spam",
+        "bulk mail",
+        "bulk",
+        "wiadomości-śmieci",
+        "niechciane",
+        "unerwünscht",
+        "werbung",
+        "courrier indésirable",
+        "indésirables",
+        "pourriel",
+        "correo no deseado",
+        "no deseado",
+        "correo basura",
+        "posta indesiderata",
+        "indesiderata",
+        "ongewenst",
+        "ongewenste e-mail",
         "lixo eletrônico",
-        "спам", "нежелательная почта",
-        "skräp", "skräppost",
+        "спам",
+        "нежелательная почта",
+        "skräp",
+        "skräppost",
         "uønsket",
         "roskaposti",
         "nevyžádaná pošta",
     ],
     "archive": [
-        "archive", "archives", "all mail",
+        "archive",
+        "archives",
+        "all mail",
         "archiwum",
         "archiv",
         "archivio",
         "archivo",
         "archief",
-        "arquivo", "arquivo morto",
-        "архив", "вся почта",
+        "arquivo",
+        "arquivo morto",
+        "архив",
+        "вся почта",
         "arkiv",
         "arkisto",
     ],

backend/app/sync/health.py

@@ -0,0 +1,109 @@
+"""An account's connection health.
+
+Telling "the mail server is not there right now" from everything else.
+
+``is_temporary`` decides whether a failure is worth another try later: the
+server could not be reached, timed out, dropped the connection, or answered
+"not now" (SMTP 4xx). Sends, user actions and the sync loops all retry on it,
+and only on it — a refused password or a permanent 5xx is not retried.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import UTC, datetime
+
+from aioimaplib import aioimaplib
+from sqlalchemy import update
+
+from app.providers.base import ProviderAuthError, ProviderConnectError
+
+logger = logging.getLogger(__name__)
+
+
+def is_temporary(exc: BaseException) -> bool:
+    if isinstance(exc, (ProviderConnectError, TimeoutError, OSError)):
+        return True
+    # aioimaplib's own: a command that timed out, or a reply cut off.
+    if isinstance(exc, (aioimaplib.CommandTimeout, aioimaplib.IncompleteRead)):
+        return True
+    # After the connection drops, aioimaplib refuses every further command as
+    # "illegal in state LOGOUT" — the connection is gone, not the command wrong.
+    return isinstance(exc, aioimaplib.Abort) and "illegal in state" in str(exc)
+
+
+# Failed connections in a row before an account shows as unreachable, and
+# refused sign-ins in a row before its syncing stops ("auth_failed": retrying a
+# wrong password only gets the account locked by the provider). Changing the
+# password in Settings re-runs the initial sync, which sets it live again.
+UNREACHABLE_AFTER = 3
+AUTH_GIVE_UP_AFTER = 3
+
+
+class AccountHealth:
+    """One account's run of failures, shared by its sync loops."""
+
+    def __init__(self, account_id: uuid.UUID) -> None:
+        self.account_id = account_id
+        self.failures = 0
+        self.auth_failures = 0
+        # Whether the account is marked unreachable; None = not known yet (a
+        # mark can survive a worker restart), so the first success checks.
+        self.marked: bool | None = None
+        # Set once the sign-in was refused too often: the account is no longer
+        # synced (state auth_failed), so its loops should stop now.
+        self.gave_up = False
+
+    async def ok(self) -> None:
+        """The server answered: reset the counts, and clear an unreachable mark."""
+        self.failures = 0
+        self.auth_failures = 0
+        if self.marked is not False:
+            await _set_sync_error(self.account_id, None)
+            self.marked = False
+
+    async def failed(self, exc: BaseException) -> None:
+        if isinstance(exc, ProviderAuthError):
+            self.auth_failures += 1
+            if self.auth_failures >= AUTH_GIVE_UP_AFTER:
+                from app.sync.engine import _set_state
+
+                logger.warning(
+                    "account %s: sign-in refused %d times; stopping sync",
+                    self.account_id,
+                    self.auth_failures,
+                )
+                await _set_state(self.account_id, "auth_failed", exc.detail[:500])
+                self.gave_up = True
+            return
+        if not is_temporary(exc):
+            return
+        self.failures += 1
+        if self.failures >= UNREACHABLE_AFTER and self.marked is not True:
+            await _set_sync_error(self.account_id, "unreachable")
+            self.marked = True
+
+
+async def _set_sync_error(account_id: uuid.UUID, error: str | None) -> None:
+    """Mark (or clear) an account's sync problem; tell the user's open apps."""
+    from app.db import session_scope
+    from app.events import get_event_bus
+    from app.models import Account
+
+    now = datetime.now(UTC)
+    async with session_scope() as session:
+        cond = Account.sync_error.isnot(None) if error is None else Account.sync_error.is_(None)
+        user_id = await session.scalar(
+            update(Account)
+            .where(Account.id == account_id, cond)
+            .values(sync_error=error, sync_error_since=None if error is None else now)
+            .returning(Account.user_id)
+        )
+    if user_id is None:
+        return  # nothing changed
+    logger.warning("account %s: %s", account_id, error or "reachable again")
+    await get_event_bus().publish(
+        "account.state_changed",
+        {"user_id": str(user_id), "account_id": str(account_id), "sync_error": error},
+    )

backend/app/sync/provider_factory.py

@@ -3,6 +3,9 @@
 This module runs only inside the worker/sync context; ``decrypt_secret`` (the
 credential vault) is invoked here to obtain the plaintext mailbox password. It
 must never be imported by API request code.
+
+A native account (the user's hosted mailbox, App. D §D.4) has no stored
+password: the worker signs in as itself, acting for the user.
 """
 
 from __future__ import annotations
@@ -14,6 +17,10 @@ from app.security.crypto import decrypt_secret
 
 def build_provider(account: Account) -> ImapSmtpProvider:
     """Construct an :class:`ImapSmtpProvider` for ``account`` (password auth, M0)."""
+    if account.kind == "native":
+        from app.hosted.native import native_provider_config
+
+        return ImapSmtpProvider(native_provider_config(account.email))
     imap = account.imap_config
     smtp = account.smtp_config
     password = decrypt_secret(account.secret_ct).decode("utf-8")
@@ -26,5 +33,6 @@ def build_provider(account: Account) -> ImapSmtpProvider:
         smtp_tls=str(smtp.get("tls", "ssl")),
         username=account.email,
         password=password,
+        public_only=True,  # a server the user named: never one inside our network
     )
     return ImapSmtpProvider(config)

backend/app/sync/runner.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
+import random
 import uuid
 from collections.abc import Awaitable, Callable
 
@@ -26,11 +27,18 @@ from sqlalchemy import select
 from app.config import get_settings
 from app.db import session_scope
 from app.models import Account
+from app.providers.base import ProviderAuthError
 from app.sync.engine import live_loop, poll_all_loop, secondary_poll_loop
+from app.sync.health import AccountHealth, is_temporary
 
 logger = logging.getLogger(__name__)
 
+# After a loop fails: wait 30 s, doubling with each failure in a row up to 10 min
+# (±20%, so accounts on one dead server don't retry in step). A loop that ran a
+# good while before failing starts again from 30 s.
 _RECONNECT_BACKOFF = 30.0
+_MAX_BACKOFF = 600.0
+_HEALTHY_RUN_S = 300.0
 _VERIFY_INTERVAL = 60.0
 
 Tier = str  # "hot" | "warm" | "cold"
@@ -49,6 +57,8 @@ class AccountRunner:
         # coordinator reaps such runners and releases their lease.
         self.dead = asyncio.Event()
         self.verify_interval = _VERIFY_INTERVAL
+        # Failures in a row across this account's loops: unreachable / sign-in failed.
+        self.health = AccountHealth(account_id)
 
     def start(self) -> None:
         """Launch the verify task and the current tier's work loops."""
@@ -98,13 +108,15 @@ class AccountRunner:
             await asyncio.gather(*tasks, return_exceptions=True)
 
     async def _run_live(self) -> None:
-        await live_loop(self.account_id, "INBOX", self._stop)
+        await live_loop(self.account_id, "INBOX", self._stop, self.health)
 
     async def _run_secondary(self) -> None:
-        await secondary_poll_loop(self.account_id, self._stop, get_settings().poll_warm)
+        await secondary_poll_loop(
+            self.account_id, self._stop, get_settings().poll_warm, self.health
+        )
 
     async def _run_poll(self, interval: float) -> None:
-        await poll_all_loop(self.account_id, interval, self._stop)
+        await poll_all_loop(self.account_id, interval, self._stop, self.health)
 
     async def _verify_loop(self) -> None:
         """Periodically confirm the account is still live; flag ``dead`` if not.
@@ -129,13 +141,47 @@ class AccountRunner:
         return state == "live"
 
     async def _supervise(self, run: Callable[[], Awaitable[None]], label: str) -> None:
-        """Run a loop, restarting it (with backoff) until stopped or cancelled."""
+        """Run a loop, restarting it (with growing backoff) until stopped or cancelled."""
+        loop = asyncio.get_running_loop()
+        fails = 0
         while not self._stop.is_set():
+            started = loop.time()
             try:
                 await run()
             except asyncio.CancelledError:
                 raise
-            except Exception:
-                logger.exception("%s loop for account %s crashed; retrying", label, self.account_id)
+            except Exception as exc:
+                if loop.time() - started > _HEALTHY_RUN_S:
+                    fails = 0  # it had been working: a fresh failure, not the same outage
+                fails += 1
+                delay = min(_MAX_BACKOFF, _RECONNECT_BACKOFF * 2 ** (fails - 1))
+                delay *= random.uniform(0.8, 1.2)
+                try:
+                    await self.health.failed(exc)
+                except Exception:  # noqa: BLE001 - never let bookkeeping kill the loop
+                    logger.exception("account %s: recording the failure failed", self.account_id)
+                if self.health.gave_up:
+                    # Sign-in refused too often: not synced any more (the health
+                    # tracker logged why). Flag the runner so the coordinator
+                    # reaps it and releases the lease.
+                    self.dead.set()
+                    return
+                if is_temporary(exc) or isinstance(exc, ProviderAuthError):
+                    # Expected while a server is down or a password is wrong: one
+                    # line, not a traceback every half minute.
+                    logger.warning(
+                        "%s loop for account %s: %s; retrying in %.0fs",
+                        label,
+                        self.account_id,
+                        getattr(exc, "detail", None) or exc or type(exc).__name__,
+                        delay,
+                    )
+                else:
+                    logger.exception(
+                        "%s loop for account %s crashed; retrying in %.0fs",
+                        label,
+                        self.account_id,
+                        delay,
+                    )
                 with contextlib.suppress(TimeoutError):
-                    await asyncio.wait_for(self._stop.wait(), timeout=_RECONNECT_BACKOFF)
+                    await asyncio.wait_for(self._stop.wait(), timeout=delay)

backend/app/threading_/threader.py

@@ -11,12 +11,12 @@ from __future__ import annotations
 import uuid
 from datetime import timedelta
 
-from sqlalchemy import ColumnElement, delete, literal, select, update
+from sqlalchemy import ColumnElement, delete, exists, func, literal, select, update
 from sqlalchemy import any_ as sa_any
 from sqlalchemy import or_ as sa_or
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.models import Message, Thread
+from app.models import Draft, Message, Thread
 from app.threading_.normalize import is_synthetic, normalize_subject
 
 _SUBJECT_FALLBACK_WINDOW = timedelta(days=7)
@@ -172,18 +172,38 @@ async def _create_thread(session: AsyncSession, message: Message) -> uuid.UUID:
 
 
 async def denormalize_thread(session: AsyncSession, thread_id: uuid.UUID) -> None:
-    """Recompute a thread's denormalized fields from its messages (SPEC §7.2.6)."""
+    """Recompute a thread's denormalized fields from its messages (SPEC §7.2.6).
+
+    The thread row is locked *before* its messages are read: two transactions
+    recounting one thread at once (read in the webmail while a sync brings the
+    same flag from a phone) then take turns, and the second reads what the
+    first committed — instead of both reading first and the slower one writing
+    an old count (a message shown read, its thread still "unread"). The
+    messages are reloaded from the database, not taken from the session's
+    copies, which a bulk UPDATE may have left behind."""
+    thread = await session.scalar(
+        select(Thread)
+        .where(Thread.id == thread_id)
+        .with_for_update()
+        .execution_options(populate_existing=True)
+    )
+    if thread is None:
+        return
     messages = list(
         await session.scalars(
             select(Message)
             .where(Message.thread_id == thread_id)
             .order_by(Message.internal_date.desc())
+            .execution_options(populate_existing=True)
         )
     )
-    thread = await session.get(Thread, thread_id)
-    if thread is None:
-        return
     if not messages:
+        # A reply still being written keeps its (now empty) thread: the draft
+        # points at it.
+        if await session.scalar(select(exists().where(Draft.thread_id == thread_id))):
+            thread.message_count = 0
+            thread.unread_count = 0
+            return
         await session.execute(delete(Thread).where(Thread.id == thread_id))
         return
 
@@ -241,3 +261,38 @@ async def thread_message(session: AsyncSession, message: Message) -> uuid.UUID:
     await denormalize_thread(session, thread_id)
     await session.flush()
     return thread_id
+
+
+async def repair_threads(session: AsyncSession, user_id: uuid.UUID) -> int:
+    """Recount the user's threads whose stored counters disagree with their
+    messages (and drop threads left without any). A safety net run nightly
+    (app/indexer/gc.py): every write path recounts its threads, but a missed
+    one would otherwise show a read message as unread until it changed again.
+    Returns how many were repaired."""
+    seen = Message.flags.any("\\Seen")  # type: ignore[arg-type]
+    actual = (
+        select(
+            Message.thread_id.label("tid"),
+            func.count().label("n"),
+            func.count().filter(~seen).label("unread"),
+        )
+        .where(Message.user_id == user_id, Message.thread_id.is_not(None))
+        .group_by(Message.thread_id)
+        .subquery()
+    )
+    stale = list(
+        await session.scalars(
+            select(Thread.id)
+            .outerjoin(actual, actual.c.tid == Thread.id)
+            .where(
+                Thread.user_id == user_id,
+                (actual.c.tid.is_(None) & (Thread.message_count != 0))
+                | (actual.c.tid.is_(None) & ~exists().where(Draft.thread_id == Thread.id))
+                | (actual.c.n != Thread.message_count)
+                | (actual.c.unread != Thread.unread_count),
+            )
+        )
+    )
+    for tid in sorted(stale):
+        await denormalize_thread(session, tid)
+    return len(stale)

backend/app/translation.py

@@ -0,0 +1,423 @@
+"""Email translation, on this server: the translator service (translator/:
+Mozilla's Firefox Translations models and fastText language identification)
+in its own container, reached only by the API. A message's text never leaves
+the server.
+
+- A message's language is detected the first time it is shown (from its own
+  text: quoted replies and signatures left out) and kept on the message, so the
+  reading pane can offer "Translate" when it differs from the interface language.
+- A translation is of the message HTML the user sees (already sanitized; the
+  result is sanitized again), cached per message, target language and whether
+  remote images were shown. It is made paragraph by paragraph, as plain text
+  (translation_prep.split_pieces), each paragraph from its own language.
+- The languages on offer are those installed on the service (admin panel).
+
+Off when ``TRANSLATOR_URL`` is empty.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import logging
+import re
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import httpx
+
+from app import translation_prep as prep
+from app.completion import own_text
+from app.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+DETECT_CHARS = 1500  # of the message's own text: plenty to tell the language
+MIN_CONFIDENCE = 50.0  # below this, the detector is guessing
+MAX_HTML_CHARS = 150_000  # a translation request bigger than this is refused
+_TIMEOUT = httpx.Timeout(120.0, connect=5.0)  # long messages take a while on a CPU
+_LANG_CACHE_S = 60
+
+
+class TranslationError(Exception):
+    """Translation is off, the service is unreachable, or it refused the text."""
+
+
+@dataclass(frozen=True)
+class Language:
+    code: str
+    name: str
+
+
+def enabled() -> bool:
+    return bool(get_settings().translator_url.strip())
+
+
+def _url(path: str) -> str:
+    return get_settings().translator_url.rstrip("/") + path
+
+
+async def _call(method: str, path: str, payload: dict[str, object] | None = None) -> object:
+    if not enabled():
+        raise TranslationError("translation is not set up on this server")
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.request(method, _url(path), json=payload)
+    except httpx.HTTPError as exc:
+        raise TranslationError(f"the translation service is unreachable: {exc}") from exc
+    if resp.status_code not in (200, 202):
+        detail = resp.text[:200]
+        raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
+    return resp.json()
+
+
+async def _post(path: str, payload: dict[str, object]) -> object:
+    return await _call("POST", path, payload)
+
+
+_languages: tuple[float, list[Language]] = (0.0, [])
+
+
+async def languages() -> list[Language]:
+    """The installed languages, English among them (cached for a minute). The
+    browser names them in the interface language; ``name`` is the code."""
+    global _languages
+    if not enabled():
+        return []
+    stamp, cached = _languages
+    if cached and time.monotonic() - stamp < _LANG_CACHE_S:
+        return cached
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
+            resp = await client.get(_url("/languages"))
+        resp.raise_for_status()
+        codes = resp.json()["installed"]
+    except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc:
+        logger.info("translation languages unavailable: %s", exc)
+        return []
+    found = [Language(code=str(c), name=str(c)) for c in codes]
+    _languages = (time.monotonic(), found)
+    return found
+
+
+def forget_languages() -> None:
+    """After an install or removal: ask the service again."""
+    global _languages
+    _languages = (0.0, [])
+
+
+async def status() -> dict[str, object]:
+    """The service's own account (admin panel): installed, installing (bytes
+    done of total), available from Mozilla, and why an install failed."""
+    async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=3.0)) as client:
+        try:
+            resp = await client.get(_url("/languages"))
+            resp.raise_for_status()
+            data = resp.json()
+        except (httpx.HTTPError, ValueError) as exc:
+            raise TranslationError(f"the translation service is unreachable: {exc}") from exc
+    return data if isinstance(data, dict) else {}
+
+
+async def install(code: str) -> None:
+    await _call("POST", f"/languages/{code}")
+    forget_languages()
+
+
+async def remove(code: str) -> None:
+    await _call("DELETE", f"/languages/{code}")
+    forget_languages()
+
+
+_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
+MIN_LETTERS = 25  # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
+MAX_DETECT = 60  # pieces of one message asked about
+MIN_SHARE = 0.15  # of a message's letters, for its language to count
+
+
+async def _detect_many(texts: list[str]) -> list[str]:
+    """The language of each of ``texts`` ('' where unsure), in one request."""
+    if not texts:
+        return []
+    result = await _post("/detect", {"q": [t[:DETECT_CHARS] for t in texts]})
+    if not isinstance(result, list) or len(result) != len(texts):
+        return [""] * len(texts)
+    return [
+        str(r.get("language", ""))
+        if isinstance(r, dict) and float(r.get("confidence", 0)) >= MIN_CONFIDENCE
+        else ""
+        for r in result
+    ]
+
+
+async def _detect_one(text: str) -> str:
+    return (await _detect_many([text]))[0]
+
+
+def _weight(text: str) -> int:
+    """How much text this is: letters, a Chinese/Japanese/Korean sign as a word."""
+    return sum(3 if _CJK.match(c) else 1 for c in text if c.isalpha())
+
+
+async def languages_of(parts: list[str]) -> list[str]:
+    """The language of each of ``parts`` (paragraphs, in order). Parts too
+    short to tell take the language of the next part that could be told (a
+    greeting goes with what follows it), the last ones that of the one before;
+    parts without letters get ''."""
+    # Addresses say nothing about a language ("E-mail: lianshaojuan.lsj@…" → Estonian).
+    parts = [_ADDRESS.sub(" ", part) for part in parts]
+    told: list[str] = [""] * len(parts)
+    asked = [
+        i
+        for i, part in enumerate(parts)
+        if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
+    ][:MAX_DETECT]
+    for i, language in zip(asked, await _detect_many([parts[i] for i in asked]), strict=True):
+        told[i] = language
+    out = list(told)
+    following = ""
+    for i in reversed(range(len(parts))):
+        if told[i]:
+            following = told[i]
+        elif any(c.isalpha() for c in parts[i]):
+            out[i] = following
+    previous = ""
+    for i, part in enumerate(parts):
+        if told[i]:
+            previous = told[i]
+        elif not out[i] and any(c.isalpha() for c in part):
+            out[i] = previous
+    return out
+
+
+async def detect(text: str) -> str:
+    """The languages of ``text`` (a message's plain text), most of it first,
+    comma-separated ("zh-Hans,pl,en"); '' if unsure. A message can be in
+    several: each language that makes up a good part of it counts."""
+    own = own_text(text)
+    if len(" ".join(own.split())) < 20:
+        return ""  # a line or two says too little
+    lines: list[str] = []
+    total = 0
+    for line in (x.strip() for x in own.splitlines()):
+        if line and total < DETECT_CHARS * 4:
+            lines.append(line)
+            total += len(line)
+    found = await languages_of(lines)
+    weight: dict[str, int] = {}
+    for line, language in zip(lines, found, strict=True):
+        if language:
+            weight[language] = weight.get(language, 0) + _weight(_ADDRESS.sub(" ", line))
+    if not weight:
+        return await _detect_one(" ".join(own.split()))
+    all_of_it = sum(weight.values())
+    ranked = sorted(weight, key=lambda k: weight[k], reverse=True)
+    return ",".join(k for k in ranked if weight[k] / all_of_it >= MIN_SHARE)
+
+
+def _base(code: str) -> str:
+    return code.lower().split("-")[0]
+
+
+async def _texts(texts: list[str], source: str, target: str) -> list[str]:
+    """``texts`` translated, in batches."""
+    out: list[str] = []
+    for start in range(0, len(texts), 100):
+        batch = texts[start : start + 100]
+        result = await _post("/translate", {"q": batch, "source": source, "target": target})
+        got = result.get("translatedText") if isinstance(result, dict) else None
+        if not isinstance(got, list) or len(got) != len(batch):
+            raise TranslationError("the translation service sent no translation")
+        out += [str(x) for x in got]
+    return out
+
+
+def _kept(name: str, translated: str) -> bool:
+    """Whether ``name`` came through a translation: each of its words still
+    there, give or take an ending ("Lukas" → "Lukasie": yes; "Milan" →
+    "Mediolan": no)."""
+    return all(
+        word[: max(3, len(word) - 2)] in translated for word in name.split() if len(word) >= 2
+    )
+
+
+# A capitalised word that starts a sentence ("Enclosed is a summary…").
+_SENTENCE_START = re.compile(r"(?:^|(?<=[.!?…]\s))(?P<word>[^\W\d_][^\W\d_'’-]{2,})\b")
+
+
+def _copied_starts(core: str, text: str, names: list[str]) -> list[str]:
+    """Words that start a sentence of ``core`` and came through ``text``
+    untranslated, and are no known name: the models take a capitalised
+    sentence start for a name ("Enclosed is…" → "Enclosed jest…")."""
+    found: list[str] = []
+    for m in _SENTENCE_START.finditer(core):
+        word = m.group("word")
+        if not word[0].isupper() or word.isupper() or any(word in n for n in names):
+            continue
+        if re.search(rf"(?<!\w){re.escape(word)}(?!\w)", text):
+            found.append(word)
+    return found
+
+
+def _lowered(core: str, words: list[str]) -> str:
+    """``core`` with ``words`` lower-cased where they start a sentence."""
+    return _SENTENCE_START.sub(
+        lambda m: m.group(0).lower() if m.group("word") in words else m.group(0), core
+    )
+
+
+def _capitalised(text: str) -> str:
+    """Sentence starts capitalised again (after ``_lowered``): the first
+    letter, and one after a sentence's end — a full stop after a word of four
+    letters or more, so not after "r.", "np.", "tzw." (abbreviations)."""
+    text = re.sub(r"^(\W*)([^\W\d_])", lambda m: m.group(1) + m.group(2).upper(), text)
+    return re.sub(
+        r"((?:[^\W\d_]{4,}[.…]|[!?])\s+)([^\W\d_])",
+        lambda m: m.group(1) + m.group(2).upper(),
+        text,
+    )
+
+
+def _as_before(text: str, first: str) -> str:
+    """A word after a comma capitalised in ``text`` but not in ``first`` (the
+    first try) is lower-cased: lower-cased input makes the models capitalise
+    elsewhere ("…2026, od 13 lipca" → "…2026 r., Od 13 lipca")."""
+
+    def repl(m: re.Match[str]) -> str:
+        word = m.group(2)
+        lower = word[0].lower() + word[1:]
+        return m.group(1) + (lower if f", {lower}" in first else word)
+
+    return re.sub(r"(,\s+)([^\W\d_][\w'’-]*)", repl, text)
+
+
+async def _with_names_kept(core: str, names: list[str], source: str, target: str) -> str:
+    """``core`` translated around ``names``, which are left as they are."""
+    pattern = re.compile("(" + "|".join(re.escape(n) for n in names) + ")")
+    parts = pattern.split(core)
+    todo = [
+        (i, m.groups())
+        for i, part in enumerate(parts)
+        if i % 2 == 0 and (m := _EDGE_SPACE.match(part)) and any(c.isalnum() for c in part)
+    ]
+    done = await _texts([groups[1] for _, groups in todo], source, target)
+    for (i, (lead, _, trail)), text in zip(todo, done, strict=True):
+        parts[i] = f"{lead}{text}{trail}"
+    return "".join(parts)
+
+
+async def _translate_paragraphs(
+    html: str, target: str, sources: list[str], names: Iterable[str] = ()
+) -> str:
+    """``html`` in ``target``, as plain text paragraph by paragraph: from the
+    one language in ``sources``, or (several, or none known) each paragraph
+    from its own. Paragraphs already in ``target``, or in a language that is
+    not installed, are left as they are.
+
+    ``names`` go along with their sentence, which then translates as a whole
+    (kept apart, "I came across <Imaginary Pixels>" lost its verb: "Przyszedłem
+    Imaginary Pixels"). Only a piece where one came out changed ("Milan" →
+    "Mediolan") is translated again with it left as it is."""
+    watch = sorted({n.strip() for n in names if len(n.strip()) >= 2}, key=len, reverse=True)
+    pieces, paragraphs = prep.split_pieces(html)
+    if len(sources) == 1:
+        found = [sources[0]] * len(paragraphs)
+    else:
+        found = await languages_of([p.text for p in paragraphs])
+    installed = {x.code for x in await languages()}
+    # Chinese and Japanese go without spaces: two pieces of one sentence
+    # ("…奎琳，</span>我从…") need one between them once translated.
+    followed = {i for p in paragraphs for i in p.pieces[:-1]}
+    by_language: dict[str, list[int]] = {}
+    for paragraph, language in zip(paragraphs, found, strict=True):
+        if language in installed and _base(language) != _base(target):
+            by_language.setdefault(language, []).extend(paragraph.pieces)
+    for language, where in by_language.items():
+        todo: list[tuple[int, str, str, str]] = []
+        for i in where:
+            m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
+            lead, core, trail = m.groups() if m else ("", "", "")
+            # A list marker stays out: the models repeat it ("1. Thursday" →
+            # "1. 1. Czwartek").
+            marker = _LIST_MARKER.match(core)
+            if marker:
+                lead, core = lead + marker.group(0), core[marker.end() :]
+            if not trail and i in followed and _base(language) in _NO_SPACES:
+                trail = " "
+            if any(c.isalnum() for c in core):
+                todo.append((i, lead, core, trail))
+        done = await _texts([core for _, _, core, _ in todo], language, target)
+        # Sentence starts that came through untranslated: once more, lower-case.
+        again = [
+            (k, words)
+            for k, ((_, _, core, _), text) in enumerate(zip(todo, done, strict=True))
+            if (words := _copied_starts(core, text, watch))
+        ]
+        if again:
+            retried = await _texts(
+                [_lowered(todo[k][2], words) for k, words in again], language, target
+            )
+            for (k, words), text in zip(again, retried, strict=True):
+                # Still there: a name after all, and the first try stands.
+                if len(_copied_starts(todo[k][2], text, watch)) < len(words):
+                    done[k] = _as_before(_capitalised(text), done[k])
+        for (i, lead, core, trail), text in zip(todo, done, strict=True):
+            lost = [n for n in watch if n in core and not _kept(n, text)]
+            if lost:
+                text = await _with_names_kept(core, lost, language, target)
+            pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
+    return "".join(pieces)
+
+
+_NO_SPACES = {"zh", "ja", "th"}
+_LIST_MARKER = re.compile(r"(?:\d{1,3}[.)]|[a-zA-Z][.)]|[-•*–])\s+(?=\S)")
+_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
+
+
+# Web and mail addresses in the text: the models break them up ("https: / /
+# vimeo.com"), so they are marked translate="no" and not sent.
+_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
+_TAG = re.compile(r"(<[^>]*>)")
+
+
+def protect_addresses(html: str) -> str:
+    """Mark the addresses in ``html``'s text as not to be translated (style
+    blocks and tag attributes are left alone)."""
+    out: list[str] = []
+    in_style = False
+    for piece in _TAG.split(html):
+        if piece.startswith("<"):
+            lowered = piece.lower()
+            if lowered.startswith("<style"):
+                in_style = True
+            elif lowered.startswith("</style"):
+                in_style = False
+            out.append(piece)
+        elif in_style or not piece:
+            out.append(piece)
+        else:
+            out.append(_ADDRESS.sub(lambda m: f'<span translate="no">{m.group(0)}</span>', piece))
+    return "".join(out)
+
+
+# Bumped when what goes to the service, or what is done with the result,
+# changes: translations kept from before are made again.
+PIPELINE_VERSION = 8
+
+
+async def translate_html(
+    html: str, source: str, target: str, keep: Iterable[str] = (), names: Iterable[str] = ()
+) -> str:
+    """``html`` (a sanitized message body) in ``target``; source "auto", a code,
+    or several ("zh-Hans,pl,en": then each paragraph from its own language).
+    ``keep``: words always left as they are; ``names``: left as they are if
+    the model would change them (translation_prep)."""
+    if len(html) > MAX_HTML_CHARS:
+        raise TranslationError("this message is too long to translate")
+    sources = [x for x in (source or "").split(",") if x and x != "auto"]
+    prepared = prep.join_wrapped_lines(html)
+    for language in sources:
+        prepared = prep.translate_phrases(prepared, language, target)
+    candidates = {*names, *prep.find_names(prepared, sources[0] if sources else "")}
+    prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, keep)))
+    return prep.tidy(await _translate_paragraphs(prepared, target, sources, candidates))

backend/app/translation_prep.py

@@ -0,0 +1,332 @@
+"""Getting a message ready for translation, and tidying what comes back.
+
+A message is translated as plain text, piece by piece (the text between
+tags), and the models stumble on things a reader would not: a sentence
+wrapped onto two lines becomes two sentences ("ask you about" / "Plan for next
+month"), "Best," becomes "Najlepsze," and a company name gets translated word
+by word. So before a message goes out:
+
+- lines that carry on a sentence (no end punctuation, next starts lowercase)
+  are joined, in HTML (<br>, one <div> per line) and in wrapped plain text;
+- greetings and sign-offs that make a whole line are put in the target
+  language from a small phrase table (English, Polish, German);
+- names are marked translate="no" and not sent — the people on the message,
+  the user's own names and signature names, and capitalised multi-word names
+  mid-sentence — but only the names the model would change ("Milan" →
+  "Mediolan"): a kept name splits its sentence into pieces translated alone
+  ("Dear <kept>Lukas Remis</kept>," → "Kochanie."), so a name the model
+  leaves alone stays in the sentence;
+- so are emoji, file names ("A_0023C017_p1CAC.mov") and text without a letter;
+- the message is translated paragraph by paragraph (split_pieces), each from
+  its own language when there are several; those in the target stay;
+- and afterwards, a kept word gets back the spaces around it.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import re
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+
+KEEP_OPEN = '<span translate="no">'
+_TAG = re.compile(r"(<[^>]*>)")
+_LOWER_START = "a-ząćęłńóśźżäöüßàâçéèêëîïôûùüÿñ"
+_UPPER = "A-ZĄĆĘŁŃÓŚŹŻÄÖÜÀÂÇÉÈÊËÎÏÔÛÙÜŸÑ"
+
+# --- joining wrapped lines -------------------------------------------------------
+
+# A line break between a word (or comma) and a lowercase word carries a sentence on.
+_HTML_SOFT_BREAK = re.compile(
+    rf"([\w,;–-])[ \t]*(?:<br\s*/?>|</div>\s*<div[^>]*>|</p>\s*<p[^>]*>)\s*(?=[{_LOWER_START}])"
+)
+_TEXT_SOFT_BREAK = re.compile(rf"([^\s.!?:;])[ \t]*\n[ \t]*(?=[{_LOWER_START}])")
+
+
+def join_wrapped_lines(html: str) -> str:
+    html = _HTML_SOFT_BREAK.sub(r"\1 ", html)
+    # Plain text (wm-plaintext): newlines inside text, outside tags.
+    return "".join(
+        p if p.startswith("<") else _TEXT_SOFT_BREAK.sub(r"\1 ", p) for p in _TAG.split(html)
+    )
+
+
+# --- greetings and sign-offs -----------------------------------------------------
+
+# source language → the line as written (lower case) → target language → line.
+_PHRASES: dict[str, dict[str, dict[str, str]]] = {
+    "en": {
+        "best": {"pl": "Pozdrawiam", "de": "Viele Grüße"},
+        "best regards": {"pl": "Pozdrawiam serdecznie", "de": "Beste Grüße"},
+        "best wishes": {"pl": "Najlepsze życzenia", "de": "Beste Grüße"},
+        "kind regards": {"pl": "Z pozdrowieniami", "de": "Freundliche Grüße"},
+        "warm regards": {"pl": "Serdecznie pozdrawiam", "de": "Herzliche Grüße"},
+        "regards": {"pl": "Pozdrawiam", "de": "Grüße"},
+        "cheers": {"pl": "Pozdrawiam", "de": "Viele Grüße"},
+        "sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"},
+        "yours sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"},
+        "thanks": {"pl": "Dzięki", "de": "Danke"},
+        "many thanks": {"pl": "Wielkie dzięki", "de": "Vielen Dank"},
+        "thank you": {"pl": "Dziękuję", "de": "Vielen Dank"},
+        "thanks in advance": {"pl": "Z góry dziękuję", "de": "Vielen Dank im Voraus"},
+        "all the best": {"pl": "Wszystkiego dobrego", "de": "Alles Gute"},
+        "talk soon": {"pl": "Do usłyszenia", "de": "Bis bald"},
+        "have a great day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"},
+        "have a nice day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"},
+    },
+    "pl": {
+        "pozdrawiam": {"en": "Best regards", "de": "Viele Grüße"},
+        "pozdrawiam serdecznie": {"en": "Warm regards", "de": "Herzliche Grüße"},
+        "serdecznie pozdrawiam": {"en": "Warm regards", "de": "Herzliche Grüße"},
+        "z poważaniem": {"en": "Kind regards", "de": "Mit freundlichen Grüßen"},
+        "z pozdrowieniami": {"en": "Kind regards", "de": "Freundliche Grüße"},
+        "dziękuję": {"en": "Thank you", "de": "Vielen Dank"},
+        "dzięki": {"en": "Thanks", "de": "Danke"},
+        "z góry dziękuję": {"en": "Thanks in advance", "de": "Vielen Dank im Voraus"},
+        "miłego dnia": {"en": "Have a nice day", "de": "Einen schönen Tag noch"},
+        "do usłyszenia": {"en": "Talk soon", "de": "Bis bald"},
+    },
+    "de": {
+        "mit freundlichen grüßen": {"en": "Kind regards", "pl": "Z poważaniem"},
+        "freundliche grüße": {"en": "Kind regards", "pl": "Z pozdrowieniami"},
+        "viele grüße": {"en": "Best regards", "pl": "Pozdrawiam"},
+        "beste grüße": {"en": "Best regards", "pl": "Pozdrawiam"},
+        "liebe grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"},
+        "herzliche grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"},
+        "danke": {"en": "Thanks", "pl": "Dzięki"},
+        "vielen dank": {"en": "Many thanks", "pl": "Bardzo dziękuję"},
+        "bis bald": {"en": "Talk soon", "pl": "Do usłyszenia"},
+    },
+}
+# A whole line: after a tag or line start, before a tag or line end.
+_LINE = re.compile(r"(^|>|\n)([ \t]*)([^<>\n]{2,40}?)([ \t]*[,.!]?)([ \t]*)(?=<|\n|$)")
+
+
+def _base(code: str) -> str:
+    return code.lower().split("-")[0]
+
+
+def translate_phrases(html: str, source: str, target: str) -> str:
+    table = _PHRASES.get(_base(source), {})
+    if not table:
+        return html
+    tgt = _base(target)
+
+    def repl(m: re.Match[str]) -> str:
+        found = table.get(" ".join(m.group(3).lower().split()), {}).get(tgt)
+        if found is None:
+            return m.group(0)
+        return f"{m.group(1)}{m.group(2)}{KEEP_OPEN}{found}{m.group(4).strip()}</span>{m.group(5)}"
+
+    # The pattern never crosses a tag: it stops at "<" and ">".
+    return _LINE.sub(repl, html)
+
+
+# --- names ------------------------------------------------------------------------
+
+# A capitalised word; dots only inside it ("U.S"), never at its end — a name
+# does not run on over a sentence's end ("NukeX. My work…").
+_WORD = rf"[{_UPPER}][\w'’&-]*(?:\.[\w'’&-]+)*"
+# Two or more capitalised words in a row ("Imaginary Pixels", "Trusted Partner
+# Network"), after a lowercase word: a name, not the start of a sentence.
+_MULTI_NAME = re.compile(
+    rf"(?<=[{_LOWER_START},] )({_WORD}(?: (?:&|of|the|de|von|van|{_WORD}))* {_WORD})"
+)
+# Languages that capitalise every noun: capitals say nothing about names there.
+_CAPITALISE_NOUNS = {"de", "lb"}
+
+
+def _title_case(text: str) -> bool:
+    """A Headline Like This: nearly every longer word capitalised (short ones
+    — of, and, in — often are not). A sentence full of names is not one."""
+    words = [w for w in re.findall(r"[^\W\d_][\w'’-]*", text) if len(w) >= 4]
+    return len(words) >= 3 and sum(w[0].isupper() for w in words) / len(words) > 0.8
+
+
+def find_names(html: str, source: str) -> set[str]:
+    """Capitalised multi-word names in the middle of sentences (not in headings
+    or Title Case lines, and not for languages that capitalise all nouns)."""
+    if _base(source) in _CAPITALISE_NOUNS:
+        return set()
+    found: set[str] = set()
+    for piece in _TAG.split(html):
+        if piece.startswith("<") or _title_case(piece):
+            continue
+        found.update(m.group(1).rstrip(".") for m in _MULTI_NAME.finditer(piece))
+    return found
+
+
+_SIG_NAME = re.compile(rf"{_WORD}(?: (?:&|of|the|{_WORD}))* {_WORD}")
+
+
+def find_names_in_signature(text: str) -> list[str]:
+    """Capitalised multi-word names in a signature ("Imaginary Pixels"): the
+    user's company or title, which a translation should leave alone."""
+    return [m.group(0).rstrip(".") for m in _SIG_NAME.finditer(text)]
+
+
+def keep_terms(html: str, terms: Iterable[str]) -> str:
+    """Mark each of ``terms`` in ``html``'s text translate="no" (longest first;
+    whole words; style blocks and tags left alone)."""
+    words = sorted({t.strip() for t in terms if len(t.strip()) >= 2}, key=len, reverse=True)
+    if not words:
+        return html
+    pattern = re.compile(r"(?<![\w>])(" + "|".join(re.escape(w) for w in words) + r")(?![\w<])")
+    return _mark(html, pattern)
+
+
+# Emoji and other pictographs (with their skin tones, joiners and variation
+# selectors): on its own, an emoji comes back as a made-up word ("😊" →
+# "Uzasadnienie").
+_SYMBOLS = re.compile(
+    "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff"
+    "\U0001f000-\U0001faff]"
+    "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff"
+    "\U0001f000-\U0001faff\ufe0f\u200d\u20e3]*"
+)
+# File names and codes with underscores ("A_0023C017_260521_p1CAC.mov",
+# "final_v2"): the models split them at every "_" and dot.
+_FILE_NAME = re.compile(
+    r"(?<![\w./-])(?:[\w-]*_[\w-]*(?:\.[\w-]+)*|[\w-]+(?:\.[\w-]+)*\.[a-z][a-z0-9]{1,4})(?![\w/-])",
+    re.IGNORECASE,
+)
+# International phone numbers ("+48 780 549 877"): the models space out the
+# "+". Only those: a kept piece splits its sentence, and plain numbers
+# ("1300 - 2500 Euro") come through anyway.
+_PHONE = re.compile(r"(?<![\w+])\+\d[\d ().-]{5,}\d(?!\w)")
+# Only names with an extension a file would have, not every "e.g." or "U.S".
+_FILE_EXT = re.compile(r"\.[a-z][a-z0-9]{1,4}$", re.IGNORECASE)
+
+
+def keep_symbols(html: str) -> str:
+    """Mark emoji, file names, handles ("Jacqueline_"), phone numbers and text
+    without a letter in ``html`` translate="no"."""
+    html = _mark(_mark(html, _SYMBOLS), _PHONE)
+
+    def is_file(m: re.Match[str]) -> bool:
+        word = m.group(0)
+        if "_" in word and word.strip("_"):
+            return True
+        return bool(_FILE_EXT.search(word)) and any(c.isdigit() or c in "_-" for c in word)
+
+    return _map_text(_mark(html, _FILE_NAME, keep=is_file), _keep_letterless)
+
+
+_ENTITY = re.compile(r"&#?\w+;")
+_EDGES = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
+
+
+def _keep_letterless(text: str) -> str:
+    """A piece of text with no letter or digit — an invisible byte-order mark,
+    a lone symbol — kept as it is: the models make up a word for it ("\ufeff"
+    → "Uzasadnienie")."""
+    if any(c.isalnum() for c in _ENTITY.sub("", text)):
+        return text
+    lead, core, trail = _EDGES.match(text).groups()  # type: ignore[union-attr]
+    if not _ENTITY.sub("", core).strip():
+        return text  # only spaces and &nbsp;
+    return f"{lead}{KEEP_OPEN}{core}</span>{trail}"
+
+
+def _mark(
+    html: str, pattern: re.Pattern[str], keep: Callable[[re.Match[str]], bool] | None = None
+) -> str:
+    """Wrap ``pattern``'s matches in ``html``'s text in translate="no" spans
+    (where ``keep(match)`` says so)."""
+
+    def repl(m: re.Match[str]) -> str:
+        if keep is not None and not keep(m):
+            return m.group(0)
+        return f"{KEEP_OPEN}{m.group(0)}</span>"
+
+    return _map_text(html, lambda text: pattern.sub(repl, text))
+
+
+def _map_text(html: str, fn: Callable[[str], str]) -> str:
+    """``fn`` applied to each piece of ``html``'s text; style blocks, tags and
+    text already kept are left alone."""
+    out: list[str] = []
+    in_style = in_keep = False
+    for piece in _TAG.split(html):
+        if piece.startswith("<"):
+            low = piece.lower()
+            if low.startswith("<style"):
+                in_style = True
+            elif low.startswith("</style"):
+                in_style = False
+            elif low.startswith(KEEP_OPEN):
+                in_keep = True
+            elif in_keep and low.startswith("</span"):
+                in_keep = False
+            out.append(piece)
+        elif in_style or in_keep or not piece:
+            out.append(piece)
+        else:
+            out.append(fn(piece))
+    return "".join(out)
+
+
+# --- messages in more than one language ------------------------------------------
+
+# Tags that end a paragraph (a line): the text between two of them is one
+# piece of one language.
+_BLOCK = re.compile(
+    r"</?(?:p|div|li|ul|ol|table|thead|tbody|tfoot|tr|td|th|h[1-6]|blockquote|pre|"
+    r"section|article|header|footer|center|br|hr)\b",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class Paragraph:
+    """One paragraph of a message: its text (for telling its language) and
+    where its translatable pieces are in ``split_pieces``' list."""
+
+    text: str = ""
+    pieces: list[int] = field(default_factory=list)
+
+
+def split_pieces(html: str) -> tuple[list[str], list[Paragraph]]:
+    """``html`` as a list of tags and text, and its paragraphs. Text in style
+    blocks is no paragraph's; text already kept counts for the language but
+    is not translated."""
+    pieces = _TAG.split(html)
+    paragraphs: list[Paragraph] = [Paragraph()]
+    in_style = in_keep = False
+    for i, piece in enumerate(pieces):
+        if piece.startswith("<"):
+            low = piece.lower()
+            if low.startswith("<style"):
+                in_style = True
+            elif low.startswith("</style"):
+                in_style = False
+            elif low.startswith(KEEP_OPEN):
+                in_keep = True
+            elif in_keep and low.startswith("</span"):
+                in_keep = False
+            if _BLOCK.match(piece) and (paragraphs[-1].text.strip() or paragraphs[-1].pieces):
+                paragraphs.append(Paragraph())
+        elif piece and not in_style:
+            paragraphs[-1].text += html_lib.unescape(piece)
+            if not in_keep:
+                paragraphs[-1].pieces.append(i)
+    return pieces, [p for p in paragraphs if p.text.strip()]
+
+
+# --- after translation -------------------------------------------------------------
+
+# A kept span can lose the space beside it ("Bruton</span>i twoje").
+# Only kept spans: they still carry translate="no" here (the sanitizer that
+# runs next takes it off); other spans are the sender's and stay as they are.
+_KEPT = re.compile(r'(<span translate="no">[^<]*</span>)')
+
+
+def tidy(html: str) -> str:
+    def repl(m: re.Match[str]) -> str:
+        before = html[m.start() - 1] if m.start() > 0 else ""
+        after = html[m.end()] if m.end() < len(html) else ""
+        lead = " " if re.match(r"[\w,;:)]", before) else ""
+        trail = " " if re.match(r"[\w(„\"“]", after) else ""
+        return f"{lead}{m.group(1)}{trail}"
+
+    return _KEPT.sub(repl, html)

backend/app/ws.py

@@ -18,6 +18,7 @@ from app.events import get_event_bus
 from app.identity.cookies import SESSION_COOKIE_INSECURE, SESSION_COOKIE_SECURE
 from app.identity.sessions import resolve_session
 from app.redis_client import get_redis
+from app.security.cross_site import is_cross_site
 from app.sync import leases
 
 logger = logging.getLogger(__name__)
@@ -27,6 +28,12 @@ router = APIRouter()
 
 @router.websocket("/ws")
 async def ws_endpoint(websocket: WebSocket) -> None:
+    # Only our own pages may listen in: a socket another site opens is refused
+    # even if the browser sent the session cookie with it (cross-site WebSocket
+    # hijacking; SameSite=Lax already stops current browsers from sending it).
+    if is_cross_site(websocket.headers):
+        await websocket.close(code=1008)
+        return
     token = websocket.cookies.get(SESSION_COOKIE_SECURE) or websocket.cookies.get(
         SESSION_COOKIE_INSECURE, ""
     )
@@ -55,6 +62,10 @@ async def ws_endpoint(websocket: WebSocket) -> None:
             payload = data.get("payload", {})
             if str(payload.get("user_id")) == user_id:
                 await websocket.send_json(data)
+                if data.get("event") == "user.signed_out":
+                    # The session this socket was opened with no longer exists.
+                    await websocket.close(code=1008)
+                    return
 
     async def receive() -> None:
         while True:

backend/migrations/versions/0019_user_blocked.py

@@ -0,0 +1,29 @@
+"""administrator can block a user
+
+``users.blocked_at`` is set while an administrator has blocked the account: it
+cannot sign in and its sessions stop working. NULL means not blocked. A
+timestamp rather than a flag so the admin panel can say since when.
+
+Revision ID: 0019
+Revises: 0018
+Create Date: 2026-09-30
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0019"
+down_revision: str | None = "0018"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE users ADD COLUMN blocked_at timestamptz")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE users DROP COLUMN IF EXISTS blocked_at")

backend/migrations/versions/0020_system_mail.py

@@ -0,0 +1,48 @@
+"""outgoing system mail (password reset) configured in the admin panel
+
+The server sends its own mail — today only password-reset links — through an
+SMTP account the administrator sets up in the admin panel. The password is
+envelope-encrypted with MASTER_KEY like every mail-account secret. ``public_url``
+is where users reach the app; reset links are built from it, never from the
+request's Host header, which an attacker controls.
+
+Revision ID: 0020
+Revises: 0019
+Create Date: 2026-09-30
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0020"
+down_revision: str | None = "0019"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_COLUMNS = (
+    ("smtp_host", "text NOT NULL DEFAULT ''"),
+    ("smtp_port", "integer NOT NULL DEFAULT 587"),
+    ("smtp_security", "text NOT NULL DEFAULT 'starttls'"),
+    ("smtp_username", "text NOT NULL DEFAULT ''"),
+    ("smtp_secret_ct", "bytea"),
+    ("smtp_from", "text NOT NULL DEFAULT ''"),
+    ("public_url", "text NOT NULL DEFAULT ''"),
+)
+
+
+def upgrade() -> None:
+    for name, ddl in _COLUMNS:
+        op.execute(f"ALTER TABLE app_settings ADD COLUMN {name} {ddl}")
+    op.execute(
+        "ALTER TABLE app_settings ADD CONSTRAINT ck_app_settings_smtp_security "
+        "CHECK (smtp_security IN ('starttls', 'ssl', 'none'))"
+    )
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE app_settings DROP CONSTRAINT IF EXISTS ck_app_settings_smtp_security")
+    for name, _ in reversed(_COLUMNS):
+        op.execute(f"ALTER TABLE app_settings DROP COLUMN IF EXISTS {name}")

backend/migrations/versions/0021_retries_and_account_health.py

@@ -0,0 +1,43 @@
+"""send retries for 24 hours, and an account's sync health
+
+A send the mail server could not take (SMTP unreachable, a temporary 4xx) is no
+longer failed at once: the outbox row goes to ``retrying`` and is tried again
+with growing gaps until ``retry_until`` (24 hours after the first failure).
+``attempts`` counts the tries, ``next_attempt_at`` is when the next one runs.
+
+An account whose server cannot be reached for a while carries ``sync_error``
+(``unreachable``) and ``sync_error_since``; both clear on the next success.
+
+Revision ID: 0021
+Revises: 0020
+Create Date: 2026-10-06
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0021"
+down_revision: str | None = "0020"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_COLUMNS = (
+    ("outbox", "attempts", "integer NOT NULL DEFAULT 0"),
+    ("outbox", "next_attempt_at", "timestamptz"),
+    ("outbox", "retry_until", "timestamptz"),
+    ("accounts", "sync_error", "text"),
+    ("accounts", "sync_error_since", "timestamptz"),
+)
+
+
+def upgrade() -> None:
+    for table, name, ddl in _COLUMNS:
+        op.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
+
+
+def downgrade() -> None:
+    for table, name, _ in reversed(_COLUMNS):
+        op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS {name}")

backend/migrations/versions/0022_push_notifications.py

@@ -0,0 +1,54 @@
+"""push notifications about new mail (Web Push, RFC 8030/8291/8292)
+
+``push_subscriptions`` holds one row per browser (or installed app) that asked
+to be told about new mail: the push service URL it gave, and the two keys the
+message is encrypted to. ``session_sid`` is the sign-in it was made under — once
+that session is gone (signed out, revoked, expired) nothing is sent there any
+more, so a signed-out device never shows a subject line again.
+
+The server signs every push with its own key pair (VAPID). It is generated on
+first use and kept in ``app_settings``; the private half is vault-encrypted
+with MASTER_KEY like every other server secret.
+
+Revision ID: 0022
+Revises: 0021
+Create Date: 2026-10-08
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0022"
+down_revision: str | None = "0021"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE push_subscriptions (
+          id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+          user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          endpoint text NOT NULL UNIQUE,
+          p256dh text NOT NULL,
+          auth text NOT NULL,
+          session_sid text NOT NULL,
+          user_agent text NOT NULL DEFAULT '',
+          created_at timestamptz NOT NULL DEFAULT now(),
+          last_used_at timestamptz
+        )
+        """
+    )
+    op.execute("CREATE INDEX ix_push_subscriptions_user ON push_subscriptions (user_id)")
+    op.execute("ALTER TABLE app_settings ADD COLUMN vapid_public text NOT NULL DEFAULT ''")
+    op.execute("ALTER TABLE app_settings ADD COLUMN vapid_private_ct bytea")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE app_settings DROP COLUMN IF EXISTS vapid_private_ct")
+    op.execute("ALTER TABLE app_settings DROP COLUMN IF EXISTS vapid_public")
+    op.execute("DROP TABLE IF EXISTS push_subscriptions")

backend/migrations/versions/0023_attachment_part_index.py

@@ -0,0 +1,36 @@
+"""attachments are kept once: inside their message
+
+An attachment used to be stored twice — inside the raw message blob, and again
+as a blob of its own. Now only the raw message is kept; ``part_index`` says
+which MIME part of it (its position in a walk of the message) holds the
+attachment, so it can be taken out when downloaded. ``blob_sha256`` stays: the
+attachment's content hash, which names its thumbnail and checks what is taken
+out. Rows stored before this have no index; they are found by that hash.
+
+Existing separate copies are removed by a background pass on the worker
+(app/indexer/attachments.py ``dedupe_existing``), only after checking each one
+can be taken out of its message again.
+
+Revision ID: 0023
+Revises: 0022
+Create Date: 2026-10-08
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0023"
+down_revision: str | None = "0022"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE attachments ADD COLUMN part_index integer")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE attachments DROP COLUMN IF EXISTS part_index")

backend/migrations/versions/0024_app_passwords.py

@@ -0,0 +1,43 @@
+"""app passwords: one password per mail app (SPEC App. D §D.5)
+
+Mail apps (Thunderbird, a phone's Mail app) will sign in to the user's hosted
+mailbox over IMAP/SMTP, where no 2FA code can be asked for. Each app gets its
+own generated password instead, which can be revoked on its own; the main
+password with 2FA stays for the webmail. Stored as argon2id hashes, the form
+the mail server's SQL directory reads (§D.3).
+
+Revision ID: 0024
+Revises: 0023
+Create Date: 2026-10-09
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0024"
+down_revision: str | None = "0023"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE user_app_passwords (
+          id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+          user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          name text NOT NULL,
+          password_hash text NOT NULL,
+          created_at timestamptz NOT NULL DEFAULT now(),
+          last_used_at timestamptz
+        )
+        """
+    )
+    op.execute("CREATE INDEX ix_user_app_passwords_user ON user_app_passwords (user_id)")
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS user_app_passwords")

backend/migrations/versions/0025_hosted_mailboxes.py

@@ -0,0 +1,75 @@
+"""hosted mailboxes: native accounts and the mail server's directory (SPEC App. D)
+
+* ``accounts.kind`` — ``external`` (any IMAP/SMTP server, the user's own
+  password in the vault) or ``native`` (the user's mailbox on the hosted mail
+  server: permanent, signed in to without any stored password, §D.4).
+* ``mail_directory`` — the view the mail server's SQL directory reads (§D.3):
+  one row per user, its address — and no password: the webmail password never
+  opens the mailbox in a mail app, only app passwords do (the mail server's
+  own, §D.5) — and ``active`` false for a blocked user. Plus the service accounts of
+  ``mail_service_accounts``: the webmail's own worker, which may act for any
+  mailbox and nothing else.
+* ``user_app_passwords.server_id`` — the mail server's id of the same app
+  password (§D.5), to revoke it there too.
+
+The mail server reads the view as its own role, given SELECT on it alone by
+``app.hosted.provision`` (the role is created there, with its password).
+
+Revision ID: 0025
+Revises: 0024
+Create Date: 2026-10-11
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0025"
+down_revision: str | None = "0024"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE accounts ADD COLUMN kind text NOT NULL DEFAULT 'external'")
+    op.execute(
+        "ALTER TABLE accounts ADD CONSTRAINT ck_accounts_kind "
+        "CHECK (kind IN ('external', 'native'))"
+    )
+    # One mailbox per user.
+    op.execute("CREATE UNIQUE INDEX uq_accounts_native ON accounts (user_id) WHERE kind = 'native'")
+    op.execute("ALTER TABLE user_app_passwords ADD COLUMN server_id text")
+    op.execute(
+        """
+        CREATE TABLE mail_service_accounts (
+          name citext PRIMARY KEY,
+          secret text NOT NULL,
+          description text NOT NULL DEFAULT ''
+        )
+        """
+    )
+    op.execute(
+        """
+        CREATE VIEW mail_directory AS
+          SELECT u.handle AS name,
+                 NULL::text AS secret,
+                 'individual'::text AS type,
+                 u.handle::text AS description,
+                 u.blocked_at IS NULL AS active
+            FROM users u
+          UNION ALL
+          SELECT s.name, s.secret, 'individual'::text, s.description, true
+            FROM mail_service_accounts s
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW IF EXISTS mail_directory")
+    op.execute("DROP TABLE IF EXISTS mail_service_accounts")
+    op.execute("ALTER TABLE user_app_passwords DROP COLUMN IF EXISTS server_id")
+    op.execute("DROP INDEX IF EXISTS uq_accounts_native")
+    op.execute("ALTER TABLE accounts DROP CONSTRAINT IF EXISTS ck_accounts_kind")
+    op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS kind")