Loading splash: animated logo over a dark overlay on desktop

index.html carries the overlay markup and styles inline so it paints before
any script; public/splash.js (external, for the CSP) plays the logo
animation from icons/ultimo-splash.html, holds, fades out after ~3.25 s and
removes itself. Skipped at phone sizes (same query as useIsMobile) and in
pop-out compose windows; a click skips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.env.example

@@ -21,3 +21,5 @@ RSPAMD_CONTROLLER_PASSWORD=devpass
 
 # Spelling/grammar check in the compose window (LanguageTool). Empty = off.
 LANGUAGETOOL_URL=http://127.0.0.1:8010
+# Email translation (the translator service, translator/). Empty = off.
+TRANSLATOR_URL=http://127.0.0.1:5005

Makefile

@@ -1,5 +1,5 @@
 up:        ## start services
-	docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool
+	docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool translator
 down:      ## stop services (data kept)
 	docker compose -f docker/docker-compose.yml down
 reset-db:  ## DESTRUCTIVE: wipe db + search + blobs, re-migrate

backend/app/accounts/discovery.py

@@ -125,9 +125,7 @@ def parse_autoconfig_xml(xml_text: str) -> CandidateConfig | None:
     smtp = _server("outgoingServer", "smtp")
     if imap is None or smtp is None:
         return None
-    return CandidateConfig(
-        imap, smtp, "autoconfig", auth_strategy=_strategy_from_auth(imap.auth)
-    )
+    return CandidateConfig(imap, smtp, "autoconfig", auth_strategy=_strategy_from_auth(imap.auth))
 
 
 # --- DNS SRV (RFC 6186) ------------------------------------------------------

backend/app/accounts/schemas.py

@@ -102,3 +102,4 @@ class AccountDetailOut(BaseModel):
     backfill_done: int
     imap_config: dict[str, object]
     smtp_config: dict[str, object]
+    kind: str = "external"

backend/app/accounts/service.py

@@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.accounts.schemas import CreateAccountRequest
 from app.accounts.types import ServerConfig
 from app.indexer import blobstore
-from app.models import Account, Attachment, Message, Thread
+from app.models import Account, Attachment, ComposeAttachment, Message, Thread
 from app.security.crypto import encrypt_secret
 
 
@@ -127,6 +127,13 @@ async def delete_account(
                 .where(Attachment.user_id == user_id, Attachment.blob_sha256 == sha)
                 .limit(1)
             )
+        if still_used is None:
+            # A forwarded file waiting in a draft shares the hash: keep it for that.
+            still_used = await session.scalar(
+                select(ComposeAttachment.id)
+                .where(ComposeAttachment.user_id == user_id, ComposeAttachment.blob_sha256 == sha)
+                .limit(1)
+            )
         if still_used is None:
             blobstore.remove(user_id, sha)
     return True

backend/app/accounts/ssrf.py

@@ -2,9 +2,10 @@
 
 All autodiscovery HTTP is HTTPS-only, 5 s timeout, response capped at 256 KB, no
 off-domain redirects, and the target host must resolve exclusively to public IPs
-(reject private/loopback/link-local/reserved ranges). A residual TOCTOU gap
-(DNS rebinding between our resolve and httpx's own resolve) is accepted for M1;
-a pinned-IP transport can close it later.
+(reject private/loopback/link-local/reserved ranges). The request then goes to
+the very address that was checked — the name is not looked up a second time —
+so a DNS answer that changes in between (DNS rebinding) cannot point it inward.
+TLS is still verified against the host name.
 """
 
 from __future__ import annotations
@@ -26,15 +27,12 @@ class SsrfError(Exception):
 
 
 def _is_public_ip(ip: str) -> bool:
-    addr = ipaddress.ip_address(ip)
-    return not (
-        addr.is_private
-        or addr.is_loopback
-        or addr.is_link_local
-        or addr.is_multicast
-        or addr.is_reserved
-        or addr.is_unspecified
-    )
+    addr = ipaddress.ip_address(ip.split("%", 1)[0])  # drop an IPv6 zone index
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
+        addr = addr.ipv4_mapped  # ::ffff:127.0.0.1 is 127.0.0.1
+    # is_global: not private, loopback, link-local, reserved, unspecified, nor
+    # shared/carrier-grade NAT (100.64.0.0/10, which is_private misses).
+    return addr.is_global and not addr.is_multicast
 
 
 async def _resolve_all(host: str) -> list[str]:
@@ -43,8 +41,10 @@ async def _resolve_all(host: str) -> list[str]:
     return [info[4][0] for info in infos]
 
 
-async def assert_host_is_public(host: str) -> None:
-    """Resolve ``host`` and reject if any address is non-public (SPEC §10.6)."""
+async def assert_host_is_public(host: str) -> list[str]:
+    """Resolve ``host`` and reject if any address is non-public (SPEC §10.6).
+
+    Returns the addresses checked, for the caller to connect to."""
     try:
         addresses = await _resolve_all(host)
     except socket.gaierror as exc:
@@ -54,6 +54,31 @@ async def assert_host_is_public(host: str) -> None:
     for ip in addresses:
         if not _is_public_ip(ip):
             raise SsrfError(f"{host} resolves to non-public address {ip}")
+    return addresses
+
+
+async def _pinned(url: str) -> tuple[str, dict[str, str], dict[str, str]]:
+    """``url`` aimed at the address just checked: (url, headers, extensions).
+
+    The URL names the IP, the Host header keeps the name (virtual hosting), and
+    TLS gets the name for SNI and for checking the certificate."""
+    parts = urlsplit(url)
+    if parts.scheme != "https":
+        raise SsrfError("only https is permitted")
+    hostname = parts.hostname
+    if not hostname:
+        raise SsrfError("missing host")
+    addresses = await assert_host_is_public(hostname)
+    # IPv4 first: containers often have no IPv6 route, though DNS returns AAAA.
+    ip = next((a for a in addresses if ":" not in a), addresses[0])
+    literal = f"[{ip}]" if ":" in ip else ip
+    netloc = f"{literal}:{parts.port}" if parts.port else literal
+    host_header = f"{hostname}:{parts.port}" if parts.port else hostname
+    return (
+        parts._replace(netloc=netloc).geturl(),
+        {"Host": host_header},
+        {"sni_hostname": hostname},
+    )
 
 
 async def guarded_get(url: str) -> str | None:
@@ -63,12 +88,7 @@ async def guarded_get(url: str) -> str | None:
     (any non-200 response). Raises :class:`SsrfError` on a policy violation
     (bad scheme, private IP, oversize); lets httpx network errors propagate.
     """
-    parts = urlsplit(url)
-    if parts.scheme != "https":
-        raise SsrfError("only https is permitted")
-    if not parts.hostname:
-        raise SsrfError("missing host")
-    await assert_host_is_public(parts.hostname)
+    target, pin_headers, extensions = await _pinned(url)
 
     async with (
         httpx.AsyncClient(
@@ -76,7 +96,7 @@ async def guarded_get(url: str) -> str | None:
             timeout=HTTP_TIMEOUT,
             headers={"User-Agent": "webmail-autodiscovery/1.0"},
         ) as client,
-        client.stream("GET", url) as resp,
+        client.stream("GET", target, headers=pin_headers, extensions=extensions) as resp,
     ):
         if resp.status_code != 200:
             return None
@@ -114,16 +134,17 @@ async def guarded_request(
     fetch ICS feeds. HTTPS-only, public host required, no off-domain redirects, body
     capped at ``max_bytes``. ``auth`` is an (user, password) basic-auth pair. Returns
     status + headers + body (all statuses; the caller decides what's an error)."""
-    parts = urlsplit(url)
-    if parts.scheme != "https":
-        raise SsrfError("only https is permitted")
-    if not parts.hostname:
-        raise SsrfError("missing host")
-    await assert_host_is_public(parts.hostname)
+    target, pin_headers, extensions = await _pinned(url)
 
     async with (
         httpx.AsyncClient(follow_redirects=False, timeout=timeout, auth=auth) as client,
-        client.stream(method, url, headers=headers, content=content) as resp,
+        client.stream(
+            method,
+            target,
+            headers={**(headers or {}), **pin_headers},
+            content=content,
+            extensions=extensions,
+        ) as resp,
     ):
         chunks: list[bytes] = []
         total = 0

backend/app/accounts/testing.py

@@ -14,6 +14,7 @@ import aiosmtplib
 
 from app.accounts.ssrf import SsrfError, assert_host_is_public
 from app.accounts.types import ServerConfig
+from app.config import smtp_helo_name
 from app.providers.base import ProviderAuthError, ProviderConnectError
 from app.providers.imap_smtp import ImapSmtpProvider, ProviderConfig
 
@@ -53,6 +54,7 @@ async def _test_smtp(config: ServerConfig, username: str, password: str) -> None
         use_tls=config.tls == "ssl",
         start_tls=config.tls == "starttls",
         timeout=_SMTP_TIMEOUT,
+        local_hostname=smtp_helo_name(),
     )
     try:
         await smtp.connect()
@@ -89,6 +91,7 @@ async def test_account(
             smtp_tls=smtp.tls,
             username=username,
             password=password,
+            public_only=True,
         )
     )
     try:

backend/app/api/__init__.py

@@ -15,7 +15,9 @@ def register_routers(app: FastAPI) -> None:
     from app.api.actions import router as actions_router
     from app.api.admin import router as admin_router
     from app.api.avatar import router as avatar_router
+    from app.api.completion import router as completion_router
     from app.api.compose import router as compose_router
+    from app.api.invitations import router as invitations_router
     from app.api.mailboxes import router as mailboxes_router
     from app.api.messages import router as messages_router
     from app.api.plugins import router as plugins_router
@@ -24,8 +26,12 @@ def register_routers(app: FastAPI) -> None:
     from app.api.saved_searches import router as saved_searches_router
     from app.api.search import router as search_router
     from app.api.settings import router as settings_router
+    from app.api.snooze import router as snooze_router
     from app.api.spellcheck import router as spellcheck_router
+    from app.api.subscriptions import router as subscriptions_router
+    from app.api.templates import router as templates_router
     from app.api.threads import router as threads_router
+    from app.api.translation import router as translation_router
     from app.identity.routes import router as auth_router
     from app.ws import router as ws_router
 
@@ -46,9 +52,19 @@ def register_routers(app: FastAPI) -> None:
         admin_router,
         spellcheck_router,
         push_router,
+        invitations_router,
+        templates_router,
+        snooze_router,
+        subscriptions_router,
+        completion_router,
+        translation_router,
     ):
         app.include_router(router, prefix="/api")
     app.include_router(ws_router, prefix="/api")
+    # CalDAV / CardDAV (SPEC Appendix E): at the root, where clients look for it.
+    from app.dav.routes import router as dav_router
+
+    app.include_router(dav_router)
 
     # Mount discovered plugin routers at /api/plugins/{name} (SPEC §11). Hooks are
     # subscribed in the lifespan startup so they register once per process.

backend/app/api/accounts.py

@@ -104,6 +104,7 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
             sync_error=a.sync_error,
             sync_error_since=a.sync_error_since.isoformat() if a.sync_error_since else None,
             state_detail=a.state_detail,
+            kind=a.kind,
         )
         for a in accounts
     ]
@@ -225,6 +226,8 @@ async def update_account(
     # re-encrypt the secret and re-sync. The password must be supplied (§10.2: the
     # stored secret is only decrypted in the worker, never here).
     conn_change = body.imap is not None or body.smtp is not None or body.password is not None
+    if conn_change and account.kind == "native":
+        raise_api_error(400, "native_account", "the settings of your own mailbox cannot be changed")
     if conn_change:
         if not body.password:
             raise_api_error(
@@ -267,10 +270,20 @@ async def update_account(
 
 @router.delete("/{account_id}", dependencies=[Depends(require_csrf)])
 async def remove_account(account_id: uuid.UUID, user: CurrentUser, db: DbSession) -> dict[str, str]:
+    kind = await db.scalar(
+        select(Account.kind).where(Account.id == account_id, Account.user_id == user.id)
+    )
+    if kind == "native":
+        raise_api_error(409, "native_account", "your own mailbox cannot be removed")
     ok = await delete_account(db, user_id=user.id, account_id=account_id)
     if not ok:
         raise_api_error(404, "not_found", "account not found")
     await db.commit()
+    # Removing an account for the user's own address (the mail at its old
+    # provider) makes room for their hosted mailbox's (App. D §D.4).
+    from app.hosted import native
+
+    await native.ensure_and_sync(db, user)
     return {"status": "deleted"}
 
 
@@ -288,4 +301,5 @@ def _detail(account: Account) -> AccountDetailOut:
         backfill_done=account.backfill_done,
         imap_config=account.imap_config,
         smtp_config=account.smtp_config,
+        kind=account.kind,
     )

backend/app/api/actions.py

@@ -125,9 +125,7 @@ async def mark_all_read(
         return {"changed": 0}
     messages = list(
         await db.scalars(
-            select(Message).where(
-                Message.thread_id.in_(thread_ids), Message.user_id == user.id
-            )
+            select(Message).where(Message.thread_id.in_(thread_ids), Message.user_id == user.id)
         )
     )
     add, remove = sanitize_flags(["\\Seen"]), sanitize_flags([])
@@ -137,9 +135,7 @@ async def mark_all_read(
     await _push_and_publish(user.id, changed, add, remove)
     bus = get_event_bus()
     for tid in thread_ids:
-        await bus.publish(
-            "thread.updated", {"user_id": str(user.id), "thread_id": str(tid)}
-        )
+        await bus.publish("thread.updated", {"user_id": str(user.id), "thread_id": str(tid)})
     return {"changed": len(changed)}
 
 

backend/app/api/admin.py

@@ -3,25 +3,44 @@
 The backend seam for the admin panel: the registration gate — the first account
 on a fresh install becomes the administrator and closes signup behind itself,
 and this is how that gets reopened — the default quota, and managing users
-(search, per-user quota, block, delete). An administrator cannot block or delete
-their own account, so the instance can never lock out its last admin that way.
+(search, per-user quota, block, delete). An administrator can change their own
+quota but cannot block or delete their own account, so the instance can never
+lock out its last admin that way.
 """
 
 from __future__ import annotations
 
+import base64
 import datetime as dt
+import re
 import uuid
 from typing import Annotated
 
-from fastapi import APIRouter, Depends, Query
+from fastapi import APIRouter, Depends, Path, Query
+from pydantic import BaseModel
 from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import translation
 from app.api.errors import raise_api_error
+from app.config import get_settings
 from app.db import get_session
+from app.hosted.admin import MailServerError
+from app.identity import invitations
 from app.identity.deps import CurrentAdmin, require_csrf
-from app.identity.mailer import LOGO_HTML, MailerError, mailer_config, send_mail
+from app.identity.mailer import (
+    LOGO_CID,
+    LOGO_DATA_URL,
+    LOGO_HTML,
+    MailerError,
+    mailer_config,
+    send_mail,
+)
 from app.identity.schemas import (
+    AdminInvitePreviewIn,
+    AdminInvitePreviewOut,
+    AdminInviteTemplateIn,
+    AdminInviteTemplateOut,
     AdminMailIn,
     AdminMailOut,
     AdminMailTestIn,
@@ -45,6 +64,7 @@ def _out(row: AppSettings) -> AdminSettingsOut:
     return AdminSettingsOut(
         registration_open=row.registration_open,
         default_quota_bytes=row.default_quota_bytes,
+        invitations_enabled=row.invitations_enabled,
     )
 
 
@@ -67,6 +87,8 @@ async def write_settings(
         row.registration_open = body.registration_open
     if body.default_quota_bytes is not None:
         row.default_quota_bytes = body.default_quota_bytes
+    if body.invitations_enabled is not None:
+        row.invitations_enabled = body.invitations_enabled
     row.updated_at = dt.datetime.now(dt.UTC)
     await db.commit()
     return _out(row)
@@ -140,6 +162,109 @@ async def test_mail(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -
     return MessageOut()
 
 
+# --- Invitation email ------------------------------------------------------------
+
+_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+")
+_SAMPLE = {"invitee_name": "Anna Nowak", "inviter_name": "Jan Kowalski"}
+
+
+def _template_out(row: AppSettings) -> AdminInviteTemplateOut:
+    return AdminInviteTemplateOut(
+        subject=row.invite_subject,
+        html=row.invite_template,
+        from_address=row.invite_from,
+        default_subject=invitations.DEFAULT_SUBJECT,
+        default_html=invitations.DEFAULT_TEMPLATE,
+        default_from=row.smtp_from,
+        placeholders=list(invitations.PLACEHOLDERS),
+    )
+
+
+@router.get("/invitations/template", response_model=AdminInviteTemplateOut)
+async def read_invite_template(admin: CurrentAdmin, db: DbSession) -> AdminInviteTemplateOut:
+    return _template_out(await app_settings(db))
+
+
+@router.put(
+    "/invitations/template",
+    response_model=AdminInviteTemplateOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def write_invite_template(
+    body: AdminInviteTemplateIn, admin: CurrentAdmin, db: DbSession
+) -> AdminInviteTemplateOut:
+    """Upload the invitation email's HTML template, subject and sender."""
+    row = await app_settings(db, for_update=True)
+    if body.html is not None:
+        html = body.html.strip()
+        if html and (problem := invitations.template_problem(html)):
+            raise_api_error(400, "invalid_template", problem)
+        row.invite_template = html
+    if body.subject is not None:
+        row.invite_subject = " ".join(body.subject.split())
+    if body.from_address is not None:
+        sender = body.from_address.strip()
+        if sender and not _EMAIL_RE.fullmatch(sender):
+            raise_api_error(400, "invalid_email", "the sender must be an email address")
+        row.invite_from = sender
+    row.updated_at = dt.datetime.now(dt.UTC)
+    await db.commit()
+    return _template_out(row)
+
+
+def _sample(row: AppSettings, subject: str, html: str) -> invitations.RenderedMail:
+    """The invitation as a friend would get it, with sample names."""
+    if html.strip() and (problem := invitations.template_problem(html)):
+        raise_api_error(400, "invalid_template", problem)
+    draft = AppSettings(invite_subject=subject, invite_template=html.strip())
+    base = row.public_url.rstrip("/") or f"https://{get_settings().app_domain}"
+    return invitations.render(draft, link=f"{base}/register#invite=sample", **_SAMPLE)
+
+
+@router.post(
+    "/invitations/preview",
+    response_model=AdminInvitePreviewOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def preview_invite(
+    body: AdminInvitePreviewIn, admin: CurrentAdmin, db: DbSession
+) -> AdminInvitePreviewOut:
+    """Render a template (saved or not) with sample names, for the panel to show."""
+    mail = _sample(await app_settings(db), body.subject, body.html)
+    html = mail.html
+    # The preview frame has no message parts: show the images it would carry.
+    for part in mail.inline:
+        data = base64.b64encode(part.data).decode()
+        html = html.replace(f"cid:{part.cid}", f"data:{part.mime_type};base64,{data}")
+    html = html.replace(f"cid:{LOGO_CID}", LOGO_DATA_URL)
+    return AdminInvitePreviewOut(subject=mail.subject, html=html)
+
+
+@router.post("/invitations/test", response_model=MessageOut, dependencies=[Depends(require_csrf)])
+async def test_invite(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -> MessageOut:
+    """Send the saved invitation email, with sample names and a dummy link."""
+    row = await app_settings(db)
+    cfg = mailer_config(row)
+    if cfg is None:
+        raise_api_error(
+            409, "mail_not_configured", "set the SMTP server, From address and public address first"
+        )
+    mail = _sample(row, row.invite_subject, row.invite_template)
+    try:
+        await send_mail(
+            cfg,
+            body.to.strip(),
+            mail.subject,
+            mail.text,
+            mail.html,
+            sender=invitations.sender_address(row),
+            inline=mail.inline,
+        )
+    except MailerError as exc:
+        raise_api_error(502, "mail_failed", f"sending failed: {exc}")
+    return MessageOut()
+
+
 # --- Users --------------------------------------------------------------------
 
 
@@ -156,12 +281,15 @@ def _user_out(user: User, used_bytes: int, accounts: list[str]) -> AdminUserOut:
     )
 
 
-async def _target(db: AsyncSession, admin: User, user_id: uuid.UUID) -> User:
-    """The user an admin action is aimed at; never the admin themselves."""
+async def _target(
+    db: AsyncSession, admin: User, user_id: uuid.UUID, *, allow_self: bool = False
+) -> User:
+    """The user an admin action is aimed at. Blocking and deleting are never
+    allowed on the admin themselves; a quota change is (``allow_self``)."""
     user = await db.scalar(select(User).where(User.id == user_id))
     if user is None:
         raise_api_error(404, "not_found", "user not found")
-    if user.id == admin.id:
+    if user.id == admin.id and not allow_self:
         raise_api_error(409, "cannot_modify_self", "you cannot do this to your own account")
     return user
 
@@ -179,11 +307,17 @@ async def patch_user(
     user_id: uuid.UUID, body: AdminUserPatch, admin: CurrentAdmin, db: DbSession
 ) -> AdminUserOut:
     """Change a user's quota and/or block or unblock them."""
-    user = await _target(db, admin, user_id)
+    # Your own quota is fine to change; blocking yourself is not.
+    user = await _target(db, admin, user_id, allow_self=body.blocked is None)
     if body.quota_bytes is not None:
         user.quota_bytes = body.quota_bytes
     if body.blocked is not None:
+        try:
             await set_blocked(db, user, body.blocked)
+        except MailServerError as exc:
+            raise_api_error(
+                502, "mail_server_unavailable", f"the mail server did not answer: {exc}"
+            )
     await db.commit()
     used = await db.scalar(
         select(func.coalesce(func.sum(Message.size_bytes), 0)).where(Message.user_id == user.id)
@@ -200,5 +334,67 @@ async def patch_user(
 async def remove_user(user_id: uuid.UUID, admin: CurrentAdmin, db: DbSession) -> dict[str, str]:
     """Delete a user and all of their data. Cannot be undone."""
     user = await _target(db, admin, user_id)
+    try:
         await delete_user(db, user)
+    except MailServerError as exc:
+        raise_api_error(502, "mail_server_unavailable", f"the mail server did not answer: {exc}")
     return {"status": "deleted"}
+
+
+# --- email translation languages (translator service) ------------------------------
+
+
+class InstallProgressOut(BaseModel):
+    done: int
+    total: int
+
+
+class AdminTranslationOut(BaseModel):
+    enabled: bool  # TRANSLATOR_URL is set
+    reachable: bool
+    installed: list[str] = []
+    installing: dict[str, InstallProgressOut] = {}
+    pending: list[str] = []  # waiting to be tried again (no network last time)
+    available: list[str] = []  # what Mozilla has models for (empty: offline)
+    errors: dict[str, str] = {}  # why a language's last install failed
+
+
+_LANG = Path(pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$")
+
+
+@router.get("/translation", response_model=AdminTranslationOut)
+async def translation_status(admin: CurrentAdmin) -> AdminTranslationOut:
+    if not translation.enabled():
+        return AdminTranslationOut(enabled=False, reachable=False)
+    try:
+        data = await translation.status()
+    except translation.TranslationError:
+        return AdminTranslationOut(enabled=True, reachable=False)
+    return AdminTranslationOut(enabled=True, reachable=True, **data)
+
+
+@router.post(
+    "/translation/languages/{code}",
+    response_model=AdminTranslationOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def install_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
+    """Download a language's models (to and from English) in the background."""
+    try:
+        await translation.install(code)
+    except translation.TranslationError as exc:
+        raise_api_error(502, "translator_failed", str(exc))
+    return await translation_status(admin)
+
+
+@router.delete(
+    "/translation/languages/{code}",
+    response_model=AdminTranslationOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def remove_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
+    try:
+        await translation.remove(code)
+    except translation.TranslationError as exc:
+        raise_api_error(502, "translator_failed", str(exc))
+    return await translation_status(admin)

backend/app/api/completion.py

@@ -0,0 +1,44 @@
+"""Sentence completion for the compose window (app/completion.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import completion
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import CompletionIndexState
+from app.queue import enqueue_job
+
+router = APIRouter(prefix="/completion", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class CompletionIn(BaseModel):
+    # The text before the cursor (its last line is all that matters). A POST,
+    # not a query string: what the user is typing stays out of access logs.
+    before: str = Field(max_length=2000)
+
+
+class CompletionOut(BaseModel):
+    suggestion: str | None
+
+
+@router.post("", response_model=CompletionOut)
+async def suggest(body: CompletionIn, user: CurrentUser, db: DbSession) -> CompletionOut:
+    before = body.before[-300:]
+    built = await db.scalar(
+        select(CompletionIndexState.built_at).where(CompletionIndexState.user_id == user.id)
+    )
+    if built is None or built < dt.datetime.now(dt.UTC) - completion.REBUILD_AFTER:
+        # In the background; until it is built, the starter phrases still answer.
+        await enqueue_job("build_index_job", str(user.id), _job_id=f"complete:{user.id}")
+    personal = await completion.personal_phrases(db, user.id, completion.keys_for(before))
+    return CompletionOut(suggestion=completion.complete(personal, before))

backend/app/api/compose.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import uuid
 from datetime import UTC, datetime, timedelta
 from typing import Annotated
@@ -15,7 +17,8 @@ from app.api.schemas import AttachmentOut
 from app.config import get_settings
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
-from app.indexer import blobstore
+from app.indexer import attachments as att_store
+from app.indexer import blobstore, rawstore
 from app.models import Attachment, BodyPart, ComposeAttachment, Draft, Mailbox, Message, Outbox
 from app.outbox.jobs import RETRY_WINDOW
 from app.outbox.schemas import (
@@ -206,7 +209,15 @@ async def adopt_remote_draft(message_id: uuid.UUID, user: CurrentUser, db: DbSes
         )
     )
     attachment_ids: list[str] = []
+    if att_rows:
+        with contextlib.suppress(FileNotFoundError):  # fetched if a hosted mailbox's
+            await rawstore.ensure(msg)
     for a in att_rows:
+        # Kept only inside the message: give the compose attachment a blob of its own.
+        if not await asyncio.to_thread(
+            att_store.materialize, user.id, msg.blob_sha256, a.blob_sha256, a.part_index
+        ):
+            continue
         ca = ComposeAttachment(
             user_id=user.id,
             filename=a.filename,

backend/app/api/errors.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from typing import NoReturn
 
 from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
@@ -29,7 +30,23 @@ async def _handler(request: Request, exc: StarletteHTTPException) -> JSONRespons
     return JSONResponse(status_code=exc.status_code, content=body, headers=headers)
 
 
+async def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+    """A malformed request, in the usual error shape. Names the first field at
+    fault and what is wrong with it — enough to show the user — but not the
+    whole request model the framework's own answer would spell out."""
+    errors = exc.errors()
+    first = errors[0] if errors else {}
+    loc = [str(p) for p in first.get("loc", ()) if p not in ("body", "query", "path")]
+    field = loc[-1] if loc else ""
+    reason = str(first.get("msg", "invalid value"))
+    message = f"{field}: {reason}" if field else reason
+    return JSONResponse(
+        status_code=422, content={"error": {"code": "validation_error", "message": message}}
+    )
+
+
 def register_error_handlers(app: FastAPI) -> None:
     """Install the error-shape handler for all HTTP exceptions."""
     app.add_exception_handler(StarletteHTTPException, _handler)  # type: ignore[arg-type]
     app.add_exception_handler(HTTPException, _handler)  # type: ignore[arg-type]
+    app.add_exception_handler(RequestValidationError, _validation_handler)  # type: ignore[arg-type]

backend/app/api/invitations.py

@@ -0,0 +1,123 @@
+"""Invite a friend (app/identity/invitations.py): what the sidebar entry and its
+dialog need — how many invitations are left, and sending one."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity import invitations
+from app.identity.deps import CurrentUser, require_csrf
+from app.identity.mailer import MailerError, mailer_config, send_mail
+from app.identity.service import app_settings
+from app.models import Invitation
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/invitations", tags=["invitations"])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+")
+
+
+class SentInvitationOut(BaseModel):
+    invitee_name: str
+    invitee_email: str
+    created_at: str
+    used: bool
+    expired: bool
+
+
+class InvitationsOut(BaseModel):
+    # Invitations are switched on and system mail can send them.
+    available: bool
+    remaining: int
+    per_user: int
+    sent: list[SentInvitationOut]
+
+
+class InvitationIn(BaseModel):
+    name: str = Field(min_length=1, max_length=100)
+    email: str = Field(min_length=3, max_length=320)
+
+
+@router.get("", response_model=InvitationsOut)
+async def read(user: CurrentUser, db: DbSession) -> InvitationsOut:
+    row = await app_settings(db)
+    now = dt.datetime.now(dt.UTC)
+    return InvitationsOut(
+        available=row.invitations_enabled and mailer_config(row) is not None,
+        remaining=await invitations.remaining(db, user.id),
+        per_user=invitations.PER_USER,
+        sent=[
+            SentInvitationOut(
+                invitee_name=i.invitee_name,
+                invitee_email=i.invitee_email,
+                created_at=i.created_at.isoformat(),
+                used=i.used_at is not None,
+                expired=i.used_at is None and i.expires_at <= now,
+            )
+            for i in await invitations.sent(db, user.id)
+        ],
+    )
+
+
+@router.post("", response_model=InvitationsOut, dependencies=[Depends(require_csrf)])
+async def send(body: InvitationIn, user: CurrentUser, db: DbSession) -> InvitationsOut:
+    """Email a friend a single-use sign-up link. Sent before answering, so the
+    user hears at once if it could not go out (and keeps the invitation)."""
+    name = " ".join(body.name.split())
+    email = body.email.strip()
+    if not name:
+        raise_api_error(400, "invalid_name", "enter your friend's name")
+    if not _EMAIL_RE.fullmatch(email):
+        raise_api_error(400, "invalid_email", "enter a valid email address")
+    row = await app_settings(db)
+    if not row.invitations_enabled:
+        raise_api_error(403, "invitations_disabled", "invitations are switched off")
+    cfg = mailer_config(row)
+    if cfg is None:
+        raise_api_error(503, "mail_not_configured", "the server cannot send email yet")
+    try:
+        invitation, token = await invitations.create(db, user, name=name, email=email)
+    except invitations.InvitationError as exc:
+        raise_api_error(409, exc.code, exc.message)
+    await db.commit()
+
+    # The token rides in the fragment: never sent to a server, so it stays out
+    # of access logs and Referer headers (as with password-reset links).
+    link = f"{cfg.public_url}/register#invite={token}"
+    mail = invitations.render(
+        row,
+        invitee_name=name,
+        inviter_name=await invitations.inviter_name(db, user),
+        link=link,
+    )
+    try:
+        await send_mail(
+            cfg,
+            email,
+            mail.subject,
+            mail.text,
+            mail.html,
+            sender=invitations.sender_address(row),
+            reply_to=user.handle,  # an answer reaches the friend who invited
+            inline=mail.inline,
+        )
+    except MailerError as exc:
+        logger.warning("invitation from %s could not be sent: %s", user.handle, exc)
+        # Not sent: the user keeps the invitation.
+        await db.execute(delete(Invitation).where(Invitation.id == invitation.id))
+        await db.commit()
+        raise_api_error(502, "mail_failed", "the invitation could not be sent — try again later")
+    return await read(user, db)

backend/app/api/mailboxes.py

@@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Draft, Mailbox, Message
+from app.models import Draft, Mailbox, Message, Thread
 from app.outbox.service import not_being_sent
 
 router = APIRouter(prefix="/mailboxes", tags=["mail"])
@@ -30,6 +30,7 @@ class MailboxOut(BaseModel):
 class FolderCounts(BaseModel):
     inbox_unread: int  # unread messages sitting in Inbox mailboxes
     drafts: int  # drafts shown in the Drafts folder (local + un-adopted remote)
+    snoozed: int = 0  # conversations snoozed out of the Inbox
 
 
 @router.get("", response_model=list[MailboxOut])
@@ -54,8 +55,17 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
             Message.user_id == user.id,
             Mailbox.special_use == "inbox",
             ~Message.flags.contains(["\\Seen"]),
+            # A snoozed conversation is out of the Inbox until it wakes.
+            ~Message.thread_id.in_(
+                select(Thread.id).where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+            ),
         )
     )
+    snoozed = await db.scalar(
+        select(func.count())
+        .select_from(Thread)
+        .where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+    )
 
     # Drafts folder = local drafts + IMAP-Drafts messages we haven't adopted (mirror
     # UIDs excluded), matching what list_drafts + list_remote_drafts return.
@@ -77,4 +87,8 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
     )
     remote = sum(1 for r in remote_rows if (r.account_id, r.uid) not in owned)
 
-    return FolderCounts(inbox_unread=int(inbox_unread or 0), drafts=int(local or 0) + remote)
+    return FolderCounts(
+        inbox_unread=int(inbox_unread or 0),
+        drafts=int(local or 0) + remote,
+        snoozed=int(snoozed or 0),
+    )

backend/app/api/messages.py

@@ -3,8 +3,10 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import html as html_lib
 import io
+import logging
 import uuid
 import zipfile
 from typing import Annotated
@@ -13,18 +15,37 @@ from urllib.parse import quote
 from fastapi import APIRouter, Depends, Query
 from fastapi.responses import FileResponse, HTMLResponse, Response
 from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import translation
 from app.api.errors import raise_api_error
 from app.api.schemas import AttachmentOut
-from app.api.settings import add_allowed_image_sender, allowed_image_senders
+from app.api.settings import (
+    add_allowed_image_sender,
+    allowed_image_senders,
+    blocks_trackers,
+)
+from app.completion import html_to_text
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
-from app.indexer import blobstore, thumbs
+from app.indexer import attachments as att_store
+from app.indexer import rawstore, thumbs
 from app.indexer.mime import parse_message
 from app.indexer.sanitize import SANITIZER_VERSION, sanitize_html
 from app.indexer.thumbs import thumb_type
-from app.models import Attachment, BodyPart, ComposeAttachment, Message
+from app.models import (
+    Account,
+    Attachment,
+    BodyPart,
+    ComposeAttachment,
+    Message,
+    MessageTranslation,
+    User,
+)
+from app.translation_prep import find_names_in_signature
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/messages", tags=["mail"])
 
@@ -86,6 +107,31 @@ def _wrap(fragment: str) -> str:
     return _DOC_TEMPLATE.format(body=fragment)
 
 
+# The sanitized message as a document of its own. The app shows it in a
+# sandboxed frame (frontend MessageBody), but this URL can also be opened
+# directly in a tab, on our own origin: there, this policy is what keeps any
+# script the sanitizer might ever miss from running — no scripts, no plugins,
+# no forms, a sandbox of its own origin, and nothing framing it elsewhere.
+_MESSAGE_CSP = (
+    "default-src 'none'; script-src 'none'; object-src 'none'; base-uri 'none'; "
+    "form-action 'none'; frame-ancestors 'self'; img-src 'self' data: https: http:; "
+    "style-src 'self' 'unsafe-inline'; font-src 'self' data: https:; media-src 'self'; "
+    "sandbox allow-popups allow-popups-to-escape-sandbox"
+)
+
+
+def _html(document: str, trackers: int = 0, extra: dict[str, str] | None = None) -> HTMLResponse:
+    # The page reads what it needs from headers: the body is the message.
+    return HTMLResponse(
+        content=document,
+        headers={
+            "Content-Security-Policy": _MESSAGE_CSP,
+            "X-Trackers-Blocked": str(trackers),
+            **(extra or {}),
+        },
+    )
+
+
 async def _owned_message(db: AsyncSession, user_id: uuid.UUID, message_id: uuid.UUID) -> Message:
     msg = await db.scalar(
         select(Message).where(Message.id == message_id, Message.user_id == user_id)
@@ -99,54 +145,167 @@ def _sender_email(message: Message) -> str:
     return message.from_addr[0].get("email", "").lower() if message.from_addr else ""
 
 
+async def _render_fragment(
+    db: AsyncSession, message: Message, allow_remote: bool, block: bool
+) -> tuple[str, int]:
+    """The sanitized body the user sees, and how many trackers came out of it.
+
+    Re-sanitizes from the raw blob when remote images are allowed, the stored
+    copy predates the current sanitizer (SPEC §10.4), or trackers are to stay
+    in; otherwise the stored copy is served as is.
+    """
+    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message.id)))
+    html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
+
+    stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
+    # The stored copy has its trackers taken out; with blocking switched off, a
+    # message that had some is rendered afresh with them left in.
+    trackers_kept = not block and html_part is not None and html_part.trackers_blocked > 0
+    if html_part is not None and not allow_remote and not stale and not trackers_kept:
+        return html_part.content, html_part.trackers_blocked if block else 0
+
+    raw = None
+    if html_part is not None or allow_remote:  # a plain-text message has nothing to redo
+        try:
+            raw = await rawstore.ensure_bytes(message)
+        except FileNotFoundError:
+            raw = None
+    parsed = parse_message(raw) if raw is not None else None
+    if parsed is not None and parsed.html_body is not None:
+        if stale and html_part is not None:
+            # The stored copy is always the plain one: no remote images, trackers out.
+            upgraded = sanitize_html(parsed.html_body, message.id)
+            html_part.content = upgraded.html
+            html_part.sanitizer_v = upgraded.sanitizer_v
+            html_part.trackers_blocked = upgraded.trackers_blocked
+            await db.commit()
+            if block and not allow_remote:
+                return upgraded.html, upgraded.trackers_blocked
+        result = sanitize_html(
+            parsed.html_body, message.id, allow_remote_images=allow_remote, block_trackers=block
+        )
+        return result.html, result.trackers_blocked
+
+    if html_part is not None:
+        return html_part.content, html_part.trackers_blocked
+    text_part = next((p for p in parts if p.kind == "text"), None)
+    body = html_lib.escape(text_part.content) if text_part else ""
+    return f'<div class="wm-plaintext">{body}</div>', 0
+
+
+async def _content_language(db: AsyncSession, message: Message) -> str:
+    """The message's language, found (and kept) the first time it is shown."""
+    if message.content_language is not None or not translation.enabled():
+        return message.content_language or ""
+    text = await db.scalar(
+        select(BodyPart.content).where(BodyPart.message_id == message.id, BodyPart.kind == "text")
+    )
+    try:
+        found = await translation.detect(text or message.snippet or "")
+    except translation.TranslationError as exc:
+        logger.info("language detection unavailable: %s", exc)
+        return ""  # not kept: tried again once the service is back
+    message.content_language = found
+    await db.commit()
+    return found
+
+
 @router.get("/{message_id}/html", response_class=HTMLResponse)
 async def message_html(
     message_id: uuid.UUID,
     user: CurrentUser,
     db: DbSession,
     images: bool = Query(False),
+    translate: str | None = Query(None, pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$"),
 ) -> HTMLResponse:
     """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
 
-    Re-sanitizes from the raw blob when the sender is image-approved, or when the
-    stored copy predates the current sanitizer version (SPEC §10.4). ``images=1``
-    is a one-time "Load images" — render remote images for this request only,
-    without persisting the sender as trusted.
+    ``images=1`` is a one-time "Load images" — render remote images for this
+    request only, without persisting the sender as trusted. ``translate=pl``
+    returns it translated (app/translation.py). Headers tell the page what it
+    got: X-Trackers-Blocked, X-Content-Language (the message's own language,
+    when known) and, translated, X-Translated-From / X-Translated-To.
     """
     message = await _owned_message(db, user.id, message_id)
     allow_remote = images or _sender_email(message) in allowed_image_senders(user)
+    block = blocks_trackers(user)
+    fragment, trackers = await _render_fragment(db, message, allow_remote, block)
 
-    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
-    html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
-
-    stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
-    if html_part is not None and not allow_remote and not stale:
-        return HTMLResponse(content=_wrap(html_part.content))
-
-    # Re-sanitize the original HTML (image-approved sender, or a stale stored copy).
-    # If the raw blob is unavailable, fall back to whatever is already stored.
-    if allow_remote or stale:
-        try:
-            raw = blobstore.load(user.id, message.blob_sha256)
-        except FileNotFoundError:
-            raw = None
-        if raw is not None:
-            parsed = parse_message(raw)
-            if parsed.html_body is not None:
-                result = sanitize_html(
-                    parsed.html_body, message.id, allow_remote_images=allow_remote
+    headers: dict[str, str] = {}
+    language = await _content_language(db, message)
+    if language:
+        headers["X-Content-Language"] = language
+    if translate:
+        fragment, source = await _translated(
+            db, user, message, fragment, language, translate, allow_remote, block
         )
-                if html_part is not None and not allow_remote:
-                    html_part.content = result.html  # persist the upgraded sanitization
-                    html_part.sanitizer_v = result.sanitizer_v
-                    await db.commit()
-                return HTMLResponse(content=_wrap(result.html))
+        headers["X-Translated-From"] = source
+        headers["X-Translated-To"] = translate
+    return _html(_wrap(fragment), trackers, headers)
 
-    if html_part is not None:
-        return HTMLResponse(content=_wrap(html_part.content))
-    text_part = next((p for p in parts if p.kind == "text"), None)
-    body = html_lib.escape(text_part.content) if text_part else ""
-    return HTMLResponse(content=_wrap(f'<div class="wm-plaintext">{body}</div>'))
+
+async def _names(db: AsyncSession, user: User, message: Message) -> list[str]:
+    """Names a translation leaves as they are if the model would change them:
+    the people on the message (and the sender's single names, as signed:
+    "Milan"), the user's own names (accounts, signatures)."""
+    names: list[str] = []
+    for people in (message.from_addr, message.to_addr, message.cc_addr):
+        names += [p.get("name", "") for p in people if p.get("name")]
+    for p in message.from_addr:
+        names += [w for w in str(p.get("name", "")).split() if len(w) >= 3 and w[0].isupper()]
+    rows = await db.execute(
+        select(Account.display_name, Account.signature).where(Account.user_id == user.id)
+    )
+    for display_name, signature in rows:
+        if display_name:
+            names.append(display_name)
+        if signature:
+            names += find_names_in_signature(html_to_text(signature))
+    return names
+
+
+async def _translated(
+    db: AsyncSession,
+    user: User,
+    message: Message,
+    fragment: str,
+    language: str,
+    target: str,
+    allow_remote: bool,
+    block: bool,
+) -> tuple[str, str]:
+    """The fragment in ``target`` — from the cache, or translated and kept."""
+    version = SANITIZER_VERSION * 1000 + translation.PIPELINE_VERSION
+    cached = await db.get(MessageTranslation, (message.id, target, allow_remote))
+    if cached is not None and cached.sanitizer_v == version:
+        return cached.html, cached.source
+    try:
+        out = await translation.translate_html(
+            fragment, language or "auto", target, names=await _names(db, user, message)
+        )
+    except translation.TranslationError as exc:
+        raise_api_error(503, "translation_failed", str(exc))
+    # The translation comes back as HTML: cleaned like any HTML we show.
+    clean = sanitize_html(
+        out, message.id, allow_remote_images=allow_remote, block_trackers=block
+    ).html
+    source = language or "auto"
+    stmt = insert(MessageTranslation).values(
+        message_id=message.id,
+        target=target,
+        images=allow_remote,
+        source=source,
+        html=clean,
+        sanitizer_v=version,
+    )
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["message_id", "target", "images"],
+            set_={"html": clean, "source": source, "sanitizer_v": version},
+        )
+    )
+    await db.commit()
+    return clean, source
 
 
 @router.post("/{message_id}/allow-remote-images", dependencies=[Depends(require_csrf)])
@@ -176,7 +335,7 @@ async def message_cid(
     )
     if att is None:
         raise_api_error(404, "not_found", "inline part not found")
-    data = blobstore.load(msg.user_id, att.blob_sha256)
+    data = await _attachment_bytes(msg, att)
     media_type = att.mime_type if att.mime_type in _INLINE_SAFE else "application/octet-stream"
     return Response(content=data, media_type=media_type)
 
@@ -190,6 +349,24 @@ def _content_disposition(disposition: str, filename: str) -> str:
     return f"{disposition}; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename)}"
 
 
+async def _raw_ready(msg: Message) -> None:
+    """Have the message's raw source here (a hosted mailbox's is fetched from
+    the mail server when not cached); a miss is left to the reader to report."""
+    with contextlib.suppress(FileNotFoundError):
+        await rawstore.ensure(msg)
+
+
+async def _attachment_bytes(msg: Message, att: Attachment) -> bytes:
+    """An attachment's bytes, taken out of its message (404 if they cannot be)."""
+    await _raw_ready(msg)
+    try:
+        return await asyncio.to_thread(
+            att_store.load, msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index
+        )
+    except FileNotFoundError:
+        raise_api_error(404, "not_found", "attachment content missing")
+
+
 async def _real_attachments(db: AsyncSession, message_id: uuid.UUID) -> list[Attachment]:
     """Downloadable (non-inline) attachments for a message, in a stable order."""
     rows = await db.scalars(
@@ -221,9 +398,11 @@ async def forward_attachments(
     message_id: uuid.UUID, user: CurrentUser, db: DbSession
 ) -> list[AttachmentOut]:
     """Attach a message's files to a forward of it: one compose attachment per
-    real (non-inline) attachment, sharing the stored blob — nothing is copied.
+    real (non-inline) attachment. Each is taken out of the message into a blob
+    of its own (shared, by hash, with any other use of the same file).
     Returns them as the composer lists its attachments."""
-    await _owned_message(db, user.id, message_id)
+    msg = await _owned_message(db, user.id, message_id)
+    await _raw_ready(msg)
     rows = [
         ComposeAttachment(
             user_id=user.id,
@@ -233,7 +412,9 @@ async def forward_attachments(
             blob_sha256=a.blob_sha256,
         )
         for a in await _real_attachments(db, message_id)
-        if blobstore.exists(user.id, a.blob_sha256)
+        if await asyncio.to_thread(
+            att_store.materialize, user.id, msg.blob_sha256, a.blob_sha256, a.part_index
+        )
     ]
     db.add_all(rows)
     await db.commit()
@@ -276,7 +457,7 @@ async def download_attachments_zip(
                 name = f"{stem}-{seen[name]}{dot}{ext}" if dot else f"{name}-{seen[name]}"
             else:
                 seen[name] = 1
-            zf.writestr(name, blobstore.load(msg.user_id, att.blob_sha256))
+            zf.writestr(name, await _attachment_bytes(msg, att))
 
     return Response(
         content=buffer.getvalue(),
@@ -301,7 +482,16 @@ async def attachment_thumbnail(
     kind = thumb_type(att.mime_type, att.filename) if att is not None else None
     if att is None or kind is None:
         raise_api_error(404, "not_found", "no thumbnail")
-    data = await asyncio.to_thread(thumbs.thumbnail, msg.user_id, att.blob_sha256, kind)
+    if not thumbs.made(msg.user_id, att.blob_sha256):
+        await _raw_ready(msg)
+    data = await asyncio.to_thread(
+        thumbs.thumbnail,
+        msg.user_id,
+        att.blob_sha256,
+        kind,
+        att.size_bytes,
+        lambda: att_store.load(msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index),
+    )
     if data is None:
         raise_api_error(404, "not_found", "no thumbnail")
     # An attachment never changes under its id: the browser may keep this.
@@ -329,8 +519,15 @@ async def message_attachment(
     )
     if att is None:
         raise_api_error(404, "not_found", "attachment not found")
-    path = blobstore.blob_path(msg.user_id, att.blob_sha256)
-    if not path.exists():
+    if not att_store.has_file(msg.user_id, att.blob_sha256):
+        await _raw_ready(msg)
+    try:
+        # Taken out of the message into a short-lived file (kept a day after
+        # its last use), so it can be streamed with HTTP Range like before.
+        path = await asyncio.to_thread(
+            att_store.file, msg.user_id, msg.blob_sha256, att.blob_sha256, att.part_index
+        )
+    except FileNotFoundError:
         raise_api_error(404, "not_found", "attachment content missing")
     inline_type = None if download else _inline_type(att.mime_type, att.filename)
     disposition = "inline" if inline_type else "attachment"

backend/app/api/schemas.py

@@ -27,6 +27,8 @@ class ThreadSummary(BaseModel):
     has_attachments: bool
     pinned: bool
     last_message_at: dt.datetime
+    # Set while the thread is snoozed: when it comes back to the Inbox.
+    snoozed_until: dt.datetime | None = None
 
 
 class ThreadListOut(BaseModel):
@@ -64,6 +66,13 @@ class MessageEnvelope(BaseModel):
     snippet: str
     # Other accounts this same delivery also landed in (collapsed duplicates).
     also_in_account_ids: list[uuid.UUID] = []
+    # Bulk mail: the Subscriptions group it belongs to, how it can be left
+    # (one_click | mailto | web), and whether the user already did.
+    subscription_key: str | None = None
+    unsubscribe_via: str | None = None
+    unsubscribed: bool = False
+    # Its language, once found (the first time it is shown): "" unknown.
+    content_language: str = ""
 
 
 class ThreadDetail(BaseModel):
@@ -72,6 +81,7 @@ class ThreadDetail(BaseModel):
     order: str  # "asc" | "desc"
     pinned: bool
     messages: list[MessageEnvelope]  # always ASC (SPEC §9.4)
+    snoozed_until: dt.datetime | None = None
 
 
 class FlagChangeRequest(BaseModel):
@@ -102,6 +112,8 @@ class AccountOut(BaseModel):
     sync_error_since: str | None = None
     # Why the state is what it is (e.g. the server's reply to a refused sign-in).
     state_detail: str = ""
+    # "native": the user's own hosted mailbox — cannot be removed (App. D §D.4).
+    kind: str = "external"
 
 
 class SearchThreadHit(BaseModel):
@@ -112,3 +124,5 @@ class SearchThreadHit(BaseModel):
 class SearchOut(BaseModel):
     threads: list[SearchThreadHit]
     cursor: str | None = None
+    # Conversations matching in all (first page only; None where not counted).
+    total: int | None = None

backend/app/api/search.py

@@ -29,13 +29,22 @@ async def search(
     q: str = Query(""),
     cursor: str | None = None,
     limit: int = Query(50, ge=1, le=200),
+    filter: str | None = Query(None, pattern="^(unread|pinned)$"),
 ) -> SearchOut:
+    """One page of matching messages, grouped into conversations. ``filter``
+    keeps conversations with unread mail / pinned ones (the list's Unread and
+    Pinned tabs). The first page also says how many conversations match in
+    all, where the search engine can count them."""
     if not q.strip():
         return SearchOut(threads=[], cursor=None)
 
     backend = get_search_backend()
     query = to_search_query(q, user.id)
     hits, next_cursor = await backend.query(query, limit=limit, cursor=cursor)
+    total: int | None = None
+    count = getattr(backend, "count_threads", None)
+    if cursor is None and count is not None:
+        total = await count(query, thread_filter=filter)
 
     # Group hits by thread, preserving order and keeping the first highlight.
     order: list[uuid.UUID] = []
@@ -48,7 +57,7 @@ async def search(
             highlights[hit.thread_id] = hit.highlights
 
     if not order:
-        return SearchOut(threads=[], cursor=next_cursor)
+        return SearchOut(threads=[], cursor=next_cursor, total=total)
 
     threads = {
         t.id: t
@@ -64,5 +73,9 @@ async def search(
         )
         for tid in order
         if tid in threads
+        and (
+            filter is None
+            or (threads[tid].unread_count > 0 if filter == "unread" else threads[tid].pinned)
+        )
     ]
-    return SearchOut(threads=results, cursor=next_cursor)
+    return SearchOut(threads=results, cursor=next_cursor, total=total)

backend/app/api/settings.py

@@ -27,6 +27,7 @@ _ALLOWED_INVITE_FROM = {"contacts", "anyone"}
 
 # "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de".
 _LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*")
+_UI_LANGUAGE = re.compile(r"[a-z]{2,3}(-[A-Za-z]{2,4})?")
 
 
 class ReminderPref(BaseModel):
@@ -52,6 +53,16 @@ class SettingsOut(BaseModel):
     # ("auto" detects it per message).
     spellcheck_enabled: bool = True
     spellcheck_language: str = "auto"
+    # Gmail-style keyboard shortcuts in the mail view (Settings → Keyboard shortcuts).
+    shortcuts_enabled: bool = True
+    # Remove tracking pixels from mail (Settings → Privacy).
+    block_trackers: bool = True
+    # Grey sentence endings in the compose window, taken with Tab.
+    completion_enabled: bool = True
+    # Interface language ("en", "pl", …); "" until chosen — the web app then
+    # picks one from the browser on first sign-in and saves it here.
+    ui_language: str = ""
+    # Names and words email translation leaves as they are.
 
 
 class SettingsPatch(BaseModel):
@@ -68,6 +79,10 @@ class SettingsPatch(BaseModel):
     invitation_file: bool | None = None
     spellcheck_enabled: bool | None = None
     spellcheck_language: str | None = None
+    shortcuts_enabled: bool | None = None
+    block_trackers: bool | None = None
+    completion_enabled: bool | None = None
+    ui_language: str | None = None
 
 
 class QuotaOut(BaseModel):
@@ -82,6 +97,11 @@ def allowed_image_senders(user: User) -> set[str]:
     return set()
 
 
+def blocks_trackers(user: User) -> bool:
+    """Whether tracking pixels are taken out of this user's mail (on unless switched off)."""
+    return bool(user.settings.get("block_trackers", True))
+
+
 def add_allowed_image_sender(user: User, email: str) -> None:
     current = sorted(allowed_image_senders(user))
     if email.lower() not in current:
@@ -127,6 +147,10 @@ def _settings_out(user: User) -> SettingsOut:
         allowed_image_senders=sorted(allowed_image_senders(user)),
         spellcheck_enabled=bool(s.get("spellcheck_enabled", True)),
         spellcheck_language=str(s.get("spellcheck_language", "auto")),
+        shortcuts_enabled=bool(s.get("shortcuts_enabled", True)),
+        block_trackers=blocks_trackers(user),
+        completion_enabled=bool(s.get("completion_enabled", True)),
+        ui_language=str(s.get("ui_language", "")),
     )
 
 
@@ -175,6 +199,14 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession)
         updates["spellcheck_enabled"] = body.spellcheck_enabled
     if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language):
         updates["spellcheck_language"] = body.spellcheck_language
+    if body.shortcuts_enabled is not None:
+        updates["shortcuts_enabled"] = body.shortcuts_enabled
+    if body.block_trackers is not None:
+        updates["block_trackers"] = body.block_trackers
+    if body.completion_enabled is not None:
+        updates["completion_enabled"] = body.completion_enabled
+    if body.ui_language is not None and _UI_LANGUAGE.fullmatch(body.ui_language):
+        updates["ui_language"] = body.ui_language
     user.settings = updates
     await db.commit()
     return _settings_out(user)

backend/app/api/snooze.py

@@ -0,0 +1,76 @@
+"""Snooze endpoints: hide a conversation from the Inbox until a set time."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import snooze
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.events import get_event_bus
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Thread
+
+router = APIRouter(prefix="/threads", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class SnoozeIn(BaseModel):
+    until: dt.datetime
+
+
+class SnoozeOut(BaseModel):
+    thread_id: uuid.UUID
+    snoozed_until: dt.datetime | None
+
+
+async def _thread(db: AsyncSession, user_id: uuid.UUID, thread_id: uuid.UUID) -> Thread:
+    thread = await db.scalar(
+        select(Thread).where(Thread.id == thread_id, Thread.user_id == user_id)
+    )
+    if thread is None:
+        raise_api_error(404, "not_found", "thread not found")
+    return thread
+
+
+async def _announce(user_id: uuid.UUID, thread_id: uuid.UUID) -> None:
+    await get_event_bus().publish(
+        "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}
+    )
+
+
+@router.post("/{thread_id}/snooze", response_model=SnoozeOut)
+async def snooze_thread(
+    thread_id: uuid.UUID, body: SnoozeIn, user: CurrentUser, db: DbSession
+) -> SnoozeOut:
+    until = body.until
+    if until.tzinfo is None:
+        raise_api_error(400, "invalid_time", "the time needs a time zone")
+    now = dt.datetime.now(dt.UTC)
+    if until <= now:
+        raise_api_error(400, "invalid_time", "pick a time in the future")
+    if until > now + snooze.MAX_SNOOZE:
+        raise_api_error(400, "invalid_time", "snooze for at most a year")
+    thread = await _thread(db, user.id, thread_id)
+    snooze.snooze(thread, until, now)
+    await db.commit()
+    await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=thread.snoozed_until)
+
+
+@router.post("/{thread_id}/unsnooze", response_model=SnoozeOut)
+async def unsnooze_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -> SnoozeOut:
+    thread = await _thread(db, user.id, thread_id)
+    if thread.snoozed_until is not None:
+        snooze.wake(thread, dt.datetime.now(dt.UTC))
+        await db.commit()
+        await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=None)

backend/app/api/subscriptions.py

@@ -0,0 +1,372 @@
+"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import case, func, literal, or_, select
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.elements import ColumnElement
+
+from app import subscriptions as subs
+from app.accounts.ssrf import guarded_request
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Mailbox, Message, SubscriptionState
+from app.outbox.schemas import DraftIn, DraftPayload
+from app.outbox.service import DraftError, create_outbox, upsert_draft
+from app.queue import enqueue_job
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+# RFC 8058 senders should stop within two days of a one-click request.
+GRACE = dt.timedelta(days=2)
+# Not counted: mail the user sent, and mail already thrown away (Trash, Spam) —
+# a sender whose mail was all deleted drops off the list.
+_LEFT_OUT = ("sent", "drafts", "trash", "junk")
+
+
+class SubscriptionOut(BaseModel):
+    key: str
+    name: str
+    email: str
+    list_id: str | None
+    total: int
+    unread: int
+    last_30_days: int
+    per_month: float
+    first_at: dt.datetime
+    last_at: dt.datetime
+    last_subject: str
+    one_click: bool
+    has_mailto: bool
+    has_web: bool
+    unsubscribed_at: dt.datetime | None
+    method: str | None
+    still_sending: bool
+
+
+class SubscriptionListOut(BaseModel):
+    subscriptions: list[SubscriptionOut]  # one page
+    total: int  # senders in all
+    pending: int  # messages whose headers are still being read
+
+
+class KeyIn(BaseModel):
+    key: str
+
+
+class UnsubscribeIn(KeyIn):
+    # From the reading pane: this message's own header, wherever it is filed
+    # (Spam included, which the page's groups leave out).
+    message_id: uuid.UUID | None = None
+
+
+class UnsubscribeOut(BaseModel):
+    status: str  # done | open
+    method: str  # one_click | mailto | web
+    url: str | None = None
+
+
+def _sender() -> ColumnElement[Any]:
+    return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, ""))
+
+
+def _key() -> ColumnElement[Any]:
+    return case(
+        (Message.list_id.isnot(None), literal("list:") + Message.list_id),
+        else_=literal("from:") + _sender(),
+    )
+
+
+def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]:
+    return [
+        Message.user_id == user_id,
+        Message.list_unsubscribe.isnot(None),
+        Message.list_unsubscribe != "",
+        or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)),
+    ]
+
+
+async def _start_backfill(user_id: uuid.UUID) -> None:
+    # One job per user at a time: arq refuses a second while the first exists.
+    await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}")
+
+
+@router.get("", response_model=SubscriptionListOut)
+async def list_subscriptions(
+    user: CurrentUser,
+    db: DbSession,
+    offset: int = Query(0, ge=0),
+    limit: int = Query(50, ge=1, le=200),
+    order: str = Query("desc"),  # desc: heard from most recently first | asc: longest ago
+) -> SubscriptionListOut:
+    """One page of senders, by when their latest mail came."""
+    pending = await subs.pending_count(db, user.id)
+    if pending:
+        await _start_backfill(user.id)
+
+    now = dt.datetime.now(dt.UTC)
+    key = _key().label("key")
+    seen = Message.flags.contains(["\\Seen"])
+    groups = (
+        select(
+            key,
+            func.count().label("total"),
+            func.count().filter(~seen).label("unread"),
+            func.count()
+            .filter(Message.internal_date > now - dt.timedelta(days=30))
+            .label("recent"),
+            func.min(Message.internal_date).label("first_at"),
+            func.max(Message.internal_date).label("last_at"),
+        )
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id))
+        .group_by(key)
+    )
+    total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0
+    agg_rows = (
+        await db.execute(
+            groups.order_by(
+                func.max(Message.internal_date).asc()
+                if order == "asc"
+                else func.max(Message.internal_date).desc(),
+                key,
+            )
+            .offset(offset)
+            .limit(limit)
+        )
+    ).all()
+    page_keys = [r.key for r in agg_rows]
+    if not page_keys:
+        return SubscriptionListOut(subscriptions=[], total=total, pending=pending)
+
+    in_page = _key().in_(page_keys)
+    latest_rows = (
+        await db.execute(
+            select(
+                key,
+                Message.from_addr,
+                Message.subject,
+                Message.list_id,
+                Message.list_unsubscribe,
+                Message.list_unsubscribe_post,
+            )
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(*_bulk(user.id), in_page)
+            .distinct(key)
+            .order_by(key, Message.internal_date.desc())
+        )
+    ).all()
+    latest = {r.key: r for r in latest_rows}
+    states = {
+        s.key: s
+        for s in await db.scalars(
+            select(SubscriptionState).where(
+                SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys)
+            )
+        )
+    }
+    # Senders still mailing after the grace period, for the whole page at once.
+    still_sending = set(
+        await db.scalars(
+            select(key)
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .join(
+                SubscriptionState,
+                (SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()),
+            )
+            .where(
+                *_bulk(user.id),
+                in_page,
+                Message.internal_date > SubscriptionState.unsubscribed_at + GRACE,
+            )
+            .distinct()
+        )
+    )
+
+    out: list[SubscriptionOut] = []
+    for row in agg_rows:
+        last = latest.get(row.key)
+        if last is None:
+            continue
+        sender = last.from_addr[0] if last.from_addr else {}
+        targets = subs.unsubscribe_targets(last.list_unsubscribe or "")
+        state = states.get(row.key)
+        months = max(1.0, (row.last_at - row.first_at).days / 30.44)
+        out.append(
+            SubscriptionOut(
+                key=row.key,
+                name=sender.get("name") or sender.get("email", ""),
+                email=sender.get("email", ""),
+                list_id=last.list_id,
+                total=row.total,
+                unread=row.unread,
+                last_30_days=row.recent,
+                per_month=round(row.total / months, 1),
+                first_at=row.first_at,
+                last_at=row.last_at,
+                last_subject=last.subject,
+                one_click=bool(last.list_unsubscribe_post and targets.https),
+                has_mailto=targets.mailto is not None,
+                has_web=targets.https is not None,
+                unsubscribed_at=state.unsubscribed_at if state else None,
+                method=state.method if state else None,
+                still_sending=row.key in still_sending,
+            )
+        )
+    return SubscriptionListOut(subscriptions=out, total=total, pending=pending)
+
+
+@router.get("/threads", response_model=list[uuid.UUID])
+async def subscription_threads(
+    user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200)
+) -> list[uuid.UUID]:
+    """The group's conversations, newest first (for the reading pane)."""
+    rows = await db.execute(
+        select(Message.thread_id, func.max(Message.internal_date).label("at"))
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None))
+        .group_by(Message.thread_id)
+        .order_by(func.max(Message.internal_date).desc())
+        .limit(limit)
+    )
+    return [r.thread_id for r in rows if r.thread_id is not None]
+
+
+async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message:
+    message = await db.scalar(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user_id), _key() == key)
+        .order_by(Message.internal_date.desc())
+        .limit(1)
+    )
+    if message is None:
+        raise_api_error(404, "not_found", "no mail from this sender")
+    return message
+
+
+async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None:
+    stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method)
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["user_id", "key"],
+            set_={"method": method, "unsubscribed_at": func.now()},
+        )
+    )
+    await db.commit()
+
+
+async def _one_click(url: str) -> bool:
+    try:
+        resp = await guarded_request(
+            "POST",
+            url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            content="List-Unsubscribe=One-Click",
+            max_bytes=256 * 1024,
+        )
+    except Exception as exc:  # SSRF refusal, network, TLS: any failure tries the next way
+        logger.info("one-click unsubscribe failed: %s", exc)
+        return False
+    return 200 <= resp.status < 400
+
+
+async def _send_mailto(
+    db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets
+) -> bool:
+    assert targets.mailto is not None
+    body = DraftIn(
+        account_id=message.account_id,
+        kind="new",
+        payload=DraftPayload(
+            to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body
+        ),
+    )
+    try:
+        draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None)
+    except DraftError:
+        return False
+    outbox = await create_outbox(db, user_id=user_id, draft=draft)
+    await db.commit()
+    await enqueue_job(
+        "send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds)
+    )
+    return True
+
+
+@router.post("/unsubscribe", response_model=UnsubscribeOut)
+async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut:
+    """Unsubscribe from a sender: one-click if offered, else by mail, else the web page."""
+    if body.message_id is not None:
+        found = await db.scalar(
+            select(Message).where(
+                Message.id == body.message_id,
+                Message.user_id == user.id,
+                Message.list_unsubscribe.isnot(None),
+                Message.list_unsubscribe != "",
+            )
+        )
+        if found is None:
+            raise_api_error(404, "not_found", "this message offers no way to unsubscribe")
+        message = found
+    else:
+        message = await _latest_with_header(db, user.id, body.key)
+    targets = subs.unsubscribe_targets(message.list_unsubscribe or "")
+    if message.list_unsubscribe_post and targets.https and await _one_click(targets.https):
+        await _record(db, user.id, body.key, "one_click")
+        return UnsubscribeOut(status="done", method="one_click")
+    if targets.mailto and await _send_mailto(db, user.id, message, targets):
+        await _record(db, user.id, body.key, "mailto")
+        return UnsubscribeOut(status="done", method="mailto")
+    if targets.https:
+        # The sender wants it done on their page: the user opens it.
+        await _record(db, user.id, body.key, "web")
+        return UnsubscribeOut(status="open", method="web", url=targets.https)
+    raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe")
+
+
+@router.post("/trash")
+async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]:
+    """Move every conversation from this sender to Trash.
+
+    Only conversations made up entirely of this sender's mail: one where the
+    user replied, or someone else wrote, stays where it is.
+    """
+    rows = (
+        await db.execute(
+            select(Message.thread_id, _key().label("key"), Mailbox.special_use)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(
+                Message.user_id == user.id,
+                Message.thread_id.in_(
+                    select(Message.thread_id)
+                    .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                    .where(*_bulk(user.id), _key() == body.key)
+                ),
+            )
+        )
+    ).all()
+    own: dict[uuid.UUID, bool] = {}
+    for thread_id, key, special in rows:
+        if thread_id is None or special == "trash":
+            continue
+        own[thread_id] = own.get(thread_id, True) and key == body.key
+    moved = [tid for tid, only_theirs in own.items() if only_theirs]
+    for tid in moved:
+        await enqueue_job("move_thread_to_folder_job", str(tid), "trash")
+    return {"moved": len(moved)}

backend/app/api/templates.py

@@ -0,0 +1,129 @@
+"""Templates: saved replies, inserted from the compose window's More menu."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+import nh3
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import MailTemplate
+
+router = APIRouter(prefix="/templates", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+MAX_TEMPLATES = 200
+MAX_BODY_BYTES = 256 * 1024
+
+# What the compose editor writes. A template goes back into the editor as HTML,
+# so it is cleaned like any HTML the app renders: no scripts, no handlers.
+_TAGS = {
+    "a", "b", "blockquote", "br", "code", "div", "em", "font", "h1", "h2", "h3",
+    "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strike", "strong",
+    "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "u", "ul",
+}  # fmt: skip
+_GENERIC = {"style", "class", "dir", "align"}
+_SPECIFIC = {
+    "a": {"href", "title"},
+    "img": {"src", "alt", "width", "height"},
+    "font": {"color", "face", "size"},
+    "td": {"colspan", "rowspan"},
+    "th": {"colspan", "rowspan"},
+}
+# nh3 takes no wildcard: the generic attributes go onto every tag.
+_ATTRS = {tag: _GENERIC | _SPECIFIC.get(tag, set()) for tag in _TAGS}
+
+
+class TemplateOut(BaseModel):
+    id: uuid.UUID
+    name: str
+    subject: str
+    body_html: str
+    updated_at: dt.datetime
+
+
+class TemplateIn(BaseModel):
+    name: str = Field(min_length=1, max_length=80)
+    subject: str = Field(default="", max_length=500)
+    body_html: str = ""
+
+
+def _out(t: MailTemplate) -> TemplateOut:
+    return TemplateOut(
+        id=t.id, name=t.name, subject=t.subject, body_html=t.body_html, updated_at=t.updated_at
+    )
+
+
+def clean_body(html: str) -> str:
+    return nh3.clean(
+        html,
+        tags=_TAGS,
+        attributes=_ATTRS,
+        url_schemes={"http", "https", "mailto", "data"},
+        link_rel="noopener noreferrer",
+    )
+
+
+@router.get("", response_model=list[TemplateOut])
+async def list_templates(user: CurrentUser, db: DbSession) -> list[TemplateOut]:
+    rows = await db.scalars(
+        select(MailTemplate)
+        .where(MailTemplate.user_id == user.id)
+        .order_by(func.lower(MailTemplate.name))
+    )
+    return [_out(t) for t in rows]
+
+
+@router.post("", response_model=TemplateOut)
+async def save_template(body: TemplateIn, user: CurrentUser, db: DbSession) -> TemplateOut:
+    """Save a template. Saving under a name that exists replaces that template."""
+    name = " ".join(body.name.split())
+    if not name:
+        raise_api_error(400, "invalid", "a template needs a name")
+    if len(body.body_html.encode()) > MAX_BODY_BYTES:
+        raise_api_error(400, "too_large", "the template is too large")
+    html = clean_body(body.body_html)
+    subject = " ".join(body.subject.split())
+    existing = await db.scalar(
+        select(MailTemplate).where(
+            MailTemplate.user_id == user.id, func.lower(MailTemplate.name) == name.lower()
+        )
+    )
+    if existing is not None:
+        existing.name, existing.subject, existing.body_html = name, subject, html
+        existing.updated_at = dt.datetime.now(dt.UTC)
+        saved = existing
+    else:
+        count = await db.scalar(
+            select(func.count()).select_from(MailTemplate).where(MailTemplate.user_id == user.id)
+        )
+        if (count or 0) >= MAX_TEMPLATES:
+            raise_api_error(409, "too_many", f"at most {MAX_TEMPLATES} templates")
+        saved = MailTemplate(user_id=user.id, name=name, subject=subject, body_html=html)
+        db.add(saved)
+    await db.commit()
+    await db.refresh(saved)
+    return _out(saved)
+
+
+@router.delete("/{template_id}")
+async def delete_template(
+    template_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    row = await db.scalar(
+        select(MailTemplate).where(MailTemplate.id == template_id, MailTemplate.user_id == user.id)
+    )
+    if row is None:
+        raise_api_error(404, "not_found", "template not found")
+    await db.delete(row)
+    await db.commit()
+    return {"status": "deleted"}

backend/app/api/threads.py

@@ -4,13 +4,14 @@ from __future__ import annotations
 
 import uuid
 from collections.abc import Collection
-from typing import Annotated
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Query
 from sqlalchemy import Text, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.sql.elements import ColumnElement
 
+from app import subscriptions as subs
 from app.api.errors import raise_api_error
 from app.api.schemas import (
     Address,
@@ -19,10 +20,10 @@ from app.api.schemas import (
     ThreadListOut,
     ThreadSummary,
 )
-from app.api.settings import allowed_image_senders
+from app.api.settings import allowed_image_senders, blocks_trackers
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Mailbox, Message, Thread, User
+from app.models import BodyPart, Mailbox, Message, SubscriptionState, Thread, User
 
 router = APIRouter(prefix="/threads", tags=["mail"])
 
@@ -50,6 +51,7 @@ def _summary(
         has_attachments=thread.has_attachments,
         pinned=thread.pinned,
         last_message_at=thread.last_message_at,
+        snoozed_until=thread.snoozed_until,
     )
 
 
@@ -74,6 +76,7 @@ def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnv
         images_allowed=bool(sender) and sender in allowed_senders,
         size_bytes=msg.size_bytes,
         snippet=msg.snippet,
+        content_language=msg.content_language or "",
     )
 
 
@@ -130,6 +133,10 @@ async def list_threads(
             .where(Message.user_id == user.id, Mailbox.special_use == special)
         )
         conditions.append(Thread.id.in_(in_folder))
+        if special == "inbox":
+            conditions.append(Thread.snoozed_until.is_(None))  # back when it wakes
+    elif view == "snoozed":
+        conditions.append(Thread.snoozed_until.isnot(None))
     elif view != "unified":
         raise_api_error(400, "bad_view", "unsupported view")
 
@@ -144,11 +151,20 @@ async def list_threads(
         )
     ) or 0
 
-    order_by = (
-        (Thread.last_message_at.asc(), Thread.id.asc())
-        if order == "asc"
-        else (Thread.last_message_at.desc(), Thread.id.desc())
+    # A thread woken from snooze sorts as if it had just arrived.
+    sort_at = func.greatest(
+        Thread.last_message_at, func.coalesce(Thread.unsnoozed_at, Thread.last_message_at)
     )
+    if view == "snoozed":
+        # Soonest back first, whatever the list order.
+        order_by: tuple[ColumnElement[Any], ...] = (
+            Thread.snoozed_until.asc(),
+            Thread.id.asc(),
+        )
+    elif order == "asc":
+        order_by = (sort_at.asc(), Thread.id.asc())
+    else:
+        order_by = (sort_at.desc(), Thread.id.desc())
     stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit)
     threads = list(await db.scalars(stmt))
 
@@ -177,6 +193,41 @@ async def list_threads(
     )
 
 
+async def _mark_subscriptions(
+    db: AsyncSession,
+    user_id: uuid.UUID,
+    messages: list[Message],
+    envelopes: list[MessageEnvelope],
+) -> None:
+    """Tell the reading pane which messages can be unsubscribed from, and how."""
+    keys: dict[uuid.UUID, str] = {}
+    for m, env in zip(messages, envelopes, strict=True):
+        if not m.list_unsubscribe:
+            continue
+        targets = subs.unsubscribe_targets(m.list_unsubscribe)
+        if m.list_unsubscribe_post and targets.https:
+            env.unsubscribe_via = "one_click"
+        elif targets.mailto:
+            env.unsubscribe_via = "mailto"
+        elif targets.https:
+            env.unsubscribe_via = "web"
+        else:
+            continue
+        sender = m.from_addr[0].get("email", "") if m.from_addr else ""
+        env.subscription_key = keys[m.id] = subs.group_key(m.list_id, sender)
+    if keys:
+        done = set(
+            await db.scalars(
+                select(SubscriptionState.key).where(
+                    SubscriptionState.user_id == user_id,
+                    SubscriptionState.key.in_(set(keys.values())),
+                )
+            )
+        )
+        for env in envelopes:
+            env.unsubscribed = env.subscription_key in done
+
+
 def _visible_in(current_special: str | None) -> ColumnElement[bool]:
     """Which of a thread's messages a view shows (see ``_REMOVAL_FOLDERS``): Trash/Spam
     show only their own messages; every other view hides them."""
@@ -281,6 +332,20 @@ async def get_thread(
         env = _envelope(m, allowed_senders)
         env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"]
         envelopes.append(env)
+    await _mark_subscriptions(db, user.id, ordered, envelopes)
+    if not blocks_trackers(user) and ordered:
+        # Trackers left in are remote images: offer "Load images" for them too.
+        with_trackers = set(
+            await db.scalars(
+                select(BodyPart.message_id).where(
+                    BodyPart.message_id.in_([m.id for m in ordered]),
+                    BodyPart.kind == "html_sanitized",
+                    BodyPart.trackers_blocked > 0,
+                )
+            )
+        )
+        for env in envelopes:
+            env.has_remote_images = env.has_remote_images or env.id in with_trackers
 
     order = "desc" if sort == "desc" else "asc"
     return ThreadDetail(
@@ -289,4 +354,5 @@ async def get_thread(
         order=order,
         pinned=thread.pinned,
         messages=envelopes,
+        snoozed_until=thread.snoozed_until,
     )

backend/app/api/translation.py

@@ -0,0 +1,30 @@
+"""What email translation offers (app/translation.py), for the reading pane."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+from app import translation
+from app.identity.deps import CurrentUser
+
+router = APIRouter(prefix="/translation", tags=["mail"])
+
+
+class LanguageOut(BaseModel):
+    code: str
+    name: str
+
+
+class TranslationInfoOut(BaseModel):
+    available: bool
+    languages: list[LanguageOut]
+
+
+@router.get("/languages", response_model=TranslationInfoOut)
+async def translation_languages(user: CurrentUser) -> TranslationInfoOut:
+    found = await translation.languages()
+    return TranslationInfoOut(
+        available=len(found) > 1,  # English alone translates nothing
+        languages=[LanguageOut(code=x.code, name=x.name) for x in found],
+    )

backend/app/completion.py

@@ -0,0 +1,391 @@
+"""Sentence completion in the compose window: phrases the user writes, finished.
+
+No AI: a per-user index of the word sequences the user has written at least
+twice, built from their own sent mail (quoted replies and signatures cut off),
+plus a few common email phrases per language to start from. Typing "I look
+forward" finds what usually follows those words — "to hearing from you." — and
+the compose window shows it in grey until Tab takes it.
+
+It works in any language the user writes in: matching is on the words just
+typed, so a Polish sentence only ever meets Polish phrases.
+
+The index lives in ``completion_phrases`` (context → continuation, with how
+often it was seen) and is rebuilt in the background at most once a day, when
+the compose window asks for suggestions (:func:`build_index_job`).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import html as html_lib
+import logging
+import re
+import uuid
+from collections import Counter
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import delete, select
+from sqlalchemy.dialects.postgresql import insert
+
+from app.db import session_scope
+from app.models import (
+    Account,
+    BodyPart,
+    CompletionIndexState,
+    CompletionPhrase,
+    Mailbox,
+    Message,
+)
+
+logger = logging.getLogger(__name__)
+
+START = "^"  # a sentence's start, as a word: "^ pozdr" completes the first word
+MAX_CONTEXT = 3  # words before the cursor matched on
+MAX_CONTINUATION = 6  # words suggested at most
+MIN_COUNT = 2  # seen this often to be offered; one-offs are noise (and private)
+EXTEND_SHARE = 0.6  # a longer suggestion only while it is what usually follows
+CORPUS_MESSAGES = 500  # most recent sent messages read
+CORPUS_CHARS = 3000  # of each
+REBUILD_AFTER = dt.timedelta(days=1)
+
+# Where the user's own words end and quoted mail or a signature begins: a reply
+# header ("On … wrote:", Polish Gmail's "… napisał(a):", …), a forwarded or
+# original-message marker, or an Outlook-style header block.
+_QUOTE_START = re.compile(
+    r"^\s*(?:"
+    r"-{2,}\s*(?:original message|forwarded message|wiadomość oryginalna"
+    r"|wiadomość przekazana dalej|ursprüngliche nachricht|weitergeleitete nachricht)"
+    r"|(?:from|od|von|de):\s.+"
+    r"|sent from my\s"
+    r")",
+    re.IGNORECASE,
+)
+_QUOTE_HEADER_END = re.compile(
+    # German names the writer after the verb: "Am … schrieb Anna:".
+    r"(?:wrote|napisał\(a\)|napisała?|pisze|schrieb(?:\s+.{0,100})?|a écrit|escribió)\s*:\s*$",
+    re.IGNORECASE,
+)
+_SIGNATURE = re.compile(r"^--\s*$")
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+|\n+")
+_WORD = re.compile(r"\S+")
+_EDGE_PUNCT = re.compile(r"^[\"'“”„«»(\[–—-]+|[\"'“”„«»)\],.;:!?…–—-]+$")
+# Words a phrase is never learned across: numbers (phones, dates, prices),
+# addresses and links — useless to complete, and often private.
+_NOT_A_WORD = re.compile(r"\d|@|www\.|https?:", re.IGNORECASE)
+
+
+def html_to_text(html: str) -> str:
+    """An HTML signature as text lines (block tags and <br> break lines)."""
+    text = re.sub(r"(?i)<br\s*/?>|</(?:p|div|li|tr|h[1-6])\s*>", "\n", html)
+    text = re.sub(r"<[^>]+>", "", text)
+    return html_lib.unescape(text)
+
+
+def own_text(text: str, signatures: Iterable[str] = ()) -> str:
+    """The part of a sent message the user wrote: no quoted mail, no signature
+    (the ``--`` kind, or one of ``signatures`` — the accounts' own, as text)."""
+    sig_lines = {norm_line(x) for sig in signatures for x in sig.splitlines() if norm_line(x)}
+    kept: list[str] = []
+    for line in text.splitlines():
+        if _SIGNATURE.match(line) or _QUOTE_START.match(line) or _QUOTE_HEADER_END.search(line):
+            break
+        if norm_line(line) in sig_lines:
+            continue
+        if line.lstrip().startswith(">"):
+            continue  # an inline-quoted line
+        kept.append(line)
+    return "\n".join(kept)
+
+
+def norm_line(line: str) -> str:
+    return " ".join(line.lower().split())
+
+
+def norm(word: str) -> str:
+    """A word as matched: lower case, outer punctuation off ("Hello," -> "hello")."""
+    return _EDGE_PUNCT.sub("", word.lower())
+
+
+def sentences(text: str) -> Iterable[list[str]]:
+    for chunk in _SENTENCE_END.split(text):
+        words = _WORD.findall(chunk)
+        if words:
+            yield words
+
+
+@dataclass(frozen=True)
+class Phrase:
+    context: str  # normalized words before the cursor, space-joined ("^ i look")
+    continuation: str  # the words that follow, as written ("forward to hearing")
+    count: int
+
+
+def build_phrases(texts: Iterable[str], weight: int = 1) -> dict[tuple[str, str], int]:
+    """(context, continuation) -> count, for every sequence seen MIN_COUNT times.
+
+    Counts each sentence's word sequences (with the start marker), keeps the
+    frequent ones, and splits each into "what was typed" and "what follows".
+    """
+    counts: Counter[tuple[str, ...]] = Counter()
+    surface: dict[tuple[str, ...], tuple[str, ...]] = {}
+    longest = MAX_CONTEXT + MAX_CONTINUATION
+    for text in texts:
+        for words in sentences(text):
+            seq = [START, *words]
+            keys = [START, *("" if _NOT_A_WORD.search(w) else norm(w) for w in words)]
+            for i in range(len(seq)):
+                for n in range(2, min(longest, len(seq) - i) + 1):
+                    gram = tuple(keys[i : i + n])
+                    if "" in gram:
+                        break  # a lone punctuation mark: no phrase across it
+                    counts[gram] += weight
+                    surface.setdefault(gram, tuple(seq[i : i + n]))
+    out: dict[tuple[str, str], int] = {}
+    for gram, count in counts.items():
+        if count < MIN_COUNT:
+            continue
+        written = surface[gram]
+        for k in range(1, min(MAX_CONTEXT, len(gram) - 1) + 1):
+            rest = len(gram) - k
+            if rest > MAX_CONTINUATION:
+                continue
+            key = (" ".join(gram[:k]), " ".join(written[k:]))
+            out[key] = max(out.get(key, 0), count)
+    return out
+
+
+# A few common phrases per language, for users with little sent mail yet.
+STARTER_PHRASES: dict[str, list[str]] = {
+    "en": [
+        "I look forward to hearing from you.",
+        "I'm looking forward to hearing from you.",
+        "Looking forward to hearing from you.",
+        "Looking forward to working with you.",
+        "Thank you for your email.",
+        "Thank you for your help.",
+        "Thank you for your quick reply.",
+        "Thanks in advance.",
+        "Please let me know if you have any questions.",
+        "Let me know if you have any questions.",
+        "Let me know what you think.",
+        "Please find attached the",
+        "I hope this email finds you well.",
+        "I hope you're doing well.",
+        "Sorry for the late reply.",
+        "I would be happy to help.",
+        "Feel free to contact me if you need anything else.",
+        "Could you please send me the",
+        "Please confirm at your earliest convenience.",
+        "Have a great day!",
+        "Have a nice weekend!",
+        "Best regards,",
+        "Kind regards,",
+    ],
+    "pl": [
+        "Z poważaniem,",
+        "Pozdrawiam serdecznie,",
+        "Dziękuję za wiadomość.",
+        "Dziękuję za szybką odpowiedź.",
+        "Dziękuję za pomoc.",
+        "Z góry dziękuję.",
+        "W załączniku przesyłam",
+        "Proszę o informację zwrotną.",
+        "W razie pytań proszę o kontakt.",
+        "Czekam na odpowiedź.",
+        "Przepraszam za opóźnienie w odpowiedzi.",
+        "Będę wdzięczny za odpowiedź.",
+        "Będę wdzięczna za odpowiedź.",
+        "Zgodnie z naszą rozmową",
+        "Proszę o potwierdzenie.",
+        "Daj znać, co myślisz.",
+        "Miłego dnia!",
+        "Miłego weekendu!",
+    ],
+    "de": [
+        "Mit freundlichen Grüßen",
+        "Viele Grüße",
+        "Beste Grüße",
+        "Vielen Dank für Ihre E-Mail.",
+        "Vielen Dank für Ihre schnelle Antwort.",
+        "Vielen Dank im Voraus.",
+        "Ich freue mich auf Ihre Antwort.",
+        "Bei Fragen stehe ich Ihnen gerne zur Verfügung.",
+        "Im Anhang finden Sie",
+        "Entschuldigen Sie die späte Antwort.",
+        "Ich hoffe, es geht Ihnen gut.",
+        "Schönen Tag noch!",
+        "Schönes Wochenende!",
+    ],
+}
+
+# Built once: every starter phrase counts as seen MIN_COUNT times.
+_STARTERS: dict[str, Counter[str]] = {}
+for (_ctx, _cont), _n in build_phrases(
+    (p for phrases in STARTER_PHRASES.values() for p in phrases), weight=MIN_COUNT
+).items():
+    _STARTERS.setdefault(_ctx, Counter())[_cont] = _n
+
+_LEADING_PUNCT = re.compile(r"^[\"'“”„«»(\[]+")
+
+
+def _candidates(phrases: dict[str, Counter[str]], context: str, partial: str) -> Counter[str]:
+    """Continuations of ``context`` whose first word starts with ``partial``."""
+    out: Counter[str] = Counter()
+    for continuation, count in phrases.get(context, Counter()).items():
+        first = continuation.split(" ", 1)[0]
+        if partial and not first.lower().startswith(partial.lower()):
+            continue
+        if partial and len(first) <= len(partial):
+            continue  # that word is typed in full already
+        out[continuation] = count
+    return out
+
+
+def choose(candidates: Counter[str]) -> str | None:
+    """The suggestion: the most frequent next word, extended word by word only
+    while the longer phrase is what usually follows (EXTEND_SHARE)."""
+    if not candidates:
+        return None
+    by_first: Counter[str] = Counter()
+    for continuation, count in candidates.items():
+        first = continuation.split(" ", 1)[0]
+        by_first[first] = max(by_first[first], count)
+    chosen, chosen_count = by_first.most_common(1)[0]
+    while True:
+        depth = chosen.count(" ") + 1
+        longer = [
+            (c, n)
+            for c, n in candidates.items()
+            if c.startswith(chosen + " ") and c.count(" ") == depth
+        ]
+        if not longer:
+            return chosen
+        best, best_count = max(longer, key=lambda x: x[1])
+        if best_count < max(MIN_COUNT, EXTEND_SHARE * chosen_count):
+            return chosen
+        chosen, chosen_count = best, best_count
+
+
+def contexts(before: str) -> tuple[list[str], str]:
+    """For the text before the cursor: the contexts to try, longest first, and
+    the partly typed word ('' after a space). No contexts: nothing to suggest
+    (an empty sentence, or the cursor right after a punctuation mark)."""
+    if not before or not before.strip():
+        return [], ""
+    last = before[-1]
+    if not last.isspace() and not last.isalnum() and last not in "'’-":
+        return [], ""  # "you," or "end." — not mid-word, and no space typed yet
+    tail = _SENTENCE_END.split(before)[-1]
+    words = _WORD.findall(tail)
+    partial = ""
+    if words and not last.isspace():
+        partial = _LEADING_PUNCT.sub("", words.pop())
+    if not words and not partial:
+        return [], ""  # a new sentence with nothing in it yet
+    keys = [START, *(norm(w) for w in words)]
+    if "" in keys:  # a lone punctuation mark: the phrase starts after it
+        keys = keys[len(keys) - keys[::-1].index("") :]
+    if not keys:
+        return [], ""
+    return [" ".join(keys[-k:]) for k in range(min(MAX_CONTEXT, len(keys)), 0, -1)], partial
+
+
+def suggest(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """What to show after ``before`` (the text up to the cursor), or None.
+
+    The returned text starts where the cursor is: the rest of a partly typed
+    word, or (after a space) the next words. The user's own phrases and the
+    starter phrases count together.
+    """
+    tried, partial = contexts(before)
+    for context in tried:
+        merged = _candidates(personal, context, partial)
+        for cont, count in _candidates(_STARTERS, context, partial).items():
+            merged[cont] = merged.get(cont, 0) + count
+        if " " not in context and not partial:
+            # A single word before a space is a weak hint: only a strong habit.
+            merged = Counter({c: n for c, n in merged.items() if n >= 2 * MIN_COUNT})
+        best = choose(merged)
+        if best:
+            return best[len(partial) :] if partial else best
+    return None
+
+
+def complete(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """:func:`suggest`, and for a word typed in full but no space yet ("I look
+    forward"), what follows it — with the space ("␣to hearing from you.")."""
+    found = suggest(personal, before)
+    if found is None and before[-1:].isalnum():
+        after_space = suggest(personal, before + " ")
+        if after_space:
+            return " " + after_space
+    return found
+
+
+def keys_for(before: str) -> list[str]:
+    """The contexts :func:`complete` may look up, to fetch them from the index."""
+    keys = contexts(before)[0]
+    if before[-1:].isalnum():
+        keys += contexts(before + " ")[0]
+    return keys
+
+
+async def personal_phrases(
+    session: Any, user_id: uuid.UUID, keys: list[str]
+) -> dict[str, Counter[str]]:
+    rows = await session.execute(
+        select(
+            CompletionPhrase.context, CompletionPhrase.continuation, CompletionPhrase.count
+        ).where(CompletionPhrase.user_id == user_id, CompletionPhrase.context.in_(keys))
+    )
+    out: dict[str, Counter[str]] = {}
+    for context, continuation, count in rows:
+        out.setdefault(context, Counter())[continuation] = count
+    return out
+
+
+async def build_index_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Rebuild a user's phrase index from their recent sent mail. Returns its size."""
+    uid = uuid.UUID(user_id)
+    async with session_scope() as session:
+        texts = list(
+            await session.scalars(
+                select(BodyPart.content)
+                .join(Message, Message.id == BodyPart.message_id)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.user_id == uid,
+                    Mailbox.special_use == "sent",
+                    BodyPart.kind == "text",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(CORPUS_MESSAGES)
+            )
+        )
+        signatures = [
+            html_to_text(sig)
+            for sig in await session.scalars(
+                select(Account.signature).where(Account.user_id == uid, Account.signature != "")
+            )
+        ]
+        phrases = build_phrases(own_text(t, signatures)[:CORPUS_CHARS] for t in texts)
+        await session.execute(delete(CompletionPhrase).where(CompletionPhrase.user_id == uid))
+        rows = [
+            {"user_id": uid, "context": c, "continuation": cont, "count": n}
+            for (c, cont), n in phrases.items()
+        ]
+        for start in range(0, len(rows), 1000):
+            await session.execute(insert(CompletionPhrase), rows[start : start + 1000])
+        stmt = insert(CompletionIndexState).values(user_id=uid, built_at=dt.datetime.now(dt.UTC))
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=["user_id"], set_={"built_at": stmt.excluded.built_at}
+            )
+        )
+        await session.commit()
+    logger.info(
+        "completion index for %s: %d phrases from %d messages", user_id, len(rows), len(texts)
+    )
+    return len(rows)

backend/app/config.py

@@ -71,11 +71,53 @@ class Settings(BaseSettings):
     # Spelling and grammar checking in the compose window (LanguageTool server,
     # e.g. http://languagetool:8010). Empty: the check is off for everyone.
     languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL")
+    # Email translation: the translator service (translator/, app/translation.py),
+    # e.g. http://translator:5000. Empty: translation is off.
+    translator_url: str = Field(default="", alias="TRANSLATOR_URL")
+    # New passwords are checked against known data breaches (Have I Been Pwned's
+    # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server).
+    password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK")
+    # The name the webmail gives itself when it connects to a mail server to
+    # send (SMTP EHLO). Empty: APP_DOMAIN — not the container's random name.
+    smtp_helo_name: str = Field(default="", alias="SMTP_HELO_NAME")
     # Auto-purge Trash/Spam messages older than this many days (0 disables).
     trash_retention_days: int = Field(default=30, alias="TRASH_RETENTION_DAYS")
 
+    # --- Hosted mailboxes (SPEC Appendix D, app/hosted/) -----------------------
+    # Off: no mail server, byte-identical to before (every account external).
+    hosted_mailboxes: bool = Field(default=False, alias="HOSTED_MAILBOXES")
+    # The mail server's management API (JMAP), signed in to as its recovery admin.
+    stalwart_url: str = Field(default="http://stalwart:8080", alias="STALWART_URL")
+    stalwart_admin_secret: str = Field(default="", alias="STALWART_ADMIN_SECRET")
+    # The webmail worker's own sign-in, which may act for any mailbox
+    # (``user%worker``): no user's password is ever stored (§D.3).
+    stalwart_master_secret: str = Field(default="", alias="STALWART_MASTER_SECRET")
+    # The mail server's role on Postgres (its database, and the directory view).
+    stalwart_db_password: str = Field(default="", alias="STALWART_DB_PASSWORD")
+    stalwart_db_name: str = Field(default="stalwart", alias="STALWART_DB_NAME")
+    # Where the mail server reaches Postgres (its own network view of it).
+    stalwart_pg_host: str = Field(default="postgres", alias="STALWART_PG_HOST")
+    # IMAP/SMTP as the webmail reaches them (TLS, certificate not checked: the
+    # traffic stays on the Docker network or loopback).
+    stalwart_imap_host: str = Field(default="stalwart", alias="STALWART_IMAP_HOST")
+    stalwart_imap_port: int = Field(default=993, alias="STALWART_IMAP_PORT")
+    stalwart_smtp_host: str = Field(default="stalwart", alias="STALWART_SMTP_HOST")
+    stalwart_smtp_port: int = Field(default=465, alias="STALWART_SMTP_PORT")
+    # The mail server's public name (SMTP greeting, reverse DNS, what mail apps
+    # are told to connect to). Default: mail.APP_DOMAIN.
+    mail_hostname: str = Field(default="", alias="MAIL_HOSTNAME")
+    # Raw messages of hosted mailboxes are kept only in a cache (§D.6 item 2);
+    # beyond this per user, the least recently used go first.
+    raw_cache_mb: int = Field(default=256, alias="RAW_CACHE_MB")
+
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
     """Return the process-wide settings singleton."""
     return Settings()  # values are sourced from the environment
+
+
+def smtp_helo_name() -> str:
+    """The name to introduce ourselves with to SMTP servers (EHLO)."""
+    settings = get_settings()
+    return settings.smtp_helo_name.strip() or settings.app_domain

backend/app/dav/__init__.py

@@ -0,0 +1 @@
+"""CalDAV / CardDAV server (SPEC Appendix E)."""

backend/app/dav/auth.py

@@ -0,0 +1,93 @@
+"""Signing in a DAV client: HTTP Basic with an app password (SPEC App. E.2).
+
+Only app passwords: the account password (with or without 2FA) never works
+here, so a lost phone is one revocation away and 2FA cannot be bypassed. A
+sync makes many requests, each with the credentials; a successful check is
+remembered for a few minutes under a hash of the password, and forgotten as
+soon as the user's app passwords change (``app_passwords`` bumps a counter).
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import datetime as dt
+import hashlib
+import uuid
+
+from sqlalchemy import select, update
+
+from app.db import session_scope
+from app.identity import app_passwords
+from app.identity.service import resolve_handle
+from app.models import User, UserAppPassword
+from app.redis_client import get_redis
+
+CACHE_S = 300
+FAIL_WINDOW_S = 900
+MAX_FAILS_PER_ADDRESS = 10
+MAX_FAILS_PER_IP = 30
+_LAST_USED_EVERY = dt.timedelta(minutes=5)
+
+
+class Throttled(Exception):
+    """Too many failed sign-ins recently."""
+
+
+def _credentials(header: str | None) -> tuple[str, str] | None:
+    if not header or not header.lower().startswith("basic "):
+        return None
+    try:
+        decoded = base64.b64decode(header[6:].strip(), validate=True).decode("utf-8")
+    except (binascii.Error, UnicodeDecodeError):
+        return None
+    login, sep, password = decoded.partition(":")
+    return (login.strip(), password) if sep and login.strip() else None
+
+
+async def authenticate(header: str | None, ip: str) -> User | None:
+    """The user these Basic credentials sign in, or None. Raises Throttled."""
+    creds = _credentials(header)
+    if creds is None:
+        return None
+    login, password = creds
+    redis = get_redis()
+    handle = resolve_handle(login)
+    fail_keys = (f"dav:fail:addr:{handle}", f"dav:fail:ip:{ip}")
+    fails = [int(await redis.get(k) or 0) for k in fail_keys]
+    if fails[0] >= MAX_FAILS_PER_ADDRESS or fails[1] >= MAX_FAILS_PER_IP:
+        raise Throttled
+
+    async with session_scope() as db:
+        user = await db.scalar(select(User).where(User.handle == handle))
+        if user is None or user.blocked_at is not None:
+            await _failed(fail_keys)
+            return None
+        gen = int(await redis.get(app_passwords.generation_key(user.id)) or 0)
+        digest = hashlib.sha256(f"{user.id}:{password}".encode()).hexdigest()
+        cache_key = f"dav:auth:{user.id}:{gen}:{digest}"
+        if await redis.get(cache_key):
+            return user
+        row = await app_passwords.verify(db, user.id, password)
+        if row is None:
+            await _failed(fail_keys)
+            return None
+        await redis.set(cache_key, str(row.id), ex=CACHE_S)
+        now = dt.datetime.now(dt.UTC)
+        if row.last_used_at is None or now - row.last_used_at > _LAST_USED_EVERY:
+            await db.execute(
+                update(UserAppPassword).where(UserAppPassword.id == row.id).values(last_used_at=now)
+            )
+        db.expunge(user)
+        return user
+
+
+async def _failed(keys: tuple[str, ...]) -> None:
+    redis = get_redis()
+    for key in keys:
+        await redis.incr(key)
+        await redis.expire(key, FAIL_WINDOW_S)
+
+
+def user_id_of(user: User) -> uuid.UUID:
+    return user.id

backend/app/dav/provider.py

@@ -0,0 +1,118 @@
+"""What a plugin implements to serve its collections over DAV (SPEC App. E.1).
+
+The core (app/dav/routes.py) speaks the protocol; a provider only knows its
+own data: collections (calendars, address books) and the resources in them
+(one iCalendar object / one vCard each), named, with an ETag. Every method
+gets the signed-in user's id; providers open the user's RLS session
+(``sdk(name).session(user_id)``) themselves.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Literal, Protocol
+
+
+class DavError(Exception):
+    """A request the provider refuses: ``status`` is the HTTP answer
+    (403 read-only, 404 missing, 409 conflict, 412 precondition failed, 415
+    unsupported data); ``condition`` an optional DAV precondition element
+    (e.g. ``{urn:ietf:params:xml:ns:caldav}valid-calendar-data``)."""
+
+    def __init__(self, status: int, message: str = "", condition: str | None = None) -> None:
+        super().__init__(message or str(status))
+        self.status = status
+        self.condition = condition
+
+
+@dataclass
+class Collection:
+    id: str
+    displayname: str
+    sync_token: int  # the latest change number; also the CTag
+    description: str = ""
+    color: str | None = None  # "#rrggbb"
+    read_only: bool = False
+    components: list[str] = field(default_factory=lambda: ["VEVENT"])  # calendars only
+
+
+@dataclass
+class Resource:
+    name: str  # last path segment, e.g. "abc.ics"
+    etag: str  # without quotes
+    data: str | None = None  # the iCalendar / vCard text, when asked for
+
+
+@dataclass
+class Changes:
+    sync_token: int
+    changed: list[str]  # resource names changed or added since the old token
+    deleted: list[str]  # resource names gone since the old token
+
+
+class DavProvider(Protocol):
+    kind: Literal["calendar", "addressbook"]
+    content_type: str  # "text/calendar; charset=utf-8" / "text/vcard; charset=utf-8"
+    extension: str  # ".ics" / ".vcf"
+
+    async def collections(self, user_id: uuid.UUID) -> list[Collection]: ...
+
+    async def resources(self, user_id: uuid.UUID, collection_id: str) -> list[Resource]: ...
+
+    async def get(
+        self, user_id: uuid.UUID, collection_id: str, names: list[str] | None
+    ) -> list[Resource]: ...
+
+    async def put(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        name: str,
+        data: str,
+        *,
+        if_match: str | None,
+        if_none_match: bool,
+    ) -> tuple[str, bool]:
+        """Store ``data`` as ``name``; returns (new ETag, created)."""
+        ...
+
+    async def delete(
+        self, user_id: uuid.UUID, collection_id: str, name: str, *, if_match: str | None
+    ) -> None: ...
+
+    async def changes(self, user_id: uuid.UUID, collection_id: str, since: int) -> Changes | None:
+        """What changed after change number ``since``; None when that token is
+        too old to answer (the client then resyncs from scratch)."""
+        ...
+
+    async def create_collection(
+        self, user_id: uuid.UUID, collection_id: str, displayname: str, color: str | None
+    ) -> None:
+        """Create a collection at the id the client chose (MKCALENDAR / MKCOL);
+        DavError(403) when that id cannot be used."""
+        ...
+
+    async def update_collection(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        *,
+        displayname: str | None = None,
+        color: str | None = None,
+        description: str | None = None,
+    ) -> None: ...
+
+    async def delete_collection(self, user_id: uuid.UUID, collection_id: str) -> None: ...
+
+    async def query(
+        self,
+        user_id: uuid.UUID,
+        collection_id: str,
+        start: datetime | None,
+        end: datetime | None,
+    ) -> list[str] | None:
+        """Calendars: names of resources with an occurrence overlapping the
+        range; None when the provider cannot filter (the core then returns all)."""
+        ...

backend/app/dav/routes.py

@@ -0,0 +1,595 @@
+"""The CalDAV / CardDAV endpoints (SPEC Appendix E).
+
+    /.well-known/caldav, /.well-known/carddav   → 301 /dav/
+    /dav/                                        root: points at the principal
+    /dav/principals/{user}/                      the signed-in user
+    /dav/calendars/{user}/[{calendar}/[{name}]]  calendar home, calendars, events
+    /dav/addressbooks/{user}/[{book}/[{name}]]   address-book home, books, contacts
+
+The data comes from the plugins' providers (app/dav/provider.py): the calendar
+plugin's for ``calendars``, the contacts plugin's for ``addressbooks``. Signed
+in with HTTP Basic and an app password only (app/dav/auth.py); session cookies
+are never looked at, so a web page cannot drive these endpoints.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+import xml.etree.ElementTree as ET
+from typing import Any
+from urllib.parse import quote, unquote, urlsplit
+
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import RedirectResponse
+from sqlalchemy import select
+
+from app.dav import auth
+from app.dav.provider import Collection, DavError, DavProvider, Resource
+from app.dav.xml import (
+    CR,
+    CS,
+    A,
+    BadRequest,
+    C,
+    D,
+    Multistatus,
+    PropValue,
+    el,
+    error_body,
+    href,
+    mkcol_props,
+    parse,
+    propfind_request,
+    proppatch_request,
+    q,
+    report_request,
+)
+from app.db import session_scope
+from app.models import Account, User
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+METHODS = [
+    "OPTIONS",
+    "PROPFIND",
+    "PROPPATCH",
+    "REPORT",
+    "GET",
+    "HEAD",
+    "PUT",
+    "DELETE",
+    "MKCALENDAR",
+    "MKCOL",
+]
+DAV_HEADER = "1, 3, calendar-access, addressbook, extended-mkcol"
+XML_TYPE = "application/xml; charset=utf-8"
+HOMES = {"calendars": "calendar", "addressbooks": "addressbook"}
+SYNC_PREFIX = "http://ultimo.email/ns/sync/"
+
+
+def _providers() -> dict[str, DavProvider]:
+    from app.plugins import loader
+
+    out: dict[str, DavProvider] = {}
+    for plugin in loader.get_plugins():
+        if plugin.dav is not None:
+            out[plugin.dav.kind] = plugin.dav
+    return out
+
+
+# ---- entry points ------------------------------------------------------------------
+
+
+@router.api_route("/.well-known/caldav", methods=["GET", "HEAD", "PROPFIND", "OPTIONS"])
+@router.api_route("/.well-known/carddav", methods=["GET", "HEAD", "PROPFIND", "OPTIONS"])
+async def well_known() -> Response:
+    return RedirectResponse("/dav/", status_code=301)
+
+
+@router.api_route("/dav", methods=METHODS)
+@router.api_route("/dav/{path:path}", methods=METHODS)
+async def dav(request: Request, path: str = "") -> Response:
+    method = request.method.upper()
+    if method == "OPTIONS":
+        return Response(
+            status_code=200,
+            headers={"DAV": DAV_HEADER, "Allow": ", ".join(METHODS), "Content-Length": "0"},
+        )
+    ip = request.client.host if request.client else ""
+    try:
+        user = await auth.authenticate(request.headers.get("authorization"), ip)
+    except auth.Throttled:
+        return Response(status_code=429, headers={"Retry-After": "900"})
+    if user is None:
+        return Response(
+            status_code=401,
+            headers={"WWW-Authenticate": 'Basic realm="Ultimo", charset="UTF-8"'},
+        )
+    parts = [unquote(p) for p in path.split("/") if p]
+    ctx = _Ctx(request, user, parts)
+    try:
+        return await ctx.handle(method)
+    except BadRequest as err:
+        return Response(status_code=400, content=str(err))
+    except DavError as err:
+        body = error_body(err.condition) if err.condition else b""
+        return Response(
+            status_code=err.status,
+            content=body,
+            media_type=XML_TYPE if body else None,
+        )
+
+
+class _Ctx:
+    def __init__(self, request: Request, user: User, parts: list[str]) -> None:
+        self.request = request
+        self.user = user
+        self.uid = str(user.id)
+        self.parts = parts
+        self.providers = _providers()
+
+    # -- paths -----------------------------------------------------------------------
+
+    def principal(self) -> str:
+        return f"/dav/principals/{self.uid}/"
+
+    def home(self, segment: str) -> str:
+        return f"/dav/{segment}/{self.uid}/"
+
+    def coll_href(self, segment: str, coll_id: str) -> str:
+        return f"{self.home(segment)}{quote(coll_id, safe='')}/"
+
+    def res_href(self, segment: str, coll_id: str, name: str) -> str:
+        return f"{self.coll_href(segment, coll_id)}{quote(name, safe='')}"
+
+    # -- dispatch --------------------------------------------------------------------
+
+    async def handle(self, method: str) -> Response:
+        p = self.parts
+        if not p:
+            return await self.root(method)
+        if p[0] == "principals":
+            if len(p) != 2 or p[1] != self.uid:
+                raise DavError(404)
+            return await self.principal_req(method)
+        if p[0] in HOMES and len(p) >= 2:
+            if p[1] != self.uid:
+                raise DavError(403)
+            provider = self.providers.get(HOMES[p[0]])
+            if provider is None:
+                raise DavError(404)
+            if len(p) == 2:
+                return await self.home_req(method, p[0], provider)
+            if len(p) == 3:
+                return await self.collection_req(method, p[0], provider, p[2])
+            if len(p) == 4:
+                return await self.resource_req(method, p[0], provider, p[2], p[3])
+        raise DavError(404)
+
+    async def body(self) -> ET.Element | None:
+        return parse(await self.request.body())
+
+    def depth(self) -> int:
+        value = (self.request.headers.get("depth") or "1").strip().lower()
+        return 0 if value == "0" else 1
+
+    @staticmethod
+    def multistatus(ms: Multistatus) -> Response:
+        return Response(status_code=207, content=ms.to_bytes(), media_type=XML_TYPE)
+
+    async def respond_props(
+        self, items: list[tuple[str, dict[str, PropValue]]], method_root: ET.Element | None
+    ) -> Response:
+        req = propfind_request(method_root)
+        ms = Multistatus()
+        for path, props in items:
+            if req.names_only:
+                ms.response(path, dict.fromkeys(props))
+            elif req.names is None:
+                ms.response(path, props)
+            else:
+                found = {n: props[n] for n in req.names if n in props}
+                missing = [n for n in req.names if n not in props]
+                ms.response(path, found, missing)
+        return self.multistatus(ms)
+
+    # -- root and principal ----------------------------------------------------------
+
+    def common(self) -> dict[str, PropValue]:
+        return {
+            q(D, "current-user-principal"): [href(self.principal())],
+            q(D, "principal-URL"): [href(self.principal())],
+            q(D, "owner"): [href(self.principal())],
+        }
+
+    async def root(self, method: str) -> Response:
+        if method != "PROPFIND":
+            return Response(status_code=200 if method in ("GET", "HEAD") else 405)
+        props = {**self.common(), q(D, "resourcetype"): [el(q(D, "collection"))]}
+        return await self.respond_props([("/dav/", props)], await self.body())
+
+    async def addresses(self) -> list[str]:
+        async with session_scope() as db:
+            emails = list(
+                await db.scalars(select(Account.email).where(Account.user_id == self.user.id))
+            )
+        out = [self.user.handle, *emails]
+        return list(dict.fromkeys(e.lower() for e in out))
+
+    async def principal_props(self) -> dict[str, PropValue]:
+        props = {
+            **self.common(),
+            q(D, "resourcetype"): [el(q(D, "principal")), el(q(D, "collection"))],
+            q(D, "displayname"): self.user.handle,
+            q(C, "calendar-user-address-set"): [
+                href(f"mailto:{a}") for a in await self.addresses()
+            ],
+            q(C, "calendar-user-type"): "INDIVIDUAL",
+            q(D, "principal-collection-set"): [href("/dav/principals/")],
+            q(D, "supported-report-set"): [],
+        }
+        if "calendar" in self.providers:
+            props[q(C, "calendar-home-set")] = [href(self.home("calendars"))]
+        if "addressbook" in self.providers:
+            props[q(CR, "addressbook-home-set")] = [href(self.home("addressbooks"))]
+        return props
+
+    async def principal_req(self, method: str) -> Response:
+        if method == "PROPFIND":
+            return await self.respond_props(
+                [(self.principal(), await self.principal_props())], await self.body()
+            )
+        if method == "REPORT":  # principal-property-search, expand-property: nothing to add
+            return self.multistatus(Multistatus())
+        return Response(status_code=405)
+
+    # -- homes -------------------------------------------------------------------------
+
+    def privileges(self, read_only: bool) -> list[ET.Element]:
+        names = ["read", "read-current-user-privilege-set"]
+        if not read_only:
+            names += ["write", "write-content", "write-properties", "bind", "unbind"]
+        return [el(q(D, "privilege"), children=[el(q(D, n))]) for n in names]
+
+    def reports(self, kind: str) -> list[ET.Element]:
+        ns, reports = (
+            (C, ["calendar-query", "calendar-multiget"])
+            if kind == "calendar"
+            else (CR, ["addressbook-query", "addressbook-multiget"])
+        )
+        items = [(ns, r) for r in reports] + [(D, "sync-collection")]
+        return [
+            el(q(D, "supported-report"), children=[el(q(D, "report"), children=[el(q(n, r))])])
+            for n, r in items
+        ]
+
+    async def home_req(self, method: str, segment: str, provider: DavProvider) -> Response:
+        if method == "PROPFIND":
+            home_props: dict[str, PropValue] = {
+                **self.common(),
+                q(D, "resourcetype"): [el(q(D, "collection"))],
+                q(D, "displayname"): "Calendars" if provider.kind == "calendar" else "Contacts",
+                q(D, "current-user-privilege-set"): self.privileges(False),
+            }
+            items = [(self.home(segment), home_props)]
+            if self.depth():
+                for coll in await provider.collections(self.user.id):
+                    items.append(
+                        (self.coll_href(segment, coll.id), self.collection_props(provider, coll))
+                    )
+            return await self.respond_props(items, await self.body())
+        if method == "REPORT":
+            return self.multistatus(Multistatus())
+        if method in ("GET", "HEAD"):
+            return Response(status_code=200)
+        return Response(status_code=405)
+
+    # -- collections -------------------------------------------------------------------
+
+    def collection_props(self, provider: DavProvider, coll: Collection) -> dict[str, PropValue]:
+        token = f"{SYNC_PREFIX}{coll.sync_token}"
+        props: dict[str, PropValue] = {
+            **self.common(),
+            q(D, "displayname"): coll.displayname,
+            q(D, "current-user-privilege-set"): self.privileges(coll.read_only),
+            q(D, "supported-report-set"): self.reports(provider.kind),
+            q(D, "sync-token"): token,
+            q(CS, "getctag"): str(coll.sync_token),
+        }
+        if provider.kind == "calendar":
+            props[q(D, "resourcetype")] = [el(q(D, "collection")), el(q(C, "calendar"))]
+            props[q(C, "supported-calendar-component-set")] = [
+                el(q(C, "comp"), attrs={"name": comp}) for comp in coll.components
+            ]
+            props[q(C, "supported-calendar-data")] = [
+                el(q(C, "calendar-data"), attrs={"content-type": "text/calendar", "version": "2.0"})
+            ]
+            props[q(C, "calendar-description")] = coll.description
+            if coll.color:
+                props[q(A, "calendar-color")] = coll.color
+        else:
+            props[q(D, "resourcetype")] = [el(q(D, "collection")), el(q(CR, "addressbook"))]
+            props[q(CR, "supported-address-data")] = [
+                el(
+                    q(CR, "address-data-type"),
+                    attrs={"content-type": "text/vcard", "version": "3.0"},
+                ),
+                el(
+                    q(CR, "address-data-type"),
+                    attrs={"content-type": "text/vcard", "version": "4.0"},
+                ),
+            ]
+            props[q(CR, "addressbook-description")] = coll.description
+        return props
+
+    async def find(self, provider: DavProvider, coll_id: str) -> Collection:
+        for coll in await provider.collections(self.user.id):
+            if coll.id == coll_id:
+                return coll
+        raise DavError(404)
+
+    def resource_props(
+        self, provider: DavProvider, res: Resource, with_data: bool
+    ) -> dict[str, PropValue]:
+        props: dict[str, PropValue] = {
+            q(D, "getetag"): f'"{res.etag}"',
+            q(D, "getcontenttype"): provider.content_type,
+            q(D, "resourcetype"): [],
+        }
+        if with_data and res.data is not None:
+            data_prop = (
+                q(C, "calendar-data") if provider.kind == "calendar" else q(CR, "address-data")
+            )
+            props[data_prop] = res.data
+        return props
+
+    async def collection_req(
+        self, method: str, segment: str, provider: DavProvider, coll_id: str
+    ) -> Response:
+        if method == "MKCALENDAR" or method == "MKCOL":
+            return await self.make_collection(segment, provider, coll_id)
+        coll = await self.find(provider, coll_id)
+        if method == "PROPFIND":
+            items = [(self.coll_href(segment, coll.id), self.collection_props(provider, coll))]
+            if self.depth():
+                for res in await provider.resources(self.user.id, coll.id):
+                    items.append(
+                        (
+                            self.res_href(segment, coll.id, res.name),
+                            self.resource_props(provider, res, False),
+                        )
+                    )
+            return await self.respond_props(items, await self.body())
+        if method == "REPORT":
+            return await self.report(segment, provider, coll)
+        if method == "PROPPATCH":
+            return await self.proppatch(segment, provider, coll)
+        if method == "DELETE":
+            if coll.read_only:
+                raise DavError(403)
+            await provider.delete_collection(self.user.id, coll.id)
+            return Response(status_code=204)
+        if method in ("GET", "HEAD"):
+            return Response(status_code=200)
+        return Response(status_code=405)
+
+    async def make_collection(self, segment: str, provider: DavProvider, coll_id: str) -> Response:
+        if any(c.id == coll_id for c in await provider.collections(self.user.id)):
+            raise DavError(405)  # exists already
+        root = await self.body()
+        props = mkcol_props(root)
+        rtype = props.get(q(D, "resourcetype"))
+        if self.request.method.upper() == "MKCOL" and (
+            rtype is None
+            or rtype.find(q(CR, "addressbook")) is None
+            or provider.kind != "addressbook"
+        ):
+            raise DavError(403)
+        if self.request.method.upper() == "MKCALENDAR" and provider.kind != "calendar":
+            raise DavError(403)
+        name_el = props.get(q(D, "displayname"))
+        color_el = props.get(q(A, "calendar-color"))
+        await provider.create_collection(
+            self.user.id,
+            coll_id,
+            (name_el.text or "").strip() if name_el is not None else "",
+            (color_el.text or "").strip() if color_el is not None else None,
+        )
+        return Response(status_code=201)
+
+    async def proppatch(self, segment: str, provider: DavProvider, coll: Collection) -> Response:
+        patch = proppatch_request(await self.body())
+        if coll.read_only:
+            raise DavError(403)
+        values: dict[str, Any] = {}
+        for name, value in patch.set.items():
+            text = (value.text or "").strip()
+            if name == q(D, "displayname"):
+                values["displayname"] = text
+            elif name == q(A, "calendar-color"):
+                values["color"] = text
+            elif name in (q(C, "calendar-description"), q(CR, "addressbook-description")):
+                values["description"] = text
+        if values:
+            await provider.update_collection(self.user.id, coll.id, **values)
+        # Properties not kept (calendar order, …) are accepted and ignored:
+        # clients treat a refusal as a failed sync.
+        ms = Multistatus()
+        ms.response(self.coll_href(segment, coll.id), dict.fromkeys([*patch.set, *patch.remove]))
+        return self.multistatus(ms)
+
+    async def report(self, segment: str, provider: DavProvider, coll: Collection) -> Response:
+        root = await self.body()
+        if root is None:
+            raise BadRequest("empty report")
+        rep = report_request(root)
+        wants_data = rep.props is None or any(
+            p in (q(C, "calendar-data"), q(CR, "address-data")) for p in rep.props
+        )
+        ms = Multistatus()
+
+        def add(res: Resource) -> None:
+            props = self.resource_props(provider, res, wants_data)
+            if rep.props is None:
+                ms.response(self.res_href(segment, coll.id, res.name), props)
+            else:
+                found = {n: props[n] for n in rep.props if n in props}
+                missing = [n for n in rep.props if n not in props]
+                ms.response(self.res_href(segment, coll.id, res.name), found, missing)
+
+        if rep.kind in ("calendar-multiget", "addressbook-multiget"):
+            names = [_last_segment(h) for h in rep.hrefs]
+            got = {r.name: r for r in await provider.get(self.user.id, coll.id, names)}
+            for h, name in zip(rep.hrefs, names, strict=True):
+                if name in got:
+                    add(got[name])
+                else:
+                    ms.response(h, {}, status=404)
+            return self.multistatus(ms)
+
+        if rep.kind in ("calendar-query", "addressbook-query"):
+            names_q = (
+                await provider.query(self.user.id, coll.id, rep.start, rep.end)
+                if rep.kind == "calendar-query"
+                else None
+            )
+            for res in (
+                await provider.get(self.user.id, coll.id, names_q)
+                if wants_data
+                else (
+                    await provider.resources(self.user.id, coll.id)
+                    if names_q is None
+                    else [
+                        r
+                        for r in await provider.resources(self.user.id, coll.id)
+                        if r.name in set(names_q)
+                    ]
+                )
+            ):
+                add(res)
+            return self.multistatus(ms)
+
+        # sync-collection (RFC 6578)
+        since = _token_number(rep.sync_token)
+        if rep.sync_token and since is None:
+            raise DavError(403, condition=q(D, "valid-sync-token"))
+        if since is None:
+            changed = None
+            deleted: list[str] = []
+            token = coll.sync_token
+        else:
+            changes = await provider.changes(self.user.id, coll.id, since)
+            if changes is None:
+                raise DavError(403, condition=q(D, "valid-sync-token"))
+            changed, deleted, token = changes.changed, changes.deleted, changes.sync_token
+        if changed is None:
+            resources = (
+                await provider.get(self.user.id, coll.id, None)
+                if wants_data
+                else await provider.resources(self.user.id, coll.id)
+            )
+        elif changed:
+            resources = (
+                await provider.get(self.user.id, coll.id, changed)
+                if wants_data
+                else [
+                    r
+                    for r in await provider.resources(self.user.id, coll.id)
+                    if r.name in set(changed)
+                ]
+            )
+        else:
+            resources = []
+        present = {r.name for r in resources}
+        for res in resources:
+            add(res)
+        for name in deleted:
+            if name not in present:
+                ms.response(self.res_href(segment, coll.id, name), {}, status=404)
+        ms.sync_token(f"{SYNC_PREFIX}{token}")
+        return self.multistatus(ms)
+
+    # -- resources ---------------------------------------------------------------------
+
+    async def resource_req(
+        self, method: str, segment: str, provider: DavProvider, coll_id: str, name: str
+    ) -> Response:
+        coll = await self.find(provider, coll_id)
+        if method in ("GET", "HEAD"):
+            got = await provider.get(self.user.id, coll.id, [name])
+            if not got:
+                raise DavError(404)
+            res = got[0]
+            content = (res.data or "").encode() if method == "GET" else b""
+            return Response(
+                status_code=200,
+                content=content,
+                media_type=provider.content_type,
+                headers={"ETag": f'"{res.etag}"'},
+            )
+        if method == "PROPFIND":
+            got = await provider.get(self.user.id, coll.id, [name])
+            if not got:
+                raise DavError(404)
+            return await self.respond_props(
+                [
+                    (
+                        self.res_href(segment, coll.id, name),
+                        self.resource_props(provider, got[0], True),
+                    )
+                ],
+                await self.body(),
+            )
+        if method == "PUT":
+            if coll.read_only:
+                raise DavError(403, condition=q(D, "need-privileges"))
+            data = (await self.request.body()).decode("utf-8", "replace")
+            etag, created = await provider.put(
+                self.user.id,
+                coll.id,
+                name,
+                data,
+                if_match=_etag_header(self.request.headers.get("if-match")),
+                if_none_match=(self.request.headers.get("if-none-match") or "").strip() == "*",
+            )
+            return Response(status_code=201 if created else 204, headers={"ETag": f'"{etag}"'})
+        if method == "DELETE":
+            if coll.read_only:
+                raise DavError(403)
+            await provider.delete(
+                self.user.id,
+                coll.id,
+                name,
+                if_match=_etag_header(self.request.headers.get("if-match")),
+            )
+            return Response(status_code=204)
+        return Response(status_code=405)
+
+
+def _last_segment(h: str) -> str:
+    path = urlsplit(h).path
+    return unquote(path.rstrip("/").rsplit("/", 1)[-1])
+
+
+def _token_number(token: str) -> int | None:
+    if not token:
+        return None
+    tail = token.rsplit("/", 1)[-1]
+    return int(tail) if tail.isdigit() else None
+
+
+def _etag_header(value: str | None) -> str | None:
+    if not value:
+        return None
+    value = value.strip()
+    if value == "*":
+        return "*"
+    if value.startswith("W/"):
+        value = value[2:]
+    return value.strip('"')
+
+
+__all__ = ["router", "uuid"]

backend/app/dav/xml.py

@@ -0,0 +1,236 @@
+"""WebDAV XML: reading request bodies, writing multistatus answers (RFC 4918,
+4791, 6352, 6578). Properties are ``{namespace}name`` strings, as ElementTree
+names them."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from defusedxml import ElementTree as SafeET
+
+D = "DAV:"
+C = "urn:ietf:params:xml:ns:caldav"
+CR = "urn:ietf:params:xml:ns:carddav"
+CS = "http://calendarserver.org/ns/"
+A = "http://apple.com/ns/ical/"
+
+for _prefix, _ns in (("d", D), ("cal", C), ("card", CR), ("cs", CS), ("ical", A)):
+    ET.register_namespace(_prefix, _ns)
+
+
+def q(ns: str, name: str) -> str:
+    return f"{{{ns}}}{name}"
+
+
+class BadRequest(Exception):
+    pass
+
+
+def parse(body: bytes) -> ET.Element | None:
+    if not body.strip():
+        return None
+    try:
+        root = SafeET.fromstring(body)
+    except Exception as err:  # noqa: BLE001 - any malformed XML is the client's error
+        raise BadRequest(str(err)) from err
+    return root  # type: ignore[no-any-return]
+
+
+@dataclass
+class PropRequest:
+    """PROPFIND: which properties (None = all)."""
+
+    names: list[str] | None = None
+    names_only: bool = False
+
+
+def propfind_request(root: ET.Element | None) -> PropRequest:
+    if root is None or root.find(q(D, "allprop")) is not None:
+        return PropRequest()
+    if root.find(q(D, "propname")) is not None:
+        return PropRequest(names_only=True)
+    prop = root.find(q(D, "prop"))
+    if prop is None:
+        return PropRequest()
+    return PropRequest(names=[child.tag for child in prop])
+
+
+def requested_props(root: ET.Element) -> list[str] | None:
+    prop = root.find(q(D, "prop"))
+    if prop is None:
+        return None
+    return [child.tag for child in prop]
+
+
+@dataclass
+class Report:
+    # calendar-query | calendar-multiget | addressbook-query | addressbook-multiget
+    # | sync-collection
+    kind: str
+    props: list[str] | None
+    hrefs: list[str] = field(default_factory=list)
+    start: datetime | None = None
+    end: datetime | None = None
+    sync_token: str = ""
+    limit: int | None = None
+
+
+def _ical_time(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        return datetime.strptime(value.strip(), "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
+    except ValueError:
+        return None
+
+
+def report_request(root: ET.Element) -> Report:
+    tag = root.tag
+    names = {
+        q(C, "calendar-query"): "calendar-query",
+        q(C, "calendar-multiget"): "calendar-multiget",
+        q(CR, "addressbook-query"): "addressbook-query",
+        q(CR, "addressbook-multiget"): "addressbook-multiget",
+        q(D, "sync-collection"): "sync-collection",
+    }
+    kind = names.get(tag)
+    if kind is None:
+        raise BadRequest(f"unsupported report {tag}")
+    rep = Report(kind=kind, props=requested_props(root))
+    rep.hrefs = [(h.text or "").strip() for h in root.findall(q(D, "href"))]
+    if kind == "calendar-query":
+        tr = root.find(f".//{q(C, 'time-range')}")
+        if tr is not None:
+            rep.start = _ical_time(tr.get("start"))
+            rep.end = _ical_time(tr.get("end"))
+    if kind == "sync-collection":
+        rep.sync_token = (root.findtext(q(D, "sync-token")) or "").strip()
+        nresults = root.find(f"{q(D, 'limit')}/{q(D, 'nresults')}")
+        if nresults is not None and (nresults.text or "").strip().isdigit():
+            rep.limit = int((nresults.text or "").strip())
+    return rep
+
+
+@dataclass
+class PropPatch:
+    set: dict[str, ET.Element] = field(default_factory=dict)
+    remove: list[str] = field(default_factory=list)
+
+
+def proppatch_request(root: ET.Element | None) -> PropPatch:
+    out = PropPatch()
+    if root is None:
+        return out
+    for action in root:
+        prop = action.find(q(D, "prop"))
+        if prop is None:
+            continue
+        for child in prop:
+            if action.tag == q(D, "set"):
+                out.set[child.tag] = child
+            elif action.tag == q(D, "remove"):
+                out.remove.append(child.tag)
+    return out
+
+
+def mkcol_props(root: ET.Element | None) -> dict[str, ET.Element]:
+    """MKCALENDAR / extended MKCOL: the properties to set on the new collection."""
+    if root is None:
+        return {}
+    out: dict[str, ET.Element] = {}
+    for sett in root.findall(q(D, "set")):
+        prop = sett.find(q(D, "prop"))
+        if prop is not None:
+            for child in prop:
+                out[child.tag] = child
+    return out
+
+
+# ---- answers --------------------------------------------------------------------------
+
+# A property's value: text, a list of child elements, or an element builder.
+PropValue = str | list[ET.Element] | None
+
+
+class Multistatus:
+    def __init__(self) -> None:
+        self.root = ET.Element(q(D, "multistatus"))
+
+    def response(
+        self,
+        href: str,
+        found: dict[str, PropValue],
+        missing: list[str] | None = None,
+        status: int | None = None,
+    ) -> None:
+        resp = ET.SubElement(self.root, q(D, "response"))
+        ET.SubElement(resp, q(D, "href")).text = href
+        if status is not None:
+            ET.SubElement(resp, q(D, "status")).text = _status_line(status)
+            return
+        if found:
+            ps = ET.SubElement(resp, q(D, "propstat"))
+            prop = ET.SubElement(ps, q(D, "prop"))
+            for name, value in found.items():
+                el = ET.SubElement(prop, name)
+                if isinstance(value, str):
+                    el.text = value
+                elif isinstance(value, list):
+                    el.extend(value)
+            ET.SubElement(ps, q(D, "status")).text = _status_line(200)
+        if missing:
+            ps = ET.SubElement(resp, q(D, "propstat"))
+            prop = ET.SubElement(ps, q(D, "prop"))
+            for name in missing:
+                ET.SubElement(prop, name)
+            ET.SubElement(ps, q(D, "status")).text = _status_line(404)
+
+    def sync_token(self, token: str) -> None:
+        ET.SubElement(self.root, q(D, "sync-token")).text = token
+
+    def to_bytes(self) -> bytes:
+        body: bytes = ET.tostring(self.root, encoding="utf-8")
+        return b'<?xml version="1.0" encoding="utf-8"?>\n' + body
+
+
+def el(
+    tag: str,
+    text: str | None = None,
+    children: list[ET.Element] | None = None,
+    attrs: dict[str, str] | None = None,
+) -> ET.Element:
+    e = ET.Element(tag, attrs or {})
+    if text is not None:
+        e.text = text
+    if children:
+        e.extend(children)
+    return e
+
+
+def href(path: str) -> ET.Element:
+    return el(q(D, "href"), path)
+
+
+def error_body(condition: str) -> bytes:
+    root = ET.Element(q(D, "error"))
+    ET.SubElement(root, condition)
+    return b'<?xml version="1.0" encoding="utf-8"?>\n' + bytes(ET.tostring(root, encoding="utf-8"))
+
+
+_REASONS = {
+    200: "OK",
+    201: "Created",
+    204: "No Content",
+    207: "Multi-Status",
+    403: "Forbidden",
+    404: "Not Found",
+    409: "Conflict",
+    412: "Precondition Failed",
+    507: "Insufficient Storage",
+}
+
+
+def _status_line(code: int) -> str:
+    return f"HTTP/1.1 {code} {_REASONS.get(code, '')}".rstrip()

backend/app/hosted/__init__.py

@@ -0,0 +1,34 @@
+"""Hosted mailboxes (SPEC Appendix D): every user's own mailbox on the Stalwart
+mail server beside the app.
+
+* :mod:`app.hosted.admin` — the mail server's management API (JMAP);
+* :mod:`app.hosted.provision` — idempotent setup: its database and role, the
+  directory over the ``mail_directory`` view, the worker's sign-in;
+* :mod:`app.hosted.native` — the permanent account in the webmail, and how the
+  worker signs in to it without any stored password;
+* :mod:`app.hosted.rawcache` — raw messages kept once, on the mail server: the
+  webmail keeps only a bounded cache of them (§D.6 item 2).
+
+All of it is inert while ``HOSTED_MAILBOXES`` is off.
+"""
+
+from __future__ import annotations
+
+from app.config import get_settings
+
+# The webmail worker's sign-in on the mail server: ``user@domain%WORKER`` acts
+# for that user (Stalwart's master-user login, given the impersonate role only).
+WORKER_LOCALPART = "webmail-worker"
+
+
+def enabled() -> bool:
+    return get_settings().hosted_mailboxes
+
+
+def worker_address() -> str:
+    return f"{WORKER_LOCALPART}@{get_settings().app_domain}"
+
+
+def mail_hostname() -> str:
+    settings = get_settings()
+    return settings.mail_hostname or f"mail.{settings.app_domain}"

backend/app/hosted/admin.py

@@ -0,0 +1,177 @@
+"""The mail server's management API (Stalwart 0.16, JMAP ``x:`` methods).
+
+Signed in to as the recovery admin (``STALWART_RECOVERY_ADMIN`` in the mail
+server's environment, ``STALWART_ADMIN_SECRET`` here). Objects are Stalwart's
+registry: ``x:Account``, ``x:Domain``, ``x:Directory``, ``x:Role``,
+``x:AppPassword``… with the usual JMAP ``/get``, ``/query``, ``/set``.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Any
+
+import httpx
+
+from app.config import get_settings
+from app.hosted import worker_address
+
+ADMIN_USER = "admin"
+_USING = ["urn:ietf:params:jmap:core", "urn:stalwart:jmap"]
+_TIMEOUT = 15.0
+
+
+class MailServerError(Exception):
+    """The mail server refused a call, or could not be reached."""
+
+
+def _basic(user: str, secret: str) -> str:
+    return "Basic " + base64.b64encode(f"{user}:{secret}".encode()).decode()
+
+
+def _admin_auth() -> str:
+    return _basic(ADMIN_USER, get_settings().stalwart_admin_secret)
+
+
+async def call(
+    *method_calls: tuple[str, dict[str, Any]], auth: str | None = None
+) -> list[dict[str, Any]]:
+    """Run JMAP method calls in one request; return each call's arguments.
+
+    Raises :class:`MailServerError` on a method error or a refused ``set``."""
+    body = {
+        "using": _USING,
+        "methodCalls": [[name, args, str(i)] for i, (name, args) in enumerate(method_calls)],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.post(
+                f"{get_settings().stalwart_url}/jmap",
+                json=body,
+                headers={"Authorization": auth or _admin_auth()},
+            )
+    except httpx.HTTPError as exc:
+        raise MailServerError(f"mail server unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        raise MailServerError(f"mail server answered {resp.status_code}: {resp.text[:300]}")
+    out: list[dict[str, Any]] = []
+    for name, args, _ in resp.json()["methodResponses"]:
+        if name == "error":
+            raise MailServerError(f"{args.get('type')}: {args.get('description', '')}")
+        for kind in ("notCreated", "notUpdated", "notDestroyed"):
+            if args.get(kind):
+                raise MailServerError(f"{name} {kind}: {args[kind]}")
+        out.append(args)
+    return out
+
+
+async def get_all(obj: str, **args: Any) -> list[dict[str, Any]]:
+    """Every object of a type (registry objects are few)."""
+    (res,) = await call((f"x:{obj}/get", {"ids": None, **args}))
+    return list(res["list"])
+
+
+async def get_one(obj: str, object_id: str = "singleton") -> dict[str, Any]:
+    (res,) = await call((f"x:{obj}/get", {"ids": [object_id]}))
+    if not res["list"]:
+        raise MailServerError(f"{obj} {object_id} not found")
+    return dict(res["list"][0])
+
+
+async def create(obj: str, value: dict[str, Any], **args: Any) -> str:
+    (res,) = await call((f"x:{obj}/set", {"create": {"n": value}, **args}))
+    return str(res["created"]["n"]["id"])
+
+
+async def update(obj: str, object_id: str, patch: dict[str, Any], **args: Any) -> None:
+    await call((f"x:{obj}/set", {"update": {object_id: patch}, **args}))
+
+
+async def destroy(obj: str, ids: list[str], **args: Any) -> None:
+    if ids:
+        await call((f"x:{obj}/set", {"destroy": ids, **args}))
+
+
+# ---- accounts ----------------------------------------------------------------------
+
+
+async def account_id(email: str) -> str | None:
+    """The mail server's id of a mailbox, if it knows it yet.
+
+    Accounts from the directory reach the server's registry the first time
+    they are used (a sign-in, a delivery): see :func:`ensure_account`."""
+    local = email.split("@", 1)[0].lower()
+    (res,) = await call(("x:Account/query", {"filter": {"name": local}}))
+    ids = list(res.get("ids") or [])
+    if not ids:
+        return None
+    for acc in (await call(("x:Account/get", {"ids": ids, "properties": ["emailAddress"]})))[0][
+        "list"
+    ]:
+        if str(acc.get("emailAddress", "")).lower() == email.lower():
+            return str(acc["id"])
+    return None
+
+
+async def ensure_account(email: str) -> str:
+    """The mailbox's id, bringing it into the server's registry if new: a
+    sign-in as the worker acting for it does that (§D.3)."""
+    found = await account_id(email)
+    if found is not None:
+        return found
+    auth = _basic(f"{email}%{worker_address()}", get_settings().stalwart_master_secret)
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(
+                f"{get_settings().stalwart_url}/jmap/session", headers={"Authorization": auth}
+            )
+    except httpx.HTTPError as exc:
+        raise MailServerError(f"mail server unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        raise MailServerError(f"cannot open the mailbox {email}: {resp.status_code}")
+    found = await account_id(email)
+    if found is None:
+        raise MailServerError(f"mailbox {email} not found after sign-in")
+    return found
+
+
+async def set_blocked(email: str, blocked: bool) -> None:
+    """A blocked user's mailbox cannot be signed in to at all — app passwords
+    and the worker included. Mail to it is still accepted."""
+    acc = await account_id(email)
+    if acc is None:
+        return  # never used: the directory alone keeps it shut
+    permissions: dict[str, Any] = (
+        {"@type": "Merge", "enabledPermissions": {}, "disabledPermissions": {"authenticate": True}}
+        if blocked
+        else {"@type": "Inherit"}
+    )
+    await update("Account", acc, {"permissions": permissions})
+
+
+async def delete_account(email: str) -> None:
+    """Remove the mailbox and all its mail from the mail server."""
+    acc = await account_id(email)
+    if acc is not None:
+        await destroy("Account", [acc])
+
+
+# ---- app passwords (§D.5) -------------------------------------------------------------
+
+
+async def create_app_password(email: str, description: str) -> tuple[str, str]:
+    """A new app password for the mailbox: (server id, the password itself)."""
+    acc = await ensure_account(email)
+    (res,) = await call(
+        ("x:AppPassword/set", {"accountId": acc, "create": {"n": {"description": description}}})
+    )
+    created = res["created"]["n"]
+    return str(created["id"]), str(created["secret"])
+
+
+async def delete_app_passwords(email: str, server_ids: list[str]) -> None:
+    acc = await account_id(email)
+    if acc is not None and server_ids:
+        (res,) = await call(("x:AppPassword/get", {"accountId": acc, "ids": server_ids}))
+        present = [str(p["id"]) for p in res["list"]]
+        await destroy("AppPassword", present, accountId=acc)

backend/app/hosted/native.py

@@ -0,0 +1,154 @@
+"""The user's own mailbox as a webmail account (SPEC App. D §D.4).
+
+Every user has one ``kind = 'native'`` account: their handle's mailbox on the
+hosted mail server. It is created with the user (and, for users from before,
+by :func:`ensure_all`, run by provisioning), cannot be removed, and its server
+settings are not the user's to change.
+
+No password of the user's is stored for it: the worker signs in as itself,
+acting for the user (``user@domain%webmail-worker``, §D.3), over the Docker
+network (or loopback in dev) — TLS, with the certificate left unchecked since
+the traffic never leaves the host.
+"""
+
+from __future__ import annotations
+
+import logging
+import ssl
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.hosted import enabled, worker_address
+from app.models import Account, User
+from app.providers.imap_smtp import ProviderConfig
+from app.security.crypto import encrypt_secret
+
+logger = logging.getLogger(__name__)
+
+KIND = "native"
+
+
+def is_native(account: Account) -> bool:
+    return account.kind == KIND
+
+
+def _server(host: str, port: int) -> dict[str, object]:
+    return {"host": host, "port": port, "tls": "ssl", "auth": "password"}
+
+
+def internal_tls() -> ssl.SSLContext:
+    """TLS to the mail server on the internal network: its certificate is for
+    its public name (or self-signed), not the container's."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def native_provider_config(email: str) -> ProviderConfig:
+    """IMAP/SMTP to the user's mailbox, signed in as the worker acting for them.
+
+    Read from the settings, not the account row: the mail server's address is
+    the deployment's, and may change."""
+    settings = get_settings()
+    return ProviderConfig(
+        imap_host=settings.stalwart_imap_host,
+        imap_port=settings.stalwart_imap_port,
+        imap_tls="ssl",
+        smtp_host=settings.stalwart_smtp_host,
+        smtp_port=settings.stalwart_smtp_port,
+        smtp_tls="ssl",
+        username=f"{email}%{worker_address()}",
+        password=settings.stalwart_master_secret,
+        tls_context=internal_tls(),
+        internal=True,
+    )
+
+
+async def native_account(db: AsyncSession, user_id: uuid.UUID) -> Account | None:
+    account: Account | None = await db.scalar(
+        select(Account).where(Account.user_id == user_id, Account.kind == KIND)
+    )
+    return account
+
+
+async def ensure(db: AsyncSession, user: User) -> Account | None:
+    """The user's native account, created (and its first sync queued by the
+    caller, see :func:`ensure_and_sync`) if missing. None when hosted
+    mailboxes are off. Flushes, does not commit."""
+    if not enabled():
+        return None
+    existing = await native_account(db, user.id)
+    if existing is not None:
+        return existing
+    # An account for the same address added by hand: the same address at
+    # another provider (the mail before it moved here — e.g. while the domain
+    # still points elsewhere). It stays as it is; the mailbox account is added
+    # once it is removed (app/api/accounts.py).
+    same = await db.scalar(
+        select(Account.id).where(Account.user_id == user.id, Account.email == user.handle)
+    )
+    if same is not None:
+        logger.info("hosted: %s already has an account for its address; not added", user.handle)
+        return None
+    settings = get_settings()
+    imap = _server(settings.stalwart_imap_host, settings.stalwart_imap_port)
+    smtp = _server(settings.stalwart_smtp_host, settings.stalwart_smtp_port)
+    account = Account(
+        user_id=user.id,
+        email=user.handle,
+        kind=KIND,
+        imap_config=imap,
+        smtp_config=smtp,
+        secret_ct=encrypt_secret(b""),  # nothing: the worker signs in as itself
+        discovery_src="hosted",
+        state="backfilling",
+        sync_window="all",
+        # The mail server's own spam filter files into Junk: show it.
+        sync_spam=True,
+    )
+    db.add(account)
+    await db.flush()
+    return account
+
+
+async def ensure_and_sync(db: AsyncSession, user: User) -> Account | None:
+    """:func:`ensure`, committed, and the account's first sync queued if new."""
+    before = await native_account(db, user.id)
+    account = await ensure(db, user)
+    await db.commit()
+    if account is not None and before is None:
+        from app.queue import enqueue_job
+
+        await enqueue_job("sync_bootstrap_account", str(account.id))
+    return account
+
+
+async def ensure_all() -> int:
+    """Give every user without one their native account (users from before
+    hosted mailboxes). Returns how many were created."""
+    from app.db import session_scope
+
+    created = 0
+    async with session_scope() as db:
+        users = list(
+            await db.scalars(
+                select(User).where(
+                    ~select(Account.id)
+                    .where(Account.user_id == User.id, Account.kind == KIND)
+                    .exists()
+                )
+            )
+        )
+    for user in users:
+        async with session_scope() as db:
+            fresh = await db.get(User, user.id)
+            if fresh is None:
+                continue
+            await ensure_and_sync(db, fresh)
+            created += 1
+            logger.info("hosted: mailbox account for %s", fresh.handle)
+    return created

backend/app/hosted/provision.py

@@ -0,0 +1,323 @@
+"""Set up the mail server for hosted mailboxes (SPEC App. D §D.2/§D.3). Idempotent.
+
+Run after the migrations (the ``migrate`` service in production; in dev
+``python -m app.hosted.provision``). Two halves:
+
+1. **Postgres** (as the app's own, privileged database user): the mail
+   server's role ``stalwart`` (password ``STALWART_DB_PASSWORD``), which owns its
+   database and may read the ``mail_directory`` view of the app's database and
+   nothing else there; and the worker's sign-in in ``mail_service_accounts``.
+   The mail server cannot start before this has run once.
+2. **The mail server** (its management API, as the recovery admin): the mail
+   domain and host name, mail kept on disk (not in Postgres), the SQL directory
+   over ``mail_directory`` as the only source of accounts, the worker's
+   impersonate-only role, the submission port for mail apps.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import asyncpg  # type: ignore[import-untyped]
+
+from app.config import get_settings
+from app.hosted import admin, mail_hostname, worker_address
+from app.identity.passwords import hash_password, verify_password
+
+logger = logging.getLogger(__name__)
+
+DB_ROLE = "stalwart"
+DIRECTORY_NAME = "Ultimo webmail users"
+WORKER_ROLE_NAME = "Ultimo webmail worker"
+BLOB_PATH = "/var/lib/stalwart/blobs"
+
+# The directory's queries ($1: the address signed in with / mail is for). A
+# blocked user cannot sign in, but still receives mail.
+QUERY_LOGIN = (
+    "SELECT name, secret, type, description FROM mail_directory WHERE name = $1 AND active"
+)
+QUERY_RECIPIENT = (
+    "SELECT name, NULL::text AS secret, type, description FROM mail_directory WHERE name = $1"
+)
+
+
+def _pg_dsn() -> str:
+    return get_settings().database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+
+
+def _app_db_name() -> str:
+    return urlparse(_pg_dsn()).path.lstrip("/")
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+async def provision_database() -> None:
+    settings = get_settings()
+    if not settings.stalwart_db_password or not settings.stalwart_master_secret:
+        raise RuntimeError("STALWART_DB_PASSWORD and STALWART_MASTER_SECRET must be set")
+    conn = await asyncpg.connect(_pg_dsn())
+    try:
+        password = _quote_literal(settings.stalwart_db_password)
+        if await conn.fetchval("SELECT 1 FROM pg_roles WHERE rolname = $1", DB_ROLE):
+            await conn.execute(f"ALTER ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        else:
+            await conn.execute(f"CREATE ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        db = settings.stalwart_db_name
+        if not await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db):
+            await conn.execute(f'CREATE DATABASE "{db}" OWNER {DB_ROLE}')
+            logger.info("hosted: created database %s", db)
+        await conn.execute(f"GRANT SELECT ON mail_directory TO {DB_ROLE}")
+
+        # The worker's sign-in: re-hashed only when the secret changed.
+        name = worker_address()
+        current = await conn.fetchval(
+            "SELECT secret FROM mail_service_accounts WHERE name = $1", name
+        )
+        if current is None or not verify_password(current, settings.stalwart_master_secret):
+            await conn.execute(
+                """
+                INSERT INTO mail_service_accounts (name, secret, description)
+                VALUES ($1, $2, 'Webmail worker')
+                ON CONFLICT (name) DO UPDATE SET secret = EXCLUDED.secret
+                """,
+                name,
+                hash_password(settings.stalwart_master_secret),
+            )
+    finally:
+        await conn.close()
+
+
+# ---- the mail server ------------------------------------------------------------------
+
+
+async def _wait_for_server(timeout_s: float = 120.0) -> None:
+    deadline = asyncio.get_running_loop().time() + timeout_s
+    while True:
+        try:
+            await admin.get_one("SystemSettings")
+            return
+        except admin.MailServerError:
+            if asyncio.get_running_loop().time() > deadline:
+                raise
+            await asyncio.sleep(2)
+
+
+def _by(items: list[dict[str, Any]], key: str, value: str) -> dict[str, Any] | None:
+    return next((i for i in items if i.get(key) == value), None)
+
+
+async def _domain() -> str:
+    name = get_settings().app_domain.lower()
+    found = _by(await admin.get_all("Domain"), "name", name)
+    if found is not None:
+        return str(found["id"])
+    domain_id = await admin.create("Domain", {"name": name})
+    logger.info("hosted: created mail domain %s", name)
+    return domain_id
+
+
+async def _directory() -> str:
+    settings = get_settings()
+    dsn = urlparse(_pg_dsn())
+    value: dict[str, Any] = {
+        "@type": "Sql",
+        "description": DIRECTORY_NAME,
+        "columnEmail": "name",
+        "columnSecret": "secret",
+        "columnClass": "type",
+        "columnDescription": "description",
+        "queryLogin": QUERY_LOGIN,
+        "queryRecipient": QUERY_RECIPIENT,
+        "queryMemberOf": None,
+        "queryEmailAliases": None,
+        "store": {
+            "@type": "PostgreSql",
+            "host": settings.stalwart_pg_host,
+            "port": 5432 if settings.stalwart_pg_host == "postgres" else (dsn.port or 5432),
+            "database": _app_db_name(),
+            "authUsername": DB_ROLE,
+            "authSecret": {"@type": "EnvironmentVariable", "variableName": "STALWART_DB_PASSWORD"},
+            "useTls": False,
+            "allowInvalidCerts": False,
+            "poolMaxConnections": 5,
+            "poolRecyclingMethod": "fast",
+            "readReplicas": {},
+            "timeout": 15000,
+            "options": None,
+        },
+    }
+    found = _by(await admin.get_all("Directory"), "description", DIRECTORY_NAME)
+    if found is None:
+        directory_id = await admin.create("Directory", value)
+        logger.info("hosted: created the SQL directory")
+    else:
+        directory_id = str(found["id"])
+        await admin.update("Directory", directory_id, value)
+    await admin.update("Authentication", "singleton", {"directoryId": directory_id})
+    return directory_id
+
+
+async def _worker_role() -> str:
+    perms = {"impersonate": True, "authenticate": True}
+    found = _by(await admin.get_all("Role"), "description", WORKER_ROLE_NAME)
+    if found is not None:
+        role_id = str(found["id"])
+        await admin.update("Role", role_id, {"enabledPermissions": perms})
+        return role_id
+    return await admin.create(
+        "Role", {"description": WORKER_ROLE_NAME, "enabledPermissions": perms}
+    )
+
+
+async def _worker_account(role_id: str) -> None:
+    """The worker may act for any mailbox and do nothing as itself."""
+    # Signing in brings it from the directory into the server's registry. With
+    # its role already set that sign-in fails (it may not open a session as
+    # itself) — after the account exists, which is all this needs.
+    worker = worker_address()
+    acc = await admin.account_id(worker)
+    if acc is None:
+        import httpx
+
+        async with httpx.AsyncClient(timeout=15) as client:
+            await client.get(
+                f"{get_settings().stalwart_url}/jmap/session",
+                auth=(worker, get_settings().stalwart_master_secret),
+            )
+        acc = await admin.account_id(worker)
+    if acc is None:
+        raise admin.MailServerError("the worker account did not appear on the mail server")
+    await admin.update("Account", acc, {"roles": {"@type": "Custom", "roleIds": {role_id: True}}})
+
+
+async def _reload() -> None:
+    """Apply the settings now. Stalwart 0.16.19 ships a few spam-list rules its
+    own validator rejects (``bit_and`` in SURBL tags); they never load, but
+    they make every reload fail — so they are removed, which changes nothing."""
+    for _ in range(10):
+        try:
+            await admin.create("Action", {"@type": "ReloadSettings"})
+            return
+        except admin.MailServerError as exc:
+            match = re.search(r"'object': '(SpamDnsblServer)', 'id': '(\w+)'", str(exc))
+            if match is None:
+                raise
+            logger.info("hosted: removing the invalid built-in rule %s", match.group(2))
+            await admin.destroy(match.group(1), [match.group(2)])
+    raise admin.MailServerError("settings reload kept failing")
+
+
+async def _logging() -> None:
+    """Log to the console (``docker logs``), not to files inside the container."""
+    for tracer in await admin.get_all("Tracer"):
+        if tracer.get("@type") != "Stdout":
+            await admin.update(
+                "Tracer",
+                str(tracer["id"]),
+                {
+                    "@type": "Stdout",
+                    "ansi": False,
+                    "buffered": False,
+                    "enable": True,
+                    "level": "info",
+                    "events": {},
+                    "eventsPolicy": "exclude",
+                },
+            )
+
+
+async def _blob_store() -> bool:
+    """Mail on disk (``DATA_ROOT/stalwart``), not inside Postgres: database
+    backups stay small. Takes a restart; True when one is needed."""
+    current = await admin.get_one("BlobStore")
+    if current.get("@type") == "FileSystem" and current.get("path") == BLOB_PATH:
+        return False
+    await admin.update(
+        "BlobStore", "singleton", {"@type": "FileSystem", "path": BLOB_PATH, "depth": 2}
+    )
+    logger.info("hosted: mail is now kept on disk (takes effect after a restart)")
+    return True
+
+
+async def _listeners() -> None:
+    """Add submission with STARTTLS (587) beside the defaults: smtp 25, submissions
+    465, imaps 993 (and http 8080 for this API)."""
+    listeners = await admin.get_all("NetworkListener")
+    if _by(listeners, "name", "submission") is None:
+        await admin.create(
+            "NetworkListener",
+            {"name": "submission", "bind": {"[::]:587": True}, "protocol": "smtp", "useTls": True},
+        )
+
+
+# Folder names for new mailboxes: the plain ones mail apps expect, not
+# Stalwart's Outlook-style defaults ("Deleted Items", "Junk Mail", "Sent
+# Items"). Archive is made too: the webmail archives into it.
+FOLDERS = {
+    "trash": "Trash",
+    "junk": "Junk",
+    "drafts": "Drafts",
+    "sent": "Sent",
+    "archive": "Archive",
+}
+
+
+async def _folders() -> None:
+    folders = {
+        use: {"name": name, "create": True, "subscribe": True, "aliases": {}}
+        for use, name in FOLDERS.items()
+    }
+    await admin.update("Email", "singleton", {"defaultFolders": folders})
+
+
+async def provision_server() -> bool:
+    """Set the mail server up; True when it must be restarted to finish."""
+    await _wait_for_server()
+    await _logging()
+    restart = await _blob_store()
+    domain_id = await _domain()
+    await admin.update(
+        "SystemSettings",
+        "singleton",
+        {"defaultDomainId": domain_id, "defaultHostname": mail_hostname()},
+    )
+    await _listeners()
+    await _folders()
+    await _directory()
+    role_id = await _worker_role()
+    await _reload()
+    await _worker_account(role_id)
+    return restart
+
+
+async def provision() -> bool:
+    from app.hosted import native
+
+    await provision_database()
+    restart = await provision_server()
+    created = await native.ensure_all()
+    if created:
+        logger.info("hosted: added the mailbox account of %d existing user(s)", created)
+    return restart
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    if not get_settings().hosted_mailboxes:
+        logger.info("hosted mailboxes are off (HOSTED_MAILBOXES): nothing to set up")
+        return
+    if asyncio.run(provision()):
+        logger.warning("hosted mailboxes: RESTART the mail server (stalwart) to finish")
+    else:
+        logger.info("hosted mailboxes: mail server ready")
+
+
+if __name__ == "__main__":
+    main()

backend/app/identity/app_passwords.py

@@ -0,0 +1,140 @@
+"""App passwords: one generated password per mail app (SPEC App. D §D.5).
+
+A mail app (Thunderbird, a phone's Mail app) signs in to the hosted mailbox
+over IMAP/SMTP, and a calendar or contacts app to the DAV server — never with
+the account password (no 2FA code can be asked for there), always with one of
+these. Each gets a password of its own, shown once, that can be
+revoked without touching the main password or the other apps. The webmail
+itself never accepts one.
+
+With hosted mailboxes on, the mail server makes the password (``app_`` and 36
+more letters and digits: its format names the credential) and checks it for
+IMAP/SMTP itself; its argon2id hash is kept here too, for the DAV server, with
+the server's id to revoke it there. Without, it is 16 random letters, for DAV
+only. Either way the hash is of the bare password: apps send exactly that, so
+it is copied and entered without the spaces it is shown with.
+"""
+
+from __future__ import annotations
+
+import secrets
+import string
+import uuid
+
+from sqlalchemy import delete, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import hosted
+from app.identity.passwords import hash_password, verify_password
+from app.models import User, UserAppPassword
+
+LENGTH = 16  # 26^16 ≈ 2^75
+SERVER_LENGTH = 40  # the mail server's: "app_" + base32 of a 4-byte id and 18 random bytes
+MAX_PER_USER = 25
+NAME_MAX = 60
+_ALPHABET = string.ascii_lowercase
+
+
+def generation_key(user_id: uuid.UUID) -> str:
+    """Redis counter bumped whenever the user's app passwords change: DAV
+    sign-ins cached under the old value stop counting (app/dav/auth.py)."""
+    return f"apppw:gen:{user_id}"
+
+
+async def _changed(user_id: uuid.UUID) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().incr(generation_key(user_id))
+
+
+class TooMany(Exception):
+    """The user already has MAX_PER_USER app passwords."""
+
+
+def generate() -> str:
+    return "".join(secrets.choice(_ALPHABET) for _ in range(LENGTH))
+
+
+async def list_for(db: AsyncSession, user_id: uuid.UUID) -> list[UserAppPassword]:
+    rows = await db.scalars(
+        select(UserAppPassword)
+        .where(UserAppPassword.user_id == user_id)
+        .order_by(UserAppPassword.created_at.desc())
+    )
+    return list(rows)
+
+
+async def create(db: AsyncSession, user_id: uuid.UUID, name: str) -> tuple[UserAppPassword, str]:
+    """A new app password named ``name``; returns the row and the password
+    itself, which is never available again."""
+    count = await db.scalar(
+        select(func.count()).select_from(UserAppPassword).where(UserAppPassword.user_id == user_id)
+    )
+    if (count or 0) >= MAX_PER_USER:
+        raise TooMany
+    name = name.strip()[:NAME_MAX]
+    server_id: str | None = None
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        handle = await db.scalar(select(User.handle).where(User.id == user_id))
+        server_id, password = await mail_admin.create_app_password(str(handle), name)
+    else:
+        password = generate()
+    row = UserAppPassword(
+        user_id=user_id, name=name, password_hash=hash_password(password), server_id=server_id
+    )
+    db.add(row)
+    await db.commit()
+    return row, password
+
+
+async def _revoke_on_server(
+    db: AsyncSession, user_id: uuid.UUID, rows: list[UserAppPassword]
+) -> None:
+    """Revoke on the mail server too (first: if it cannot be reached, the
+    password keeps working there, so nothing is revoked here either)."""
+    server_ids = [r.server_id for r in rows if r.server_id]
+    if server_ids and hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        handle = await db.scalar(select(User.handle).where(User.id == user_id))
+        await mail_admin.delete_app_passwords(str(handle), server_ids)
+
+
+async def revoke(db: AsyncSession, user_id: uuid.UUID, app_password_id: uuid.UUID) -> bool:
+    rows = list(
+        await db.scalars(
+            select(UserAppPassword).where(
+                UserAppPassword.id == app_password_id, UserAppPassword.user_id == user_id
+            )
+        )
+    )
+    await _revoke_on_server(db, user_id, rows)
+    result = await db.execute(
+        delete(UserAppPassword).where(
+            UserAppPassword.id == app_password_id, UserAppPassword.user_id == user_id
+        )
+    )
+    await db.commit()
+    await _changed(user_id)
+    return bool(result.rowcount)  # type: ignore[attr-defined]
+
+
+async def revoke_all(db: AsyncSession, user_id: uuid.UUID) -> None:
+    """Every app signs in again (the main password was reset)."""
+    await _revoke_on_server(db, user_id, await list_for(db, user_id))
+    await db.execute(delete(UserAppPassword).where(UserAppPassword.user_id == user_id))
+    await db.commit()
+    await _changed(user_id)
+
+
+async def verify(db: AsyncSession, user_id: uuid.UUID, password: str) -> UserAppPassword | None:
+    """The app password ``password`` matches, if any — for the mail server's
+    sign-in and for tests; never for the webmail's own sign-in."""
+    if len(password) not in (LENGTH, SERVER_LENGTH):
+        return None
+    for row in await list_for(db, user_id):
+        if verify_password(row.password_hash, password):
+            return row
+    return None

backend/app/identity/invitations.py

@@ -0,0 +1,240 @@
+"""Invitations: a user sends a friend a single-use sign-up link.
+
+While the administrator has invitations switched on, every user may send
+``PER_USER`` of them. A link lets exactly one account be created — even with
+public registration closed — and works for ``VALID_FOR``; one that ran out
+unused is handed back to its sender. Only a hash of the link's token is kept.
+
+The email is built from the administrator's template (or the built-in one)
+with these placeholders, written ``{{name}}``:
+
+    invitee_name   the friend's name, as the sender typed it
+    inviter_name   the sender's name (their mail account's display name,
+                   else their address)
+    invite_link    the sign-up address itself
+    invite_button  a ready-made button linking to it
+    logo           the Ultimo Email logo
+
+Names are HTML-escaped where they go into the HTML: they come from users and
+end up in mail to strangers.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import hashlib
+import html as html_lib
+import re
+import secrets
+import uuid
+from dataclasses import dataclass
+
+import nh3
+from sqlalchemy import func, or_, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.identity.mailer import LOGO_HTML
+from app.models import Account, AppSettings, Invitation, User
+from app.outbox.inline_images import InlinePart, data_image
+
+PER_USER = 2
+VALID_FOR = dt.timedelta(days=14)
+MAX_TEMPLATE_BYTES = 500 * 1024
+
+PLACEHOLDERS = ("invitee_name", "inviter_name", "invite_link", "invite_button", "logo")
+_PLACEHOLDER = re.compile(r"\{\{\s*([a-z_]+)\s*\}\}")
+_DATA_SRC = re.compile(r"""(\bsrc\s*=\s*)(["'])(data:image/[^"']+)\2""", re.IGNORECASE)
+
+DEFAULT_SUBJECT = "{{inviter_name}} invites you to Ultimo Email"
+_BODY = "font-family:Arial,Helvetica,sans-serif;font-size:15px;color:#1f2937;line-height:1.5"
+_MUTED = "font-size:13px;color:#6b7280"
+_BUTTON = (
+    "display:inline-block;background:#2563eb;color:#ffffff;text-decoration:none;"
+    "padding:10px 18px;border-radius:8px;font-weight:bold"
+)
+DEFAULT_TEMPLATE = f"""<!doctype html>
+<html><body style="{_BODY}">
+{{{{logo}}}}
+<p>Hi {{{{invitee_name}}}},</p>
+<p><b>{{{{inviter_name}}}}</b> invites you to Ultimo Email — private email,
+calendar and contacts in one place.</p>
+<p>{{{{invite_button}}}}</p>
+<p style="{_MUTED}">Or paste this address into your browser:<br>{{{{invite_link}}}}</p>
+<p style="{_MUTED}">The invitation works once and expires in 14 days. If you
+weren't expecting it, you can ignore this email.</p>
+</body></html>
+"""
+
+
+class InvitationError(Exception):
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+@dataclass(frozen=True)
+class RenderedMail:
+    subject: str
+    text: str
+    html: str
+    inline: list[InlinePart]
+
+
+def hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def sender_address(row: AppSettings) -> str:
+    """The invitation's From. Empty: the system-mail From address, the one the
+    SMTP account is known to be allowed to send as."""
+    return row.invite_from.strip() or row.smtp_from
+
+
+def template_problem(html: str) -> str | None:
+    """Why ``html`` cannot be used as the invitation template, or None."""
+    if len(html.encode()) > MAX_TEMPLATE_BYTES:
+        return f"the template is larger than {MAX_TEMPLATE_BYTES // 1024} KB"
+    used = set(_PLACEHOLDER.findall(html))
+    unknown = sorted(used - set(PLACEHOLDERS))
+    if unknown:
+        return "unknown placeholder: " + ", ".join("{{" + u + "}}" for u in unknown)
+    if not used & {"invite_link", "invite_button"}:
+        return "the template must contain {{invite_link}} or {{invite_button}}"
+    return None
+
+
+def render(row: AppSettings, *, invitee_name: str, inviter_name: str, link: str) -> RenderedMail:
+    """The invitation email for one friend, from the configured template."""
+    template = row.invite_template or DEFAULT_TEMPLATE
+    subject_template = row.invite_subject or DEFAULT_SUBJECT
+    esc_link = html_lib.escape(link, quote=True)
+    html_values = {
+        "invitee_name": html_lib.escape(invitee_name),
+        "inviter_name": html_lib.escape(inviter_name),
+        "invite_link": f'<a href="{esc_link}">{esc_link}</a>',
+        "invite_button": f'<a href="{esc_link}" style="{_BUTTON}">Accept the invitation</a>',
+        "logo": LOGO_HTML,
+    }
+    text_values = {
+        "invitee_name": invitee_name,
+        "inviter_name": inviter_name,
+        "invite_link": link,
+        "invite_button": link,
+        "logo": "",
+    }
+
+    def fill(source: str, values: dict[str, str]) -> str:
+        return _PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), source)
+
+    # An image written into the template itself (data:) goes as an inline part:
+    # Gmail does not show data: images.
+    inline: list[InlinePart] = []
+
+    def to_cid(match: re.Match[str]) -> str:
+        part = data_image(match.group(3), get_settings().app_domain)
+        if part is None:
+            return match.group(0)
+        inline.append(part)
+        return f"{match.group(1)}{match.group(2)}cid:{part.cid}{match.group(2)}"
+
+    html = _DATA_SRC.sub(to_cid, fill(template, html_values))
+    text = fill(template, text_values)
+    # Keep the paragraphs and line breaks the HTML shows once its tags are gone.
+    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"</(?:p|div|h[1-6]|li|tr|table)\s*>", "\n\n", text, flags=re.IGNORECASE)
+    text = nh3.clean(text, tags=set(), attributes={}, strip_comments=True)
+    text = html_lib.unescape(re.sub(r"\n\s*\n\s*", "\n\n", text)).strip() + "\n"
+    subject = " ".join(fill(subject_template, text_values).split())
+    return RenderedMail(subject=subject, text=text, html=html, inline=inline)
+
+
+async def inviter_name(db: AsyncSession, user: User) -> str:
+    """How the friend is told who invited them."""
+    names = (
+        await db.execute(
+            select(Account.email, Account.display_name).where(Account.user_id == user.id)
+        )
+    ).all()
+    own = next((n for e, n in names if e.lower() == user.handle.lower() and n), None)
+    return own or next((n for _, n in names if n), None) or user.handle
+
+
+def _counts(now: dt.datetime):  # type: ignore[no-untyped-def]
+    # Used, or still valid: an invitation that expired unused is given back.
+    return or_(Invitation.used_at.isnot(None), Invitation.expires_at > now)
+
+
+async def remaining(db: AsyncSession, user_id: uuid.UUID) -> int:
+    now = dt.datetime.now(dt.UTC)
+    taken = await db.scalar(
+        select(func.count())
+        .select_from(Invitation)
+        .where(Invitation.inviter_id == user_id, _counts(now))
+    )
+    return max(0, PER_USER - int(taken or 0))
+
+
+async def sent(db: AsyncSession, user_id: uuid.UUID) -> list[Invitation]:
+    return list(
+        await db.scalars(
+            select(Invitation)
+            .where(Invitation.inviter_id == user_id)
+            .order_by(Invitation.created_at.desc())
+        )
+    )
+
+
+async def create(db: AsyncSession, user: User, *, name: str, email: str) -> tuple[Invitation, str]:
+    """A new invitation from ``user`` (not committed): the row and the token.
+    Raises :class:`InvitationError` when the user has none left."""
+    # One at a time per user, so two quick clicks cannot both take the last one.
+    await db.execute(select(User.id).where(User.id == user.id).with_for_update())
+    if await remaining(db, user.id) <= 0:
+        raise InvitationError("no_invitations_left", "you have no invitations left")
+    token = secrets.token_urlsafe(32)
+    invitation = Invitation(
+        inviter_id=user.id,
+        token_hash=hash_token(token),
+        invitee_name=name,
+        invitee_email=email,
+        expires_at=dt.datetime.now(dt.UTC) + VALID_FOR,
+    )
+    db.add(invitation)
+    await db.flush()
+    return invitation, token
+
+
+async def find_valid(db: AsyncSession, token: str) -> Invitation | None:
+    """The unused, unexpired invitation behind ``token``, without using it."""
+    invitation: Invitation | None = await db.scalar(
+        select(Invitation).where(
+            Invitation.token_hash == hash_token(token),
+            Invitation.used_at.is_(None),
+            Invitation.expires_at > dt.datetime.now(dt.UTC),
+        )
+    )
+    return invitation
+
+
+async def claim(db: AsyncSession, token: str) -> uuid.UUID | None:
+    """Use the invitation up (in the caller's transaction): its id, or None if
+    it is unknown, used or expired. Atomic — two sign-ups with one link cannot
+    both get it."""
+    return await db.scalar(
+        update(Invitation)
+        .where(
+            Invitation.token_hash == hash_token(token),
+            Invitation.used_at.is_(None),
+            Invitation.expires_at > dt.datetime.now(dt.UTC),
+        )
+        .values(used_at=func.now())
+        .returning(Invitation.id)
+    )
+
+
+async def record_user(db: AsyncSession, invitation_id: uuid.UUID, user_id: uuid.UUID) -> None:
+    await db.execute(
+        update(Invitation).where(Invitation.id == invitation_id).values(used_by=user_id)
+    )

backend/app/identity/mailer.py

@@ -1,6 +1,7 @@
 """Outgoing system mail (SPEC §6.1): mail the server sends on its own behalf.
 
-Today that is only the password-reset link. It goes out through the SMTP
+Password-reset links, and invitations users send their friends
+(app/identity/invitations.py). It goes out through the SMTP
 account the administrator configured in the admin panel (``app_settings``),
 never through a user's own mail account — the whole point is that the user
 cannot sign in. The administrator's server is trusted, so unlike user mail
@@ -10,19 +11,26 @@ on the LAN).
 
 from __future__ import annotations
 
+import base64
 import html as html_lib
 import logging
+from collections.abc import Sequence
 from dataclasses import dataclass
 from email.message import EmailMessage
-from email.utils import formataddr, make_msgid
+from email.utils import formataddr, formatdate, make_msgid
 from functools import cache
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import aiosmtplib
 
+from app.config import smtp_helo_name
 from app.models import AppSettings
 from app.security.crypto import decrypt_secret
 
+if TYPE_CHECKING:
+    from app.outbox.inline_images import InlinePart
+
 logger = logging.getLogger(__name__)
 
 SENDER_NAME = "Ultimo Email"
@@ -33,9 +41,9 @@ _TIMEOUT = 20.0
 # because mail clients (Gmail first) do not display SVG. Rendered at twice the
 # size it is shown at, for sharp high-DPI screens.
 _LOGO_FILE = Path(__file__).parent / "assets" / "ultimo-logo.png"
-_LOGO_CID = "ultimo-logo@ultimo.email"
+LOGO_CID = "ultimo-logo@ultimo.email"
 LOGO_HTML = (
-    f'<p style="text-align:center;margin:0 0 24px"><img src="cid:{_LOGO_CID}" '
+    f'<p style="text-align:center;margin:0 0 24px"><img src="cid:{LOGO_CID}" '
     'width="180" height="47" alt="Ultimo Email" style="display:inline-block;border:0"></p>'
 )
 
@@ -45,6 +53,10 @@ def _logo() -> bytes:
     return _LOGO_FILE.read_bytes()
 
 
+# For previews in the admin panel, which show the HTML without message parts.
+LOGO_DATA_URL = "data:image/png;base64," + base64.b64encode(_LOGO_FILE.read_bytes()).decode()
+
+
 class MailerError(Exception):
     """System mail is not configured, or the SMTP server refused the message."""
 
@@ -76,19 +88,49 @@ def mailer_config(row: AppSettings) -> MailerConfig | None:
     )
 
 
-async def send_mail(cfg: MailerConfig, to: str, subject: str, text: str, html: str) -> None:
+async def send_mail(
+    cfg: MailerConfig,
+    to: str,
+    subject: str,
+    text: str,
+    html: str,
+    *,
+    sender: str | None = None,
+    reply_to: str | None = None,
+    inline: Sequence[InlinePart] = (),
+) -> None:
+    """Send one system message. ``sender`` replaces the configured From address
+    (the SMTP account must be allowed to send as it); ``inline`` are images the
+    HTML names by ``cid:``."""
     msg = EmailMessage()
-    msg["From"] = formataddr((SENDER_NAME, cfg.sender))
+    msg["From"] = formataddr((SENDER_NAME, sender or cfg.sender))
     msg["To"] = to
+    if reply_to:
+        msg["Reply-To"] = reply_to
     msg["Subject"] = subject
+    # RFC 5322 requires it, and mail without one scores as spam (Gmail).
+    msg["Date"] = formatdate(localtime=True)
     msg["Message-ID"] = make_msgid(domain=cfg.sender.rsplit("@", 1)[-1])
     msg["Auto-Submitted"] = "auto-generated"  # RFC 3834: do not auto-reply to this
     msg.set_content(text)
     msg.add_alternative(html, subtype="html")
-    if f"cid:{_LOGO_CID}" in html:
+    if f"cid:{LOGO_CID}" in html:
         html_part = msg.get_body(("html",))
         if isinstance(html_part, EmailMessage):
-            html_part.add_related(_logo(), maintype="image", subtype="png", cid=f"<{_LOGO_CID}>")
+            html_part.add_related(_logo(), maintype="image", subtype="png", cid=f"<{LOGO_CID}>")
+    if inline:
+        html_part = msg.get_body(("html",))
+        assert isinstance(html_part, EmailMessage)
+        for part in inline:
+            maintype, _, subtype = part.mime_type.partition("/")
+            html_part.add_related(
+                part.data,
+                maintype=maintype,
+                subtype=subtype,
+                cid=f"<{part.cid}>",
+                filename=part.filename,
+                disposition="inline",
+            )
     try:
         await aiosmtplib.send(
             msg,
@@ -99,6 +141,7 @@ async def send_mail(cfg: MailerConfig, to: str, subject: str, text: str, html: s
             use_tls=cfg.security == "ssl",
             start_tls=cfg.security == "starttls",
             timeout=_TIMEOUT,
+            local_hostname=smtp_helo_name(),
         )
     except (aiosmtplib.SMTPException, OSError) as exc:
         raise MailerError(str(exc) or type(exc).__name__) from exc

backend/app/identity/password_policy.py

@@ -0,0 +1,145 @@
+"""What a new account password must not be (sign-up and reset).
+
+Length is bounded by the request schemas (8–128). On top of that a password is
+refused when it is one of the passwords every guessing attack tries first, is
+made of the user's own address, is one character repeated, or has turned up in
+a known data breach. The breach check asks Have I Been Pwned's range API with
+the first five hex digits of the password's SHA-1 only (k-anonymity: neither the
+password nor its hash leaves the server) and is skipped, not failed, when that
+service cannot be reached — a sign-up never hangs on someone else's outage.
+Set ``PASSWORD_BREACH_CHECK=false`` to never ask it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+
+import httpx
+
+from app.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+MIN_LENGTH = 8
+MAX_LENGTH = 128
+
+_RANGE_API = "https://api.pwnedpasswords.com/range/"
+_TIMEOUT = 3.0
+
+# The front of every guessing list (and of every breach), lower-cased.
+_COMMON = frozenset(
+    {
+        "password",
+        "password1",
+        "password12",
+        "password123",
+        "password1234",
+        "passw0rd",
+        "p@ssw0rd",
+        "p@ssword",
+        "12345678",
+        "123456789",
+        "1234567890",
+        "0123456789",
+        "87654321",
+        "11111111",
+        "00000000",
+        "12341234",
+        "qwertyui",
+        "qwerty123",
+        "qwerty1234",
+        "qwertyuiop",
+        "1q2w3e4r",
+        "1q2w3e4r5t",
+        "1qaz2wsx",
+        "zaq12wsx",
+        "asdfghjk",
+        "asdfasdf",
+        "iloveyou",
+        "iloveyou1",
+        "sunshine",
+        "princess",
+        "football",
+        "baseball",
+        "welcome1",
+        "welcome123",
+        "letmein1",
+        "trustno1",
+        "superman",
+        "batman123",
+        "dragon123",
+        "monkey123",
+        "abc12345",
+        "abcd1234",
+        "aa123456",
+        "qazwsxedc",
+        "changeme",
+        "changeme1",
+        "administrator",
+        "admin123",
+        "adminadmin",
+        "computer",
+        "internet",
+        "whatever",
+        "starwars",
+        "michelle",
+        "jennifer",
+        "charlie1",
+        "zaq1zaq1",
+        "haslo123",
+        "polska123",
+        "kochamcie",
+        "zaq1@wsx",
+        "qwerty12",
+        "1234qwer",
+        "q1w2e3r4",
+    }
+)
+
+
+class PasswordRejected(Exception):
+    """The password may not be used; ``code`` and ``message`` say why."""
+
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+async def _breached(password: str) -> bool:
+    digest = hashlib.sha1(password.encode("utf-8")).hexdigest().upper()  # noqa: S324 - HIBP's format
+    prefix, suffix = digest[:5], digest[5:]
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(_RANGE_API + prefix, headers={"Add-Padding": "true"})
+            resp.raise_for_status()
+    except httpx.HTTPError as exc:
+        logger.warning("breached-password check skipped: %s", exc)
+        return False
+    for line in resp.text.splitlines():
+        candidate, _, count = line.partition(":")
+        if candidate.strip() == suffix and count.strip() not in ("", "0"):  # padding rows say 0
+            return True
+    return False
+
+
+async def check_new_password(password: str, handle: str) -> None:
+    """Raise :class:`PasswordRejected` if ``password`` may not be used by ``handle``."""
+    lowered = password.lower()
+    if len(set(password)) <= 2:
+        raise PasswordRejected("weak_password", "this password is too easy to guess")
+    if lowered in _COMMON:
+        raise PasswordRejected("weak_password", "this is one of the most common passwords")
+    localpart = handle.split("@", 1)[0].lower()
+    if len(localpart) >= 3 and localpart in lowered:
+        # The name with a year or a few characters added ("jsmith2026") is the
+        # first thing anyone who knows the address will try.
+        rest = lowered.replace(localpart, "", 1)
+        if len(rest) < 4 or all(not c.isalpha() for c in rest):
+            raise PasswordRejected("weak_password", "the password must not be your address")
+    if get_settings().password_breach_check and await _breached(password):
+        raise PasswordRejected(
+            "breached_password",
+            "this password has appeared in a data breach — please choose another",
+        )

backend/app/identity/routes.py

@@ -6,18 +6,20 @@ from __future__ import annotations
 import logging
 import re
 import uuid
-from typing import Annotated
+from typing import Annotated, NoReturn
 
 from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import hosted
 from app.api.errors import raise_api_error
 from app.config import get_settings
 from app.db import get_session
 from app.events import get_event_bus
+from app.hosted.admin import MailServerError
+from app.identity import app_passwords, invitations, trusted_devices
 from app.identity import totp as totp_mod
-from app.identity import trusted_devices
 from app.identity.cookies import (
     clear_auth_cookies,
     new_csrf_token,
@@ -25,6 +27,7 @@ from app.identity.cookies import (
 )
 from app.identity.deps import CurrentSession, CurrentUser, require_csrf
 from app.identity.mailer import MailerConfig, MailerError, mailer_config, reset_mail, send_mail
+from app.identity.password_policy import PasswordRejected, check_new_password
 from app.identity.passwords import hash_password, verify_password
 from app.identity.reset import (
     consume_token,
@@ -34,6 +37,11 @@ from app.identity.reset import (
     request_allowed,
 )
 from app.identity.schemas import (
+    AppPasswordCreatedOut,
+    AppPasswordIn,
+    AppPasswordOut,
+    InvitationCheckIn,
+    InvitationCheckOut,
     LoginRequest,
     MessageOut,
     PasswordResetCheckIn,
@@ -102,6 +110,7 @@ def _user_out(user: User) -> UserOut:
         settings=user.settings,
         totp_enabled=user.totp_secret is not None,
         is_admin=user.is_admin,
+        mail_host=hosted.mail_hostname() if hosted.enabled() else None,
     )
 
 
@@ -109,26 +118,68 @@ def _user_out(user: User) -> UserOut:
 async def registration(db: DbSession) -> RegistrationStatusOut:
     """Whether signup is open. Unauthenticated: the signup page needs it to render."""
     status = await registration_status(db)
-    return RegistrationStatusOut(open=status.open, bootstrap=status.bootstrap)
+    return RegistrationStatusOut(
+        open=status.open, bootstrap=status.bootstrap, domain=get_settings().app_domain
+    )
 
 
 _REGISTRATION_ERROR_STATUS = {"handle_taken": 409, "registration_closed": 403}
 
 
+@router.post("/invitation/check", response_model=InvitationCheckOut)
+async def invitation_check(body: InvitationCheckIn, db: DbSession) -> InvitationCheckOut:
+    """Whether an invitation link still works, and who it is from — for the
+    sign-up page. Does not use it up."""
+    if not (await app_settings(db)).invitations_enabled:
+        return InvitationCheckOut(valid=False)
+    invitation = await invitations.find_valid(db, body.token)
+    if invitation is None:
+        return InvitationCheckOut(valid=False)
+    inviter = await db.get(User, invitation.inviter_id)
+    return InvitationCheckOut(
+        valid=True,
+        invitee_name=invitation.invitee_name,
+        inviter_name=await invitations.inviter_name(db, inviter) if inviter else "",
+    )
+
+
 @router.post("/register", response_model=UserOut)
 async def register(
     body: RegisterRequest, request: Request, response: Response, db: DbSession
 ) -> UserOut:
+    try:
+        await check_new_password(body.password, resolve_handle(body.localpart))
+    except PasswordRejected as exc:
+        raise_api_error(400, exc.code, exc.message)
+    invitation_id: uuid.UUID | None = None
+    if body.invite:
+        # Used up in this same transaction: if the sign-up fails (the address
+        # is taken, …) the rollback hands the invitation back.
+        if (await app_settings(db)).invitations_enabled:
+            invitation_id = await invitations.claim(db, body.invite)
+        if invitation_id is None:
+            raise_api_error(
+                400, "invalid_invitation", "this invitation has already been used or has expired"
+            )
     try:
         user = await register_user(
             db,
             localpart=body.localpart,
             password=body.password,
             recovery_email=body.recovery_email,
+            invited=invitation_id is not None,
         )
     except RegistrationError as exc:
         raise_api_error(_REGISTRATION_ERROR_STATUS.get(exc.code, 400), exc.code, exc.message)
+    if invitation_id is not None:
+        await invitations.record_user(db, invitation_id, user.id)
     await db.commit()
+    # The user's own mailbox (App. D §D.4): it exists on the mail server as soon
+    # as the user does (the directory reads the users table); this adds it to
+    # the webmail.
+    from app.hosted import native
+
+    await native.ensure_and_sync(db, user)
     await _establish_session(request, response, user)
     return _user_out(user)
 
@@ -300,6 +351,11 @@ async def password_reset(body: PasswordResetIn, db: DbSession) -> MessageOut:
     user = await db.get(User, user_id) if user_id else None
     if user is None:
         raise_api_error(400, "invalid_token", "this link is invalid or has expired")
+    # Before the 2FA code and the link are spent: a refused password costs neither.
+    try:
+        await check_new_password(body.password, user.handle)
+    except PasswordRejected as exc:
+        raise_api_error(400, exc.code, exc.message)
     if user.totp_secret is not None:
         if not body.totp:
             raise_api_error(401, "totp_required", "a 2FA code is required")
@@ -312,12 +368,85 @@ async def password_reset(body: PasswordResetIn, db: DbSession) -> MessageOut:
     user.password_hash = hash_password(body.password)
     await db.commit()
     await destroy_all_sessions(user.id)  # force re-login everywhere after reset
+    # Mail apps sign in again too: the reset may be the owner taking the account back.
+    try:
+        await app_passwords.revoke_all(db, user.id)
+    except MailServerError:
+        # The new password is set; the app passwords stay listed (and working)
+        # until the mail server answers, so the user can still revoke them.
+        logger.exception("password reset: could not revoke app passwords on the mail server")
     # Whoever reset the password may be the owner taking the account back: no
     # device keeps skipping the code on the old trust.
     await trusted_devices.revoke_all(user.id)
     return MessageOut()
 
 
+# --- App passwords (SPEC App. D §D.5) -----------------------------------------
+
+
+def _mail_server_down() -> NoReturn:
+    raise_api_error(
+        502, "mail_server_unavailable", "the mail server is not answering; try again shortly"
+    )
+
+
+def _app_password_out(row: object) -> AppPasswordOut:
+    from app.models import UserAppPassword
+
+    assert isinstance(row, UserAppPassword)
+    return AppPasswordOut(
+        id=str(row.id),
+        name=row.name,
+        created_at=row.created_at.isoformat(),
+        last_used_at=row.last_used_at.isoformat() if row.last_used_at else None,
+    )
+
+
+@router.get("/app-passwords", response_model=list[AppPasswordOut])
+async def list_app_passwords(user: CurrentUser, db: DbSession) -> list[AppPasswordOut]:
+    return [_app_password_out(r) for r in await app_passwords.list_for(db, user.id)]
+
+
+@router.post(
+    "/app-passwords", response_model=AppPasswordCreatedOut, dependencies=[Depends(require_csrf)]
+)
+async def create_app_password(
+    body: AppPasswordIn, user: CurrentUser, db: DbSession
+) -> AppPasswordCreatedOut:
+    """A new password for one mail app. Takes the account password: an app
+    password opens the mailbox without a 2FA code."""
+    if not verify_password(user.password_hash, body.password):
+        raise_api_error(403, "invalid_password", "incorrect password")
+    if not body.name.strip():
+        raise_api_error(400, "invalid_name", "give the app password a name")
+    try:
+        row, password = await app_passwords.create(db, user.id, body.name)
+    except app_passwords.TooMany:
+        raise_api_error(
+            409, "too_many", f"you can have up to {app_passwords.MAX_PER_USER} app passwords"
+        )
+    except MailServerError:
+        _mail_server_down()
+    return AppPasswordCreatedOut(**_app_password_out(row).model_dump(), app_password=password)
+
+
+@router.delete(
+    "/app-passwords/{app_password_id}",
+    response_model=MessageOut,
+    dependencies=[Depends(require_csrf)],
+)
+async def revoke_app_password(
+    app_password_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> MessageOut:
+    try:
+        revoked = await app_passwords.revoke(db, user.id, app_password_id)
+    except MailServerError:
+        _mail_server_down()
+    if not revoked:
+        raise_api_error(404, "not_found", "app password not found")
+    return MessageOut()
+
+
 @router.put("/recovery-email", response_model=UserOut, dependencies=[Depends(require_csrf)])
 async def set_recovery_email(body: RecoveryEmailIn, user: CurrentUser, db: DbSession) -> UserOut:
     """Add, change or remove the recovery address. Takes the password: whoever

backend/app/identity/schemas.py

@@ -10,8 +10,11 @@ from pydantic import BaseModel, Field
 
 class RegisterRequest(BaseModel):
     localpart: str = Field(min_length=3, max_length=30)
-    password: str = Field(min_length=8, max_length=1024)
+    password: str = Field(min_length=8, max_length=128)
     recovery_email: str | None = None
+    # A friend's invitation (app/identity/invitations.py): signs up even while
+    # public registration is closed, and is used up by it.
+    invite: str | None = Field(default=None, max_length=200)
 
 
 class LoginRequest(BaseModel):
@@ -49,7 +52,7 @@ class PasswordResetRequestIn(BaseModel):
 
 class PasswordResetIn(BaseModel):
     token: str = Field(min_length=1)
-    password: str = Field(min_length=8, max_length=1024)
+    password: str = Field(min_length=8, max_length=128)
     totp: str | None = None  # required when the account has 2FA enabled
 
 
@@ -63,6 +66,23 @@ class PasswordResetCheckOut(BaseModel):
     totp_required: bool = False
 
 
+class AppPasswordIn(BaseModel):
+    name: str = Field(min_length=1, max_length=60)  # "Thunderbird on laptop"
+    password: str = Field(min_length=1, max_length=1024)  # the account password, to confirm
+
+
+class AppPasswordOut(BaseModel):
+    id: str
+    name: str
+    created_at: str
+    last_used_at: str | None = None
+
+
+class AppPasswordCreatedOut(AppPasswordOut):
+    # The generated password itself: in this answer only, never again.
+    app_password: str
+
+
 class RecoveryEmailIn(BaseModel):
     password: str = Field(min_length=1, max_length=1024)
     # None (or blank) removes the recovery address.
@@ -76,6 +96,19 @@ class UserOut(BaseModel):
     settings: dict[str, object]
     totp_enabled: bool = False
     is_admin: bool = False
+    # The hosted mail server's name for mail apps (IMAP 993, SMTP 465/587), or
+    # None when there are no hosted mailboxes (SPEC App. D).
+    mail_host: str | None = None
+
+
+class InvitationCheckIn(BaseModel):
+    token: str = Field(min_length=1, max_length=200)
+
+
+class InvitationCheckOut(BaseModel):
+    valid: bool
+    invitee_name: str = ""
+    inviter_name: str = ""
 
 
 class RegistrationStatusOut(BaseModel):
@@ -84,17 +117,23 @@ class RegistrationStatusOut(BaseModel):
     open: bool
     # True when the instance has no users yet, so signup creates the administrator.
     bootstrap: bool
+    # The domain every handle lives under (localpart@domain), shown beside the
+    # handle field.
+    domain: str
 
 
 class AdminSettingsOut(BaseModel):
     registration_open: bool
     default_quota_bytes: int
+    # Users may send friends sign-up invitations (app/identity/invitations.py).
+    invitations_enabled: bool
 
 
 class AdminSettingsIn(BaseModel):
     """Merge semantics: omitted fields are left as they are."""
 
     registration_open: bool | None = None
+    invitations_enabled: bool | None = None
     # Applies to users created after the change; existing mailboxes keep theirs.
     default_quota_bytes: int | None = Field(default=None, gt=0)
 
@@ -125,6 +164,36 @@ class AdminMailIn(BaseModel):
     public_url: str | None = Field(default=None, max_length=500)
 
 
+class AdminInviteTemplateOut(BaseModel):
+    # Empty strings: the built-in subject / template / sender is used.
+    subject: str
+    html: str
+    from_address: str
+    default_subject: str
+    default_html: str
+    default_from: str
+    placeholders: list[str]
+
+
+class AdminInviteTemplateIn(BaseModel):
+    """Merge semantics: omitted fields are left as they are; an empty string
+    goes back to the built-in one."""
+
+    subject: str | None = Field(default=None, max_length=300)
+    html: str | None = None
+    from_address: str | None = Field(default=None, max_length=320)
+
+
+class AdminInvitePreviewIn(BaseModel):
+    subject: str = Field(default="", max_length=300)
+    html: str = ""
+
+
+class AdminInvitePreviewOut(BaseModel):
+    subject: str
+    html: str
+
+
 class AdminMailTestIn(BaseModel):
     to: str = Field(min_length=3, max_length=320)
 

backend/app/identity/service.py

@@ -97,7 +97,12 @@ async def registration_status(db: AsyncSession) -> RegistrationStatus:
 
 
 async def register_user(
-    db: AsyncSession, *, localpart: str, password: str, recovery_email: str | None
+    db: AsyncSession,
+    *,
+    localpart: str,
+    password: str,
+    recovery_email: str | None,
+    invited: bool = False,
 ) -> User:
     """Create a new user. Raises :class:`RegistrationError` on validation/uniqueness.
 
@@ -114,7 +119,7 @@ async def register_user(
 
     settings = await app_settings(db, for_update=True)
     bootstrap = not await _has_users(db)
-    if not bootstrap and not settings.registration_open:
+    if not bootstrap and not settings.registration_open and not invited:
         raise RegistrationError("registration_closed", "registration is closed on this server")
 
     user = User(

backend/app/identity/user_admin.py

@@ -67,7 +67,16 @@ async def search_users(db: AsyncSession, q: str) -> list[tuple[User, int, list[s
 
 
 async def set_blocked(db: AsyncSession, user: User, blocked: bool) -> None:
-    """Block or unblock. Blocking also signs the user out everywhere."""
+    """Block or unblock. Blocking also signs the user out everywhere, and shuts
+    their hosted mailbox to every sign-in (mail apps included); it still
+    receives mail. The mail server goes first: if it cannot be told, nothing
+    changes."""
+    from app import hosted
+
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        await mail_admin.set_blocked(user.handle, blocked)
     if blocked and user.blocked_at is None:
         user.blocked_at = dt.datetime.now(dt.UTC)
         await db.flush()
@@ -97,8 +106,18 @@ async def _plugin_tables(db: AsyncSession) -> list[str]:
 
 
 async def delete_user(db: AsyncSession, user: User) -> None:
-    """Delete ``user`` and everything they own. Commits."""
+    """Delete ``user`` and everything they own. Commits.
+
+    Their hosted mailbox goes first, with all its mail: if the mail server
+    cannot be reached nothing is deleted — a mailbox left behind would be
+    handed, mail and all, to whoever registers the address next."""
+    from app import hosted
+
     user_id = user.id
+    if hosted.enabled():
+        from app.hosted import admin as mail_admin
+
+        await mail_admin.delete_account(user.handle)
 
     # Mail accounts first, one by one: this is what stops their sync runners and
     # tidies threads, exactly as when the user removes an account themselves.

backend/app/indexer/attachments.py

@@ -0,0 +1,244 @@
+"""Attachments, kept once: inside their message (SPEC App. D §D.6 item 1).
+
+An attachment is not stored apart from its message any more — it was, which
+kept most of a mailbox's bytes twice (once base64 in the raw message, once
+decoded on its own). It is taken out of the raw message when wanted:
+
+* :func:`load` — its bytes (downloads, zip, inline images, forwarding, thumbnails);
+* :func:`file` — a file holding it, for streaming with HTTP Range (videos):
+  taken out into ``{user}/cache/`` and kept a day after its last use;
+* :func:`dedupe_existing` — the one-time pass removing the separate copies
+  stored before, each only after checking it can be taken out of its message.
+
+Blocking (parsing, disk): call these in a thread — after
+:func:`app.indexer.rawstore.ensure` for the message, which fetches a hosted
+mailbox's message from its mail server when it is not cached.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import logging
+import time
+import uuid
+from collections import defaultdict
+from pathlib import Path
+
+from app.indexer import blobstore, rawstore
+from app.indexer.mime import load_message, part_payload
+
+logger = logging.getLogger(__name__)
+
+CACHE_KEEP_S = 24 * 3600  # a taken-out file is kept this long after its last use
+CACHE_MAX_BYTES = 1024**3  # per user; the least recently used go first beyond it
+# part_index of a row whose content is not in its message (kept as its own file).
+NOT_IN_MESSAGE = -1
+
+
+def _legacy(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The separate copy stored before attachments were kept once, if still there."""
+    path = blobstore.blob_path(user_id, sha)
+    return path if path.exists() else None
+
+
+def load(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bytes:
+    """An attachment's bytes. Raises FileNotFoundError when they cannot be had."""
+    legacy = _legacy(user_id, sha)
+    if legacy is not None:
+        return legacy.read_bytes()
+    raw = rawstore.locate(user_id, message_sha)
+    if part_index == NOT_IN_MESSAGE or raw is None:
+        raise FileNotFoundError(sha)
+    payload = part_payload(load_message(path=raw), part_index, sha)
+    if payload is None:
+        raise FileNotFoundError(sha)
+    return payload
+
+
+def materialize(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bool:
+    """Make sure the attachment exists as a blob of its own — for a compose
+    attachment (forward, a draft reopened), which is read by its hash. False
+    when its bytes cannot be had."""
+    if blobstore.exists(user_id, sha):
+        return True
+    try:
+        blobstore.store(user_id, load(user_id, message_sha, sha, part_index))
+    except FileNotFoundError:
+        return False
+    return True
+
+
+def _cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / "cache"
+
+
+def prune_cache(cache: Path) -> None:
+    """Drop streaming copies past their keep time, and the oldest beyond the cap."""
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file()]
+    now = time.time()
+    total = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > CACHE_KEEP_S or total > CACHE_MAX_BYTES:
+            path.unlink(missing_ok=True)
+
+
+def has_file(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether :func:`file` can answer without reading the message."""
+    return _legacy(user_id, sha) is not None or (_cache_dir(user_id) / sha).exists()
+
+
+def file(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> Path:
+    """A file holding the attachment, to stream from. Raises FileNotFoundError."""
+    legacy = _legacy(user_id, sha)
+    if legacy is not None:
+        return legacy
+    cache = _cache_dir(user_id)
+    path = cache / sha
+    if path.exists():
+        path.touch()  # used again: kept longer
+        return path
+    data = load(user_id, message_sha, sha, part_index)
+    cache.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+    tmp.write_bytes(data)
+    tmp.replace(path)
+    with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+        prune_cache(cache)
+    return path
+
+
+def drop_cached(user_id: uuid.UUID) -> None:
+    """Forget every taken-out file of a user (account removed)."""
+    cache = _cache_dir(user_id)
+    if cache.exists():
+        for path in cache.iterdir():
+            path.unlink(missing_ok=True)
+
+
+# ---- removing the copies stored before ------------------------------------------------
+
+# A file this recent may belong to work in progress (a sent message waiting
+# for its Sent copy is referenced only by its job): left for the next pass.
+_MIN_AGE_S = 3600
+
+
+def _part_indexes(raw_path: Path) -> dict[str, int]:
+    """Content hash → MIME part position, for every leaf part of a message."""
+    out: dict[str, int] = {}
+    for index, part in enumerate(load_message(path=raw_path).walk()):
+        if part.is_multipart():
+            continue
+        payload = part.get_payload(decode=True)
+        if isinstance(payload, bytes):
+            out.setdefault(hashlib.sha256(payload).hexdigest(), index)
+    return out
+
+
+async def dedupe_existing(batch: int = 200, *, user_id: uuid.UUID | None = None) -> dict[str, int]:
+    """Record where each older attachment sits in its message, and delete its
+    separate copy once it is known to be there. Idempotent; resumable."""
+    import asyncio
+
+    from sqlalchemy import select, update
+
+    from app.db import session_scope
+    from app.models import Attachment, ComposeAttachment, Message
+
+    stats = {"rows": 0, "found": 0, "not_found": 0, "files_removed": 0, "bytes_freed": 0}
+    while True:
+        async with session_scope() as db:
+            rows = (
+                await db.execute(
+                    select(
+                        Attachment.id,
+                        Attachment.user_id,
+                        Attachment.blob_sha256,
+                        Message.blob_sha256,
+                    )
+                    .join(Message, Message.id == Attachment.message_id)
+                    .where(
+                        Attachment.part_index.is_(None),
+                        *([Attachment.user_id == user_id] if user_id is not None else []),
+                    )
+                    .order_by(Attachment.message_id)
+                    .limit(batch)
+                )
+            ).all()
+        if not rows:
+            return stats
+        by_message: dict[tuple[uuid.UUID, str], list[tuple[uuid.UUID, str]]] = defaultdict(list)
+        for att_id, owner, sha, msg_sha in rows:
+            by_message[(owner, msg_sha)].append((att_id, sha))
+
+        located: dict[uuid.UUID, int] = {}
+        for (owner, msg_sha), atts in by_message.items():
+            raw = rawstore.locate(owner, msg_sha)
+            try:
+                indexes = await asyncio.to_thread(_part_indexes, raw) if raw is not None else {}
+            except Exception:  # noqa: BLE001 - unreadable message: keep the copies
+                logger.warning("attachments: cannot parse %s", raw, exc_info=True)
+                indexes = {}
+            for att_id, sha in atts:
+                located[att_id] = indexes.get(sha, NOT_IN_MESSAGE)
+
+        async with session_scope() as db:
+            for att_id, index in located.items():
+                await db.execute(
+                    update(Attachment).where(Attachment.id == att_id).values(part_index=index)
+                )
+        stats["rows"] += len(located)
+        stats["found"] += sum(1 for i in located.values() if i != NOT_IN_MESSAGE)
+        stats["not_found"] += sum(1 for i in located.values() if i == NOT_IN_MESSAGE)
+
+        # A separate copy goes only when every attachment row with that content
+        # is found in its message and nothing else uses the file.
+        candidates = {
+            (owner, sha)
+            for att_id, owner, sha, _ in rows
+            if located.get(att_id, NOT_IN_MESSAGE) != NOT_IN_MESSAGE
+        }
+        async with session_scope() as db:
+            for owner, sha in candidates:
+                path = blobstore.blob_path(owner, sha)
+                if not path.exists() or time.time() - path.stat().st_mtime < _MIN_AGE_S:
+                    continue
+                in_use = (
+                    await db.scalar(
+                        select(Message.id)
+                        .where(Message.user_id == owner, Message.blob_sha256 == sha)
+                        .limit(1)
+                    )
+                    or await db.scalar(
+                        select(ComposeAttachment.id)
+                        .where(
+                            ComposeAttachment.user_id == owner,
+                            ComposeAttachment.blob_sha256 == sha,
+                        )
+                        .limit(1)
+                    )
+                    or await db.scalar(
+                        select(Attachment.id)
+                        .where(
+                            Attachment.user_id == owner,
+                            Attachment.blob_sha256 == sha,
+                            (Attachment.part_index.is_(None))
+                            | (Attachment.part_index == NOT_IN_MESSAGE),
+                        )
+                        .limit(1)
+                    )
+                )
+                if in_use:
+                    continue
+                size = path.stat().st_size
+                path.unlink(missing_ok=True)
+                stats["files_removed"] += 1
+                stats["bytes_freed"] += size
+
+
+async def dedupe_existing_job(ctx: dict[str, object]) -> dict[str, int]:
+    """arq job (bulk queue): :func:`dedupe_existing`, run once after an update."""
+    stats = await dedupe_existing()
+    logger.info("attachments kept once: %s", stats)
+    return stats

backend/app/indexer/gc.py

@@ -0,0 +1,217 @@
+"""The nightly clean-up of stored files nothing uses any more.
+
+Messages leave in many ways — Delete forever, Empty Trash, the Trash/Spam
+retention purge, an expunge on the server, a folder re-synced — and none of
+them removes the message's file: files are stored by content, so one file can
+serve several rows (the same email in two folders, a forwarded file in a
+draft), and removing it is only safe once nothing refers to it. This pass does
+that for every user, whatever the way the rows went:
+
+* uploads (compose attachments) no draft has referred to for a week — left by
+  a sent message (its draft is gone) or a composer closed without saving;
+* message/attachment files no row refers to;
+* attachment previews (thumbnails) of attachments that are gone;
+* streaming copies past their keep time (app/indexer/attachments.py);
+* cached raw messages of a hosted mailbox beyond the cache's room
+  (app/indexer/rawstore.py).
+
+It also recounts threads whose unread/message counters disagree with their
+messages, and drops threads left empty (app/threading_/threader.py).
+
+A file younger than an hour is left alone: it may belong to work in progress
+(a message being stored, a sent message waiting for its Sent copy).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import re
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import delete, or_, select
+
+from app.config import get_settings
+from app.indexer import attachments as att_store
+from app.indexer import rawstore
+
+logger = logging.getLogger(__name__)
+
+MIN_AGE_S = 3600
+UPLOAD_KEEP = dt.timedelta(days=7)
+_SHA = re.compile(r"^[0-9a-f]{64}$")
+
+
+# ---- files a job still needs ---------------------------------------------------------
+#
+# A sent message whose Sent copy could not be filed yet is referenced only by
+# its retrying job (app/outbox/jobs.py), for up to a day: held here meanwhile.
+# Entries older than HOLD_MAX are ignored, in case a job died without letting go.
+
+HOLD_MAX_S = 3 * 24 * 3600
+
+
+def _hold_key(user_id: uuid.UUID | str) -> str:
+    return f"blob:held:{user_id}"
+
+
+async def hold(user_id: uuid.UUID | str, sha: str) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().zadd(_hold_key(user_id), {sha: time.time()})
+
+
+async def release(user_id: uuid.UUID | str, sha: str) -> None:
+    from app.redis_client import get_redis
+
+    await get_redis().zrem(_hold_key(user_id), sha)
+
+
+async def _held(user_id: uuid.UUID) -> set[str]:
+    from app.redis_client import get_redis
+
+    redis = get_redis()
+    oldest = time.time() - HOLD_MAX_S
+    await redis.zremrangebyscore(_hold_key(user_id), "-inf", oldest)
+    return {
+        m.decode() if isinstance(m, bytes) else str(m)
+        for m in await redis.zrangebyscore(_hold_key(user_id), oldest, "+inf")
+    }
+
+
+def _draft_upload_ids(payloads: list[dict[str, Any]]) -> set[uuid.UUID]:
+    out: set[uuid.UUID] = set()
+    for payload in payloads:
+        for value in payload.get("attachment_ids") or []:
+            try:
+                out.add(uuid.UUID(str(value)))
+            except ValueError:
+                continue
+    return out
+
+
+def _old(path: Path, now: float) -> bool:
+    return now - path.stat().st_mtime >= MIN_AGE_S
+
+
+async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
+    """Clean one user's stored files; returns what was removed."""
+    from app.db import session_scope
+    from app.models import Account, Attachment, ComposeAttachment, Draft, Message
+
+    stats = {"uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
+    async with session_scope() as db:
+        # Uploads no draft refers to, once a week old.
+        used = _draft_upload_ids(
+            list(await db.scalars(select(Draft.payload).where(Draft.user_id == user_id)))
+        )
+        stale = select(ComposeAttachment.id).where(
+            ComposeAttachment.user_id == user_id,
+            ComposeAttachment.created_at < dt.datetime.now(dt.UTC) - UPLOAD_KEEP,
+        )
+        if used:
+            stale = stale.where(ComposeAttachment.id.not_in(used))
+        gone = list(await db.scalars(stale))
+        if gone:
+            await db.execute(delete(ComposeAttachment).where(ComposeAttachment.id.in_(gone)))
+        stats["uploads"] = len(gone)
+
+    async with session_scope() as db:
+        # Every file still referred to. Attachments count only where their file
+        # is their only copy (not found in the message, or not yet looked for).
+        # A hosted mailbox's messages are kept by its mail server: their raw
+        # file here (left by a Sent copy filed late) is not needed.
+        referenced = set(
+            await db.scalars(
+                select(Message.blob_sha256)
+                .join(Account, Account.id == Message.account_id)
+                .where(Message.user_id == user_id, Account.kind != "native")
+            )
+        )
+        referenced |= set(
+            await db.scalars(
+                select(ComposeAttachment.blob_sha256).where(ComposeAttachment.user_id == user_id)
+            )
+        )
+        referenced |= set(
+            await db.scalars(
+                select(Attachment.blob_sha256).where(
+                    Attachment.user_id == user_id,
+                    or_(
+                        Attachment.part_index.is_(None),
+                        Attachment.part_index == att_store.NOT_IN_MESSAGE,
+                    ),
+                )
+            )
+        )
+        referenced |= await _held(user_id)
+        attachment_shas = set(
+            await db.scalars(select(Attachment.blob_sha256).where(Attachment.user_id == user_id))
+        )
+
+    # Counters out of step with their messages (see repair_threads).
+    from app.threading_.threader import repair_threads
+
+    async with session_scope() as db:
+        stats["threads_repaired"] = await repair_threads(db, user_id)
+
+    root = Path(get_settings().blob_root) / str(user_id)
+    if not root.is_dir():
+        return stats
+    now = time.time()
+    for path in root.glob("??/*"):
+        if not path.is_file() or not _SHA.match(path.name) or path.name in referenced:
+            continue
+        if not _old(path, now):
+            continue
+        size = path.stat().st_size
+        path.unlink(missing_ok=True)
+        stats["files"] += 1
+        stats["bytes"] += size
+    thumbs = root / "thumbs"
+    if thumbs.is_dir():
+        for path in thumbs.iterdir():
+            sha = path.name.split("-", 1)[0]
+            if (
+                path.is_file()
+                and _SHA.match(sha)
+                and sha not in attachment_shas
+                and _old(path, now)
+            ):
+                path.unlink(missing_ok=True)
+                stats["thumbs"] += 1
+    cache = root / "cache"
+    if cache.is_dir():
+        att_store.prune_cache(cache)
+    rawstore.prune(user_id)
+    return stats
+
+
+async def sweep(user_id: uuid.UUID | None = None) -> dict[str, int]:
+    """:func:`sweep_user` for one user, or every user."""
+    from app.db import session_scope
+    from app.models import User
+
+    async with session_scope() as db:
+        users = [user_id] if user_id is not None else list(await db.scalars(select(User.id)))
+    total = {"users": 0, "uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
+    for uid in users:
+        try:
+            stats = await sweep_user(uid)
+        except Exception:  # noqa: BLE001 - one user's trouble must not stop the rest
+            logger.warning("stored-file clean-up failed for user %s", uid, exc_info=True)
+            continue
+        total["users"] += 1
+        for key, value in stats.items():
+            total[key] += value
+    return total
+
+
+async def sweep_job(ctx: dict[str, Any]) -> dict[str, int]:
+    """arq cron job: the nightly :func:`sweep`."""
+    stats = await sweep()
+    logger.info("stored-file clean-up: %s", stats)
+    return stats

backend/app/indexer/indexer.py

@@ -19,12 +19,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.concurrency import loop_semaphore
 from app.config import get_settings
-from app.indexer import blobstore
+from app.indexer import blobstore, rawstore
 from app.indexer.mime import ParsedEmail, parse_message, parse_message_file
 from app.indexer.sanitize import sanitize_html
-from app.models import Attachment, BodyPart, KnownRecipient, Message
+from app.models import Account, Attachment, BodyPart, KnownRecipient, Message
 from app.providers.base import RemoteMessage
 from app.search.base import SearchDoc
+from app.subscriptions import read_list_headers
 from app.threading_.normalize import synthesize_message_id
 from app.threading_.threader import thread_message
 
@@ -118,6 +119,13 @@ async def _resolve_message_id(
     return parsed_id
 
 
+async def _keeps_raw(session: AsyncSession, account_id: uuid.UUID) -> bool:
+    """Whether the raw message is kept for good: not for a hosted mailbox, whose
+    mail server holds it (it is only cached, App. D §D.6 item 2)."""
+    kind = await session.scalar(select(Account.kind).where(Account.id == account_id))
+    return kind != "native"
+
+
 async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmail:
     """Parse under the PARSE_CONCURRENCY cap; stream large messages from disk (§7).
 
@@ -128,8 +136,9 @@ async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmai
     settings = get_settings()
     threshold = settings.stream_parse_mb * 1024 * 1024
     async with loop_semaphore("mime_parse", "parse", settings.parse_concurrency):
-        if len(raw) > threshold:
-            return parse_message_file(blobstore.blob_path(user_id, sha))
+        path = rawstore.locate(user_id, sha) if len(raw) > threshold else None
+        if path is not None:
+            return parse_message_file(path)
         return parse_message(raw)
 
 
@@ -154,8 +163,12 @@ async def persist_message(
     if existing is not None:
         return None
 
+    if await _keeps_raw(session, account_id):
         sha = blobstore.store(user_id, remote.raw)
+    else:
+        sha = rawstore.put_cached(user_id, remote.raw)
     parsed = await _parse_bounded(user_id, sha, remote.raw)
+    list_headers = read_list_headers(remote.raw)  # for the Subscriptions page
 
     message_id_hdr = await _resolve_message_id(
         session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw
@@ -196,6 +209,9 @@ async def persist_message(
         has_attachments=parsed.has_attachments,
         snippet=parsed.snippet,
         blob_sha256=sha,
+        list_id=list_headers.list_id,
+        list_unsubscribe=list_headers.unsubscribe,
+        list_unsubscribe_post=list_headers.one_click,
     )
     session.add(message)
     await session.flush()
@@ -213,11 +229,13 @@ async def persist_message(
                 kind="html_sanitized",
                 content=result.html,
                 sanitizer_v=result.sanitizer_v,
+                trackers_blocked=result.trackers_blocked,
             )
         )
 
+    # Attachments are not stored apart: they stay inside the raw blob, and are
+    # taken out of it when wanted (app/indexer/attachments.py).
     for att in parsed.attachments:
-        att_sha = blobstore.store(user_id, att.payload)
         session.add(
             Attachment(
                 message_id=message.id,
@@ -226,7 +244,8 @@ async def persist_message(
                 mime_type=att.mime_type,
                 size_bytes=len(att.payload),
                 content_id=att.content_id,
-                blob_sha256=att_sha,
+                blob_sha256=blobstore.sha256_hex(att.payload),
+                part_index=att.part_index,
             )
         )
 

backend/app/indexer/mime.py

@@ -41,6 +41,9 @@ class ParsedAttachment:
     mime_type: str
     content_id: str | None
     payload: bytes
+    # The part's position in ``msg.walk()``: where to find it again in the raw
+    # message, which is the only copy kept (attachments are not stored apart).
+    part_index: int = -1
 
 
 @dataclass
@@ -114,7 +117,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
     # or inline resource if it has a filename, a Content-ID, or attachment disposition;
     # the body text/plain and text/html parts have none of these and are skipped.
     out: list[ParsedAttachment] = []
-    for part in msg.walk():
+    for index, part in enumerate(msg.walk()):
         if part.is_multipart() or not isinstance(part, EmailMessage):
             continue
         cid = part.get("Content-ID")
@@ -137,13 +140,14 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
                 mime_type=part.get_content_type(),
                 content_id=cid.strip("<>") if (inline and cid is not None) else None,
                 payload=payload,
+                part_index=index,
             )
         )
     # An invitation sent only as the text/calendar alternative (Outlook, often) has
     # no file to show: keep it as invite.ics, so the event can be seen, answered
     # and downloaded like a Gmail one. Not when the message attaches its own .ics.
     if not any(a.filename.lower().endswith(".ics") for a in out):
-        for part in msg.walk():
+        for index, part in enumerate(msg.walk()):
             if part.get_content_type() != "text/calendar" or part.get_filename():
                 continue
             payload = part.get_payload(decode=True)
@@ -154,6 +158,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
                         mime_type="text/calendar",
                         content_id=None,
                         payload=payload,
+                        part_index=index,
                     )
                 )
                 break
@@ -215,3 +220,36 @@ def _parse_email(msg: EmailMessage) -> ParsedEmail:
         html_body=_scrub(html_body) if html_body is not None else None,
         attachments=_extract_attachments(msg),
     )
+
+
+def load_message(
+    raw: bytes | None = None, path: str | os.PathLike[str] | None = None
+) -> EmailMessage:
+    """The parsed tree of a raw message, from memory or straight from its file."""
+    if path is not None:
+        with open(path, "rb") as handle:
+            msg = email.message_from_binary_file(handle, policy=default_policy)
+    else:
+        msg = email.message_from_bytes(raw or b"", policy=default_policy)
+    assert isinstance(msg, EmailMessage)
+    return msg
+
+
+def part_payload(msg: EmailMessage, part_index: int | None, sha256: str) -> bytes | None:
+    """The decoded bytes of the attachment whose content hashes to ``sha256``:
+    the part at ``part_index`` when that is it, else whichever part is (rows
+    stored before the index was kept). None when the message has no such part."""
+    import hashlib
+
+    parts = list(msg.walk())
+    if part_index is not None and 0 <= part_index < len(parts):
+        payload = parts[part_index].get_payload(decode=True)
+        if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
+            return payload
+    for part in parts:
+        if part.is_multipart():
+            continue
+        payload = part.get_payload(decode=True)
+        if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
+            return payload
+    return None

backend/app/indexer/rawstore.py

@@ -0,0 +1,170 @@
+"""Where a message's raw source is (SPEC App. D §D.6 item 2).
+
+An external account's messages are kept whole in the blob store, as always.
+A hosted mailbox's are not: the mail server beside the app holds them, so the
+webmail keeps only what it reads them for (headers, the sanitized body, the
+snippet, the search index) and a **cache** of raw messages — the newest ones
+on arrival, the ones opened since — bounded per user (``RAW_CACHE_MB``,
+least recently used out first). Anything else is fetched from the mail server
+when needed: downloading an attachment, "show original", forwarding.
+
+Layout: ``${BLOB_ROOT}/{user_id}/rawcache/{sha256}``, the same content hash
+``messages.blob_sha256`` names.
+
+* :func:`locate` — the file, if it is here (blob store or cache); cheap, sync;
+* :func:`ensure` — the file, fetched from the mail server if need be (async):
+  call it before handing the work to a thread that reads the file.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import time
+import uuid
+from pathlib import Path
+
+from app.config import get_settings
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+CACHE_DIR = "rawcache"
+KEEP_S = 30 * 24 * 3600  # unused this long: dropped whatever the room
+PRUNE_EVERY = 50
+_writes: dict[uuid.UUID, int] = {}
+
+
+def cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / CACHE_DIR
+
+
+def cache_path(user_id: uuid.UUID, sha: str) -> Path:
+    return cache_dir(user_id) / sha
+
+
+def locate(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The message's raw file, if it is kept or cached here."""
+    kept = blobstore.blob_path(user_id, sha)
+    if kept.exists():
+        return kept
+    cached = cache_path(user_id, sha)
+    if cached.exists():
+        with contextlib.suppress(OSError):
+            cached.touch()  # used again: kept longer
+        return cached
+    return None
+
+
+def load(user_id: uuid.UUID, sha: str) -> bytes:
+    """The raw bytes if they are here. Raises FileNotFoundError."""
+    path = locate(user_id, sha)
+    if path is None:
+        raise FileNotFoundError(sha)
+    return path.read_bytes()
+
+
+def put_cached(user_id: uuid.UUID, data: bytes, sha: str | None = None) -> str:
+    """Cache a raw message (``sha`` names it; its content hash by default)."""
+    sha = sha or blobstore.sha256_hex(data)
+    if blobstore.exists(user_id, sha):
+        return sha  # kept for good already (another account has the same message)
+    path = cache_path(user_id, sha)
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+        tmp.write_bytes(data)
+        tmp.replace(path)
+    else:
+        path.touch()
+    # Pruning lists the whole cache: done every PRUNE_EVERY writes, not each
+    # (a backfill writes thousands); meanwhile it overshoots by that much at most.
+    writes = _writes.get(user_id, 0) + 1
+    _writes[user_id] = writes % PRUNE_EVERY
+    if writes >= PRUNE_EVERY:
+        with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+            prune(user_id)
+    return sha
+
+
+def prune(user_id: uuid.UUID, *, max_bytes: int | None = None) -> int:
+    """Drop cached messages unused for KEEP_S, and the least recently used
+    beyond the cap. Returns bytes freed."""
+    cache = cache_dir(user_id)
+    if not cache.is_dir():
+        return 0
+    cap = max_bytes if max_bytes is not None else get_settings().raw_cache_mb * 1024 * 1024
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file() and not p.name.endswith(".tmp")]
+    now = time.time()
+    total = freed = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > KEEP_S or total > cap:
+            path.unlink(missing_ok=True)
+            freed += st.st_size
+    return freed
+
+
+def drop(user_id: uuid.UUID, sha: str) -> None:
+    cache_path(user_id, sha).unlink(missing_ok=True)
+
+
+async def ensure(message: object) -> Path:
+    """The raw file of ``message`` (a :class:`~app.models.Message`), fetched
+    from the hosted mail server into the cache when it is not here. Raises
+    FileNotFoundError when it cannot be had (an external account's message
+    missing from the blob store; a native one gone from the server)."""
+    from app.models import Message
+
+    assert isinstance(message, Message)
+    found = locate(message.user_id, message.blob_sha256)
+    if found is not None:
+        return found
+    raw = await _fetch_native(message)
+    if raw is None:
+        raise FileNotFoundError(message.blob_sha256)
+    got = blobstore.sha256_hex(raw)
+    if got != message.blob_sha256:
+        # The server's copy differs from what was indexed (it should not):
+        # cached under the indexed name, so the message can still be read.
+        logger.warning("raw of message %s changed on the mail server", message.id)
+    put_cached(message.user_id, raw, message.blob_sha256)
+    return cache_path(message.user_id, message.blob_sha256)
+
+
+async def ensure_bytes(message: object) -> bytes:
+    path = await ensure(message)
+    import asyncio
+
+    return await asyncio.to_thread(path.read_bytes)
+
+
+async def _fetch_native(message: object) -> bytes | None:
+    from sqlalchemy import select
+
+    from app.db import session_scope
+    from app.models import Account, Mailbox, Message
+
+    assert isinstance(message, Message)
+    async with session_scope() as db:
+        row = (
+            await db.execute(
+                select(Account.kind, Account.email, Mailbox.path)
+                .join(Mailbox, Mailbox.account_id == Account.id)
+                .where(Account.id == message.account_id, Mailbox.id == message.mailbox_id)
+            )
+        ).first()
+    if row is None or row.kind != "native":
+        return None
+    from app.hosted.native import native_provider_config
+    from app.providers.base import ProviderError
+    from app.providers.imap_smtp import ImapSmtpProvider
+
+    provider = ImapSmtpProvider(native_provider_config(row.email))
+    try:
+        return await provider.fetch_raw(row.path, message.uid)
+    except ProviderError as exc:
+        logger.warning("cannot fetch message %s from the mail server: %s", message.id, exc)
+        return None
+    finally:
+        await provider.close()

backend/app/indexer/sanitize.py

@@ -17,7 +17,16 @@ import nh3
 
 # Bump when the sanitization policy changes; body_parts.sanitizer_v records it so a
 # render re-sanitizes stale content from the raw blob (SPEC §10.4).
-SANITIZER_VERSION = 4
+# v5: this server's own inline-image URLs quoted in a message load from here.
+# v6: links keep only http(s) and mailto (no data: pages).
+# v7: tracking pixels are removed (and counted), even when images are allowed.
+# v8: CSS sizes in other units (in, em, %, pt) no longer read as pixels — v7
+#     took 0.31in signature logos for trackers.
+SANITIZER_VERSION = 8
+
+# This server's inline-image URL, however it ended up written: "/api/…",
+# "https://host/api/…", or mangled to "http:///api/…" by an earlier copy.
+INTERNAL_CID = re.compile(r"/api/messages/([0-9a-fA-F-]{36})/cid/([^\"'?#\s<>]+)")
 
 _ALLOWED_TAGS: set[str] = {
     # structure
@@ -148,6 +157,94 @@ class SanitizeResult:
     html: str
     has_remote_images: bool
     sanitizer_v: int
+    trackers_blocked: int = 0
+
+
+# --- Tracking pixels ---------------------------------------------------------------
+# A remote image that is invisible (1×1, or hidden by CSS) has no use but to tell
+# the sender the mail was opened, and by whom. A few well-known open-tracking URL
+# shapes are caught whatever their size. Matching runs on the raw HTML before nh3:
+# a miss only leaves an image in, and nh3 still cleans whatever remains.
+_IMG_TAG = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
+_ATTR = re.compile(r"""([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("[^"]*"|'[^']*'|[^\s"'>]+)""")
+_CSS_HIDDEN = re.compile(
+    r"display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?![.\d]*[1-9])",
+    re.IGNORECASE,
+)
+_TRACKER_URL = re.compile(
+    r"//[^/]*mailtrack\.io/"
+    r"|/track/open\b"
+    r"|/wf/open\b"
+    r"|/open\.(?:php|aspx|gif)\b"
+    r"|[?&]tracking[_-]?pixel=",
+    re.IGNORECASE,
+)
+
+
+_PX = re.compile(r"\s*(\d+(?:\.\d*)?|\.\d+)\s*(px)?\s*$", re.IGNORECASE)
+
+
+def _px(value: str | None) -> float | None:
+    """A size in pixels ("1", "1px"), or None for anything else: other units
+    (0.31in is a 30-pixel signature logo), auto, %, or malformed (".", "1.2.3")."""
+    if value is None:
+        return None
+    match = _PX.match(value)
+    return float(match.group(1)) if match else None
+
+
+def _tiny(value: str | None) -> bool:
+    px = _px(value)
+    return px is not None and px <= 2
+
+
+def _css(style: str) -> dict[str, str]:
+    """An inline style's declarations, lower-cased, without !important."""
+    out: dict[str, str] = {}
+    for decl in style.split(";"):
+        name, _, value = decl.partition(":")
+        value = value.lower().replace("!important", "").strip()
+        if name.strip() and value:
+            out[name.strip().lower()] = value
+    return out
+
+
+def is_tracking_pixel(tag: str) -> bool:
+    """Whether an ``<img …>`` tag is a remote open-tracking pixel."""
+    attrs = {name.lower(): value.strip("\"'") for name, value in _ATTR.findall(tag)}
+    src = attrs.get("src", "").strip()
+    if not src.lower().startswith(("http:", "https:", "//")):
+        return False  # inline (cid:/data:) images can't report anything
+    if _TRACKER_URL.search(src):
+        return True
+    style = attrs.get("style", "")
+    if _CSS_HIDDEN.search(style):
+        return True
+    # The size the browser draws: a style width/height overrides the attribute
+    # (and "auto", % or another unit is no tiny size); a max-width/max-height
+    # of a pixel or two caps it; height "auto" follows the width.
+    css = _css(style)
+    wide = _tiny(css["width"]) if "width" in css else _tiny(attrs.get("width"))
+    tall = _tiny(css["height"]) if "height" in css else _tiny(attrs.get("height"))
+    wide = wide or _tiny(css.get("max-width"))
+    tall = tall or _tiny(css.get("max-height"))
+    if css.get("height") == "auto":
+        tall = wide
+    return wide and tall
+
+
+def strip_tracking_pixels(html: str) -> tuple[str, int]:
+    """Remove tracking pixels from raw HTML; returns the HTML and how many."""
+    count = 0
+
+    def repl(match: re.Match[str]) -> str:
+        nonlocal count
+        if is_tracking_pixel(match.group(0)):
+            count += 1
+            return ""
+        return match.group(0)
+
+    return _IMG_TAG.sub(repl, html), count
 
 
 def _sanitize_style(value: str, allow_remote: bool) -> str | None:
@@ -182,26 +279,47 @@ def _sanitize_style_elements(html: str, allow_remote: bool, state: dict[str, boo
 
 
 def sanitize_html(
-    html: str, message_id: uuid.UUID, *, allow_remote_images: bool = False
+    html: str,
+    message_id: uuid.UUID,
+    *,
+    allow_remote_images: bool = False,
+    block_trackers: bool = True,
 ) -> SanitizeResult:
     """Sanitize ``html`` for the message ``message_id``.
 
     Returns the safe HTML, whether any remote images were blocked, and the
     sanitizer version. Pass ``allow_remote_images=True`` after the user approves
     a sender to render remote images and CSS backgrounds (SPEC §9.4).
+    ``block_trackers=False`` (Settings → Privacy) leaves tracking pixels in; they
+    are remote images like any other then.
     """
     state = {"has_remote": False}
+    trackers = 0
+    if block_trackers:
+        html, trackers = strip_tracking_pixels(html)
 
     def attribute_filter(tag: str, attr: str, value: str) -> str | None:
         if attr == "style":
             return _sanitize_style(value, allow_remote_images)
         if tag == "a" and attr == "target":
             return "_blank"
+        if attr == "href":
+            # A link goes to the web or to a mail address: data: (a page made
+            # up by the sender) and cid: are kept for inline images only.
+            scheme = value.strip().lower().split(":", 1)[0] if ":" in value else ""
+            return value if scheme in ("", "http", "https", "mailto") else None
         if tag == "img" and attr == "src":
             lowered = value.strip().lower()
             if lowered.startswith("cid:"):
                 content_id = value.strip()[4:]
                 return f"/api/messages/{message_id}/cid/{content_id}"
+            # This server's own inline-image URL, quoted from another message
+            # (by an earlier version, or another of the user's own mails) —
+            # possibly mangled to "http:///api/…", which a browser takes for a
+            # host named "api": load it from here, never look that host up.
+            internal = INTERNAL_CID.search(value)
+            if internal is not None and not lowered.startswith("data:"):
+                return f"/api/messages/{internal.group(1)}/cid/{internal.group(2)}"
             if lowered.startswith(("http:", "https:")):
                 state["has_remote"] = True
                 return value if allow_remote_images else None  # block by default (FR-23)
@@ -229,4 +347,5 @@ def sanitize_html(
         html=cleaned,
         has_remote_images=state["has_remote"],
         sanitizer_v=SANITIZER_VERSION,
+        trackers_blocked=trackers,
     )

backend/app/indexer/thumbs.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 import io
 import logging
 import uuid
+from collections.abc import Callable
 from pathlib import Path
 
 from app.indexer import blobstore
@@ -86,8 +87,21 @@ def _render(data: bytes, mime_type: str) -> bytes | None:
     return out.getvalue()
 
 
-def thumbnail(user_id: uuid.UUID, sha: str, mime_type: str) -> bytes | None:
-    """The thumbnail of a stored attachment, or None if it cannot have one.
+def made(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether the attachment's thumbnail (or its absence) is settled already."""
+    return _path(user_id, sha, ".webp").exists() or _path(user_id, sha, ".none").exists()
+
+
+def thumbnail(
+    user_id: uuid.UUID,
+    sha: str,
+    mime_type: str,
+    size: int,
+    load: Callable[[], bytes],
+) -> bytes | None:
+    """The thumbnail of an attachment (content hash ``sha``, ``size`` bytes,
+    read with ``load`` only when it is not made yet), or None if it cannot
+    have one.
 
     Blocking (decoding is CPU work): call it in a thread.
     """
@@ -97,11 +111,14 @@ def thumbnail(user_id: uuid.UUID, sha: str, mime_type: str) -> bytes | None:
     none = _path(user_id, sha, ".none")
     if none.exists() or not can_thumbnail(mime_type):
         return None
-    path = blobstore.blob_path(user_id, sha)
-    if not path.exists() or path.stat().st_size > _MAX_INPUT_BYTES:
+    if size > _MAX_INPUT_BYTES:
         return None
     try:
-        thumb = _render(path.read_bytes(), mime_type)
+        data = load()
+    except FileNotFoundError:
+        return None
+    try:
+        thumb = _render(data, mime_type)
     except Exception as exc:  # noqa: BLE001 - any undecodable file: no thumbnail
         logger.info("no thumbnail for blob %s (%s): %s", sha, mime_type, exc)
         thumb = None

backend/app/main.py

@@ -37,6 +37,10 @@ def create_app() -> FastAPI:
 
     register_error_handlers(app)
 
+    from app.security.cross_site import CrossSiteGuard
+
+    app.add_middleware(CrossSiteGuard)
+
     @app.get("/api/healthz")
     async def healthz() -> dict[str, str]:
         """Liveness/readiness probe: checks DB and Redis (dev-setup §6)."""

backend/app/models.py

@@ -87,6 +87,13 @@ class AppSettings(Base):
     # take it (base64url, uncompressed P-256 point).
     vapid_public: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     vapid_private_ct: Mapped[bytes | None] = mapped_column(BYTEA)
+    # Users may invite friends (app/identity/invitations.py); the invitation
+    # email's From address (empty: postmaster@APP_DOMAIN), subject and HTML
+    # template (empty: the built-in ones).
+    invitations_enabled: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
+    invite_from: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    invite_subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    invite_template: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     updated_at: Mapped[dt.datetime] = _now()
 
 
@@ -131,6 +138,10 @@ class Account(Base):
     imap_config: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     smtp_config: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     secret_ct: Mapped[bytes] = mapped_column(BYTEA, nullable=False)
+    # external: any IMAP/SMTP server (the user's password in secret_ct);
+    # native: the user's own mailbox on the hosted mail server — permanent, no
+    # stored password (SPEC App. D §D.4; app/hosted/).
+    kind: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'external'"))
     discovery_src: Mapped[str] = mapped_column(Text, nullable=False)
     state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'backfilling'"))
     state_detail: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
@@ -161,9 +172,10 @@ class Mailbox(Base):
     )
     path: Mapped[str] = mapped_column(Text, nullable=False)
     special_use: Mapped[str | None] = mapped_column(Text)
-    uidvalidity: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-    highestmodseq: Mapped[int | None] = mapped_column(Integer)
-    last_seen_uid: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # IMAP ids are unsigned 32-bit (MODSEQ 63-bit): bigint, not integer.
+    uidvalidity: Mapped[int] = mapped_column(BigInteger, nullable=False, server_default=text("0"))
+    highestmodseq: Mapped[int | None] = mapped_column(BigInteger)
+    last_seen_uid: Mapped[int] = mapped_column(BigInteger, nullable=False, server_default=text("0"))
 
 
 class Thread(Base):
@@ -187,6 +199,12 @@ class Thread(Base):
     has_attachments: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     last_message_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    # Snooze: out of the Inbox until this time (NULL when not snoozed). A message
+    # that arrives after ``snoozed_at`` wakes it early; ``unsnoozed_at`` (when it
+    # last woke) puts it back at the top of the Inbox.
+    snoozed_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    snoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    unsnoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
 
 class Message(Base):
@@ -214,7 +232,7 @@ class Message(Base):
     restore_mailbox_id: Mapped[uuid.UUID | None] = mapped_column(
         UUID(as_uuid=True), ForeignKey("mailboxes.id", ondelete="SET NULL")
     )
-    uid: Mapped[int] = mapped_column(Integer, nullable=False)
+    uid: Mapped[int] = mapped_column(BigInteger, nullable=False)
     thread_id: Mapped[uuid.UUID | None] = mapped_column(
         UUID(as_uuid=True), ForeignKey("threads.id")
     )
@@ -251,6 +269,17 @@ class Message(Base):
     snippet: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
     body_indexed: Mapped[str | None] = mapped_column(TSVECTOR)
+    # Mailing-list headers, for the Subscriptions page. ``list_unsubscribe`` is
+    # NULL until read (a backfill job fills older mail), '' when there is none.
+    list_id: Mapped[str | None] = mapped_column(Text)
+    list_unsubscribe: Mapped[str | None] = mapped_column(Text)
+    # RFC 8058: the sender takes a one-click unsubscribe POST.
+    list_unsubscribe_post: Mapped[bool] = mapped_column(
+        nullable=False, server_default=text("false")
+    )
+    # The language of its text (app/translation.py), found the first time it is
+    # shown: NULL not looked at yet, '' could not tell.
+    content_language: Mapped[str | None] = mapped_column(Text)
 
 
 class BodyPart(Base):
@@ -264,6 +293,8 @@ class BodyPart(Base):
     content: Mapped[str] = mapped_column(Text, nullable=False)
     # SPEC §10.4: re-sanitize on render if the sanitizer version changed.
     sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # Tracking pixels taken out of this HTML (shown as "N trackers blocked").
+    trackers_blocked: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
 
 
 class Attachment(Base):
@@ -278,7 +309,11 @@ class Attachment(Base):
     mime_type: Mapped[str] = mapped_column(Text, nullable=False)
     size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
     content_id: Mapped[str | None] = mapped_column(Text)
+    # The content's hash (names its thumbnail, checks what is taken out). The
+    # bytes themselves live only inside the message's raw blob, at MIME part
+    # ``part_index`` (NULL for rows stored before that was recorded).
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
+    part_index: Mapped[int | None] = mapped_column(Integer)
 
 
 class ComposeAttachment(Base):
@@ -317,7 +352,7 @@ class Draft(Base):
     payload: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
     # UID of the mirrored copy in the account's IMAP Drafts folder (so the draft
     # is visible/continuable on other mail clients). Replaced on each edit.
-    imap_uid: Mapped[int | None] = mapped_column(Integer)
+    imap_uid: Mapped[int | None] = mapped_column(BigInteger)
     updated_at: Mapped[dt.datetime] = _now()
 
 
@@ -349,6 +384,117 @@ class SavedSearch(Base):
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("true"))
 
 
+class UserAppPassword(Base):
+    """A password for one mail app (IMAP/SMTP), revocable on its own (§D.5)."""
+
+    __tablename__ = "user_app_passwords"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)  # "Thunderbird on laptop"
+    password_hash: Mapped[str] = mapped_column(Text, nullable=False)  # argon2id
+    created_at: Mapped[dt.datetime] = _now()
+    # Set by the mail server's sign-in, once there is one.
+    last_used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    # The hosted mail server's id of the same password (§D.5), when it has one.
+    server_id: Mapped[str | None] = mapped_column(Text)
+
+
+class Invitation(Base):
+    """A single-use link that lets one person sign up, sent by a user."""
+
+    __tablename__ = "invitations"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    inviter_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    # sha256 of the token in the link; the token itself is only in the email.
+    token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    invitee_name: Mapped[str] = mapped_column(Text, nullable=False)
+    invitee_email: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+    expires_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    used_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL")
+    )
+
+
+class SubscriptionState(Base):
+    """What a user did about a bulk sender on the Subscriptions page."""
+
+    __tablename__ = "subscription_states"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    # The sender group: "list:<List-Id>" or "from:<address>".
+    key: Mapped[str] = mapped_column(Text, primary_key=True)
+    unsubscribed_at: Mapped[dt.datetime] = _now()
+    method: Mapped[str] = mapped_column(Text, nullable=False)  # one_click | mailto | web
+
+
+class MailTemplate(Base):
+    """A saved reply, inserted from the compose window."""
+
+    __tablename__ = "mail_templates"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)
+    subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    body_html: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    created_at: Mapped[dt.datetime] = _now()
+    updated_at: Mapped[dt.datetime] = _now()
+
+
+class MessageTranslation(Base):
+    """A message's HTML translated into a language (app/translation.py), kept so
+    it is translated once. ``images``: with remote images shown or not."""
+
+    __tablename__ = "message_translations"
+
+    message_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
+    )
+    target: Mapped[str] = mapped_column(Text, primary_key=True)
+    images: Mapped[bool] = mapped_column(primary_key=True)
+    source: Mapped[str] = mapped_column(Text, nullable=False)
+    html: Mapped[str] = mapped_column(Text, nullable=False)
+    sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+
+
+class CompletionPhrase(Base):
+    """Sentence completion (app/completion.py): after these words, the user
+    usually writes this — seen ``count`` times in their sent mail."""
+
+    __tablename__ = "completion_phrases"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    context: Mapped[str] = mapped_column(Text, primary_key=True)  # normalized words
+    continuation: Mapped[str] = mapped_column(Text, primary_key=True)  # as written
+    count: Mapped[int] = mapped_column(Integer, nullable=False)
+
+
+class CompletionIndexState(Base):
+    """When a user's completion index was last built."""
+
+    __tablename__ = "completion_index_state"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    built_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+
+
 class PushSubscription(Base):
     """A browser or installed app to tell about new mail (Web Push)."""
 

backend/app/outbox/draft_sync.py

@@ -17,6 +17,7 @@ from sqlalchemy import select
 
 from app.db import session_scope
 from app.models import Account, Draft, Mailbox, Message
+from app.outbox import inline_images
 from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
 from app.sync.provider_factory import build_provider
@@ -68,7 +69,15 @@ async def push_draft_to_imap_job(ctx: dict[str, Any], draft_id: str) -> None:
             else None
         )
         attachments = await load_attachments(session, user_id, draft)
-        _, sent_raw, _, _ = build_mime(account, draft, reply_to, attachments)
+        html, inline = await inline_images.embed(
+            session,
+            user_id,
+            str(draft.payload.get("html", "") or ""),
+            account.email.rsplit("@", 1)[-1],
+        )
+        _, sent_raw, _, _ = build_mime(
+            account, draft, reply_to, attachments, html=html, inline=inline
+        )
         drafts_mb = await session.scalar(
             select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )

backend/app/outbox/inline_images.py

@@ -0,0 +1,153 @@
+"""Inline images in an outgoing message's HTML.
+
+A reply quotes the original as the reading pane shows it, so an image embedded
+in the original (``cid:`` — a signature logo, a pasted screenshot) appears in
+the quote as this server's own URL, ``/api/messages/{id}/cid/{content-id}``.
+Sent as is, that URL means nothing to the recipient (and to anyone else,
+including ourselves once the original is gone). Before sending, each such image
+is put back into the message as an inline part with a new Content-ID — as any
+mail program does — and one that can no longer be had is left out.
+
+An image written into the HTML itself (``data:image/…;base64,…`` — a signature
+logo pasted into the editor) goes the same way: Gmail and other mail programs
+do not show ``data:`` images in mail they receive, so each becomes an inline
+part too (one per distinct image).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import binascii
+import hashlib
+import re
+import uuid
+from dataclasses import dataclass
+from email.utils import make_msgid
+from urllib.parse import unquote
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.indexer import attachments as att_store
+from app.indexer import rawstore
+from app.indexer.sanitize import INTERNAL_CID
+from app.models import Attachment, Message
+
+_DATA_IMG = re.compile(r"^data:(image/[a-z0-9.+-]+);base64,(.*)$", re.IGNORECASE | re.DOTALL)
+_EXT = {"jpeg": "jpg", "svg+xml": "svg"}
+
+_IMG = re.compile(r"<img\b[^>]*?\bsrc\s*=\s*([\"'])(.*?)\1[^>]*>", re.IGNORECASE | re.DOTALL)
+
+
+@dataclass(frozen=True)
+class InlinePart:
+    cid: str  # without angle brackets
+    mime_type: str
+    filename: str
+    data: bytes
+
+
+def data_image(src: str, domain: str) -> InlinePart | None:
+    """The inline part for a ``data:image/…;base64`` URL, or None if it is not one."""
+    m = _DATA_IMG.match(src.strip())
+    if m is None:
+        return None
+    try:
+        data = base64.b64decode(re.sub(r"\s+", "", m.group(2)), validate=True)
+    except (binascii.Error, ValueError):
+        return None
+    if not data:
+        return None
+    mime_type = m.group(1).lower()
+    subtype = mime_type.split("/", 1)[1]
+    name = f"image.{_EXT.get(subtype, subtype)}"
+    return InlinePart(make_msgid(domain=domain)[1:-1], mime_type, name, data)
+
+
+def internal_cid(src: str) -> tuple[uuid.UUID, str] | None:
+    """The (message id, content id) an image URL of this server points at."""
+    value = src.strip()
+    if value.lower().startswith(("cid:", "data:")):
+        return None
+    m = INTERNAL_CID.search(value)
+    if m is None:
+        return None
+    try:
+        return uuid.UUID(m.group(1)), unquote(m.group(2))
+    except ValueError:
+        return None
+
+
+async def embed(
+    session: AsyncSession, user_id: uuid.UUID, html: str, domain: str
+) -> tuple[str, list[InlinePart]]:
+    """``html`` with this server's image URLs turned into ``cid:`` references,
+    and the parts to send them in."""
+    parts: dict[tuple[uuid.UUID, str], InlinePart | None] = {}
+    # Images written into the HTML, by content: the same logo twice is one part.
+    data_parts: dict[str, InlinePart | None] = {}
+
+    def data_key(src: str) -> str:
+        return hashlib.sha256(src.strip().encode()).hexdigest()
+
+    for match in _IMG.finditer(html):
+        src = match.group(2)
+        if src.strip().lower().startswith("data:"):
+            key = data_key(src)
+            if key not in data_parts:
+                data_parts[key] = data_image(src, domain)
+            continue
+        target = internal_cid(src)
+        if target is None or target in parts:
+            continue
+        message_id, content_id = target
+        found = (
+            await session.execute(
+                select(Attachment, Message)
+                .join(Message, Message.id == Attachment.message_id)
+                .where(
+                    Attachment.message_id == message_id,
+                    Attachment.content_id == content_id,
+                    Message.user_id == user_id,
+                )
+                .limit(1)
+            )
+        ).first()
+        part: InlinePart | None = None
+        if found is not None:
+            att, message = found
+            try:
+                await rawstore.ensure(message)  # a hosted mailbox's: fetched if need be
+                data = await asyncio.to_thread(
+                    att_store.load, user_id, message.blob_sha256, att.blob_sha256, att.part_index
+                )
+                cid = make_msgid(domain=domain)[1:-1]
+                part = InlinePart(cid, att.mime_type, att.filename, data)
+            except FileNotFoundError:
+                part = None
+        parts[target] = part
+
+    def replace(match: re.Match[str]) -> str:
+        quote = match.group(1)
+        src = match.group(2)
+        if src.strip().lower().startswith("data:"):
+            data_part = data_parts.get(data_key(src))
+            if data_part is None:
+                return match.group(0)  # not an image we can carry: left as it was
+            return match.group(0).replace(
+                f"src={quote}{src}{quote}", f"src={quote}cid:{data_part.cid}{quote}", 1
+            )
+        target = internal_cid(match.group(2))
+        if target is None:
+            return match.group(0)
+        part = parts.get(target)
+        if part is None:
+            return ""  # gone: no broken image for the recipient
+        return match.group(0).replace(
+            f"src={quote}{match.group(2)}{quote}", f"src={quote}cid:{part.cid}{quote}", 1
+        )
+
+    out = _IMG.sub(replace, html) if parts or data_parts else html
+    quoted = [p for p in parts.values() if p is not None]
+    return out, quoted + [p for p in data_parts.values() if p is not None]

backend/app/outbox/jobs.py

@@ -14,15 +14,17 @@ from datetime import UTC, datetime, timedelta
 from email.utils import getaddresses
 from typing import Any
 
+from arq import Retry
 from sqlalchemy import select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import session_scope
 from app.events import get_event_bus
-from app.indexer import blobstore
+from app.indexer import blobstore, gc
 from app.indexer.indexer import persist_message
 from app.models import Account, Draft, KnownRecipient, Mailbox, Message, Outbox
+from app.outbox import inline_images
 from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
 from app.outbox.service import delete_mirrored_draft_row
@@ -171,7 +173,21 @@ async def _send_failed(
 
 async def file_sent_copy_job(ctx: dict[str, Any], account_id: str, user_id: str, sha: str) -> None:
     """File a sent message's copy in Sent, after the send itself succeeded but
-    the IMAP server could not be reached. Retried (arq Retry) for ~24 hours."""
+    the IMAP server could not be reached. Retried (arq Retry) for ~24 hours.
+
+    Until then the copy's file is referenced only by this job: it is held from
+    the nightly clean-up (app/indexer/gc.py) and let go once filed or given up."""
+    try:
+        await _file_sent_copy(ctx, account_id, user_id, sha)
+    except Retry:
+        raise  # tried again later: the file is still needed
+    except BaseException:
+        await gc.release(user_id, sha)
+        raise
+    await gc.release(user_id, sha)
+
+
+async def _file_sent_copy(ctx: dict[str, Any], account_id: str, user_id: str, sha: str) -> None:
     uid = uuid.UUID(user_id)
     async with session_scope() as session:
         account = await session.get(Account, uuid.UUID(account_id))
@@ -236,8 +252,16 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
             return
         user_id = outbox.user_id
         attachments = await load_attachments(session, user_id, draft)
+        # Images quoted from the original travel inside the message, not as
+        # links to this server (app/outbox/inline_images.py).
+        html, inline = await inline_images.embed(
+            session,
+            user_id,
+            str(draft.payload.get("html", "") or ""),
+            account.email.rsplit("@", 1)[-1],
+        )
         transmit_raw, sent_raw, envelope_from, envelope_to = build_mime(
-            account, draft, reply_to, attachments
+            account, draft, reply_to, attachments, html=html, inline=inline
         )
         sent_path = await session.scalar(
             select(Mailbox.path).where(
@@ -271,6 +295,7 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
             except Exception as exc:  # noqa: BLE001
                 logger.warning("outbox %s sent; filing it in Sent failed, will retry: %s", oid, exc)
                 sha = blobstore.store(user_id, sent_raw)
+                await gc.hold(user_id, sha)  # until filed: only the job refers to it
                 await enqueue_job("file_sent_copy_job", str(account_obj.id), str(user_id), sha)
         # Remove the mirrored Drafts-folder copy now that it has been sent.
         if drafts_path and draft_imap_uid:
@@ -313,6 +338,12 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
             # need to read the outbox row. payload to/cc are "Name <email>" strings.
             "to": _addr_dicts(draft.payload.get("to")),
             "cc": _addr_dicts(draft.payload.get("cc")),
+            # What it answers: "reply" / "reply_all" / "forward" / "new", and the
+            # message (a reply says the user wanted that mail — antispam learns it).
+            "kind": draft.kind,
+            "reply_to_message_id": (
+                str(draft.reply_to_message_id) if draft.reply_to_message_id else None
+            ),
         },
     )
     await bus.publish(

backend/app/outbox/mime_build.py

@@ -14,6 +14,7 @@ from email.utils import format_datetime, make_msgid
 import nh3
 
 from app.models import Account, Draft, Message
+from app.outbox.inline_images import InlinePart
 
 
 def _html_to_text(html: str) -> str:
@@ -49,6 +50,9 @@ def build_mime(
     draft: Draft,
     reply_to: Message | None,
     attachments: list[tuple[str, str, bytes]] | None = None,
+    *,
+    html: str | None = None,
+    inline: list[InlinePart] | None = None,
 ) -> tuple[bytes, bytes, str, list[str]]:
     """Return ``(transmit_raw, sent_raw, envelope_from, envelope_to)`` for ``draft``.
 
@@ -59,13 +63,17 @@ def build_mime(
 
     ``attachments`` is a list of ``(filename, mime_type, data)`` uploaded via the
     compose Attach action; each is added as a file attachment part.
+
+    ``html`` replaces the draft's HTML when given (images already turned into
+    ``cid:`` references by app/outbox/inline_images.py), and ``inline`` are the
+    image parts those references name, sent alongside the HTML.
     """
     payload = draft.payload
     to = _addr_list(payload.get("to"))
     cc = _addr_list(payload.get("cc"))
     bcc = _addr_list(payload.get("bcc"))
     subject = str(payload.get("subject", ""))
-    html = str(payload.get("html", "") or "")
+    html = html if html is not None else str(payload.get("html", "") or "")
     text = str(payload.get("text", "") or "")
 
     msg = EmailMessage()
@@ -91,6 +99,19 @@ def build_mime(
     if html:
         msg.set_content(text or _html_to_text(html))
         msg.add_alternative(html, subtype="html")
+        if inline:
+            html_part = next(p for p in msg.walk() if p.get_content_type() == "text/html")
+            assert isinstance(html_part, EmailMessage)
+            for part in inline:
+                maintype, _, subtype = part.mime_type.partition("/")
+                html_part.add_related(
+                    part.data,
+                    maintype=maintype or "image",
+                    subtype=subtype or "png",
+                    cid=f"<{part.cid}>",
+                    filename=part.filename,
+                    disposition="inline",
+                )
     else:
         msg.set_content(text)
 

backend/app/plugins/actions.py

@@ -23,7 +23,7 @@ from sqlalchemy import select
 
 from app.db import session_scope
 from app.events import get_event_bus
-from app.indexer import blobstore
+from app.indexer import rawstore
 from app.models import Account, Mailbox, Message, Thread, User
 
 
@@ -39,9 +39,8 @@ class PluginMessages:
             )
             if msg is None or not msg.blob_sha256:
                 return None
-            sha = msg.blob_sha256
         try:
-            raw = blobstore.load(user_id, sha)
+            raw = await rawstore.ensure_bytes(msg)
         except FileNotFoundError:
             return None
         return message_from_bytes(raw).get(name)
@@ -60,9 +59,8 @@ class PluginMessages:
             )
             if msg is None or not msg.blob_sha256:
                 return None
-            sha = msg.blob_sha256
         try:
-            return blobstore.load(user_id, sha)
+            return await rawstore.ensure_bytes(msg)
         except FileNotFoundError:
             return None
 

backend/app/plugins/http.py

@@ -21,9 +21,7 @@ class PluginHttp:
         content: bytes | str | None = None,
         auth: tuple[str, str] | None = None,
     ) -> GuardedResponse:
-        return await guarded_request(
-            method, url, headers=headers, content=content, auth=auth
-        )
+        return await guarded_request(method, url, headers=headers, content=content, auth=auth)
 
     async def get(self, url: str, **kw: object) -> GuardedResponse:
         return await self.request("GET", url, **kw)  # type: ignore[arg-type]

backend/app/plugins/sdk.py

@@ -179,6 +179,12 @@ class PluginSdk:
         CalDAV/CardDAV/ICS. Returns status + headers + body."""
         return PluginHttp()
 
+    # --- notifications ------------------------------------------------------
+    @property
+    def push(self) -> PluginPush:
+        """Notifications on the user's devices (Web Push) — calendar reminders."""
+        return PluginPush(self.name)
+
     # --- secret vault -------------------------------------------------------
     @property
     def vault(self) -> PluginVault:
@@ -187,6 +193,39 @@ class PluginSdk:
         return PluginVault()
 
 
+class PluginPush:
+    """``sdk.push``: show a notification on every device where the user turned
+    notifications on (Settings → Accounts → Notifications)."""
+
+    def __init__(self, plugin: str) -> None:
+        self._plugin = plugin
+
+    async def notify(
+        self,
+        user_id: uuid.UUID,
+        *,
+        title: str,
+        body: str = "",
+        url: str = "/mail",
+        tag: str | None = None,
+        sticky: bool = False,
+    ) -> int:
+        """Send one notification; returns how many devices it went to. ``tag``
+        replaces an earlier notification with the same tag (it is prefixed with
+        the plugin's name); ``sticky`` keeps it on screen until dismissed."""
+        from app.push.service import deliver
+
+        payload: dict[str, object] = {
+            "type": f"plugin.{self._plugin}",
+            "title": title[:200],
+            "body": body[:500],
+            "url": url if url.startswith("/") else "/mail",
+            "tag": f"{self._plugin}-{tag}" if tag else self._plugin,
+            "sticky": sticky,
+        }
+        return await deliver(user_id, [payload])
+
+
 def sdk(name: str) -> PluginSdk:
     """Return the SDK facade bound to plugin ``name`` (SPEC §11)."""
     return PluginSdk(name)

backend/app/plugins/types.py

@@ -112,6 +112,9 @@ class Plugin:
     # Core plugins (calendar, contacts) ship builtin=True: always installed, never
     # offered for removal in the UI, and the uninstall path refuses --drop-data.
     builtin: bool = False
+    # Serves its collections to CalDAV / CardDAV clients (app/dav/provider.py
+    # DavProvider; SPEC Appendix E).
+    dav: Any = None
 
     def __post_init__(self) -> None:
         if not _NAME_RE.fullmatch(self.name):

backend/app/providers/imap_smtp/provider.py

@@ -20,7 +20,7 @@ import aiosmtplib
 from aioimaplib import aioimaplib
 
 from app.concurrency import loop_semaphore
-from app.config import get_settings
+from app.config import get_settings, smtp_helo_name
 from app.providers.base import (
     FlagChange,
     MailboxDelta,
@@ -122,6 +122,16 @@ def _date_criteria(since: date | None, before: date | None) -> str:
     return " ".join(parts) if parts else "ALL"
 
 
+async def _require_public(host: str) -> None:
+    """Refuse a user-named server that now resolves to a non-public address."""
+    from app.accounts.ssrf import SsrfError, assert_host_is_public
+
+    try:
+        await assert_host_is_public(host)
+    except SsrfError as exc:
+        raise ProviderConnectError(f"refusing to connect to {host}: {exc}") from exc
+
+
 @dataclass(frozen=True)
 class ProviderConfig:
     imap_host: str
@@ -132,6 +142,16 @@ class ProviderConfig:
     smtp_tls: str  # "ssl" | "starttls" | "plain"
     username: str
     password: str  # plaintext; caller decrypts per §10.2
+    # TLS settings other than the default (certificate checked): the hosted
+    # mail server on the internal network (app/hosted/native.py).
+    tls_context: ssl.SSLContext | None = None
+    # The hosted mail server: every user's mailbox is on that one host, so the
+    # per-host connection cap (which spares outside providers) does not apply.
+    internal: bool = False
+    # A server the user named (every external account): its name is checked
+    # again before each connection, so DNS changed after the account was added
+    # cannot turn the worker on the internal network (DNS rebinding).
+    public_only: bool = False
 
 
 class ImapSmtpProvider(MailProvider):
@@ -169,8 +189,11 @@ class ImapSmtpProvider(MailProvider):
         account = loop_semaphore(
             "imap_account", f"{cfg.username}@{cfg.imap_host}", _PER_ACCOUNT_CONN_CAP
         )
-        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
         await account.acquire()  # consistent order (account, then host) avoids deadlock
+        if cfg.internal:
+            self._conn_slots = [account]
+            return
+        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
         try:
             await host.acquire()
         except BaseException:
@@ -187,13 +210,18 @@ class ImapSmtpProvider(MailProvider):
         if self._imap is not None:
             return self._imap
         cfg = self._cfg
+        if cfg.public_only:
+            await _require_public(cfg.imap_host)
         await self._acquire_conn_slots()
         try:
             try:
                 imap: aioimaplib.IMAP4
                 if cfg.imap_tls == "ssl":
                     imap = _IMAP4_SSL(
-                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
+                        host=cfg.imap_host,
+                        port=cfg.imap_port,
+                        timeout=_CONNECT_TIMEOUT,
+                        ssl_context=cfg.tls_context,
                     )
                 else:
                     imap = _IMAP4(host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT)
@@ -356,6 +384,15 @@ class ImapSmtpProvider(MailProvider):
         resp = await imap.uid_search("ALL")
         return set(parse_search_uids(resp.lines))
 
+    async def fetch_raw(self, mailbox: str, uid: int) -> bytes | None:
+        """One message's raw bytes, or None if it is not (or no longer) there."""
+        await self._select(mailbox)
+        imap = await self._connect()
+        for message in await self._fetch_messages(imap, [uid]):
+            if message.uid == uid:
+                return message.raw
+        return None
+
     async def _fetch_messages(self, imap: aioimaplib.IMAP4, uids: list[int]) -> list[RemoteMessage]:
         if not uids:
             return []
@@ -474,12 +511,12 @@ class ImapSmtpProvider(MailProvider):
         """Create a mailbox on the server (idempotent — ignores 'already exists')."""
         imap = await self._connect()
         with contextlib.suppress(Exception):
-            await imap.create(path)
+            await imap.create(self._wire(path))
 
     async def move(self, uid: int, src: str, dst: str) -> int:
         await self._select(src)
         imap = await self._connect()
-        resp = await imap.uid("move", str(uid), dst)
+        resp = await imap.uid("move", str(uid), self._wire(dst))
         if resp.result != "OK":
             raise ProviderConnectError(f"MOVE {uid} -> {dst} failed")
         for line in resp.lines:
@@ -519,9 +556,15 @@ class ImapSmtpProvider(MailProvider):
 
     async def send(self, raw: bytes, envelope_from: str, envelope_to: list[str]) -> None:
         cfg = self._cfg
+        if cfg.public_only:
+            await _require_public(cfg.smtp_host)
         use_tls = cfg.smtp_tls == "ssl"
         start_tls = cfg.smtp_tls == "starttls"
-        host_slot = loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
+        host_slot = (
+            contextlib.nullcontext()
+            if cfg.internal
+            else loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
+        )
         try:
             async with host_slot:  # bound simultaneous sockets to the SMTP host (§4)
                 await aiosmtplib.send(
@@ -534,7 +577,10 @@ class ImapSmtpProvider(MailProvider):
                     password=cfg.password,
                     use_tls=use_tls,
                     start_tls=start_tls,
+                    tls_context=cfg.tls_context,
                     timeout=_CONNECT_TIMEOUT,
+                    # Our own name, not the container's random host name.
+                    local_hostname=smtp_helo_name(),
                 )
         except aiosmtplib.SMTPException as exc:
             raise _smtp_error(exc) from exc

backend/app/push/service.py

@@ -200,6 +200,26 @@ async def notify(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> int:
     pushes were delivered to push services."""
     from app.db import session_scope
 
+    async with session_scope() as db:
+        if not await db.scalar(
+            select(PushSubscription.id).where(PushSubscription.user_id == user_id).limit(1)
+        ):
+            return 0
+        messages = await _news(db, user_id, message_ids)
+        if not messages:
+            return 0
+        payloads = _payloads(messages, await _unread(db, user_id))
+    return await deliver(user_id, payloads)
+
+
+async def deliver(user_id: uuid.UUID, payloads: list[dict[str, object]]) -> int:
+    """Push ``payloads`` to every device of the user that is still signed in
+    (for new mail, and for plugins: calendar reminders); returns how many were
+    handed to push services. Devices that are gone are forgotten."""
+    from app.db import session_scope
+
+    if not payloads:
+        return 0
     async with session_scope() as db:
         # Plain values: the rows expire when the key pair is first saved below.
         subs = [
@@ -210,10 +230,6 @@ async def notify(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> int:
         ]
         if not subs:
             return 0
-        messages = await _news(db, user_id, message_ids)
-        if not messages:
-            return 0
-        payloads = _payloads(messages, await _unread(db, user_id))
         key = await vapid(db)
         subject = _subject(await db.get(AppSettings, 1))
 

backend/app/queue.py

@@ -21,7 +21,14 @@ QUEUE_INTERACTIVE = "interactive"
 QUEUE_BULK = "bulk"
 
 # Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
-_BULK_JOBS = frozenset({"sync_bootstrap_account"})
+_BULK_JOBS = frozenset(
+    {
+        "sync_bootstrap_account",
+        "dedupe_existing_job",
+        "backfill_list_headers_job",
+        "build_index_job",
+    }
+)
 
 
 def queue_for(job_name: str) -> str:

backend/app/search/pg.py

@@ -13,6 +13,7 @@ import base64
 import datetime as dt
 import html
 import re
+from typing import Any
 from uuid import UUID
 
 from sqlalchemy import (
@@ -20,6 +21,7 @@ from sqlalchemy import (
     any_,
     case,
     cast,
+    distinct,
     exists,
     func,
     literal,
@@ -32,7 +34,7 @@ from sqlalchemy import (
 from sqlalchemy.sql.elements import ColumnElement
 
 from app.db import get_sessionmaker
-from app.models import Attachment, Mailbox, Message
+from app.models import Attachment, Mailbox, Message, Thread
 from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
 
 _TS_CONFIG = "simple"  # language-neutral; avoids wrong-language stemming (M0)
@@ -52,6 +54,8 @@ def _prefix_tsquery(value: str) -> ColumnElement[object]:
     lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text)
     prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g")
     return func.to_tsquery(_TS_CONFIG, prefixed)
+
+
 _RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
 
 
@@ -254,20 +258,9 @@ class PgSearchBackend(SearchBackend):
             .join(Mailbox, Mailbox.id == Message.mailbox_id)
             .where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
         )
-        for cond in conditions:
+        for cond in self._scope(groups, conditions):
             stmt = stmt.where(cond)
 
-        # Exclude Trash/Junk from search by default (Gmail-style), so deleting a
-        # message from search results removes it. An explicit `in:trash`/`in:junk`
-        # overrides this to search there on purpose.
-        if not _targets_removal_folder(groups):
-            stmt = stmt.where(
-                or_(
-                    Mailbox.special_use.is_(None),
-                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
-                )
-            )
-
         decoded = _decode_cursor(cursor)
         if decoded is not None:
             cur_hit, cur_date, cur_id = decoded
@@ -301,6 +294,41 @@ class PgSearchBackend(SearchBackend):
         )
         return hits, next_cursor
 
+    @staticmethod
+    def _scope(groups: list[list[dict[str, object]]], conditions: list[Any]) -> list[Any]:
+        """The query's conditions, plus leaving out Trash/Junk by default
+        (Gmail-style), so deleting a message from search results removes it. An
+        explicit `in:trash`/`in:junk` overrides this to search there on purpose."""
+        out = list(conditions)
+        if not _targets_removal_folder(groups):
+            out.append(
+                or_(
+                    Mailbox.special_use.is_(None),
+                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
+                )
+            )
+        return out
+
+    async def count_threads(self, q: SearchQuery, *, thread_filter: str | None = None) -> int:
+        """How many conversations match (``thread_filter``: "unread" / "pinned"
+        keeps only conversations with unread mail / pinned ones)."""
+        groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
+        assert isinstance(groups, list)
+        stmt = (
+            select(func.count(distinct(Message.thread_id)))
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
+        )
+        for cond in self._scope(groups, self._build_conditions(groups)):
+            stmt = stmt.where(cond)
+        if thread_filter in ("unread", "pinned"):
+            stmt = stmt.join(Thread, Thread.id == Message.thread_id).where(
+                Thread.unread_count > 0 if thread_filter == "unread" else Thread.pinned.is_(True)
+            )
+        async with get_sessionmaker()() as session:
+            return int(await session.scalar(stmt) or 0)
+
 
 _REMOVAL_FOLDERS = ("trash", "junk")
 

backend/app/security/cross_site.py

@@ -0,0 +1,91 @@
+"""Refuse state-changing API requests that another web site makes a browser send.
+
+The double-submit CSRF token (app/identity/deps.py ``require_csrf``) guards
+everything a signed-in user can change. The requests made before anyone is
+signed in — sign-in itself, sign-up, a password reset — have no token yet, so a
+hostile page could make a visitor's browser sign in to an account of the
+attacker's choosing (login CSRF): whatever the victim then writes, files or
+uploads lands in the attacker's account.
+
+Browsers say where a request comes from: ``Sec-Fetch-Site`` in every current
+browser, ``Origin`` on any cross-origin POST in older ones. A request another
+site started is refused here, before any route runs. Mail apps, calendar apps
+and servers send neither header and are not affected (DAV lives outside /api
+anyway).
+"""
+
+from __future__ import annotations
+
+import json
+from urllib.parse import urlsplit
+
+from starlette.datastructures import Headers
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+from app.config import get_settings
+
+_SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+_BODY = json.dumps(
+    {"error": {"code": "cross_site", "message": "request refused: sent from another site"}}
+).encode()
+
+
+def _hostname(value: str) -> str:
+    """``host[:port]`` -> ``host``, lower-cased (IPv6 brackets kept)."""
+    value = value.strip().lower()
+    if value.startswith("["):
+        return value.split("]", 1)[0] + "]"
+    return value.rsplit(":", 1)[0] if value.count(":") == 1 else value
+
+
+def is_cross_site(headers: Headers) -> bool:
+    """Whether a browser reports this request as started by another site."""
+    fetch_site = headers.get("sec-fetch-site")
+    if fetch_site is not None:
+        # "same-origin", "same-site" (our own subdomains), "none" (typed or
+        # bookmarked): all fine. Only another site is refused.
+        return fetch_site.strip().lower() == "cross-site"
+    origin = headers.get("origin")
+    if origin is None:
+        return False  # not a browser, or a same-origin request from an old one
+    if origin.strip().lower() == "null":
+        return True  # sandboxed frame, data: URL, file: page
+    origin_host = (urlsplit(origin.strip()).hostname or "").lower()
+    # Our own domain, whatever a reverse proxy did to the Host header.
+    domain = get_settings().app_domain.lower()
+    if origin_host == domain or origin_host.endswith("." + domain):
+        return False
+    # Otherwise the name the browser used: Host, or the one a reverse proxy saw.
+    ours = {_hostname(headers.get("host", ""))}
+    forwarded = headers.get("x-forwarded-host")
+    if forwarded:
+        ours.add(_hostname(forwarded.split(",")[0]))
+    return origin_host not in ours
+
+
+class CrossSiteGuard:
+    """ASGI middleware: 403 on cross-site POST/PUT/PATCH/DELETE under /api/."""
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if (
+            scope["type"] == "http"
+            and scope["method"] not in _SAFE_METHODS
+            and scope["path"].startswith("/api/")
+            and is_cross_site(Headers(scope=scope))
+        ):
+            await send(
+                {
+                    "type": "http.response.start",
+                    "status": 403,
+                    "headers": [
+                        (b"content-type", b"application/json"),
+                        (b"content-length", str(len(_BODY)).encode()),
+                    ],
+                }
+            )
+            await send({"type": "http.response.body", "body": _BODY})
+            return
+        await self.app(scope, receive, send)

backend/app/snooze.py

@@ -0,0 +1,92 @@
+"""Snooze: a conversation leaves the Inbox until a chosen time, then comes back.
+
+Kept in this database only (the mail server knows nothing of it): a snoozed
+thread is hidden from the Inbox view and listed under Snoozed. The wake job,
+run every minute, brings back threads whose time has come — or that got a new
+message meanwhile, as Gmail does — at the top of the Inbox, the newest message
+marked unread.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import or_, select
+
+from app.db import session_scope
+from app.events import get_event_bus
+from app.models import Mailbox, Message, Thread
+from app.queue import enqueue_job
+from app.sync.actions import apply_flags_to_messages
+
+logger = logging.getLogger(__name__)
+
+MAX_SNOOZE = dt.timedelta(days=366)
+_SEEN = frozenset({"\\Seen"})
+
+
+def snooze(thread: Thread, until: dt.datetime, now: dt.datetime) -> None:
+    thread.snoozed_until = until
+    thread.snoozed_at = now
+
+
+def wake(thread: Thread, now: dt.datetime) -> None:
+    thread.snoozed_until = None
+    thread.snoozed_at = None
+    thread.unsnoozed_at = now
+
+
+async def wake_due_job(ctx: dict[str, Any]) -> int:
+    """Cron (every minute): wake snoozed threads that are due. Returns how many."""
+    now = dt.datetime.now(dt.UTC)
+    woken: list[tuple[uuid.UUID, uuid.UUID]] = []
+    unread: list[tuple[uuid.UUID, Message]] = []
+    async with session_scope() as session:
+        due = list(
+            await session.scalars(
+                select(Thread)
+                .where(
+                    Thread.snoozed_until.isnot(None),
+                    or_(Thread.snoozed_until <= now, Thread.last_message_at > Thread.snoozed_at),
+                )
+                .with_for_update(skip_locked=True)
+            )
+        )
+        for thread in due:
+            on_time = thread.snoozed_until is not None and thread.snoozed_until <= now
+            wake(thread, now)
+            woken.append((thread.user_id, thread.id))
+            if not on_time:
+                continue  # a new message woke it: that message is unread already
+            # Back as new: the newest Inbox message becomes unread again.
+            newest = await session.scalar(
+                select(Message)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.thread_id == thread.id,
+                    Message.user_id == thread.user_id,
+                    Mailbox.special_use == "inbox",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(1)
+            )
+            if newest is not None and "\\Seen" in newest.flags:
+                await apply_flags_to_messages(session, [newest], frozenset(), _SEEN)
+                unread.append((thread.user_id, newest))
+        await session.commit()
+
+    bus = get_event_bus()
+    for user_id, message in unread:
+        await enqueue_job("push_flags_job", str(message.id), [], ["\\Seen"])
+        await bus.publish(
+            "flags.changed",
+            {"user_id": str(user_id), "message_id": str(message.id), "flags": list(message.flags)},
+        )
+    for user_id, thread_id in woken:
+        await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
+    if woken:
+        logger.info("woke %d snoozed thread(s)", len(woken))
+    return len(woken)

backend/app/subscriptions.py

@@ -0,0 +1,151 @@
+"""Subscriptions: the bulk senders mailing a user, and unsubscribing from them.
+
+No service records what a user signed up for, so the list is worked out from
+received mail, as Gmail does: a message is a subscription's when it carries a
+``List-Unsubscribe`` header (required of bulk senders by Gmail and Yahoo since
+2024). Messages are grouped by ``List-Id`` when there is one (a company can run
+several lists), otherwise by sender address.
+
+The headers are read when a message is stored (indexer.persist_message); older
+mail is read once by :func:`backfill_list_headers_job`. Unsubscribing uses the
+sender's one-click POST (RFC 8058) when offered, else mails the ``mailto:``
+address, else hands the web page back for the user to open.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from dataclasses import dataclass
+from email.parser import BytesHeaderParser
+from email.policy import default as default_policy
+from typing import Any
+from urllib.parse import parse_qs, unquote, urlsplit
+
+from sqlalchemy import select, update
+
+from app.db import session_scope
+from app.indexer import rawstore
+from app.models import Message
+
+logger = logging.getLogger(__name__)
+
+_MAX_HEADER = 2000
+_ANGLE = re.compile(r"<([^<>]+)>")
+_BATCH = 200
+
+
+@dataclass(frozen=True)
+class ListHeaders:
+    list_id: str | None
+    unsubscribe: str  # '' when the message has none
+    one_click: bool
+
+
+def read_list_headers(raw: bytes) -> ListHeaders:
+    """The mailing-list headers of a raw message (headers only are parsed)."""
+    end = raw.find(b"\r\n\r\n")
+    if end < 0:
+        end = raw.find(b"\n\n")
+    head = raw if end < 0 else raw[: end + 2]
+    try:
+        msg = BytesHeaderParser(policy=default_policy).parsebytes(head)
+        list_id_raw = str(msg.get("List-Id") or "")
+        unsubscribe = str(msg.get("List-Unsubscribe") or "")
+        post = str(msg.get("List-Unsubscribe-Post") or "")
+    except Exception:  # a malformed header must not stop the message being stored
+        return ListHeaders(None, "", False)
+    list_id = None
+    if list_id_raw:
+        match = _ANGLE.search(list_id_raw)
+        list_id = (match.group(1) if match else list_id_raw).strip().lower()[:300] or None
+    unsubscribe = " ".join(unsubscribe.split())[:_MAX_HEADER]
+    one_click = "list-unsubscribe=one-click" in post.replace(" ", "").lower()
+    return ListHeaders(list_id, unsubscribe, one_click and bool(unsubscribe))
+
+
+@dataclass(frozen=True)
+class UnsubscribeTargets:
+    https: str | None
+    mailto: str | None  # the address
+    mailto_subject: str
+    mailto_body: str
+
+
+def unsubscribe_targets(header: str) -> UnsubscribeTargets:
+    """The https and mailto targets listed in a ``List-Unsubscribe`` header."""
+    https = mailto = None
+    subject, body = "unsubscribe", "unsubscribe"
+    for target in _ANGLE.findall(header):
+        target = target.strip()
+        lowered = target.lower()
+        if https is None and lowered.startswith("https://"):
+            https = target
+        elif mailto is None and lowered.startswith("mailto:"):
+            parts = urlsplit(target)
+            address = unquote(parts.path).strip()
+            if "@" in address and "\n" not in address and "\r" not in address:
+                mailto = address
+                query = parse_qs(parts.query)
+                subject = (query.get("subject") or [subject])[0][:200]
+                body = (query.get("body") or [body])[0][:2000]
+    return UnsubscribeTargets(https, mailto, subject, body)
+
+
+def group_key(list_id: str | None, sender: str) -> str:
+    return f"list:{list_id}" if list_id else f"from:{sender.lower()}"
+
+
+async def pending_count(session: Any, user_id: uuid.UUID) -> int:
+    from sqlalchemy import func
+
+    return int(
+        await session.scalar(
+            select(func.count())
+            .select_from(Message)
+            .where(Message.user_id == user_id, Message.list_unsubscribe.is_(None))
+        )
+        or 0
+    )
+
+
+async def backfill_list_headers_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Read the list headers of a user's older mail (stored before this feature).
+
+    Works in batches; a message whose raw copy can't be had is marked as having
+    none, so it is not tried again. Returns how many messages were read.
+    """
+    uid = uuid.UUID(user_id)
+    done = 0
+    while True:
+        async with session_scope() as session:
+            batch = list(
+                await session.scalars(
+                    select(Message)
+                    .where(Message.user_id == uid, Message.list_unsubscribe.is_(None))
+                    .limit(_BATCH)
+                )
+            )
+            if not batch:
+                break
+            for message in batch:
+                try:
+                    raw = await rawstore.ensure_bytes(message)
+                    headers = read_list_headers(raw)
+                except Exception:  # missing blob, server unreachable: give up on it
+                    headers = ListHeaders(None, "", False)
+                await session.execute(
+                    update(Message)
+                    .where(Message.id == message.id)
+                    .values(
+                        list_id=headers.list_id,
+                        list_unsubscribe=headers.unsubscribe,
+                        list_unsubscribe_post=headers.one_click,
+                    )
+                )
+            await session.commit()
+            done += len(batch)
+    if done:
+        logger.info("read list headers of %d message(s) for user %s", done, user_id)
+    return done

backend/app/sync/actions.py

@@ -77,7 +77,7 @@ async def apply_flags_to_messages(
         if message.thread_id is not None:
             thread_ids.add(message.thread_id)
     await session.flush()
-    for tid in thread_ids:
+    for tid in sorted(thread_ids):  # one lock order across transactions
         await denormalize_thread(session, tid)
     return thread_ids
 
@@ -302,7 +302,8 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
 
     Each message returns to ``restore_mailbox_id`` (the folder it was in before it
     was archived/trashed) — e.g. a deleted Sent message goes back to Sent. Messages
-    with no recorded origin fall back to the account's Inbox.
+    with no recorded origin fall back to the account's Inbox. Only messages in
+    Archive, Trash or Spam move: the thread's others (your reply in Sent) stay.
     """
     tid = uuid.UUID(thread_id)
     async with session_scope() as session:
@@ -319,10 +320,13 @@ async def restore_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
             await session.scalars(select(Mailbox).where(Mailbox.account_id.in_(account_ids)))
         )
         path_by_id = {mb.id: mb.path for mb in mailboxes}
+        removal_ids = {mb.id for mb in mailboxes if mb.special_use in _REMOVAL_FOLDERS}
         inbox_by_account = {mb.account_id: mb.id for mb in mailboxes if mb.special_use == "inbox"}
         # (message_id, account_id, source_path, target_id) — resolve the target now.
         plan: list[tuple[uuid.UUID, uuid.UUID, str, uuid.UUID, int]] = []
         for m in messages:
+            if m.mailbox_id not in removal_ids:
+                continue  # never removed: nothing to restore
             target_id = m.restore_mailbox_id
             if target_id is None or target_id not in path_by_id:
                 target_id = inbox_by_account.get(m.account_id)
@@ -504,7 +508,7 @@ async def empty_folder_job(ctx: dict[str, Any], user_id: str, special_use: str)
         await session.flush()
         # Threads with no messages left get dropped (drafts un-linked first); the
         # rest are re-denormalized so their counts/snippet reflect the removal.
-        for tid in thread_ids:
+        for tid in sorted(thread_ids):
             remaining = await session.scalar(
                 select(func.count()).select_from(Message).where(Message.thread_id == tid)
             )
@@ -584,7 +588,7 @@ async def autopurge_trash_spam(ctx: dict[str, Any]) -> None:
         )
         await session.execute(delete(Message).where(Message.id.in_(msg_ids)))
         await session.flush()
-        for tid in thread_ids:
+        for tid in sorted(thread_ids):
             remaining = await session.scalar(
                 select(func.count()).select_from(Message).where(Message.thread_id == tid)
             )

backend/app/sync/engine.py

@@ -172,9 +172,7 @@ class _BackfillProgress:
         self._mailboxes.add(mailbox_path)
         async with session_scope() as session:
             await session.execute(
-                update(Account)
-                .where(Account.id == self.account.id)
-                .values(backfill_done=self.done)
+                update(Account).where(Account.id == self.account.id).values(backfill_done=self.done)
             )
         await get_event_bus().publish(
             "sync.progress",
@@ -321,16 +319,28 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
             dedupe = gmail and mailbox.special_use == "archive"
             if since is not None:  # 3months: single windowed pass
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, since=since,
+                    provider,
+                    account_obj,
+                    mailbox,
+                    progress,
+                    since=since,
                     dedupe_account_scope=dedupe,
                 )
             else:  # all: recent window first (usable fast), then the remainder
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, since=recent_cutoff,
+                    provider,
+                    account_obj,
+                    mailbox,
+                    progress,
+                    since=recent_cutoff,
                     dedupe_account_scope=dedupe,
                 )
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, before=recent_cutoff,
+                    provider,
+                    account_obj,
+                    mailbox,
+                    progress,
+                    before=recent_cutoff,
                     dedupe_account_scope=dedupe,
                 )
         await _set_state(account_id, "live")
@@ -354,7 +364,18 @@ async def apply_delta(
     if delta.new_uidvalidity is not None:
         # UIDVALIDITY changed: wipe this mailbox's rows and re-backfill (SPEC §6.3).
         async with session_scope() as session:
+            # Their threads are recounted: a message not brought back by the
+            # re-import (gone meanwhile) must not stay counted.
+            wiped_threads: set[uuid.UUID] = {
+                tid
+                for tid in await session.scalars(
+                    select(Message.thread_id).where(Message.mailbox_id == mailbox.id)
+                )
+                if tid is not None
+            }
             await session.execute(delete(Message).where(Message.mailbox_id == mailbox.id))
+            for tid in sorted(wiped_threads):
+                await denormalize_thread(session, tid)
             await session.execute(
                 update(Mailbox)
                 .where(Mailbox.id == mailbox.id)
@@ -428,7 +449,7 @@ async def apply_delta(
             )
         # After the deletes, so a thread left with no messages is removed rather
         # than recounted (denormalize_thread handles that case).
-        for tid in touched_threads:
+        for tid in sorted(touched_threads):  # one lock order (denormalize_thread)
             await denormalize_thread(session, tid)
 
         updates = {"last_seen_uid": max_uid}
@@ -480,9 +501,7 @@ async def apply_delta(
         await _reconcile_drafts(provider, account, mailbox)
 
 
-async def _reconcile_drafts(
-    provider: ImapSmtpProvider, account: Account, mailbox: Mailbox
-) -> None:
+async def _reconcile_drafts(provider: ImapSmtpProvider, account: Account, mailbox: Mailbox) -> None:
     """Drop DB rows for drafts that no longer exist on the server (were expunged)."""
     from app.db import session_scope
 
@@ -537,13 +556,13 @@ async def live_loop(
         await provider.verify()  # signed in: the account is reachable
         if health is not None:
             await health.ok()
+        first = True
         while not stop.is_set():
-            try:
-                await provider.idle(mailbox_path)
-            except ProviderNotSupported:
-                await asyncio.sleep(idle_fallback)
-            if stop.is_set():
-                break
+            # Catch up before each IDLE, not only after it: IDLE reports what
+            # arrives from now on, so mail that came while this loop was not
+            # running (a new account's first minutes, a reconnect, a worker
+            # restart) would otherwise wait for the next new message or the
+            # IDLE timeout (10 min) to show.
             async with session_scope() as session:
                 mailbox = await session.scalar(
                     select(Mailbox).where(
@@ -551,14 +570,23 @@ async def live_loop(
                     )
                 )
                 mailbox_obj = mailbox
-            if mailbox_obj is None:
-                break
+            if mailbox_obj is not None:
                 delta = await provider.poll(
                     mailbox_path,
                     last_seen_uid=mailbox_obj.last_seen_uid,
                     highestmodseq=mailbox_obj.highestmodseq,
                 )
                 await apply_delta(provider, account_obj, mailbox_obj, delta)
+            elif not first:
+                break  # the folder is gone
+            # else: the account's first sync has not recorded the folder yet.
+            first = False
+            if stop.is_set():
+                break
+            try:
+                await provider.idle(mailbox_path)
+            except ProviderNotSupported:
+                await asyncio.sleep(idle_fallback)
     finally:
         await provider.close()
 
@@ -619,9 +647,7 @@ async def poll_folders_once(
         await provider.close()
 
 
-async def _poll_secondary_once(
-    account_id: uuid.UUID, health: AccountHealth | None = None
-) -> None:
+async def _poll_secondary_once(account_id: uuid.UUID, health: AccountHealth | None = None) -> None:
     await poll_folders_once(account_id, include_inbox=False, health=health)
 
 

backend/app/sync/folder_names.py

@@ -28,24 +28,52 @@ def _compact(name: str) -> str:
 # Localized folder names by special_use. Written readably; compacted at load.
 _NAMES: dict[str, list[str]] = {
     "sent": [
-        "sent", "sent items", "sent mail", "sent messages",  # English / Outlook / Gmail
-        "wysłane", "elementy wysłane", "wiadomości wysłane",  # Polish
-        "gesendet", "gesendete objekte", "gesendete elemente",  # German
-        "envoyés", "éléments envoyés", "messages envoyés", "courrier envoyé",  # French
-        "enviados", "elementos enviados", "correo enviado",  # Spanish
-        "inviata", "posta inviata", "messaggi inviati",  # Italian
-        "verzonden", "verzonden items", "verzonden berichten",  # Dutch
-        "enviadas", "itens enviados", "mensagens enviadas",  # Portuguese
-        "отправленные", "исходящие",  # Russian
-        "skickat", "skickade", "skickade objekt",  # Swedish
-        "sendt", "sendte", "sendte elementer",  # Norwegian / Danish
+        "sent",
+        "sent items",
+        "sent mail",
+        "sent messages",  # English / Outlook / Gmail
+        "wysłane",
+        "elementy wysłane",
+        "wiadomości wysłane",  # Polish
+        "gesendet",
+        "gesendete objekte",
+        "gesendete elemente",  # German
+        "envoyés",
+        "éléments envoyés",
+        "messages envoyés",
+        "courrier envoyé",  # French
+        "enviados",
+        "elementos enviados",
+        "correo enviado",  # Spanish
+        "inviata",
+        "posta inviata",
+        "messaggi inviati",  # Italian
+        "verzonden",
+        "verzonden items",
+        "verzonden berichten",  # Dutch
+        "enviadas",
+        "itens enviados",
+        "mensagens enviadas",  # Portuguese
+        "отправленные",
+        "исходящие",  # Russian
+        "skickat",
+        "skickade",
+        "skickade objekt",  # Swedish
+        "sendt",
+        "sendte",
+        "sendte elementer",  # Norwegian / Danish
         "lähetetyt",  # Finnish
-        "odeslané", "odeslaná pošta",  # Czech
+        "odeslané",
+        "odeslaná pošta",  # Czech
     ],
     "drafts": [
-        "drafts", "draft",
-        "robocze", "kopie robocze", "wersje robocze",
-        "entwürfe", "entwurf",
+        "drafts",
+        "draft",
+        "robocze",
+        "kopie robocze",
+        "wersje robocze",
+        "entwürfe",
+        "entwurf",
         "brouillons",
         "borradores",
         "bozze",
@@ -58,44 +86,80 @@ _NAMES: dict[str, list[str]] = {
         "koncepty",  # Czech
     ],
     "trash": [
-        "trash", "deleted", "deleted items", "deleted messages", "bin", "recycle bin",
+        "trash",
+        "deleted",
+        "deleted items",
+        "deleted messages",
+        "bin",
+        "recycle bin",
         "kosz",
-        "papierkorb", "gelöschte objekte", "gelöschte elemente",
-        "corbeille", "éléments supprimés",
-        "papelera", "elementos eliminados",
-        "cestino", "posta eliminata",
-        "prullenbak", "verwijderde items",
-        "lixeira", "lixo", "itens excluídos",
-        "корзина", "удаленные",
-        "papperskorg", "borttaget",
-        "papirkurv", "slettet",
-        "roskakori", "poistetut",
+        "papierkorb",
+        "gelöschte objekte",
+        "gelöschte elemente",
+        "corbeille",
+        "éléments supprimés",
+        "papelera",
+        "elementos eliminados",
+        "cestino",
+        "posta eliminata",
+        "prullenbak",
+        "verwijderde items",
+        "lixeira",
+        "lixo",
+        "itens excluídos",
+        "корзина",
+        "удаленные",
+        "papperskorg",
+        "borttaget",
+        "papirkurv",
+        "slettet",
+        "roskakori",
+        "poistetut",
         "koš",  # Czech
     ],
     "junk": [
-        "junk", "junk email", "junk e-mail", "spam", "bulk mail", "bulk",
-        "wiadomości-śmieci", "niechciane",
-        "unerwünscht", "werbung",
-        "courrier indésirable", "indésirables", "pourriel",
-        "correo no deseado", "no deseado", "correo basura",
-        "posta indesiderata", "indesiderata",
-        "ongewenst", "ongewenste e-mail",
+        "junk",
+        "junk email",
+        "junk e-mail",
+        "spam",
+        "bulk mail",
+        "bulk",
+        "wiadomości-śmieci",
+        "niechciane",
+        "unerwünscht",
+        "werbung",
+        "courrier indésirable",
+        "indésirables",
+        "pourriel",
+        "correo no deseado",
+        "no deseado",
+        "correo basura",
+        "posta indesiderata",
+        "indesiderata",
+        "ongewenst",
+        "ongewenste e-mail",
         "lixo eletrônico",
-        "спам", "нежелательная почта",
-        "skräp", "skräppost",
+        "спам",
+        "нежелательная почта",
+        "skräp",
+        "skräppost",
         "uønsket",
         "roskaposti",
         "nevyžádaná pošta",
     ],
     "archive": [
-        "archive", "archives", "all mail",
+        "archive",
+        "archives",
+        "all mail",
         "archiwum",
         "archiv",
         "archivio",
         "archivo",
         "archief",
-        "arquivo", "arquivo morto",
-        "архив", "вся почта",
+        "arquivo",
+        "arquivo morto",
+        "архив",
+        "вся почта",
         "arkiv",
         "arkisto",
     ],

backend/app/sync/provider_factory.py

@@ -3,6 +3,9 @@
 This module runs only inside the worker/sync context; ``decrypt_secret`` (the
 credential vault) is invoked here to obtain the plaintext mailbox password. It
 must never be imported by API request code.
+
+A native account (the user's hosted mailbox, App. D §D.4) has no stored
+password: the worker signs in as itself, acting for the user.
 """
 
 from __future__ import annotations
@@ -14,6 +17,10 @@ from app.security.crypto import decrypt_secret
 
 def build_provider(account: Account) -> ImapSmtpProvider:
     """Construct an :class:`ImapSmtpProvider` for ``account`` (password auth, M0)."""
+    if account.kind == "native":
+        from app.hosted.native import native_provider_config
+
+        return ImapSmtpProvider(native_provider_config(account.email))
     imap = account.imap_config
     smtp = account.smtp_config
     password = decrypt_secret(account.secret_ct).decode("utf-8")
@@ -26,5 +33,6 @@ def build_provider(account: Account) -> ImapSmtpProvider:
         smtp_tls=str(smtp.get("tls", "ssl")),
         username=account.email,
         password=password,
+        public_only=True,  # a server the user named: never one inside our network
     )
     return ImapSmtpProvider(config)

backend/app/threading_/threader.py

@@ -11,12 +11,12 @@ from __future__ import annotations
 import uuid
 from datetime import timedelta
 
-from sqlalchemy import ColumnElement, delete, literal, select, update
+from sqlalchemy import ColumnElement, delete, exists, func, literal, select, update
 from sqlalchemy import any_ as sa_any
 from sqlalchemy import or_ as sa_or
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.models import Message, Thread
+from app.models import Draft, Message, Thread
 from app.threading_.normalize import is_synthetic, normalize_subject
 
 _SUBJECT_FALLBACK_WINDOW = timedelta(days=7)
@@ -172,18 +172,38 @@ async def _create_thread(session: AsyncSession, message: Message) -> uuid.UUID:
 
 
 async def denormalize_thread(session: AsyncSession, thread_id: uuid.UUID) -> None:
-    """Recompute a thread's denormalized fields from its messages (SPEC §7.2.6)."""
+    """Recompute a thread's denormalized fields from its messages (SPEC §7.2.6).
+
+    The thread row is locked *before* its messages are read: two transactions
+    recounting one thread at once (read in the webmail while a sync brings the
+    same flag from a phone) then take turns, and the second reads what the
+    first committed — instead of both reading first and the slower one writing
+    an old count (a message shown read, its thread still "unread"). The
+    messages are reloaded from the database, not taken from the session's
+    copies, which a bulk UPDATE may have left behind."""
+    thread = await session.scalar(
+        select(Thread)
+        .where(Thread.id == thread_id)
+        .with_for_update()
+        .execution_options(populate_existing=True)
+    )
+    if thread is None:
+        return
     messages = list(
         await session.scalars(
             select(Message)
             .where(Message.thread_id == thread_id)
             .order_by(Message.internal_date.desc())
+            .execution_options(populate_existing=True)
         )
     )
-    thread = await session.get(Thread, thread_id)
-    if thread is None:
-        return
     if not messages:
+        # A reply still being written keeps its (now empty) thread: the draft
+        # points at it.
+        if await session.scalar(select(exists().where(Draft.thread_id == thread_id))):
+            thread.message_count = 0
+            thread.unread_count = 0
+            return
         await session.execute(delete(Thread).where(Thread.id == thread_id))
         return
 
@@ -241,3 +261,38 @@ async def thread_message(session: AsyncSession, message: Message) -> uuid.UUID:
     await denormalize_thread(session, thread_id)
     await session.flush()
     return thread_id
+
+
+async def repair_threads(session: AsyncSession, user_id: uuid.UUID) -> int:
+    """Recount the user's threads whose stored counters disagree with their
+    messages (and drop threads left without any). A safety net run nightly
+    (app/indexer/gc.py): every write path recounts its threads, but a missed
+    one would otherwise show a read message as unread until it changed again.
+    Returns how many were repaired."""
+    seen = Message.flags.any("\\Seen")  # type: ignore[arg-type]
+    actual = (
+        select(
+            Message.thread_id.label("tid"),
+            func.count().label("n"),
+            func.count().filter(~seen).label("unread"),
+        )
+        .where(Message.user_id == user_id, Message.thread_id.is_not(None))
+        .group_by(Message.thread_id)
+        .subquery()
+    )
+    stale = list(
+        await session.scalars(
+            select(Thread.id)
+            .outerjoin(actual, actual.c.tid == Thread.id)
+            .where(
+                Thread.user_id == user_id,
+                (actual.c.tid.is_(None) & (Thread.message_count != 0))
+                | (actual.c.tid.is_(None) & ~exists().where(Draft.thread_id == Thread.id))
+                | (actual.c.n != Thread.message_count)
+                | (actual.c.unread != Thread.unread_count),
+            )
+        )
+    )
+    for tid in sorted(stale):
+        await denormalize_thread(session, tid)
+    return len(stale)

backend/app/translation.py

@@ -0,0 +1,423 @@
+"""Email translation, on this server: the translator service (translator/:
+Mozilla's Firefox Translations models and fastText language identification)
+in its own container, reached only by the API. A message's text never leaves
+the server.
+
+- A message's language is detected the first time it is shown (from its own
+  text: quoted replies and signatures left out) and kept on the message, so the
+  reading pane can offer "Translate" when it differs from the interface language.
+- A translation is of the message HTML the user sees (already sanitized; the
+  result is sanitized again), cached per message, target language and whether
+  remote images were shown. It is made paragraph by paragraph, as plain text
+  (translation_prep.split_pieces), each paragraph from its own language.
+- The languages on offer are those installed on the service (admin panel).
+
+Off when ``TRANSLATOR_URL`` is empty.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import logging
+import re
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import httpx
+
+from app import translation_prep as prep
+from app.completion import own_text
+from app.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+DETECT_CHARS = 1500  # of the message's own text: plenty to tell the language
+MIN_CONFIDENCE = 50.0  # below this, the detector is guessing
+MAX_HTML_CHARS = 150_000  # a translation request bigger than this is refused
+_TIMEOUT = httpx.Timeout(120.0, connect=5.0)  # long messages take a while on a CPU
+_LANG_CACHE_S = 60
+
+
+class TranslationError(Exception):
+    """Translation is off, the service is unreachable, or it refused the text."""
+
+
+@dataclass(frozen=True)
+class Language:
+    code: str
+    name: str
+
+
+def enabled() -> bool:
+    return bool(get_settings().translator_url.strip())
+
+
+def _url(path: str) -> str:
+    return get_settings().translator_url.rstrip("/") + path
+
+
+async def _call(method: str, path: str, payload: dict[str, object] | None = None) -> object:
+    if not enabled():
+        raise TranslationError("translation is not set up on this server")
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.request(method, _url(path), json=payload)
+    except httpx.HTTPError as exc:
+        raise TranslationError(f"the translation service is unreachable: {exc}") from exc
+    if resp.status_code not in (200, 202):
+        detail = resp.text[:200]
+        raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
+    return resp.json()
+
+
+async def _post(path: str, payload: dict[str, object]) -> object:
+    return await _call("POST", path, payload)
+
+
+_languages: tuple[float, list[Language]] = (0.0, [])
+
+
+async def languages() -> list[Language]:
+    """The installed languages, English among them (cached for a minute). The
+    browser names them in the interface language; ``name`` is the code."""
+    global _languages
+    if not enabled():
+        return []
+    stamp, cached = _languages
+    if cached and time.monotonic() - stamp < _LANG_CACHE_S:
+        return cached
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
+            resp = await client.get(_url("/languages"))
+        resp.raise_for_status()
+        codes = resp.json()["installed"]
+    except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc:
+        logger.info("translation languages unavailable: %s", exc)
+        return []
+    found = [Language(code=str(c), name=str(c)) for c in codes]
+    _languages = (time.monotonic(), found)
+    return found
+
+
+def forget_languages() -> None:
+    """After an install or removal: ask the service again."""
+    global _languages
+    _languages = (0.0, [])
+
+
+async def status() -> dict[str, object]:
+    """The service's own account (admin panel): installed, installing (bytes
+    done of total), available from Mozilla, and why an install failed."""
+    async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=3.0)) as client:
+        try:
+            resp = await client.get(_url("/languages"))
+            resp.raise_for_status()
+            data = resp.json()
+        except (httpx.HTTPError, ValueError) as exc:
+            raise TranslationError(f"the translation service is unreachable: {exc}") from exc
+    return data if isinstance(data, dict) else {}
+
+
+async def install(code: str) -> None:
+    await _call("POST", f"/languages/{code}")
+    forget_languages()
+
+
+async def remove(code: str) -> None:
+    await _call("DELETE", f"/languages/{code}")
+    forget_languages()
+
+
+_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
+MIN_LETTERS = 25  # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
+MAX_DETECT = 60  # pieces of one message asked about
+MIN_SHARE = 0.15  # of a message's letters, for its language to count
+
+
+async def _detect_many(texts: list[str]) -> list[str]:
+    """The language of each of ``texts`` ('' where unsure), in one request."""
+    if not texts:
+        return []
+    result = await _post("/detect", {"q": [t[:DETECT_CHARS] for t in texts]})
+    if not isinstance(result, list) or len(result) != len(texts):
+        return [""] * len(texts)
+    return [
+        str(r.get("language", ""))
+        if isinstance(r, dict) and float(r.get("confidence", 0)) >= MIN_CONFIDENCE
+        else ""
+        for r in result
+    ]
+
+
+async def _detect_one(text: str) -> str:
+    return (await _detect_many([text]))[0]
+
+
+def _weight(text: str) -> int:
+    """How much text this is: letters, a Chinese/Japanese/Korean sign as a word."""
+    return sum(3 if _CJK.match(c) else 1 for c in text if c.isalpha())
+
+
+async def languages_of(parts: list[str]) -> list[str]:
+    """The language of each of ``parts`` (paragraphs, in order). Parts too
+    short to tell take the language of the next part that could be told (a
+    greeting goes with what follows it), the last ones that of the one before;
+    parts without letters get ''."""
+    # Addresses say nothing about a language ("E-mail: lianshaojuan.lsj@…" → Estonian).
+    parts = [_ADDRESS.sub(" ", part) for part in parts]
+    told: list[str] = [""] * len(parts)
+    asked = [
+        i
+        for i, part in enumerate(parts)
+        if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
+    ][:MAX_DETECT]
+    for i, language in zip(asked, await _detect_many([parts[i] for i in asked]), strict=True):
+        told[i] = language
+    out = list(told)
+    following = ""
+    for i in reversed(range(len(parts))):
+        if told[i]:
+            following = told[i]
+        elif any(c.isalpha() for c in parts[i]):
+            out[i] = following
+    previous = ""
+    for i, part in enumerate(parts):
+        if told[i]:
+            previous = told[i]
+        elif not out[i] and any(c.isalpha() for c in part):
+            out[i] = previous
+    return out
+
+
+async def detect(text: str) -> str:
+    """The languages of ``text`` (a message's plain text), most of it first,
+    comma-separated ("zh-Hans,pl,en"); '' if unsure. A message can be in
+    several: each language that makes up a good part of it counts."""
+    own = own_text(text)
+    if len(" ".join(own.split())) < 20:
+        return ""  # a line or two says too little
+    lines: list[str] = []
+    total = 0
+    for line in (x.strip() for x in own.splitlines()):
+        if line and total < DETECT_CHARS * 4:
+            lines.append(line)
+            total += len(line)
+    found = await languages_of(lines)
+    weight: dict[str, int] = {}
+    for line, language in zip(lines, found, strict=True):
+        if language:
+            weight[language] = weight.get(language, 0) + _weight(_ADDRESS.sub(" ", line))
+    if not weight:
+        return await _detect_one(" ".join(own.split()))
+    all_of_it = sum(weight.values())
+    ranked = sorted(weight, key=lambda k: weight[k], reverse=True)
+    return ",".join(k for k in ranked if weight[k] / all_of_it >= MIN_SHARE)
+
+
+def _base(code: str) -> str:
+    return code.lower().split("-")[0]
+
+
+async def _texts(texts: list[str], source: str, target: str) -> list[str]:
+    """``texts`` translated, in batches."""
+    out: list[str] = []
+    for start in range(0, len(texts), 100):
+        batch = texts[start : start + 100]
+        result = await _post("/translate", {"q": batch, "source": source, "target": target})
+        got = result.get("translatedText") if isinstance(result, dict) else None
+        if not isinstance(got, list) or len(got) != len(batch):
+            raise TranslationError("the translation service sent no translation")
+        out += [str(x) for x in got]
+    return out
+
+
+def _kept(name: str, translated: str) -> bool:
+    """Whether ``name`` came through a translation: each of its words still
+    there, give or take an ending ("Lukas" → "Lukasie": yes; "Milan" →
+    "Mediolan": no)."""
+    return all(
+        word[: max(3, len(word) - 2)] in translated for word in name.split() if len(word) >= 2
+    )
+
+
+# A capitalised word that starts a sentence ("Enclosed is a summary…").
+_SENTENCE_START = re.compile(r"(?:^|(?<=[.!?…]\s))(?P<word>[^\W\d_][^\W\d_'’-]{2,})\b")
+
+
+def _copied_starts(core: str, text: str, names: list[str]) -> list[str]:
+    """Words that start a sentence of ``core`` and came through ``text``
+    untranslated, and are no known name: the models take a capitalised
+    sentence start for a name ("Enclosed is…" → "Enclosed jest…")."""
+    found: list[str] = []
+    for m in _SENTENCE_START.finditer(core):
+        word = m.group("word")
+        if not word[0].isupper() or word.isupper() or any(word in n for n in names):
+            continue
+        if re.search(rf"(?<!\w){re.escape(word)}(?!\w)", text):
+            found.append(word)
+    return found
+
+
+def _lowered(core: str, words: list[str]) -> str:
+    """``core`` with ``words`` lower-cased where they start a sentence."""
+    return _SENTENCE_START.sub(
+        lambda m: m.group(0).lower() if m.group("word") in words else m.group(0), core
+    )
+
+
+def _capitalised(text: str) -> str:
+    """Sentence starts capitalised again (after ``_lowered``): the first
+    letter, and one after a sentence's end — a full stop after a word of four
+    letters or more, so not after "r.", "np.", "tzw." (abbreviations)."""
+    text = re.sub(r"^(\W*)([^\W\d_])", lambda m: m.group(1) + m.group(2).upper(), text)
+    return re.sub(
+        r"((?:[^\W\d_]{4,}[.…]|[!?])\s+)([^\W\d_])",
+        lambda m: m.group(1) + m.group(2).upper(),
+        text,
+    )
+
+
+def _as_before(text: str, first: str) -> str:
+    """A word after a comma capitalised in ``text`` but not in ``first`` (the
+    first try) is lower-cased: lower-cased input makes the models capitalise
+    elsewhere ("…2026, od 13 lipca" → "…2026 r., Od 13 lipca")."""
+
+    def repl(m: re.Match[str]) -> str:
+        word = m.group(2)
+        lower = word[0].lower() + word[1:]
+        return m.group(1) + (lower if f", {lower}" in first else word)
+
+    return re.sub(r"(,\s+)([^\W\d_][\w'’-]*)", repl, text)
+
+
+async def _with_names_kept(core: str, names: list[str], source: str, target: str) -> str:
+    """``core`` translated around ``names``, which are left as they are."""
+    pattern = re.compile("(" + "|".join(re.escape(n) for n in names) + ")")
+    parts = pattern.split(core)
+    todo = [
+        (i, m.groups())
+        for i, part in enumerate(parts)
+        if i % 2 == 0 and (m := _EDGE_SPACE.match(part)) and any(c.isalnum() for c in part)
+    ]
+    done = await _texts([groups[1] for _, groups in todo], source, target)
+    for (i, (lead, _, trail)), text in zip(todo, done, strict=True):
+        parts[i] = f"{lead}{text}{trail}"
+    return "".join(parts)
+
+
+async def _translate_paragraphs(
+    html: str, target: str, sources: list[str], names: Iterable[str] = ()
+) -> str:
+    """``html`` in ``target``, as plain text paragraph by paragraph: from the
+    one language in ``sources``, or (several, or none known) each paragraph
+    from its own. Paragraphs already in ``target``, or in a language that is
+    not installed, are left as they are.
+
+    ``names`` go along with their sentence, which then translates as a whole
+    (kept apart, "I came across <Imaginary Pixels>" lost its verb: "Przyszedłem
+    Imaginary Pixels"). Only a piece where one came out changed ("Milan" →
+    "Mediolan") is translated again with it left as it is."""
+    watch = sorted({n.strip() for n in names if len(n.strip()) >= 2}, key=len, reverse=True)
+    pieces, paragraphs = prep.split_pieces(html)
+    if len(sources) == 1:
+        found = [sources[0]] * len(paragraphs)
+    else:
+        found = await languages_of([p.text for p in paragraphs])
+    installed = {x.code for x in await languages()}
+    # Chinese and Japanese go without spaces: two pieces of one sentence
+    # ("…奎琳，</span>我从…") need one between them once translated.
+    followed = {i for p in paragraphs for i in p.pieces[:-1]}
+    by_language: dict[str, list[int]] = {}
+    for paragraph, language in zip(paragraphs, found, strict=True):
+        if language in installed and _base(language) != _base(target):
+            by_language.setdefault(language, []).extend(paragraph.pieces)
+    for language, where in by_language.items():
+        todo: list[tuple[int, str, str, str]] = []
+        for i in where:
+            m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
+            lead, core, trail = m.groups() if m else ("", "", "")
+            # A list marker stays out: the models repeat it ("1. Thursday" →
+            # "1. 1. Czwartek").
+            marker = _LIST_MARKER.match(core)
+            if marker:
+                lead, core = lead + marker.group(0), core[marker.end() :]
+            if not trail and i in followed and _base(language) in _NO_SPACES:
+                trail = " "
+            if any(c.isalnum() for c in core):
+                todo.append((i, lead, core, trail))
+        done = await _texts([core for _, _, core, _ in todo], language, target)
+        # Sentence starts that came through untranslated: once more, lower-case.
+        again = [
+            (k, words)
+            for k, ((_, _, core, _), text) in enumerate(zip(todo, done, strict=True))
+            if (words := _copied_starts(core, text, watch))
+        ]
+        if again:
+            retried = await _texts(
+                [_lowered(todo[k][2], words) for k, words in again], language, target
+            )
+            for (k, words), text in zip(again, retried, strict=True):
+                # Still there: a name after all, and the first try stands.
+                if len(_copied_starts(todo[k][2], text, watch)) < len(words):
+                    done[k] = _as_before(_capitalised(text), done[k])
+        for (i, lead, core, trail), text in zip(todo, done, strict=True):
+            lost = [n for n in watch if n in core and not _kept(n, text)]
+            if lost:
+                text = await _with_names_kept(core, lost, language, target)
+            pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
+    return "".join(pieces)
+
+
+_NO_SPACES = {"zh", "ja", "th"}
+_LIST_MARKER = re.compile(r"(?:\d{1,3}[.)]|[a-zA-Z][.)]|[-•*–])\s+(?=\S)")
+_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
+
+
+# Web and mail addresses in the text: the models break them up ("https: / /
+# vimeo.com"), so they are marked translate="no" and not sent.
+_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
+_TAG = re.compile(r"(<[^>]*>)")
+
+
+def protect_addresses(html: str) -> str:
+    """Mark the addresses in ``html``'s text as not to be translated (style
+    blocks and tag attributes are left alone)."""
+    out: list[str] = []
+    in_style = False
+    for piece in _TAG.split(html):
+        if piece.startswith("<"):
+            lowered = piece.lower()
+            if lowered.startswith("<style"):
+                in_style = True
+            elif lowered.startswith("</style"):
+                in_style = False
+            out.append(piece)
+        elif in_style or not piece:
+            out.append(piece)
+        else:
+            out.append(_ADDRESS.sub(lambda m: f'<span translate="no">{m.group(0)}</span>', piece))
+    return "".join(out)
+
+
+# Bumped when what goes to the service, or what is done with the result,
+# changes: translations kept from before are made again.
+PIPELINE_VERSION = 8
+
+
+async def translate_html(
+    html: str, source: str, target: str, keep: Iterable[str] = (), names: Iterable[str] = ()
+) -> str:
+    """``html`` (a sanitized message body) in ``target``; source "auto", a code,
+    or several ("zh-Hans,pl,en": then each paragraph from its own language).
+    ``keep``: words always left as they are; ``names``: left as they are if
+    the model would change them (translation_prep)."""
+    if len(html) > MAX_HTML_CHARS:
+        raise TranslationError("this message is too long to translate")
+    sources = [x for x in (source or "").split(",") if x and x != "auto"]
+    prepared = prep.join_wrapped_lines(html)
+    for language in sources:
+        prepared = prep.translate_phrases(prepared, language, target)
+    candidates = {*names, *prep.find_names(prepared, sources[0] if sources else "")}
+    prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, keep)))
+    return prep.tidy(await _translate_paragraphs(prepared, target, sources, candidates))

backend/app/translation_prep.py

@@ -0,0 +1,332 @@
+"""Getting a message ready for translation, and tidying what comes back.
+
+A message is translated as plain text, piece by piece (the text between
+tags), and the models stumble on things a reader would not: a sentence
+wrapped onto two lines becomes two sentences ("ask you about" / "Plan for next
+month"), "Best," becomes "Najlepsze," and a company name gets translated word
+by word. So before a message goes out:
+
+- lines that carry on a sentence (no end punctuation, next starts lowercase)
+  are joined, in HTML (<br>, one <div> per line) and in wrapped plain text;
+- greetings and sign-offs that make a whole line are put in the target
+  language from a small phrase table (English, Polish, German);
+- names are marked translate="no" and not sent — the people on the message,
+  the user's own names and signature names, and capitalised multi-word names
+  mid-sentence — but only the names the model would change ("Milan" →
+  "Mediolan"): a kept name splits its sentence into pieces translated alone
+  ("Dear <kept>Lukas Remis</kept>," → "Kochanie."), so a name the model
+  leaves alone stays in the sentence;
+- so are emoji, file names ("A_0023C017_p1CAC.mov") and text without a letter;
+- the message is translated paragraph by paragraph (split_pieces), each from
+  its own language when there are several; those in the target stay;
+- and afterwards, a kept word gets back the spaces around it.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import re
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+
+KEEP_OPEN = '<span translate="no">'
+_TAG = re.compile(r"(<[^>]*>)")
+_LOWER_START = "a-ząćęłńóśźżäöüßàâçéèêëîïôûùüÿñ"
+_UPPER = "A-ZĄĆĘŁŃÓŚŹŻÄÖÜÀÂÇÉÈÊËÎÏÔÛÙÜŸÑ"
+
+# --- joining wrapped lines -------------------------------------------------------
+
+# A line break between a word (or comma) and a lowercase word carries a sentence on.
+_HTML_SOFT_BREAK = re.compile(
+    rf"([\w,;–-])[ \t]*(?:<br\s*/?>|</div>\s*<div[^>]*>|</p>\s*<p[^>]*>)\s*(?=[{_LOWER_START}])"
+)
+_TEXT_SOFT_BREAK = re.compile(rf"([^\s.!?:;])[ \t]*\n[ \t]*(?=[{_LOWER_START}])")
+
+
+def join_wrapped_lines(html: str) -> str:
+    html = _HTML_SOFT_BREAK.sub(r"\1 ", html)
+    # Plain text (wm-plaintext): newlines inside text, outside tags.
+    return "".join(
+        p if p.startswith("<") else _TEXT_SOFT_BREAK.sub(r"\1 ", p) for p in _TAG.split(html)
+    )
+
+
+# --- greetings and sign-offs -----------------------------------------------------
+
+# source language → the line as written (lower case) → target language → line.
+_PHRASES: dict[str, dict[str, dict[str, str]]] = {
+    "en": {
+        "best": {"pl": "Pozdrawiam", "de": "Viele Grüße"},
+        "best regards": {"pl": "Pozdrawiam serdecznie", "de": "Beste Grüße"},
+        "best wishes": {"pl": "Najlepsze życzenia", "de": "Beste Grüße"},
+        "kind regards": {"pl": "Z pozdrowieniami", "de": "Freundliche Grüße"},
+        "warm regards": {"pl": "Serdecznie pozdrawiam", "de": "Herzliche Grüße"},
+        "regards": {"pl": "Pozdrawiam", "de": "Grüße"},
+        "cheers": {"pl": "Pozdrawiam", "de": "Viele Grüße"},
+        "sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"},
+        "yours sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"},
+        "thanks": {"pl": "Dzięki", "de": "Danke"},
+        "many thanks": {"pl": "Wielkie dzięki", "de": "Vielen Dank"},
+        "thank you": {"pl": "Dziękuję", "de": "Vielen Dank"},
+        "thanks in advance": {"pl": "Z góry dziękuję", "de": "Vielen Dank im Voraus"},
+        "all the best": {"pl": "Wszystkiego dobrego", "de": "Alles Gute"},
+        "talk soon": {"pl": "Do usłyszenia", "de": "Bis bald"},
+        "have a great day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"},
+        "have a nice day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"},
+    },
+    "pl": {
+        "pozdrawiam": {"en": "Best regards", "de": "Viele Grüße"},
+        "pozdrawiam serdecznie": {"en": "Warm regards", "de": "Herzliche Grüße"},
+        "serdecznie pozdrawiam": {"en": "Warm regards", "de": "Herzliche Grüße"},
+        "z poważaniem": {"en": "Kind regards", "de": "Mit freundlichen Grüßen"},
+        "z pozdrowieniami": {"en": "Kind regards", "de": "Freundliche Grüße"},
+        "dziękuję": {"en": "Thank you", "de": "Vielen Dank"},
+        "dzięki": {"en": "Thanks", "de": "Danke"},
+        "z góry dziękuję": {"en": "Thanks in advance", "de": "Vielen Dank im Voraus"},
+        "miłego dnia": {"en": "Have a nice day", "de": "Einen schönen Tag noch"},
+        "do usłyszenia": {"en": "Talk soon", "de": "Bis bald"},
+    },
+    "de": {
+        "mit freundlichen grüßen": {"en": "Kind regards", "pl": "Z poważaniem"},
+        "freundliche grüße": {"en": "Kind regards", "pl": "Z pozdrowieniami"},
+        "viele grüße": {"en": "Best regards", "pl": "Pozdrawiam"},
+        "beste grüße": {"en": "Best regards", "pl": "Pozdrawiam"},
+        "liebe grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"},
+        "herzliche grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"},
+        "danke": {"en": "Thanks", "pl": "Dzięki"},
+        "vielen dank": {"en": "Many thanks", "pl": "Bardzo dziękuję"},
+        "bis bald": {"en": "Talk soon", "pl": "Do usłyszenia"},
+    },
+}
+# A whole line: after a tag or line start, before a tag or line end.
+_LINE = re.compile(r"(^|>|\n)([ \t]*)([^<>\n]{2,40}?)([ \t]*[,.!]?)([ \t]*)(?=<|\n|$)")
+
+
+def _base(code: str) -> str:
+    return code.lower().split("-")[0]
+
+
+def translate_phrases(html: str, source: str, target: str) -> str:
+    table = _PHRASES.get(_base(source), {})
+    if not table:
+        return html
+    tgt = _base(target)
+
+    def repl(m: re.Match[str]) -> str:
+        found = table.get(" ".join(m.group(3).lower().split()), {}).get(tgt)
+        if found is None:
+            return m.group(0)
+        return f"{m.group(1)}{m.group(2)}{KEEP_OPEN}{found}{m.group(4).strip()}</span>{m.group(5)}"
+
+    # The pattern never crosses a tag: it stops at "<" and ">".
+    return _LINE.sub(repl, html)
+
+
+# --- names ------------------------------------------------------------------------
+
+# A capitalised word; dots only inside it ("U.S"), never at its end — a name
+# does not run on over a sentence's end ("NukeX. My work…").
+_WORD = rf"[{_UPPER}][\w'’&-]*(?:\.[\w'’&-]+)*"
+# Two or more capitalised words in a row ("Imaginary Pixels", "Trusted Partner
+# Network"), after a lowercase word: a name, not the start of a sentence.
+_MULTI_NAME = re.compile(
+    rf"(?<=[{_LOWER_START},] )({_WORD}(?: (?:&|of|the|de|von|van|{_WORD}))* {_WORD})"
+)
+# Languages that capitalise every noun: capitals say nothing about names there.
+_CAPITALISE_NOUNS = {"de", "lb"}
+
+
+def _title_case(text: str) -> bool:
+    """A Headline Like This: nearly every longer word capitalised (short ones
+    — of, and, in — often are not). A sentence full of names is not one."""
+    words = [w for w in re.findall(r"[^\W\d_][\w'’-]*", text) if len(w) >= 4]
+    return len(words) >= 3 and sum(w[0].isupper() for w in words) / len(words) > 0.8
+
+
+def find_names(html: str, source: str) -> set[str]:
+    """Capitalised multi-word names in the middle of sentences (not in headings
+    or Title Case lines, and not for languages that capitalise all nouns)."""
+    if _base(source) in _CAPITALISE_NOUNS:
+        return set()
+    found: set[str] = set()
+    for piece in _TAG.split(html):
+        if piece.startswith("<") or _title_case(piece):
+            continue
+        found.update(m.group(1).rstrip(".") for m in _MULTI_NAME.finditer(piece))
+    return found
+
+
+_SIG_NAME = re.compile(rf"{_WORD}(?: (?:&|of|the|{_WORD}))* {_WORD}")
+
+
+def find_names_in_signature(text: str) -> list[str]:
+    """Capitalised multi-word names in a signature ("Imaginary Pixels"): the
+    user's company or title, which a translation should leave alone."""
+    return [m.group(0).rstrip(".") for m in _SIG_NAME.finditer(text)]
+
+
+def keep_terms(html: str, terms: Iterable[str]) -> str:
+    """Mark each of ``terms`` in ``html``'s text translate="no" (longest first;
+    whole words; style blocks and tags left alone)."""
+    words = sorted({t.strip() for t in terms if len(t.strip()) >= 2}, key=len, reverse=True)
+    if not words:
+        return html
+    pattern = re.compile(r"(?<![\w>])(" + "|".join(re.escape(w) for w in words) + r")(?![\w<])")
+    return _mark(html, pattern)
+
+
+# Emoji and other pictographs (with their skin tones, joiners and variation
+# selectors): on its own, an emoji comes back as a made-up word ("😊" →
+# "Uzasadnienie").
+_SYMBOLS = re.compile(
+    "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff"
+    "\U0001f000-\U0001faff]"
+    "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff"
+    "\U0001f000-\U0001faff\ufe0f\u200d\u20e3]*"
+)
+# File names and codes with underscores ("A_0023C017_260521_p1CAC.mov",
+# "final_v2"): the models split them at every "_" and dot.
+_FILE_NAME = re.compile(
+    r"(?<![\w./-])(?:[\w-]*_[\w-]*(?:\.[\w-]+)*|[\w-]+(?:\.[\w-]+)*\.[a-z][a-z0-9]{1,4})(?![\w/-])",
+    re.IGNORECASE,
+)
+# International phone numbers ("+48 780 549 877"): the models space out the
+# "+". Only those: a kept piece splits its sentence, and plain numbers
+# ("1300 - 2500 Euro") come through anyway.
+_PHONE = re.compile(r"(?<![\w+])\+\d[\d ().-]{5,}\d(?!\w)")
+# Only names with an extension a file would have, not every "e.g." or "U.S".
+_FILE_EXT = re.compile(r"\.[a-z][a-z0-9]{1,4}$", re.IGNORECASE)
+
+
+def keep_symbols(html: str) -> str:
+    """Mark emoji, file names, handles ("Jacqueline_"), phone numbers and text
+    without a letter in ``html`` translate="no"."""
+    html = _mark(_mark(html, _SYMBOLS), _PHONE)
+
+    def is_file(m: re.Match[str]) -> bool:
+        word = m.group(0)
+        if "_" in word and word.strip("_"):
+            return True
+        return bool(_FILE_EXT.search(word)) and any(c.isdigit() or c in "_-" for c in word)
+
+    return _map_text(_mark(html, _FILE_NAME, keep=is_file), _keep_letterless)
+
+
+_ENTITY = re.compile(r"&#?\w+;")
+_EDGES = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
+
+
+def _keep_letterless(text: str) -> str:
+    """A piece of text with no letter or digit — an invisible byte-order mark,
+    a lone symbol — kept as it is: the models make up a word for it ("\ufeff"
+    → "Uzasadnienie")."""
+    if any(c.isalnum() for c in _ENTITY.sub("", text)):
+        return text
+    lead, core, trail = _EDGES.match(text).groups()  # type: ignore[union-attr]
+    if not _ENTITY.sub("", core).strip():
+        return text  # only spaces and &nbsp;
+    return f"{lead}{KEEP_OPEN}{core}</span>{trail}"
+
+
+def _mark(
+    html: str, pattern: re.Pattern[str], keep: Callable[[re.Match[str]], bool] | None = None
+) -> str:
+    """Wrap ``pattern``'s matches in ``html``'s text in translate="no" spans
+    (where ``keep(match)`` says so)."""
+
+    def repl(m: re.Match[str]) -> str:
+        if keep is not None and not keep(m):
+            return m.group(0)
+        return f"{KEEP_OPEN}{m.group(0)}</span>"
+
+    return _map_text(html, lambda text: pattern.sub(repl, text))
+
+
+def _map_text(html: str, fn: Callable[[str], str]) -> str:
+    """``fn`` applied to each piece of ``html``'s text; style blocks, tags and
+    text already kept are left alone."""
+    out: list[str] = []
+    in_style = in_keep = False
+    for piece in _TAG.split(html):
+        if piece.startswith("<"):
+            low = piece.lower()
+            if low.startswith("<style"):
+                in_style = True
+            elif low.startswith("</style"):
+                in_style = False
+            elif low.startswith(KEEP_OPEN):
+                in_keep = True
+            elif in_keep and low.startswith("</span"):
+                in_keep = False
+            out.append(piece)
+        elif in_style or in_keep or not piece:
+            out.append(piece)
+        else:
+            out.append(fn(piece))
+    return "".join(out)
+
+
+# --- messages in more than one language ------------------------------------------
+
+# Tags that end a paragraph (a line): the text between two of them is one
+# piece of one language.
+_BLOCK = re.compile(
+    r"</?(?:p|div|li|ul|ol|table|thead|tbody|tfoot|tr|td|th|h[1-6]|blockquote|pre|"
+    r"section|article|header|footer|center|br|hr)\b",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class Paragraph:
+    """One paragraph of a message: its text (for telling its language) and
+    where its translatable pieces are in ``split_pieces``' list."""
+
+    text: str = ""
+    pieces: list[int] = field(default_factory=list)
+
+
+def split_pieces(html: str) -> tuple[list[str], list[Paragraph]]:
+    """``html`` as a list of tags and text, and its paragraphs. Text in style
+    blocks is no paragraph's; text already kept counts for the language but
+    is not translated."""
+    pieces = _TAG.split(html)
+    paragraphs: list[Paragraph] = [Paragraph()]
+    in_style = in_keep = False
+    for i, piece in enumerate(pieces):
+        if piece.startswith("<"):
+            low = piece.lower()
+            if low.startswith("<style"):
+                in_style = True
+            elif low.startswith("</style"):
+                in_style = False
+            elif low.startswith(KEEP_OPEN):
+                in_keep = True
+            elif in_keep and low.startswith("</span"):
+                in_keep = False
+            if _BLOCK.match(piece) and (paragraphs[-1].text.strip() or paragraphs[-1].pieces):
+                paragraphs.append(Paragraph())
+        elif piece and not in_style:
+            paragraphs[-1].text += html_lib.unescape(piece)
+            if not in_keep:
+                paragraphs[-1].pieces.append(i)
+    return pieces, [p for p in paragraphs if p.text.strip()]
+
+
+# --- after translation -------------------------------------------------------------
+
+# A kept span can lose the space beside it ("Bruton</span>i twoje").
+# Only kept spans: they still carry translate="no" here (the sanitizer that
+# runs next takes it off); other spans are the sender's and stay as they are.
+_KEPT = re.compile(r'(<span translate="no">[^<]*</span>)')
+
+
+def tidy(html: str) -> str:
+    def repl(m: re.Match[str]) -> str:
+        before = html[m.start() - 1] if m.start() > 0 else ""
+        after = html[m.end()] if m.end() < len(html) else ""
+        lead = " " if re.match(r"[\w,;:)]", before) else ""
+        trail = " " if re.match(r"[\w(„\"“]", after) else ""
+        return f"{lead}{m.group(1)}{trail}"
+
+    return _KEPT.sub(repl, html)

backend/app/ws.py

@@ -18,6 +18,7 @@ from app.events import get_event_bus
 from app.identity.cookies import SESSION_COOKIE_INSECURE, SESSION_COOKIE_SECURE
 from app.identity.sessions import resolve_session
 from app.redis_client import get_redis
+from app.security.cross_site import is_cross_site
 from app.sync import leases
 
 logger = logging.getLogger(__name__)
@@ -27,6 +28,12 @@ router = APIRouter()
 
 @router.websocket("/ws")
 async def ws_endpoint(websocket: WebSocket) -> None:
+    # Only our own pages may listen in: a socket another site opens is refused
+    # even if the browser sent the session cookie with it (cross-site WebSocket
+    # hijacking; SameSite=Lax already stops current browsers from sending it).
+    if is_cross_site(websocket.headers):
+        await websocket.close(code=1008)
+        return
     token = websocket.cookies.get(SESSION_COOKIE_SECURE) or websocket.cookies.get(
         SESSION_COOKIE_INSECURE, ""
     )

backend/migrations/versions/0023_attachment_part_index.py

@@ -0,0 +1,36 @@
+"""attachments are kept once: inside their message
+
+An attachment used to be stored twice — inside the raw message blob, and again
+as a blob of its own. Now only the raw message is kept; ``part_index`` says
+which MIME part of it (its position in a walk of the message) holds the
+attachment, so it can be taken out when downloaded. ``blob_sha256`` stays: the
+attachment's content hash, which names its thumbnail and checks what is taken
+out. Rows stored before this have no index; they are found by that hash.
+
+Existing separate copies are removed by a background pass on the worker
+(app/indexer/attachments.py ``dedupe_existing``), only after checking each one
+can be taken out of its message again.
+
+Revision ID: 0023
+Revises: 0022
+Create Date: 2026-10-08
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0023"
+down_revision: str | None = "0022"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE attachments ADD COLUMN part_index integer")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE attachments DROP COLUMN IF EXISTS part_index")

backend/migrations/versions/0024_app_passwords.py

@@ -0,0 +1,43 @@
+"""app passwords: one password per mail app (SPEC App. D §D.5)
+
+Mail apps (Thunderbird, a phone's Mail app) will sign in to the user's hosted
+mailbox over IMAP/SMTP, where no 2FA code can be asked for. Each app gets its
+own generated password instead, which can be revoked on its own; the main
+password with 2FA stays for the webmail. Stored as argon2id hashes, the form
+the mail server's SQL directory reads (§D.3).
+
+Revision ID: 0024
+Revises: 0023
+Create Date: 2026-10-09
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0024"
+down_revision: str | None = "0023"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE user_app_passwords (
+          id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+          user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          name text NOT NULL,
+          password_hash text NOT NULL,
+          created_at timestamptz NOT NULL DEFAULT now(),
+          last_used_at timestamptz
+        )
+        """
+    )
+    op.execute("CREATE INDEX ix_user_app_passwords_user ON user_app_passwords (user_id)")
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS user_app_passwords")

backend/migrations/versions/0025_hosted_mailboxes.py

@@ -0,0 +1,75 @@
+"""hosted mailboxes: native accounts and the mail server's directory (SPEC App. D)
+
+* ``accounts.kind`` — ``external`` (any IMAP/SMTP server, the user's own
+  password in the vault) or ``native`` (the user's mailbox on the hosted mail
+  server: permanent, signed in to without any stored password, §D.4).
+* ``mail_directory`` — the view the mail server's SQL directory reads (§D.3):
+  one row per user, its address — and no password: the webmail password never
+  opens the mailbox in a mail app, only app passwords do (the mail server's
+  own, §D.5) — and ``active`` false for a blocked user. Plus the service accounts of
+  ``mail_service_accounts``: the webmail's own worker, which may act for any
+  mailbox and nothing else.
+* ``user_app_passwords.server_id`` — the mail server's id of the same app
+  password (§D.5), to revoke it there too.
+
+The mail server reads the view as its own role, given SELECT on it alone by
+``app.hosted.provision`` (the role is created there, with its password).
+
+Revision ID: 0025
+Revises: 0024
+Create Date: 2026-10-11
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0025"
+down_revision: str | None = "0024"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE accounts ADD COLUMN kind text NOT NULL DEFAULT 'external'")
+    op.execute(
+        "ALTER TABLE accounts ADD CONSTRAINT ck_accounts_kind "
+        "CHECK (kind IN ('external', 'native'))"
+    )
+    # One mailbox per user.
+    op.execute("CREATE UNIQUE INDEX uq_accounts_native ON accounts (user_id) WHERE kind = 'native'")
+    op.execute("ALTER TABLE user_app_passwords ADD COLUMN server_id text")
+    op.execute(
+        """
+        CREATE TABLE mail_service_accounts (
+          name citext PRIMARY KEY,
+          secret text NOT NULL,
+          description text NOT NULL DEFAULT ''
+        )
+        """
+    )
+    op.execute(
+        """
+        CREATE VIEW mail_directory AS
+          SELECT u.handle AS name,
+                 NULL::text AS secret,
+                 'individual'::text AS type,
+                 u.handle::text AS description,
+                 u.blocked_at IS NULL AS active
+            FROM users u
+          UNION ALL
+          SELECT s.name, s.secret, 'individual'::text, s.description, true
+            FROM mail_service_accounts s
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW IF EXISTS mail_directory")
+    op.execute("DROP TABLE IF EXISTS mail_service_accounts")
+    op.execute("ALTER TABLE user_app_passwords DROP COLUMN IF EXISTS server_id")
+    op.execute("DROP INDEX IF EXISTS uq_accounts_native")
+    op.execute("ALTER TABLE accounts DROP CONSTRAINT IF EXISTS ck_accounts_kind")
+    op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS kind")

backend/migrations/versions/0026_imap_ids_bigint.py

@@ -0,0 +1,40 @@
+"""IMAP ids as 64-bit integers
+
+UIDVALIDITY and UIDs are unsigned 32-bit numbers (RFC 9051), MODSEQ up to 63
+bits (RFC 7162); Postgres ``integer`` is signed 32-bit. Servers that hand out
+values above 2^31 — the hosted mail server's UIDVALIDITY, for one — could not
+be stored ("value out of int32 range") and their folders never synced.
+
+Revision ID: 0026
+Revises: 0025
+Create Date: 2026-10-11
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0026"
+down_revision: str | None = "0025"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_COLUMNS = (
+    ("mailboxes", "uidvalidity"),
+    ("mailboxes", "highestmodseq"),
+    ("mailboxes", "last_seen_uid"),
+    ("messages", "uid"),
+    ("drafts", "imap_uid"),
+)
+
+
+def upgrade() -> None:
+    for table, column in _COLUMNS:
+        op.execute(f"ALTER TABLE {table} ALTER COLUMN {column} TYPE bigint")
+
+
+def downgrade() -> None:
+    for table, column in _COLUMNS:
+        op.execute(f"ALTER TABLE {table} ALTER COLUMN {column} TYPE integer")

backend/migrations/versions/0027_invitations.py

@@ -0,0 +1,63 @@
+"""invitations: users invite friends, even while sign-up is closed
+
+An administrator switch lets every user send a few single-use invitation links
+by email; a link opens sign-up for the person it was sent to even when public
+registration is off. The email is built from a template the administrator
+uploads (or the built-in one) and sent from the instance's own address.
+
+Revision ID: 0027
+Revises: 0026
+Create Date: 2026-10-14
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0027"
+down_revision: str | None = "0026"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE app_settings
+          ADD COLUMN invitations_enabled boolean NOT NULL DEFAULT false,
+          ADD COLUMN invite_from text NOT NULL DEFAULT '',
+          ADD COLUMN invite_subject text NOT NULL DEFAULT '',
+          ADD COLUMN invite_template text NOT NULL DEFAULT ''
+        """
+    )
+    op.execute(
+        """
+        CREATE TABLE invitations (
+          id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+          inviter_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          token_hash text NOT NULL UNIQUE,
+          invitee_name text NOT NULL,
+          invitee_email text NOT NULL,
+          created_at timestamptz NOT NULL DEFAULT now(),
+          expires_at timestamptz NOT NULL,
+          used_at timestamptz,
+          used_by uuid REFERENCES users(id) ON DELETE SET NULL
+        )
+        """
+    )
+    op.execute("CREATE INDEX ix_invitations_inviter ON invitations (inviter_id)")
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS invitations")
+    op.execute(
+        """
+        ALTER TABLE app_settings
+          DROP COLUMN IF EXISTS invitations_enabled,
+          DROP COLUMN IF EXISTS invite_from,
+          DROP COLUMN IF EXISTS invite_subject,
+          DROP COLUMN IF EXISTS invite_template
+        """
+    )

backend/migrations/versions/0028_snooze_templates_subscriptions.py

@@ -0,0 +1,108 @@
+"""snooze, templates, subscriptions, tracker counts
+
+- Snooze: a thread can leave the Inbox until a set time (``snoozed_until``);
+  ``snoozed_at`` lets a reply that arrives meanwhile wake it early, and
+  ``unsnoozed_at`` brings a woken thread to the top of the Inbox.
+- Templates: saved replies, per user, inserted from the compose window.
+- Subscriptions: the mailing-list headers of each message (``List-Id``,
+  ``List-Unsubscribe``, one-click support), read during sync. NULL
+  ``list_unsubscribe`` means "not read yet" (filled in by a backfill job), ''
+  means the message has none. ``subscription_states`` records what the user did
+  about a sender (unsubscribed when).
+- Tracker blocking: how many tracking pixels were removed from a message's HTML.
+
+Revision ID: 0028
+Revises: 0027
+Create Date: 2026-10-18
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0028"
+down_revision: str | None = "0027"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE threads
+          ADD COLUMN snoozed_until timestamptz,
+          ADD COLUMN snoozed_at timestamptz,
+          ADD COLUMN unsnoozed_at timestamptz
+        """
+    )
+    op.execute(
+        "CREATE INDEX ix_threads_snoozed ON threads (snoozed_until) WHERE snoozed_until IS NOT NULL"
+    )
+    op.execute(
+        """
+        ALTER TABLE messages
+          ADD COLUMN list_id text,
+          ADD COLUMN list_unsubscribe text,
+          ADD COLUMN list_unsubscribe_post boolean NOT NULL DEFAULT false
+        """
+    )
+    op.execute(
+        "CREATE INDEX ix_messages_user_list ON messages (user_id) "
+        "WHERE list_unsubscribe IS NOT NULL AND list_unsubscribe <> ''"
+    )
+    op.execute(
+        "CREATE INDEX ix_messages_list_unread ON messages (user_id) WHERE list_unsubscribe IS NULL"
+    )
+    op.execute(
+        """
+        CREATE TABLE subscription_states (
+          user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          key text NOT NULL,
+          unsubscribed_at timestamptz NOT NULL DEFAULT now(),
+          method text NOT NULL,
+          PRIMARY KEY (user_id, key)
+        )
+        """
+    )
+    op.execute(
+        """
+        CREATE TABLE mail_templates (
+          id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+          user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          name text NOT NULL,
+          subject text NOT NULL DEFAULT '',
+          body_html text NOT NULL DEFAULT '',
+          created_at timestamptz NOT NULL DEFAULT now(),
+          updated_at timestamptz NOT NULL DEFAULT now()
+        )
+        """
+    )
+    op.execute("CREATE INDEX ix_mail_templates_user ON mail_templates (user_id)")
+    op.execute("ALTER TABLE body_parts ADD COLUMN trackers_blocked integer NOT NULL DEFAULT 0")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE body_parts DROP COLUMN IF EXISTS trackers_blocked")
+    op.execute("DROP TABLE IF EXISTS mail_templates")
+    op.execute("DROP TABLE IF EXISTS subscription_states")
+    op.execute("DROP INDEX IF EXISTS ix_messages_list_unread")
+    op.execute("DROP INDEX IF EXISTS ix_messages_user_list")
+    op.execute(
+        """
+        ALTER TABLE messages
+          DROP COLUMN IF EXISTS list_id,
+          DROP COLUMN IF EXISTS list_unsubscribe,
+          DROP COLUMN IF EXISTS list_unsubscribe_post
+        """
+    )
+    op.execute("DROP INDEX IF EXISTS ix_threads_snoozed")
+    op.execute(
+        """
+        ALTER TABLE threads
+          DROP COLUMN IF EXISTS snoozed_until,
+          DROP COLUMN IF EXISTS snoozed_at,
+          DROP COLUMN IF EXISTS unsnoozed_at
+        """
+    )

backend/migrations/versions/0029_completion.py

@@ -0,0 +1,49 @@
+"""sentence completion: a per-user index of phrases from their sent mail
+
+``completion_phrases`` maps the words before the cursor (normalized) to what
+the user usually writes next, with how often; rebuilt from the sent mail at
+most daily (app/completion.py). The primary key leads with (user_id, context),
+which is the lookup.
+
+Revision ID: 0029
+Revises: 0028
+Create Date: 2026-10-18
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0029"
+down_revision: str | None = "0028"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE completion_phrases (
+          user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+          context text NOT NULL,
+          continuation text NOT NULL,
+          count integer NOT NULL,
+          PRIMARY KEY (user_id, context, continuation)
+        )
+        """
+    )
+    op.execute(
+        """
+        CREATE TABLE completion_index_state (
+          user_id uuid PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
+          built_at timestamptz NOT NULL
+        )
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS completion_index_state")
+    op.execute("DROP TABLE IF EXISTS completion_phrases")

backend/migrations/versions/0030_translation.py

@@ -0,0 +1,45 @@
+"""email translation: a message's detected language, and translations kept
+
+``messages.content_language``: NULL until the message is first shown, '' when
+the language could not be told. ``message_translations``: one translated HTML
+per message, target language and remote-images choice (app/translation.py);
+dropped with the message.
+
+Revision ID: 0030
+Revises: 0029
+Create Date: 2026-10-18
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0030"
+down_revision: str | None = "0029"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE messages ADD COLUMN content_language text")
+    op.execute(
+        """
+        CREATE TABLE message_translations (
+          message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
+          target text NOT NULL,
+          images boolean NOT NULL,
+          source text NOT NULL,
+          html text NOT NULL,
+          sanitizer_v integer NOT NULL,
+          created_at timestamptz NOT NULL DEFAULT now(),
+          PRIMARY KEY (message_id, target, images)
+        )
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS message_translations")
+    op.execute("ALTER TABLE messages DROP COLUMN IF EXISTS content_language")

backend/scripts/backfill_delivery_id.py

@@ -16,9 +16,7 @@ from app.models import Message
 
 async def main() -> None:
     async with session_scope() as session:
-        rows = list(
-            await session.scalars(select(Message).where(Message.delivery_id.is_(None)))
-        )
+        rows = list(await session.scalars(select(Message).where(Message.delivery_id.is_(None))))
         fixed = 0
         for m in rows:
             try:

backend/scripts/reindex_address_parts.py

@@ -42,9 +42,7 @@ async def main() -> None:
                 update(Message)
                 .where(Message.id == msg.id)
                 .values(
-                    body_indexed=Message.body_indexed.op("||")(
-                        func.to_tsvector(_TS_CONFIG, blob)
-                    )
+                    body_indexed=Message.body_indexed.op("||")(func.to_tsvector(_TS_CONFIG, blob))
                 )
             )
             updated += 1

backend/tests/accounts/test_discovery.py

@@ -37,10 +37,14 @@ AUTOCONFIG_XML = """<?xml version="1.0"?>
 def _skip_ssrf(monkeypatch) -> None:  # type: ignore[no-untyped-def]
     """Bypass the real DNS/public-IP check so respx can intercept the fetch."""
 
-    async def ok(_host: str) -> None:
-        return None
+    async def ok(_host: str) -> list[str]:
+        return ["93.184.216.34"]
+
+    async def unpinned(url: str) -> tuple[str, dict[str, str], dict[str, str]]:
+        return url, {}, {}  # respx matches on the host name, not the pinned IP
 
     monkeypatch.setattr(ssrf, "assert_host_is_public", ok)
+    monkeypatch.setattr(ssrf, "_pinned", unpinned)
 
 
 # --- XML parsing -------------------------------------------------------------

backend/tests/api/conftest.py

@@ -0,0 +1,149 @@
+"""A signed-in user with an API client, for endpoint tests in this package."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from email.message import EmailMessage
+
+import httpx
+import pytest_asyncio
+from httpx import ASGITransport
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+from app.identity.passwords import hash_password
+from app.identity.sessions import create_session
+from app.indexer.indexer import persist_message
+from app.main import create_app
+from app.models import Account, Mailbox, Message, User
+from app.providers.base import RemoteMessage
+
+CSRF = "csrf"
+HDR = {"X-CSRF": CSRF}
+
+
+@dataclass
+class UserCtx:
+    client: httpx.AsyncClient
+    user_id: uuid.UUID
+    session: AsyncSession  # for arranging rows directly
+
+
+@pytest_asyncio.fixture
+async def user_ctx() -> AsyncIterator[UserCtx]:
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    session = sm()
+    user = User(handle=f"t{uuid.uuid4().hex[:10]}@test.local", password_hash=hash_password("pw"))
+    session.add(user)
+    await session.commit()
+    user_id = user.id  # kept: a test may expire the session's objects
+    token = await create_session(user_id, "pytest")
+    client = httpx.AsyncClient(
+        transport=ASGITransport(app=create_app()),
+        base_url="http://testserver",
+        cookies={"session": token, "csrf": CSRF},
+    )
+    try:
+        yield UserCtx(client=client, user_id=user_id, session=session)
+    finally:
+        await client.aclose()
+        await session.rollback()
+        await session.execute(delete(User).where(User.id == user_id))
+        await session.commit()
+        await session.close()
+        await engine.dispose()
+
+
+@dataclass
+class MailCtx(UserCtx):
+    account_id: uuid.UUID
+    inbox_id: uuid.UUID
+    sent_id: uuid.UUID
+
+    async def deliver(
+        self,
+        raw: bytes,
+        uid: int,
+        *,
+        seen: bool = False,
+        at: dt.datetime | None = None,
+        folder: str = "inbox",
+    ) -> Message:
+        """Store a message the way sync does (indexer.persist_message), in the
+        Inbox or (``folder="sent"``) the Sent folder."""
+        remote = RemoteMessage(
+            uid=uid,
+            raw=raw,
+            flags=frozenset({"\\Seen"}) if seen else frozenset(),
+            internal_date=at or dt.datetime.now(dt.UTC),
+        )
+        result = await persist_message(
+            self.session,
+            user_id=self.user_id,
+            account_id=self.account_id,
+            mailbox_id=self.sent_id if folder == "sent" else self.inbox_id,
+            mailbox_special_use=folder,
+            remote=remote,
+        )
+        await self.session.commit()
+        assert result is not None
+        return result.message
+
+
+def raw_mail(
+    subject: str,
+    *,
+    sender: str = "News <news@shop.example>",
+    headers: dict[str, str] | None = None,
+    mid: str | None = None,
+    html: str | None = None,
+) -> bytes:
+    m = EmailMessage()
+    m["From"] = sender
+    m["To"] = "me@example.com"
+    m["Subject"] = subject
+    m["Message-ID"] = mid or f"<{uuid.uuid4().hex}@shop.example>"
+    for name, value in (headers or {}).items():
+        m[name] = value
+    m.set_content(f"{subject} body")
+    if html is not None:
+        m.add_alternative(html, subtype="html")
+    return m.as_bytes()
+
+
+@pytest_asyncio.fixture
+async def mail_ctx(user_ctx: UserCtx) -> MailCtx:
+    s = user_ctx.session
+    account = Account(
+        user_id=user_ctx.user_id,
+        email="me@example.com",
+        imap_config={},
+        smtp_config={},
+        secret_ct=b"",
+        discovery_src="manual",
+    )
+    s.add(account)
+    await s.flush()
+    inbox = Mailbox(
+        account_id=account.id, user_id=user_ctx.user_id, path="INBOX", special_use="inbox"
+    )
+    trash = Mailbox(
+        account_id=account.id, user_id=user_ctx.user_id, path="Trash", special_use="trash"
+    )
+    sent = Mailbox(account_id=account.id, user_id=user_ctx.user_id, path="Sent", special_use="sent")
+    s.add_all([inbox, trash, sent])
+    await s.commit()
+    return MailCtx(
+        client=user_ctx.client,
+        user_id=user_ctx.user_id,
+        session=s,
+        account_id=account.id,
+        inbox_id=inbox.id,
+        sent_id=sent.id,
+    )

backend/tests/api/test_admin_translation.py

@@ -0,0 +1,82 @@
+"""Email translation's languages in the admin panel (app/api/admin.py), with
+the translator service replaced by a stand-in."""
+
+from __future__ import annotations
+
+import pytest
+
+from app import translation
+from tests.api.test_admin_users import World, world  # noqa: F401 — the fixture
+
+
+@pytest.fixture
+def service(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[str]]:
+    state: dict[str, list[str]] = {"installed": ["en", "pl"], "calls": []}
+
+    async def status() -> dict[str, object]:
+        return {
+            "installed": list(state["installed"]),
+            "installing": {"de": {"done": 10, "total": 40}} if "de" in state["calls"] else {},
+            "available": ["en", "pl", "de", "hu"],
+            "errors": {},
+        }
+
+    async def install(code: str) -> None:
+        state["calls"].append(code)
+
+    async def remove(code: str) -> None:
+        state["installed"].remove(code)
+
+    monkeypatch.setattr(translation, "enabled", lambda: True)
+    monkeypatch.setattr(translation, "status", status)
+    monkeypatch.setattr(translation, "install", install)
+    monkeypatch.setattr(translation, "remove", remove)
+    return state
+
+
+async def test_admin_installs_and_removes_languages(
+    world: World,  # noqa: F811
+    service: dict[str, list[str]],
+) -> None:
+    async with await world.client(await world.user(admin=True)) as c:
+        got = (await c.get("/api/admin/translation")).json()
+        assert got["reachable"] and got["installed"] == ["en", "pl"]
+        assert got["available"] == ["en", "pl", "de", "hu"]
+
+        got = (await c.post("/api/admin/translation/languages/de")).json()
+        assert service["calls"] == ["de"]
+        assert got["installing"] == {"de": {"done": 10, "total": 40}}
+
+        got = (await c.delete("/api/admin/translation/languages/pl")).json()
+        assert got["installed"] == ["en"]
+
+        assert (await c.post("/api/admin/translation/languages/x;rm")).status_code == 422
+
+
+async def test_only_admins(world: World, service: dict[str, list[str]]) -> None:  # noqa: F811
+    async with await world.client(await world.user()) as c:
+        assert (await c.get("/api/admin/translation")).status_code == 403
+        assert (await c.post("/api/admin/translation/languages/de")).status_code == 403
+    assert service["calls"] == []
+
+
+async def test_unreachable_service_is_reported(
+    world: World,  # noqa: F811
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    async def down() -> dict[str, object]:
+        raise translation.TranslationError("down")
+
+    monkeypatch.setattr(translation, "enabled", lambda: True)
+    monkeypatch.setattr(translation, "status", down)
+    async with await world.client(await world.user(admin=True)) as c:
+        got = (await c.get("/api/admin/translation")).json()
+    assert got == {
+        "enabled": True,
+        "reachable": False,
+        "installed": [],
+        "installing": {},
+        "pending": [],
+        "available": [],
+        "errors": {},
+    }

backend/tests/api/test_admin_users.py

@@ -168,6 +168,20 @@ async def test_admin_cannot_block_or_delete_themselves(world: World) -> None:
         assert (await c.delete(f"/api/admin/users/{admin.id}")).status_code == 409
 
 
+async def test_admin_can_change_their_own_quota(world: World) -> None:
+    admin = await world.user(admin=True)
+    async with await world.client(admin) as c:
+        resp = await c.patch(f"/api/admin/users/{admin.id}", json={"quota_bytes": 5 * 2**30})
+        assert resp.status_code == 200
+        assert resp.json()["quota_bytes"] == 5 * 2**30
+        # Quota together with a block is still refused: the block is the problem.
+        assert (
+            await c.patch(
+                f"/api/admin/users/{admin.id}", json={"quota_bytes": 2**30, "blocked": True}
+            )
+        ).status_code == 409
+
+
 async def test_delete_removes_the_user_and_their_data(world: World, bus: _Bus) -> None:
     admin = await world.user(admin=True)
     target = await world.user()

backend/tests/api/test_api.py

@@ -310,7 +310,8 @@ async def test_attachment_download_non_ascii_filename(api: ApiCtx) -> None:
     await api.session.commit()
 
     aid = next(
-        a["id"] for a in (await api.client.get(f"/api/messages/{mid}/attachments")).json()
+        a["id"]
+        for a in (await api.client.get(f"/api/messages/{mid}/attachments")).json()
         if a["filename"] == name
     )
     resp = await api.client.get(f"/api/messages/{mid}/attachments/{aid}")
@@ -390,7 +391,7 @@ async def test_folder_counts(api: ApiCtx) -> None:
     # All three seeded inbox messages are unread; no drafts yet.
     resp = await api.client.get("/api/mailboxes/counts")
     assert resp.status_code == 200
-    assert resp.json() == {"inbox_unread": 3, "drafts": 0}
+    assert resp.json() == {"inbox_unread": 3, "drafts": 0, "snoozed": 0}
 
     # Marking one message \Seen lowers the unread count.
     msg = await api.session.get(Message, api.message_ids[0])
@@ -404,8 +405,11 @@ async def test_folder_counts(api: ApiCtx) -> None:
 async def test_thread_collapses_cross_account_duplicate(api: ApiCtx) -> None:
     assert api.session is not None
     # The kickoff thread's first message; give it a real delivery_id.
-    kickoff = next(t for t in (await api.client.get("/api/threads")).json()["threads"]
-                   if t["message_count"] == 2)
+    kickoff = next(
+        t
+        for t in (await api.client.get("/api/threads")).json()["threads"]
+        if t["message_count"] == 2
+    )
     detail = (await api.client.get(f"/api/threads/{kickoff['id']}")).json()
     first_id = uuid.UUID(detail["messages"][0]["id"])
     first = await api.session.get(Message, first_id)
@@ -414,20 +418,32 @@ async def test_thread_collapses_cross_account_duplicate(api: ApiCtx) -> None:
 
     # A second account receives the SAME email (same delivery_id) into the same thread.
     acct2 = Account(
-        user_id=api.user_id, email="me2@example.com", imap_config={}, smtp_config={},
-        secret_ct=b"", discovery_src="manual",
+        user_id=api.user_id,
+        email="me2@example.com",
+        imap_config={},
+        smtp_config={},
+        secret_ct=b"",
+        discovery_src="manual",
     )
     api.session.add(acct2)
     await api.session.flush()
     inbox2 = Mailbox(account_id=acct2.id, user_id=api.user_id, path="INBOX", special_use="inbox")
     api.session.add(inbox2)
     await api.session.flush()
-    api.session.add(Message(
-        user_id=api.user_id, account_id=acct2.id, mailbox_id=inbox2.id, uid=1,
-        thread_id=first.thread_id, message_id_hdr="<synthetic@synthetic.local>",
-        delivery_id="<shared@x.com>", subject=first.subject,
-        internal_date=datetime.now(UTC), blob_sha256="z" * 64,  # later -> first stays the rep
-    ))
+    api.session.add(
+        Message(
+            user_id=api.user_id,
+            account_id=acct2.id,
+            mailbox_id=inbox2.id,
+            uid=1,
+            thread_id=first.thread_id,
+            message_id_hdr="<synthetic@synthetic.local>",
+            delivery_id="<shared@x.com>",
+            subject=first.subject,
+            internal_date=datetime.now(UTC),
+            blob_sha256="z" * 64,  # later -> first stays the rep
+        )
+    )
     await api.session.commit()
 
     body = (await api.client.get(f"/api/threads/{kickoff['id']}")).json()

backend/tests/api/test_completion_api.py

@@ -0,0 +1,41 @@
+"""Sentence completion over the API, from the user's sent mail."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from app.api import completion as api_completion
+from app.completion import build_index_job
+from tests.api.conftest import HDR, MailCtx, raw_mail
+
+
+async def _suggest(ctx: MailCtx, before: str) -> str | None:
+    resp = await ctx.client.post("/api/completion", json={"before": before}, headers=HDR)
+    assert resp.status_code == 200, resp.text
+    return resp.json()["suggestion"]
+
+
+async def test_suggests_from_sent_mail(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None:
+    queued: list[tuple[Any, ...]] = []
+
+    async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None:
+        queued.append((name, *args))
+
+    monkeypatch.setattr(api_completion, "enqueue_job", fake_enqueue)
+    body = "Hi Anna,\nThe renders will be ready by Thursday evening.\n\nOn Mon Anna wrote:\n> ok"
+    for uid in (1, 2):
+        await mail_ctx.deliver(
+            raw_mail("Renders", sender="me@example.com").replace(b"Renders body", body.encode()),
+            uid,
+            folder="sent",
+        )
+
+    # No index yet: a build is asked for, and the starter phrases still answer.
+    assert await _suggest(mail_ctx, "I look forward ") == "to hearing from you."
+    assert queued and queued[0][0] == "build_index_job"
+
+    assert await build_index_job({}, str(mail_ctx.user_id)) > 0
+    assert await _suggest(mail_ctx, "The renders will be re") == "ady by Thursday evening."
+    assert await _suggest(mail_ctx, "> ok") is None  # quoted text was not learned

backend/tests/api/test_saved_searches.py

@@ -104,9 +104,7 @@ async def test_saved_search_crud_and_upsert(ctx: Ctx) -> None:
 
 async def test_saved_search_requires_csrf(ctx: Ctx) -> None:
     # No CSRF header -> rejected (state-changing route).
-    resp = await ctx.client.post(
-        "/api/saved-searches", json={"name": "x", "query": "y"}
-    )
+    resp = await ctx.client.post("/api/saved-searches", json={"name": "x", "query": "y"})
     assert resp.status_code == 403
 
 

backend/tests/api/test_snooze.py

@@ -0,0 +1,80 @@
+"""Snooze: a conversation leaves the Inbox until a set time, then comes back."""
+
+from __future__ import annotations
+
+import datetime as dt
+
+from sqlalchemy import select
+
+from app.models import Message, Thread
+from app.snooze import wake_due_job
+from tests.api.conftest import HDR, MailCtx, raw_mail
+
+
+def _in(days: float = 1) -> str:
+    return (dt.datetime.now(dt.UTC) + dt.timedelta(days=days)).isoformat()
+
+
+async def _inbox_ids(ctx: MailCtx) -> list[str]:
+    resp = await ctx.client.get("/api/threads", params={"view": "folder:inbox"})
+    return [t["id"] for t in resp.json()["threads"]]
+
+
+async def test_snoozed_thread_leaves_the_inbox(mail_ctx: MailCtx) -> None:
+    old = await mail_ctx.deliver(
+        raw_mail("Old"), 1, at=dt.datetime.now(dt.UTC) - dt.timedelta(days=2)
+    )
+    new = await mail_ctx.deliver(raw_mail("New"), 2)
+    c = mail_ctx.client
+    resp = await c.post(f"/api/threads/{new.thread_id}/snooze", json={"until": _in()}, headers=HDR)
+    assert resp.status_code == 200, resp.text
+
+    assert await _inbox_ids(mail_ctx) == [str(old.thread_id)]
+    snoozed = (await c.get("/api/threads", params={"view": "snoozed"})).json()["threads"]
+    assert [t["id"] for t in snoozed] == [str(new.thread_id)]
+    assert snoozed[0]["snoozed_until"] is not None
+    counts = (await c.get("/api/mailboxes/counts")).json()
+    assert counts["snoozed"] == 1 and counts["inbox_unread"] == 1
+
+    # Unsnooze: straight back, at the top.
+    await c.post(f"/api/threads/{new.thread_id}/unsnooze", headers=HDR)
+    assert await _inbox_ids(mail_ctx) == [str(new.thread_id), str(old.thread_id)]
+
+
+async def test_snooze_needs_a_future_time(mail_ctx: MailCtx) -> None:
+    msg = await mail_ctx.deliver(raw_mail("x"), 1)
+    for until in (_in(-1), _in(400), "2030-01-01T09:00:00"):
+        resp = await mail_ctx.client.post(
+            f"/api/threads/{msg.thread_id}/snooze", json={"until": until}, headers=HDR
+        )
+        assert resp.status_code == 400, until
+
+
+async def test_due_thread_wakes_on_top_and_unread(mail_ctx: MailCtx) -> None:
+    week_ago = dt.datetime.now(dt.UTC) - dt.timedelta(days=7)
+    snoozed = await mail_ctx.deliver(raw_mail("Snoozed"), 1, seen=True, at=week_ago)
+    newer = await mail_ctx.deliver(raw_mail("Newer"), 2, seen=True)
+    thread = await mail_ctx.session.get(Thread, snoozed.thread_id)
+    assert thread is not None
+    thread.snoozed_until = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=1)
+    thread.snoozed_at = week_ago
+    await mail_ctx.session.commit()
+
+    assert await wake_due_job({}) >= 1
+    assert await _inbox_ids(mail_ctx) == [str(snoozed.thread_id), str(newer.thread_id)]
+    snoozed_id = snoozed.id
+    mail_ctx.session.expire_all()
+    flags = await mail_ctx.session.scalar(select(Message.flags).where(Message.id == snoozed_id))
+    assert flags is not None and "\\Seen" not in flags
+
+
+async def test_a_reply_wakes_a_snoozed_thread(mail_ctx: MailCtx) -> None:
+    first = await mail_ctx.deliver(raw_mail("Question", mid="<q@x>"), 1)
+    await mail_ctx.client.post(
+        f"/api/threads/{first.thread_id}/snooze", json={"until": _in(3)}, headers=HDR
+    )
+    m = raw_mail("Re: Question", headers={"In-Reply-To": "<q@x>", "References": "<q@x>"})
+    await mail_ctx.deliver(m, 2, at=dt.datetime.now(dt.UTC) + dt.timedelta(seconds=5))
+
+    await wake_due_job({})
+    assert await _inbox_ids(mail_ctx) == [str(first.thread_id)]

backend/tests/api/test_subscriptions.py

@@ -0,0 +1,239 @@
+"""Subscriptions: bulk senders worked out from List-* headers, and unsubscribing."""
+
+from __future__ import annotations
+
+import datetime as dt
+from typing import Any
+
+import pytest
+from sqlalchemy import select, update
+
+from app.api import subscriptions as api_subs
+from app.models import Draft, Mailbox, Message
+from app.subscriptions import backfill_list_headers_job, read_list_headers, unsubscribe_targets
+from tests.api.conftest import HDR, MailCtx, raw_mail
+
+NEWS = {
+    "List-Id": '"Weekly news" <weekly.shop.example>',
+    "List-Unsubscribe": "<https://shop.example/u/1>, <mailto:leave@shop.example?subject=stop>",
+    "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
+}
+
+
+def test_read_list_headers() -> None:
+    h = read_list_headers(raw_mail("x", headers=NEWS))
+    assert h.list_id == "weekly.shop.example"
+    assert h.unsubscribe.startswith("<https://shop.example/u/1>")
+    assert h.one_click is True
+    plain = read_list_headers(raw_mail("hi"))
+    assert (plain.list_id, plain.unsubscribe, plain.one_click) == (None, "", False)
+
+
+def test_unsubscribe_targets() -> None:
+    t = unsubscribe_targets(NEWS["List-Unsubscribe"])
+    assert t.https == "https://shop.example/u/1"
+    assert t.mailto == "leave@shop.example" and t.mailto_subject == "stop"
+    assert unsubscribe_targets("<http://insecure.example/u>").https is None
+    assert unsubscribe_targets("<mailto:a@b.c%0d%0aBcc:x@y.z>").mailto is None
+
+
+async def _list(ctx: MailCtx) -> list[dict[str, Any]]:
+    resp = await ctx.client.get("/api/subscriptions")
+    assert resp.status_code == 200, resp.text
+    return list(resp.json()["subscriptions"])
+
+
+async def test_groups_by_list_then_sender(mail_ctx: MailCtx) -> None:
+    await mail_ctx.deliver(raw_mail("Issue 1", headers=NEWS), 1, seen=True)
+    await mail_ctx.deliver(raw_mail("Issue 2", headers=NEWS), 2)
+    await mail_ctx.deliver(
+        raw_mail(
+            "Sale",
+            sender="Deals <deals@other.example>",
+            headers={"List-Unsubscribe": "<mailto:u@other.example>"},
+        ),
+        3,
+    )
+    await mail_ctx.deliver(raw_mail("Hi from a friend", sender="ann@friend.example"), 4)
+
+    subs = {s["key"]: s for s in await _list(mail_ctx)}
+    assert set(subs) == {"list:weekly.shop.example", "from:deals@other.example"}
+    news = subs["list:weekly.shop.example"]
+    assert (news["total"], news["unread"], news["name"]) == (2, 1, "News")
+    assert news["one_click"] and news["has_mailto"] and news["has_web"]
+    assert news["last_subject"] == "Issue 2"
+    threads = await mail_ctx.client.get(
+        "/api/subscriptions/threads", params={"key": "list:weekly.shop.example"}
+    )
+    assert len(threads.json()) == 2
+
+
+async def test_one_click_unsubscribe(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None:
+    await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1)
+    posted: list[str] = []
+
+    async def fake_one_click(url: str) -> bool:
+        posted.append(url)
+        return True
+
+    monkeypatch.setattr(api_subs, "_one_click", fake_one_click)
+    resp = await mail_ctx.client.post(
+        "/api/subscriptions/unsubscribe", json={"key": "list:weekly.shop.example"}, headers=HDR
+    )
+    assert resp.json() == {"status": "done", "method": "one_click", "url": None}
+    assert posted == ["https://shop.example/u/1"]
+    (news,) = await _list(mail_ctx)
+    assert news["unsubscribed_at"] is not None and news["still_sending"] is False
+
+
+async def test_mailto_unsubscribe_and_still_sending(
+    mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    enqueued: list[tuple[Any, ...]] = []
+
+    async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None:
+        enqueued.append((name, *args))
+
+    monkeypatch.setattr(api_subs, "enqueue_job", fake_enqueue)
+    headers = {"List-Unsubscribe": "<mailto:leave@shop.example?subject=stop>"}
+    await mail_ctx.deliver(raw_mail("Issue", headers=headers), 1)
+    resp = await mail_ctx.client.post(
+        "/api/subscriptions/unsubscribe", json={"key": "from:news@shop.example"}, headers=HDR
+    )
+    assert resp.json()["method"] == "mailto"
+    assert [e[0] for e in enqueued] == ["send_job"]
+    draft = await mail_ctx.session.scalar(select(Draft).where(Draft.user_id == mail_ctx.user_id))
+    assert draft is not None
+    assert draft.payload["to"] == ["leave@shop.example"] and draft.payload["subject"] == "stop"
+
+    # Mail still arriving after the grace period is flagged.
+    later = dt.datetime.now(dt.UTC) + dt.timedelta(days=3)
+    await mail_ctx.deliver(raw_mail("Issue again", headers=headers), 2, at=later)
+    (sub,) = await _list(mail_ctx)
+    assert sub["still_sending"] is True
+
+
+async def test_web_only_unsubscribe_hands_back_the_page(mail_ctx: MailCtx) -> None:
+    headers = {"List-Unsubscribe": "<https://shop.example/prefs>"}
+    await mail_ctx.deliver(raw_mail("Issue", headers=headers), 1)
+    resp = await mail_ctx.client.post(
+        "/api/subscriptions/unsubscribe", json={"key": "from:news@shop.example"}, headers=HDR
+    )
+    assert resp.json() == {"status": "open", "method": "web", "url": "https://shop.example/prefs"}
+
+
+async def test_backfill_reads_older_mail(mail_ctx: MailCtx) -> None:
+    msg = await mail_ctx.deliver(raw_mail("Old issue", headers=NEWS), 1)
+    await mail_ctx.session.execute(
+        update(Message).where(Message.id == msg.id).values(list_unsubscribe=None, list_id=None)
+    )
+    await mail_ctx.session.commit()
+    resp = await mail_ctx.client.get("/api/subscriptions")
+    assert resp.json()["pending"] == 1
+
+    assert await backfill_list_headers_job({}, str(mail_ctx.user_id)) == 1
+    resp = await mail_ctx.client.get("/api/subscriptions")
+    assert resp.json()["pending"] == 0
+    assert [s["key"] for s in resp.json()["subscriptions"]] == ["list:weekly.shop.example"]
+
+
+async def test_trash_all_leaves_mixed_conversations(
+    mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    moved: list[str] = []
+
+    async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None:
+        moved.append(args[0])
+
+    monkeypatch.setattr(api_subs, "enqueue_job", fake_enqueue)
+    only = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS, mid="<i1@shop>"), 1)
+    await mail_ctx.deliver(raw_mail("Issue 2", headers=NEWS, mid="<i2@shop>"), 2)
+    reply = raw_mail(
+        "Re: Issue 2",
+        sender="me@example.com",
+        headers={"In-Reply-To": "<i2@shop>", "References": "<i2@shop>"},
+    )
+    await mail_ctx.deliver(reply, 3)
+    resp = await mail_ctx.client.post(
+        "/api/subscriptions/trash", json={"key": "list:weekly.shop.example"}, headers=HDR
+    )
+    assert resp.json() == {"moved": 1}
+    assert moved == [str(only.thread_id)]
+
+
+async def test_deleted_mail_is_left_out(mail_ctx: MailCtx) -> None:
+    msg = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1)
+    trash = await mail_ctx.session.scalar(
+        select(Mailbox.id).where(
+            Mailbox.account_id == mail_ctx.account_id, Mailbox.special_use == "trash"
+        )
+    )
+    await mail_ctx.session.execute(
+        update(Message).where(Message.id == msg.id).values(mailbox_id=trash)
+    )
+    await mail_ctx.session.commit()
+    assert await _list(mail_ctx) == []
+
+
+async def test_reading_pane_offers_unsubscribe(
+    mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    async def fake_one_click(url: str) -> bool:
+        return True
+
+    monkeypatch.setattr(api_subs, "_one_click", fake_one_click)
+    news = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1)
+    plain = await mail_ctx.deliver(raw_mail("Hi", sender="ann@friend.example"), 2)
+    c = mail_ctx.client
+
+    (env,) = (await c.get(f"/api/threads/{news.thread_id}")).json()["messages"]
+    assert env["subscription_key"] == "list:weekly.shop.example"
+    assert env["unsubscribe_via"] == "one_click" and env["unsubscribed"] is False
+    (other,) = (await c.get(f"/api/threads/{plain.thread_id}")).json()["messages"]
+    assert other["subscription_key"] is None and other["unsubscribe_via"] is None
+
+    resp = await c.post(
+        "/api/subscriptions/unsubscribe",
+        json={"key": env["subscription_key"], "message_id": env["id"]},
+        headers=HDR,
+    )
+    assert resp.json()["status"] == "done"
+    (env,) = (await c.get(f"/api/threads/{news.thread_id}")).json()["messages"]
+    assert env["unsubscribed"] is True
+
+    # A message without the header can't be used.
+    resp = await c.post(
+        "/api/subscriptions/unsubscribe",
+        json={"key": "from:ann@friend.example", "message_id": str(plain.id)},
+        headers=HDR,
+    )
+    assert resp.status_code == 404
+
+
+async def test_senders_come_in_pages_newest_first(mail_ctx: MailCtx) -> None:
+    now = dt.datetime.now(dt.UTC)
+    for i, name in enumerate(["old", "mid", "new"]):
+        await mail_ctx.deliver(
+            raw_mail(
+                f"From {name}",
+                sender=f"{name}@lists.example",
+                headers={"List-Unsubscribe": f"<mailto:u@{name}.example>"},
+            ),
+            i + 1,
+            at=now - dt.timedelta(days=10 - i),
+        )
+    c = mail_ctx.client
+    first = (await c.get("/api/subscriptions", params={"limit": 2})).json()
+    assert first["total"] == 3
+    assert [s["email"] for s in first["subscriptions"]] == [
+        "new@lists.example",
+        "mid@lists.example",
+    ]
+    rest = (await c.get("/api/subscriptions", params={"limit": 2, "offset": 2})).json()
+    assert [s["email"] for s in rest["subscriptions"]] == ["old@lists.example"]
+    assert (await c.get("/api/subscriptions", params={"offset": 9})).json()["subscriptions"] == []
+    oldest = (await c.get("/api/subscriptions", params={"limit": 2, "order": "asc"})).json()
+    assert [s["email"] for s in oldest["subscriptions"]] == [
+        "old@lists.example",
+        "mid@lists.example",
+    ]

backend/tests/api/test_templates.py

@@ -0,0 +1,54 @@
+"""Templates: saved replies from the compose window's More menu."""
+
+from __future__ import annotations
+
+from tests.api.conftest import HDR, UserCtx
+
+
+async def test_save_list_replace_delete(user_ctx: UserCtx) -> None:
+    c = user_ctx.client
+    assert (await c.get("/api/templates")).json() == []
+
+    resp = await c.post(
+        "/api/templates",
+        json={"name": " Thanks ", "subject": "Re: offer", "body_html": "<p>Thank <b>you</b></p>"},
+        headers=HDR,
+    )
+    assert resp.status_code == 200, resp.text
+    first = resp.json()
+    assert first["name"] == "Thanks" and first["subject"] == "Re: offer"
+
+    # Same name, any case: replaces rather than adding a second one.
+    resp = await c.post(
+        "/api/templates", json={"name": "thanks", "body_html": "<p>Cheers</p>"}, headers=HDR
+    )
+    assert resp.json()["id"] == first["id"]
+    await c.post("/api/templates", json={"name": "Apology", "body_html": "x"}, headers=HDR)
+    listed = (await c.get("/api/templates")).json()
+    assert [t["name"] for t in listed] == ["Apology", "thanks"]
+    assert listed[1]["body_html"] == "<p>Cheers</p>"
+
+    assert (await c.delete(f"/api/templates/{first['id']}", headers=HDR)).status_code == 200
+    assert [t["name"] for t in (await c.get("/api/templates")).json()] == ["Apology"]
+
+
+async def test_template_html_is_cleaned(user_ctx: UserCtx) -> None:
+    resp = await user_ctx.client.post(
+        "/api/templates",
+        json={
+            "name": "x",
+            "body_html": '<p onclick="evil()">Hi<script>alert(1)</script>'
+            '<img src="x" onerror="evil()"><a href="javascript:evil()">l</a></p>',
+        },
+        headers=HDR,
+    )
+    html = resp.json()["body_html"]
+    assert "script" not in html and "onclick" not in html and "onerror" not in html
+    assert "javascript:" not in html and "Hi" in html
+
+
+async def test_template_needs_a_name(user_ctx: UserCtx) -> None:
+    resp = await user_ctx.client.post(
+        "/api/templates", json={"name": "   ", "body_html": "x"}, headers=HDR
+    )
+    assert resp.status_code == 400

backend/tests/api/test_trackers.py

@@ -0,0 +1,45 @@
+"""Tracker blocking (Settings → Privacy): on by default, and it can be switched off."""
+
+from __future__ import annotations
+
+from tests.api.conftest import HDR, MailCtx, raw_mail
+
+PIXEL = "https://t.example.com/open.gif?u=42"
+HTML = f'<p>Hello</p><img src="{PIXEL}" width="1" height="1">'
+
+
+async def _html(ctx: MailCtx, message_id: object, images: bool = False) -> tuple[str, str]:
+    resp = await ctx.client.get(
+        f"/api/messages/{message_id}/html", params={"images": "1"} if images else {}
+    )
+    assert resp.status_code == 200
+    return resp.text, resp.headers["X-Trackers-Blocked"]
+
+
+async def test_trackers_blocked_by_default(mail_ctx: MailCtx) -> None:
+    msg = await mail_ctx.deliver(raw_mail("News", html=HTML), 1)
+    assert (await mail_ctx.client.get("/api/settings")).json()["block_trackers"] is True
+    body, count = await _html(mail_ctx, msg.id, images=True)
+    assert count == "1" and PIXEL not in body and "Hello" in body
+    (env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"]
+    assert env["has_remote_images"] is False  # nothing left worth loading
+
+
+async def test_switched_off_trackers_are_ordinary_remote_images(mail_ctx: MailCtx) -> None:
+    msg = await mail_ctx.deliver(raw_mail("News", html=HTML), 1)
+    resp = await mail_ctx.client.patch("/api/settings", json={"block_trackers": False}, headers=HDR)
+    assert resp.json()["block_trackers"] is False
+
+    # Still a remote image: not loaded until images are allowed, but offered.
+    body, count = await _html(mail_ctx, msg.id)
+    assert count == "0" and PIXEL not in body
+    (env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"]
+    assert env["has_remote_images"] is True
+
+    body, count = await _html(mail_ctx, msg.id, images=True)
+    assert count == "0" and PIXEL in body.replace("&amp;", "&")
+
+    # Back on: gone again.
+    await mail_ctx.client.patch("/api/settings", json={"block_trackers": True}, headers=HDR)
+    body, count = await _html(mail_ctx, msg.id, images=True)
+    assert count == "1" and PIXEL not in body

backend/tests/api/test_translation.py

@@ -0,0 +1,86 @@
+"""Email translation over the API, with LibreTranslate replaced by a stand-in."""
+
+from __future__ import annotations
+
+import pytest
+
+from app import translation
+from tests.api.conftest import MailCtx, raw_mail
+
+TEXT = "Hello Lukas, I would like to ask about the project schedule for next month."
+
+
+@pytest.fixture
+def fake_service(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
+    calls: dict[str, object] = {"detect": 0, "translate": 0}
+
+    async def detect(text: str) -> str:
+        calls["detect"] = int(calls["detect"]) + 1  # type: ignore[call-overload]
+        return "en"
+
+    async def translate_html(
+        html: str, source: str, target: str, keep: object = (), names: object = ()
+    ) -> str:
+        calls["translate"] = int(calls["translate"]) + 1  # type: ignore[call-overload]
+        calls["keep"] = list(keep)  # type: ignore[call-overload]
+        calls["names"] = list(names)  # type: ignore[call-overload]
+        assert source == "en" and target == "pl"
+        return "<p>Cześć Łukasz</p><script>alert(1)</script>"
+
+    monkeypatch.setattr(translation, "enabled", lambda: True)
+    monkeypatch.setattr(translation, "detect", detect)
+    monkeypatch.setattr(translation, "translate_html", translate_html)
+    return calls
+
+
+async def test_language_is_found_once_and_kept(
+    mail_ctx: MailCtx, fake_service: dict[str, object]
+) -> None:
+    msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
+    for _ in range(2):
+        resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html")
+        assert resp.headers["X-Content-Language"] == "en"
+    assert fake_service["detect"] == 1
+    (env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"]
+    assert env["content_language"] == "en"
+
+
+async def test_translation_is_cleaned_and_kept(
+    mail_ctx: MailCtx, fake_service: dict[str, object]
+) -> None:
+    msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
+    for _ in range(2):
+        resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html", params={"translate": "pl"})
+        assert resp.status_code == 200
+        assert "Cześć Łukasz" in resp.text and "alert(1)" not in resp.text
+        assert resp.headers["X-Translated-From"] == "en"
+        assert resp.headers["X-Translated-To"] == "pl"
+    assert fake_service["translate"] == 1  # the second came from the cache
+    # The people on the message are names to keep if the model would change them.
+    assert "News" in fake_service["names"]  # type: ignore[operator]
+
+
+async def test_service_trouble_is_a_clear_error(
+    mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch, fake_service: dict[str, object]
+) -> None:
+    async def broken(
+        html: str, source: str, target: str, keep: object = (), names: object = ()
+    ) -> str:
+        raise translation.TranslationError("the translation service is unreachable")
+
+    monkeypatch.setattr(translation, "translate_html", broken)
+    msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
+    resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html", params={"translate": "pl"})
+    assert resp.status_code == 503
+    assert resp.json()["error"]["code"] == "translation_failed"
+    # The message itself still shows.
+    assert (await mail_ctx.client.get(f"/api/messages/{msg.id}/html")).status_code == 200
+
+
+async def test_off_without_a_service(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(translation, "enabled", lambda: False)
+    msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
+    resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html")
+    assert "X-Content-Language" not in resp.headers
+    info = (await mail_ctx.client.get("/api/translation/languages")).json()
+    assert info == {"available": False, "languages": []}

backend/tests/conftest.py

@@ -18,6 +18,21 @@ os.environ.setdefault(
 os.environ["REDIS_URL"] = "redis://127.0.0.1:6380/15"
 os.environ.setdefault("MASTER_KEY", base64.b64encode(b"\x00" * 32).decode())
 os.environ.setdefault("BLOB_ROOT", "./var/blobs-test")
+# Hosted mailboxes off: tests that need the mail server turn them on themselves
+# (tests/hosted/), against the dev stack's Stalwart container.
+os.environ["HOSTED_MAILBOXES"] = "false"
+# No calls to the breached-password service from tests (app/identity/password_policy.py).
+os.environ["PASSWORD_BREACH_CHECK"] = "false"
+for _key, _value in {
+    "STALWART_URL": "http://127.0.0.1:8180",
+    "STALWART_ADMIN_SECRET": "stalwart_admin_dev",
+    "STALWART_MASTER_SECRET": "stalwart_master_dev",
+    "STALWART_IMAP_HOST": "127.0.0.1",
+    "STALWART_IMAP_PORT": "1993",
+    "STALWART_SMTP_HOST": "127.0.0.1",
+    "STALWART_SMTP_PORT": "1465",
+}.items():
+    os.environ.setdefault(_key, _value)
 
 from collections.abc import AsyncIterator  # noqa: E402