Productivity features, UI translations, local email translation

- Snooze (with a Snoozed folder), keyboard shortcuts with a Settings
  section, mail templates (Compose > More > Templates), a Subscriptions
  folder with bulk unsubscribe/delete, and tracker blocking with a
  Privacy toggle (on by default)
- Own dialogs instead of browser popups; emoji picker in compose; compose
  windows survive a visit to Settings
- Sentence completion learned from the user's sent mail (no AI)
- Interface in English and Polish (one file per language, English as the
  fallback), picked from the browser on first run, then kept
- Email translation on the server via LibreTranslate: Translate button
  and "Translate to...", messages in several languages translated part
  by part, names/addresses/emoji/file names/phone numbers kept, an
  animated line while translating
- Antispam "Mark as spam" loads with the message, not when the menu opens

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/api/__init__.py

@@ -15,6 +15,7 @@ def register_routers(app: FastAPI) -> None:
     from app.api.actions import router as actions_router
     from app.api.admin import router as admin_router
     from app.api.avatar import router as avatar_router
+    from app.api.completion import router as completion_router
     from app.api.compose import router as compose_router
     from app.api.invitations import router as invitations_router
     from app.api.mailboxes import router as mailboxes_router
@@ -25,8 +26,12 @@ def register_routers(app: FastAPI) -> None:
     from app.api.saved_searches import router as saved_searches_router
     from app.api.search import router as search_router
     from app.api.settings import router as settings_router
+    from app.api.snooze import router as snooze_router
     from app.api.spellcheck import router as spellcheck_router
+    from app.api.subscriptions import router as subscriptions_router
+    from app.api.templates import router as templates_router
     from app.api.threads import router as threads_router
+    from app.api.translation import router as translation_router
     from app.identity.routes import router as auth_router
     from app.ws import router as ws_router
 
@@ -48,6 +53,11 @@ def register_routers(app: FastAPI) -> None:
         spellcheck_router,
         push_router,
         invitations_router,
+        templates_router,
+        snooze_router,
+        subscriptions_router,
+        completion_router,
+        translation_router,
     ):
         app.include_router(router, prefix="/api")
     app.include_router(ws_router, prefix="/api")

backend/app/api/completion.py

@@ -0,0 +1,44 @@
+"""Sentence completion for the compose window (app/completion.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import completion
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import CompletionIndexState
+from app.queue import enqueue_job
+
+router = APIRouter(prefix="/completion", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class CompletionIn(BaseModel):
+    # The text before the cursor (its last line is all that matters). A POST,
+    # not a query string: what the user is typing stays out of access logs.
+    before: str = Field(max_length=2000)
+
+
+class CompletionOut(BaseModel):
+    suggestion: str | None
+
+
+@router.post("", response_model=CompletionOut)
+async def suggest(body: CompletionIn, user: CurrentUser, db: DbSession) -> CompletionOut:
+    before = body.before[-300:]
+    built = await db.scalar(
+        select(CompletionIndexState.built_at).where(CompletionIndexState.user_id == user.id)
+    )
+    if built is None or built < dt.datetime.now(dt.UTC) - completion.REBUILD_AFTER:
+        # In the background; until it is built, the starter phrases still answer.
+        await enqueue_job("build_index_job", str(user.id), _job_id=f"complete:{user.id}")
+    personal = await completion.personal_phrases(db, user.id, completion.keys_for(before))
+    return CompletionOut(suggestion=completion.complete(personal, before))

backend/app/api/mailboxes.py

@@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Draft, Mailbox, Message
+from app.models import Draft, Mailbox, Message, Thread
 from app.outbox.service import not_being_sent
 
 router = APIRouter(prefix="/mailboxes", tags=["mail"])
@@ -30,6 +30,7 @@ class MailboxOut(BaseModel):
 class FolderCounts(BaseModel):
     inbox_unread: int  # unread messages sitting in Inbox mailboxes
     drafts: int  # drafts shown in the Drafts folder (local + un-adopted remote)
+    snoozed: int = 0  # conversations snoozed out of the Inbox
 
 
 @router.get("", response_model=list[MailboxOut])
@@ -54,8 +55,17 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
             Message.user_id == user.id,
             Mailbox.special_use == "inbox",
             ~Message.flags.contains(["\\Seen"]),
+            # A snoozed conversation is out of the Inbox until it wakes.
+            ~Message.thread_id.in_(
+                select(Thread.id).where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+            ),
         )
     )
+    snoozed = await db.scalar(
+        select(func.count())
+        .select_from(Thread)
+        .where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None))
+    )
 
     # Drafts folder = local drafts + IMAP-Drafts messages we haven't adopted (mirror
     # UIDs excluded), matching what list_drafts + list_remote_drafts return.
@@ -77,4 +87,8 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts:
     )
     remote = sum(1 for r in remote_rows if (r.account_id, r.uid) not in owned)
 
-    return FolderCounts(inbox_unread=int(inbox_unread or 0), drafts=int(local or 0) + remote)
+    return FolderCounts(
+        inbox_unread=int(inbox_unread or 0),
+        drafts=int(local or 0) + remote,
+        snoozed=int(snoozed or 0),
+    )

backend/app/api/messages.py

@@ -6,6 +6,7 @@ import asyncio
 import contextlib
 import html as html_lib
 import io
+import logging
 import uuid
 import zipfile
 from typing import Annotated
@@ -14,11 +15,18 @@ from urllib.parse import quote
 from fastapi import APIRouter, Depends, Query
 from fastapi.responses import FileResponse, HTMLResponse, Response
 from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app import translation
 from app.api.errors import raise_api_error
 from app.api.schemas import AttachmentOut
-from app.api.settings import add_allowed_image_sender, allowed_image_senders
+from app.api.settings import (
+    add_allowed_image_sender,
+    allowed_image_senders,
+    blocks_trackers,
+)
+from app.completion import html_to_text
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
 from app.indexer import attachments as att_store
@@ -26,7 +34,18 @@ from app.indexer import rawstore, thumbs
 from app.indexer.mime import parse_message
 from app.indexer.sanitize import SANITIZER_VERSION, sanitize_html
 from app.indexer.thumbs import thumb_type
-from app.models import Attachment, BodyPart, ComposeAttachment, Message
+from app.models import (
+    Account,
+    Attachment,
+    BodyPart,
+    ComposeAttachment,
+    Message,
+    MessageTranslation,
+    User,
+)
+from app.translation_prep import find_names_in_signature
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/messages", tags=["mail"])
 
@@ -101,8 +120,16 @@ _MESSAGE_CSP = (
 )
 
 
-def _html(document: str) -> HTMLResponse:
-    return HTMLResponse(content=document, headers={"Content-Security-Policy": _MESSAGE_CSP})
+def _html(document: str, trackers: int = 0, extra: dict[str, str] | None = None) -> HTMLResponse:
+    # The page reads what it needs from headers: the body is the message.
+    return HTMLResponse(
+        content=document,
+        headers={
+            "Content-Security-Policy": _MESSAGE_CSP,
+            "X-Trackers-Blocked": str(trackers),
+            **(extra or {}),
+        },
+    )
 
 
 async def _owned_message(db: AsyncSession, user_id: uuid.UUID, message_id: uuid.UUID) -> Message:
@@ -118,54 +145,167 @@ def _sender_email(message: Message) -> str:
     return message.from_addr[0].get("email", "").lower() if message.from_addr else ""
 
 
+async def _render_fragment(
+    db: AsyncSession, message: Message, allow_remote: bool, block: bool
+) -> tuple[str, int]:
+    """The sanitized body the user sees, and how many trackers came out of it.
+
+    Re-sanitizes from the raw blob when remote images are allowed, the stored
+    copy predates the current sanitizer (SPEC §10.4), or trackers are to stay
+    in; otherwise the stored copy is served as is.
+    """
+    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message.id)))
+    html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
+
+    stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
+    # The stored copy has its trackers taken out; with blocking switched off, a
+    # message that had some is rendered afresh with them left in.
+    trackers_kept = not block and html_part is not None and html_part.trackers_blocked > 0
+    if html_part is not None and not allow_remote and not stale and not trackers_kept:
+        return html_part.content, html_part.trackers_blocked if block else 0
+
+    raw = None
+    if html_part is not None or allow_remote:  # a plain-text message has nothing to redo
+        try:
+            raw = await rawstore.ensure_bytes(message)
+        except FileNotFoundError:
+            raw = None
+    parsed = parse_message(raw) if raw is not None else None
+    if parsed is not None and parsed.html_body is not None:
+        if stale and html_part is not None:
+            # The stored copy is always the plain one: no remote images, trackers out.
+            upgraded = sanitize_html(parsed.html_body, message.id)
+            html_part.content = upgraded.html
+            html_part.sanitizer_v = upgraded.sanitizer_v
+            html_part.trackers_blocked = upgraded.trackers_blocked
+            await db.commit()
+            if block and not allow_remote:
+                return upgraded.html, upgraded.trackers_blocked
+        result = sanitize_html(
+            parsed.html_body, message.id, allow_remote_images=allow_remote, block_trackers=block
+        )
+        return result.html, result.trackers_blocked
+
+    if html_part is not None:
+        return html_part.content, html_part.trackers_blocked
+    text_part = next((p for p in parts if p.kind == "text"), None)
+    body = html_lib.escape(text_part.content) if text_part else ""
+    return f'<div class="wm-plaintext">{body}</div>', 0
+
+
+async def _content_language(db: AsyncSession, message: Message) -> str:
+    """The message's language, found (and kept) the first time it is shown."""
+    if message.content_language is not None or not translation.enabled():
+        return message.content_language or ""
+    text = await db.scalar(
+        select(BodyPart.content).where(BodyPart.message_id == message.id, BodyPart.kind == "text")
+    )
+    try:
+        found = await translation.detect(text or message.snippet or "")
+    except translation.TranslationError as exc:
+        logger.info("language detection unavailable: %s", exc)
+        return ""  # not kept: tried again once the service is back
+    message.content_language = found
+    await db.commit()
+    return found
+
+
 @router.get("/{message_id}/html", response_class=HTMLResponse)
 async def message_html(
     message_id: uuid.UUID,
     user: CurrentUser,
     db: DbSession,
     images: bool = Query(False),
+    translate: str | None = Query(None, pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$"),
 ) -> HTMLResponse:
     """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
 
-    Re-sanitizes from the raw blob when the sender is image-approved, or when the
-    stored copy predates the current sanitizer version (SPEC §10.4). ``images=1``
-    is a one-time "Load images" — render remote images for this request only,
-    without persisting the sender as trusted.
+    ``images=1`` is a one-time "Load images" — render remote images for this
+    request only, without persisting the sender as trusted. ``translate=pl``
+    returns it translated (app/translation.py). Headers tell the page what it
+    got: X-Trackers-Blocked, X-Content-Language (the message's own language,
+    when known) and, translated, X-Translated-From / X-Translated-To.
     """
     message = await _owned_message(db, user.id, message_id)
     allow_remote = images or _sender_email(message) in allowed_image_senders(user)
+    block = blocks_trackers(user)
+    fragment, trackers = await _render_fragment(db, message, allow_remote, block)
 
-    parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
-    html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
+    headers: dict[str, str] = {}
+    language = await _content_language(db, message)
+    if language:
+        headers["X-Content-Language"] = language
+    if translate:
+        fragment, source = await _translated(
+            db, user, message, fragment, language, translate, allow_remote, block
+        )
+        headers["X-Translated-From"] = source
+        headers["X-Translated-To"] = translate
+    return _html(_wrap(fragment), trackers, headers)
 
-    stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
-    if html_part is not None and not allow_remote and not stale:
-        return _html(_wrap(html_part.content))
 
-    # Re-sanitize the original HTML (image-approved sender, or a stale stored copy).
-    # If the raw blob is unavailable, fall back to whatever is already stored.
-    if allow_remote or stale:
-        try:
-            raw = await rawstore.ensure_bytes(message)
-        except FileNotFoundError:
-            raw = None
-        if raw is not None:
-            parsed = parse_message(raw)
-            if parsed.html_body is not None:
-                result = sanitize_html(
-                    parsed.html_body, message.id, allow_remote_images=allow_remote
-                )
-                if html_part is not None and not allow_remote:
-                    html_part.content = result.html  # persist the upgraded sanitization
-                    html_part.sanitizer_v = result.sanitizer_v
-                    await db.commit()
-                return _html(_wrap(result.html))
+async def _names(db: AsyncSession, user: User, message: Message) -> list[str]:
+    """Names a translation leaves as they are if the model would change them:
+    the people on the message (and the sender's single names, as signed:
+    "Milan"), the user's own names (accounts, signatures)."""
+    names: list[str] = []
+    for people in (message.from_addr, message.to_addr, message.cc_addr):
+        names += [p.get("name", "") for p in people if p.get("name")]
+    for p in message.from_addr:
+        names += [w for w in str(p.get("name", "")).split() if len(w) >= 3 and w[0].isupper()]
+    rows = await db.execute(
+        select(Account.display_name, Account.signature).where(Account.user_id == user.id)
+    )
+    for display_name, signature in rows:
+        if display_name:
+            names.append(display_name)
+        if signature:
+            names += find_names_in_signature(html_to_text(signature))
+    return names
 
-    if html_part is not None:
-        return _html(_wrap(html_part.content))
-    text_part = next((p for p in parts if p.kind == "text"), None)
-    body = html_lib.escape(text_part.content) if text_part else ""
-    return _html(_wrap(f'<div class="wm-plaintext">{body}</div>'))
+
+async def _translated(
+    db: AsyncSession,
+    user: User,
+    message: Message,
+    fragment: str,
+    language: str,
+    target: str,
+    allow_remote: bool,
+    block: bool,
+) -> tuple[str, str]:
+    """The fragment in ``target`` — from the cache, or translated and kept."""
+    version = SANITIZER_VERSION * 1000 + translation.PIPELINE_VERSION
+    cached = await db.get(MessageTranslation, (message.id, target, allow_remote))
+    if cached is not None and cached.sanitizer_v == version:
+        return cached.html, cached.source
+    try:
+        out = await translation.translate_html(
+            fragment, language or "auto", target, names=await _names(db, user, message)
+        )
+    except translation.TranslationError as exc:
+        raise_api_error(503, "translation_failed", str(exc))
+    # The translation comes back as HTML: cleaned like any HTML we show.
+    clean = sanitize_html(
+        out, message.id, allow_remote_images=allow_remote, block_trackers=block
+    ).html
+    source = language or "auto"
+    stmt = insert(MessageTranslation).values(
+        message_id=message.id,
+        target=target,
+        images=allow_remote,
+        source=source,
+        html=clean,
+        sanitizer_v=version,
+    )
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["message_id", "target", "images"],
+            set_={"html": clean, "source": source, "sanitizer_v": version},
+        )
+    )
+    await db.commit()
+    return clean, source
 
 
 @router.post("/{message_id}/allow-remote-images", dependencies=[Depends(require_csrf)])

backend/app/api/schemas.py

@@ -27,6 +27,8 @@ class ThreadSummary(BaseModel):
     has_attachments: bool
     pinned: bool
     last_message_at: dt.datetime
+    # Set while the thread is snoozed: when it comes back to the Inbox.
+    snoozed_until: dt.datetime | None = None
 
 
 class ThreadListOut(BaseModel):
@@ -64,6 +66,13 @@ class MessageEnvelope(BaseModel):
     snippet: str
     # Other accounts this same delivery also landed in (collapsed duplicates).
     also_in_account_ids: list[uuid.UUID] = []
+    # Bulk mail: the Subscriptions group it belongs to, how it can be left
+    # (one_click | mailto | web), and whether the user already did.
+    subscription_key: str | None = None
+    unsubscribe_via: str | None = None
+    unsubscribed: bool = False
+    # Its language, once found (the first time it is shown): "" unknown.
+    content_language: str = ""
 
 
 class ThreadDetail(BaseModel):
@@ -72,6 +81,7 @@ class ThreadDetail(BaseModel):
     order: str  # "asc" | "desc"
     pinned: bool
     messages: list[MessageEnvelope]  # always ASC (SPEC §9.4)
+    snoozed_until: dt.datetime | None = None
 
 
 class FlagChangeRequest(BaseModel):

backend/app/api/settings.py

@@ -27,6 +27,7 @@ _ALLOWED_INVITE_FROM = {"contacts", "anyone"}
 
 # "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de".
 _LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*")
+_UI_LANGUAGE = re.compile(r"[a-z]{2,3}(-[A-Za-z]{2,4})?")
 
 
 class ReminderPref(BaseModel):
@@ -52,6 +53,16 @@ class SettingsOut(BaseModel):
     # ("auto" detects it per message).
     spellcheck_enabled: bool = True
     spellcheck_language: str = "auto"
+    # Gmail-style keyboard shortcuts in the mail view (Settings → Keyboard shortcuts).
+    shortcuts_enabled: bool = True
+    # Remove tracking pixels from mail (Settings → Privacy).
+    block_trackers: bool = True
+    # Grey sentence endings in the compose window, taken with Tab.
+    completion_enabled: bool = True
+    # Interface language ("en", "pl", …); "" until chosen — the web app then
+    # picks one from the browser on first sign-in and saves it here.
+    ui_language: str = ""
+    # Names and words email translation leaves as they are.
 
 
 class SettingsPatch(BaseModel):
@@ -68,6 +79,10 @@ class SettingsPatch(BaseModel):
     invitation_file: bool | None = None
     spellcheck_enabled: bool | None = None
     spellcheck_language: str | None = None
+    shortcuts_enabled: bool | None = None
+    block_trackers: bool | None = None
+    completion_enabled: bool | None = None
+    ui_language: str | None = None
 
 
 class QuotaOut(BaseModel):
@@ -82,6 +97,11 @@ def allowed_image_senders(user: User) -> set[str]:
     return set()
 
 
+def blocks_trackers(user: User) -> bool:
+    """Whether tracking pixels are taken out of this user's mail (on unless switched off)."""
+    return bool(user.settings.get("block_trackers", True))
+
+
 def add_allowed_image_sender(user: User, email: str) -> None:
     current = sorted(allowed_image_senders(user))
     if email.lower() not in current:
@@ -127,6 +147,10 @@ def _settings_out(user: User) -> SettingsOut:
         allowed_image_senders=sorted(allowed_image_senders(user)),
         spellcheck_enabled=bool(s.get("spellcheck_enabled", True)),
         spellcheck_language=str(s.get("spellcheck_language", "auto")),
+        shortcuts_enabled=bool(s.get("shortcuts_enabled", True)),
+        block_trackers=blocks_trackers(user),
+        completion_enabled=bool(s.get("completion_enabled", True)),
+        ui_language=str(s.get("ui_language", "")),
     )
 
 
@@ -175,6 +199,14 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession)
         updates["spellcheck_enabled"] = body.spellcheck_enabled
     if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language):
         updates["spellcheck_language"] = body.spellcheck_language
+    if body.shortcuts_enabled is not None:
+        updates["shortcuts_enabled"] = body.shortcuts_enabled
+    if body.block_trackers is not None:
+        updates["block_trackers"] = body.block_trackers
+    if body.completion_enabled is not None:
+        updates["completion_enabled"] = body.completion_enabled
+    if body.ui_language is not None and _UI_LANGUAGE.fullmatch(body.ui_language):
+        updates["ui_language"] = body.ui_language
     user.settings = updates
     await db.commit()
     return _settings_out(user)

backend/app/api/snooze.py

@@ -0,0 +1,76 @@
+"""Snooze endpoints: hide a conversation from the Inbox until a set time."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app import snooze
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.events import get_event_bus
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Thread
+
+router = APIRouter(prefix="/threads", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+
+class SnoozeIn(BaseModel):
+    until: dt.datetime
+
+
+class SnoozeOut(BaseModel):
+    thread_id: uuid.UUID
+    snoozed_until: dt.datetime | None
+
+
+async def _thread(db: AsyncSession, user_id: uuid.UUID, thread_id: uuid.UUID) -> Thread:
+    thread = await db.scalar(
+        select(Thread).where(Thread.id == thread_id, Thread.user_id == user_id)
+    )
+    if thread is None:
+        raise_api_error(404, "not_found", "thread not found")
+    return thread
+
+
+async def _announce(user_id: uuid.UUID, thread_id: uuid.UUID) -> None:
+    await get_event_bus().publish(
+        "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}
+    )
+
+
+@router.post("/{thread_id}/snooze", response_model=SnoozeOut)
+async def snooze_thread(
+    thread_id: uuid.UUID, body: SnoozeIn, user: CurrentUser, db: DbSession
+) -> SnoozeOut:
+    until = body.until
+    if until.tzinfo is None:
+        raise_api_error(400, "invalid_time", "the time needs a time zone")
+    now = dt.datetime.now(dt.UTC)
+    if until <= now:
+        raise_api_error(400, "invalid_time", "pick a time in the future")
+    if until > now + snooze.MAX_SNOOZE:
+        raise_api_error(400, "invalid_time", "snooze for at most a year")
+    thread = await _thread(db, user.id, thread_id)
+    snooze.snooze(thread, until, now)
+    await db.commit()
+    await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=thread.snoozed_until)
+
+
+@router.post("/{thread_id}/unsnooze", response_model=SnoozeOut)
+async def unsnooze_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -> SnoozeOut:
+    thread = await _thread(db, user.id, thread_id)
+    if thread.snoozed_until is not None:
+        snooze.wake(thread, dt.datetime.now(dt.UTC))
+        await db.commit()
+        await _announce(user.id, thread_id)
+    return SnoozeOut(thread_id=thread_id, snoozed_until=None)

backend/app/api/subscriptions.py

@@ -0,0 +1,372 @@
+"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py)."""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import case, func, literal, or_, select
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.elements import ColumnElement
+
+from app import subscriptions as subs
+from app.accounts.ssrf import guarded_request
+from app.api.errors import raise_api_error
+from app.config import get_settings
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import Mailbox, Message, SubscriptionState
+from app.outbox.schemas import DraftIn, DraftPayload
+from app.outbox.service import DraftError, create_outbox, upsert_draft
+from app.queue import enqueue_job
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+# RFC 8058 senders should stop within two days of a one-click request.
+GRACE = dt.timedelta(days=2)
+# Not counted: mail the user sent, and mail already thrown away (Trash, Spam) —
+# a sender whose mail was all deleted drops off the list.
+_LEFT_OUT = ("sent", "drafts", "trash", "junk")
+
+
+class SubscriptionOut(BaseModel):
+    key: str
+    name: str
+    email: str
+    list_id: str | None
+    total: int
+    unread: int
+    last_30_days: int
+    per_month: float
+    first_at: dt.datetime
+    last_at: dt.datetime
+    last_subject: str
+    one_click: bool
+    has_mailto: bool
+    has_web: bool
+    unsubscribed_at: dt.datetime | None
+    method: str | None
+    still_sending: bool
+
+
+class SubscriptionListOut(BaseModel):
+    subscriptions: list[SubscriptionOut]  # one page
+    total: int  # senders in all
+    pending: int  # messages whose headers are still being read
+
+
+class KeyIn(BaseModel):
+    key: str
+
+
+class UnsubscribeIn(KeyIn):
+    # From the reading pane: this message's own header, wherever it is filed
+    # (Spam included, which the page's groups leave out).
+    message_id: uuid.UUID | None = None
+
+
+class UnsubscribeOut(BaseModel):
+    status: str  # done | open
+    method: str  # one_click | mailto | web
+    url: str | None = None
+
+
+def _sender() -> ColumnElement[Any]:
+    return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, ""))
+
+
+def _key() -> ColumnElement[Any]:
+    return case(
+        (Message.list_id.isnot(None), literal("list:") + Message.list_id),
+        else_=literal("from:") + _sender(),
+    )
+
+
+def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]:
+    return [
+        Message.user_id == user_id,
+        Message.list_unsubscribe.isnot(None),
+        Message.list_unsubscribe != "",
+        or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)),
+    ]
+
+
+async def _start_backfill(user_id: uuid.UUID) -> None:
+    # One job per user at a time: arq refuses a second while the first exists.
+    await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}")
+
+
+@router.get("", response_model=SubscriptionListOut)
+async def list_subscriptions(
+    user: CurrentUser,
+    db: DbSession,
+    offset: int = Query(0, ge=0),
+    limit: int = Query(50, ge=1, le=200),
+    order: str = Query("desc"),  # desc: heard from most recently first | asc: longest ago
+) -> SubscriptionListOut:
+    """One page of senders, by when their latest mail came."""
+    pending = await subs.pending_count(db, user.id)
+    if pending:
+        await _start_backfill(user.id)
+
+    now = dt.datetime.now(dt.UTC)
+    key = _key().label("key")
+    seen = Message.flags.contains(["\\Seen"])
+    groups = (
+        select(
+            key,
+            func.count().label("total"),
+            func.count().filter(~seen).label("unread"),
+            func.count()
+            .filter(Message.internal_date > now - dt.timedelta(days=30))
+            .label("recent"),
+            func.min(Message.internal_date).label("first_at"),
+            func.max(Message.internal_date).label("last_at"),
+        )
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id))
+        .group_by(key)
+    )
+    total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0
+    agg_rows = (
+        await db.execute(
+            groups.order_by(
+                func.max(Message.internal_date).asc()
+                if order == "asc"
+                else func.max(Message.internal_date).desc(),
+                key,
+            )
+            .offset(offset)
+            .limit(limit)
+        )
+    ).all()
+    page_keys = [r.key for r in agg_rows]
+    if not page_keys:
+        return SubscriptionListOut(subscriptions=[], total=total, pending=pending)
+
+    in_page = _key().in_(page_keys)
+    latest_rows = (
+        await db.execute(
+            select(
+                key,
+                Message.from_addr,
+                Message.subject,
+                Message.list_id,
+                Message.list_unsubscribe,
+                Message.list_unsubscribe_post,
+            )
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(*_bulk(user.id), in_page)
+            .distinct(key)
+            .order_by(key, Message.internal_date.desc())
+        )
+    ).all()
+    latest = {r.key: r for r in latest_rows}
+    states = {
+        s.key: s
+        for s in await db.scalars(
+            select(SubscriptionState).where(
+                SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys)
+            )
+        )
+    }
+    # Senders still mailing after the grace period, for the whole page at once.
+    still_sending = set(
+        await db.scalars(
+            select(key)
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .join(
+                SubscriptionState,
+                (SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()),
+            )
+            .where(
+                *_bulk(user.id),
+                in_page,
+                Message.internal_date > SubscriptionState.unsubscribed_at + GRACE,
+            )
+            .distinct()
+        )
+    )
+
+    out: list[SubscriptionOut] = []
+    for row in agg_rows:
+        last = latest.get(row.key)
+        if last is None:
+            continue
+        sender = last.from_addr[0] if last.from_addr else {}
+        targets = subs.unsubscribe_targets(last.list_unsubscribe or "")
+        state = states.get(row.key)
+        months = max(1.0, (row.last_at - row.first_at).days / 30.44)
+        out.append(
+            SubscriptionOut(
+                key=row.key,
+                name=sender.get("name") or sender.get("email", ""),
+                email=sender.get("email", ""),
+                list_id=last.list_id,
+                total=row.total,
+                unread=row.unread,
+                last_30_days=row.recent,
+                per_month=round(row.total / months, 1),
+                first_at=row.first_at,
+                last_at=row.last_at,
+                last_subject=last.subject,
+                one_click=bool(last.list_unsubscribe_post and targets.https),
+                has_mailto=targets.mailto is not None,
+                has_web=targets.https is not None,
+                unsubscribed_at=state.unsubscribed_at if state else None,
+                method=state.method if state else None,
+                still_sending=row.key in still_sending,
+            )
+        )
+    return SubscriptionListOut(subscriptions=out, total=total, pending=pending)
+
+
+@router.get("/threads", response_model=list[uuid.UUID])
+async def subscription_threads(
+    user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200)
+) -> list[uuid.UUID]:
+    """The group's conversations, newest first (for the reading pane)."""
+    rows = await db.execute(
+        select(Message.thread_id, func.max(Message.internal_date).label("at"))
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None))
+        .group_by(Message.thread_id)
+        .order_by(func.max(Message.internal_date).desc())
+        .limit(limit)
+    )
+    return [r.thread_id for r in rows if r.thread_id is not None]
+
+
+async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message:
+    message = await db.scalar(
+        select(Message)
+        .join(Mailbox, Mailbox.id == Message.mailbox_id)
+        .where(*_bulk(user_id), _key() == key)
+        .order_by(Message.internal_date.desc())
+        .limit(1)
+    )
+    if message is None:
+        raise_api_error(404, "not_found", "no mail from this sender")
+    return message
+
+
+async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None:
+    stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method)
+    await db.execute(
+        stmt.on_conflict_do_update(
+            index_elements=["user_id", "key"],
+            set_={"method": method, "unsubscribed_at": func.now()},
+        )
+    )
+    await db.commit()
+
+
+async def _one_click(url: str) -> bool:
+    try:
+        resp = await guarded_request(
+            "POST",
+            url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            content="List-Unsubscribe=One-Click",
+            max_bytes=256 * 1024,
+        )
+    except Exception as exc:  # SSRF refusal, network, TLS: any failure tries the next way
+        logger.info("one-click unsubscribe failed: %s", exc)
+        return False
+    return 200 <= resp.status < 400
+
+
+async def _send_mailto(
+    db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets
+) -> bool:
+    assert targets.mailto is not None
+    body = DraftIn(
+        account_id=message.account_id,
+        kind="new",
+        payload=DraftPayload(
+            to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body
+        ),
+    )
+    try:
+        draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None)
+    except DraftError:
+        return False
+    outbox = await create_outbox(db, user_id=user_id, draft=draft)
+    await db.commit()
+    await enqueue_job(
+        "send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds)
+    )
+    return True
+
+
+@router.post("/unsubscribe", response_model=UnsubscribeOut)
+async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut:
+    """Unsubscribe from a sender: one-click if offered, else by mail, else the web page."""
+    if body.message_id is not None:
+        found = await db.scalar(
+            select(Message).where(
+                Message.id == body.message_id,
+                Message.user_id == user.id,
+                Message.list_unsubscribe.isnot(None),
+                Message.list_unsubscribe != "",
+            )
+        )
+        if found is None:
+            raise_api_error(404, "not_found", "this message offers no way to unsubscribe")
+        message = found
+    else:
+        message = await _latest_with_header(db, user.id, body.key)
+    targets = subs.unsubscribe_targets(message.list_unsubscribe or "")
+    if message.list_unsubscribe_post and targets.https and await _one_click(targets.https):
+        await _record(db, user.id, body.key, "one_click")
+        return UnsubscribeOut(status="done", method="one_click")
+    if targets.mailto and await _send_mailto(db, user.id, message, targets):
+        await _record(db, user.id, body.key, "mailto")
+        return UnsubscribeOut(status="done", method="mailto")
+    if targets.https:
+        # The sender wants it done on their page: the user opens it.
+        await _record(db, user.id, body.key, "web")
+        return UnsubscribeOut(status="open", method="web", url=targets.https)
+    raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe")
+
+
+@router.post("/trash")
+async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]:
+    """Move every conversation from this sender to Trash.
+
+    Only conversations made up entirely of this sender's mail: one where the
+    user replied, or someone else wrote, stays where it is.
+    """
+    rows = (
+        await db.execute(
+            select(Message.thread_id, _key().label("key"), Mailbox.special_use)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(
+                Message.user_id == user.id,
+                Message.thread_id.in_(
+                    select(Message.thread_id)
+                    .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                    .where(*_bulk(user.id), _key() == body.key)
+                ),
+            )
+        )
+    ).all()
+    own: dict[uuid.UUID, bool] = {}
+    for thread_id, key, special in rows:
+        if thread_id is None or special == "trash":
+            continue
+        own[thread_id] = own.get(thread_id, True) and key == body.key
+    moved = [tid for tid, only_theirs in own.items() if only_theirs]
+    for tid in moved:
+        await enqueue_job("move_thread_to_folder_job", str(tid), "trash")
+    return {"moved": len(moved)}

backend/app/api/templates.py

@@ -0,0 +1,129 @@
+"""Templates: saved replies, inserted from the compose window's More menu."""
+
+from __future__ import annotations
+
+import datetime as dt
+import uuid
+from typing import Annotated
+
+import nh3
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import raise_api_error
+from app.db import get_session
+from app.identity.deps import CurrentUser, require_csrf
+from app.models import MailTemplate
+
+router = APIRouter(prefix="/templates", tags=["compose"], dependencies=[Depends(require_csrf)])
+
+DbSession = Annotated[AsyncSession, Depends(get_session)]
+
+MAX_TEMPLATES = 200
+MAX_BODY_BYTES = 256 * 1024
+
+# What the compose editor writes. A template goes back into the editor as HTML,
+# so it is cleaned like any HTML the app renders: no scripts, no handlers.
+_TAGS = {
+    "a", "b", "blockquote", "br", "code", "div", "em", "font", "h1", "h2", "h3",
+    "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strike", "strong",
+    "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "u", "ul",
+}  # fmt: skip
+_GENERIC = {"style", "class", "dir", "align"}
+_SPECIFIC = {
+    "a": {"href", "title"},
+    "img": {"src", "alt", "width", "height"},
+    "font": {"color", "face", "size"},
+    "td": {"colspan", "rowspan"},
+    "th": {"colspan", "rowspan"},
+}
+# nh3 takes no wildcard: the generic attributes go onto every tag.
+_ATTRS = {tag: _GENERIC | _SPECIFIC.get(tag, set()) for tag in _TAGS}
+
+
+class TemplateOut(BaseModel):
+    id: uuid.UUID
+    name: str
+    subject: str
+    body_html: str
+    updated_at: dt.datetime
+
+
+class TemplateIn(BaseModel):
+    name: str = Field(min_length=1, max_length=80)
+    subject: str = Field(default="", max_length=500)
+    body_html: str = ""
+
+
+def _out(t: MailTemplate) -> TemplateOut:
+    return TemplateOut(
+        id=t.id, name=t.name, subject=t.subject, body_html=t.body_html, updated_at=t.updated_at
+    )
+
+
+def clean_body(html: str) -> str:
+    return nh3.clean(
+        html,
+        tags=_TAGS,
+        attributes=_ATTRS,
+        url_schemes={"http", "https", "mailto", "data"},
+        link_rel="noopener noreferrer",
+    )
+
+
+@router.get("", response_model=list[TemplateOut])
+async def list_templates(user: CurrentUser, db: DbSession) -> list[TemplateOut]:
+    rows = await db.scalars(
+        select(MailTemplate)
+        .where(MailTemplate.user_id == user.id)
+        .order_by(func.lower(MailTemplate.name))
+    )
+    return [_out(t) for t in rows]
+
+
+@router.post("", response_model=TemplateOut)
+async def save_template(body: TemplateIn, user: CurrentUser, db: DbSession) -> TemplateOut:
+    """Save a template. Saving under a name that exists replaces that template."""
+    name = " ".join(body.name.split())
+    if not name:
+        raise_api_error(400, "invalid", "a template needs a name")
+    if len(body.body_html.encode()) > MAX_BODY_BYTES:
+        raise_api_error(400, "too_large", "the template is too large")
+    html = clean_body(body.body_html)
+    subject = " ".join(body.subject.split())
+    existing = await db.scalar(
+        select(MailTemplate).where(
+            MailTemplate.user_id == user.id, func.lower(MailTemplate.name) == name.lower()
+        )
+    )
+    if existing is not None:
+        existing.name, existing.subject, existing.body_html = name, subject, html
+        existing.updated_at = dt.datetime.now(dt.UTC)
+        saved = existing
+    else:
+        count = await db.scalar(
+            select(func.count()).select_from(MailTemplate).where(MailTemplate.user_id == user.id)
+        )
+        if (count or 0) >= MAX_TEMPLATES:
+            raise_api_error(409, "too_many", f"at most {MAX_TEMPLATES} templates")
+        saved = MailTemplate(user_id=user.id, name=name, subject=subject, body_html=html)
+        db.add(saved)
+    await db.commit()
+    await db.refresh(saved)
+    return _out(saved)
+
+
+@router.delete("/{template_id}")
+async def delete_template(
+    template_id: uuid.UUID, user: CurrentUser, db: DbSession
+) -> dict[str, str]:
+    row = await db.scalar(
+        select(MailTemplate).where(MailTemplate.id == template_id, MailTemplate.user_id == user.id)
+    )
+    if row is None:
+        raise_api_error(404, "not_found", "template not found")
+    await db.delete(row)
+    await db.commit()
+    return {"status": "deleted"}

backend/app/api/threads.py

@@ -4,13 +4,14 @@ from __future__ import annotations
 
 import uuid
 from collections.abc import Collection
-from typing import Annotated
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Query
 from sqlalchemy import Text, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.sql.elements import ColumnElement
 
+from app import subscriptions as subs
 from app.api.errors import raise_api_error
 from app.api.schemas import (
     Address,
@@ -19,10 +20,10 @@ from app.api.schemas import (
     ThreadListOut,
     ThreadSummary,
 )
-from app.api.settings import allowed_image_senders
+from app.api.settings import allowed_image_senders, blocks_trackers
 from app.db import get_session
 from app.identity.deps import CurrentUser
-from app.models import Mailbox, Message, Thread, User
+from app.models import BodyPart, Mailbox, Message, SubscriptionState, Thread, User
 
 router = APIRouter(prefix="/threads", tags=["mail"])
 
@@ -50,6 +51,7 @@ def _summary(
         has_attachments=thread.has_attachments,
         pinned=thread.pinned,
         last_message_at=thread.last_message_at,
+        snoozed_until=thread.snoozed_until,
     )
 
 
@@ -74,6 +76,7 @@ def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnv
         images_allowed=bool(sender) and sender in allowed_senders,
         size_bytes=msg.size_bytes,
         snippet=msg.snippet,
+        content_language=msg.content_language or "",
     )
 
 
@@ -130,6 +133,10 @@ async def list_threads(
             .where(Message.user_id == user.id, Mailbox.special_use == special)
         )
         conditions.append(Thread.id.in_(in_folder))
+        if special == "inbox":
+            conditions.append(Thread.snoozed_until.is_(None))  # back when it wakes
+    elif view == "snoozed":
+        conditions.append(Thread.snoozed_until.isnot(None))
     elif view != "unified":
         raise_api_error(400, "bad_view", "unsupported view")
 
@@ -144,11 +151,20 @@ async def list_threads(
         )
     ) or 0
 
-    order_by = (
-        (Thread.last_message_at.asc(), Thread.id.asc())
-        if order == "asc"
-        else (Thread.last_message_at.desc(), Thread.id.desc())
+    # A thread woken from snooze sorts as if it had just arrived.
+    sort_at = func.greatest(
+        Thread.last_message_at, func.coalesce(Thread.unsnoozed_at, Thread.last_message_at)
     )
+    if view == "snoozed":
+        # Soonest back first, whatever the list order.
+        order_by: tuple[ColumnElement[Any], ...] = (
+            Thread.snoozed_until.asc(),
+            Thread.id.asc(),
+        )
+    elif order == "asc":
+        order_by = (sort_at.asc(), Thread.id.asc())
+    else:
+        order_by = (sort_at.desc(), Thread.id.desc())
     stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit)
     threads = list(await db.scalars(stmt))
 
@@ -177,6 +193,41 @@ async def list_threads(
     )
 
 
+async def _mark_subscriptions(
+    db: AsyncSession,
+    user_id: uuid.UUID,
+    messages: list[Message],
+    envelopes: list[MessageEnvelope],
+) -> None:
+    """Tell the reading pane which messages can be unsubscribed from, and how."""
+    keys: dict[uuid.UUID, str] = {}
+    for m, env in zip(messages, envelopes, strict=True):
+        if not m.list_unsubscribe:
+            continue
+        targets = subs.unsubscribe_targets(m.list_unsubscribe)
+        if m.list_unsubscribe_post and targets.https:
+            env.unsubscribe_via = "one_click"
+        elif targets.mailto:
+            env.unsubscribe_via = "mailto"
+        elif targets.https:
+            env.unsubscribe_via = "web"
+        else:
+            continue
+        sender = m.from_addr[0].get("email", "") if m.from_addr else ""
+        env.subscription_key = keys[m.id] = subs.group_key(m.list_id, sender)
+    if keys:
+        done = set(
+            await db.scalars(
+                select(SubscriptionState.key).where(
+                    SubscriptionState.user_id == user_id,
+                    SubscriptionState.key.in_(set(keys.values())),
+                )
+            )
+        )
+        for env in envelopes:
+            env.unsubscribed = env.subscription_key in done
+
+
 def _visible_in(current_special: str | None) -> ColumnElement[bool]:
     """Which of a thread's messages a view shows (see ``_REMOVAL_FOLDERS``): Trash/Spam
     show only their own messages; every other view hides them."""
@@ -281,6 +332,20 @@ async def get_thread(
         env = _envelope(m, allowed_senders)
         env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"]
         envelopes.append(env)
+    await _mark_subscriptions(db, user.id, ordered, envelopes)
+    if not blocks_trackers(user) and ordered:
+        # Trackers left in are remote images: offer "Load images" for them too.
+        with_trackers = set(
+            await db.scalars(
+                select(BodyPart.message_id).where(
+                    BodyPart.message_id.in_([m.id for m in ordered]),
+                    BodyPart.kind == "html_sanitized",
+                    BodyPart.trackers_blocked > 0,
+                )
+            )
+        )
+        for env in envelopes:
+            env.has_remote_images = env.has_remote_images or env.id in with_trackers
 
     order = "desc" if sort == "desc" else "asc"
     return ThreadDetail(
@@ -289,4 +354,5 @@ async def get_thread(
         order=order,
         pinned=thread.pinned,
         messages=envelopes,
+        snoozed_until=thread.snoozed_until,
     )

backend/app/api/translation.py

@@ -0,0 +1,30 @@
+"""What email translation offers (app/translation.py), for the reading pane."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+from app import translation
+from app.identity.deps import CurrentUser
+
+router = APIRouter(prefix="/translation", tags=["mail"])
+
+
+class LanguageOut(BaseModel):
+    code: str
+    name: str
+
+
+class TranslationInfoOut(BaseModel):
+    available: bool
+    languages: list[LanguageOut]
+
+
+@router.get("/languages", response_model=TranslationInfoOut)
+async def translation_languages(user: CurrentUser) -> TranslationInfoOut:
+    found = await translation.languages()
+    return TranslationInfoOut(
+        available=bool(found),
+        languages=[LanguageOut(code=x.code, name=x.name) for x in found],
+    )

backend/app/completion.py

@@ -0,0 +1,391 @@
+"""Sentence completion in the compose window: phrases the user writes, finished.
+
+No AI: a per-user index of the word sequences the user has written at least
+twice, built from their own sent mail (quoted replies and signatures cut off),
+plus a few common email phrases per language to start from. Typing "I look
+forward" finds what usually follows those words — "to hearing from you." — and
+the compose window shows it in grey until Tab takes it.
+
+It works in any language the user writes in: matching is on the words just
+typed, so a Polish sentence only ever meets Polish phrases.
+
+The index lives in ``completion_phrases`` (context → continuation, with how
+often it was seen) and is rebuilt in the background at most once a day, when
+the compose window asks for suggestions (:func:`build_index_job`).
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import html as html_lib
+import logging
+import re
+import uuid
+from collections import Counter
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import delete, select
+from sqlalchemy.dialects.postgresql import insert
+
+from app.db import session_scope
+from app.models import (
+    Account,
+    BodyPart,
+    CompletionIndexState,
+    CompletionPhrase,
+    Mailbox,
+    Message,
+)
+
+logger = logging.getLogger(__name__)
+
+START = "^"  # a sentence's start, as a word: "^ pozdr" completes the first word
+MAX_CONTEXT = 3  # words before the cursor matched on
+MAX_CONTINUATION = 6  # words suggested at most
+MIN_COUNT = 2  # seen this often to be offered; one-offs are noise (and private)
+EXTEND_SHARE = 0.6  # a longer suggestion only while it is what usually follows
+CORPUS_MESSAGES = 500  # most recent sent messages read
+CORPUS_CHARS = 3000  # of each
+REBUILD_AFTER = dt.timedelta(days=1)
+
+# Where the user's own words end and quoted mail or a signature begins: a reply
+# header ("On … wrote:", Polish Gmail's "… napisał(a):", …), a forwarded or
+# original-message marker, or an Outlook-style header block.
+_QUOTE_START = re.compile(
+    r"^\s*(?:"
+    r"-{2,}\s*(?:original message|forwarded message|wiadomość oryginalna"
+    r"|wiadomość przekazana dalej|ursprüngliche nachricht|weitergeleitete nachricht)"
+    r"|(?:from|od|von|de):\s.+"
+    r"|sent from my\s"
+    r")",
+    re.IGNORECASE,
+)
+_QUOTE_HEADER_END = re.compile(
+    # German names the writer after the verb: "Am … schrieb Anna:".
+    r"(?:wrote|napisał\(a\)|napisała?|pisze|schrieb(?:\s+.{0,100})?|a écrit|escribió)\s*:\s*$",
+    re.IGNORECASE,
+)
+_SIGNATURE = re.compile(r"^--\s*$")
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+|\n+")
+_WORD = re.compile(r"\S+")
+_EDGE_PUNCT = re.compile(r"^[\"'“”„«»(\[–—-]+|[\"'“”„«»)\],.;:!?…–—-]+$")
+# Words a phrase is never learned across: numbers (phones, dates, prices),
+# addresses and links — useless to complete, and often private.
+_NOT_A_WORD = re.compile(r"\d|@|www\.|https?:", re.IGNORECASE)
+
+
+def html_to_text(html: str) -> str:
+    """An HTML signature as text lines (block tags and <br> break lines)."""
+    text = re.sub(r"(?i)<br\s*/?>|</(?:p|div|li|tr|h[1-6])\s*>", "\n", html)
+    text = re.sub(r"<[^>]+>", "", text)
+    return html_lib.unescape(text)
+
+
+def own_text(text: str, signatures: Iterable[str] = ()) -> str:
+    """The part of a sent message the user wrote: no quoted mail, no signature
+    (the ``--`` kind, or one of ``signatures`` — the accounts' own, as text)."""
+    sig_lines = {norm_line(x) for sig in signatures for x in sig.splitlines() if norm_line(x)}
+    kept: list[str] = []
+    for line in text.splitlines():
+        if _SIGNATURE.match(line) or _QUOTE_START.match(line) or _QUOTE_HEADER_END.search(line):
+            break
+        if norm_line(line) in sig_lines:
+            continue
+        if line.lstrip().startswith(">"):
+            continue  # an inline-quoted line
+        kept.append(line)
+    return "\n".join(kept)
+
+
+def norm_line(line: str) -> str:
+    return " ".join(line.lower().split())
+
+
+def norm(word: str) -> str:
+    """A word as matched: lower case, outer punctuation off ("Hello," -> "hello")."""
+    return _EDGE_PUNCT.sub("", word.lower())
+
+
+def sentences(text: str) -> Iterable[list[str]]:
+    for chunk in _SENTENCE_END.split(text):
+        words = _WORD.findall(chunk)
+        if words:
+            yield words
+
+
+@dataclass(frozen=True)
+class Phrase:
+    context: str  # normalized words before the cursor, space-joined ("^ i look")
+    continuation: str  # the words that follow, as written ("forward to hearing")
+    count: int
+
+
+def build_phrases(texts: Iterable[str], weight: int = 1) -> dict[tuple[str, str], int]:
+    """(context, continuation) -> count, for every sequence seen MIN_COUNT times.
+
+    Counts each sentence's word sequences (with the start marker), keeps the
+    frequent ones, and splits each into "what was typed" and "what follows".
+    """
+    counts: Counter[tuple[str, ...]] = Counter()
+    surface: dict[tuple[str, ...], tuple[str, ...]] = {}
+    longest = MAX_CONTEXT + MAX_CONTINUATION
+    for text in texts:
+        for words in sentences(text):
+            seq = [START, *words]
+            keys = [START, *("" if _NOT_A_WORD.search(w) else norm(w) for w in words)]
+            for i in range(len(seq)):
+                for n in range(2, min(longest, len(seq) - i) + 1):
+                    gram = tuple(keys[i : i + n])
+                    if "" in gram:
+                        break  # a lone punctuation mark: no phrase across it
+                    counts[gram] += weight
+                    surface.setdefault(gram, tuple(seq[i : i + n]))
+    out: dict[tuple[str, str], int] = {}
+    for gram, count in counts.items():
+        if count < MIN_COUNT:
+            continue
+        written = surface[gram]
+        for k in range(1, min(MAX_CONTEXT, len(gram) - 1) + 1):
+            rest = len(gram) - k
+            if rest > MAX_CONTINUATION:
+                continue
+            key = (" ".join(gram[:k]), " ".join(written[k:]))
+            out[key] = max(out.get(key, 0), count)
+    return out
+
+
+# A few common phrases per language, for users with little sent mail yet.
+STARTER_PHRASES: dict[str, list[str]] = {
+    "en": [
+        "I look forward to hearing from you.",
+        "I'm looking forward to hearing from you.",
+        "Looking forward to hearing from you.",
+        "Looking forward to working with you.",
+        "Thank you for your email.",
+        "Thank you for your help.",
+        "Thank you for your quick reply.",
+        "Thanks in advance.",
+        "Please let me know if you have any questions.",
+        "Let me know if you have any questions.",
+        "Let me know what you think.",
+        "Please find attached the",
+        "I hope this email finds you well.",
+        "I hope you're doing well.",
+        "Sorry for the late reply.",
+        "I would be happy to help.",
+        "Feel free to contact me if you need anything else.",
+        "Could you please send me the",
+        "Please confirm at your earliest convenience.",
+        "Have a great day!",
+        "Have a nice weekend!",
+        "Best regards,",
+        "Kind regards,",
+    ],
+    "pl": [
+        "Z poważaniem,",
+        "Pozdrawiam serdecznie,",
+        "Dziękuję za wiadomość.",
+        "Dziękuję za szybką odpowiedź.",
+        "Dziękuję za pomoc.",
+        "Z góry dziękuję.",
+        "W załączniku przesyłam",
+        "Proszę o informację zwrotną.",
+        "W razie pytań proszę o kontakt.",
+        "Czekam na odpowiedź.",
+        "Przepraszam za opóźnienie w odpowiedzi.",
+        "Będę wdzięczny za odpowiedź.",
+        "Będę wdzięczna za odpowiedź.",
+        "Zgodnie z naszą rozmową",
+        "Proszę o potwierdzenie.",
+        "Daj znać, co myślisz.",
+        "Miłego dnia!",
+        "Miłego weekendu!",
+    ],
+    "de": [
+        "Mit freundlichen Grüßen",
+        "Viele Grüße",
+        "Beste Grüße",
+        "Vielen Dank für Ihre E-Mail.",
+        "Vielen Dank für Ihre schnelle Antwort.",
+        "Vielen Dank im Voraus.",
+        "Ich freue mich auf Ihre Antwort.",
+        "Bei Fragen stehe ich Ihnen gerne zur Verfügung.",
+        "Im Anhang finden Sie",
+        "Entschuldigen Sie die späte Antwort.",
+        "Ich hoffe, es geht Ihnen gut.",
+        "Schönen Tag noch!",
+        "Schönes Wochenende!",
+    ],
+}
+
+# Built once: every starter phrase counts as seen MIN_COUNT times.
+_STARTERS: dict[str, Counter[str]] = {}
+for (_ctx, _cont), _n in build_phrases(
+    (p for phrases in STARTER_PHRASES.values() for p in phrases), weight=MIN_COUNT
+).items():
+    _STARTERS.setdefault(_ctx, Counter())[_cont] = _n
+
+_LEADING_PUNCT = re.compile(r"^[\"'“”„«»(\[]+")
+
+
+def _candidates(phrases: dict[str, Counter[str]], context: str, partial: str) -> Counter[str]:
+    """Continuations of ``context`` whose first word starts with ``partial``."""
+    out: Counter[str] = Counter()
+    for continuation, count in phrases.get(context, Counter()).items():
+        first = continuation.split(" ", 1)[0]
+        if partial and not first.lower().startswith(partial.lower()):
+            continue
+        if partial and len(first) <= len(partial):
+            continue  # that word is typed in full already
+        out[continuation] = count
+    return out
+
+
+def choose(candidates: Counter[str]) -> str | None:
+    """The suggestion: the most frequent next word, extended word by word only
+    while the longer phrase is what usually follows (EXTEND_SHARE)."""
+    if not candidates:
+        return None
+    by_first: Counter[str] = Counter()
+    for continuation, count in candidates.items():
+        first = continuation.split(" ", 1)[0]
+        by_first[first] = max(by_first[first], count)
+    chosen, chosen_count = by_first.most_common(1)[0]
+    while True:
+        depth = chosen.count(" ") + 1
+        longer = [
+            (c, n)
+            for c, n in candidates.items()
+            if c.startswith(chosen + " ") and c.count(" ") == depth
+        ]
+        if not longer:
+            return chosen
+        best, best_count = max(longer, key=lambda x: x[1])
+        if best_count < max(MIN_COUNT, EXTEND_SHARE * chosen_count):
+            return chosen
+        chosen, chosen_count = best, best_count
+
+
+def contexts(before: str) -> tuple[list[str], str]:
+    """For the text before the cursor: the contexts to try, longest first, and
+    the partly typed word ('' after a space). No contexts: nothing to suggest
+    (an empty sentence, or the cursor right after a punctuation mark)."""
+    if not before or not before.strip():
+        return [], ""
+    last = before[-1]
+    if not last.isspace() and not last.isalnum() and last not in "'’-":
+        return [], ""  # "you," or "end." — not mid-word, and no space typed yet
+    tail = _SENTENCE_END.split(before)[-1]
+    words = _WORD.findall(tail)
+    partial = ""
+    if words and not last.isspace():
+        partial = _LEADING_PUNCT.sub("", words.pop())
+    if not words and not partial:
+        return [], ""  # a new sentence with nothing in it yet
+    keys = [START, *(norm(w) for w in words)]
+    if "" in keys:  # a lone punctuation mark: the phrase starts after it
+        keys = keys[len(keys) - keys[::-1].index("") :]
+    if not keys:
+        return [], ""
+    return [" ".join(keys[-k:]) for k in range(min(MAX_CONTEXT, len(keys)), 0, -1)], partial
+
+
+def suggest(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """What to show after ``before`` (the text up to the cursor), or None.
+
+    The returned text starts where the cursor is: the rest of a partly typed
+    word, or (after a space) the next words. The user's own phrases and the
+    starter phrases count together.
+    """
+    tried, partial = contexts(before)
+    for context in tried:
+        merged = _candidates(personal, context, partial)
+        for cont, count in _candidates(_STARTERS, context, partial).items():
+            merged[cont] = merged.get(cont, 0) + count
+        if " " not in context and not partial:
+            # A single word before a space is a weak hint: only a strong habit.
+            merged = Counter({c: n for c, n in merged.items() if n >= 2 * MIN_COUNT})
+        best = choose(merged)
+        if best:
+            return best[len(partial) :] if partial else best
+    return None
+
+
+def complete(personal: dict[str, Counter[str]], before: str) -> str | None:
+    """:func:`suggest`, and for a word typed in full but no space yet ("I look
+    forward"), what follows it — with the space ("␣to hearing from you.")."""
+    found = suggest(personal, before)
+    if found is None and before[-1:].isalnum():
+        after_space = suggest(personal, before + " ")
+        if after_space:
+            return " " + after_space
+    return found
+
+
+def keys_for(before: str) -> list[str]:
+    """The contexts :func:`complete` may look up, to fetch them from the index."""
+    keys = contexts(before)[0]
+    if before[-1:].isalnum():
+        keys += contexts(before + " ")[0]
+    return keys
+
+
+async def personal_phrases(
+    session: Any, user_id: uuid.UUID, keys: list[str]
+) -> dict[str, Counter[str]]:
+    rows = await session.execute(
+        select(
+            CompletionPhrase.context, CompletionPhrase.continuation, CompletionPhrase.count
+        ).where(CompletionPhrase.user_id == user_id, CompletionPhrase.context.in_(keys))
+    )
+    out: dict[str, Counter[str]] = {}
+    for context, continuation, count in rows:
+        out.setdefault(context, Counter())[continuation] = count
+    return out
+
+
+async def build_index_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Rebuild a user's phrase index from their recent sent mail. Returns its size."""
+    uid = uuid.UUID(user_id)
+    async with session_scope() as session:
+        texts = list(
+            await session.scalars(
+                select(BodyPart.content)
+                .join(Message, Message.id == BodyPart.message_id)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.user_id == uid,
+                    Mailbox.special_use == "sent",
+                    BodyPart.kind == "text",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(CORPUS_MESSAGES)
+            )
+        )
+        signatures = [
+            html_to_text(sig)
+            for sig in await session.scalars(
+                select(Account.signature).where(Account.user_id == uid, Account.signature != "")
+            )
+        ]
+        phrases = build_phrases(own_text(t, signatures)[:CORPUS_CHARS] for t in texts)
+        await session.execute(delete(CompletionPhrase).where(CompletionPhrase.user_id == uid))
+        rows = [
+            {"user_id": uid, "context": c, "continuation": cont, "count": n}
+            for (c, cont), n in phrases.items()
+        ]
+        for start in range(0, len(rows), 1000):
+            await session.execute(insert(CompletionPhrase), rows[start : start + 1000])
+        stmt = insert(CompletionIndexState).values(user_id=uid, built_at=dt.datetime.now(dt.UTC))
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=["user_id"], set_={"built_at": stmt.excluded.built_at}
+            )
+        )
+        await session.commit()
+    logger.info(
+        "completion index for %s: %d phrases from %d messages", user_id, len(rows), len(texts)
+    )
+    return len(rows)

backend/app/config.py

@@ -71,6 +71,9 @@ class Settings(BaseSettings):
     # Spelling and grammar checking in the compose window (LanguageTool server,
     # e.g. http://languagetool:8010). Empty: the check is off for everyone.
     languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL")
+    # Email translation: a LibreTranslate server (app/translation.py), e.g.
+    # http://libretranslate:5000. Empty: translation is off.
+    libretranslate_url: str = Field(default="", alias="LIBRETRANSLATE_URL")
     # New passwords are checked against known data breaches (Have I Been Pwned's
     # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server).
     password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK")

backend/app/indexer/indexer.py

@@ -25,6 +25,7 @@ from app.indexer.sanitize import sanitize_html
 from app.models import Account, Attachment, BodyPart, KnownRecipient, Message
 from app.providers.base import RemoteMessage
 from app.search.base import SearchDoc
+from app.subscriptions import read_list_headers
 from app.threading_.normalize import synthesize_message_id
 from app.threading_.threader import thread_message
 
@@ -167,6 +168,7 @@ async def persist_message(
     else:
         sha = rawstore.put_cached(user_id, remote.raw)
     parsed = await _parse_bounded(user_id, sha, remote.raw)
+    list_headers = read_list_headers(remote.raw)  # for the Subscriptions page
 
     message_id_hdr = await _resolve_message_id(
         session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw
@@ -207,6 +209,9 @@ async def persist_message(
         has_attachments=parsed.has_attachments,
         snippet=parsed.snippet,
         blob_sha256=sha,
+        list_id=list_headers.list_id,
+        list_unsubscribe=list_headers.unsubscribe,
+        list_unsubscribe_post=list_headers.one_click,
     )
     session.add(message)
     await session.flush()
@@ -224,6 +229,7 @@ async def persist_message(
                 kind="html_sanitized",
                 content=result.html,
                 sanitizer_v=result.sanitizer_v,
+                trackers_blocked=result.trackers_blocked,
             )
         )
 

backend/app/indexer/sanitize.py

@@ -19,7 +19,10 @@ import nh3
 # render re-sanitizes stale content from the raw blob (SPEC §10.4).
 # v5: this server's own inline-image URLs quoted in a message load from here.
 # v6: links keep only http(s) and mailto (no data: pages).
-SANITIZER_VERSION = 6
+# v7: tracking pixels are removed (and counted), even when images are allowed.
+# v8: CSS sizes in other units (in, em, %, pt) no longer read as pixels — v7
+#     took 0.31in signature logos for trackers.
+SANITIZER_VERSION = 8
 
 # This server's inline-image URL, however it ended up written: "/api/…",
 # "https://host/api/…", or mangled to "http:///api/…" by an earlier copy.
@@ -154,6 +157,94 @@ class SanitizeResult:
     html: str
     has_remote_images: bool
     sanitizer_v: int
+    trackers_blocked: int = 0
+
+
+# --- Tracking pixels ---------------------------------------------------------------
+# A remote image that is invisible (1×1, or hidden by CSS) has no use but to tell
+# the sender the mail was opened, and by whom. A few well-known open-tracking URL
+# shapes are caught whatever their size. Matching runs on the raw HTML before nh3:
+# a miss only leaves an image in, and nh3 still cleans whatever remains.
+_IMG_TAG = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
+_ATTR = re.compile(r"""([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("[^"]*"|'[^']*'|[^\s"'>]+)""")
+_CSS_HIDDEN = re.compile(
+    r"display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?![.\d]*[1-9])",
+    re.IGNORECASE,
+)
+_TRACKER_URL = re.compile(
+    r"//[^/]*mailtrack\.io/"
+    r"|/track/open\b"
+    r"|/wf/open\b"
+    r"|/open\.(?:php|aspx|gif)\b"
+    r"|[?&]tracking[_-]?pixel=",
+    re.IGNORECASE,
+)
+
+
+_PX = re.compile(r"\s*(\d+(?:\.\d*)?|\.\d+)\s*(px)?\s*$", re.IGNORECASE)
+
+
+def _px(value: str | None) -> float | None:
+    """A size in pixels ("1", "1px"), or None for anything else: other units
+    (0.31in is a 30-pixel signature logo), auto, %, or malformed (".", "1.2.3")."""
+    if value is None:
+        return None
+    match = _PX.match(value)
+    return float(match.group(1)) if match else None
+
+
+def _tiny(value: str | None) -> bool:
+    px = _px(value)
+    return px is not None and px <= 2
+
+
+def _css(style: str) -> dict[str, str]:
+    """An inline style's declarations, lower-cased, without !important."""
+    out: dict[str, str] = {}
+    for decl in style.split(";"):
+        name, _, value = decl.partition(":")
+        value = value.lower().replace("!important", "").strip()
+        if name.strip() and value:
+            out[name.strip().lower()] = value
+    return out
+
+
+def is_tracking_pixel(tag: str) -> bool:
+    """Whether an ``<img …>`` tag is a remote open-tracking pixel."""
+    attrs = {name.lower(): value.strip("\"'") for name, value in _ATTR.findall(tag)}
+    src = attrs.get("src", "").strip()
+    if not src.lower().startswith(("http:", "https:", "//")):
+        return False  # inline (cid:/data:) images can't report anything
+    if _TRACKER_URL.search(src):
+        return True
+    style = attrs.get("style", "")
+    if _CSS_HIDDEN.search(style):
+        return True
+    # The size the browser draws: a style width/height overrides the attribute
+    # (and "auto", % or another unit is no tiny size); a max-width/max-height
+    # of a pixel or two caps it; height "auto" follows the width.
+    css = _css(style)
+    wide = _tiny(css["width"]) if "width" in css else _tiny(attrs.get("width"))
+    tall = _tiny(css["height"]) if "height" in css else _tiny(attrs.get("height"))
+    wide = wide or _tiny(css.get("max-width"))
+    tall = tall or _tiny(css.get("max-height"))
+    if css.get("height") == "auto":
+        tall = wide
+    return wide and tall
+
+
+def strip_tracking_pixels(html: str) -> tuple[str, int]:
+    """Remove tracking pixels from raw HTML; returns the HTML and how many."""
+    count = 0
+
+    def repl(match: re.Match[str]) -> str:
+        nonlocal count
+        if is_tracking_pixel(match.group(0)):
+            count += 1
+            return ""
+        return match.group(0)
+
+    return _IMG_TAG.sub(repl, html), count
 
 
 def _sanitize_style(value: str, allow_remote: bool) -> str | None:
@@ -188,15 +279,24 @@ def _sanitize_style_elements(html: str, allow_remote: bool, state: dict[str, boo
 
 
 def sanitize_html(
-    html: str, message_id: uuid.UUID, *, allow_remote_images: bool = False
+    html: str,
+    message_id: uuid.UUID,
+    *,
+    allow_remote_images: bool = False,
+    block_trackers: bool = True,
 ) -> SanitizeResult:
     """Sanitize ``html`` for the message ``message_id``.
 
     Returns the safe HTML, whether any remote images were blocked, and the
     sanitizer version. Pass ``allow_remote_images=True`` after the user approves
     a sender to render remote images and CSS backgrounds (SPEC §9.4).
+    ``block_trackers=False`` (Settings → Privacy) leaves tracking pixels in; they
+    are remote images like any other then.
     """
     state = {"has_remote": False}
+    trackers = 0
+    if block_trackers:
+        html, trackers = strip_tracking_pixels(html)
 
     def attribute_filter(tag: str, attr: str, value: str) -> str | None:
         if attr == "style":
@@ -247,4 +347,5 @@ def sanitize_html(
         html=cleaned,
         has_remote_images=state["has_remote"],
         sanitizer_v=SANITIZER_VERSION,
+        trackers_blocked=trackers,
     )

backend/app/models.py

@@ -199,6 +199,12 @@ class Thread(Base):
     has_attachments: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     last_message_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+    # Snooze: out of the Inbox until this time (NULL when not snoozed). A message
+    # that arrives after ``snoozed_at`` wakes it early; ``unsnoozed_at`` (when it
+    # last woke) puts it back at the top of the Inbox.
+    snoozed_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    snoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
+    unsnoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
 
 class Message(Base):
@@ -263,6 +269,17 @@ class Message(Base):
     snippet: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
     blob_sha256: Mapped[str] = mapped_column(Text, nullable=False)
     body_indexed: Mapped[str | None] = mapped_column(TSVECTOR)
+    # Mailing-list headers, for the Subscriptions page. ``list_unsubscribe`` is
+    # NULL until read (a backfill job fills older mail), '' when there is none.
+    list_id: Mapped[str | None] = mapped_column(Text)
+    list_unsubscribe: Mapped[str | None] = mapped_column(Text)
+    # RFC 8058: the sender takes a one-click unsubscribe POST.
+    list_unsubscribe_post: Mapped[bool] = mapped_column(
+        nullable=False, server_default=text("false")
+    )
+    # The language of its text (app/translation.py), found the first time it is
+    # shown: NULL not looked at yet, '' could not tell.
+    content_language: Mapped[str | None] = mapped_column(Text)
 
 
 class BodyPart(Base):
@@ -276,6 +293,8 @@ class BodyPart(Base):
     content: Mapped[str] = mapped_column(Text, nullable=False)
     # SPEC §10.4: re-sanitize on render if the sanitizer version changed.
     sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    # Tracking pixels taken out of this HTML (shown as "N trackers blocked").
+    trackers_blocked: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
 
 
 class Attachment(Base):
@@ -404,6 +423,78 @@ class Invitation(Base):
     )
 
 
+class SubscriptionState(Base):
+    """What a user did about a bulk sender on the Subscriptions page."""
+
+    __tablename__ = "subscription_states"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    # The sender group: "list:<List-Id>" or "from:<address>".
+    key: Mapped[str] = mapped_column(Text, primary_key=True)
+    unsubscribed_at: Mapped[dt.datetime] = _now()
+    method: Mapped[str] = mapped_column(Text, nullable=False)  # one_click | mailto | web
+
+
+class MailTemplate(Base):
+    """A saved reply, inserted from the compose window."""
+
+    __tablename__ = "mail_templates"
+
+    id: Mapped[uuid.UUID] = _uuid_pk()
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    name: Mapped[str] = mapped_column(Text, nullable=False)
+    subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    body_html: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
+    created_at: Mapped[dt.datetime] = _now()
+    updated_at: Mapped[dt.datetime] = _now()
+
+
+class MessageTranslation(Base):
+    """A message's HTML translated into a language (app/translation.py), kept so
+    it is translated once. ``images``: with remote images shown or not."""
+
+    __tablename__ = "message_translations"
+
+    message_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
+    )
+    target: Mapped[str] = mapped_column(Text, primary_key=True)
+    images: Mapped[bool] = mapped_column(primary_key=True)
+    source: Mapped[str] = mapped_column(Text, nullable=False)
+    html: Mapped[str] = mapped_column(Text, nullable=False)
+    sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[dt.datetime] = _now()
+
+
+class CompletionPhrase(Base):
+    """Sentence completion (app/completion.py): after these words, the user
+    usually writes this — seen ``count`` times in their sent mail."""
+
+    __tablename__ = "completion_phrases"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    context: Mapped[str] = mapped_column(Text, primary_key=True)  # normalized words
+    continuation: Mapped[str] = mapped_column(Text, primary_key=True)  # as written
+    count: Mapped[int] = mapped_column(Integer, nullable=False)
+
+
+class CompletionIndexState(Base):
+    """When a user's completion index was last built."""
+
+    __tablename__ = "completion_index_state"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    built_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False)
+
+
 class PushSubscription(Base):
     """A browser or installed app to tell about new mail (Web Push)."""
 

backend/app/queue.py

@@ -21,7 +21,14 @@ QUEUE_INTERACTIVE = "interactive"
 QUEUE_BULK = "bulk"
 
 # Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
-_BULK_JOBS = frozenset({"sync_bootstrap_account", "dedupe_existing_job"})
+_BULK_JOBS = frozenset(
+    {
+        "sync_bootstrap_account",
+        "dedupe_existing_job",
+        "backfill_list_headers_job",
+        "build_index_job",
+    }
+)
 
 
 def queue_for(job_name: str) -> str:

backend/app/snooze.py

@@ -0,0 +1,92 @@
+"""Snooze: a conversation leaves the Inbox until a chosen time, then comes back.
+
+Kept in this database only (the mail server knows nothing of it): a snoozed
+thread is hidden from the Inbox view and listed under Snoozed. The wake job,
+run every minute, brings back threads whose time has come — or that got a new
+message meanwhile, as Gmail does — at the top of the Inbox, the newest message
+marked unread.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import or_, select
+
+from app.db import session_scope
+from app.events import get_event_bus
+from app.models import Mailbox, Message, Thread
+from app.queue import enqueue_job
+from app.sync.actions import apply_flags_to_messages
+
+logger = logging.getLogger(__name__)
+
+MAX_SNOOZE = dt.timedelta(days=366)
+_SEEN = frozenset({"\\Seen"})
+
+
+def snooze(thread: Thread, until: dt.datetime, now: dt.datetime) -> None:
+    thread.snoozed_until = until
+    thread.snoozed_at = now
+
+
+def wake(thread: Thread, now: dt.datetime) -> None:
+    thread.snoozed_until = None
+    thread.snoozed_at = None
+    thread.unsnoozed_at = now
+
+
+async def wake_due_job(ctx: dict[str, Any]) -> int:
+    """Cron (every minute): wake snoozed threads that are due. Returns how many."""
+    now = dt.datetime.now(dt.UTC)
+    woken: list[tuple[uuid.UUID, uuid.UUID]] = []
+    unread: list[tuple[uuid.UUID, Message]] = []
+    async with session_scope() as session:
+        due = list(
+            await session.scalars(
+                select(Thread)
+                .where(
+                    Thread.snoozed_until.isnot(None),
+                    or_(Thread.snoozed_until <= now, Thread.last_message_at > Thread.snoozed_at),
+                )
+                .with_for_update(skip_locked=True)
+            )
+        )
+        for thread in due:
+            on_time = thread.snoozed_until is not None and thread.snoozed_until <= now
+            wake(thread, now)
+            woken.append((thread.user_id, thread.id))
+            if not on_time:
+                continue  # a new message woke it: that message is unread already
+            # Back as new: the newest Inbox message becomes unread again.
+            newest = await session.scalar(
+                select(Message)
+                .join(Mailbox, Mailbox.id == Message.mailbox_id)
+                .where(
+                    Message.thread_id == thread.id,
+                    Message.user_id == thread.user_id,
+                    Mailbox.special_use == "inbox",
+                )
+                .order_by(Message.internal_date.desc())
+                .limit(1)
+            )
+            if newest is not None and "\\Seen" in newest.flags:
+                await apply_flags_to_messages(session, [newest], frozenset(), _SEEN)
+                unread.append((thread.user_id, newest))
+        await session.commit()
+
+    bus = get_event_bus()
+    for user_id, message in unread:
+        await enqueue_job("push_flags_job", str(message.id), [], ["\\Seen"])
+        await bus.publish(
+            "flags.changed",
+            {"user_id": str(user_id), "message_id": str(message.id), "flags": list(message.flags)},
+        )
+    for user_id, thread_id in woken:
+        await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
+    if woken:
+        logger.info("woke %d snoozed thread(s)", len(woken))
+    return len(woken)

backend/app/subscriptions.py

@@ -0,0 +1,151 @@
+"""Subscriptions: the bulk senders mailing a user, and unsubscribing from them.
+
+No service records what a user signed up for, so the list is worked out from
+received mail, as Gmail does: a message is a subscription's when it carries a
+``List-Unsubscribe`` header (required of bulk senders by Gmail and Yahoo since
+2024). Messages are grouped by ``List-Id`` when there is one (a company can run
+several lists), otherwise by sender address.
+
+The headers are read when a message is stored (indexer.persist_message); older
+mail is read once by :func:`backfill_list_headers_job`. Unsubscribing uses the
+sender's one-click POST (RFC 8058) when offered, else mails the ``mailto:``
+address, else hands the web page back for the user to open.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from dataclasses import dataclass
+from email.parser import BytesHeaderParser
+from email.policy import default as default_policy
+from typing import Any
+from urllib.parse import parse_qs, unquote, urlsplit
+
+from sqlalchemy import select, update
+
+from app.db import session_scope
+from app.indexer import rawstore
+from app.models import Message
+
+logger = logging.getLogger(__name__)
+
+_MAX_HEADER = 2000
+_ANGLE = re.compile(r"<([^<>]+)>")
+_BATCH = 200
+
+
+@dataclass(frozen=True)
+class ListHeaders:
+    list_id: str | None
+    unsubscribe: str  # '' when the message has none
+    one_click: bool
+
+
+def read_list_headers(raw: bytes) -> ListHeaders:
+    """The mailing-list headers of a raw message (headers only are parsed)."""
+    end = raw.find(b"\r\n\r\n")
+    if end < 0:
+        end = raw.find(b"\n\n")
+    head = raw if end < 0 else raw[: end + 2]
+    try:
+        msg = BytesHeaderParser(policy=default_policy).parsebytes(head)
+        list_id_raw = str(msg.get("List-Id") or "")
+        unsubscribe = str(msg.get("List-Unsubscribe") or "")
+        post = str(msg.get("List-Unsubscribe-Post") or "")
+    except Exception:  # a malformed header must not stop the message being stored
+        return ListHeaders(None, "", False)
+    list_id = None
+    if list_id_raw:
+        match = _ANGLE.search(list_id_raw)
+        list_id = (match.group(1) if match else list_id_raw).strip().lower()[:300] or None
+    unsubscribe = " ".join(unsubscribe.split())[:_MAX_HEADER]
+    one_click = "list-unsubscribe=one-click" in post.replace(" ", "").lower()
+    return ListHeaders(list_id, unsubscribe, one_click and bool(unsubscribe))
+
+
+@dataclass(frozen=True)
+class UnsubscribeTargets:
+    https: str | None
+    mailto: str | None  # the address
+    mailto_subject: str
+    mailto_body: str
+
+
+def unsubscribe_targets(header: str) -> UnsubscribeTargets:
+    """The https and mailto targets listed in a ``List-Unsubscribe`` header."""
+    https = mailto = None
+    subject, body = "unsubscribe", "unsubscribe"
+    for target in _ANGLE.findall(header):
+        target = target.strip()
+        lowered = target.lower()
+        if https is None and lowered.startswith("https://"):
+            https = target
+        elif mailto is None and lowered.startswith("mailto:"):
+            parts = urlsplit(target)
+            address = unquote(parts.path).strip()
+            if "@" in address and "\n" not in address and "\r" not in address:
+                mailto = address
+                query = parse_qs(parts.query)
+                subject = (query.get("subject") or [subject])[0][:200]
+                body = (query.get("body") or [body])[0][:2000]
+    return UnsubscribeTargets(https, mailto, subject, body)
+
+
+def group_key(list_id: str | None, sender: str) -> str:
+    return f"list:{list_id}" if list_id else f"from:{sender.lower()}"
+
+
+async def pending_count(session: Any, user_id: uuid.UUID) -> int:
+    from sqlalchemy import func
+
+    return int(
+        await session.scalar(
+            select(func.count())
+            .select_from(Message)
+            .where(Message.user_id == user_id, Message.list_unsubscribe.is_(None))
+        )
+        or 0
+    )
+
+
+async def backfill_list_headers_job(ctx: dict[str, Any], user_id: str) -> int:
+    """Read the list headers of a user's older mail (stored before this feature).
+
+    Works in batches; a message whose raw copy can't be had is marked as having
+    none, so it is not tried again. Returns how many messages were read.
+    """
+    uid = uuid.UUID(user_id)
+    done = 0
+    while True:
+        async with session_scope() as session:
+            batch = list(
+                await session.scalars(
+                    select(Message)
+                    .where(Message.user_id == uid, Message.list_unsubscribe.is_(None))
+                    .limit(_BATCH)
+                )
+            )
+            if not batch:
+                break
+            for message in batch:
+                try:
+                    raw = await rawstore.ensure_bytes(message)
+                    headers = read_list_headers(raw)
+                except Exception:  # missing blob, server unreachable: give up on it
+                    headers = ListHeaders(None, "", False)
+                await session.execute(
+                    update(Message)
+                    .where(Message.id == message.id)
+                    .values(
+                        list_id=headers.list_id,
+                        list_unsubscribe=headers.unsubscribe,
+                        list_unsubscribe_post=headers.one_click,
+                    )
+                )
+            await session.commit()
+            done += len(batch)
+    if done:
+        logger.info("read list headers of %d message(s) for user %s", done, user_id)
+    return done

backend/app/translation.py

@@ -0,0 +1,303 @@
+"""Email translation, on this server: LibreTranslate (Argos models) in its own
+container, reached only by the API. A message's text never leaves the server.
+
+- A message's language is detected the first time it is shown (from its own
+  text: quoted replies and signatures left out) and kept on the message, so the
+  reading pane can offer "Translate" when it differs from the interface language.
+- A translation is of the message HTML the user sees (already sanitized; the
+  result is sanitized again), cached per message, target language and whether
+  remote images were shown.
+
+Off when ``LIBRETRANSLATE_URL`` is empty.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import html as html_lib
+import logging
+import re
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import httpx
+
+from app import translation_prep as prep
+from app.completion import own_text
+from app.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+DETECT_CHARS = 1500  # of the message's own text: plenty to tell the language
+MIN_CONFIDENCE = 50.0  # below this, LibreTranslate is guessing
+MAX_HTML_CHARS = 150_000  # a translation request bigger than this is refused
+_TIMEOUT = httpx.Timeout(120.0, connect=5.0)  # long messages take a while on a CPU
+_LANG_CACHE_S = 600
+
+
+class TranslationError(Exception):
+    """Translation is off, the service is unreachable, or it refused the text."""
+
+
+@dataclass(frozen=True)
+class Language:
+    code: str
+    name: str
+
+
+def enabled() -> bool:
+    return bool(get_settings().libretranslate_url.strip())
+
+
+def _url(path: str) -> str:
+    return get_settings().libretranslate_url.rstrip("/") + path
+
+
+async def _post(path: str, payload: dict[str, object]) -> object:
+    if not enabled():
+        raise TranslationError("translation is not set up on this server")
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.post(_url(path), json=payload)
+    except httpx.HTTPError as exc:
+        raise TranslationError(f"the translation service is unreachable: {exc}") from exc
+    if resp.status_code != 200:
+        detail = resp.text[:200]
+        raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
+    return resp.json()
+
+
+_languages: tuple[float, list[Language]] = (0.0, [])
+
+
+async def languages() -> list[Language]:
+    """What the service can translate between (cached for a few minutes)."""
+    global _languages
+    stamp, cached = _languages
+    if cached and time.monotonic() - stamp < _LANG_CACHE_S:
+        return cached
+    if not enabled():
+        return []
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
+            resp = await client.get(_url("/languages"))
+        resp.raise_for_status()
+        rows = resp.json()
+    except (httpx.HTTPError, ValueError) as exc:
+        logger.info("translation languages unavailable: %s", exc)
+        return []
+    found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r]
+    _languages = (time.monotonic(), found)
+    return found
+
+
+_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
+MIN_LETTERS = 25  # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
+MAX_DETECT = 60  # pieces of one message asked about
+MIN_SHARE = 0.15  # of a message's letters, for its language to count
+
+
+async def _detect_one(text: str) -> str:
+    result = await _post("/detect", {"q": text[:DETECT_CHARS]})
+    if not isinstance(result, list) or not result:
+        return ""
+    best = result[0]
+    if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE:
+        return ""
+    return str(best.get("language", ""))
+
+
+def _weight(text: str) -> int:
+    """How much text this is: letters, a Chinese/Japanese/Korean sign as a word."""
+    return sum(3 if _CJK.match(c) else 1 for c in text if c.isalpha())
+
+
+async def languages_of(parts: list[str]) -> list[str]:
+    """The language of each of ``parts`` (paragraphs, in order). Parts too
+    short to tell take the language of the next part that could be told (a
+    greeting goes with what follows it), the last ones that of the one before;
+    parts without letters get ''."""
+    # Addresses say nothing about a language ("E-mail: lianshaojuan.lsj@…" → Estonian).
+    parts = [_ADDRESS.sub(" ", part) for part in parts]
+    told: list[str] = [""] * len(parts)
+    asked = [
+        i
+        for i, part in enumerate(parts)
+        if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
+    ][:MAX_DETECT]
+    gate = asyncio.Semaphore(4)
+
+    async def one(i: int) -> None:
+        async with gate:
+            told[i] = await _detect_one(parts[i])
+
+    await asyncio.gather(*(one(i) for i in asked))
+    out = list(told)
+    following = ""
+    for i in reversed(range(len(parts))):
+        if told[i]:
+            following = told[i]
+        elif any(c.isalpha() for c in parts[i]):
+            out[i] = following
+    previous = ""
+    for i, part in enumerate(parts):
+        if told[i]:
+            previous = told[i]
+        elif not out[i] and any(c.isalpha() for c in part):
+            out[i] = previous
+    return out
+
+
+async def detect(text: str) -> str:
+    """The languages of ``text`` (a message's plain text), most of it first,
+    comma-separated ("zh-Hans,pl,en"); '' if unsure. A message can be in
+    several: each language that makes up a good part of it counts."""
+    own = own_text(text)
+    if len(" ".join(own.split())) < 20:
+        return ""  # a line or two says too little
+    lines: list[str] = []
+    total = 0
+    for line in (x.strip() for x in own.splitlines()):
+        if line and total < DETECT_CHARS * 4:
+            lines.append(line)
+            total += len(line)
+    found = await languages_of(lines)
+    weight: dict[str, int] = {}
+    for line, language in zip(lines, found, strict=True):
+        if language:
+            weight[language] = weight.get(language, 0) + _weight(_ADDRESS.sub(" ", line))
+    if not weight:
+        return await _detect_one(" ".join(own.split()))
+    all_of_it = sum(weight.values())
+    ranked = sorted(weight, key=lambda k: weight[k], reverse=True)
+    return ",".join(k for k in ranked if weight[k] / all_of_it >= MIN_SHARE)
+
+
+def _base(code: str) -> str:
+    return code.lower().split("-")[0]
+
+
+async def _translate_paragraphs(html: str, target: str) -> str:
+    """``html`` in more than one language, in ``target``: each paragraph from
+    its own language; paragraphs already in ``target`` are left as they are."""
+    pieces, paragraphs = prep.split_pieces(html)
+    found = await languages_of([p.text for p in paragraphs])
+    # Chinese and Japanese go without spaces: two pieces of one sentence
+    # ("…奎琳，</span>我从…") need one between them once translated.
+    followed = {i for p in paragraphs for i in p.pieces[:-1]}
+    by_language: dict[str, list[int]] = {}
+    for paragraph, language in zip(paragraphs, found, strict=True):
+        if language and _base(language) != _base(target):
+            by_language.setdefault(language, []).extend(paragraph.pieces)
+    for language, where in by_language.items():
+        todo: list[tuple[int, str, str, str]] = []
+        for i in where:
+            m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
+            lead, core, trail = m.groups() if m else ("", "", "")
+            if not trail and i in followed and _base(language) in _NO_SPACES:
+                trail = " "
+            if any(c.isalnum() for c in core):
+                todo.append((i, lead, core, trail))
+        for start in range(0, len(todo), 100):
+            batch = todo[start : start + 100]
+            result = await _post(
+                "/translate",
+                {"q": [core for _, _, core, _ in batch], "source": language, "target": target},
+            )
+            out = result.get("translatedText") if isinstance(result, dict) else None
+            if not isinstance(out, list) or len(out) != len(batch):
+                raise TranslationError("the translation service sent no translation")
+            for (i, lead, _, trail), text in zip(batch, out, strict=True):
+                pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
+    return "".join(pieces)
+
+
+_NO_SPACES = {"zh", "ja", "th"}
+_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
+
+
+# Web and mail addresses in the text: the models break them up ("https: / /
+# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps.
+_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
+_TAG = re.compile(r"(<[^>]*>)")
+
+
+def protect_addresses(html: str) -> str:
+    """Mark the addresses in ``html``'s text as not to be translated (style
+    blocks and tag attributes are left alone)."""
+    out: list[str] = []
+    in_style = False
+    for piece in _TAG.split(html):
+        if piece.startswith("<"):
+            lowered = piece.lower()
+            if lowered.startswith("<style"):
+                in_style = True
+            elif lowered.startswith("</style"):
+                in_style = False
+            out.append(piece)
+        elif in_style or not piece:
+            out.append(piece)
+        else:
+            out.append(_ADDRESS.sub(lambda m: f'<span translate="no">{m.group(0)}</span>', piece))
+    return "".join(out)
+
+
+_PUNCT = re.compile(r"[\W_]+")
+MAX_NAMES = 200  # names checked for one message
+
+
+async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]:
+    """Of ``names``, those the model changes when it translates them alone
+    ("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one
+    request for them all. If the check fails, all of them."""
+    wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES]
+    if not wanted:
+        return set()
+    try:
+        result = await _post(
+            "/translate", {"q": wanted, "source": source or "auto", "target": target}
+        )
+    except TranslationError:
+        return set(wanted)
+    out = result.get("translatedText") if isinstance(result, dict) else None
+    if not isinstance(out, list) or len(out) != len(wanted):
+        return set(wanted)
+
+    def same(a: str, b: str) -> bool:
+        return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold()
+
+    return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))}
+
+
+# Bumped when what goes to the service, or what is done with the result,
+# changes: translations kept from before are made again.
+PIPELINE_VERSION = 5
+
+
+async def translate_html(
+    html: str, source: str, target: str, keep: Iterable[str] = (), names: Iterable[str] = ()
+) -> str:
+    """``html`` (a sanitized message body) in ``target``; source "auto", a code,
+    or several ("zh-Hans,pl,en": then each paragraph from its own language).
+    ``keep``: words always left as they are; ``names``: left as they are if
+    the model would change them (translation_prep)."""
+    if len(html) > MAX_HTML_CHARS:
+        raise TranslationError("this message is too long to translate")
+    sources = [x for x in (source or "").split(",") if x and x != "auto"]
+    prepared = prep.join_wrapped_lines(html)
+    for language in sources:
+        prepared = prep.translate_phrases(prepared, language, target)
+    single = sources[0] if len(sources) == 1 else "auto"
+    candidates = {*names, *prep.find_names(prepared, single if sources else "")}
+    terms = {*keep, *await names_to_keep(candidates, single, target)}
+    prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms)))
+    if len(sources) > 1:
+        return prep.tidy(await _translate_paragraphs(prepared, target))
+    result = await _post(
+        "/translate",
+        {"q": prepared, "source": source or "auto", "target": target, "format": "html"},
+    )
+    if not isinstance(result, dict) or "translatedText" not in result:
+        raise TranslationError("the translation service sent no translation")
+    return prep.tidy(str(result["translatedText"]))

backend/app/translation_prep.py

@@ -0,0 +1,332 @@
+"""Getting a message ready for translation, and tidying what comes back.
+
+LibreTranslate translates each piece of text between tags on its own, and its
+small models stumble on things a reader would not: a sentence wrapped onto two
+lines becomes two sentences ("ask you about" / "Plan for next month"),
+"Best," becomes "Najlepsze," and a company name gets translated word by word.
+So before a message goes out:
+
+- lines that carry on a sentence (no end punctuation, next starts lowercase)
+  are joined, in HTML (<br>, one <div> per line) and in wrapped plain text;
+- greetings and sign-offs that make a whole line are put in the target
+  language from a small phrase table (English, Polish, German);
+- names are marked translate="no", which LibreTranslate keeps — the people
+  on the message, the user's own names and signature names, and capitalised
+  multi-word names mid-sentence — but only the names the model would change
+  ("Milan" → "Mediolan"): a kept name splits its sentence, and LibreTranslate
+  translates each piece alone ("Dear <kept>Lukas Remis</kept>," →
+  "Kochanie."), so a name the model leaves alone stays in the sentence;
+- so are emoji, file names ("A_0023C017_p1CAC.mov") and text without a letter;
+- a message in more than one language is translated paragraph by paragraph,
+  each from its own language (split_pieces); those in the target stay;
+- and afterwards, the space LibreTranslate drops after a kept word comes back.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import re
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+
+KEEP_OPEN = '<span translate="no">'
+_TAG = re.compile(r"(<[^>]*>)")
+_LOWER_START = "a-ząćęłńóśźżäöüßàâçéèêëîïôûùüÿñ"
+_UPPER = "A-ZĄĆĘŁŃÓŚŹŻÄÖÜÀÂÇÉÈÊËÎÏÔÛÙÜŸÑ"
+
+# --- joining wrapped lines -------------------------------------------------------
+
+# A line break between a word (or comma) and a lowercase word carries a sentence on.
+_HTML_SOFT_BREAK = re.compile(
+    rf"([\w,;–-])[ \t]*(?:<br\s*/?>|</div>\s*<div[^>]*>|</p>\s*<p[^>]*>)\s*(?=[{_LOWER_START}])"
+)
+_TEXT_SOFT_BREAK = re.compile(rf"([^\s.!?:;])[ \t]*\n[ \t]*(?=[{_LOWER_START}])")
+
+
+def join_wrapped_lines(html: str) -> str:
+    html = _HTML_SOFT_BREAK.sub(r"\1 ", html)
+    # Plain text (wm-plaintext): newlines inside text, outside tags.
+    return "".join(
+        p if p.startswith("<") else _TEXT_SOFT_BREAK.sub(r"\1 ", p) for p in _TAG.split(html)
+    )
+
+
+# --- greetings and sign-offs -----------------------------------------------------
+
+# source language → the line as written (lower case) → target language → line.
+_PHRASES: dict[str, dict[str, dict[str, str]]] = {
+    "en": {
+        "best": {"pl": "Pozdrawiam", "de": "Viele Grüße"},
+        "best regards": {"pl": "Pozdrawiam serdecznie", "de": "Beste Grüße"},
+        "best wishes": {"pl": "Najlepsze życzenia", "de": "Beste Grüße"},
+        "kind regards": {"pl": "Z pozdrowieniami", "de": "Freundliche Grüße"},
+        "warm regards": {"pl": "Serdecznie pozdrawiam", "de": "Herzliche Grüße"},
+        "regards": {"pl": "Pozdrawiam", "de": "Grüße"},
+        "cheers": {"pl": "Pozdrawiam", "de": "Viele Grüße"},
+        "sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"},
+        "yours sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"},
+        "thanks": {"pl": "Dzięki", "de": "Danke"},
+        "many thanks": {"pl": "Wielkie dzięki", "de": "Vielen Dank"},
+        "thank you": {"pl": "Dziękuję", "de": "Vielen Dank"},
+        "thanks in advance": {"pl": "Z góry dziękuję", "de": "Vielen Dank im Voraus"},
+        "all the best": {"pl": "Wszystkiego dobrego", "de": "Alles Gute"},
+        "talk soon": {"pl": "Do usłyszenia", "de": "Bis bald"},
+        "have a great day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"},
+        "have a nice day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"},
+    },
+    "pl": {
+        "pozdrawiam": {"en": "Best regards", "de": "Viele Grüße"},
+        "pozdrawiam serdecznie": {"en": "Warm regards", "de": "Herzliche Grüße"},
+        "serdecznie pozdrawiam": {"en": "Warm regards", "de": "Herzliche Grüße"},
+        "z poważaniem": {"en": "Kind regards", "de": "Mit freundlichen Grüßen"},
+        "z pozdrowieniami": {"en": "Kind regards", "de": "Freundliche Grüße"},
+        "dziękuję": {"en": "Thank you", "de": "Vielen Dank"},
+        "dzięki": {"en": "Thanks", "de": "Danke"},
+        "z góry dziękuję": {"en": "Thanks in advance", "de": "Vielen Dank im Voraus"},
+        "miłego dnia": {"en": "Have a nice day", "de": "Einen schönen Tag noch"},
+        "do usłyszenia": {"en": "Talk soon", "de": "Bis bald"},
+    },
+    "de": {
+        "mit freundlichen grüßen": {"en": "Kind regards", "pl": "Z poważaniem"},
+        "freundliche grüße": {"en": "Kind regards", "pl": "Z pozdrowieniami"},
+        "viele grüße": {"en": "Best regards", "pl": "Pozdrawiam"},
+        "beste grüße": {"en": "Best regards", "pl": "Pozdrawiam"},
+        "liebe grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"},
+        "herzliche grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"},
+        "danke": {"en": "Thanks", "pl": "Dzięki"},
+        "vielen dank": {"en": "Many thanks", "pl": "Bardzo dziękuję"},
+        "bis bald": {"en": "Talk soon", "pl": "Do usłyszenia"},
+    },
+}
+# A whole line: after a tag or line start, before a tag or line end.
+_LINE = re.compile(r"(^|>|\n)([ \t]*)([^<>\n]{2,40}?)([ \t]*[,.!]?)([ \t]*)(?=<|\n|$)")
+
+
+def _base(code: str) -> str:
+    return code.lower().split("-")[0]
+
+
+def translate_phrases(html: str, source: str, target: str) -> str:
+    table = _PHRASES.get(_base(source), {})
+    if not table:
+        return html
+    tgt = _base(target)
+
+    def repl(m: re.Match[str]) -> str:
+        found = table.get(" ".join(m.group(3).lower().split()), {}).get(tgt)
+        if found is None:
+            return m.group(0)
+        return f"{m.group(1)}{m.group(2)}{KEEP_OPEN}{found}{m.group(4).strip()}</span>{m.group(5)}"
+
+    # The pattern never crosses a tag: it stops at "<" and ">".
+    return _LINE.sub(repl, html)
+
+
+# --- names ------------------------------------------------------------------------
+
+# A capitalised word; dots only inside it ("U.S"), never at its end — a name
+# does not run on over a sentence's end ("NukeX. My work…").
+_WORD = rf"[{_UPPER}][\w'’&-]*(?:\.[\w'’&-]+)*"
+# Two or more capitalised words in a row ("Imaginary Pixels", "Trusted Partner
+# Network"), after a lowercase word: a name, not the start of a sentence.
+_MULTI_NAME = re.compile(
+    rf"(?<=[{_LOWER_START},] )({_WORD}(?: (?:&|of|the|de|von|van|{_WORD}))* {_WORD})"
+)
+# Languages that capitalise every noun: capitals say nothing about names there.
+_CAPITALISE_NOUNS = {"de", "lb"}
+
+
+def _title_case(text: str) -> bool:
+    """A Headline Like This: nearly every longer word capitalised (short ones
+    — of, and, in — often are not). A sentence full of names is not one."""
+    words = [w for w in re.findall(r"[^\W\d_][\w'’-]*", text) if len(w) >= 4]
+    return len(words) >= 3 and sum(w[0].isupper() for w in words) / len(words) > 0.8
+
+
+def find_names(html: str, source: str) -> set[str]:
+    """Capitalised multi-word names in the middle of sentences (not in headings
+    or Title Case lines, and not for languages that capitalise all nouns)."""
+    if _base(source) in _CAPITALISE_NOUNS:
+        return set()
+    found: set[str] = set()
+    for piece in _TAG.split(html):
+        if piece.startswith("<") or _title_case(piece):
+            continue
+        found.update(m.group(1).rstrip(".") for m in _MULTI_NAME.finditer(piece))
+    return found
+
+
+_SIG_NAME = re.compile(rf"{_WORD}(?: (?:&|of|the|{_WORD}))* {_WORD}")
+
+
+def find_names_in_signature(text: str) -> list[str]:
+    """Capitalised multi-word names in a signature ("Imaginary Pixels"): the
+    user's company or title, which a translation should leave alone."""
+    return [m.group(0).rstrip(".") for m in _SIG_NAME.finditer(text)]
+
+
+def keep_terms(html: str, terms: Iterable[str]) -> str:
+    """Mark each of ``terms`` in ``html``'s text translate="no" (longest first;
+    whole words; style blocks and tags left alone)."""
+    words = sorted({t.strip() for t in terms if len(t.strip()) >= 2}, key=len, reverse=True)
+    if not words:
+        return html
+    pattern = re.compile(r"(?<![\w>])(" + "|".join(re.escape(w) for w in words) + r")(?![\w<])")
+    return _mark(html, pattern)
+
+
+# Emoji and other pictographs (with their skin tones, joiners and variation
+# selectors): on its own, an emoji comes back as a made-up word ("😊" →
+# "Uzasadnienie").
+_SYMBOLS = re.compile(
+    "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff"
+    "\U0001f000-\U0001faff]"
+    "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff"
+    "\U0001f000-\U0001faff\ufe0f\u200d\u20e3]*"
+)
+# File names and codes with underscores ("A_0023C017_260521_p1CAC.mov",
+# "final_v2"): the models split them at every "_" and dot.
+_FILE_NAME = re.compile(
+    r"(?<![\w./-])(?:[\w-]*_[\w-]*(?:\.[\w-]+)*|[\w-]+(?:\.[\w-]+)*\.[a-z][a-z0-9]{1,4})(?![\w/-])",
+    re.IGNORECASE,
+)
+# International phone numbers ("+48 780 549 877"): the models space out the
+# "+". Only those: a kept piece splits its sentence, and plain numbers
+# ("1300 - 2500 Euro") come through anyway.
+_PHONE = re.compile(r"(?<![\w+])\+\d[\d ().-]{5,}\d(?!\w)")
+# Only names with an extension a file would have, not every "e.g." or "U.S".
+_FILE_EXT = re.compile(r"\.[a-z][a-z0-9]{1,4}$", re.IGNORECASE)
+
+
+def keep_symbols(html: str) -> str:
+    """Mark emoji, file names, handles ("Jacqueline_"), phone numbers and text
+    without a letter in ``html`` translate="no"."""
+    html = _mark(_mark(html, _SYMBOLS), _PHONE)
+
+    def is_file(m: re.Match[str]) -> bool:
+        word = m.group(0)
+        if "_" in word and word.strip("_"):
+            return True
+        return bool(_FILE_EXT.search(word)) and any(c.isdigit() or c in "_-" for c in word)
+
+    return _map_text(_mark(html, _FILE_NAME, keep=is_file), _keep_letterless)
+
+
+_ENTITY = re.compile(r"&#?\w+;")
+_EDGES = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
+
+
+def _keep_letterless(text: str) -> str:
+    """A piece of text with no letter or digit — an invisible byte-order mark,
+    a lone symbol — kept as it is: the models make up a word for it ("\ufeff"
+    → "Uzasadnienie")."""
+    if any(c.isalnum() for c in _ENTITY.sub("", text)):
+        return text
+    lead, core, trail = _EDGES.match(text).groups()  # type: ignore[union-attr]
+    if not _ENTITY.sub("", core).strip():
+        return text  # only spaces and &nbsp;
+    return f"{lead}{KEEP_OPEN}{core}</span>{trail}"
+
+
+def _mark(
+    html: str, pattern: re.Pattern[str], keep: Callable[[re.Match[str]], bool] | None = None
+) -> str:
+    """Wrap ``pattern``'s matches in ``html``'s text in translate="no" spans
+    (where ``keep(match)`` says so)."""
+
+    def repl(m: re.Match[str]) -> str:
+        if keep is not None and not keep(m):
+            return m.group(0)
+        return f"{KEEP_OPEN}{m.group(0)}</span>"
+
+    return _map_text(html, lambda text: pattern.sub(repl, text))
+
+
+def _map_text(html: str, fn: Callable[[str], str]) -> str:
+    """``fn`` applied to each piece of ``html``'s text; style blocks, tags and
+    text already kept are left alone."""
+    out: list[str] = []
+    in_style = in_keep = False
+    for piece in _TAG.split(html):
+        if piece.startswith("<"):
+            low = piece.lower()
+            if low.startswith("<style"):
+                in_style = True
+            elif low.startswith("</style"):
+                in_style = False
+            elif low.startswith(KEEP_OPEN):
+                in_keep = True
+            elif in_keep and low.startswith("</span"):
+                in_keep = False
+            out.append(piece)
+        elif in_style or in_keep or not piece:
+            out.append(piece)
+        else:
+            out.append(fn(piece))
+    return "".join(out)
+
+
+# --- messages in more than one language ------------------------------------------
+
+# Tags that end a paragraph (a line): the text between two of them is one
+# piece of one language.
+_BLOCK = re.compile(
+    r"</?(?:p|div|li|ul|ol|table|thead|tbody|tfoot|tr|td|th|h[1-6]|blockquote|pre|"
+    r"section|article|header|footer|center|br|hr)\b",
+    re.IGNORECASE,
+)
+
+
+@dataclass
+class Paragraph:
+    """One paragraph of a message: its text (for telling its language) and
+    where its translatable pieces are in ``split_pieces``' list."""
+
+    text: str = ""
+    pieces: list[int] = field(default_factory=list)
+
+
+def split_pieces(html: str) -> tuple[list[str], list[Paragraph]]:
+    """``html`` as a list of tags and text, and its paragraphs. Text in style
+    blocks is no paragraph's; text already kept counts for the language but
+    is not translated."""
+    pieces = _TAG.split(html)
+    paragraphs: list[Paragraph] = [Paragraph()]
+    in_style = in_keep = False
+    for i, piece in enumerate(pieces):
+        if piece.startswith("<"):
+            low = piece.lower()
+            if low.startswith("<style"):
+                in_style = True
+            elif low.startswith("</style"):
+                in_style = False
+            elif low.startswith(KEEP_OPEN):
+                in_keep = True
+            elif in_keep and low.startswith("</span"):
+                in_keep = False
+            if _BLOCK.match(piece) and (paragraphs[-1].text.strip() or paragraphs[-1].pieces):
+                paragraphs.append(Paragraph())
+        elif piece and not in_style:
+            paragraphs[-1].text += html_lib.unescape(piece)
+            if not in_keep:
+                paragraphs[-1].pieces.append(i)
+    return pieces, [p for p in paragraphs if p.text.strip()]
+
+
+# --- after translation -------------------------------------------------------------
+
+# LibreTranslate drops the space after a kept span ("Bruton</span>i twoje").
+# Only kept spans: they still carry translate="no" here (the sanitizer that
+# runs next takes it off); other spans are the sender's and stay as they are.
+_KEPT = re.compile(r'(<span translate="no">[^<]*</span>)')
+
+
+def tidy(html: str) -> str:
+    def repl(m: re.Match[str]) -> str:
+        before = html[m.start() - 1] if m.start() > 0 else ""
+        after = html[m.end()] if m.end() < len(html) else ""
+        lead = " " if re.match(r"[\w,;:)]", before) else ""
+        trail = " " if re.match(r"[\w(„\"“]", after) else ""
+        return f"{lead}{m.group(1)}{trail}"
+
+    return _KEPT.sub(repl, html)