M6 hosted mailboxes (off by default); dark message mode; fixes

Hosted mailboxes (SPEC App. D), behind HOSTED_MAILBOXES (off: unchanged):
- Stalwart mail server, provisioned by `python -m app.hosted.provision`;
  directory read from a Postgres view, app passwords only for mail apps
- native account per user, the worker signs in as master for the user
- raw messages of hosted mailboxes only cached, fetched back on demand
- migrations 0025 (accounts.kind, directory view), 0026 (IMAP ids bigint)

Mail:
- dark message mode switch in the reading pane (Thunderbird's algorithm)
- moves and folder creation quote the folder name ("Deleted Items"):
  swipe-to-delete / Trash no longer bounce back
- thread unread counters: lock before recount, sorted lock order,
  UIDVALIDITY re-import recounts, nightly repair_threads
- sidebar accounts sorted alphabetically, more space above headings

Settings: Security headings renamed; antispam accounts are explicit
(none ticked = none filtered), input widths, Activity hidden.
Plugin bundles served with Cache-Control: no-cache.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/indexer/attachments.py

@@ -10,7 +10,9 @@ decoded on its own). It is taken out of the raw message when wanted:
 * :func:`dedupe_existing` — the one-time pass removing the separate copies
   stored before, each only after checking it can be taken out of its message.
 
-Blocking (parsing, disk): call these in a thread.
+Blocking (parsing, disk): call these in a thread — after
+:func:`app.indexer.rawstore.ensure` for the message, which fetches a hosted
+mailbox's message from its mail server when it is not cached.
 """
 
 from __future__ import annotations
@@ -23,7 +25,7 @@ import uuid
 from collections import defaultdict
 from pathlib import Path
 
-from app.indexer import blobstore
+from app.indexer import blobstore, rawstore
 from app.indexer.mime import load_message, part_payload
 
 logger = logging.getLogger(__name__)
@@ -45,8 +47,8 @@ def load(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None)
     legacy = _legacy(user_id, sha)
     if legacy is not None:
         return legacy.read_bytes()
-    raw = blobstore.blob_path(user_id, message_sha)
-    if part_index == NOT_IN_MESSAGE or not raw.exists():
+    raw = rawstore.locate(user_id, message_sha)
+    if part_index == NOT_IN_MESSAGE or raw is None:
         raise FileNotFoundError(sha)
     payload = part_payload(load_message(path=raw), part_index, sha)
     if payload is None:
@@ -82,6 +84,11 @@ def prune_cache(cache: Path) -> None:
             path.unlink(missing_ok=True)
 
 
+def has_file(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether :func:`file` can answer without reading the message."""
+    return _legacy(user_id, sha) is not None or (_cache_dir(user_id) / sha).exists()
+
+
 def file(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> Path:
     """A file holding the attachment, to stream from. Raises FileNotFoundError."""
     legacy = _legacy(user_id, sha)
@@ -167,9 +174,9 @@ async def dedupe_existing(batch: int = 200, *, user_id: uuid.UUID | None = None)
 
         located: dict[uuid.UUID, int] = {}
         for (owner, msg_sha), atts in by_message.items():
-            raw = blobstore.blob_path(owner, msg_sha)
+            raw = rawstore.locate(owner, msg_sha)
             try:
-                indexes = await asyncio.to_thread(_part_indexes, raw) if raw.exists() else {}
+                indexes = await asyncio.to_thread(_part_indexes, raw) if raw is not None else {}
             except Exception:  # noqa: BLE001 - unreadable message: keep the copies
                 logger.warning("attachments: cannot parse %s", raw, exc_info=True)
                 indexes = {}

backend/app/indexer/gc.py

@@ -11,7 +11,12 @@ that for every user, whatever the way the rows went:
   a sent message (its draft is gone) or a composer closed without saving;
 * message/attachment files no row refers to;
 * attachment previews (thumbnails) of attachments that are gone;
-* streaming copies past their keep time (app/indexer/attachments.py).
+* streaming copies past their keep time (app/indexer/attachments.py);
+* cached raw messages of a hosted mailbox beyond the cache's room
+  (app/indexer/rawstore.py).
+
+It also recounts threads whose unread/message counters disagree with their
+messages, and drops threads left empty (app/threading_/threader.py).
 
 A file younger than an hour is left alone: it may belong to work in progress
 (a message being stored, a sent message waiting for its Sent copy).
@@ -31,6 +36,7 @@ from sqlalchemy import delete, or_, select
 
 from app.config import get_settings
 from app.indexer import attachments as att_store
+from app.indexer import rawstore
 
 logger = logging.getLogger(__name__)
 
@@ -94,9 +100,9 @@ def _old(path: Path, now: float) -> bool:
 async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
     """Clean one user's stored files; returns what was removed."""
     from app.db import session_scope
-    from app.models import Attachment, ComposeAttachment, Draft, Message
+    from app.models import Account, Attachment, ComposeAttachment, Draft, Message
 
-    stats = {"uploads": 0, "files": 0, "bytes": 0, "thumbs": 0}
+    stats = {"uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
     async with session_scope() as db:
         # Uploads no draft refers to, once a week old.
         used = _draft_upload_ids(
@@ -116,8 +122,14 @@ async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
     async with session_scope() as db:
         # Every file still referred to. Attachments count only where their file
         # is their only copy (not found in the message, or not yet looked for).
+        # A hosted mailbox's messages are kept by its mail server: their raw
+        # file here (left by a Sent copy filed late) is not needed.
         referenced = set(
-            await db.scalars(select(Message.blob_sha256).where(Message.user_id == user_id))
+            await db.scalars(
+                select(Message.blob_sha256)
+                .join(Account, Account.id == Message.account_id)
+                .where(Message.user_id == user_id, Account.kind != "native")
+            )
         )
         referenced |= set(
             await db.scalars(
@@ -140,6 +152,12 @@ async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
             await db.scalars(select(Attachment.blob_sha256).where(Attachment.user_id == user_id))
         )
 
+    # Counters out of step with their messages (see repair_threads).
+    from app.threading_.threader import repair_threads
+
+    async with session_scope() as db:
+        stats["threads_repaired"] = await repair_threads(db, user_id)
+
     root = Path(get_settings().blob_root) / str(user_id)
     if not root.is_dir():
         return stats
@@ -168,6 +186,7 @@ async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
     cache = root / "cache"
     if cache.is_dir():
         att_store.prune_cache(cache)
+    rawstore.prune(user_id)
     return stats
 
 
@@ -178,7 +197,7 @@ async def sweep(user_id: uuid.UUID | None = None) -> dict[str, int]:
 
     async with session_scope() as db:
         users = [user_id] if user_id is not None else list(await db.scalars(select(User.id)))
-    total = {"users": 0, "uploads": 0, "files": 0, "bytes": 0, "thumbs": 0}
+    total = {"users": 0, "uploads": 0, "files": 0, "bytes": 0, "thumbs": 0, "threads_repaired": 0}
     for uid in users:
         try:
             stats = await sweep_user(uid)

backend/app/indexer/indexer.py

@@ -19,10 +19,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.concurrency import loop_semaphore
 from app.config import get_settings
-from app.indexer import blobstore
+from app.indexer import blobstore, rawstore
 from app.indexer.mime import ParsedEmail, parse_message, parse_message_file
 from app.indexer.sanitize import sanitize_html
-from app.models import Attachment, BodyPart, KnownRecipient, Message
+from app.models import Account, Attachment, BodyPart, KnownRecipient, Message
 from app.providers.base import RemoteMessage
 from app.search.base import SearchDoc
 from app.threading_.normalize import synthesize_message_id
@@ -118,6 +118,13 @@ async def _resolve_message_id(
     return parsed_id
 
 
+async def _keeps_raw(session: AsyncSession, account_id: uuid.UUID) -> bool:
+    """Whether the raw message is kept for good: not for a hosted mailbox, whose
+    mail server holds it (it is only cached, App. D §D.6 item 2)."""
+    kind = await session.scalar(select(Account.kind).where(Account.id == account_id))
+    return kind != "native"
+
+
 async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmail:
     """Parse under the PARSE_CONCURRENCY cap; stream large messages from disk (§7).
 
@@ -128,8 +135,9 @@ async def _parse_bounded(user_id: uuid.UUID, sha: str, raw: bytes) -> ParsedEmai
     settings = get_settings()
     threshold = settings.stream_parse_mb * 1024 * 1024
     async with loop_semaphore("mime_parse", "parse", settings.parse_concurrency):
-        if len(raw) > threshold:
-            return parse_message_file(blobstore.blob_path(user_id, sha))
+        path = rawstore.locate(user_id, sha) if len(raw) > threshold else None
+        if path is not None:
+            return parse_message_file(path)
         return parse_message(raw)
 
 
@@ -154,7 +162,10 @@ async def persist_message(
     if existing is not None:
         return None
 
-    sha = blobstore.store(user_id, remote.raw)
+    if await _keeps_raw(session, account_id):
+        sha = blobstore.store(user_id, remote.raw)
+    else:
+        sha = rawstore.put_cached(user_id, remote.raw)
     parsed = await _parse_bounded(user_id, sha, remote.raw)
 
     message_id_hdr = await _resolve_message_id(

backend/app/indexer/rawstore.py

@@ -0,0 +1,170 @@
+"""Where a message's raw source is (SPEC App. D §D.6 item 2).
+
+An external account's messages are kept whole in the blob store, as always.
+A hosted mailbox's are not: the mail server beside the app holds them, so the
+webmail keeps only what it reads them for (headers, the sanitized body, the
+snippet, the search index) and a **cache** of raw messages — the newest ones
+on arrival, the ones opened since — bounded per user (``RAW_CACHE_MB``,
+least recently used out first). Anything else is fetched from the mail server
+when needed: downloading an attachment, "show original", forwarding.
+
+Layout: ``${BLOB_ROOT}/{user_id}/rawcache/{sha256}``, the same content hash
+``messages.blob_sha256`` names.
+
+* :func:`locate` — the file, if it is here (blob store or cache); cheap, sync;
+* :func:`ensure` — the file, fetched from the mail server if need be (async):
+  call it before handing the work to a thread that reads the file.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import time
+import uuid
+from pathlib import Path
+
+from app.config import get_settings
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+CACHE_DIR = "rawcache"
+KEEP_S = 30 * 24 * 3600  # unused this long: dropped whatever the room
+PRUNE_EVERY = 50
+_writes: dict[uuid.UUID, int] = {}
+
+
+def cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / CACHE_DIR
+
+
+def cache_path(user_id: uuid.UUID, sha: str) -> Path:
+    return cache_dir(user_id) / sha
+
+
+def locate(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The message's raw file, if it is kept or cached here."""
+    kept = blobstore.blob_path(user_id, sha)
+    if kept.exists():
+        return kept
+    cached = cache_path(user_id, sha)
+    if cached.exists():
+        with contextlib.suppress(OSError):
+            cached.touch()  # used again: kept longer
+        return cached
+    return None
+
+
+def load(user_id: uuid.UUID, sha: str) -> bytes:
+    """The raw bytes if they are here. Raises FileNotFoundError."""
+    path = locate(user_id, sha)
+    if path is None:
+        raise FileNotFoundError(sha)
+    return path.read_bytes()
+
+
+def put_cached(user_id: uuid.UUID, data: bytes, sha: str | None = None) -> str:
+    """Cache a raw message (``sha`` names it; its content hash by default)."""
+    sha = sha or blobstore.sha256_hex(data)
+    if blobstore.exists(user_id, sha):
+        return sha  # kept for good already (another account has the same message)
+    path = cache_path(user_id, sha)
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+        tmp.write_bytes(data)
+        tmp.replace(path)
+    else:
+        path.touch()
+    # Pruning lists the whole cache: done every PRUNE_EVERY writes, not each
+    # (a backfill writes thousands); meanwhile it overshoots by that much at most.
+    writes = _writes.get(user_id, 0) + 1
+    _writes[user_id] = writes % PRUNE_EVERY
+    if writes >= PRUNE_EVERY:
+        with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+            prune(user_id)
+    return sha
+
+
+def prune(user_id: uuid.UUID, *, max_bytes: int | None = None) -> int:
+    """Drop cached messages unused for KEEP_S, and the least recently used
+    beyond the cap. Returns bytes freed."""
+    cache = cache_dir(user_id)
+    if not cache.is_dir():
+        return 0
+    cap = max_bytes if max_bytes is not None else get_settings().raw_cache_mb * 1024 * 1024
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file() and not p.name.endswith(".tmp")]
+    now = time.time()
+    total = freed = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > KEEP_S or total > cap:
+            path.unlink(missing_ok=True)
+            freed += st.st_size
+    return freed
+
+
+def drop(user_id: uuid.UUID, sha: str) -> None:
+    cache_path(user_id, sha).unlink(missing_ok=True)
+
+
+async def ensure(message: object) -> Path:
+    """The raw file of ``message`` (a :class:`~app.models.Message`), fetched
+    from the hosted mail server into the cache when it is not here. Raises
+    FileNotFoundError when it cannot be had (an external account's message
+    missing from the blob store; a native one gone from the server)."""
+    from app.models import Message
+
+    assert isinstance(message, Message)
+    found = locate(message.user_id, message.blob_sha256)
+    if found is not None:
+        return found
+    raw = await _fetch_native(message)
+    if raw is None:
+        raise FileNotFoundError(message.blob_sha256)
+    got = blobstore.sha256_hex(raw)
+    if got != message.blob_sha256:
+        # The server's copy differs from what was indexed (it should not):
+        # cached under the indexed name, so the message can still be read.
+        logger.warning("raw of message %s changed on the mail server", message.id)
+    put_cached(message.user_id, raw, message.blob_sha256)
+    return cache_path(message.user_id, message.blob_sha256)
+
+
+async def ensure_bytes(message: object) -> bytes:
+    path = await ensure(message)
+    import asyncio
+
+    return await asyncio.to_thread(path.read_bytes)
+
+
+async def _fetch_native(message: object) -> bytes | None:
+    from sqlalchemy import select
+
+    from app.db import session_scope
+    from app.models import Account, Mailbox, Message
+
+    assert isinstance(message, Message)
+    async with session_scope() as db:
+        row = (
+            await db.execute(
+                select(Account.kind, Account.email, Mailbox.path)
+                .join(Mailbox, Mailbox.account_id == Account.id)
+                .where(Account.id == message.account_id, Mailbox.id == message.mailbox_id)
+            )
+        ).first()
+    if row is None or row.kind != "native":
+        return None
+    from app.hosted.native import native_provider_config
+    from app.providers.base import ProviderError
+    from app.providers.imap_smtp import ImapSmtpProvider
+
+    provider = ImapSmtpProvider(native_provider_config(row.email))
+    try:
+        return await provider.fetch_raw(row.path, message.uid)
+    except ProviderError as exc:
+        logger.warning("cannot fetch message %s from the mail server: %s", message.id, exc)
+        return None
+    finally:
+        await provider.close()

backend/app/indexer/thumbs.py

@@ -87,6 +87,11 @@ def _render(data: bytes, mime_type: str) -> bytes | None:
     return out.getvalue()
 
 
+def made(user_id: uuid.UUID, sha: str) -> bool:
+    """Whether the attachment's thumbnail (or its absence) is settled already."""
+    return _path(user_id, sha, ".webp").exists() or _path(user_id, sha, ".none").exists()
+
+
 def thumbnail(
     user_id: uuid.UUID,
     sha: str,