M6 hosted mailboxes (off by default); dark message mode; fixes

Hosted mailboxes (SPEC App. D), behind HOSTED_MAILBOXES (off: unchanged): - Stalwart mail server, provisioned by `python -m app.hosted.provision`; directory read from a Postgres view, app passwords only for mail apps - native account per user, the worker signs in as master for the user - raw messages of hosted mailboxes only cached, fetched back on demand - migrations 0025 (accounts.kind, directory view), 0026 (IMAP ids bigint) Mail: - dark message mode switch in the reading pane (Thunderbird's algorithm) - moves and folder creation quote the folder name ("Deleted Items"): swipe-to-delete / Trash no longer bounce back - thread unread counters: lock before recount, sorted lock order, UIDVALIDITY re-import recounts, nightly repair_threads - sidebar accounts sorted alphabetically, more space above headings Settings: Security headings renamed; antispam accounts are explicit (none ticked = none filtered), input widths, Activity hidden. Plugin bundles served with Cache-Control: no-cache. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 19:25:10 +02:00
parent d45fbf2c44
commit b88855931e
62 changed files with 2427 additions and 188 deletions
@@ -0,0 +1,170 @@
+"""Where a message's raw source is (SPEC App. D §D.6 item 2).
+
+An external account's messages are kept whole in the blob store, as always.
+A hosted mailbox's are not: the mail server beside the app holds them, so the
+webmail keeps only what it reads them for (headers, the sanitized body, the
+snippet, the search index) and a **cache** of raw messages — the newest ones
+on arrival, the ones opened since — bounded per user (``RAW_CACHE_MB``,
+least recently used out first). Anything else is fetched from the mail server
+when needed: downloading an attachment, "show original", forwarding.
+
+Layout: ``${BLOB_ROOT}/{user_id}/rawcache/{sha256}``, the same content hash
+``messages.blob_sha256`` names.
+
+* :func:`locate` — the file, if it is here (blob store or cache); cheap, sync;
+* :func:`ensure` — the file, fetched from the mail server if need be (async):
+  call it before handing the work to a thread that reads the file.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import time
+import uuid
+from pathlib import Path
+
+from app.config import get_settings
+from app.indexer import blobstore
+
+logger = logging.getLogger(__name__)
+
+CACHE_DIR = "rawcache"
+KEEP_S = 30 * 24 * 3600  # unused this long: dropped whatever the room
+PRUNE_EVERY = 50
+_writes: dict[uuid.UUID, int] = {}
+
+
+def cache_dir(user_id: uuid.UUID) -> Path:
+    return blobstore.blob_path(user_id, "00").parent.parent / CACHE_DIR
+
+
+def cache_path(user_id: uuid.UUID, sha: str) -> Path:
+    return cache_dir(user_id) / sha
+
+
+def locate(user_id: uuid.UUID, sha: str) -> Path | None:
+    """The message's raw file, if it is kept or cached here."""
+    kept = blobstore.blob_path(user_id, sha)
+    if kept.exists():
+        return kept
+    cached = cache_path(user_id, sha)
+    if cached.exists():
+        with contextlib.suppress(OSError):
+            cached.touch()  # used again: kept longer
+        return cached
+    return None
+
+
+def load(user_id: uuid.UUID, sha: str) -> bytes:
+    """The raw bytes if they are here. Raises FileNotFoundError."""
+    path = locate(user_id, sha)
+    if path is None:
+        raise FileNotFoundError(sha)
+    return path.read_bytes()
+
+
+def put_cached(user_id: uuid.UUID, data: bytes, sha: str | None = None) -> str:
+    """Cache a raw message (``sha`` names it; its content hash by default)."""
+    sha = sha or blobstore.sha256_hex(data)
+    if blobstore.exists(user_id, sha):
+        return sha  # kept for good already (another account has the same message)
+    path = cache_path(user_id, sha)
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+        tmp.write_bytes(data)
+        tmp.replace(path)
+    else:
+        path.touch()
+    # Pruning lists the whole cache: done every PRUNE_EVERY writes, not each
+    # (a backfill writes thousands); meanwhile it overshoots by that much at most.
+    writes = _writes.get(user_id, 0) + 1
+    _writes[user_id] = writes % PRUNE_EVERY
+    if writes >= PRUNE_EVERY:
+        with contextlib.suppress(OSError):  # a file removed meanwhile by another request
+            prune(user_id)
+    return sha
+
+
+def prune(user_id: uuid.UUID, *, max_bytes: int | None = None) -> int:
+    """Drop cached messages unused for KEEP_S, and the least recently used
+    beyond the cap. Returns bytes freed."""
+    cache = cache_dir(user_id)
+    if not cache.is_dir():
+        return 0
+    cap = max_bytes if max_bytes is not None else get_settings().raw_cache_mb * 1024 * 1024
+    files = [(p, p.stat()) for p in cache.iterdir() if p.is_file() and not p.name.endswith(".tmp")]
+    now = time.time()
+    total = freed = 0
+    for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
+        total += st.st_size
+        if now - st.st_mtime > KEEP_S or total > cap:
+            path.unlink(missing_ok=True)
+            freed += st.st_size
+    return freed
+
+
+def drop(user_id: uuid.UUID, sha: str) -> None:
+    cache_path(user_id, sha).unlink(missing_ok=True)
+
+
+async def ensure(message: object) -> Path:
+    """The raw file of ``message`` (a :class:`~app.models.Message`), fetched
+    from the hosted mail server into the cache when it is not here. Raises
+    FileNotFoundError when it cannot be had (an external account's message
+    missing from the blob store; a native one gone from the server)."""
+    from app.models import Message
+
+    assert isinstance(message, Message)
+    found = locate(message.user_id, message.blob_sha256)
+    if found is not None:
+        return found
+    raw = await _fetch_native(message)
+    if raw is None:
+        raise FileNotFoundError(message.blob_sha256)
+    got = blobstore.sha256_hex(raw)
+    if got != message.blob_sha256:
+        # The server's copy differs from what was indexed (it should not):
+        # cached under the indexed name, so the message can still be read.
+        logger.warning("raw of message %s changed on the mail server", message.id)
+    put_cached(message.user_id, raw, message.blob_sha256)
+    return cache_path(message.user_id, message.blob_sha256)
+
+
+async def ensure_bytes(message: object) -> bytes:
+    path = await ensure(message)
+    import asyncio
+
+    return await asyncio.to_thread(path.read_bytes)
+
+
+async def _fetch_native(message: object) -> bytes | None:
+    from sqlalchemy import select
+
+    from app.db import session_scope
+    from app.models import Account, Mailbox, Message
+
+    assert isinstance(message, Message)
+    async with session_scope() as db:
+        row = (
+            await db.execute(
+                select(Account.kind, Account.email, Mailbox.path)
+                .join(Mailbox, Mailbox.account_id == Account.id)
+                .where(Account.id == message.account_id, Mailbox.id == message.mailbox_id)
+            )
+        ).first()
+    if row is None or row.kind != "native":
+        return None
+    from app.hosted.native import native_provider_config
+    from app.providers.base import ProviderError
+    from app.providers.imap_smtp import ImapSmtpProvider
+
+    provider = ImapSmtpProvider(native_provider_config(row.email))
+    try:
+        return await provider.fetch_raw(row.path, message.uid)
+    except ProviderError as exc:
+        logger.warning("cannot fetch message %s from the mail server: %s", message.id, exc)
+        return None
+    finally:
+        await provider.close()