M6 hosted mailboxes (off by default); dark message mode; fixes

Hosted mailboxes (SPEC App. D), behind HOSTED_MAILBOXES (off: unchanged): - Stalwart mail server, provisioned by `python -m app.hosted.provision`; directory read from a Postgres view, app passwords only for mail apps - native account per user, the worker signs in as master for the user - raw messages of hosted mailboxes only cached, fetched back on demand - migrations 0025 (accounts.kind, directory view), 0026 (IMAP ids bigint) Mail: - dark message mode switch in the reading pane (Thunderbird's algorithm) - moves and folder creation quote the folder name ("Deleted Items"): swipe-to-delete / Trash no longer bounce back - thread unread counters: lock before recount, sorted lock order, UIDVALIDITY re-import recounts, nightly repair_threads - sidebar accounts sorted alphabetically, more space above headings Settings: Security headings renamed; antispam accounts are explicit (none ticked = none filtered), input widths, Activity hidden. Plugin bundles served with Cache-Control: no-cache. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 19:25:10 +02:00
parent d45fbf2c44
commit b88855931e
62 changed files with 2427 additions and 188 deletions
@@ -0,0 +1,323 @@
+"""Set up the mail server for hosted mailboxes (SPEC App. D §D.2/§D.3). Idempotent.
+
+Run after the migrations (the ``migrate`` service in production; in dev
+``python -m app.hosted.provision``). Two halves:
+
+1. **Postgres** (as the app's own, privileged database user): the mail
+   server's role ``stalwart`` (password ``STALWART_DB_PASSWORD``), which owns its
+   database and may read the ``mail_directory`` view of the app's database and
+   nothing else there; and the worker's sign-in in ``mail_service_accounts``.
+   The mail server cannot start before this has run once.
+2. **The mail server** (its management API, as the recovery admin): the mail
+   domain and host name, mail kept on disk (not in Postgres), the SQL directory
+   over ``mail_directory`` as the only source of accounts, the worker's
+   impersonate-only role, the submission port for mail apps.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+import asyncpg  # type: ignore[import-untyped]
+
+from app.config import get_settings
+from app.hosted import admin, mail_hostname, worker_address
+from app.identity.passwords import hash_password, verify_password
+
+logger = logging.getLogger(__name__)
+
+DB_ROLE = "stalwart"
+DIRECTORY_NAME = "Ultimo webmail users"
+WORKER_ROLE_NAME = "Ultimo webmail worker"
+BLOB_PATH = "/var/lib/stalwart/blobs"
+
+# The directory's queries ($1: the address signed in with / mail is for). A
+# blocked user cannot sign in, but still receives mail.
+QUERY_LOGIN = (
+    "SELECT name, secret, type, description FROM mail_directory WHERE name = $1 AND active"
+)
+QUERY_RECIPIENT = (
+    "SELECT name, NULL::text AS secret, type, description FROM mail_directory WHERE name = $1"
+)
+
+
+def _pg_dsn() -> str:
+    return get_settings().database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+
+
+def _app_db_name() -> str:
+    return urlparse(_pg_dsn()).path.lstrip("/")
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+async def provision_database() -> None:
+    settings = get_settings()
+    if not settings.stalwart_db_password or not settings.stalwart_master_secret:
+        raise RuntimeError("STALWART_DB_PASSWORD and STALWART_MASTER_SECRET must be set")
+    conn = await asyncpg.connect(_pg_dsn())
+    try:
+        password = _quote_literal(settings.stalwart_db_password)
+        if await conn.fetchval("SELECT 1 FROM pg_roles WHERE rolname = $1", DB_ROLE):
+            await conn.execute(f"ALTER ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        else:
+            await conn.execute(f"CREATE ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
+        db = settings.stalwart_db_name
+        if not await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db):
+            await conn.execute(f'CREATE DATABASE "{db}" OWNER {DB_ROLE}')
+            logger.info("hosted: created database %s", db)
+        await conn.execute(f"GRANT SELECT ON mail_directory TO {DB_ROLE}")
+
+        # The worker's sign-in: re-hashed only when the secret changed.
+        name = worker_address()
+        current = await conn.fetchval(
+            "SELECT secret FROM mail_service_accounts WHERE name = $1", name
+        )
+        if current is None or not verify_password(current, settings.stalwart_master_secret):
+            await conn.execute(
+                """
+                INSERT INTO mail_service_accounts (name, secret, description)
+                VALUES ($1, $2, 'Webmail worker')
+                ON CONFLICT (name) DO UPDATE SET secret = EXCLUDED.secret
+                """,
+                name,
+                hash_password(settings.stalwart_master_secret),
+            )
+    finally:
+        await conn.close()
+
+
+# ---- the mail server ------------------------------------------------------------------
+
+
+async def _wait_for_server(timeout_s: float = 120.0) -> None:
+    deadline = asyncio.get_running_loop().time() + timeout_s
+    while True:
+        try:
+            await admin.get_one("SystemSettings")
+            return
+        except admin.MailServerError:
+            if asyncio.get_running_loop().time() > deadline:
+                raise
+            await asyncio.sleep(2)
+
+
+def _by(items: list[dict[str, Any]], key: str, value: str) -> dict[str, Any] | None:
+    return next((i for i in items if i.get(key) == value), None)
+
+
+async def _domain() -> str:
+    name = get_settings().app_domain.lower()
+    found = _by(await admin.get_all("Domain"), "name", name)
+    if found is not None:
+        return str(found["id"])
+    domain_id = await admin.create("Domain", {"name": name})
+    logger.info("hosted: created mail domain %s", name)
+    return domain_id
+
+
+async def _directory() -> str:
+    settings = get_settings()
+    dsn = urlparse(_pg_dsn())
+    value: dict[str, Any] = {
+        "@type": "Sql",
+        "description": DIRECTORY_NAME,
+        "columnEmail": "name",
+        "columnSecret": "secret",
+        "columnClass": "type",
+        "columnDescription": "description",
+        "queryLogin": QUERY_LOGIN,
+        "queryRecipient": QUERY_RECIPIENT,
+        "queryMemberOf": None,
+        "queryEmailAliases": None,
+        "store": {
+            "@type": "PostgreSql",
+            "host": settings.stalwart_pg_host,
+            "port": 5432 if settings.stalwart_pg_host == "postgres" else (dsn.port or 5432),
+            "database": _app_db_name(),
+            "authUsername": DB_ROLE,
+            "authSecret": {"@type": "EnvironmentVariable", "variableName": "STALWART_DB_PASSWORD"},
+            "useTls": False,
+            "allowInvalidCerts": False,
+            "poolMaxConnections": 5,
+            "poolRecyclingMethod": "fast",
+            "readReplicas": {},
+            "timeout": 15000,
+            "options": None,
+        },
+    }
+    found = _by(await admin.get_all("Directory"), "description", DIRECTORY_NAME)
+    if found is None:
+        directory_id = await admin.create("Directory", value)
+        logger.info("hosted: created the SQL directory")
+    else:
+        directory_id = str(found["id"])
+        await admin.update("Directory", directory_id, value)
+    await admin.update("Authentication", "singleton", {"directoryId": directory_id})
+    return directory_id
+
+
+async def _worker_role() -> str:
+    perms = {"impersonate": True, "authenticate": True}
+    found = _by(await admin.get_all("Role"), "description", WORKER_ROLE_NAME)
+    if found is not None:
+        role_id = str(found["id"])
+        await admin.update("Role", role_id, {"enabledPermissions": perms})
+        return role_id
+    return await admin.create(
+        "Role", {"description": WORKER_ROLE_NAME, "enabledPermissions": perms}
+    )
+
+
+async def _worker_account(role_id: str) -> None:
+    """The worker may act for any mailbox and do nothing as itself."""
+    # Signing in brings it from the directory into the server's registry. With
+    # its role already set that sign-in fails (it may not open a session as
+    # itself) — after the account exists, which is all this needs.
+    worker = worker_address()
+    acc = await admin.account_id(worker)
+    if acc is None:
+        import httpx
+
+        async with httpx.AsyncClient(timeout=15) as client:
+            await client.get(
+                f"{get_settings().stalwart_url}/jmap/session",
+                auth=(worker, get_settings().stalwart_master_secret),
+            )
+        acc = await admin.account_id(worker)
+    if acc is None:
+        raise admin.MailServerError("the worker account did not appear on the mail server")
+    await admin.update("Account", acc, {"roles": {"@type": "Custom", "roleIds": {role_id: True}}})
+
+
+async def _reload() -> None:
+    """Apply the settings now. Stalwart 0.16.19 ships a few spam-list rules its
+    own validator rejects (``bit_and`` in SURBL tags); they never load, but
+    they make every reload fail — so they are removed, which changes nothing."""
+    for _ in range(10):
+        try:
+            await admin.create("Action", {"@type": "ReloadSettings"})
+            return
+        except admin.MailServerError as exc:
+            match = re.search(r"'object': '(SpamDnsblServer)', 'id': '(\w+)'", str(exc))
+            if match is None:
+                raise
+            logger.info("hosted: removing the invalid built-in rule %s", match.group(2))
+            await admin.destroy(match.group(1), [match.group(2)])
+    raise admin.MailServerError("settings reload kept failing")
+
+
+async def _logging() -> None:
+    """Log to the console (``docker logs``), not to files inside the container."""
+    for tracer in await admin.get_all("Tracer"):
+        if tracer.get("@type") != "Stdout":
+            await admin.update(
+                "Tracer",
+                str(tracer["id"]),
+                {
+                    "@type": "Stdout",
+                    "ansi": False,
+                    "buffered": False,
+                    "enable": True,
+                    "level": "info",
+                    "events": {},
+                    "eventsPolicy": "exclude",
+                },
+            )
+
+
+async def _blob_store() -> bool:
+    """Mail on disk (``DATA_ROOT/stalwart``), not inside Postgres: database
+    backups stay small. Takes a restart; True when one is needed."""
+    current = await admin.get_one("BlobStore")
+    if current.get("@type") == "FileSystem" and current.get("path") == BLOB_PATH:
+        return False
+    await admin.update(
+        "BlobStore", "singleton", {"@type": "FileSystem", "path": BLOB_PATH, "depth": 2}
+    )
+    logger.info("hosted: mail is now kept on disk (takes effect after a restart)")
+    return True
+
+
+async def _listeners() -> None:
+    """Add submission with STARTTLS (587) beside the defaults: smtp 25, submissions
+    465, imaps 993 (and http 8080 for this API)."""
+    listeners = await admin.get_all("NetworkListener")
+    if _by(listeners, "name", "submission") is None:
+        await admin.create(
+            "NetworkListener",
+            {"name": "submission", "bind": {"[::]:587": True}, "protocol": "smtp", "useTls": True},
+        )
+
+
+# Folder names for new mailboxes: the plain ones mail apps expect, not
+# Stalwart's Outlook-style defaults ("Deleted Items", "Junk Mail", "Sent
+# Items"). Archive is made too: the webmail archives into it.
+FOLDERS = {
+    "trash": "Trash",
+    "junk": "Junk",
+    "drafts": "Drafts",
+    "sent": "Sent",
+    "archive": "Archive",
+}
+
+
+async def _folders() -> None:
+    folders = {
+        use: {"name": name, "create": True, "subscribe": True, "aliases": {}}
+        for use, name in FOLDERS.items()
+    }
+    await admin.update("Email", "singleton", {"defaultFolders": folders})
+
+
+async def provision_server() -> bool:
+    """Set the mail server up; True when it must be restarted to finish."""
+    await _wait_for_server()
+    await _logging()
+    restart = await _blob_store()
+    domain_id = await _domain()
+    await admin.update(
+        "SystemSettings",
+        "singleton",
+        {"defaultDomainId": domain_id, "defaultHostname": mail_hostname()},
+    )
+    await _listeners()
+    await _folders()
+    await _directory()
+    role_id = await _worker_role()
+    await _reload()
+    await _worker_account(role_id)
+    return restart
+
+
+async def provision() -> bool:
+    from app.hosted import native
+
+    await provision_database()
+    restart = await provision_server()
+    created = await native.ensure_all()
+    if created:
+        logger.info("hosted: added the mailbox account of %d existing user(s)", created)
+    return restart
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    if not get_settings().hosted_mailboxes:
+        logger.info("hosted mailboxes are off (HOSTED_MAILBOXES): nothing to set up")
+        return
+    if asyncio.run(provision()):
+        logger.warning("hosted mailboxes: RESTART the mail server (stalwart) to finish")
+    else:
+        logger.info("hosted mailboxes: mail server ready")
+
+
+if __name__ == "__main__":
+    main()