webmail-client/backend/app/hosted/provision.py

"""Set up the mail server for hosted mailboxes (SPEC App. D §D.2/§D.3). Idempotent.

Run after the migrations (the ``migrate`` service in production; in dev
``python -m app.hosted.provision``). Two halves:

1. **Postgres** (as the app's own, privileged database user): the mail
   server's role ``stalwart`` (password ``STALWART_DB_PASSWORD``), which owns its
   database and may read the ``mail_directory`` view of the app's database and
   nothing else there; and the worker's sign-in in ``mail_service_accounts``.
   The mail server cannot start before this has run once.
2. **The mail server** (its management API, as the recovery admin): the mail
   domain and host name, mail kept on disk (not in Postgres), the SQL directory
   over ``mail_directory`` as the only source of accounts, the worker's
   impersonate-only role, the submission port for mail apps.
"""

from __future__ import annotations

import asyncio
import logging
import re
from typing import Any
from urllib.parse import urlparse

import asyncpg  # type: ignore[import-untyped]

from app.config import get_settings
from app.hosted import admin, mail_hostname, worker_address
from app.identity.passwords import hash_password, verify_password

logger = logging.getLogger(__name__)

DB_ROLE = "stalwart"
DIRECTORY_NAME = "Ultimo webmail users"
WORKER_ROLE_NAME = "Ultimo webmail worker"
BLOB_PATH = "/var/lib/stalwart/blobs"

# The directory's queries ($1: the address signed in with / mail is for). A
# blocked user cannot sign in, but still receives mail.
QUERY_LOGIN = (
    "SELECT name, secret, type, description FROM mail_directory WHERE name = $1 AND active"
)
QUERY_RECIPIENT = (
    "SELECT name, NULL::text AS secret, type, description FROM mail_directory WHERE name = $1"
)


def _pg_dsn() -> str:
    return get_settings().database_url.replace("postgresql+asyncpg://", "postgresql://", 1)


def _app_db_name() -> str:
    return urlparse(_pg_dsn()).path.lstrip("/")


def _quote_literal(value: str) -> str:
    return "'" + value.replace("'", "''") + "'"


async def provision_database() -> None:
    settings = get_settings()
    if not settings.stalwart_db_password or not settings.stalwart_master_secret:
        raise RuntimeError("STALWART_DB_PASSWORD and STALWART_MASTER_SECRET must be set")
    conn = await asyncpg.connect(_pg_dsn())
    try:
        password = _quote_literal(settings.stalwart_db_password)
        if await conn.fetchval("SELECT 1 FROM pg_roles WHERE rolname = $1", DB_ROLE):
            await conn.execute(f"ALTER ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
        else:
            await conn.execute(f"CREATE ROLE {DB_ROLE} WITH LOGIN PASSWORD {password}")
        db = settings.stalwart_db_name
        if not await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db):
            await conn.execute(f'CREATE DATABASE "{db}" OWNER {DB_ROLE}')
            logger.info("hosted: created database %s", db)
        await conn.execute(f"GRANT SELECT ON mail_directory TO {DB_ROLE}")

        # The worker's sign-in: re-hashed only when the secret changed.
        name = worker_address()
        current = await conn.fetchval(
            "SELECT secret FROM mail_service_accounts WHERE name = $1", name
        )
        if current is None or not verify_password(current, settings.stalwart_master_secret):
            await conn.execute(
                """
                INSERT INTO mail_service_accounts (name, secret, description)
                VALUES ($1, $2, 'Webmail worker')
                ON CONFLICT (name) DO UPDATE SET secret = EXCLUDED.secret
                """,
                name,
                hash_password(settings.stalwart_master_secret),
            )
    finally:
        await conn.close()


# ---- the mail server ------------------------------------------------------------------


async def _wait_for_server(timeout_s: float = 120.0) -> None:
    deadline = asyncio.get_running_loop().time() + timeout_s
    while True:
        try:
            await admin.get_one("SystemSettings")
            return
        except admin.MailServerError:
            if asyncio.get_running_loop().time() > deadline:
                raise
            await asyncio.sleep(2)


def _by(items: list[dict[str, Any]], key: str, value: str) -> dict[str, Any] | None:
    return next((i for i in items if i.get(key) == value), None)


async def _domain() -> str:
    name = get_settings().app_domain.lower()
    found = _by(await admin.get_all("Domain"), "name", name)
    if found is not None:
        return str(found["id"])
    domain_id = await admin.create("Domain", {"name": name})
    logger.info("hosted: created mail domain %s", name)
    return domain_id


async def _directory() -> str:
    settings = get_settings()
    dsn = urlparse(_pg_dsn())
    value: dict[str, Any] = {
        "@type": "Sql",
        "description": DIRECTORY_NAME,
        "columnEmail": "name",
        "columnSecret": "secret",
        "columnClass": "type",
        "columnDescription": "description",
        "queryLogin": QUERY_LOGIN,
        "queryRecipient": QUERY_RECIPIENT,
        "queryMemberOf": None,
        "queryEmailAliases": None,
        "store": {
            "@type": "PostgreSql",
            "host": settings.stalwart_pg_host,
            "port": 5432 if settings.stalwart_pg_host == "postgres" else (dsn.port or 5432),
            "database": _app_db_name(),
            "authUsername": DB_ROLE,
            "authSecret": {"@type": "EnvironmentVariable", "variableName": "STALWART_DB_PASSWORD"},
            "useTls": False,
            "allowInvalidCerts": False,
            "poolMaxConnections": 5,
            "poolRecyclingMethod": "fast",
            "readReplicas": {},
            "timeout": 15000,
            "options": None,
        },
    }
    found = _by(await admin.get_all("Directory"), "description", DIRECTORY_NAME)
    if found is None:
        directory_id = await admin.create("Directory", value)
        logger.info("hosted: created the SQL directory")
    else:
        directory_id = str(found["id"])
        await admin.update("Directory", directory_id, value)
    await admin.update("Authentication", "singleton", {"directoryId": directory_id})
    return directory_id


async def _worker_role() -> str:
    perms = {"impersonate": True, "authenticate": True}
    found = _by(await admin.get_all("Role"), "description", WORKER_ROLE_NAME)
    if found is not None:
        role_id = str(found["id"])
        await admin.update("Role", role_id, {"enabledPermissions": perms})
        return role_id
    return await admin.create(
        "Role", {"description": WORKER_ROLE_NAME, "enabledPermissions": perms}
    )


async def _worker_account(role_id: str) -> None:
    """The worker may act for any mailbox and do nothing as itself."""
    # Signing in brings it from the directory into the server's registry. With
    # its role already set that sign-in fails (it may not open a session as
    # itself) — after the account exists, which is all this needs.
    worker = worker_address()
    acc = await admin.account_id(worker)
    if acc is None:
        import httpx

        async with httpx.AsyncClient(timeout=15) as client:
            await client.get(
                f"{get_settings().stalwart_url}/jmap/session",
                auth=(worker, get_settings().stalwart_master_secret),
            )
        acc = await admin.account_id(worker)
    if acc is None:
        raise admin.MailServerError("the worker account did not appear on the mail server")
    await admin.update("Account", acc, {"roles": {"@type": "Custom", "roleIds": {role_id: True}}})


async def _reload() -> None:
    """Apply the settings now. Stalwart 0.16.19 ships a few spam-list rules its
    own validator rejects (``bit_and`` in SURBL tags); they never load, but
    they make every reload fail — so they are removed, which changes nothing."""
    for _ in range(10):
        try:
            await admin.create("Action", {"@type": "ReloadSettings"})
            return
        except admin.MailServerError as exc:
            match = re.search(r"'object': '(SpamDnsblServer)', 'id': '(\w+)'", str(exc))
            if match is None:
                raise
            logger.info("hosted: removing the invalid built-in rule %s", match.group(2))
            await admin.destroy(match.group(1), [match.group(2)])
    raise admin.MailServerError("settings reload kept failing")


async def _logging() -> None:
    """Log to the console (``docker logs``), not to files inside the container."""
    for tracer in await admin.get_all("Tracer"):
        if tracer.get("@type") != "Stdout":
            await admin.update(
                "Tracer",
                str(tracer["id"]),
                {
                    "@type": "Stdout",
                    "ansi": False,
                    "buffered": False,
                    "enable": True,
                    "level": "info",
                    "events": {},
                    "eventsPolicy": "exclude",
                },
            )


async def _blob_store() -> bool:
    """Mail on disk (``DATA_ROOT/stalwart``), not inside Postgres: database
    backups stay small. Takes a restart; True when one is needed."""
    current = await admin.get_one("BlobStore")
    if current.get("@type") == "FileSystem" and current.get("path") == BLOB_PATH:
        return False
    await admin.update(
        "BlobStore", "singleton", {"@type": "FileSystem", "path": BLOB_PATH, "depth": 2}
    )
    logger.info("hosted: mail is now kept on disk (takes effect after a restart)")
    return True


async def _listeners() -> None:
    """Add submission with STARTTLS (587) beside the defaults: smtp 25, submissions
    465, imaps 993 (and http 8080 for this API)."""
    listeners = await admin.get_all("NetworkListener")
    if _by(listeners, "name", "submission") is None:
        await admin.create(
            "NetworkListener",
            {"name": "submission", "bind": {"[::]:587": True}, "protocol": "smtp", "useTls": True},
        )


# Folder names for new mailboxes: the plain ones mail apps expect, not
# Stalwart's Outlook-style defaults ("Deleted Items", "Junk Mail", "Sent
# Items"). Archive is made too: the webmail archives into it.
FOLDERS = {
    "trash": "Trash",
    "junk": "Junk",
    "drafts": "Drafts",
    "sent": "Sent",
    "archive": "Archive",
}


async def _folders() -> None:
    folders = {
        use: {"name": name, "create": True, "subscribe": True, "aliases": {}}
        for use, name in FOLDERS.items()
    }
    await admin.update("Email", "singleton", {"defaultFolders": folders})


async def provision_server() -> bool:
    """Set the mail server up; True when it must be restarted to finish."""
    await _wait_for_server()
    await _logging()
    restart = await _blob_store()
    domain_id = await _domain()
    await admin.update(
        "SystemSettings",
        "singleton",
        {"defaultDomainId": domain_id, "defaultHostname": mail_hostname()},
    )
    await _listeners()
    await _folders()
    await _directory()
    role_id = await _worker_role()
    await _reload()
    await _worker_account(role_id)
    return restart


async def provision() -> bool:
    from app.hosted import native

    await provision_database()
    restart = await provision_server()
    created = await native.ensure_all()
    if created:
        logger.info("hosted: added the mailbox account of %d existing user(s)", created)
    return restart


def main() -> None:
    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
    logging.getLogger("httpx").setLevel(logging.WARNING)
    if not get_settings().hosted_mailboxes:
        logger.info("hosted mailboxes are off (HOSTED_MAILBOXES): nothing to set up")
        return
    if asyncio.run(provision()):
        logger.warning("hosted mailboxes: RESTART the mail server (stalwart) to finish")
    else:
        logger.info("hosted mailboxes: mail server ready")


if __name__ == "__main__":
    main()