Installable app, push notifications about new mail, Install app entry

The webmail is now a PWA: web app manifest, icons (any + maskable, a status-bar badge), theme-coloured title bar following the chosen theme, and an app-icon badge with the unread Inbox count. The service worker is always installed; offline access is on only when it is registered as /sw.js?offline=1, so turning it on or off is an update that applies at once. Notifications (Settings → Accounts → Notifications, per device) use Web Push. Encryption (RFC 8291) and VAPID signing (RFC 8292) are implemented in app/push/webpush.py — pywebpush needs a newer cryptography than we pin — and checked against the RFC's worked example. The server makes its key pair on first use (private half encrypted with MASTER_KEY). New mail found by the live sync in an Inbox enqueues a job 5 s later that tells the user's devices about messages still unread there, under 2 hours old and not from the user; more than three become one "N new messages". A subscription belongs to the session it was made under: signed out or revoked, nothing more is sent there. Endpoints are limited to the known push services (no SSRF). Nothing shows while the app is in front (except on Apple, which requires it). Tested with real pushes through Google's service in Chromium and Opera. The mobile drawer gets "Install app": Chrome's own install dialog where the browser offers one, else short steps for iPhone/iPad, Opera and Firefox, with "Don't show this again". docs/nas-install.md §5.3 covers installing, notifications, Public URL and a later Android app (TWA). Migration 0022: push_subscriptions, app_settings.vapid_*. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:52:26 +02:00
parent 0729dbb691
commit fda4cc6f16
37 changed files with 1765 additions and 42 deletions
@@ -0,0 +1,245 @@
+"""Push notifications about new mail: the API devices subscribe through, and the
+job that decides what to tell them — delivered to a mocked push service and
+decrypted the way the browser would."""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import uuid
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+
+import httpx
+import pytest_asyncio
+import respx
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from httpx import ASGITransport
+from sqlalchemy import delete, select, update
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+from test_webpush import browser_decrypt
+
+from app.config import get_settings
+from app.identity.passwords import hash_password
+from app.identity.sessions import create_session, destroy_session
+from app.main import create_app
+from app.models import Account, AppSettings, Mailbox, Message, PushSubscription, User
+from app.push import service
+from app.push.webpush import b64u_encode
+
+CSRF = "csrf"
+ENDPOINT = "https://fcm.googleapis.com/fcm/send/test-device"
+
+
+@dataclass
+class Ctx:
+    client: httpx.AsyncClient
+    db: AsyncSession
+    user_id: uuid.UUID
+    token: str
+    inbox: Mailbox
+    archive: Mailbox
+    ua: ec.EllipticCurvePrivateKey
+    auth: bytes
+
+
+@pytest_asyncio.fixture
+async def ctx() -> AsyncIterator[Ctx]:
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    db = async_sessionmaker(engine, expire_on_commit=False)()
+    # The server's key pair lives in the one app_settings row: put it back as it was.
+    row = await db.get(AppSettings, 1)
+    saved = (row.vapid_public, row.vapid_private_ct) if row else None
+    user = User(handle=f"push{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
+    db.add(user)
+    await db.flush()
+    account = Account(
+        user_id=user.id,
+        email="me@example.com",
+        imap_config={},
+        smtp_config={},
+        secret_ct=b"x",
+        discovery_src="manual",
+        state="live",
+    )
+    db.add(account)
+    await db.flush()
+    inbox = Mailbox(account_id=account.id, user_id=user.id, path="INBOX", special_use="inbox")
+    archive = Mailbox(account_id=account.id, user_id=user.id, path="Archive", special_use="archive")
+    db.add_all([inbox, archive])
+    await db.commit()
+    token = await create_session(user.id, "pytest")
+    client = httpx.AsyncClient(
+        transport=ASGITransport(app=create_app()),
+        base_url="http://testserver",
+        cookies={"session": token, "csrf": CSRF},
+    )
+    try:
+        yield Ctx(
+            client,
+            db,
+            user.id,
+            token,
+            inbox,
+            archive,
+            ec.generate_private_key(ec.SECP256R1()),
+            b"0123456789abcdef",
+        )
+    finally:
+        await client.aclose()
+        await db.execute(delete(User).where(User.id == user.id))
+        if saved is not None:
+            await db.execute(
+                update(AppSettings)
+                .where(AppSettings.id == 1)
+                .values(vapid_public=saved[0], vapid_private_ct=saved[1])
+            )
+        await db.commit()
+        await db.close()
+        await engine.dispose()
+
+
+def _keys(c: Ctx) -> dict[str, str]:
+    public = c.ua.public_key().public_bytes(
+        serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint
+    )
+    return {"p256dh": b64u_encode(public), "auth": b64u_encode(c.auth)}
+
+
+async def _subscribe(c: Ctx, endpoint: str = ENDPOINT) -> httpx.Response:
+    return await c.client.post(
+        "/api/push/subscriptions",
+        json={"endpoint": endpoint, "keys": _keys(c)},
+        headers={"X-CSRF": CSRF},
+    )
+
+
+_uid = iter(range(1, 10_000))
+
+
+async def _mail(
+    c: Ctx,
+    *,
+    subject: str = "Hello",
+    sender: str = "alice@example.org",
+    name: str = "Alice",
+    box: Mailbox | None = None,
+    flags: list[str] | None = None,
+    age: dt.timedelta = dt.timedelta(minutes=1),
+) -> uuid.UUID:
+    m = Message(
+        user_id=c.user_id,
+        account_id=c.inbox.account_id,
+        mailbox_id=(box or c.inbox).id,
+        uid=next(_uid),
+        thread_id=None,
+        subject=subject,
+        from_addr=[{"name": name, "email": sender}],
+        internal_date=dt.datetime.now(dt.UTC) - age,
+        flags=flags or [],
+        snippet="  First   line of the message ",
+        blob_sha256="0" * 64,
+    )
+    c.db.add(m)
+    await c.db.commit()
+    return m.id
+
+
+def _decrypted(c: Ctx, route: respx.Route) -> list[dict[str, object]]:
+    return [json.loads(browser_decrypt(call.request.content, c.ua, c.auth)) for call in route.calls]
+
+
+async def test_subscribe_validates_and_hands_out_the_key(ctx: Ctx) -> None:
+    key = await ctx.client.get("/api/push/key")
+    assert key.status_code == 200 and len(key.json()["public_key"]) == 87
+    assert (await ctx.client.get("/api/push/key")).json() == key.json()  # made once
+
+    assert (await _subscribe(ctx, "https://169.254.169.254/latest")).status_code == 400
+    bad = await ctx.client.post(
+        "/api/push/subscriptions",
+        json={"endpoint": ENDPOINT, "keys": {"p256dh": "AAAA", "auth": "AAAA"}},
+        headers={"X-CSRF": CSRF},
+    )
+    assert bad.status_code == 400
+    assert (await _subscribe(ctx)).status_code == 200
+    assert (await _subscribe(ctx)).status_code == 200  # again: refreshed, not doubled
+    rows = (
+        await ctx.db.scalars(
+            select(PushSubscription).where(PushSubscription.user_id == ctx.user_id)
+        )
+    ).all()
+    assert len(rows) == 1
+
+    gone = await ctx.client.post(
+        "/api/push/subscriptions/remove", json={"endpoint": ENDPOINT}, headers={"X-CSRF": CSRF}
+    )
+    assert gone.status_code == 200
+    assert (
+        await ctx.db.scalar(select(PushSubscription).where(PushSubscription.endpoint == ENDPOINT))
+    ) is None
+
+
+@respx.mock
+async def test_new_inbox_mail_is_pushed_encrypted_and_signed(ctx: Ctx) -> None:
+    await _subscribe(ctx)
+    route = respx.post(ENDPOINT).mock(return_value=httpx.Response(201))
+    news = await _mail(ctx, subject="Lunch?")
+    skipped = [
+        await _mail(ctx, flags=["\\Seen"]),  # read on another device already
+        await _mail(ctx, box=ctx.archive),  # filed away by a filter
+        await _mail(ctx, age=dt.timedelta(days=1)),  # old mail moved back to the Inbox
+        await _mail(ctx, sender="ME@example.com"),  # from yourself
+    ]
+    assert await service.notify(ctx.user_id, [news, *skipped]) == 1
+    request = route.calls[0].request
+    assert request.headers["Content-Encoding"] == "aes128gcm"
+    assert request.headers["Authorization"].startswith("vapid t=")
+    [payload] = _decrypted(ctx, route)
+    assert payload["title"] == "Alice"
+    assert payload["body"] == "Lunch?\nFirst line of the message"
+    assert payload["url"] == "/mail" and payload["type"] == "mail"
+    assert payload["unread"] == 3  # Inbox unread: news, old, from-me
+
+
+@respx.mock
+async def test_many_at_once_become_one_summary(ctx: Ctx) -> None:
+    await _subscribe(ctx)
+    route = respx.post(ENDPOINT).mock(return_value=httpx.Response(201))
+    ids = [
+        await _mail(ctx, name=n, sender=f"{n.lower()}@example.org")
+        for n in ("Ann", "Bob", "Cy", "Dee", "Ann")
+    ]
+    assert await service.notify(ctx.user_id, ids) == 1
+    [payload] = _decrypted(ctx, route)
+    assert payload["title"] == "5 new messages"
+    assert payload["body"] == "Ann, Bob, Cy…"
+    assert payload["tag"] == "new-mail"
+
+
+@respx.mock
+async def test_signed_out_or_expired_devices_are_forgotten(ctx: Ctx) -> None:
+    await _subscribe(ctx)
+    other = "https://updates.push.services.mozilla.com/wpush/v2/other"
+    await _subscribe(ctx, other)
+    respx.post(ENDPOINT).mock(return_value=httpx.Response(410))  # browser unsubscribed
+    firefox = respx.post(other).mock(return_value=httpx.Response(201))
+    assert await service.notify(ctx.user_id, [await _mail(ctx)]) == 1
+    left = (
+        await ctx.db.scalars(
+            select(PushSubscription.endpoint).where(PushSubscription.user_id == ctx.user_id)
+        )
+    ).all()
+    assert left == [other]
+
+    await destroy_session(ctx.token)  # signed out: no subject lines on that device any more
+    assert await service.notify(ctx.user_id, [await _mail(ctx)]) == 0
+    assert firefox.call_count == 1
+    assert (
+        await ctx.db.scalar(select(PushSubscription).where(PushSubscription.user_id == ctx.user_id))
+    ) is None
+
+
+async def test_no_devices_no_work(ctx: Ctx) -> None:
+    assert await service.notify(ctx.user_id, [await _mail(ctx)]) == 0
@@ -0,0 +1,120 @@
+"""Web Push protocol: RFC 8291 encryption (against the RFC's worked example and
+by decrypting as a browser would), VAPID signatures, the push-service allow-list."""
+
+from __future__ import annotations
+
+import json
+
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+from app.push import webpush
+from app.push.webpush import b64u_decode, b64u_encode
+
+# RFC 8291 §5 / Appendix A.
+RFC_AUTH = "BTBZMqHH6r4Tts7J_aSIgg"
+RFC_UA_PUBLIC = (
+    "BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4"
+)
+RFC_AS_PRIVATE = "yfWPiYE-n46HLnH0KqZOF1fJJU3MYrct3AELtAQ-oRw"
+RFC_SALT = "DGv6ra1nlYgDCS1FRnbzlw"
+RFC_BODY = (
+    "DGv6ra1nlYgDCS1FRnbzlwAAEABBBP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27ml"
+    "mlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A_yl95bQpu6cVPT"
+    "pK4Mqgkf1CXztLVBSt2Ks3oZwbuwXPXLWyouBWLVWGNWQexSgSxsj_Qulcy4a-fN"
+)
+
+
+def _hkdf(salt: bytes, ikm: bytes, info: bytes, n: int) -> bytes:
+    return HKDF(algorithm=hashes.SHA256(), length=n, salt=salt, info=info).derive(ikm)
+
+
+def browser_decrypt(body: bytes, ua: ec.EllipticCurvePrivateKey, auth: bytes) -> bytes:
+    """What the browser does with a push body (RFC 8291 receiver side)."""
+    salt, idlen = body[:16], body[20]
+    as_public = body[21 : 21 + idlen]
+    ua_public = ua.public_key().public_bytes(
+        serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint
+    )
+    as_key = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), as_public)
+    ikm = _hkdf(
+        auth, ua.exchange(ec.ECDH(), as_key), b"WebPush: info\x00" + ua_public + as_public, 32
+    )
+    cek = _hkdf(salt, ikm, b"Content-Encoding: aes128gcm\x00", 16)
+    nonce = _hkdf(salt, ikm, b"Content-Encoding: nonce\x00", 12)
+    plain = AESGCM(cek).decrypt(nonce, body[21 + idlen :], None)
+    assert plain.endswith(b"\x02")
+    return plain[:-1]
+
+
+def test_rfc8291_worked_example() -> None:
+    sender = ec.derive_private_key(
+        int.from_bytes(b64u_decode(RFC_AS_PRIVATE), "big"), ec.SECP256R1()
+    )
+    body = webpush.encrypt(
+        b"When I grow up, I want to be a watermelon",
+        RFC_UA_PUBLIC,
+        RFC_AUTH,
+        salt=b64u_decode(RFC_SALT),
+        sender=sender,
+    )
+    assert b64u_encode(body) == RFC_BODY
+
+
+def test_browser_can_decrypt() -> None:
+    ua = ec.generate_private_key(ec.SECP256R1())
+    auth = b"0123456789abcdef"
+    p256dh = b64u_encode(
+        ua.public_key().public_bytes(
+            serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint
+        )
+    )
+    payload = json.dumps({"title": "Zażółć gęślą jaźń"}).encode()
+    body = webpush.encrypt(payload, p256dh, b64u_encode(auth))
+    assert browser_decrypt(body, ua, auth) == payload
+
+
+def test_vapid_authorization_is_a_valid_es256_jwt() -> None:
+    key = webpush.Vapid.generate()
+    again = webpush.Vapid.from_pem(key.pem())
+    assert again.public_b64 == key.public_b64
+    header = key.authorization("https://fcm.googleapis.com/fcm/send/abc", "https://ultimo.email")
+    token = header.split("t=")[1].split(",")[0]
+    assert header.endswith(f"k={key.public_b64}")
+    head, claims, sig = token.split(".")
+    assert json.loads(b64u_decode(claims)) | {"exp": 0} == {
+        "aud": "https://fcm.googleapis.com",
+        "sub": "https://ultimo.email",
+        "exp": 0,
+    }
+    raw = b64u_decode(sig)
+    der = encode_dss_signature(int.from_bytes(raw[:32], "big"), int.from_bytes(raw[32:], "big"))
+    public = ec.EllipticCurvePublicKey.from_encoded_point(
+        ec.SECP256R1(), b64u_decode(key.public_b64)
+    )
+    public.verify(der, f"{head}.{claims}".encode(), ec.ECDSA(hashes.SHA256()))  # raises if bad
+
+
+def test_only_push_services_are_allowed() -> None:
+    ok = [
+        "https://fcm.googleapis.com/fcm/send/x",
+        "https://jmt17.google.com/fcm/send/x",
+        "https://updates.push.services.mozilla.com/wpush/v2/x",
+        "https://web.push.apple.com/x",
+        "https://wns2-db5p.notify.windows.com/w/?token=x",
+    ]
+    bad = [
+        "http://fcm.googleapis.com/fcm/send/x",
+        "https://fcm.googleapis.com:8443/x",
+        "https://evil.com/fcm.googleapis.com",
+        "https://fcm.googleapis.com.evil.com/x",
+        "https://google.com.evil.com/x",
+        "https://notapple-push.apple.com.evil.com/x",
+        "https://127.0.0.1/x",
+        "https://postgres/x",
+    ]
+    assert all(webpush.allowed_endpoint(u) for u in ok)
+    assert not any(webpush.allowed_endpoint(u) for u in bad)