Installable app, push notifications about new mail, Install app entry
The webmail is now a PWA: web app manifest, icons (any + maskable, a status-bar badge), theme-coloured title bar following the chosen theme, and an app-icon badge with the unread Inbox count. The service worker is always installed; offline access is on only when it is registered as /sw.js?offline=1, so turning it on or off is an update that applies at once. Notifications (Settings → Accounts → Notifications, per device) use Web Push. Encryption (RFC 8291) and VAPID signing (RFC 8292) are implemented in app/push/webpush.py — pywebpush needs a newer cryptography than we pin — and checked against the RFC's worked example. The server makes its key pair on first use (private half encrypted with MASTER_KEY). New mail found by the live sync in an Inbox enqueues a job 5 s later that tells the user's devices about messages still unread there, under 2 hours old and not from the user; more than three become one "N new messages". A subscription belongs to the session it was made under: signed out or revoked, nothing more is sent there. Endpoints are limited to the known push services (no SSRF). Nothing shows while the app is in front (except on Apple, which requires it). Tested with real pushes through Google's service in Chromium and Opera. The mobile drawer gets "Install app": Chrome's own install dialog where the browser offers one, else short steps for iPhone/iPad, Opera and Firefox, with "Don't show this again". docs/nas-install.md §5.3 covers installing, notifications, Public URL and a later Android app (TWA). Migration 0022: push_subscriptions, app_settings.vapid_*. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1 @@
|
||||
"""Push notifications about new mail (Web Push)."""
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Push notifications about new mail: the server's keys, the devices, and the job.
|
||||
|
||||
New Inbox mail found by the live sync enqueues :func:`push_new_mail_job` a few
|
||||
seconds later, so filters and the spam check have moved what they move by then;
|
||||
the job looks again at each message and tells the user's devices only about
|
||||
mail that is still unread in an Inbox, recent, and not from the user.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.identity.sessions import session_alive
|
||||
from app.models import Account, AppSettings, Mailbox, Message, PushSubscription
|
||||
from app.push import webpush
|
||||
from app.security.crypto import decrypt_secret, encrypt_secret
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# How long after new Inbox mail is found the job runs (filters, spam check).
|
||||
NOTIFY_DELAY_S = 5
|
||||
# Mail older than this is not news: a message moved back into the Inbox, or a
|
||||
# backlog after the server was unreachable for a while.
|
||||
MAX_AGE = dt.timedelta(hours=2)
|
||||
# More new messages than this at once become a single "N new messages".
|
||||
MAX_SINGLE = 3
|
||||
SNIPPET_CHARS = 140
|
||||
|
||||
|
||||
# ---- the server's key pair ------------------------------------------------------------
|
||||
|
||||
|
||||
async def vapid(db: AsyncSession) -> webpush.Vapid:
|
||||
"""The server's VAPID key pair, made (once) on first use."""
|
||||
row = await db.get(AppSettings, 1)
|
||||
if row is not None and row.vapid_private_ct:
|
||||
return webpush.Vapid.from_pem(decrypt_secret(row.vapid_private_ct))
|
||||
row = await db.scalar(select(AppSettings).where(AppSettings.id == 1).with_for_update())
|
||||
if row is None:
|
||||
row = AppSettings(id=1)
|
||||
db.add(row)
|
||||
elif row.vapid_private_ct: # made by another request meanwhile
|
||||
return webpush.Vapid.from_pem(decrypt_secret(row.vapid_private_ct))
|
||||
key = webpush.Vapid.generate()
|
||||
row.vapid_private_ct = encrypt_secret(key.pem())
|
||||
row.vapid_public = key.public_b64
|
||||
await db.commit()
|
||||
return key
|
||||
|
||||
|
||||
def _subject(row: AppSettings | None) -> str:
|
||||
"""Who to contact about our pushes (the VAPID ``sub``): the app's address
|
||||
when it is set and https, else the postmaster of the app's domain."""
|
||||
url = row.public_url if row is not None else ""
|
||||
if url.startswith("https://"):
|
||||
return url
|
||||
return f"mailto:postmaster@{get_settings().app_domain}"
|
||||
|
||||
|
||||
# ---- devices ------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def subscribe(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
session_sid: str,
|
||||
endpoint: str,
|
||||
p256dh: str,
|
||||
auth: str,
|
||||
user_agent: str,
|
||||
) -> None:
|
||||
"""Remember a device (or refresh it). One row per endpoint: a browser that
|
||||
is now signed in as someone else is theirs from now on."""
|
||||
row = await db.scalar(select(PushSubscription).where(PushSubscription.endpoint == endpoint))
|
||||
if row is None:
|
||||
row = PushSubscription(endpoint=endpoint)
|
||||
db.add(row)
|
||||
row.user_id = user_id
|
||||
row.session_sid = session_sid
|
||||
row.p256dh = p256dh
|
||||
row.auth = auth
|
||||
row.user_agent = user_agent[:256]
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def unsubscribe(db: AsyncSession, *, user_id: uuid.UUID, endpoint: str) -> None:
|
||||
await db.execute(
|
||||
delete(PushSubscription).where(
|
||||
PushSubscription.user_id == user_id, PushSubscription.endpoint == endpoint
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
# ---- what to say --------------------------------------------------------------------
|
||||
|
||||
|
||||
def _sender(msg: Message) -> str:
|
||||
first = (msg.from_addr or [{}])[0]
|
||||
return (first.get("name") or first.get("email") or "Unknown sender").strip()
|
||||
|
||||
|
||||
def _payloads(messages: list[Message], unread: int) -> list[dict[str, object]]:
|
||||
"""One notification per message, or one summary when there are many."""
|
||||
if len(messages) > MAX_SINGLE:
|
||||
senders: list[str] = []
|
||||
for m in messages:
|
||||
name = _sender(m)
|
||||
if name not in senders:
|
||||
senders.append(name)
|
||||
shown = ", ".join(senders[:3]) + ("…" if len(senders) > 3 else "")
|
||||
return [
|
||||
{
|
||||
"type": "mail",
|
||||
"title": f"{len(messages)} new messages",
|
||||
"body": shown,
|
||||
"tag": "new-mail",
|
||||
"url": "/mail",
|
||||
"unread": unread,
|
||||
}
|
||||
]
|
||||
out: list[dict[str, object]] = []
|
||||
for m in messages:
|
||||
subject = m.subject.strip() or "(no subject)"
|
||||
snippet = " ".join(m.snippet.split())[:SNIPPET_CHARS]
|
||||
out.append(
|
||||
{
|
||||
"type": "mail",
|
||||
"title": _sender(m),
|
||||
"body": f"{subject}\n{snippet}" if snippet else subject,
|
||||
"tag": f"thread-{m.thread_id}",
|
||||
"url": f"/mail/t/{m.thread_id}" if m.thread_id else "/mail",
|
||||
"thread_id": str(m.thread_id) if m.thread_id else None,
|
||||
"unread": unread,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def _news(db: AsyncSession, user_id: uuid.UUID, ids: list[uuid.UUID]) -> list[Message]:
|
||||
"""The messages still worth a notification, oldest first."""
|
||||
own = {
|
||||
e.lower() for e in await db.scalars(select(Account.email).where(Account.user_id == user_id))
|
||||
}
|
||||
rows = await db.scalars(
|
||||
select(Message)
|
||||
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||||
.where(
|
||||
Message.user_id == user_id,
|
||||
Message.id.in_(ids),
|
||||
Mailbox.special_use == "inbox",
|
||||
~Message.flags.contains(["\\Seen"]),
|
||||
Message.internal_date >= dt.datetime.now(dt.UTC) - MAX_AGE,
|
||||
)
|
||||
.order_by(Message.internal_date)
|
||||
)
|
||||
out: list[Message] = []
|
||||
seen: set[str] = set()
|
||||
for m in rows:
|
||||
sender = ((m.from_addr or [{}])[0].get("email") or "").lower()
|
||||
if sender in own:
|
||||
continue # sent to yourself, or a copy of your own message
|
||||
key = m.delivery_id or str(m.id) # one email that reached two accounts
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(m)
|
||||
return out
|
||||
|
||||
|
||||
async def _unread(db: AsyncSession, user_id: uuid.UUID) -> int:
|
||||
"""Unread Inbox messages, for the app icon's badge."""
|
||||
count = await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Message)
|
||||
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||||
.where(
|
||||
Message.user_id == user_id,
|
||||
Mailbox.special_use == "inbox",
|
||||
~Message.flags.contains(["\\Seen"]),
|
||||
)
|
||||
)
|
||||
return int(count or 0)
|
||||
|
||||
|
||||
# ---- the job ------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def notify(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> int:
|
||||
"""Tell the user's signed-in devices about ``message_ids``; returns how many
|
||||
pushes were delivered to push services."""
|
||||
from app.db import session_scope
|
||||
|
||||
async with session_scope() as db:
|
||||
# Plain values: the rows expire when the key pair is first saved below.
|
||||
subs = [
|
||||
(row.id, row.endpoint, row.p256dh, row.auth, row.session_sid)
|
||||
for row in await db.scalars(
|
||||
select(PushSubscription).where(PushSubscription.user_id == user_id)
|
||||
)
|
||||
]
|
||||
if not subs:
|
||||
return 0
|
||||
messages = await _news(db, user_id, message_ids)
|
||||
if not messages:
|
||||
return 0
|
||||
payloads = _payloads(messages, await _unread(db, user_id))
|
||||
key = await vapid(db)
|
||||
subject = _subject(await db.get(AppSettings, 1))
|
||||
|
||||
sent = 0
|
||||
gone: list[uuid.UUID] = []
|
||||
used: list[uuid.UUID] = []
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
for sub_id, endpoint, p256dh, auth, sid in subs:
|
||||
if not await session_alive(user_id, sid):
|
||||
gone.append(sub_id) # signed out: this device hears nothing more
|
||||
continue
|
||||
try:
|
||||
for payload in payloads:
|
||||
await webpush.send(
|
||||
client,
|
||||
endpoint=endpoint,
|
||||
p256dh=p256dh,
|
||||
auth=auth,
|
||||
payload=payload,
|
||||
vapid=key,
|
||||
subject=subject,
|
||||
)
|
||||
sent += 1
|
||||
used.append(sub_id)
|
||||
except webpush.Gone:
|
||||
gone.append(sub_id)
|
||||
except webpush.Failed as err:
|
||||
logger.warning("push to %s failed: %s", endpoint.split("/")[2], err)
|
||||
|
||||
if gone or used:
|
||||
async with session_scope() as db:
|
||||
if gone:
|
||||
await db.execute(delete(PushSubscription).where(PushSubscription.id.in_(gone)))
|
||||
if used:
|
||||
await db.execute(
|
||||
update(PushSubscription)
|
||||
.where(PushSubscription.id.in_(used))
|
||||
.values(last_used_at=func.now())
|
||||
)
|
||||
return sent
|
||||
|
||||
|
||||
async def push_new_mail_job(ctx: dict[str, Any], user_id: str, message_ids: list[str]) -> int:
|
||||
"""arq job: notify about new Inbox mail (enqueued by the live sync)."""
|
||||
return await notify(uuid.UUID(user_id), [uuid.UUID(m) for m in message_ids])
|
||||
|
||||
|
||||
async def enqueue_new_mail(user_id: uuid.UUID, message_ids: list[uuid.UUID]) -> None:
|
||||
"""Called by the live sync with new Inbox mail. Never raises: a missing
|
||||
notification must not stop the sync."""
|
||||
if not message_ids:
|
||||
return
|
||||
from app.queue import enqueue_job
|
||||
|
||||
try:
|
||||
await enqueue_job(
|
||||
"push_new_mail_job",
|
||||
str(user_id),
|
||||
[str(m) for m in message_ids],
|
||||
_defer_by=NOTIFY_DELAY_S,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - best effort
|
||||
logger.warning("could not enqueue new-mail push for user %s", user_id, exc_info=True)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Web Push, the protocol (RFC 8030 delivery, RFC 8291 encryption, RFC 8292 VAPID).
|
||||
|
||||
A browser that wants notifications gives us a push service URL (Google's,
|
||||
Mozilla's, Apple's, Microsoft's) and two keys. A message is encrypted to those
|
||||
keys, so the push service only carries it, and signed with our own key pair
|
||||
(VAPID), so the push service knows it comes from the server the browser
|
||||
subscribed to. Written here rather than taken from pywebpush, which needs a
|
||||
newer ``cryptography`` than the rest of the backend pins; it is small, and
|
||||
``tests/push`` checks it against the RFC 8291 worked example.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
|
||||
|
||||
RECORD_SIZE = 4096
|
||||
# Push services only: the endpoint comes from the browser, i.e. from the user,
|
||||
# and the server posts to it — anything else would let a user aim the server at
|
||||
# an internal address (SSRF). Every browser in use delivers through one of these.
|
||||
PUSH_HOSTS = (
|
||||
"fcm.googleapis.com", # Chrome, Edge on Android, Opera, Samsung Internet
|
||||
"android.googleapis.com",
|
||||
".google.com", # Chromium builds subscribe at jmt17.google.com and the like
|
||||
".push.services.mozilla.com", # Firefox
|
||||
".push.apple.com", # Safari, and installed apps on iPhone/iPad
|
||||
".notify.windows.com", # Edge on Windows
|
||||
)
|
||||
|
||||
|
||||
def b64u_decode(value: str) -> bytes:
|
||||
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
||||
|
||||
|
||||
def b64u_encode(data: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def allowed_endpoint(endpoint: str) -> bool:
|
||||
"""Whether ``endpoint`` is an https URL of a known push service."""
|
||||
parts = urlsplit(endpoint)
|
||||
host = (parts.hostname or "").lower()
|
||||
if parts.scheme != "https" or not host or parts.port not in (None, 443):
|
||||
return False
|
||||
return any(host == h or (h.startswith(".") and host.endswith(h)) for h in PUSH_HOSTS)
|
||||
|
||||
|
||||
def _public_bytes(key: ec.EllipticCurvePrivateKey) -> bytes:
|
||||
return key.public_key().public_bytes(
|
||||
serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint
|
||||
)
|
||||
|
||||
|
||||
def _hkdf(salt: bytes, ikm: bytes, info: bytes, length: int) -> bytes:
|
||||
return HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
|
||||
|
||||
|
||||
def encrypt(
|
||||
payload: bytes,
|
||||
p256dh: str,
|
||||
auth: str,
|
||||
*,
|
||||
salt: bytes | None = None,
|
||||
sender: ec.EllipticCurvePrivateKey | None = None,
|
||||
) -> bytes:
|
||||
"""``payload`` encrypted for one browser (RFC 8291, ``aes128gcm``), as the
|
||||
request body: header (salt, record size, our one-off public key) + one
|
||||
record. ``salt`` and ``sender`` are for the RFC test vector only."""
|
||||
ua_public = b64u_decode(p256dh)
|
||||
auth_secret = b64u_decode(auth)
|
||||
sender = sender or ec.generate_private_key(ec.SECP256R1())
|
||||
as_public = _public_bytes(sender)
|
||||
ua_key = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), ua_public)
|
||||
ecdh_secret = sender.exchange(ec.ECDH(), ua_key)
|
||||
ikm = _hkdf(auth_secret, ecdh_secret, b"WebPush: info\x00" + ua_public + as_public, 32)
|
||||
salt = salt or os.urandom(16)
|
||||
cek = _hkdf(salt, ikm, b"Content-Encoding: aes128gcm\x00", 16)
|
||||
nonce = _hkdf(salt, ikm, b"Content-Encoding: nonce\x00", 12)
|
||||
# One record, no padding: 0x02 marks the last (only) record.
|
||||
record = AESGCM(cek).encrypt(nonce, payload + b"\x02", None)
|
||||
header = salt + RECORD_SIZE.to_bytes(4, "big") + bytes([len(as_public)]) + as_public
|
||||
return header + record
|
||||
|
||||
|
||||
# ---- VAPID (RFC 8292) ---------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Vapid:
|
||||
"""The server's signing key pair."""
|
||||
|
||||
private: ec.EllipticCurvePrivateKey
|
||||
|
||||
@classmethod
|
||||
def generate(cls) -> Vapid:
|
||||
return cls(ec.generate_private_key(ec.SECP256R1()))
|
||||
|
||||
@classmethod
|
||||
def from_pem(cls, pem: bytes) -> Vapid:
|
||||
key = serialization.load_pem_private_key(pem, password=None)
|
||||
if not isinstance(key, ec.EllipticCurvePrivateKey):
|
||||
raise ValueError("VAPID key is not an EC key")
|
||||
return cls(key)
|
||||
|
||||
def pem(self) -> bytes:
|
||||
return self.private.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
|
||||
@property
|
||||
def public_b64(self) -> str:
|
||||
"""The public key as browsers take it (``applicationServerKey``)."""
|
||||
return b64u_encode(_public_bytes(self.private))
|
||||
|
||||
def authorization(self, endpoint: str, subject: str, *, ttl: int = 12 * 3600) -> str:
|
||||
"""The ``Authorization`` header for a push to ``endpoint``: a JWT (ES256)
|
||||
naming the push service's origin, with our public key."""
|
||||
parts = urlsplit(endpoint)
|
||||
claims = {
|
||||
"aud": f"{parts.scheme}://{parts.netloc}",
|
||||
"exp": int(time.time()) + ttl,
|
||||
"sub": subject,
|
||||
}
|
||||
header = {"typ": "JWT", "alg": "ES256"}
|
||||
signing_input = (
|
||||
b64u_encode(json.dumps(header, separators=(",", ":")).encode())
|
||||
+ "."
|
||||
+ b64u_encode(json.dumps(claims, separators=(",", ":")).encode())
|
||||
)
|
||||
der = self.private.sign(signing_input.encode(), ec.ECDSA(hashes.SHA256()))
|
||||
r, s = decode_dss_signature(der)
|
||||
signature = r.to_bytes(32, "big") + s.to_bytes(32, "big")
|
||||
return f"vapid t={signing_input}.{b64u_encode(signature)}, k={self.public_b64}"
|
||||
|
||||
|
||||
# ---- delivery (RFC 8030) ------------------------------------------------------------
|
||||
|
||||
|
||||
class Gone(Exception):
|
||||
"""The subscription no longer exists (404/410): forget it."""
|
||||
|
||||
|
||||
class Failed(Exception):
|
||||
"""The push service refused or could not be reached; worth trying later."""
|
||||
|
||||
|
||||
async def send(
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
endpoint: str,
|
||||
p256dh: str,
|
||||
auth: str,
|
||||
payload: dict[str, object],
|
||||
vapid: Vapid,
|
||||
subject: str,
|
||||
ttl: int = 24 * 3600,
|
||||
urgency: str = "normal",
|
||||
topic: str | None = None,
|
||||
) -> None:
|
||||
"""Deliver one encrypted message; raise :class:`Gone` or :class:`Failed`."""
|
||||
if not allowed_endpoint(endpoint):
|
||||
raise Gone("not a push service")
|
||||
body = encrypt(json.dumps(payload, separators=(",", ":")).encode(), p256dh, auth)
|
||||
headers = {
|
||||
"Authorization": vapid.authorization(endpoint, subject),
|
||||
"Content-Encoding": "aes128gcm",
|
||||
"Content-Type": "application/octet-stream",
|
||||
"TTL": str(ttl),
|
||||
"Urgency": urgency,
|
||||
}
|
||||
if topic:
|
||||
headers["Topic"] = topic # replaces an undelivered message with the same topic
|
||||
try:
|
||||
resp = await client.post(endpoint, content=body, headers=headers)
|
||||
except httpx.HTTPError as err:
|
||||
raise Failed(str(err)) from err
|
||||
if resp.status_code in (404, 410):
|
||||
raise Gone(f"HTTP {resp.status_code}")
|
||||
if resp.status_code >= 400:
|
||||
raise Failed(f"HTTP {resp.status_code}: {resp.text[:200]}")
|
||||
Reference in New Issue
Block a user