Initial commit: webmail client (M0/M1)

Three-pane webmail client.
- Backend: FastAPI + async SQLAlchemy + arq worker (Postgres/Redis), IMAP/SMTP
  sync, threading, search, drafts/outbox with undo, attachments, 2FA.
- Frontend: React 18 + Vite + TypeScript + TanStack Query + Tailwind; three-pane
  UI with rich-text composer, light/dark themes, resizable panels.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-17 23:20:34 +02:00
co-authored by Claude Opus 4.8
commit a5ec0d21cb
174 changed files with 22239 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""Indexer: MIME parsing, sanitization, blob storage, DB rows (SPEC §6.4)."""
+55
View File
@@ -0,0 +1,55 @@
"""Content-addressed blob store (SPEC §4 "Blob store", §6.4).
Layout: ``${BLOB_ROOT}/{user_id}/{sha256[:2]}/{sha256}`` (filesystem default).
Content-addressed, so identical payloads dedupe. Postgres holds metadata only;
deleting a user removes their blob subtree.
"""
from __future__ import annotations
import hashlib
import uuid
from pathlib import Path
from app.config import get_settings
def _root() -> Path:
return Path(get_settings().blob_root)
def sha256_hex(data: bytes) -> str:
"""Return the hex sha256 digest of ``data``."""
return hashlib.sha256(data).hexdigest()
def blob_path(user_id: uuid.UUID, sha: str) -> Path:
"""Return the on-disk path for a user's blob (does not create it)."""
return _root() / str(user_id) / sha[:2] / sha
def store(user_id: uuid.UUID, data: bytes) -> str:
"""Store ``data`` content-addressed; return its sha256. Idempotent (dedupes)."""
sha = sha256_hex(data)
path = blob_path(user_id, sha)
if not path.exists():
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
tmp.write_bytes(data)
tmp.replace(path) # atomic within the same directory
return sha
def load(user_id: uuid.UUID, sha: str) -> bytes:
"""Load a blob by sha256. Raises FileNotFoundError if absent."""
return blob_path(user_id, sha).read_bytes()
def exists(user_id: uuid.UUID, sha: str) -> bool:
"""Return True if the blob is present."""
return blob_path(user_id, sha).exists()
def remove(user_id: uuid.UUID, sha: str) -> None:
"""Delete a blob if present (used by account/user deletion). Idempotent."""
blob_path(user_id, sha).unlink(missing_ok=True)
+202
View File
@@ -0,0 +1,202 @@
"""Indexer orchestration (SPEC §6.4).
Turns a :class:`RemoteMessage` into persisted rows: raw blob (deduped), parsed
MIME, sanitized HTML, ``messages``/``body_parts``/``attachments`` rows, JWZ
threading, and a ``known_recipients`` upsert. Returns a :class:`SearchDoc` for
the caller to index *after commit* (FR-30 inline indexing) and to drive the
``message.new`` event. Idempotent on ``(mailbox_id, uid)``.
"""
from __future__ import annotations
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.indexer import blobstore
from app.indexer.mime import ParsedEmail, parse_message
from app.indexer.sanitize import sanitize_html
from app.models import Attachment, BodyPart, KnownRecipient, Message
from app.providers.base import RemoteMessage
from app.search.base import SearchDoc
from app.threading_.normalize import synthesize_message_id
from app.threading_.threader import thread_message
_NO_RECIPIENT_HARVEST = {"junk", "sent", "drafts", "trash"}
@dataclass
class PersistResult:
message: Message
search_doc: SearchDoc
has_remote_images: bool
def _epoch(value: datetime) -> int:
return int(value.timestamp())
def _build_search_doc(message: Message, parsed: ParsedEmail, special_use: str | None) -> SearchDoc:
return SearchDoc(
message_id=message.id,
thread_id=message.thread_id or message.id,
user_id=message.user_id,
account_id=message.account_id,
mailbox_special_use=special_use,
from_=parsed.from_addr,
to=parsed.to_addr,
cc=parsed.cc_addr,
subject=parsed.subject,
body_text=parsed.text_body,
attachment_names=[a.filename for a in parsed.attachments],
has_attachments=parsed.has_attachments,
is_unread="\\Seen" not in message.flags,
is_starred="\\Flagged" in message.flags,
date=_epoch(message.internal_date),
size_bytes=message.size_bytes,
)
async def _harvest_recipient(session: AsyncSession, message: Message, parsed: ParsedEmail) -> None:
"""Upsert the From address into known_recipients (weight 1) (SPEC §6.4)."""
if not parsed.from_addr:
return
sender = parsed.from_addr[0]
email = (sender.get("email") or "").lower()
if not email:
return
stmt = pg_insert(KnownRecipient).values(
user_id=message.user_id,
email=email,
name=sender.get("name", ""),
times_used=1,
last_used_at=message.internal_date,
)
stmt = stmt.on_conflict_do_update(
index_elements=[KnownRecipient.user_id, KnownRecipient.email],
set_={
"times_used": KnownRecipient.times_used + 1,
"last_used_at": stmt.excluded.last_used_at,
"name": stmt.excluded.name,
},
)
await session.execute(stmt)
async def _resolve_message_id(
session: AsyncSession,
*,
user_id: uuid.UUID,
account_id: uuid.UUID,
parsed_id: str | None,
raw: bytes,
) -> str:
"""Resolve the Message-ID to store, applying SPEC §7.1 duplicate handling.
Missing id -> synthesize. A *duplicate* id already seen in a **different
account** is a separate delivery (e.g. a bulk sender that reuses one
Message-ID for every recipient) -> synthesize a non-authoritative id so the
two do not merge. A same-account duplicate (e.g. Gmail INBOX + All Mail
holding one message) keeps the real id so the copies stay in one thread.
"""
if parsed_id is None:
return synthesize_message_id(raw)
other_account = await session.scalar(
select(Message.account_id)
.where(Message.user_id == user_id, Message.message_id_hdr == parsed_id)
.limit(1)
)
if other_account is not None and other_account != account_id:
return synthesize_message_id(raw)
return parsed_id
async def persist_message(
session: AsyncSession,
*,
user_id: uuid.UUID,
account_id: uuid.UUID,
mailbox_id: uuid.UUID,
mailbox_special_use: str | None,
remote: RemoteMessage,
) -> PersistResult | None:
"""Persist one message. Returns None if it already exists (idempotent)."""
existing = await session.scalar(
select(Message.id).where(Message.mailbox_id == mailbox_id, Message.uid == remote.uid)
)
if existing is not None:
return None
sha = blobstore.store(user_id, remote.raw)
parsed = parse_message(remote.raw)
message_id_hdr = await _resolve_message_id(
session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw
)
message = Message(
id=uuid.uuid4(),
user_id=user_id,
account_id=account_id,
mailbox_id=mailbox_id,
uid=remote.uid,
message_id_hdr=message_id_hdr,
in_reply_to=parsed.in_reply_to,
references_hdr=parsed.references,
subject=parsed.subject,
from_addr=parsed.from_addr,
to_addr=parsed.to_addr,
cc_addr=parsed.cc_addr,
date_hdr=parsed.date,
internal_date=remote.internal_date or datetime.now(UTC),
flags=sorted(remote.flags),
size_bytes=len(remote.raw),
has_attachments=parsed.has_attachments,
snippet=parsed.snippet,
blob_sha256=sha,
)
session.add(message)
await session.flush()
# Body parts: plain text (for indexing) + sanitized HTML (for display).
has_remote_images = False
if parsed.text_body:
session.add(BodyPart(message_id=message.id, kind="text", content=parsed.text_body))
if parsed.html_body is not None:
result = sanitize_html(parsed.html_body, message.id)
has_remote_images = result.has_remote_images
session.add(
BodyPart(
message_id=message.id,
kind="html_sanitized",
content=result.html,
sanitizer_v=result.sanitizer_v,
)
)
for att in parsed.attachments:
att_sha = blobstore.store(user_id, att.payload)
session.add(
Attachment(
message_id=message.id,
user_id=user_id,
filename=att.filename,
mime_type=att.mime_type,
size_bytes=len(att.payload),
content_id=att.content_id,
blob_sha256=att_sha,
)
)
message.has_remote_images = has_remote_images
await thread_message(session, message)
if mailbox_special_use not in _NO_RECIPIENT_HARVEST:
await _harvest_recipient(session, message, parsed)
await session.flush()
doc = _build_search_doc(message, parsed, mailbox_special_use)
return PersistResult(message=message, search_doc=doc, has_remote_images=has_remote_images)
+147
View File
@@ -0,0 +1,147 @@
"""MIME parsing via the stdlib ``email`` package with ``policy.default`` (SPEC §6.4).
Primary parsing is stdlib per SPEC §3; ``mail-parser`` is available for helpers only.
Header RFC2047 decoding is automatic under ``policy.default``.
"""
from __future__ import annotations
import email
from dataclasses import dataclass, field
from datetime import datetime
from email.headerregistry import AddressHeader
from email.message import EmailMessage
from email.policy import default as default_policy
import nh3
from app.threading_.normalize import normalize_message_id, normalize_reference_list
_SNIPPET_LEN = 160
@dataclass(frozen=True)
class ParsedAttachment:
filename: str
mime_type: str
content_id: str | None
payload: bytes
@dataclass
class ParsedEmail:
message_id: str | None
in_reply_to: str | None
references: list[str] # References + In-Reply-To, normalized, in order (§6.4)
subject: str
from_addr: list[dict[str, str]]
to_addr: list[dict[str, str]]
cc_addr: list[dict[str, str]]
date: datetime | None
text_body: str
html_body: str | None
attachments: list[ParsedAttachment] = field(default_factory=list)
@property
def has_attachments(self) -> bool:
return len(self.attachments) > 0
@property
def snippet(self) -> str:
return " ".join(self.text_body.split())[:_SNIPPET_LEN]
def _addresses(msg: EmailMessage, header: str) -> list[dict[str, str]]:
value = msg[header]
if value is None:
return []
out: list[dict[str, str]] = []
if isinstance(value, AddressHeader):
for addr in value.addresses:
if addr.addr_spec:
out.append({"name": str(addr.display_name or ""), "email": addr.addr_spec})
return out
def _html_to_text(html: str) -> str:
"""Derive plain text from HTML by stripping all tags (SPEC §6.4 nh3 strip)."""
return nh3.clean(html, tags=set(), attributes={}, strip_comments=True)
def _get_body_content(msg: EmailMessage, subtype: str) -> str | None:
part = msg.get_body(preferencelist=(subtype,))
if part is None:
return None
try:
content = part.get_content()
except (LookupError, ValueError, KeyError):
payload = part.get_payload(decode=True)
if not isinstance(payload, bytes):
return None
content = payload.decode("utf-8", "replace")
return content if isinstance(content, str) else None
def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
out: list[ParsedAttachment] = []
for part in msg.iter_attachments():
if not isinstance(part, EmailMessage):
continue
payload = part.get_payload(decode=True)
if not isinstance(payload, bytes):
continue
cid = part.get("Content-ID")
out.append(
ParsedAttachment(
filename=part.get_filename() or "attachment",
mime_type=part.get_content_type(),
content_id=cid.strip("<>") if cid else None,
payload=payload,
)
)
return out
def parse_message(raw: bytes) -> ParsedEmail:
"""Parse a raw RFC822 message into a :class:`ParsedEmail`."""
msg = email.message_from_bytes(raw, policy=default_policy)
assert isinstance(msg, EmailMessage)
message_id = normalize_message_id(msg.get("Message-ID"))
in_reply_to_ids = normalize_reference_list(msg.get("In-Reply-To"))
in_reply_to = in_reply_to_ids[0] if in_reply_to_ids else None
# references_hdr = References then In-Reply-To message-ids, in order, deduped (§6.4).
references: list[str] = []
for ref in normalize_reference_list(msg.get("References")) + in_reply_to_ids:
if ref not in references:
references.append(ref)
subject = str(msg.get("Subject", "") or "")
date: datetime | None = None
date_header = msg["Date"]
if date_header is not None:
try:
date = date_header.datetime
except (AttributeError, ValueError):
date = None
html_body = _get_body_content(msg, "html")
text_body = _get_body_content(msg, "plain")
if text_body is None and html_body is not None:
text_body = _html_to_text(html_body)
return ParsedEmail(
message_id=message_id,
in_reply_to=in_reply_to,
references=references,
subject=subject,
from_addr=_addresses(msg, "From"),
to_addr=_addresses(msg, "To"),
cc_addr=_addresses(msg, "Cc"),
date=date,
text_body=text_body or "",
html_body=html_body,
attachments=_extract_attachments(msg),
)
+195
View File
@@ -0,0 +1,195 @@
"""HTML sanitization with nh3 (SPEC §6.4, §10.4, invariant I-8).
Mail HTML is hostile: sanitized server-side at index time with a broad-but-safe
allowlist (structural + presentational tags, filtered inline CSS), and rendered
inside a sandboxed iframe. ``cid:`` images are rewritten to the per-message
endpoint; remote images (and CSS ``url()``) are blocked by default and flagged so
the UI can offer "load images" (SPEC §9.4).
"""
from __future__ import annotations
import re
import uuid
from dataclasses import dataclass
import nh3
# Bump when the sanitization policy changes; body_parts.sanitizer_v records it so a
# render re-sanitizes stale content from the raw blob (SPEC §10.4).
SANITIZER_VERSION = 2
_ALLOWED_TAGS: set[str] = {
# structure
"a",
"abbr",
"address",
"article",
"aside",
"b",
"bdi",
"bdo",
"blockquote",
"br",
"caption",
"center",
"cite",
"code",
"col",
"colgroup",
"dd",
"del",
"details",
"dfn",
"div",
"dl",
"dt",
"em",
"figcaption",
"figure",
"font",
"footer",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"i",
"img",
"ins",
"kbd",
"li",
"main",
"mark",
"nav",
"ol",
"p",
"pre",
"q",
"s",
"samp",
"section",
"small",
"span",
"strike",
"strong",
"sub",
"summary",
"sup",
"table",
"tbody",
"td",
"tfoot",
"th",
"thead",
"time",
"tr",
"tt",
"u",
"ul",
"var",
"wbr",
}
# Attributes allowed on any tag (presentational email formatting).
_GENERIC_ATTRS: set[str] = {
"style",
"align",
"valign",
"dir",
"title",
"width",
"height",
"bgcolor",
"color",
"face",
"size",
"span",
"nowrap",
"lang",
}
_TAG_SPECIFIC: dict[str, set[str]] = {
"a": {"href", "target"},
"img": {"src", "alt"},
"td": {"colspan", "rowspan"},
"th": {"colspan", "rowspan", "scope"},
"table": {"cellpadding", "cellspacing", "border"},
"font": {"face", "color", "size"},
}
# nh3 has no "generic attributes" option, so expand the generic set onto every tag.
_ALLOWED_ATTRS: dict[str, set[str]] = {
tag: set(_GENERIC_ATTRS) | _TAG_SPECIFIC.get(tag, set()) for tag in _ALLOWED_TAGS
}
_URL_SCHEMES: set[str] = {"http", "https", "mailto", "cid", "data"}
_STYLE_DANGEROUS = re.compile(
r"(expression\s*\(|javascript:|vbscript:|@import|-moz-binding|behavior\s*:)", re.IGNORECASE
)
_STYLE_URL = re.compile(r"url\s*\([^)]*\)", re.IGNORECASE)
@dataclass(frozen=True)
class SanitizeResult:
html: str
has_remote_images: bool
sanitizer_v: int
def _sanitize_style(value: str, allow_remote: bool) -> str | None:
if _STYLE_DANGEROUS.search(value):
return None # drop the whole style rather than try to repair it
if not allow_remote and _STYLE_URL.search(value):
# Strip CSS url() so backgrounds can't smuggle in remote/tracking images.
return _STYLE_URL.sub("none", value)
return value
def sanitize_html(
html: str, message_id: uuid.UUID, *, allow_remote_images: bool = False
) -> SanitizeResult:
"""Sanitize ``html`` for the message ``message_id``.
Returns the safe HTML, whether any remote images were blocked, and the
sanitizer version. Pass ``allow_remote_images=True`` after the user approves
a sender to render remote images and CSS backgrounds (SPEC §9.4).
"""
state = {"has_remote": False}
def attribute_filter(tag: str, attr: str, value: str) -> str | None:
if attr == "style":
return _sanitize_style(value, allow_remote_images)
if tag == "a" and attr == "target":
return "_blank"
if tag == "img" and attr == "src":
lowered = value.strip().lower()
if lowered.startswith("cid:"):
content_id = value.strip()[4:]
return f"/api/messages/{message_id}/cid/{content_id}"
if lowered.startswith(("http:", "https:")):
state["has_remote"] = True
return value if allow_remote_images else None # block by default (FR-23)
if lowered.startswith("data:image/"):
return value
return None
return value
cleaned = nh3.clean(
html,
tags=_ALLOWED_TAGS,
attributes=_ALLOWED_ATTRS,
url_schemes=_URL_SCHEMES,
link_rel="noopener noreferrer nofollow",
strip_comments=True,
attribute_filter=attribute_filter,
)
return SanitizeResult(
html=cleaned,
has_remote_images=state["has_remote"],
sanitizer_v=SANITIZER_VERSION,
)