Initial commit: webmail client (M0/M1)
Three-pane webmail client. - Backend: FastAPI + async SQLAlchemy + arq worker (Postgres/Redis), IMAP/SMTP sync, threading, search, drafts/outbox with undo, attachments, 2FA. - Frontend: React 18 + Vite + TypeScript + TanStack Query + Tailwind; three-pane UI with rich-text composer, light/dark themes, resizable panels. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1 @@
|
||||
"""JWZ threading (SPEC §7). Trailing underscore avoids the stdlib ``threading`` clash."""
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Message-ID and subject normalization (SPEC §7.1, §7.3)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
# Repeated reply/forward prefixes to strip, case-insensitive (incl. Polish Odp:/PD:).
|
||||
_PREFIX_RE = re.compile(r"^\s*(re|fw|fwd|odp|pd)\s*:\s*", re.IGNORECASE)
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
_ANGLE_RE = re.compile(r"<([^>]*)>")
|
||||
|
||||
|
||||
def normalize_message_id(raw: str | None) -> str | None:
|
||||
"""Normalize a Message-ID per SPEC §7.1.
|
||||
|
||||
Strip angle brackets and whitespace, lowercase the domain part only.
|
||||
Return None for obviously invalid ids (no ``@``).
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
text = raw.strip()
|
||||
m = _ANGLE_RE.search(text)
|
||||
if m:
|
||||
text = m.group(1)
|
||||
text = text.strip().strip("<>").strip()
|
||||
if "@" not in text:
|
||||
return None
|
||||
local, _, domain = text.rpartition("@")
|
||||
if not local or not domain:
|
||||
return None
|
||||
return f"{local}@{domain.lower()}"
|
||||
|
||||
|
||||
def normalize_reference_list(raw: str | None) -> list[str]:
|
||||
"""Parse a References / In-Reply-To header value into normalized ids, in order."""
|
||||
if not raw:
|
||||
return []
|
||||
out: list[str] = []
|
||||
for token in _ANGLE_RE.findall(raw) or raw.split():
|
||||
norm = normalize_message_id(token)
|
||||
if norm and norm not in out:
|
||||
out.append(norm)
|
||||
return out
|
||||
|
||||
|
||||
def synthesize_message_id(raw_message: bytes) -> str:
|
||||
"""Synthesize a non-authoritative Message-ID for a message that lacks one (§7.1)."""
|
||||
digest = hashlib.sha256(raw_message).hexdigest()[:32]
|
||||
return f"{digest}@synthetic.local"
|
||||
|
||||
|
||||
def is_synthetic(message_id: str | None) -> bool:
|
||||
"""Return True if ``message_id`` is a synthesized (non-authoritative) id."""
|
||||
return message_id is not None and message_id.endswith("@synthetic.local")
|
||||
|
||||
|
||||
def normalize_subject(subject: str) -> str:
|
||||
"""Normalize a subject for the §7.3 fallback: strip repeated Re:/Fw:/... prefixes."""
|
||||
text = subject or ""
|
||||
while True:
|
||||
stripped = _PREFIX_RE.sub("", text, count=1)
|
||||
if stripped == text:
|
||||
break
|
||||
text = stripped
|
||||
return _WS_RE.sub(" ", text).strip()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""JWZ threading — incremental per-message algorithm (SPEC §7.2, §7.3).
|
||||
|
||||
Cross-account by design: candidate lookup is scoped by ``user_id`` only (I-3).
|
||||
Called after the new :class:`Message` row is flushed (so it has an ``id`` and its
|
||||
normalized id columns); assigns ``thread_id`` and recomputes the thread's
|
||||
denormalized fields.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import ColumnElement, delete, literal, select, update
|
||||
from sqlalchemy import any_ as sa_any
|
||||
from sqlalchemy import or_ as sa_or
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Message, Thread
|
||||
from app.threading_.normalize import is_synthetic, normalize_subject
|
||||
|
||||
_SUBJECT_FALLBACK_WINDOW = timedelta(days=7)
|
||||
|
||||
|
||||
def _recipient_emails(message: Message) -> set[str]:
|
||||
"""Recipient (To/Cc) emails of a message, lowercased.
|
||||
|
||||
Subject-fallback joining keys on recipients rather than all participants so a
|
||||
bulk sender that reuses one subject to many recipients (e.g. a newsletter or
|
||||
a static welcome mail) does not collapse unrelated deliveries into one thread.
|
||||
A genuine reply is always addressed to someone already in the thread.
|
||||
"""
|
||||
emails: set[str] = set()
|
||||
for field in (message.to_addr, message.cc_addr):
|
||||
for addr in field:
|
||||
email = addr.get("email")
|
||||
if email:
|
||||
emails.add(email.lower())
|
||||
return emails
|
||||
|
||||
|
||||
def _candidate_ids(message: Message) -> list[str]:
|
||||
candidates: list[str] = list(message.references_hdr)
|
||||
if message.in_reply_to:
|
||||
candidates.append(message.in_reply_to)
|
||||
# Own id may let an earlier reply (that referenced us) join when we arrive late.
|
||||
if message.message_id_hdr and not is_synthetic(message.message_id_hdr):
|
||||
candidates.append(message.message_id_hdr)
|
||||
# Dedupe, drop synthetic ids (never used to join, §7.1).
|
||||
seen: list[str] = []
|
||||
for c in candidates:
|
||||
if c and not is_synthetic(c) and c not in seen:
|
||||
seen.append(c)
|
||||
return seen
|
||||
|
||||
|
||||
async def _find_by_references(
|
||||
session: AsyncSession,
|
||||
user_id: uuid.UUID,
|
||||
message_id: uuid.UUID,
|
||||
candidates: list[str],
|
||||
own_id: str | None,
|
||||
) -> list[uuid.UUID]:
|
||||
"""Threads joined by reference (SPEC §7.2.2).
|
||||
|
||||
Matches bidirectionally: existing messages whose id is one of my candidate ids
|
||||
(I reference them / duplicate id), OR existing messages that reference *my* id
|
||||
(handles a late-arriving parent). Synthetic own-ids are never used to join (§7.1).
|
||||
"""
|
||||
conditions: list[ColumnElement[bool]] = []
|
||||
if candidates:
|
||||
conditions.append(Message.message_id_hdr.in_(candidates))
|
||||
if own_id is not None and not is_synthetic(own_id):
|
||||
# own_id = ANY(references_hdr): an existing message that references me.
|
||||
conditions.append(literal(own_id) == sa_any(Message.references_hdr))
|
||||
conditions.append(Message.in_reply_to == own_id)
|
||||
if not conditions:
|
||||
return []
|
||||
stmt = (
|
||||
select(Message.thread_id)
|
||||
.where(
|
||||
Message.user_id == user_id,
|
||||
Message.thread_id.is_not(None),
|
||||
Message.id != message_id,
|
||||
sa_or(*conditions),
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
rows = await session.scalars(stmt)
|
||||
return [tid for tid in rows if tid is not None]
|
||||
|
||||
|
||||
async def _find_by_subject(
|
||||
session: AsyncSession, user_id: uuid.UUID, message: Message
|
||||
) -> uuid.UUID | None:
|
||||
subject_norm = normalize_subject(message.subject)
|
||||
# Never join when this message both lacks references and has an empty subject (§7.3).
|
||||
if not subject_norm:
|
||||
return None
|
||||
anchor = message.internal_date
|
||||
lo = anchor - _SUBJECT_FALLBACK_WINDOW
|
||||
hi = anchor + _SUBJECT_FALLBACK_WINDOW
|
||||
stmt = (
|
||||
select(Thread)
|
||||
.where(
|
||||
Thread.user_id == user_id,
|
||||
Thread.subject_norm == subject_norm,
|
||||
Thread.last_message_at >= lo,
|
||||
Thread.last_message_at <= hi,
|
||||
)
|
||||
.order_by(Thread.last_message_at.desc())
|
||||
)
|
||||
my_recipients = _recipient_emails(message)
|
||||
if not my_recipients:
|
||||
return None
|
||||
for thread in await session.scalars(stmt):
|
||||
participant_emails = {
|
||||
(p.get("email") or "").lower() for p in thread.participants if p.get("email")
|
||||
}
|
||||
if my_recipients & participant_emails:
|
||||
return thread.id
|
||||
return None
|
||||
|
||||
|
||||
async def _merge_threads(
|
||||
session: AsyncSession, thread_ids: list[uuid.UUID]
|
||||
) -> tuple[uuid.UUID, list[uuid.UUID]]:
|
||||
"""Merge ``thread_ids`` keeping the oldest ``last_message_at`` as canonical (§7.2.5)."""
|
||||
threads = list(await session.scalars(select(Thread).where(Thread.id.in_(thread_ids))))
|
||||
canonical = min(threads, key=lambda t: t.last_message_at)
|
||||
others = [t.id for t in threads if t.id != canonical.id]
|
||||
if others:
|
||||
await session.execute(
|
||||
update(Message).where(Message.thread_id.in_(others)).values(thread_id=canonical.id)
|
||||
)
|
||||
await session.execute(delete(Thread).where(Thread.id.in_(others)))
|
||||
return canonical.id, others
|
||||
|
||||
|
||||
async def _create_thread(session: AsyncSession, message: Message) -> uuid.UUID:
|
||||
thread = Thread(
|
||||
user_id=message.user_id,
|
||||
subject_norm=normalize_subject(message.subject),
|
||||
last_message_at=message.internal_date,
|
||||
)
|
||||
session.add(thread)
|
||||
await session.flush()
|
||||
return thread.id
|
||||
|
||||
|
||||
async def denormalize_thread(session: AsyncSession, thread_id: uuid.UUID) -> None:
|
||||
"""Recompute a thread's denormalized fields from its messages (SPEC §7.2.6)."""
|
||||
messages = list(
|
||||
await session.scalars(
|
||||
select(Message)
|
||||
.where(Message.thread_id == thread_id)
|
||||
.order_by(Message.internal_date.desc())
|
||||
)
|
||||
)
|
||||
thread = await session.get(Thread, thread_id)
|
||||
if thread is None:
|
||||
return
|
||||
if not messages:
|
||||
await session.execute(delete(Thread).where(Thread.id == thread_id))
|
||||
return
|
||||
|
||||
newest = messages[0]
|
||||
participants: list[dict[str, str]] = []
|
||||
seen_emails: set[str] = set()
|
||||
for msg in messages: # newest first => recency order
|
||||
for addr in list(msg.from_addr) + list(msg.to_addr) + list(msg.cc_addr):
|
||||
email = (addr.get("email") or "").lower()
|
||||
if email and email not in seen_emails:
|
||||
seen_emails.add(email)
|
||||
participants.append({"name": addr.get("name", ""), "email": addr.get("email", "")})
|
||||
|
||||
thread.message_count = len(messages)
|
||||
thread.unread_count = sum(1 for m in messages if "\\Seen" not in m.flags)
|
||||
thread.has_attachments = any(m.has_attachments for m in messages)
|
||||
thread.pinned = any("\\Flagged" in m.flags for m in messages)
|
||||
thread.account_ids = sorted({m.account_id for m in messages})
|
||||
thread.last_message_at = newest.internal_date
|
||||
thread.snippet = newest.snippet
|
||||
thread.subject_norm = normalize_subject(newest.subject) or thread.subject_norm
|
||||
thread.participants = participants
|
||||
|
||||
|
||||
async def thread_message(session: AsyncSession, message: Message) -> uuid.UUID:
|
||||
"""Assign ``message`` to a thread (creating/attaching/merging) and denormalize.
|
||||
|
||||
Returns the resulting thread id and sets ``message.thread_id``.
|
||||
"""
|
||||
candidates = _candidate_ids(message)
|
||||
own_id = message.message_id_hdr
|
||||
thread_ids = await _find_by_references(session, message.user_id, message.id, candidates, own_id)
|
||||
|
||||
if not thread_ids:
|
||||
subject_thread = await _find_by_subject(session, message.user_id, message)
|
||||
if subject_thread is not None:
|
||||
thread_ids = [subject_thread]
|
||||
|
||||
if not thread_ids:
|
||||
thread_id = await _create_thread(session, message)
|
||||
elif len(thread_ids) == 1:
|
||||
thread_id = thread_ids[0]
|
||||
else:
|
||||
thread_id, _merged_away = await _merge_threads(session, thread_ids)
|
||||
|
||||
message.thread_id = thread_id
|
||||
await session.flush()
|
||||
await denormalize_thread(session, thread_id)
|
||||
await session.flush()
|
||||
return thread_id
|
||||
Reference in New Issue
Block a user