Initial commit: webmail client (M0/M1)
Three-pane webmail client. - Backend: FastAPI + async SQLAlchemy + arq worker (Postgres/Redis), IMAP/SMTP sync, threading, search, drafts/outbox with undo, attachments, 2FA. - Frontend: React 18 + Vite + TypeScript + TanStack Query + Tailwind; three-pane UI with rich-text composer, light/dark themes, resizable panels. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1 @@
|
||||
"""Search: engine-independent query grammar + pluggable backends (SPEC §5.2, §8)."""
|
||||
@@ -0,0 +1,58 @@
|
||||
# INTERFACE VERSION: 1
|
||||
"""SearchBackend — the frozen search contract (SPEC §5.2).
|
||||
|
||||
Both backends (``pg.py`` for M0-M1, ``meili.py`` for M2) index the same
|
||||
:class:`SearchDoc` and honour invariant I-7 (every query carries ``user_id``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchQuery:
|
||||
text: str | None # free-text remainder after operator extraction
|
||||
filters: dict[str, object] # parsed operators, keys per §8.2
|
||||
user_id: UUID # ALWAYS set (invariant I-7)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchHit:
|
||||
message_id: UUID
|
||||
thread_id: UUID
|
||||
score: float
|
||||
highlights: dict[str, str] # field -> HTML-escaped snippet with <mark>
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchDoc:
|
||||
"""The document indexed by every backend (SPEC §5.2)."""
|
||||
|
||||
message_id: UUID
|
||||
thread_id: UUID
|
||||
user_id: UUID
|
||||
account_id: UUID
|
||||
mailbox_special_use: str | None
|
||||
from_: list[dict[str, str]] # name+email
|
||||
to: list[dict[str, str]]
|
||||
cc: list[dict[str, str]]
|
||||
subject: str
|
||||
body_text: str
|
||||
attachment_names: list[str] = field(default_factory=list)
|
||||
has_attachments: bool = False
|
||||
is_unread: bool = False
|
||||
is_starred: bool = False
|
||||
date: int = 0 # unix seconds
|
||||
size_bytes: int = 0
|
||||
|
||||
|
||||
class SearchBackend(Protocol):
|
||||
async def index(self, doc: SearchDoc) -> None: ...
|
||||
async def delete(self, message_id: UUID) -> None: ...
|
||||
async def delete_user(self, user_id: UUID) -> None: ...
|
||||
async def query(
|
||||
self, q: SearchQuery, *, limit: int, cursor: str | None
|
||||
) -> tuple[list[SearchHit], str | None]: ...
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Select the active SearchBackend from config (SPEC §8.1, env SEARCH_BACKEND)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.config import get_settings
|
||||
from app.search.base import SearchBackend
|
||||
from app.search.pg import PgSearchBackend
|
||||
|
||||
_backend: SearchBackend | None = None
|
||||
|
||||
|
||||
def get_search_backend() -> SearchBackend:
|
||||
"""Return the process-wide SearchBackend (``pg`` in M0-M1, ``meili`` in M2)."""
|
||||
global _backend
|
||||
if _backend is None:
|
||||
which = get_settings().search_backend
|
||||
if which == "meili":
|
||||
# TODO(spec): Meilisearch backend lands in M2.
|
||||
raise NotImplementedError("meili backend is implemented in M2")
|
||||
_backend = PgSearchBackend()
|
||||
return _backend
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Postgres full-text SearchBackend (SPEC §5.2, §8; M0-M1).
|
||||
|
||||
Indexing populates ``messages.body_indexed`` (a ``tsvector``) from the doc's
|
||||
searchable fields. Querying translates the parsed §8.2 groups (AND of ORs) into
|
||||
SQL over messages joined to mailboxes/accounts, always filtered by ``user_id``
|
||||
(invariant I-7). Relevance ranking / typo tolerance is Meilisearch's job (M2);
|
||||
here results are date-ordered with keyset pagination.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import html
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import Text, any_, cast, exists, func, literal, not_, or_, select, tuple_, update
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from app.db import get_sessionmaker
|
||||
from app.models import Attachment, Mailbox, Message
|
||||
from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
|
||||
|
||||
_TS_CONFIG = "simple" # language-neutral; avoids wrong-language stemming (M0)
|
||||
_RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
|
||||
|
||||
|
||||
def _doc_text_blob(doc: SearchDoc) -> str:
|
||||
parts = [doc.subject, doc.body_text]
|
||||
for field in (doc.from_, doc.to, doc.cc):
|
||||
for addr in field:
|
||||
parts.append(addr.get("name", ""))
|
||||
parts.append(addr.get("email", ""))
|
||||
parts.extend(doc.attachment_names)
|
||||
return " ".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _parse_date(value: str) -> dt.datetime | None:
|
||||
try:
|
||||
return dt.datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=dt.UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_relative(value: str) -> dt.timedelta | None:
|
||||
m = re.fullmatch(r"(\d+)([dwmy])", value.strip().lower())
|
||||
if not m:
|
||||
return None
|
||||
n = int(m.group(1)) * _RELATIVE_MULT[m.group(2)]
|
||||
return dt.timedelta(days=n)
|
||||
|
||||
|
||||
def _parse_size(value: str) -> int | None:
|
||||
m = re.fullmatch(r"(\d+)([km])", value.strip().lower())
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) * (1024 if m.group(2) == "k" else 1024 * 1024)
|
||||
|
||||
|
||||
def _flag_condition(flag: str) -> ColumnElement[bool]:
|
||||
# flag = ANY(messages.flags) — the array membership operator, not func any().
|
||||
return literal(flag) == any_(Message.flags)
|
||||
|
||||
|
||||
class PgSearchBackend(SearchBackend):
|
||||
async def index(self, doc: SearchDoc) -> None:
|
||||
blob = _doc_text_blob(doc)
|
||||
async with get_sessionmaker()() as session:
|
||||
await session.execute(
|
||||
update(Message)
|
||||
.where(Message.id == doc.message_id, Message.user_id == doc.user_id)
|
||||
.values(body_indexed=func.to_tsvector(_TS_CONFIG, blob))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def delete(self, message_id: UUID) -> None:
|
||||
async with get_sessionmaker()() as session:
|
||||
await session.execute(
|
||||
update(Message).where(Message.id == message_id).values(body_indexed=None)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def delete_user(self, user_id: UUID) -> None:
|
||||
# User deletion cascades message rows (FK ON DELETE CASCADE); clearing the
|
||||
# tsvector here keeps the backend contract satisfied for partial re-index.
|
||||
async with get_sessionmaker()() as session:
|
||||
await session.execute(
|
||||
update(Message).where(Message.user_id == user_id).values(body_indexed=None)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# --- querying ------------------------------------------------------------
|
||||
|
||||
def _clause_condition(self, clause: dict[str, object]) -> ColumnElement[bool] | None:
|
||||
value = str(clause.get("value", ""))
|
||||
if clause.get("type") == "text":
|
||||
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
|
||||
key = clause.get("key")
|
||||
cond = self._filter_condition(str(key), value)
|
||||
return cond
|
||||
|
||||
def _filter_condition(self, key: str, value: str) -> ColumnElement[bool] | None:
|
||||
like = f"%{value}%"
|
||||
if key == "from":
|
||||
return cast(Message.from_addr, Text).ilike(like)
|
||||
if key == "to":
|
||||
return cast(Message.to_addr, Text).ilike(like)
|
||||
if key == "cc":
|
||||
return cast(Message.cc_addr, Text).ilike(like)
|
||||
if key == "subject":
|
||||
return Message.subject.ilike(like)
|
||||
if key == "in":
|
||||
return or_(Mailbox.special_use == value.lower(), Mailbox.path.ilike(like))
|
||||
if key == "is":
|
||||
v = value.lower()
|
||||
if v == "unread":
|
||||
return not_(_flag_condition("\\Seen"))
|
||||
if v == "read":
|
||||
return _flag_condition("\\Seen")
|
||||
if v == "starred":
|
||||
return _flag_condition("\\Flagged")
|
||||
if v == "answered":
|
||||
return _flag_condition("\\Answered")
|
||||
return None
|
||||
if key == "has":
|
||||
return Message.has_attachments.is_(True) if value.lower() == "attachment" else None
|
||||
if key == "filename":
|
||||
return exists().where(
|
||||
Attachment.message_id == Message.id, Attachment.filename.ilike(like)
|
||||
)
|
||||
if key == "before":
|
||||
d = _parse_date(value)
|
||||
return Message.internal_date < d if d else None
|
||||
if key == "after":
|
||||
d = _parse_date(value)
|
||||
return Message.internal_date >= d if d else None
|
||||
if key == "newer_than":
|
||||
delta = _parse_relative(value)
|
||||
return Message.internal_date >= func.now() - delta if delta else None
|
||||
if key == "older_than":
|
||||
delta = _parse_relative(value)
|
||||
return Message.internal_date < func.now() - delta if delta else None
|
||||
if key == "larger":
|
||||
n = _parse_size(value)
|
||||
return Message.size_bytes > n if n is not None else None
|
||||
if key == "smaller":
|
||||
n = _parse_size(value)
|
||||
return Message.size_bytes < n if n is not None else None
|
||||
return None
|
||||
|
||||
def _build_conditions(self, groups: list[list[dict[str, object]]]) -> list[ColumnElement[bool]]:
|
||||
conditions: list[ColumnElement[bool]] = []
|
||||
for group in groups:
|
||||
ors: list[ColumnElement[bool]] = []
|
||||
for clause in group:
|
||||
cond = self._clause_condition(clause)
|
||||
if cond is None:
|
||||
continue
|
||||
if clause.get("negate"):
|
||||
cond = not_(cond)
|
||||
ors.append(cond)
|
||||
if ors:
|
||||
conditions.append(or_(*ors))
|
||||
return conditions
|
||||
|
||||
async def query(
|
||||
self, q: SearchQuery, *, limit: int, cursor: str | None
|
||||
) -> tuple[list[SearchHit], str | None]:
|
||||
groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
|
||||
assert isinstance(groups, list)
|
||||
conditions = self._build_conditions(groups)
|
||||
|
||||
rank = (
|
||||
func.ts_rank_cd(Message.body_indexed, func.websearch_to_tsquery(_TS_CONFIG, q.text))
|
||||
if q.text
|
||||
else literal(0.0)
|
||||
)
|
||||
stmt = (
|
||||
select(
|
||||
Message.id,
|
||||
Message.thread_id,
|
||||
Message.subject,
|
||||
Message.snippet,
|
||||
Message.internal_date,
|
||||
rank.label("score"),
|
||||
)
|
||||
.select_from(Message)
|
||||
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||||
.where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
|
||||
)
|
||||
for cond in conditions:
|
||||
stmt = stmt.where(cond)
|
||||
|
||||
decoded = _decode_cursor(cursor)
|
||||
if decoded is not None:
|
||||
cur_date, cur_id = decoded
|
||||
stmt = stmt.where(
|
||||
tuple_(Message.internal_date, Message.id)
|
||||
< tuple_(literal(cur_date), literal(cur_id))
|
||||
)
|
||||
stmt = stmt.order_by(Message.internal_date.desc(), Message.id.desc()).limit(limit + 1)
|
||||
|
||||
terms = _positive_text_terms(groups)
|
||||
async with get_sessionmaker()() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
hits = [
|
||||
SearchHit(
|
||||
message_id=row.id,
|
||||
thread_id=row.thread_id,
|
||||
score=float(row.score or 0.0),
|
||||
highlights=_highlights(row.subject, row.snippet, terms),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
next_cursor = (
|
||||
_encode_cursor(rows[-1].internal_date, rows[-1].id) if has_more and rows else None
|
||||
)
|
||||
return hits, next_cursor
|
||||
|
||||
|
||||
def _positive_text_terms(groups: list[list[dict[str, object]]]) -> list[str]:
|
||||
terms: list[str] = []
|
||||
for group in groups:
|
||||
for clause in group:
|
||||
if clause.get("type") == "text" and not clause.get("negate"):
|
||||
terms.append(str(clause.get("value", "")))
|
||||
return terms
|
||||
|
||||
|
||||
def _highlights(subject: str, snippet: str, terms: list[str]) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
if not terms:
|
||||
return out
|
||||
marked_subject = _mark(subject, terms)
|
||||
if "<mark>" in marked_subject:
|
||||
out["subject"] = marked_subject
|
||||
marked_body = _mark(snippet, terms)
|
||||
if "<mark>" in marked_body:
|
||||
out["body"] = marked_body
|
||||
return out
|
||||
|
||||
|
||||
def _mark(value: str, terms: list[str]) -> str:
|
||||
"""HTML-escape ``value`` and wrap term matches in <mark> (SPEC §5.2 highlights)."""
|
||||
escaped = html.escape(value)
|
||||
for term in terms:
|
||||
if not term:
|
||||
continue
|
||||
pattern = re.compile(re.escape(html.escape(term)), re.IGNORECASE)
|
||||
escaped = pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", escaped)
|
||||
return escaped
|
||||
|
||||
|
||||
def _encode_cursor(date: dt.datetime, message_id: UUID) -> str:
|
||||
raw = f"{date.isoformat()}|{message_id}"
|
||||
return base64.urlsafe_b64encode(raw.encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str | None) -> tuple[dt.datetime, UUID] | None:
|
||||
if not cursor:
|
||||
return None
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
||||
date_str, id_str = raw.split("|", 1)
|
||||
return dt.datetime.fromisoformat(date_str), UUID(id_str)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Operator query grammar (SPEC §8.2), engine-independent.
|
||||
|
||||
Tokenization: whitespace split respecting ``"quoted phrases"``. A ``key:value``
|
||||
(or ``-key:value``) token with a known key becomes a filter; everything else is
|
||||
free text. ``-`` negates. ``OR`` (uppercase) groups adjacent clauses into a
|
||||
disjunction; the default combinator is AND. Unknown ``key:`` tokens are treated
|
||||
as free text (forward compatibility). The parser never raises.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID
|
||||
|
||||
from app.search.base import SearchQuery
|
||||
|
||||
# Known operator keys (SPEC §8.2). Unknown keys fall through to free text.
|
||||
KNOWN_KEYS: frozenset[str] = frozenset(
|
||||
{
|
||||
"from",
|
||||
"to",
|
||||
"cc",
|
||||
"subject",
|
||||
"account",
|
||||
"in",
|
||||
"is",
|
||||
"has",
|
||||
"filename",
|
||||
"before",
|
||||
"after",
|
||||
"newer_than",
|
||||
"older_than",
|
||||
"larger",
|
||||
"smaller",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Clause:
|
||||
"""One parsed unit: a free-text term or a ``key:value`` filter."""
|
||||
|
||||
type: str # "text" | "filter"
|
||||
key: str | None # operator key for filters, else None
|
||||
value: str
|
||||
negate: bool
|
||||
phrase: bool # value was a quoted phrase
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"type": self.type,
|
||||
"key": self.key,
|
||||
"value": self.value,
|
||||
"negate": self.negate,
|
||||
"phrase": self.phrase,
|
||||
}
|
||||
|
||||
|
||||
# A parsed query is an AND of groups; each group is an OR of clauses.
|
||||
ParsedQuery = list[list[Clause]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Token:
|
||||
text: str
|
||||
negate: bool
|
||||
starts_with_quote: bool # a bare quoted phrase -> never a key:value filter
|
||||
|
||||
|
||||
def _tokenize(query: str) -> list[_Token]:
|
||||
"""Split into tokens honouring quotes; never raises on unbalanced quotes.
|
||||
|
||||
Tracks a leading ``-`` (negation) and whether the token began with a quote
|
||||
(a bare phrase), so ``key:"quoted value"`` is still recognised as a filter
|
||||
while ``"key: value"`` stays free text.
|
||||
"""
|
||||
tokens: list[_Token] = []
|
||||
i, n = 0, len(query)
|
||||
while i < n:
|
||||
while i < n and query[i].isspace():
|
||||
i += 1
|
||||
if i >= n:
|
||||
break
|
||||
negate = False
|
||||
if query[i] == "-" and i + 1 < n and not query[i + 1].isspace():
|
||||
negate = True
|
||||
i += 1
|
||||
buf: list[str] = []
|
||||
starts_with_quote = i < n and query[i] == '"'
|
||||
while i < n and not query[i].isspace():
|
||||
if query[i] == '"':
|
||||
i += 1
|
||||
while i < n and query[i] != '"':
|
||||
buf.append(query[i])
|
||||
i += 1
|
||||
if i < n and query[i] == '"':
|
||||
i += 1 # consume closing quote
|
||||
else:
|
||||
buf.append(query[i])
|
||||
i += 1
|
||||
tokens.append(_Token(text="".join(buf), negate=negate, starts_with_quote=starts_with_quote))
|
||||
return tokens
|
||||
|
||||
|
||||
def _to_clause(token: _Token) -> Clause | None:
|
||||
if not token.text and not token.starts_with_quote:
|
||||
return None
|
||||
if not token.starts_with_quote and ":" in token.text:
|
||||
key, _, value = token.text.partition(":")
|
||||
if key.lower() in KNOWN_KEYS and value:
|
||||
return Clause(
|
||||
type="filter",
|
||||
key=key.lower(),
|
||||
value=value,
|
||||
negate=token.negate,
|
||||
phrase=(" " in value),
|
||||
)
|
||||
return Clause(
|
||||
type="text",
|
||||
key=None,
|
||||
value=token.text,
|
||||
negate=token.negate,
|
||||
phrase=token.starts_with_quote or (" " in token.text),
|
||||
)
|
||||
|
||||
|
||||
def parse(query: str) -> ParsedQuery:
|
||||
"""Parse ``query`` into AND-of-OR groups of :class:`Clause`."""
|
||||
groups: ParsedQuery = []
|
||||
pending_or = False
|
||||
for token in _tokenize(query):
|
||||
if token.text == "OR" and not token.negate and not token.starts_with_quote:
|
||||
pending_or = True
|
||||
continue
|
||||
clause = _to_clause(token)
|
||||
if clause is None:
|
||||
continue
|
||||
if pending_or and groups:
|
||||
groups[-1].append(clause)
|
||||
else:
|
||||
groups.append([clause])
|
||||
pending_or = False
|
||||
return groups
|
||||
|
||||
|
||||
def free_text(groups: ParsedQuery) -> str | None:
|
||||
"""Return the positive free-text remainder (standalone, non-negated text terms)."""
|
||||
terms = [g[0].value for g in groups if len(g) == 1 and g[0].type == "text" and not g[0].negate]
|
||||
return " ".join(terms) if terms else None
|
||||
|
||||
|
||||
def to_search_query(query: str, user_id: UUID) -> SearchQuery:
|
||||
"""Parse ``query`` into a :class:`SearchQuery` (I-7: ``user_id`` always set)."""
|
||||
groups = parse(query)
|
||||
filters: dict[str, object] = {
|
||||
"groups": [[clause.as_dict() for clause in group] for group in groups]
|
||||
}
|
||||
return SearchQuery(text=free_text(groups), filters=filters, user_id=user_id)
|
||||
Reference in New Issue
Block a user