lukas/webmail-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

search: email address search + address-first ranking; inline images; folder i18n; link security

Browse Source 
Search (PG backend):
- '@' terms match address substrings (from/to/cc), so a partial email like
  "dyrektor@wfdif" resolves and narrows predictably as you type
- index each address split into parts, so a bare domain/local word ("wfdif")
  finds the address (indexed as one lexeme otherwise); dotted terms fall back to
  an address substring
- rank sender/recipient name+address matches above body/subject-only ones
  (addr_hit tier, then recency); cursor extended to keep keyset pagination correct
- scripts/reindex_address_parts.py backfills address tokens into existing rows

Inline images:
- extract attachments via a full MIME walk instead of shallow iter_attachments(),
  which missed inline images nested in mixed>related>alternative (cid: rendered
  broken); has_attachments counts only real (non-cid) attachments
- scripts/backfill_inline_attachments.py stores previously-missed inline images

Folder-name i18n (SPEC §12 M2):
- accent/punctuation-insensitive, multi-delimiter (/ \ .) special-use inference
  across ~13 languages; curated names (no substring false positives)

Link security (reading panel):
- links never navigate the message iframe: a parent-side click interceptor opens
  http/https in a new tab (noopener) and mailto in the composer (recipient
  pre-filled); all other schemes blocked

Backend 292 tests green, ruff + mypy --strict clean; frontend tsc/eslint/vitest clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
lukas
2026-07-26 09:28:48 +02:00
co-authored by Claude Opus 4.8
parent 49738740c0
commit c023cfa15e
11 changed files with 483 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/search/pg.py
+79 -14
View File
@@ -15,7 +15,20 @@ import html
import re
from uuid import UUID
from sqlalchemy import Text, any_, cast, exists, func, literal, not_, or_, select, tuple_, update
from sqlalchemy import (
Text,
any_,
case,
cast,
exists,
func,
literal,
not_,
or_,
select,
tuple_,
update,
)
from sqlalchemy.sql.elements import ColumnElement
from app.db import get_sessionmaker
@@ -42,12 +55,23 @@ def _prefix_tsquery(value: str) -> ColumnElement[object]:
_RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
def _email_parts(email: str) -> str:
"""Split an address on @/./-/+/_ so its domain and local part are searchable
on their own — a bare "wfdif" should find dyrektor@wfdif.com.pl. Postgres
indexes the whole address as a single lexeme, so the parts are invisible to a
word search otherwise."""
return re.sub(r"[@._+\-]+", " ", email)
def _doc_text_blob(doc: SearchDoc) -> str:
parts = [doc.subject, doc.body_text]
for field in (doc.from_, doc.to, doc.cc):
for addr in field:
parts.append(addr.get("name", ""))
parts.append(addr.get("email", ""))
email = addr.get("email", "")
if email:
parts.append(email) # whole address (one lexeme, for exact/prefix)
parts.append(_email_parts(email)) # + its parts as separate tokens
parts.extend(doc.attachment_names)
return " ".join(p for p in parts if p)
@@ -114,7 +138,24 @@ class PgSearchBackend(SearchBackend):
if clause.get("phrase"):
# Quoted phrase: keep exact-adjacency semantics, no prefix.
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
return Message.body_indexed.op("@@")(_prefix_tsquery(value))
like = f"%{value}%"
addr_match = or_(
cast(Message.from_addr, Text).ilike(like),
cast(Message.to_addr, Text).ilike(like),
cast(Message.cc_addr, Text).ilike(like),
)
if "@" in value:
# A term with '@' is an address search: match the address substring
# only (like from:/to:/cc:), so it narrows predictably as you type —
# not the split-word body matching that made a partial term over-hit.
return addr_match
fts = Message.body_indexed.op("@@")(_prefix_tsquery(value))
if "." in value:
# A dotted term (e.g. "wfdif.com") stays one host lexeme in FTS, so
# also match it as an address substring. Bare words don't need this:
# address parts are indexed as their own tokens (see _doc_text_blob).
return or_(fts, addr_match)
return fts
key = clause.get("key")
cond = self._filter_condition(str(key), value)
return cond
@@ -189,7 +230,11 @@ class PgSearchBackend(SearchBackend):
groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
assert isinstance(groups, list)
conditions = self._build_conditions(groups)
terms = _positive_text_terms(groups)
# Rank a match in a sender/recipient name or address above a body/subject-only
# match, so "who" results come first; ties break by recency.
addr_hit = _address_hit_expr(terms)
rank = (
func.ts_rank_cd(Message.body_indexed, _prefix_tsquery(q.text))
if q.text
@@ -203,6 +248,7 @@ class PgSearchBackend(SearchBackend):
Message.snippet,
Message.internal_date,
rank.label("score"),
addr_hit.label("addr_hit"),
)
.select_from(Message)
.join(Mailbox, Mailbox.id == Message.mailbox_id)
@@ -213,14 +259,16 @@ class PgSearchBackend(SearchBackend):
decoded = _decode_cursor(cursor)
if decoded is not None:
cur_date, cur_id = decoded
cur_hit, cur_date, cur_id = decoded
# Keyset over the full ordering (addr_hit, date, id), all descending.
stmt = stmt.where(
tuple_(Message.internal_date, Message.id)
< tuple_(literal(cur_date), literal(cur_id))
tuple_(addr_hit, Message.internal_date, Message.id)
< tuple_(literal(cur_hit), literal(cur_date), literal(cur_id))
)
stmt = stmt.order_by(Message.internal_date.desc(), Message.id.desc()).limit(limit + 1)
stmt = stmt.order_by(
addr_hit.desc(), Message.internal_date.desc(), Message.id.desc()
).limit(limit + 1)
terms = _positive_text_terms(groups)
async with get_sessionmaker()() as session:
rows = (await session.execute(stmt)).all()
@@ -236,7 +284,9 @@ class PgSearchBackend(SearchBackend):
for row in rows
]
next_cursor = (
_encode_cursor(rows[-1].internal_date, rows[-1].id) if has_more and rows else None
_encode_cursor(int(rows[-1].addr_hit), rows[-1].internal_date, rows[-1].id)
if has_more and rows
else None
)
return hits, next_cursor
@@ -250,6 +300,21 @@ def _positive_text_terms(groups: list[list[dict[str, object]]]) -> list[str]:
return terms
def _address_hit_expr(terms: list[str]) -> ColumnElement[int]:
"""1 if any positive free-text term appears in a from/to/cc name or address,
else 0. Used to sort address/name matches ahead of body/subject-only ones."""
conds: list[ColumnElement[bool]] = []
for t in terms:
if t:
like = f"%{t}%"
conds.append(cast(Message.from_addr, Text).ilike(like))
conds.append(cast(Message.to_addr, Text).ilike(like))
conds.append(cast(Message.cc_addr, Text).ilike(like))
if not conds:
return literal(0)
return case((or_(*conds), 1), else_=0)
def _highlights(subject: str, snippet: str, terms: list[str]) -> dict[str, str]:
out: dict[str, str] = {}
if not terms:
@@ -274,17 +339,17 @@ def _mark(value: str, terms: list[str]) -> str:
return escaped
def _encode_cursor(date: dt.datetime, message_id: UUID) -> str:
raw = f"{date.isoformat()}|{message_id}"
def _encode_cursor(addr_hit: int, date: dt.datetime, message_id: UUID) -> str:
raw = f"{addr_hit}|{date.isoformat()}|{message_id}"
return base64.urlsafe_b64encode(raw.encode()).decode()
def _decode_cursor(cursor: str | None) -> tuple[dt.datetime, UUID] | None:
def _decode_cursor(cursor: str | None) -> tuple[int, dt.datetime, UUID] | None:
if not cursor:
return None
try:
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
date_str, id_str = raw.split("|", 1)
return dt.datetime.fromisoformat(date_str), UUID(id_str)
hit_str, date_str, id_str = raw.split("|", 2)
return int(hit_str), dt.datetime.fromisoformat(date_str), UUID(id_str)
except (ValueError, TypeError):
return None