search: email address search + address-first ranking; inline images; folder i18n; link security
Search (PG backend):
- '@' terms match address substrings (from/to/cc), so a partial email like
"dyrektor@wfdif" resolves and narrows predictably as you type
- index each address split into parts, so a bare domain/local word ("wfdif")
finds the address (indexed as one lexeme otherwise); dotted terms fall back to
an address substring
- rank sender/recipient name+address matches above body/subject-only ones
(addr_hit tier, then recency); cursor extended to keep keyset pagination correct
- scripts/reindex_address_parts.py backfills address tokens into existing rows
Inline images:
- extract attachments via a full MIME walk instead of shallow iter_attachments(),
which missed inline images nested in mixed>related>alternative (cid: rendered
broken); has_attachments counts only real (non-cid) attachments
- scripts/backfill_inline_attachments.py stores previously-missed inline images
Folder-name i18n (SPEC §12 M2):
- accent/punctuation-insensitive, multi-delimiter (/ \ .) special-use inference
across ~13 languages; curated names (no substring false positives)
Link security (reading panel):
- links never navigate the message iframe: a parent-side click interceptor opens
http/https in a new tab (noopener) and mailto in the composer (recipient
pre-filled); all other schemes blocked
Backend 292 tests green, ruff + mypy --strict clean; frontend tsc/eslint/vitest clean.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
@@ -54,7 +54,7 @@ def _build_search_doc(message: Message, parsed: ParsedEmail, special_use: str |
|
|||||||
cc=parsed.cc_addr,
|
cc=parsed.cc_addr,
|
||||||
subject=parsed.subject,
|
subject=parsed.subject,
|
||||||
body_text=parsed.text_body,
|
body_text=parsed.text_body,
|
||||||
attachment_names=[a.filename for a in parsed.attachments],
|
attachment_names=[a.filename for a in parsed.attachments if a.content_id is None],
|
||||||
has_attachments=parsed.has_attachments,
|
has_attachments=parsed.has_attachments,
|
||||||
is_unread="\\Seen" not in message.flags,
|
is_unread="\\Seen" not in message.flags,
|
||||||
is_starred="\\Flagged" in message.flags,
|
is_starred="\\Flagged" in message.flags,
|
||||||
|
|||||||
@@ -46,7 +46,9 @@ class ParsedEmail:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def has_attachments(self) -> bool:
|
def has_attachments(self) -> bool:
|
||||||
return len(self.attachments) > 0
|
# Inline resources (cid: images) carry a content_id and render in-body, not
|
||||||
|
# as attachment chips — only real attachments flip the paperclip.
|
||||||
|
return any(a.content_id is None for a in self.attachments)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def snippet(self) -> str:
|
def snippet(self) -> str:
|
||||||
@@ -85,17 +87,25 @@ def _get_body_content(msg: EmailMessage, subtype: str) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
|
def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
|
||||||
|
# Walk the whole tree, not iter_attachments(): the latter is shallow and treats
|
||||||
|
# multipart/related as a body, so an inline image nested in mixed>related>alt is
|
||||||
|
# never yielded (its cid: reference then renders broken). A part is an attachment
|
||||||
|
# or inline resource if it has a filename, a Content-ID, or attachment disposition;
|
||||||
|
# the body text/plain and text/html parts have none of these and are skipped.
|
||||||
out: list[ParsedAttachment] = []
|
out: list[ParsedAttachment] = []
|
||||||
for part in msg.iter_attachments():
|
for part in msg.walk():
|
||||||
if not isinstance(part, EmailMessage):
|
if part.is_multipart() or not isinstance(part, EmailMessage):
|
||||||
|
continue
|
||||||
|
cid = part.get("Content-ID")
|
||||||
|
filename = part.get_filename()
|
||||||
|
if not (filename or cid or part.get_content_disposition() == "attachment"):
|
||||||
continue
|
continue
|
||||||
payload = part.get_payload(decode=True)
|
payload = part.get_payload(decode=True)
|
||||||
if not isinstance(payload, bytes):
|
if not isinstance(payload, bytes):
|
||||||
continue
|
continue
|
||||||
cid = part.get("Content-ID")
|
|
||||||
out.append(
|
out.append(
|
||||||
ParsedAttachment(
|
ParsedAttachment(
|
||||||
filename=part.get_filename() or "attachment",
|
filename=filename or "attachment",
|
||||||
mime_type=part.get_content_type(),
|
mime_type=part.get_content_type(),
|
||||||
content_id=cid.strip("<>") if cid else None,
|
content_id=cid.strip("<>") if cid else None,
|
||||||
payload=payload,
|
payload=payload,
|
||||||
|
|||||||
+79
-14
@@ -15,7 +15,20 @@ import html
|
|||||||
import re
|
import re
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlalchemy import Text, any_, cast, exists, func, literal, not_, or_, select, tuple_, update
|
from sqlalchemy import (
|
||||||
|
Text,
|
||||||
|
any_,
|
||||||
|
case,
|
||||||
|
cast,
|
||||||
|
exists,
|
||||||
|
func,
|
||||||
|
literal,
|
||||||
|
not_,
|
||||||
|
or_,
|
||||||
|
select,
|
||||||
|
tuple_,
|
||||||
|
update,
|
||||||
|
)
|
||||||
from sqlalchemy.sql.elements import ColumnElement
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
|
|
||||||
from app.db import get_sessionmaker
|
from app.db import get_sessionmaker
|
||||||
@@ -42,12 +55,23 @@ def _prefix_tsquery(value: str) -> ColumnElement[object]:
|
|||||||
_RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
|
_RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
|
||||||
|
|
||||||
|
|
||||||
|
def _email_parts(email: str) -> str:
|
||||||
|
"""Split an address on @/./-/+/_ so its domain and local part are searchable
|
||||||
|
on their own — a bare "wfdif" should find dyrektor@wfdif.com.pl. Postgres
|
||||||
|
indexes the whole address as a single lexeme, so the parts are invisible to a
|
||||||
|
word search otherwise."""
|
||||||
|
return re.sub(r"[@._+\-]+", " ", email)
|
||||||
|
|
||||||
|
|
||||||
def _doc_text_blob(doc: SearchDoc) -> str:
|
def _doc_text_blob(doc: SearchDoc) -> str:
|
||||||
parts = [doc.subject, doc.body_text]
|
parts = [doc.subject, doc.body_text]
|
||||||
for field in (doc.from_, doc.to, doc.cc):
|
for field in (doc.from_, doc.to, doc.cc):
|
||||||
for addr in field:
|
for addr in field:
|
||||||
parts.append(addr.get("name", ""))
|
parts.append(addr.get("name", ""))
|
||||||
parts.append(addr.get("email", ""))
|
email = addr.get("email", "")
|
||||||
|
if email:
|
||||||
|
parts.append(email) # whole address (one lexeme, for exact/prefix)
|
||||||
|
parts.append(_email_parts(email)) # + its parts as separate tokens
|
||||||
parts.extend(doc.attachment_names)
|
parts.extend(doc.attachment_names)
|
||||||
return " ".join(p for p in parts if p)
|
return " ".join(p for p in parts if p)
|
||||||
|
|
||||||
@@ -114,7 +138,24 @@ class PgSearchBackend(SearchBackend):
|
|||||||
if clause.get("phrase"):
|
if clause.get("phrase"):
|
||||||
# Quoted phrase: keep exact-adjacency semantics, no prefix.
|
# Quoted phrase: keep exact-adjacency semantics, no prefix.
|
||||||
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
|
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
|
||||||
return Message.body_indexed.op("@@")(_prefix_tsquery(value))
|
like = f"%{value}%"
|
||||||
|
addr_match = or_(
|
||||||
|
cast(Message.from_addr, Text).ilike(like),
|
||||||
|
cast(Message.to_addr, Text).ilike(like),
|
||||||
|
cast(Message.cc_addr, Text).ilike(like),
|
||||||
|
)
|
||||||
|
if "@" in value:
|
||||||
|
# A term with '@' is an address search: match the address substring
|
||||||
|
# only (like from:/to:/cc:), so it narrows predictably as you type —
|
||||||
|
# not the split-word body matching that made a partial term over-hit.
|
||||||
|
return addr_match
|
||||||
|
fts = Message.body_indexed.op("@@")(_prefix_tsquery(value))
|
||||||
|
if "." in value:
|
||||||
|
# A dotted term (e.g. "wfdif.com") stays one host lexeme in FTS, so
|
||||||
|
# also match it as an address substring. Bare words don't need this:
|
||||||
|
# address parts are indexed as their own tokens (see _doc_text_blob).
|
||||||
|
return or_(fts, addr_match)
|
||||||
|
return fts
|
||||||
key = clause.get("key")
|
key = clause.get("key")
|
||||||
cond = self._filter_condition(str(key), value)
|
cond = self._filter_condition(str(key), value)
|
||||||
return cond
|
return cond
|
||||||
@@ -189,7 +230,11 @@ class PgSearchBackend(SearchBackend):
|
|||||||
groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
|
groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
|
||||||
assert isinstance(groups, list)
|
assert isinstance(groups, list)
|
||||||
conditions = self._build_conditions(groups)
|
conditions = self._build_conditions(groups)
|
||||||
|
terms = _positive_text_terms(groups)
|
||||||
|
|
||||||
|
# Rank a match in a sender/recipient name or address above a body/subject-only
|
||||||
|
# match, so "who" results come first; ties break by recency.
|
||||||
|
addr_hit = _address_hit_expr(terms)
|
||||||
rank = (
|
rank = (
|
||||||
func.ts_rank_cd(Message.body_indexed, _prefix_tsquery(q.text))
|
func.ts_rank_cd(Message.body_indexed, _prefix_tsquery(q.text))
|
||||||
if q.text
|
if q.text
|
||||||
@@ -203,6 +248,7 @@ class PgSearchBackend(SearchBackend):
|
|||||||
Message.snippet,
|
Message.snippet,
|
||||||
Message.internal_date,
|
Message.internal_date,
|
||||||
rank.label("score"),
|
rank.label("score"),
|
||||||
|
addr_hit.label("addr_hit"),
|
||||||
)
|
)
|
||||||
.select_from(Message)
|
.select_from(Message)
|
||||||
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||||||
@@ -213,14 +259,16 @@ class PgSearchBackend(SearchBackend):
|
|||||||
|
|
||||||
decoded = _decode_cursor(cursor)
|
decoded = _decode_cursor(cursor)
|
||||||
if decoded is not None:
|
if decoded is not None:
|
||||||
cur_date, cur_id = decoded
|
cur_hit, cur_date, cur_id = decoded
|
||||||
|
# Keyset over the full ordering (addr_hit, date, id), all descending.
|
||||||
stmt = stmt.where(
|
stmt = stmt.where(
|
||||||
tuple_(Message.internal_date, Message.id)
|
tuple_(addr_hit, Message.internal_date, Message.id)
|
||||||
< tuple_(literal(cur_date), literal(cur_id))
|
< tuple_(literal(cur_hit), literal(cur_date), literal(cur_id))
|
||||||
)
|
)
|
||||||
stmt = stmt.order_by(Message.internal_date.desc(), Message.id.desc()).limit(limit + 1)
|
stmt = stmt.order_by(
|
||||||
|
addr_hit.desc(), Message.internal_date.desc(), Message.id.desc()
|
||||||
|
).limit(limit + 1)
|
||||||
|
|
||||||
terms = _positive_text_terms(groups)
|
|
||||||
async with get_sessionmaker()() as session:
|
async with get_sessionmaker()() as session:
|
||||||
rows = (await session.execute(stmt)).all()
|
rows = (await session.execute(stmt)).all()
|
||||||
|
|
||||||
@@ -236,7 +284,9 @@ class PgSearchBackend(SearchBackend):
|
|||||||
for row in rows
|
for row in rows
|
||||||
]
|
]
|
||||||
next_cursor = (
|
next_cursor = (
|
||||||
_encode_cursor(rows[-1].internal_date, rows[-1].id) if has_more and rows else None
|
_encode_cursor(int(rows[-1].addr_hit), rows[-1].internal_date, rows[-1].id)
|
||||||
|
if has_more and rows
|
||||||
|
else None
|
||||||
)
|
)
|
||||||
return hits, next_cursor
|
return hits, next_cursor
|
||||||
|
|
||||||
@@ -250,6 +300,21 @@ def _positive_text_terms(groups: list[list[dict[str, object]]]) -> list[str]:
|
|||||||
return terms
|
return terms
|
||||||
|
|
||||||
|
|
||||||
|
def _address_hit_expr(terms: list[str]) -> ColumnElement[int]:
|
||||||
|
"""1 if any positive free-text term appears in a from/to/cc name or address,
|
||||||
|
else 0. Used to sort address/name matches ahead of body/subject-only ones."""
|
||||||
|
conds: list[ColumnElement[bool]] = []
|
||||||
|
for t in terms:
|
||||||
|
if t:
|
||||||
|
like = f"%{t}%"
|
||||||
|
conds.append(cast(Message.from_addr, Text).ilike(like))
|
||||||
|
conds.append(cast(Message.to_addr, Text).ilike(like))
|
||||||
|
conds.append(cast(Message.cc_addr, Text).ilike(like))
|
||||||
|
if not conds:
|
||||||
|
return literal(0)
|
||||||
|
return case((or_(*conds), 1), else_=0)
|
||||||
|
|
||||||
|
|
||||||
def _highlights(subject: str, snippet: str, terms: list[str]) -> dict[str, str]:
|
def _highlights(subject: str, snippet: str, terms: list[str]) -> dict[str, str]:
|
||||||
out: dict[str, str] = {}
|
out: dict[str, str] = {}
|
||||||
if not terms:
|
if not terms:
|
||||||
@@ -274,17 +339,17 @@ def _mark(value: str, terms: list[str]) -> str:
|
|||||||
return escaped
|
return escaped
|
||||||
|
|
||||||
|
|
||||||
def _encode_cursor(date: dt.datetime, message_id: UUID) -> str:
|
def _encode_cursor(addr_hit: int, date: dt.datetime, message_id: UUID) -> str:
|
||||||
raw = f"{date.isoformat()}|{message_id}"
|
raw = f"{addr_hit}|{date.isoformat()}|{message_id}"
|
||||||
return base64.urlsafe_b64encode(raw.encode()).decode()
|
return base64.urlsafe_b64encode(raw.encode()).decode()
|
||||||
|
|
||||||
|
|
||||||
def _decode_cursor(cursor: str | None) -> tuple[dt.datetime, UUID] | None:
|
def _decode_cursor(cursor: str | None) -> tuple[int, dt.datetime, UUID] | None:
|
||||||
if not cursor:
|
if not cursor:
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
||||||
date_str, id_str = raw.split("|", 1)
|
hit_str, date_str, id_str = raw.split("|", 2)
|
||||||
return dt.datetime.fromisoformat(date_str), UUID(id_str)
|
return int(hit_str), dt.datetime.fromisoformat(date_str), UUID(id_str)
|
||||||
except (ValueError, TypeError):
|
except (ValueError, TypeError):
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -1,48 +1,120 @@
|
|||||||
"""Special-use inference from folder names (SPEC §6.3 fallback heuristics).
|
"""Special-use inference from folder names (SPEC §6.3 / §12 M2 fallback).
|
||||||
|
|
||||||
Used only when the server does not advertise SPECIAL-USE. Includes Polish names
|
Used only when the server does not advertise the SPECIAL-USE extension. Matching
|
||||||
(Wysłane, Kosz, ...) alongside English/German, per the spec's i18n intent.
|
is accent- and punctuation-insensitive (so "Wysłane", "Éléments envoyés" and
|
||||||
|
"Junk E-mail" all resolve), covers the hierarchy leaf under any of the common
|
||||||
|
IMAP delimiters (``/ \\ .``), and recognises provider prefixes like
|
||||||
|
``[Gmail]/`` implicitly via the leaf. Names are curated per language rather than
|
||||||
|
substring-matched, so a folder like "Sent to John" is *not* misfiled as Sent.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
# Latin letters NFKD doesn't decompose to ASCII; fold them explicitly.
|
||||||
|
_LATIN_SPECIAL = str.maketrans(
|
||||||
|
{"ł": "l", "ø": "o", "đ": "d", "ð": "d", "ß": "ss", "æ": "ae", "œ": "oe", "ı": "i", "þ": "th"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _compact(name: str) -> str:
|
||||||
|
"""Fold to a comparable key: lowercase, strip accents/diacritics, drop every
|
||||||
|
non-alphanumeric char. "Éléments envoyés" -> "elementsenvoyes"."""
|
||||||
|
folded = unicodedata.normalize("NFKD", name.strip().lower()).translate(_LATIN_SPECIAL)
|
||||||
|
return "".join(ch for ch in folded if ch.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
# Localized folder names by special_use. Written readably; compacted at load.
|
||||||
|
_NAMES: dict[str, list[str]] = {
|
||||||
|
"sent": [
|
||||||
|
"sent", "sent items", "sent mail", "sent messages", # English / Outlook / Gmail
|
||||||
|
"wysłane", "elementy wysłane", "wiadomości wysłane", # Polish
|
||||||
|
"gesendet", "gesendete objekte", "gesendete elemente", # German
|
||||||
|
"envoyés", "éléments envoyés", "messages envoyés", "courrier envoyé", # French
|
||||||
|
"enviados", "elementos enviados", "correo enviado", # Spanish
|
||||||
|
"inviata", "posta inviata", "messaggi inviati", # Italian
|
||||||
|
"verzonden", "verzonden items", "verzonden berichten", # Dutch
|
||||||
|
"enviadas", "itens enviados", "mensagens enviadas", # Portuguese
|
||||||
|
"отправленные", "исходящие", # Russian
|
||||||
|
"skickat", "skickade", "skickade objekt", # Swedish
|
||||||
|
"sendt", "sendte", "sendte elementer", # Norwegian / Danish
|
||||||
|
"lähetetyt", # Finnish
|
||||||
|
"odeslané", "odeslaná pošta", # Czech
|
||||||
|
],
|
||||||
|
"drafts": [
|
||||||
|
"drafts", "draft",
|
||||||
|
"robocze", "kopie robocze", "wersje robocze",
|
||||||
|
"entwürfe", "entwurf",
|
||||||
|
"brouillons",
|
||||||
|
"borradores",
|
||||||
|
"bozze",
|
||||||
|
"concepten",
|
||||||
|
"rascunhos",
|
||||||
|
"черновики",
|
||||||
|
"utkast",
|
||||||
|
"kladder", # Danish
|
||||||
|
"luonnokset", # Finnish
|
||||||
|
"koncepty", # Czech
|
||||||
|
],
|
||||||
|
"trash": [
|
||||||
|
"trash", "deleted", "deleted items", "deleted messages", "bin", "recycle bin",
|
||||||
|
"kosz",
|
||||||
|
"papierkorb", "gelöschte objekte", "gelöschte elemente",
|
||||||
|
"corbeille", "éléments supprimés",
|
||||||
|
"papelera", "elementos eliminados",
|
||||||
|
"cestino", "posta eliminata",
|
||||||
|
"prullenbak", "verwijderde items",
|
||||||
|
"lixeira", "lixo", "itens excluídos",
|
||||||
|
"корзина", "удаленные",
|
||||||
|
"papperskorg", "borttaget",
|
||||||
|
"papirkurv", "slettet",
|
||||||
|
"roskakori", "poistetut",
|
||||||
|
"koš", # Czech
|
||||||
|
],
|
||||||
|
"junk": [
|
||||||
|
"junk", "junk email", "junk e-mail", "spam", "bulk mail", "bulk",
|
||||||
|
"wiadomości-śmieci", "niechciane",
|
||||||
|
"unerwünscht", "werbung",
|
||||||
|
"courrier indésirable", "indésirables", "pourriel",
|
||||||
|
"correo no deseado", "no deseado", "correo basura",
|
||||||
|
"posta indesiderata", "indesiderata",
|
||||||
|
"ongewenst", "ongewenste e-mail",
|
||||||
|
"lixo eletrônico",
|
||||||
|
"спам", "нежелательная почта",
|
||||||
|
"skräp", "skräppost",
|
||||||
|
"uønsket",
|
||||||
|
"roskaposti",
|
||||||
|
"nevyžádaná pošta",
|
||||||
|
],
|
||||||
|
"archive": [
|
||||||
|
"archive", "archives", "all mail",
|
||||||
|
"archiwum",
|
||||||
|
"archiv",
|
||||||
|
"archivio",
|
||||||
|
"archivo",
|
||||||
|
"archief",
|
||||||
|
"arquivo", "arquivo morto",
|
||||||
|
"архив", "вся почта",
|
||||||
|
"arkiv",
|
||||||
|
"arkisto",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
_NAME_MAP: dict[str, str] = {
|
_NAME_MAP: dict[str, str] = {
|
||||||
# sent
|
_compact(name): special for special, names in _NAMES.items() for name in names
|
||||||
"sent": "sent",
|
|
||||||
"sent items": "sent",
|
|
||||||
"sent mail": "sent",
|
|
||||||
"wysłane": "sent",
|
|
||||||
"wyslane": "sent",
|
|
||||||
"elementy wysłane": "sent",
|
|
||||||
"gesendet": "sent",
|
|
||||||
# drafts
|
|
||||||
"drafts": "drafts",
|
|
||||||
"draft": "drafts",
|
|
||||||
"robocze": "drafts",
|
|
||||||
"kopie robocze": "drafts",
|
|
||||||
"entwürfe": "drafts",
|
|
||||||
# trash
|
|
||||||
"trash": "trash",
|
|
||||||
"deleted": "trash",
|
|
||||||
"deleted items": "trash",
|
|
||||||
"kosz": "trash",
|
|
||||||
"papierkorb": "trash",
|
|
||||||
# junk
|
|
||||||
"junk": "junk",
|
|
||||||
"spam": "junk",
|
|
||||||
"bulk mail": "junk",
|
|
||||||
"wiadomości-śmieci": "junk",
|
|
||||||
# archive
|
|
||||||
"archive": "archive",
|
|
||||||
"archiwum": "archive",
|
|
||||||
"all mail": "archive",
|
|
||||||
"archiv": "archive",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def infer_special_use(path: str) -> str | None:
|
def infer_special_use(path: str) -> str | None:
|
||||||
"""Infer a special_use value from a mailbox path/name (case-insensitive)."""
|
"""Infer a special_use value from a mailbox path/name (accent-insensitive)."""
|
||||||
if path.upper() == "INBOX":
|
if path.strip().upper().rstrip("/.") == "INBOX":
|
||||||
return "inbox"
|
return "inbox"
|
||||||
leaf = path.replace("\\", "/").split("/")[-1].strip().lower()
|
# Take the hierarchy leaf under any common IMAP delimiter (/ \ or . for
|
||||||
return _NAME_MAP.get(leaf) or _NAME_MAP.get(path.strip().lower())
|
# Dovecot/Courier namespaces), then fold for lookup.
|
||||||
|
normalized = path
|
||||||
|
for delim in ("/", "\\", "."):
|
||||||
|
normalized = normalized.replace(delim, "\n")
|
||||||
|
segments = [s for s in normalized.split("\n") if s.strip()]
|
||||||
|
tail = segments[-1] if segments else path
|
||||||
|
return _NAME_MAP.get(_compact(tail)) or _NAME_MAP.get(_compact(path))
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
"""One-off: store inline (cid:) images that the old shallow attachment extractor
|
||||||
|
missed for messages nested as mixed>related>alternative, so their cid: references
|
||||||
|
resolve instead of rendering broken. Idempotent — skips content_ids already stored.
|
||||||
|
|
||||||
|
Run: python -m scripts.backfill_inline_attachments
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.db import get_sessionmaker
|
||||||
|
from app.indexer import blobstore
|
||||||
|
from app.indexer.mime import parse_message
|
||||||
|
from app.models import Attachment, Message
|
||||||
|
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
sm = get_sessionmaker()
|
||||||
|
added = 0
|
||||||
|
fixed_msgs = 0
|
||||||
|
async with sm() as session:
|
||||||
|
msgs = list(await session.scalars(select(Message)))
|
||||||
|
for msg in msgs:
|
||||||
|
try:
|
||||||
|
raw = blobstore.load(msg.user_id, msg.blob_sha256)
|
||||||
|
except FileNotFoundError:
|
||||||
|
continue
|
||||||
|
parsed = parse_message(raw)
|
||||||
|
inline = [a for a in parsed.attachments if a.content_id]
|
||||||
|
if not inline:
|
||||||
|
continue
|
||||||
|
have = set(
|
||||||
|
await session.scalars(
|
||||||
|
select(Attachment.content_id).where(
|
||||||
|
Attachment.message_id == msg.id, Attachment.content_id.is_not(None)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
new = 0
|
||||||
|
for att in inline:
|
||||||
|
if att.content_id in have:
|
||||||
|
continue
|
||||||
|
att_sha = blobstore.store(msg.user_id, att.payload)
|
||||||
|
session.add(
|
||||||
|
Attachment(
|
||||||
|
message_id=msg.id,
|
||||||
|
user_id=msg.user_id,
|
||||||
|
filename=att.filename,
|
||||||
|
mime_type=att.mime_type,
|
||||||
|
size_bytes=len(att.payload),
|
||||||
|
content_id=att.content_id,
|
||||||
|
blob_sha256=att_sha,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
new += 1
|
||||||
|
if new:
|
||||||
|
added += new
|
||||||
|
fixed_msgs += 1
|
||||||
|
await session.commit()
|
||||||
|
print(f"stored {added} missing inline images across {fixed_msgs} messages")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""One-off: add split-address tokens to existing messages' FTS vector.
|
||||||
|
|
||||||
|
New messages get address parts indexed via `_doc_text_blob` (so a bare "wfdif"
|
||||||
|
finds dyrektor@wfdif.com.pl). This backfills that into already-indexed rows by
|
||||||
|
appending the split-address lexemes to `body_indexed`. Idempotent — tsvector
|
||||||
|
concat merges duplicate lexemes, so re-running is harmless.
|
||||||
|
|
||||||
|
Run: python -m scripts.reindex_address_parts
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from sqlalchemy import func, select, update
|
||||||
|
|
||||||
|
from app.db import get_sessionmaker
|
||||||
|
from app.models import Message
|
||||||
|
from app.search.pg import _TS_CONFIG, _email_parts
|
||||||
|
|
||||||
|
|
||||||
|
def _address_parts_blob(msg: Message) -> str:
|
||||||
|
parts: list[str] = []
|
||||||
|
for field in (msg.from_addr, msg.to_addr, msg.cc_addr):
|
||||||
|
for addr in field:
|
||||||
|
email = addr.get("email", "")
|
||||||
|
if email:
|
||||||
|
parts.append(_email_parts(email))
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
sm = get_sessionmaker()
|
||||||
|
updated = 0
|
||||||
|
async with sm() as session:
|
||||||
|
msgs = list(await session.scalars(select(Message).where(Message.body_indexed.is_not(None))))
|
||||||
|
for msg in msgs:
|
||||||
|
blob = _address_parts_blob(msg)
|
||||||
|
if not blob:
|
||||||
|
continue
|
||||||
|
await session.execute(
|
||||||
|
update(Message)
|
||||||
|
.where(Message.id == msg.id)
|
||||||
|
.values(
|
||||||
|
body_indexed=Message.body_indexed.op("||")(
|
||||||
|
func.to_tsvector(_TS_CONFIG, blob)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
updated += 1
|
||||||
|
await session.commit()
|
||||||
|
print(f"reindexed address parts into {updated} messages")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -93,6 +93,39 @@ def test_attachment_extracted() -> None:
|
|||||||
assert parsed.attachments[0].payload == b"\x00\x01\x02data"
|
assert parsed.attachments[0].payload == b"\x00\x01\x02data"
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_image_nested_in_related_is_extracted() -> None:
|
||||||
|
# Real-world structure: mixed > related > alternative(text, html) + inline image.
|
||||||
|
# iter_attachments() misses the nested image; a full walk must find it so the
|
||||||
|
# cid: reference resolves (regression: broken inline image).
|
||||||
|
from email.mime.image import MIMEImage
|
||||||
|
from email.mime.multipart import MIMEMultipart
|
||||||
|
from email.mime.text import MIMEText
|
||||||
|
|
||||||
|
root = MIMEMultipart("mixed")
|
||||||
|
root["Subject"] = "newsletter"
|
||||||
|
root["From"] = "biuro@kipa.pl"
|
||||||
|
root["To"] = "me@example.com"
|
||||||
|
root["Message-ID"] = "<n@x.com>"
|
||||||
|
related = MIMEMultipart("related")
|
||||||
|
alt = MIMEMultipart("alternative")
|
||||||
|
alt.attach(MIMEText("text version", "plain"))
|
||||||
|
alt.attach(MIMEText('<p>Hi <img src="cid:img001"></p>', "html"))
|
||||||
|
related.attach(alt)
|
||||||
|
img = MIMEImage(b"\x89PNG\r\n\x1a\ndata", "png")
|
||||||
|
img.add_header("Content-ID", "<img001>")
|
||||||
|
img.add_header("Content-Disposition", "inline", filename="image001.png")
|
||||||
|
related.attach(img)
|
||||||
|
root.attach(related)
|
||||||
|
|
||||||
|
parsed = parse_message(root.as_bytes())
|
||||||
|
assert len(parsed.attachments) == 1
|
||||||
|
att = parsed.attachments[0]
|
||||||
|
assert att.content_id == "img001"
|
||||||
|
assert att.mime_type == "image/png"
|
||||||
|
# Inline-only: no attachment chip / paperclip.
|
||||||
|
assert parsed.has_attachments is False
|
||||||
|
|
||||||
|
|
||||||
def test_missing_message_id_returns_none() -> None:
|
def test_missing_message_id_returns_none() -> None:
|
||||||
msg = EmailMessage()
|
msg = EmailMessage()
|
||||||
msg["Subject"] = "no id"
|
msg["Subject"] = "no id"
|
||||||
|
|||||||
@@ -168,6 +168,36 @@ async def test_free_text_full_email_address(seeded: Ctx) -> None:
|
|||||||
assert await _search(seeded, "bob@corp.co") == 1 # prefix while still typing
|
assert await _search(seeded, "bob@corp.co") == 1 # prefix while still typing
|
||||||
|
|
||||||
|
|
||||||
|
async def test_free_text_email_matches_address_consistently(seeded: Ctx) -> None:
|
||||||
|
# A term with '@' is treated as an address search and matches as a substring
|
||||||
|
# of from/to/cc, so it narrows *predictably* as you type more of the address
|
||||||
|
# (a partial term never returns more than the fuller one).
|
||||||
|
assert await _search(seeded, "bob@") == 1 # local part -> bob only
|
||||||
|
assert await _search(seeded, "bob@corp") == 1
|
||||||
|
assert await _search(seeded, "bob@co") == 1
|
||||||
|
assert await _search(seeded, "@corp.com") == 3 # all three share the domain
|
||||||
|
assert await _search(seeded, "zzz@corp") == 0 # no such sender
|
||||||
|
|
||||||
|
|
||||||
|
async def test_address_matches_rank_before_body_matches(ctx: Ctx) -> None:
|
||||||
|
# Older message matches "zebra" only via its sender address; the newer one
|
||||||
|
# matches only in the body. The address/name hit must still come first.
|
||||||
|
await _ingest(ctx, frm="Zebra Corp <hello@zebra.com>", subject="hi", body="nothing here")
|
||||||
|
await _ingest(ctx, frm="Someone <someone@corp.com>", subject="a note", body="the zebra ran")
|
||||||
|
resp = await ctx.client.get("/api/search", params={"q": "zebra"})
|
||||||
|
subjects = [t["thread"]["subject"] for t in resp.json()["threads"]]
|
||||||
|
assert subjects == ["hi", "a note"] # address hit first, then body-only
|
||||||
|
|
||||||
|
|
||||||
|
async def test_free_text_bare_domain_word_matches_address(seeded: Ctx) -> None:
|
||||||
|
# A bare domain/local word finds the address even though Postgres indexes the
|
||||||
|
# whole address as one lexeme — address parts are tokenized at index time.
|
||||||
|
assert await _search(seeded, "corp") == 3 # the domain word -> all three
|
||||||
|
assert await _search(seeded, "carol") == 1 # local part -> carol only
|
||||||
|
# A dotted domain term also matches via the address-substring fallback.
|
||||||
|
assert await _search(seeded, "corp.com") == 3
|
||||||
|
|
||||||
|
|
||||||
async def test_from_operator(seeded: Ctx) -> None:
|
async def test_from_operator(seeded: Ctx) -> None:
|
||||||
assert await _search(seeded, "from:bob") == 1
|
assert await _search(seeded, "from:bob") == 1
|
||||||
assert await _search(seeded, "from:corp.com") == 3 # all three share the domain
|
assert await _search(seeded, "from:corp.com") == 3 # all three share the domain
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Folder-name special-use heuristics (SPEC §6.3)."""
|
"""Folder-name special-use heuristics (SPEC §6.3 / §12 M2)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -10,24 +10,64 @@ from app.sync.folder_names import infer_special_use
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"path,expected",
|
"path,expected",
|
||||||
[
|
[
|
||||||
|
# inbox
|
||||||
("INBOX", "inbox"),
|
("INBOX", "inbox"),
|
||||||
("inbox", "inbox"),
|
("inbox", "inbox"),
|
||||||
|
("INBOX.", "inbox"), # Dovecot may show a trailing delimiter
|
||||||
|
# English / Outlook / Gmail
|
||||||
("Sent", "sent"),
|
("Sent", "sent"),
|
||||||
("Wysłane", "sent"), # Polish
|
("Sent Items", "sent"),
|
||||||
("Elementy wysłane", "sent"),
|
("[Gmail]/Sent Mail", "sent"),
|
||||||
("Gesendet", "sent"), # German
|
|
||||||
("Drafts", "drafts"),
|
("Drafts", "drafts"),
|
||||||
("Robocze", "drafts"),
|
("Deleted Items", "trash"),
|
||||||
("Trash", "trash"),
|
("Bin", "trash"),
|
||||||
("Kosz", "trash"), # Polish
|
|
||||||
("Papierkorb", "trash"), # German
|
|
||||||
("Spam", "junk"),
|
|
||||||
("Junk", "junk"),
|
("Junk", "junk"),
|
||||||
|
("Junk E-mail", "junk"), # Outlook, punctuation-insensitive
|
||||||
|
("[Gmail]/Spam", "junk"),
|
||||||
("Archive", "archive"),
|
("Archive", "archive"),
|
||||||
|
("[Gmail]/All Mail", "archive"),
|
||||||
|
# Polish (accents)
|
||||||
|
("Wysłane", "sent"),
|
||||||
|
("Elementy wysłane", "sent"),
|
||||||
|
("Robocze", "drafts"),
|
||||||
|
("Kosz", "trash"),
|
||||||
|
("Wiadomości-śmieci", "junk"),
|
||||||
("Archiwum", "archive"),
|
("Archiwum", "archive"),
|
||||||
("INBOX/Sent", "sent"), # nested leaf
|
# German
|
||||||
|
("Gesendet", "sent"),
|
||||||
|
("Gesendete Elemente", "sent"),
|
||||||
|
("Entwürfe", "drafts"),
|
||||||
|
("Papierkorb", "trash"),
|
||||||
|
("Gelöschte Elemente", "trash"),
|
||||||
|
# French
|
||||||
|
("Éléments envoyés", "sent"),
|
||||||
|
("Brouillons", "drafts"),
|
||||||
|
("Corbeille", "trash"),
|
||||||
|
("Courrier indésirable", "junk"),
|
||||||
|
# Spanish / Italian / Dutch / Portuguese
|
||||||
|
("Enviados", "sent"),
|
||||||
|
("Papelera", "trash"),
|
||||||
|
("Posta inviata", "sent"),
|
||||||
|
("Cestino", "trash"),
|
||||||
|
("Verzonden", "sent"),
|
||||||
|
("Prullenbak", "trash"),
|
||||||
|
("Itens enviados", "sent"),
|
||||||
|
("Lixeira", "trash"),
|
||||||
|
# Russian (Cyrillic)
|
||||||
|
("Отправленные", "sent"),
|
||||||
|
("Корзина", "trash"),
|
||||||
|
("Черновики", "drafts"),
|
||||||
|
("Спам", "junk"),
|
||||||
|
# delimiters / nesting
|
||||||
|
("INBOX/Sent", "sent"),
|
||||||
|
("INBOX.Sent", "sent"), # Dovecot dot delimiter
|
||||||
|
("INBOX.Trash", "trash"),
|
||||||
|
# negatives — must not misfire
|
||||||
("Work", None),
|
("Work", None),
|
||||||
("Random Folder", None),
|
("Random Folder", None),
|
||||||
|
("Sent to John", None), # curated names, not substring match
|
||||||
|
("Newsletter", None),
|
||||||
|
("Projects/Client", None),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_infer_special_use(path: str, expected: str | None) -> None:
|
def test_infer_special_use(path: str, expected: str | None) -> None:
|
||||||
|
|||||||
@@ -12,7 +12,13 @@ import { Attachments } from "./Attachments";
|
|||||||
// measure the content to auto-size the frame — so the message shows in full and
|
// measure the content to auto-size the frame — so the message shows in full and
|
||||||
// only the thread panel scrolls — and (b) send the session cookie so inline
|
// only the thread panel scrolls — and (b) send the session cookie so inline
|
||||||
// (cid:) images authenticate and load.
|
// (cid:) images authenticate and load.
|
||||||
export function MessageBody({ message }: { message: MessageEnvelope }) {
|
export function MessageBody({
|
||||||
|
message,
|
||||||
|
onMailto,
|
||||||
|
}: {
|
||||||
|
message: MessageEnvelope;
|
||||||
|
onMailto?: (email: string) => void;
|
||||||
|
}) {
|
||||||
const t = useT();
|
const t = useT();
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
// Seed from the server's trusted-sender flag so an already-approved sender
|
// Seed from the server's trusted-sender flag so an already-approved sender
|
||||||
@@ -20,6 +26,8 @@ export function MessageBody({ message }: { message: MessageEnvelope }) {
|
|||||||
const [allowed, setAllowed] = useState(message.images_allowed);
|
const [allowed, setAllowed] = useState(message.images_allowed);
|
||||||
const frameRef = useRef<HTMLIFrameElement>(null);
|
const frameRef = useRef<HTMLIFrameElement>(null);
|
||||||
const observerRef = useRef<ResizeObserver | null>(null);
|
const observerRef = useRef<ResizeObserver | null>(null);
|
||||||
|
const onMailtoRef = useRef(onMailto);
|
||||||
|
onMailtoRef.current = onMailto;
|
||||||
const [height, setHeight] = useState(80);
|
const [height, setHeight] = useState(80);
|
||||||
|
|
||||||
const { data, isLoading, isError } = useQuery({
|
const { data, isLoading, isError } = useQuery({
|
||||||
@@ -73,11 +81,31 @@ export function MessageBody({ message }: { message: MessageEnvelope }) {
|
|||||||
[resize],
|
[resize],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Links must NEVER navigate the message iframe (security: a click would replace
|
||||||
|
// the message body, and could be used for clickjacking/phishing). The frame is
|
||||||
|
// same-origin, so the parent intercepts clicks here: http/https open in a new
|
||||||
|
// tab (noopener), mailto opens the composer, and every other scheme is blocked.
|
||||||
|
const handleFrameClick = useCallback((e: Event) => {
|
||||||
|
const el = e.target as Element | null;
|
||||||
|
const anchor = el?.closest?.("a[href]") as HTMLAnchorElement | null;
|
||||||
|
if (!anchor) return;
|
||||||
|
const raw = (anchor.getAttribute("href") || "").trim();
|
||||||
|
e.preventDefault();
|
||||||
|
if (/^mailto:/i.test(raw)) {
|
||||||
|
const email = decodeURIComponent(raw.replace(/^mailto:/i, "").split("?")[0]).trim();
|
||||||
|
if (email) onMailtoRef.current?.(email);
|
||||||
|
} else if (/^https?:\/\//i.test(raw)) {
|
||||||
|
window.open(anchor.href, "_blank", "noopener,noreferrer");
|
||||||
|
}
|
||||||
|
// else: relative / cid: / javascript: / data: / tel: — blocked, no navigation.
|
||||||
|
}, []);
|
||||||
|
|
||||||
const onLoad = () => {
|
const onLoad = () => {
|
||||||
const doc = frameRef.current?.contentDocument;
|
const doc = frameRef.current?.contentDocument;
|
||||||
if (!doc) return;
|
if (!doc) return;
|
||||||
injectQuoteToggle(doc);
|
injectQuoteToggle(doc);
|
||||||
resize();
|
resize();
|
||||||
|
doc.addEventListener("click", handleFrameClick);
|
||||||
// Re-measure as images finish loading (they change the content height).
|
// Re-measure as images finish loading (they change the content height).
|
||||||
doc.querySelectorAll("img").forEach((img) => {
|
doc.querySelectorAll("img").forEach((img) => {
|
||||||
if (!img.complete) img.addEventListener("load", resize, { once: true });
|
if (!img.complete) img.addEventListener("load", resize, { once: true });
|
||||||
|
|||||||
@@ -218,6 +218,7 @@ function MessageItem({
|
|||||||
alsoIn,
|
alsoIn,
|
||||||
onToggle,
|
onToggle,
|
||||||
onReply,
|
onReply,
|
||||||
|
onMailto,
|
||||||
onToggleRead,
|
onToggleRead,
|
||||||
onPrint,
|
onPrint,
|
||||||
onDelete,
|
onDelete,
|
||||||
@@ -235,6 +236,7 @@ function MessageItem({
|
|||||||
alsoIn: string[];
|
alsoIn: string[];
|
||||||
onToggle: () => void;
|
onToggle: () => void;
|
||||||
onReply: (kind: string) => void;
|
onReply: (kind: string) => void;
|
||||||
|
onMailto: (email: string) => void;
|
||||||
onToggleRead: () => void;
|
onToggleRead: () => void;
|
||||||
onPrint: () => void;
|
onPrint: () => void;
|
||||||
onDelete: () => void;
|
onDelete: () => void;
|
||||||
@@ -385,7 +387,7 @@ function MessageItem({
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="px-4 pb-3">
|
<div className="px-4 pb-3">
|
||||||
<MessageBody message={message} />
|
<MessageBody message={message} onMailto={onMailto} />
|
||||||
</div>
|
</div>
|
||||||
{showReplyBar && canReply && (
|
{showReplyBar && canReply && (
|
||||||
<div className="flex gap-2 border-t border-app-border px-4 py-3">
|
<div className="flex gap-2 border-t border-app-border px-4 py-3">
|
||||||
@@ -479,6 +481,16 @@ export function ThreadView({
|
|||||||
setComposing({ anchorId: msg.id, init: buildReplyInit(subject, msg, kind) });
|
setComposing({ anchorId: msg.id, init: buildReplyInit(subject, msg, kind) });
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// A mailto: link in the message body opens a fresh composer to that address,
|
||||||
|
// sent from the account this message belongs to.
|
||||||
|
const openMailto = (msg: MessageEnvelope, email: string) => {
|
||||||
|
setExpanded((prev) => new Set(prev).add(msg.id));
|
||||||
|
setComposing({
|
||||||
|
anchorId: msg.id,
|
||||||
|
init: { accountId: msg.account_id, kind: "new", to: email, subject: "" },
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (replyKindSignal && newest) openReply(newest, replyKindSignal.kind);
|
if (replyKindSignal && newest) openReply(newest, replyKindSignal.kind);
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
@@ -583,6 +595,7 @@ export function ThreadView({
|
|||||||
alsoIn={alsoIn}
|
alsoIn={alsoIn}
|
||||||
onToggle={() => toggle(message.id)}
|
onToggle={() => toggle(message.id)}
|
||||||
onReply={(kind) => openReply(message, kind)}
|
onReply={(kind) => openReply(message, kind)}
|
||||||
|
onMailto={(email) => openMailto(message, email)}
|
||||||
onToggleRead={() => toggleRead(message)}
|
onToggleRead={() => toggleRead(message)}
|
||||||
onPrint={() => void printMessages(subject, [message])}
|
onPrint={() => void printMessages(subject, [message])}
|
||||||
onDelete={() => deleteMessage(message)}
|
onDelete={() => deleteMessage(message)}
|
||||||
|
|||||||
Reference in New Issue
Block a user