search: email address search + address-first ranking; inline images; folder i18n; link security
Search (PG backend):
- '@' terms match address substrings (from/to/cc), so a partial email like
"dyrektor@wfdif" resolves and narrows predictably as you type
- index each address split into parts, so a bare domain/local word ("wfdif")
finds the address (indexed as one lexeme otherwise); dotted terms fall back to
an address substring
- rank sender/recipient name+address matches above body/subject-only ones
(addr_hit tier, then recency); cursor extended to keep keyset pagination correct
- scripts/reindex_address_parts.py backfills address tokens into existing rows
Inline images:
- extract attachments via a full MIME walk instead of shallow iter_attachments(),
which missed inline images nested in mixed>related>alternative (cid: rendered
broken); has_attachments counts only real (non-cid) attachments
- scripts/backfill_inline_attachments.py stores previously-missed inline images
Folder-name i18n (SPEC §12 M2):
- accent/punctuation-insensitive, multi-delimiter (/ \ .) special-use inference
across ~13 languages; curated names (no substring false positives)
Link security (reading panel):
- links never navigate the message iframe: a parent-side click interceptor opens
http/https in a new tab (noopener) and mailto in the composer (recipient
pre-filled); all other schemes blocked
Backend 292 tests green, ruff + mypy --strict clean; frontend tsc/eslint/vitest clean.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
"""One-off: store inline (cid:) images that the old shallow attachment extractor
|
||||
missed for messages nested as mixed>related>alternative, so their cid: references
|
||||
resolve instead of rendering broken. Idempotent — skips content_ids already stored.
|
||||
|
||||
Run: python -m scripts.backfill_inline_attachments
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import get_sessionmaker
|
||||
from app.indexer import blobstore
|
||||
from app.indexer.mime import parse_message
|
||||
from app.models import Attachment, Message
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
sm = get_sessionmaker()
|
||||
added = 0
|
||||
fixed_msgs = 0
|
||||
async with sm() as session:
|
||||
msgs = list(await session.scalars(select(Message)))
|
||||
for msg in msgs:
|
||||
try:
|
||||
raw = blobstore.load(msg.user_id, msg.blob_sha256)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
parsed = parse_message(raw)
|
||||
inline = [a for a in parsed.attachments if a.content_id]
|
||||
if not inline:
|
||||
continue
|
||||
have = set(
|
||||
await session.scalars(
|
||||
select(Attachment.content_id).where(
|
||||
Attachment.message_id == msg.id, Attachment.content_id.is_not(None)
|
||||
)
|
||||
)
|
||||
)
|
||||
new = 0
|
||||
for att in inline:
|
||||
if att.content_id in have:
|
||||
continue
|
||||
att_sha = blobstore.store(msg.user_id, att.payload)
|
||||
session.add(
|
||||
Attachment(
|
||||
message_id=msg.id,
|
||||
user_id=msg.user_id,
|
||||
filename=att.filename,
|
||||
mime_type=att.mime_type,
|
||||
size_bytes=len(att.payload),
|
||||
content_id=att.content_id,
|
||||
blob_sha256=att_sha,
|
||||
)
|
||||
)
|
||||
new += 1
|
||||
if new:
|
||||
added += new
|
||||
fixed_msgs += 1
|
||||
await session.commit()
|
||||
print(f"stored {added} missing inline images across {fixed_msgs} messages")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,56 @@
|
||||
"""One-off: add split-address tokens to existing messages' FTS vector.
|
||||
|
||||
New messages get address parts indexed via `_doc_text_blob` (so a bare "wfdif"
|
||||
finds dyrektor@wfdif.com.pl). This backfills that into already-indexed rows by
|
||||
appending the split-address lexemes to `body_indexed`. Idempotent — tsvector
|
||||
concat merges duplicate lexemes, so re-running is harmless.
|
||||
|
||||
Run: python -m scripts.reindex_address_parts
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
|
||||
from app.db import get_sessionmaker
|
||||
from app.models import Message
|
||||
from app.search.pg import _TS_CONFIG, _email_parts
|
||||
|
||||
|
||||
def _address_parts_blob(msg: Message) -> str:
|
||||
parts: list[str] = []
|
||||
for field in (msg.from_addr, msg.to_addr, msg.cc_addr):
|
||||
for addr in field:
|
||||
email = addr.get("email", "")
|
||||
if email:
|
||||
parts.append(_email_parts(email))
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
sm = get_sessionmaker()
|
||||
updated = 0
|
||||
async with sm() as session:
|
||||
msgs = list(await session.scalars(select(Message).where(Message.body_indexed.is_not(None))))
|
||||
for msg in msgs:
|
||||
blob = _address_parts_blob(msg)
|
||||
if not blob:
|
||||
continue
|
||||
await session.execute(
|
||||
update(Message)
|
||||
.where(Message.id == msg.id)
|
||||
.values(
|
||||
body_indexed=Message.body_indexed.op("||")(
|
||||
func.to_tsvector(_TS_CONFIG, blob)
|
||||
)
|
||||
)
|
||||
)
|
||||
updated += 1
|
||||
await session.commit()
|
||||
print(f"reindexed address parts into {updated} messages")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user