lukas/webmail-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

M2: Meili search + saved searches, sync windows + backfill progress, threading & image fixes

Browse Source 
Search:
- Meilisearch backend (app/search/meili.py) + factory wiring; pinned SDK 2.7
- Search-as-you-type: debounced query + Postgres prefix matching, built from
  the DB tokenizer so full email addresses match (email stays one lexeme)
- Saved searches API + sidebar virtual mailboxes; save affordance in the box

Account sync windows (SPEC §12 M2):
- Choose at setup: All (recent 30d first, then remainder), Last 3 months, New only
- Per-account sync_window + "Sync remaining mail" from settings (idempotent re-run)
- Provider windowed backfill (SINCE/BEFORE), message_count, newest_uid
- migration 0012: accounts.sync_window + backfill_total/backfill_done

Backfill progress UX:
- Floating progress card (auto-dissolves after 10s) + spinning sidebar ring that
  fills with the account colour; driven by sync.progress over WS

Thread list pagination:
- Server-side offset + accurate grand total, with filter/order in SQL; working
  prev/next across all synced mail

Reading panel:
- "Load images" one-off now actually renders images (?images=1) without trusting
  the sender; envelope carries images_allowed so a trusted sender shows no prompt

Threading:
- Subject fallback no longer merges messages that declare an explicit (different)
  parent; true siblings still group via a shared-parent reference match. Fixes
  unrelated auto-replies with the same subject landing in one thread

Backend 257 tests green, ruff + mypy --strict clean; frontend tsc/eslint/vitest clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
lukas
2026-07-23 23:00:08 +02:00
co-authored by Claude Opus 4.8
parent b4e07f11aa
commit 49738740c0
37 changed files with 1526 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sync/engine.py
+130 -12
View File
@@ -7,6 +7,8 @@ import contextlib
import logging
import random
import uuid
from dataclasses import dataclass, field
from datetime import date, timedelta
from typing import Any
from sqlalchemy import delete, select, update
@@ -27,6 +29,11 @@ logger = logging.getLogger(__name__)
# Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3).
_PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2}
# Sync-window import strategy (SPEC §12 M2). "all" imports the most recent
# _RECENT_DAYS first (so the account is usable fast), then the older remainder.
_RECENT_DAYS = 30
_WINDOW_DAYS = {"3months": 90}
async def _load_account(session: AsyncSession, account_id: uuid.UUID) -> Account:
account = await session.get(Account, account_id)
@@ -97,9 +104,8 @@ def _mailbox_order(mailboxes: list[Mailbox]) -> list[Mailbox]:
return sorted(mailboxes, key=lambda m: _PRIORITY.get(m.special_use or "", 99))
async def _emit_batch(
results: list[PersistResult], account: Account, mailbox_path: str, done: int
) -> None:
async def _index_and_notify(results: list[PersistResult], account: Account) -> None:
"""Index each persisted message and publish its ``message.new`` event."""
backend = get_search_backend()
bus = get_event_bus()
for r in results:
@@ -113,7 +119,13 @@ async def _emit_batch(
"thread_id": str(r.message.thread_id),
},
)
await bus.publish(
async def _emit_batch(
results: list[PersistResult], account: Account, mailbox_path: str, done: int
) -> None:
await _index_and_notify(results, account)
await get_event_bus().publish(
"sync.progress",
{
"user_id": str(account.user_id),
@@ -124,14 +136,61 @@ async def _emit_batch(
)
async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox: Mailbox) -> int:
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30)."""
@dataclass
class _BackfillProgress:
"""Account-level backfill counter: persists ``backfill_done`` and emits
``sync.progress`` with the running total so the UI can show a progress bar."""
account: Account
total: int
done: int = 0
_mailboxes: set[str] = field(default_factory=set)
async def record(self, results: list[PersistResult], mailbox_path: str) -> None:
from app.db import session_scope
await _index_and_notify(results, self.account)
self.done += len(results)
self._mailboxes.add(mailbox_path)
async with session_scope() as session:
await session.execute(
update(Account)
.where(Account.id == self.account.id)
.values(backfill_done=self.done)
)
await get_event_bus().publish(
"sync.progress",
{
"user_id": str(self.account.user_id),
"account_id": str(self.account.id),
"mailbox": mailbox_path,
"done": self.done,
"total": self.total,
},
)
async def backfill_mailbox(
provider: ImapSmtpProvider,
account: Account,
mailbox: Mailbox,
progress: _BackfillProgress | None = None,
*,
since: date | None = None,
before: date | None = None,
) -> int:
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30).
``since``/``before`` bound the import by date (sync window, SPEC §12 M2).
When a ``progress`` tracker is given, batches count toward the account-level
backfill total; otherwise each batch emits a standalone ``sync.progress``.
"""
from app.db import session_scope
account_id, user_id, mailbox_id = account.id, account.user_id, mailbox.id
special_use, path = mailbox.special_use, mailbox.path
done = 0
async for batch in provider.backfill(path, batch_size=200):
async for batch in provider.backfill(path, batch_size=200, since=since, before=before):
results: list[PersistResult] = []
max_uid = 0
async with session_scope() as session:
@@ -153,12 +212,41 @@ async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox
.values(last_seen_uid=max_uid)
)
done += len(results)
await _emit_batch(results, account, path, done)
if progress is not None:
await progress.record(results, path)
else:
await _emit_batch(results, account, path, done)
return done
async def _set_backfill_counts(account_id: uuid.UUID, *, total: int, done: int) -> None:
from app.db import session_scope
async with session_scope() as session:
await session.execute(
update(Account)
.where(Account.id == account_id)
.values(backfill_total=total, backfill_done=done)
)
async def _sync_new_only(provider: ImapSmtpProvider, mailboxes: list[Mailbox]) -> None:
"""``new`` window: import no history — just move each mailbox watermark to the
current tip so only mail arriving after setup is pulled (SPEC §12 M2)."""
from app.db import session_scope
for mailbox in mailboxes:
tip = await provider.newest_uid(mailbox.path)
async with session_scope() as session:
await session.execute(
update(Mailbox)
.where(Mailbox.id == mailbox.id, Mailbox.last_seen_uid < tip)
.values(last_seen_uid=tip)
)
async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
"""Full initial sync for an account: mailboxes + backfill; sets state=live."""
"""Full initial sync for an account: mailboxes + windowed backfill; state=live."""
from app.db import session_scope
async with session_scope() as session:
@@ -167,12 +255,42 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
mailboxes = _mailbox_order(await bootstrap_mailboxes(session, provider, account))
# Detach: reuse account/mailboxes values after the session closes.
account_obj = account
window = account.sync_window
mailbox_list = list(mailboxes)
total = 0
try:
if window == "new":
# No backfill: set watermarks to the tip, go live immediately.
await _sync_new_only(provider, mailbox_list)
await _set_backfill_counts(account_id, total=0, done=0)
await _set_state(account_id, "live")
return 0
since = (
date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None
)
# Progress denominator: count what we're about to import up front so the
# bar has a stable total (SPEC §12 M2 backfill-progress UX).
total = 0
for mailbox in mailbox_list:
total += await backfill_mailbox(provider, account_obj, mailbox)
total += await provider.message_count(mailbox.path, since=since)
await _set_backfill_counts(account_id, total=total, done=0)
progress = _BackfillProgress(account_obj, total)
recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS)
count = 0
for mailbox in mailbox_list:
if since is not None: # 3months: single windowed pass
count += await backfill_mailbox(
provider, account_obj, mailbox, progress, since=since
)
else: # all: recent window first (usable fast), then the remainder
count += await backfill_mailbox(
provider, account_obj, mailbox, progress, since=recent_cutoff
)
count += await backfill_mailbox(
provider, account_obj, mailbox, progress, before=recent_cutoff
)
await _set_state(account_id, "live")
except Exception as exc: # noqa: BLE001 - surface provider errors as account state
logger.exception("backfill failed for account %s", account_id)
@@ -180,7 +298,7 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
raise
finally:
await provider.close()
return total
return count
async def apply_delta(