M2: Meili search + saved searches, sync windows + backfill progress, threading & image fixes

Search:
- Meilisearch backend (app/search/meili.py) + factory wiring; pinned SDK 2.7
- Search-as-you-type: debounced query + Postgres prefix matching, built from
  the DB tokenizer so full email addresses match (email stays one lexeme)
- Saved searches API + sidebar virtual mailboxes; save affordance in the box

Account sync windows (SPEC §12 M2):
- Choose at setup: All (recent 30d first, then remainder), Last 3 months, New only
- Per-account sync_window + "Sync remaining mail" from settings (idempotent re-run)
- Provider windowed backfill (SINCE/BEFORE), message_count, newest_uid
- migration 0012: accounts.sync_window + backfill_total/backfill_done

Backfill progress UX:
- Floating progress card (auto-dissolves after 10s) + spinning sidebar ring that
  fills with the account colour; driven by sync.progress over WS

Thread list pagination:
- Server-side offset + accurate grand total, with filter/order in SQL; working
  prev/next across all synced mail

Reading panel:
- "Load images" one-off now actually renders images (?images=1) without trusting
  the sender; envelope carries images_allowed so a trusted sender shows no prompt

Threading:
- Subject fallback no longer merges messages that declare an explicit (different)
  parent; true siblings still group via a shared-parent reference match. Fixes
  unrelated auto-replies with the same subject landing in one thread

Backend 257 tests green, ruff + mypy --strict clean; frontend tsc/eslint/vitest clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
2026-07-23 23:00:08 +02:00
co-authored by Claude Opus 4.8
parent b4e07f11aa
commit 49738740c0
37 changed files with 1526 additions and 108 deletions
+9
View File
@@ -3,9 +3,13 @@
from __future__ import annotations
import uuid
from typing import Literal
from pydantic import BaseModel, Field
# How much history to import at account setup (SPEC §12 M2).
SyncWindow = Literal["all", "3months", "new"]
class ServerConfigIn(BaseModel):
host: str
@@ -56,12 +60,14 @@ class CreateAccountRequest(BaseModel):
smtp: ServerConfigIn
username: str | None = None
password: str = Field(min_length=1)
sync_window: SyncWindow = "all"
class UpdateAccountRequest(BaseModel):
display_name: str | None = None
color: str | None = None
signature: str | None = None
sync_window: SyncWindow | None = None
# Connection/credential changes require the password (re-validated + re-encrypted).
imap: ServerConfigIn | None = None
smtp: ServerConfigIn | None = None
@@ -76,5 +82,8 @@ class AccountDetailOut(BaseModel):
state: str
state_detail: str
signature: str
sync_window: str
backfill_total: int
backfill_done: int
imap_config: dict[str, object]
smtp_config: dict[str, object]
+1
View File
@@ -39,6 +39,7 @@ async def create_account(
secret_ct=encrypt_secret(req.password.encode("utf-8")),
discovery_src="manual",
state="backfilling",
sync_window=req.sync_window,
)
session.add(account)
try:
+2
View File
@@ -17,6 +17,7 @@ def register_routers(app: FastAPI) -> None:
from app.api.mailboxes import router as mailboxes_router
from app.api.messages import router as messages_router
from app.api.recipients import router as recipients_router
from app.api.saved_searches import router as saved_searches_router
from app.api.search import router as search_router
from app.api.settings import router as settings_router
from app.api.threads import router as threads_router
@@ -34,6 +35,7 @@ def register_routers(app: FastAPI) -> None:
recipients_router,
settings_router,
mailboxes_router,
saved_searches_router,
):
app.include_router(router, prefix="/api")
app.include_router(ws_router, prefix="/api")
+19 -1
View File
@@ -82,6 +82,9 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
color=a.color,
state=a.state,
signature=a.signature,
sync_window=a.sync_window,
backfill_total=a.backfill_total,
backfill_done=a.backfill_done,
)
for a in accounts
]
@@ -186,6 +189,18 @@ async def update_account(
if body.signature is not None:
account.signature = body.signature
# Broadening the sync window (e.g. "new" -> "all", "Download all mail") means
# importing history not fetched before, so re-run the backfill (SPEC §12 M2).
# persist_message is idempotent on (mailbox_id, uid), so already-synced mail
# is skipped. Narrowing keeps existing mail — nothing to fetch.
_COVERAGE = {"new": 0, "3months": 1, "all": 2}
broaden = False
if body.sync_window is not None and body.sync_window != account.sync_window:
broaden = _COVERAGE[body.sync_window] > _COVERAGE.get(account.sync_window, 0)
account.sync_window = body.sync_window
if broaden:
account.state = "backfilling"
# Connection settings and/or password: re-validate against the servers, then
# re-encrypt the secret and re-sync. The password must be supplied (§10.2: the
# stored secret is only decrypted in the worker, never here).
@@ -224,7 +239,7 @@ async def update_account(
await db.commit()
if conn_change:
if conn_change or broaden:
await enqueue_job("sync_bootstrap_account", str(account.id))
return _detail(account)
@@ -248,6 +263,9 @@ def _detail(account: Account) -> AccountDetailOut:
state=account.state,
state_detail=account.state_detail,
signature=account.signature,
sync_window=account.sync_window,
backfill_total=account.backfill_total,
backfill_done=account.backfill_done,
imap_config=account.imap_config,
smtp_config=account.smtp_config,
)
+10 -3
View File
@@ -65,14 +65,21 @@ def _sender_email(message: Message) -> str:
@router.get("/{message_id}/html", response_class=HTMLResponse)
async def message_html(message_id: uuid.UUID, user: CurrentUser, db: DbSession) -> HTMLResponse:
async def message_html(
message_id: uuid.UUID,
user: CurrentUser,
db: DbSession,
images: bool = Query(False),
) -> HTMLResponse:
"""Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
Re-sanitizes from the raw blob when the sender is image-approved, or when the
stored copy predates the current sanitizer version (SPEC §10.4).
stored copy predates the current sanitizer version (SPEC §10.4). ``images=1``
is a one-time "Load images" — render remote images for this request only,
without persisting the sender as trusted.
"""
message = await _owned_message(db, user.id, message_id)
allow_remote = _sender_email(message) in allowed_image_senders(user)
allow_remote = images or _sender_email(message) in allowed_image_senders(user)
parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
+82
View File
@@ -0,0 +1,82 @@
"""Saved searches — stored queries shown as virtual mailboxes (SPEC §12 M2)."""
from __future__ import annotations
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.errors import raise_api_error
from app.db import get_session
from app.identity.deps import CurrentUser, require_csrf
from app.models import SavedSearch
router = APIRouter(prefix="/saved-searches", tags=["search"], dependencies=[Depends(require_csrf)])
DbSession = Annotated[AsyncSession, Depends(get_session)]
class SavedSearchOut(BaseModel):
id: uuid.UUID
name: str
query: str
pinned: bool
class SavedSearchIn(BaseModel):
name: str = Field(min_length=1, max_length=80)
query: str = Field(min_length=1, max_length=500)
pinned: bool = True
def _out(s: SavedSearch) -> SavedSearchOut:
return SavedSearchOut(id=s.id, name=s.name, query=s.query, pinned=s.pinned)
@router.get("", response_model=list[SavedSearchOut])
async def list_saved_searches(user: CurrentUser, db: DbSession) -> list[SavedSearchOut]:
rows = await db.scalars(
select(SavedSearch).where(SavedSearch.user_id == user.id).order_by(SavedSearch.name)
)
return [_out(s) for s in rows]
@router.post("", response_model=SavedSearchOut)
async def create_saved_search(
body: SavedSearchIn, user: CurrentUser, db: DbSession
) -> SavedSearchOut:
name = body.name.strip()
query = body.query.strip()
if not name or not query:
raise_api_error(400, "invalid", "name and query are required")
# Upsert by (user, name): re-saving under an existing name updates its query.
existing = await db.scalar(
select(SavedSearch).where(SavedSearch.user_id == user.id, SavedSearch.name == name)
)
if existing is not None:
existing.query = query
existing.pinned = body.pinned
saved = existing
else:
saved = SavedSearch(user_id=user.id, name=name, query=query, pinned=body.pinned)
db.add(saved)
await db.commit()
return _out(saved)
@router.delete("/{search_id}")
async def delete_saved_search(
search_id: uuid.UUID, user: CurrentUser, db: DbSession
) -> dict[str, str]:
saved = await db.scalar(
select(SavedSearch).where(SavedSearch.id == search_id, SavedSearch.user_id == user.id)
)
if saved is None:
raise_api_error(404, "not_found", "saved search not found")
await db.delete(saved)
await db.commit()
return {"status": "deleted"}
+6
View File
@@ -29,6 +29,7 @@ class ThreadSummary(BaseModel):
class ThreadListOut(BaseModel):
threads: list[ThreadSummary]
cursor: str | None = None
total: int = 0
class AttachmentOut(BaseModel):
@@ -54,6 +55,8 @@ class MessageEnvelope(BaseModel):
is_starred: bool
has_attachments: bool
has_remote_images: bool
# The sender is on the user's trusted list, so remote images load without a prompt.
images_allowed: bool = False
size_bytes: int
snippet: str
# Other accounts this same delivery also landed in (collapsed duplicates).
@@ -84,6 +87,9 @@ class AccountOut(BaseModel):
color: str
state: str
signature: str = ""
sync_window: str = "all"
backfill_total: int = 0
backfill_done: int = 0
class SearchThreadHit(BaseModel):
+34 -21
View File
@@ -3,13 +3,14 @@
from __future__ import annotations
import uuid
from collections.abc import Collection
from typing import Annotated
from fastapi import APIRouter, Depends, Query
from sqlalchemy import Text, func, literal, or_, select, tuple_
from sqlalchemy import Text, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from app.api.cursors import decode_cursor, encode_cursor
from app.api.errors import raise_api_error
from app.api.schemas import (
Address,
@@ -18,6 +19,7 @@ from app.api.schemas import (
ThreadListOut,
ThreadSummary,
)
from app.api.settings import allowed_image_senders
from app.db import get_session
from app.identity.deps import CurrentUser
from app.models import Mailbox, Message, Thread, User
@@ -48,7 +50,8 @@ def _summary(thread: Thread, message_count: int | None = None) -> ThreadSummary:
)
def _envelope(msg: Message) -> MessageEnvelope:
def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnvelope:
sender = msg.from_addr[0].get("email", "").lower() if msg.from_addr else ""
return MessageEnvelope(
id=msg.id,
account_id=msg.account_id,
@@ -65,6 +68,7 @@ def _envelope(msg: Message) -> MessageEnvelope:
is_starred="\\Flagged" in msg.flags,
has_attachments=msg.has_attachments,
has_remote_images=msg.has_remote_images,
images_allowed=bool(sender) and sender in allowed_senders,
size_bytes=msg.size_bytes,
snippet=msg.snippet,
)
@@ -75,10 +79,14 @@ async def list_threads(
user: CurrentUser,
db: DbSession,
view: str = Query("unified"),
cursor: str | None = None,
limit: int = Query(50, ge=1, le=200),
offset: int = Query(0, ge=0),
filter: str = Query("all"), # all | unread | pinned
order: str = Query("desc"), # desc (newest first) | asc (oldest first)
) -> ThreadListOut:
stmt = select(Thread).where(Thread.user_id == user.id)
# Collect WHERE conditions once so the page query and the total-count query
# stay in lockstep (offset pagination needs an accurate grand total).
conditions: list[ColumnElement[bool]] = [Thread.user_id == user.id]
if view.startswith("account:"):
# A single account's inbox: threads with a message in THIS account's
@@ -97,7 +105,7 @@ async def list_threads(
Mailbox.special_use == "inbox",
)
)
stmt = stmt.where(Thread.id.in_(in_account_inbox))
conditions.append(Thread.id.in_(in_account_inbox))
elif view.startswith("folder:"):
# Threads with at least one message in a mailbox of this special_use,
# across all of the user's accounts (unified per-folder view).
@@ -107,24 +115,28 @@ async def list_threads(
.join(Mailbox, Mailbox.id == Message.mailbox_id)
.where(Message.user_id == user.id, Mailbox.special_use == special)
)
stmt = stmt.where(Thread.id.in_(in_folder))
conditions.append(Thread.id.in_(in_folder))
elif view != "unified":
raise_api_error(400, "bad_view", "unsupported view")
decoded = decode_cursor(cursor)
if decoded is not None:
cur_date, cur_id = decoded
stmt = stmt.where(
tuple_(Thread.last_message_at, Thread.id) < tuple_(literal(cur_date), literal(cur_id))
)
stmt = stmt.order_by(Thread.last_message_at.desc(), Thread.id.desc()).limit(limit + 1)
if filter == "unread":
conditions.append(Thread.unread_count > 0)
elif filter == "pinned":
conditions.append(Thread.pinned.is_(True))
threads = list(await db.scalars(stmt))
has_more = len(threads) > limit
threads = threads[:limit]
next_cursor = (
encode_cursor(threads[-1].last_message_at, threads[-1].id) if has_more and threads else None
total = (
await db.scalar(
select(func.count()).select_from(select(Thread.id).where(*conditions).subquery())
)
) or 0
order_by = (
(Thread.last_message_at.asc(), Thread.id.asc())
if order == "asc"
else (Thread.last_message_at.desc(), Thread.id.desc())
)
stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit)
threads = list(await db.scalars(stmt))
# Per-thread visible message count matching what get_thread shows for this view:
# Trash/Spam count only their own messages; every other view excludes them. This
@@ -152,7 +164,7 @@ async def list_threads(
}
return ThreadListOut(
threads=[_summary(t, counts.get(t.id)) for t in threads], cursor=next_cursor
threads=[_summary(t, counts.get(t.id)) for t in threads], total=total
)
@@ -216,9 +228,10 @@ async def get_thread(
elif m.account_id != rep.account_id and m.account_id not in also[key]:
also[key].append(m.account_id)
allowed_senders = allowed_image_senders(user)
envelopes = []
for m in ordered:
env = _envelope(m)
env = _envelope(m, allowed_senders)
env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"]
envelopes.append(env)
+5
View File
@@ -82,6 +82,11 @@ class Account(Base):
discovery_src: Mapped[str] = mapped_column(Text, nullable=False)
state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'backfilling'"))
state_detail: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
# How much history to import (SPEC §12 M2): all | 3months | new. Drives the
# initial backfill; backfill_total/backfill_done feed the progress UX.
sync_window: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'all'"))
backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
created_at: Mapped[dt.datetime] = _now()
+19 -3
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import datetime
from datetime import date, datetime
from typing import Protocol
@@ -71,9 +71,25 @@ class MailProvider(Protocol):
async def list_mailboxes(self) -> list[RemoteMailbox]: ...
def backfill(
self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
self,
mailbox: str,
*,
newest_first: bool = True,
batch_size: int = 200,
since: date | None = None,
before: date | None = None,
) -> AsyncIterator[list[RemoteMessage]]:
"""Yields batches until the mailbox is exhausted. Headers+body in one pass."""
"""Yields batches until the mailbox is exhausted. Headers+body in one pass.
``since``/``before`` bound the import by internal date (SPEC §12 M2)."""
...
async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
"""Number of messages to import (progress denominator); whole mailbox or
the ``since`` window."""
...
async def newest_uid(self, mailbox: str) -> int:
"""Current high-water UID; used by the ``new`` window to skip history."""
...
async def poll(
+43 -2
View File
@@ -13,6 +13,7 @@ import contextlib
import re
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import date
import aiosmtplib
from aioimaplib import aioimaplib
@@ -42,6 +43,21 @@ _CONNECT_TIMEOUT = 20
_PER_ACCOUNT_CONN_CAP = 4 # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
_APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
_COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")
_IMAP_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
def _imap_date(d: date) -> str:
"""RFC 3501 search date, e.g. ``5-Jul-2026`` (locale-independent)."""
return f"{d.day}-{_IMAP_MONTHS[d.month - 1]}-{d.year}"
def _date_criteria(since: date | None, before: date | None) -> str:
parts: list[str] = []
if since is not None:
parts.append(f"SINCE {_imap_date(since)}")
if before is not None:
parts.append(f"BEFORE {_imap_date(before)}")
return " ".join(parts) if parts else "ALL"
@dataclass(frozen=True)
@@ -184,12 +200,21 @@ class ImapSmtpProvider(MailProvider):
return 0
async def backfill(
self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
self,
mailbox: str,
*,
newest_first: bool = True,
batch_size: int = 200,
since: date | None = None,
before: date | None = None,
) -> AsyncIterator[list[RemoteMessage]]:
"""Yield message batches. ``since``/``before`` bound the import by IMAP
internal date (``SINCE`` inclusive, ``BEFORE`` exclusive) so the caller
can sync a recent window and/or the older remainder (SPEC §12 M2)."""
info = await self._select(mailbox)
self._uidvalidity_seen[mailbox] = info.get("uidvalidity", 0)
imap = await self._connect()
search = await imap.uid_search("ALL")
search = await imap.uid_search(_date_criteria(since, before))
uids = parse_search_uids(search.lines)
uids.sort(reverse=newest_first)
for start in range(0, len(uids), batch_size):
@@ -198,6 +223,22 @@ class ImapSmtpProvider(MailProvider):
if messages:
yield messages
async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
"""Number of messages to import for the progress denominator. Uses the
cheap ``EXISTS`` from SELECT for the whole mailbox, or a dated SEARCH."""
info = await self._select(mailbox)
if since is None:
return int(info.get("exists", 0))
imap = await self._connect()
search = await imap.uid_search(_date_criteria(since, None))
return len(parse_search_uids(search.lines))
async def newest_uid(self, mailbox: str) -> int:
"""Current high-water UID (``UIDNEXT - 1``); 0 for an empty mailbox. Used
by the ``new`` sync window to skip history and only pull future mail."""
info = await self._select(mailbox)
return max(0, int(info.get("uidnext", 1)) - 1)
async def list_uids(self, mailbox: str) -> set[int]:
"""Full set of UIDs currently present in the mailbox (expunge reconciliation).
+6 -4
View File
@@ -10,12 +10,14 @@ _backend: SearchBackend | None = None
def get_search_backend() -> SearchBackend:
"""Return the process-wide SearchBackend (``pg`` in M0-M1, ``meili`` in M2)."""
"""Return the process-wide SearchBackend (``pg`` or ``meili`` per SEARCH_BACKEND)."""
global _backend
if _backend is None:
which = get_settings().search_backend
if which == "meili":
# TODO(spec): Meilisearch backend lands in M2.
raise NotImplementedError("meili backend is implemented in M2")
_backend = PgSearchBackend()
from app.search.meili import MeiliSearchBackend
_backend = MeiliSearchBackend()
else:
_backend = PgSearchBackend()
return _backend
+267
View File
@@ -0,0 +1,267 @@
# INTERFACE VERSION: 1
"""Meilisearch SearchBackend (SPEC §5.2, §8; M2).
Typo-tolerant free-text search with highlighted snippets over a single ``messages``
index. Every query is filtered by ``user_id`` (invariant I-7). Structured §8.2
operators that map to filterable attributes — ``is`` / ``has`` / ``in`` / date /
size — become Meilisearch filters; the field operators (``from`` / ``to`` / ``cc``
/ ``subject`` / ``filename``) fold into the query text, since Meilisearch 1.8 has
no substring filter (the PG backend remains available for exact operator queries).
"""
from __future__ import annotations
import asyncio
import base64
import datetime as dt
import re
from typing import Any
from uuid import UUID
from meilisearch_python_sdk import AsyncClient
from app.config import get_settings
from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
_INDEX = "messages"
# Priority order matters: subject/senders rank above the body.
_SEARCHABLE = [
"subject",
"from_names",
"from_emails",
"to_names",
"to_emails",
"cc_names",
"cc_emails",
"body",
"attachment_names",
]
_FILTERABLE = [
"user_id",
"account_id",
"special",
"has_attachments",
"is_unread",
"is_starred",
"date",
"size_bytes",
]
_TEXT_FIELD_OPS = {"from", "to", "cc", "subject", "filename"}
_RELATIVE_DAYS = {"d": 1, "w": 7, "m": 30, "y": 365}
def _emails(field: list[dict[str, str]]) -> list[str]:
return [(a.get("email") or "").lower() for a in field if a.get("email")]
def _names(field: list[dict[str, str]]) -> list[str]:
return [a.get("name") or "" for a in field if a.get("name")]
def _to_doc(doc: SearchDoc) -> dict[str, Any]:
return {
"id": str(doc.message_id),
"thread_id": str(doc.thread_id),
"user_id": str(doc.user_id),
"account_id": str(doc.account_id),
"special": doc.mailbox_special_use or "",
"subject": doc.subject,
"body": doc.body_text,
"from_names": _names(doc.from_),
"from_emails": _emails(doc.from_),
"to_names": _names(doc.to),
"to_emails": _emails(doc.to),
"cc_names": _names(doc.cc),
"cc_emails": _emails(doc.cc),
"attachment_names": doc.attachment_names,
"has_attachments": doc.has_attachments,
"is_unread": doc.is_unread,
"is_starred": doc.is_starred,
"date": doc.date,
"size_bytes": doc.size_bytes,
}
def _parse_date_unix(value: str) -> int | None:
try:
d = dt.datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=dt.UTC)
except ValueError:
return None
return int(d.timestamp())
def _parse_relative_days(value: str) -> int | None:
m = re.fullmatch(r"(\d+)([dwmy])", value.strip().lower())
return int(m.group(1)) * _RELATIVE_DAYS[m.group(2)] if m else None
def _parse_size(value: str) -> int | None:
m = re.fullmatch(r"(\d+)([km])", value.strip().lower())
if not m:
return None
return int(m.group(1)) * (1024 if m.group(2) == "k" else 1024 * 1024)
def _op_filter(key: str, value: str) -> str | None:
"""A single §8.2 operator -> a Meilisearch filter fragment (or None)."""
now = int(dt.datetime.now(dt.UTC).timestamp())
if key == "in":
return f'special = "{value.lower()}"'
if key == "is":
v = value.lower()
if v == "unread":
return "is_unread = true"
if v == "read":
return "is_unread = false"
if v == "starred":
return "is_starred = true"
return None # 'answered' is not indexed in SearchDoc
if key == "has":
return "has_attachments = true" if value.lower() == "attachment" else None
if key == "before":
u = _parse_date_unix(value)
return f"date < {u}" if u is not None else None
if key == "after":
u = _parse_date_unix(value)
return f"date >= {u}" if u is not None else None
if key == "newer_than":
days = _parse_relative_days(value)
return f"date >= {now - days * 86400}" if days is not None else None
if key == "older_than":
days = _parse_relative_days(value)
return f"date < {now - days * 86400}" if days is not None else None
if key == "larger":
n = _parse_size(value)
return f"size_bytes > {n}" if n is not None else None
if key == "smaller":
n = _parse_size(value)
return f"size_bytes < {n}" if n is not None else None
return None
def _build_filter(groups: list[list[dict[str, Any]]], user_id: UUID) -> str:
ands = [f'user_id = "{user_id}"']
for group in groups:
ors: list[str] = []
for clause in group:
if clause.get("type") == "text":
continue
frag = _op_filter(str(clause.get("key")), str(clause.get("value", "")))
if frag is None:
continue
ors.append(f"NOT ({frag})" if clause.get("negate") else frag)
if ors:
ands.append("(" + " OR ".join(ors) + ")")
return " AND ".join(ands)
def _query_text(groups: list[list[dict[str, Any]]]) -> str:
"""Free text for Meili's `q`: text terms (negated as -term) + folded field ops."""
terms: list[str] = []
for group in groups:
for clause in group:
value = str(clause.get("value", "")).strip()
if not value:
continue
if clause.get("type") == "text":
terms.append(f"-{value}" if clause.get("negate") else value)
elif clause.get("key") in _TEXT_FIELD_OPS and not clause.get("negate"):
terms.append(value)
return " ".join(terms).strip()
class MeiliSearchBackend(SearchBackend):
def __init__(self) -> None:
cfg = get_settings()
self._client = AsyncClient(cfg.meili_url, cfg.meili_key or None)
self._index = self._client.index(_INDEX)
self._ready = False
self._lock = asyncio.Lock()
async def _ensure(self) -> None:
"""Create the index and apply settings once (idempotent)."""
if self._ready:
return
async with self._lock:
if self._ready:
return
indexes = await self._client.get_indexes()
names = {i.uid for i in indexes} if indexes else set()
if _INDEX not in names:
await self._client.create_index(_INDEX, primary_key="id")
await self._index.update_searchable_attributes(_SEARCHABLE)
await self._index.update_sortable_attributes(["date"])
task = await self._index.update_filterable_attributes(_FILTERABLE)
await self._client.wait_for_task(task.task_uid)
self._ready = True
async def index(self, doc: SearchDoc) -> None:
await self._ensure()
await self._index.add_documents([_to_doc(doc)])
async def delete(self, message_id: UUID) -> None:
await self._ensure()
await self._index.delete_document(str(message_id))
async def delete_user(self, user_id: UUID) -> None:
await self._ensure()
await self._index.delete_documents_by_filter(f'user_id = "{user_id}"')
async def query(
self, q: SearchQuery, *, limit: int, cursor: str | None
) -> tuple[list[SearchHit], str | None]:
await self._ensure()
groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
assert isinstance(groups, list)
offset = _decode_cursor(cursor)
res = await self._index.search(
_query_text(groups),
filter=_build_filter(groups, q.user_id),
offset=offset,
limit=limit + 1,
sort=["date:desc"], # stable order for offset paging; typo matching still applies
attributes_to_retrieve=["id", "thread_id"],
attributes_to_highlight=["subject", "body"],
attributes_to_crop=["body"],
crop_length=30,
highlight_pre_tag="<mark>",
highlight_post_tag="</mark>",
show_ranking_score=True,
)
rows = res.hits
has_more = len(rows) > limit
rows = rows[:limit]
hits = [_to_hit(row) for row in rows]
next_cursor = _encode_cursor(offset + limit) if has_more else None
return hits, next_cursor
def _to_hit(row: dict[str, Any]) -> SearchHit:
fmt = row.get("_formatted") or {}
highlights: dict[str, str] = {}
subject = fmt.get("subject") or ""
if "<mark>" in subject:
highlights["subject"] = subject
body = fmt.get("body") or ""
if "<mark>" in body:
highlights["body"] = body
return SearchHit(
message_id=UUID(row["id"]),
thread_id=UUID(row["thread_id"]),
score=float(row.get("_rankingScore") or 0.0),
highlights=highlights,
)
def _encode_cursor(offset: int) -> str:
return base64.urlsafe_b64encode(str(offset).encode()).decode()
def _decode_cursor(cursor: str | None) -> int:
if not cursor:
return 0
try:
return max(0, int(base64.urlsafe_b64decode(cursor.encode()).decode()))
except (ValueError, TypeError):
return 0
+21 -2
View File
@@ -23,6 +23,22 @@ from app.models import Attachment, Mailbox, Message
from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
_TS_CONFIG = "simple" # language-neutral; avoids wrong-language stemming (M0)
def _prefix_tsquery(value: str) -> ColumnElement[object]:
"""A prefix ``tsquery`` for search-as-you-type: ``invo`` finds ``invoice``.
We can't tokenize in Python and match the index: Postgres' parser keeps an
email/host/URL as a *single* lexeme (``lukas.remis@gmail.com``), so splitting
on word chars would never match it. Instead we re-parse the term with the
same ``plainto_tsquery`` used at index time, then append ``:*`` to each
lexeme to make it a prefix. ``plainto_tsquery`` strips tsquery operators and
quotes, so this is injection- and error-safe on hostile input; an all-symbol
term yields the empty query (matches nothing).
"""
lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text)
prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g")
return func.to_tsquery(_TS_CONFIG, prefixed)
_RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
@@ -95,7 +111,10 @@ class PgSearchBackend(SearchBackend):
def _clause_condition(self, clause: dict[str, object]) -> ColumnElement[bool] | None:
value = str(clause.get("value", ""))
if clause.get("type") == "text":
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
if clause.get("phrase"):
# Quoted phrase: keep exact-adjacency semantics, no prefix.
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
return Message.body_indexed.op("@@")(_prefix_tsquery(value))
key = clause.get("key")
cond = self._filter_condition(str(key), value)
return cond
@@ -172,7 +191,7 @@ class PgSearchBackend(SearchBackend):
conditions = self._build_conditions(groups)
rank = (
func.ts_rank_cd(Message.body_indexed, func.websearch_to_tsquery(_TS_CONFIG, q.text))
func.ts_rank_cd(Message.body_indexed, _prefix_tsquery(q.text))
if q.text
else literal(0.0)
)
+130 -12
View File
@@ -7,6 +7,8 @@ import contextlib
import logging
import random
import uuid
from dataclasses import dataclass, field
from datetime import date, timedelta
from typing import Any
from sqlalchemy import delete, select, update
@@ -27,6 +29,11 @@ logger = logging.getLogger(__name__)
# Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3).
_PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2}
# Sync-window import strategy (SPEC §12 M2). "all" imports the most recent
# _RECENT_DAYS first (so the account is usable fast), then the older remainder.
_RECENT_DAYS = 30
_WINDOW_DAYS = {"3months": 90}
async def _load_account(session: AsyncSession, account_id: uuid.UUID) -> Account:
account = await session.get(Account, account_id)
@@ -97,9 +104,8 @@ def _mailbox_order(mailboxes: list[Mailbox]) -> list[Mailbox]:
return sorted(mailboxes, key=lambda m: _PRIORITY.get(m.special_use or "", 99))
async def _emit_batch(
results: list[PersistResult], account: Account, mailbox_path: str, done: int
) -> None:
async def _index_and_notify(results: list[PersistResult], account: Account) -> None:
"""Index each persisted message and publish its ``message.new`` event."""
backend = get_search_backend()
bus = get_event_bus()
for r in results:
@@ -113,7 +119,13 @@ async def _emit_batch(
"thread_id": str(r.message.thread_id),
},
)
await bus.publish(
async def _emit_batch(
results: list[PersistResult], account: Account, mailbox_path: str, done: int
) -> None:
await _index_and_notify(results, account)
await get_event_bus().publish(
"sync.progress",
{
"user_id": str(account.user_id),
@@ -124,14 +136,61 @@ async def _emit_batch(
)
async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox: Mailbox) -> int:
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30)."""
@dataclass
class _BackfillProgress:
"""Account-level backfill counter: persists ``backfill_done`` and emits
``sync.progress`` with the running total so the UI can show a progress bar."""
account: Account
total: int
done: int = 0
_mailboxes: set[str] = field(default_factory=set)
async def record(self, results: list[PersistResult], mailbox_path: str) -> None:
from app.db import session_scope
await _index_and_notify(results, self.account)
self.done += len(results)
self._mailboxes.add(mailbox_path)
async with session_scope() as session:
await session.execute(
update(Account)
.where(Account.id == self.account.id)
.values(backfill_done=self.done)
)
await get_event_bus().publish(
"sync.progress",
{
"user_id": str(self.account.user_id),
"account_id": str(self.account.id),
"mailbox": mailbox_path,
"done": self.done,
"total": self.total,
},
)
async def backfill_mailbox(
provider: ImapSmtpProvider,
account: Account,
mailbox: Mailbox,
progress: _BackfillProgress | None = None,
*,
since: date | None = None,
before: date | None = None,
) -> int:
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30).
``since``/``before`` bound the import by date (sync window, SPEC §12 M2).
When a ``progress`` tracker is given, batches count toward the account-level
backfill total; otherwise each batch emits a standalone ``sync.progress``.
"""
from app.db import session_scope
account_id, user_id, mailbox_id = account.id, account.user_id, mailbox.id
special_use, path = mailbox.special_use, mailbox.path
done = 0
async for batch in provider.backfill(path, batch_size=200):
async for batch in provider.backfill(path, batch_size=200, since=since, before=before):
results: list[PersistResult] = []
max_uid = 0
async with session_scope() as session:
@@ -153,12 +212,41 @@ async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox
.values(last_seen_uid=max_uid)
)
done += len(results)
await _emit_batch(results, account, path, done)
if progress is not None:
await progress.record(results, path)
else:
await _emit_batch(results, account, path, done)
return done
async def _set_backfill_counts(account_id: uuid.UUID, *, total: int, done: int) -> None:
from app.db import session_scope
async with session_scope() as session:
await session.execute(
update(Account)
.where(Account.id == account_id)
.values(backfill_total=total, backfill_done=done)
)
async def _sync_new_only(provider: ImapSmtpProvider, mailboxes: list[Mailbox]) -> None:
"""``new`` window: import no history — just move each mailbox watermark to the
current tip so only mail arriving after setup is pulled (SPEC §12 M2)."""
from app.db import session_scope
for mailbox in mailboxes:
tip = await provider.newest_uid(mailbox.path)
async with session_scope() as session:
await session.execute(
update(Mailbox)
.where(Mailbox.id == mailbox.id, Mailbox.last_seen_uid < tip)
.values(last_seen_uid=tip)
)
async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
"""Full initial sync for an account: mailboxes + backfill; sets state=live."""
"""Full initial sync for an account: mailboxes + windowed backfill; state=live."""
from app.db import session_scope
async with session_scope() as session:
@@ -167,12 +255,42 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
mailboxes = _mailbox_order(await bootstrap_mailboxes(session, provider, account))
# Detach: reuse account/mailboxes values after the session closes.
account_obj = account
window = account.sync_window
mailbox_list = list(mailboxes)
total = 0
try:
if window == "new":
# No backfill: set watermarks to the tip, go live immediately.
await _sync_new_only(provider, mailbox_list)
await _set_backfill_counts(account_id, total=0, done=0)
await _set_state(account_id, "live")
return 0
since = (
date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None
)
# Progress denominator: count what we're about to import up front so the
# bar has a stable total (SPEC §12 M2 backfill-progress UX).
total = 0
for mailbox in mailbox_list:
total += await backfill_mailbox(provider, account_obj, mailbox)
total += await provider.message_count(mailbox.path, since=since)
await _set_backfill_counts(account_id, total=total, done=0)
progress = _BackfillProgress(account_obj, total)
recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS)
count = 0
for mailbox in mailbox_list:
if since is not None: # 3months: single windowed pass
count += await backfill_mailbox(
provider, account_obj, mailbox, progress, since=since
)
else: # all: recent window first (usable fast), then the remainder
count += await backfill_mailbox(
provider, account_obj, mailbox, progress, since=recent_cutoff
)
count += await backfill_mailbox(
provider, account_obj, mailbox, progress, before=recent_cutoff
)
await _set_state(account_id, "live")
except Exception as exc: # noqa: BLE001 - surface provider errors as account state
logger.exception("backfill failed for account %s", account_id)
@@ -180,7 +298,7 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
raise
finally:
await provider.close()
return total
return count
async def apply_delta(
+34 -4
View File
@@ -39,6 +39,20 @@ def _recipient_emails(message: Message) -> set[str]:
return emails
def _parent_refs(message: Message) -> list[str]:
"""Non-synthetic parent ids this message explicitly declares (References +
In-Reply-To). A non-empty result means the message states its parentage, so it
must not be subject-merged into an unrelated conversation (§7.3)."""
refs: list[str] = list(message.references_hdr)
if message.in_reply_to:
refs.append(message.in_reply_to)
out: list[str] = []
for r in refs:
if r and not is_synthetic(r) and r not in out:
out.append(r)
return out
def _candidate_ids(message: Message) -> list[str]:
candidates: list[str] = list(message.references_hdr)
if message.in_reply_to:
@@ -59,17 +73,26 @@ async def _find_by_references(
user_id: uuid.UUID,
message_id: uuid.UUID,
candidates: list[str],
parent_refs: list[str],
own_id: str | None,
) -> list[uuid.UUID]:
"""Threads joined by reference (SPEC §7.2.2).
Matches bidirectionally: existing messages whose id is one of my candidate ids
(I reference them / duplicate id), OR existing messages that reference *my* id
(handles a late-arriving parent). Synthetic own-ids are never used to join (§7.1).
(I reference them / duplicate id), existing messages that reference *my* id
(late-arriving parent), OR existing messages that share one of my declared
parents — a sibling reply to the same original, even when neither the parent
nor a mutual reference is present. Synthetic ids are never used to join (§7.1).
"""
conditions: list[ColumnElement[bool]] = []
if candidates:
conditions.append(Message.message_id_hdr.in_(candidates))
if parent_refs:
# Siblings: an existing message that references, or is a reply to, one of
# my parents. This groups replies to a common (possibly absent) original
# without letting the subject fallback merge replies to *different* ones.
conditions.append(Message.references_hdr.overlap(parent_refs))
conditions.append(Message.in_reply_to.in_(parent_refs))
if own_id is not None and not is_synthetic(own_id):
# own_id = ANY(references_hdr): an existing message that references me.
conditions.append(literal(own_id) == sa_any(Message.references_hdr))
@@ -191,10 +214,17 @@ async def thread_message(session: AsyncSession, message: Message) -> uuid.UUID:
Returns the resulting thread id and sets ``message.thread_id``.
"""
candidates = _candidate_ids(message)
parent_refs = _parent_refs(message)
own_id = message.message_id_hdr
thread_ids = await _find_by_references(session, message.user_id, message.id, candidates, own_id)
thread_ids = await _find_by_references(
session, message.user_id, message.id, candidates, parent_refs, own_id
)
if not thread_ids:
# Subject fallback only for messages that declare no parentage (a broken/absent
# References chain). A message that explicitly replies to a specific parent is
# never subject-merged into an unrelated thread — e.g. two auto-replies to
# different originals that happen to share the same subject (§7.3).
if not thread_ids and not parent_refs:
subject_thread = await _find_by_subject(session, message.user_id, message)
if subject_thread is not None:
thread_ids = [subject_thread]
@@ -0,0 +1,35 @@
"""add accounts.sync_window + backfill progress counters
Lets the user choose at account setup how much history to import (SPEC §12 M2):
``all`` (default; recent 30d first, then the rest), ``3months`` (windowed), or
``new`` (no backfill — only mail arriving after setup). ``backfill_total`` /
``backfill_done`` drive the initial-sync progress UX; total 0 means "nothing to
import" (the ``new`` window, or an empty account).
Revision ID: 0012
Revises: 0011
Create Date: 2026-07-23
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
revision: str = "0012"
down_revision: str | None = "0011"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute("ALTER TABLE accounts ADD COLUMN sync_window text NOT NULL DEFAULT 'all'")
op.execute("ALTER TABLE accounts ADD COLUMN backfill_total integer NOT NULL DEFAULT 0")
op.execute("ALTER TABLE accounts ADD COLUMN backfill_done integer NOT NULL DEFAULT 0")
def downgrade() -> None:
op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS backfill_done")
op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS backfill_total")
op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS sync_window")
+1
View File
@@ -28,6 +28,7 @@ dependencies = [
"nh3~=0.2",
"cryptography~=43.0",
"mail-parser~=4.0",
"meilisearch-python-sdk==2.7.0", # last release that fits the pinned Meilisearch v1.8
"python-multipart~=0.0.9",
# DEPENDENCY: SPEC §6.2 mandates accounts/presets.yaml (config, not code); the
# stdlib has no YAML parser, so PyYAML is required to load it.
+5 -4
View File
@@ -378,12 +378,13 @@ async def test_accounts_list(api: ApiCtx) -> None:
assert resp.json()[0]["email"] == "me@example.com"
async def test_thread_pagination_cursor(api: ApiCtx) -> None:
first = await api.client.get("/api/threads", params={"limit": 1})
async def test_thread_pagination_offset(api: ApiCtx) -> None:
first = await api.client.get("/api/threads", params={"limit": 1, "offset": 0})
body = first.json()
assert len(body["threads"]) == 1
assert body["cursor"] is not None
second = await api.client.get("/api/threads", params={"limit": 1, "cursor": body["cursor"]})
assert body["total"] >= 2 # more than one page of threads exists
second = await api.client.get("/api/threads", params={"limit": 1, "offset": 1})
body2 = second.json()
assert len(body2["threads"]) == 1
assert body2["total"] == body["total"] # grand total is stable across pages
assert body2["threads"][0]["id"] != body["threads"][0]["id"]
+141
View File
@@ -0,0 +1,141 @@
"""Saved searches CRUD — stored queries shown as virtual mailboxes (SPEC §12 M2)."""
from __future__ import annotations
import uuid
from collections.abc import AsyncIterator
from dataclasses import dataclass
import httpx
import pytest_asyncio
from httpx import ASGITransport
from sqlalchemy import delete
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
from app.config import get_settings
from app.identity.passwords import hash_password
from app.identity.sessions import create_session
from app.main import create_app
from app.models import User
CSRF = "csrf"
@dataclass
class Ctx:
client: httpx.AsyncClient
user_id: uuid.UUID
def _hdr() -> dict[str, str]:
return {"X-CSRF": CSRF}
@pytest_asyncio.fixture
async def ctx() -> AsyncIterator[Ctx]:
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
sm = async_sessionmaker(engine, expire_on_commit=False)
session = sm()
user = User(handle=f"ss{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
session.add(user)
await session.commit()
token = await create_session(user.id, "pytest")
client = httpx.AsyncClient(
transport=ASGITransport(app=create_app()),
base_url="http://testserver",
cookies={"session": token, "csrf": CSRF},
)
try:
yield Ctx(client=client, user_id=user.id)
finally:
await client.aclose()
await session.execute(delete(User).where(User.id == user.id))
await session.commit()
await session.close()
await engine.dispose()
async def test_saved_search_crud_and_upsert(ctx: Ctx) -> None:
# Empty to start.
resp = await ctx.client.get("/api/saved-searches")
assert resp.status_code == 200
assert resp.json() == []
# Create.
resp = await ctx.client.post(
"/api/saved-searches",
json={"name": "Unread from boss", "query": "is:unread from:boss"},
headers=_hdr(),
)
assert resp.status_code == 200
created = resp.json()
assert created["name"] == "Unread from boss"
assert created["query"] == "is:unread from:boss"
assert created["pinned"] is True
sid = created["id"]
# It lists back.
resp = await ctx.client.get("/api/saved-searches")
assert [s["name"] for s in resp.json()] == ["Unread from boss"]
# Re-saving under the same name upserts (updates query, keeps a single row).
resp = await ctx.client.post(
"/api/saved-searches",
json={"name": "Unread from boss", "query": "is:unread from:ceo"},
headers=_hdr(),
)
assert resp.status_code == 200
assert resp.json()["id"] == sid # same row
assert resp.json()["query"] == "is:unread from:ceo"
rows = (await ctx.client.get("/api/saved-searches")).json()
assert len(rows) == 1 and rows[0]["query"] == "is:unread from:ceo"
# Delete.
resp = await ctx.client.delete(f"/api/saved-searches/{sid}", headers=_hdr())
assert resp.status_code == 200
assert (await ctx.client.get("/api/saved-searches")).json() == []
# Deleting a missing row 404s.
resp = await ctx.client.delete(f"/api/saved-searches/{uuid.uuid4()}", headers=_hdr())
assert resp.status_code == 404
async def test_saved_search_requires_csrf(ctx: Ctx) -> None:
# No CSRF header -> rejected (state-changing route).
resp = await ctx.client.post(
"/api/saved-searches", json={"name": "x", "query": "y"}
)
assert resp.status_code == 403
async def test_saved_search_is_per_user(ctx: Ctx) -> None:
await ctx.client.post(
"/api/saved-searches",
json={"name": "Mine", "query": "in:inbox"},
headers=_hdr(),
)
# A different user's client sees none of the above.
other = User(handle=f"ss{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
sm = async_sessionmaker(engine, expire_on_commit=False)
session = sm()
session.add(other)
await session.commit()
token = await create_session(other.id, "pytest")
try:
client = httpx.AsyncClient(
transport=ASGITransport(app=create_app()),
base_url="http://testserver",
cookies={"session": token, "csrf": CSRF},
)
try:
assert (await client.get("/api/saved-searches")).json() == []
finally:
await client.aclose()
finally:
await session.execute(delete(User).where(User.id == other.id))
await session.commit()
await session.close()
await engine.dispose()
+79
View File
@@ -0,0 +1,79 @@
"""Meilisearch backend (SPEC §8, M2). Runs against the live Meili in the dev/CI
stack; skips if it is unreachable so the rest of the suite still runs locally."""
from __future__ import annotations
import asyncio
import time
import uuid
import pytest
from app.search.base import SearchDoc
from app.search.meili import MeiliSearchBackend
from app.search.queryparser import to_search_query
def _doc(user_id: uuid.UUID) -> SearchDoc:
return SearchDoc(
message_id=uuid.uuid4(),
thread_id=uuid.uuid4(),
user_id=user_id,
account_id=uuid.uuid4(),
mailbox_special_use="inbox",
from_=[{"name": "Bob Vendor", "email": "bob@vendor.com"}],
to=[{"name": "Me", "email": "me@x.com"}],
cc=[],
subject="Your invoice for March",
body_text="Please find attached the invoice, amount due soon.",
attachment_names=["invoice.pdf"],
has_attachments=True,
is_unread=True,
is_starred=False,
date=int(time.time()),
size_bytes=2048,
)
async def _poll(backend: MeiliSearchBackend, q: str, user_id: uuid.UUID) -> int:
"""Indexing is async in Meili; retry briefly until the doc is searchable."""
for _ in range(60):
hits, _c = await backend.query(to_search_query(q, user_id), limit=10, cursor=None)
if hits:
return len(hits)
await asyncio.sleep(0.05)
return 0
async def test_meili_typo_tolerance_and_filters() -> None:
backend = MeiliSearchBackend()
user_id = uuid.uuid4()
try:
await backend.index(_doc(user_id))
# Acceptance (§12 M2): a typo still finds the message.
hits, _c = await backend.query(
to_search_query("invocie", user_id), limit=10, cursor=None
)
if not hits: # wait for the async index task
assert await _poll(backend, "invocie", user_id) == 1
hits, _c = await backend.query(
to_search_query("invocie", user_id), limit=10, cursor=None
)
assert len(hits) == 1
assert "<mark>" in hits[0].highlights.get("subject", "")
async def n(query: str) -> int:
res, _c = await backend.query(to_search_query(query, user_id), limit=10, cursor=None)
return len(res)
assert await n("is:unread invoice") == 1
assert await n("is:read invoice") == 0 # the doc is unread
assert await n("from:vendor") == 1 # field operator folded into the query
assert await n("larger:1k invoice") == 1
assert await n("larger:5k invoice") == 0
assert await n("in:sent invoice") == 0 # doc is in the inbox
except (ConnectionError, OSError) as exc: # Meili not running locally
pytest.skip(f"Meilisearch unavailable: {exc}")
finally:
await backend.delete_user(user_id)
+18
View File
@@ -150,6 +150,24 @@ async def test_free_text(seeded: Ctx) -> None:
assert await _search(seeded, "lunch") == 1
async def test_free_text_prefix_matches_while_typing(seeded: Ctx) -> None:
# Search-as-you-type: a partial word finds the message before it is complete.
assert await _search(seeded, "reven") == 1 # -> "revenue"
assert await _search(seeded, "lun") == 1 # -> "lunch"
# Prefix is anchored at word start, not a substring match.
assert await _search(seeded, "enue") == 0
# Multiple partial words are ANDed as prefixes.
assert await _search(seeded, "quart reven") == 1 # "Quarterly revenue"
assert await _search(seeded, "quart lun") == 0
async def test_free_text_full_email_address(seeded: Ctx) -> None:
# Postgres keeps an email as a single lexeme; searching the *whole* address
# must still match (regression: prefix-splitting on word chars broke this).
assert await _search(seeded, "bob@corp.com") == 1
assert await _search(seeded, "bob@corp.co") == 1 # prefix while still typing
async def test_from_operator(seeded: Ctx) -> None:
assert await _search(seeded, "from:bob") == 1
assert await _search(seeded, "from:corp.com") == 3 # all three share the domain
+160
View File
@@ -0,0 +1,160 @@
"""Sync-window backfill strategies at account setup (SPEC §12 M2): all (recent
first, then remainder), 3months (windowed), new (no history, watermark only)."""
from __future__ import annotations
import uuid
from collections.abc import AsyncIterator
from datetime import UTC, date, datetime, timedelta
from email.message import EmailMessage
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
from app.config import get_settings
from app.identity.passwords import hash_password
from app.models import Account, Mailbox, Message, User
from app.providers.base import RemoteMailbox, RemoteMessage
from app.sync import engine as sync_engine
def _raw(mid: str) -> bytes:
m = EmailMessage()
m["Subject"] = "hi"
m["From"] = "me@example.com"
m["To"] = "bob@example.com"
m["Message-ID"] = mid
m["Date"] = "Mon, 09 Mar 2026 08:36:37 +0100"
m.set_content("body")
return m.as_bytes()
class _FakeProvider:
def __init__(self) -> None:
self.backfill_calls: list[tuple[str, date | None, date | None]] = []
self.count_calls: list[tuple[str, date | None]] = []
self.newest_calls: list[str] = []
async def list_mailboxes(self) -> list[RemoteMailbox]:
return [RemoteMailbox(path="INBOX", special_use="inbox", uidvalidity=1)]
async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
self.count_calls.append((mailbox, since))
return 3
async def newest_uid(self, mailbox: str) -> int:
self.newest_calls.append(mailbox)
return 42
async def backfill( # type: ignore[no-untyped-def]
self, mailbox, *, newest_first=True, batch_size=200, since=None, before=None
) -> AsyncIterator[list[RemoteMessage]]:
self.backfill_calls.append((mailbox, since, before))
# Distinct UID per pass so recent + remainder both persist a row.
uid = 100 if since is not None else 50
yield [
RemoteMessage(
uid=uid,
raw=_raw(f"<{mailbox}-{uid}@x.com>"),
flags=frozenset(),
internal_date=datetime.now(UTC),
)
]
async def close(self) -> None:
return None
async def _seed(sm: async_sessionmaker, window: str) -> tuple[uuid.UUID, uuid.UUID]: # type: ignore[type-arg]
session = sm()
user = User(handle=f"bf{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
session.add(user)
await session.flush()
account = Account(
user_id=user.id,
email="me@example.com",
imap_config={},
smtp_config={},
secret_ct=b"",
discovery_src="manual",
sync_window=window,
)
session.add(account)
await session.commit()
aid = account.id
await session.close()
return user.id, aid
async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None: # type: ignore[no-untyped-def]
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
sm = async_sessionmaker(engine, expire_on_commit=False)
_uid, account_id = await _seed(sm, "new")
fake = _FakeProvider()
monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake)
try:
count = await sync_engine.bootstrap_and_backfill(account_id)
assert count == 0
assert fake.backfill_calls == [] # no history fetched
assert fake.newest_calls == ["INBOX"]
session = sm()
acct = await session.get(Account, account_id)
assert acct is not None and acct.state == "live" and acct.backfill_total == 0
mb = await session.scalar(select(Mailbox).where(Mailbox.account_id == account_id))
assert mb is not None and mb.last_seen_uid == 42 # moved to the tip
msgs = await session.scalar(
select(func.count()).select_from(Message).where(Message.mailbox_id == mb.id)
)
assert msgs == 0 # nothing imported
await session.close()
finally:
await engine.dispose()
async def test_window_all_imports_recent_then_remainder(monkeypatch) -> None: # type: ignore[no-untyped-def]
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
sm = async_sessionmaker(engine, expire_on_commit=False)
_uid, account_id = await _seed(sm, "all")
fake = _FakeProvider()
monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake)
try:
await sync_engine.bootstrap_and_backfill(account_id)
# Two passes per mailbox: recent window (since set), then the older
# remainder (before set) — the "usable fast, then fill in" strategy.
cutoff = date.today() - timedelta(days=sync_engine._RECENT_DAYS)
assert fake.backfill_calls == [("INBOX", cutoff, None), ("INBOX", None, cutoff)]
assert fake.count_calls == [("INBOX", None)] # denominator = whole mailbox
session = sm()
acct = await session.get(Account, account_id)
assert acct is not None and acct.state == "live"
assert acct.backfill_total == 3 and acct.backfill_done == 2
mb = await session.scalar(select(Mailbox).where(Mailbox.account_id == account_id))
assert mb is not None
msgs = await session.scalar(
select(func.count()).select_from(Message).where(Message.mailbox_id == mb.id)
)
assert msgs == 2 # one from each pass
await session.close()
finally:
await engine.dispose()
async def test_window_3months_single_dated_pass(monkeypatch) -> None: # type: ignore[no-untyped-def]
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
sm = async_sessionmaker(engine, expire_on_commit=False)
_uid, account_id = await _seed(sm, "3months")
fake = _FakeProvider()
monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake)
try:
await sync_engine.bootstrap_and_backfill(account_id)
since = date.today() - timedelta(days=sync_engine._WINDOW_DAYS["3months"])
assert fake.backfill_calls == [("INBOX", since, None)] # single windowed pass
assert fake.count_calls == [("INBOX", since)] # denominator counts the window
await engine.dispose()
finally:
pass
+42
View File
@@ -72,6 +72,48 @@ async def test_broken_client_subject_only_joins(env: ThreadingEnv) -> None:
assert a.thread_id == b.thread_id
async def test_auto_replies_to_different_originals_do_not_join(env: ThreadingEnv) -> None:
# Two out-of-office auto-replies, same subject and same recipient, but each
# replying to a DIFFERENT (absent) original. They must not be subject-merged.
a = await env.add(
message_id="autoA@tvp.pl",
subject="Automatyczna odpowiedź: Silent Hill",
from_email="robert@tvp.pl",
to_email="me@gmail.com",
in_reply_to="orig1@mail.gmail.com",
references=["orig1@mail.gmail.com"],
)
b = await env.add(
message_id="autoB@wfdif.com.pl",
subject="Automatyczna odpowiedź: Silent Hill",
from_email="dyrektor@wfdif.com.pl",
to_email="me@gmail.com",
in_reply_to="orig2@mail.gmail.com",
references=["orig2@mail.gmail.com"],
minutes=5,
)
assert a.thread_id != b.thread_id
async def test_replies_to_same_absent_parent_join(env: ThreadingEnv) -> None:
# Two replies that reference the SAME (absent) original — and not each other —
# still thread together via the shared parent reference.
a = await env.add(
message_id="r1@x.com",
subject="Re: Topic",
in_reply_to="orig@x.com",
references=["orig@x.com"],
)
b = await env.add(
message_id="r2@x.com",
subject="Re: Topic",
in_reply_to="orig@x.com",
references=["orig@x.com"],
minutes=3,
)
assert a.thread_id == b.thread_id
async def test_distinct_topic_same_subject_does_not_join(env: ThreadingEnv) -> None:
a = await env.add(
message_id="a@x.com",
@@ -0,0 +1,69 @@
// Initial-sync progress UX (SPEC §12 M2). Shows a small floating card while any
// account is importing its history; reads the accounts query, which the WS
// `sync.progress` event invalidates so the bar advances live.
import { useQuery } from "@tanstack/react-query";
import { useEffect, useState } from "react";
import { useT } from "../i18n";
import { api } from "../lib/api";
export function BackfillBanner() {
const t = useT();
const accounts = useQuery({ queryKey: ["accounts"], queryFn: api.accounts });
const importing = (accounts.data ?? []).filter(
(a) => a.state === "backfilling" && a.backfill_total > 0,
);
const active = importing.length > 0;
// Dissolve the card 10s after it appears; the sidebar ring keeps showing
// progress until the sync finishes.
const [dismissed, setDismissed] = useState(false);
useEffect(() => {
if (!active) {
setDismissed(false);
return;
}
const timer = window.setTimeout(() => setDismissed(true), 10000);
return () => window.clearTimeout(timer);
}, [active]);
if (!active) return null;
return (
<div
className={`fixed bottom-4 left-4 z-40 w-72 space-y-2 transition-opacity duration-700 ${
dismissed ? "pointer-events-none opacity-0" : "opacity-100"
}`}
>
{importing.map((a) => {
const done = Math.min(a.backfill_done, a.backfill_total);
const pct = Math.round((done / a.backfill_total) * 100);
return (
<div
key={a.id}
className="rounded-lg border border-app-border bg-app-surface p-3 shadow-lg"
>
<div className="mb-1 flex items-center gap-2">
<span className="h-2.5 w-2.5 shrink-0 rounded-full" style={{ backgroundColor: a.color }} />
<span className="truncate text-xs font-medium text-app-fg">{a.email}</span>
</div>
<div className="mb-1.5 text-xs text-app-muted">
{t("backfill.importing", {
done: done.toLocaleString(),
total: a.backfill_total.toLocaleString(),
})}
</div>
<div className="h-1.5 w-full overflow-hidden rounded-full bg-app-hover">
<div
className="h-full rounded-full bg-app-accent transition-all duration-500"
style={{ width: `${pct}%` }}
/>
</div>
</div>
);
})}
</div>
);
}
+12 -2
View File
@@ -15,7 +15,9 @@ import { Attachments } from "./Attachments";
export function MessageBody({ message }: { message: MessageEnvelope }) {
const t = useT();
const queryClient = useQueryClient();
const [allowed, setAllowed] = useState(false);
// Seed from the server's trusted-sender flag so an already-approved sender
// loads images and shows no "blocked" prompt on reopen.
const [allowed, setAllowed] = useState(message.images_allowed);
const frameRef = useRef<HTMLIFrameElement>(null);
const observerRef = useRef<ResizeObserver | null>(null);
const [height, setHeight] = useState(80);
@@ -23,7 +25,10 @@ export function MessageBody({ message }: { message: MessageEnvelope }) {
const { data, isLoading, isError } = useQuery({
queryKey: ["message-html", message.id, allowed],
queryFn: async () => {
const resp = await fetch(`/api/messages/${message.id}/html`, { credentials: "include" });
// ?images=1 asks the server to render remote images for this one request
// (the "Load images" one-off); "Always from this sender" persists separately.
const url = `/api/messages/${message.id}/html${allowed ? "?images=1" : ""}`;
const resp = await fetch(url, { credentials: "include" });
if (!resp.ok) throw new Error("failed to load message");
return resp.text();
},
@@ -89,6 +94,11 @@ export function MessageBody({ message }: { message: MessageEnvelope }) {
if (persist) {
await api.allowRemoteImages(message.id);
queryClient.invalidateQueries({ queryKey: ["message-html", message.id] });
// Refresh thread detail so the sender's images_allowed flag reflects the
// new trust on reopen (suppresses the prompt next time).
if (message.thread_id) {
queryClient.invalidateQueries({ queryKey: ["thread", message.thread_id] });
}
}
setAllowed(true);
};
+14
View File
@@ -49,6 +49,9 @@ export const en = {
"nav.spam": "Spam",
"nav.trash": "Trash",
"nav.search": "Search mail…",
"search.save": "Save this search",
"search.saved": "Saved searches",
"search.saveNamePrompt": "Name this saved search:",
"nav.storageQuota": "Storage used",
"nav.refresh": "Refresh all folders",
@@ -176,11 +179,22 @@ export const en = {
"wizard.startReading": "Start reading now",
"wizard.manualConfig": "Enter settings manually",
"wizard.displayName": "Display name",
"wizard.syncWindow": "How much mail to sync",
"wizard.syncWindow.all": "All mail",
"wizard.syncWindow.all.hint": "Imports the last 30 days first, then the rest in the background.",
"wizard.syncWindow.3months": "Last 3 months only",
"wizard.syncWindow.3months.hint": "Older mail stays on the server until you download it.",
"wizard.syncWindow.new": "New mail only",
"wizard.syncWindow.new.hint":
"Nothing from before now — download older mail anytime from account settings.",
"settings.title": "Settings",
"settings.profile": "Profile",
"settings.security": "Security",
"settings.accounts": "Accounts",
"settings.downloadAllMail": "Sync remaining mail",
"settings.syncedPartially": "Synced partially",
"backfill.importing": "Importing messages… {done} of {total}",
"settings.appearance": "Appearance",
"settings.handle": "Handle",
"settings.recoveryEmail": "Recovery email",
+22 -2
View File
@@ -13,10 +13,12 @@ import type {
QuotaOut,
RecipientSuggestion,
RemoteDraftOut,
SavedSearch,
SearchOut,
SendResponse,
ServerConfig,
SessionInfo,
SyncWindow,
ThreadDetail,
ThreadListOut,
TotpSetup,
@@ -90,9 +92,15 @@ export const api = {
mailboxes: () => apiFetch<MailboxOut[]>("/mailboxes"),
syncNow: () => apiFetch<{ status: string }>("/accounts/sync", { method: "POST" }),
threads: (view = "unified", cursor?: string) => {
threads: (
view = "unified",
opts?: { offset?: number; limit?: number; filter?: string; order?: "asc" | "desc" },
) => {
const params = new URLSearchParams({ view });
if (cursor) params.set("cursor", cursor);
if (opts?.offset) params.set("offset", String(opts.offset));
if (opts?.limit) params.set("limit", String(opts.limit));
if (opts?.filter && opts.filter !== "all") params.set("filter", opts.filter);
if (opts?.order) params.set("order", opts.order);
return apiFetch<ThreadListOut>(`/threads?${params.toString()}`);
},
thread: (id: string, sort: "asc" | "desc" = "asc", view = "unified") =>
@@ -121,6 +129,7 @@ export const api = {
imap: ServerConfig;
smtp: ServerConfig;
password: string;
sync_window?: SyncWindow;
}) => apiFetch<AccountDetail>("/accounts", { method: "POST", body: JSON.stringify(input) }),
accountDetail: (id: string) => apiFetch<AccountDetail>(`/accounts/${id}`),
updateAccount: (
@@ -129,6 +138,7 @@ export const api = {
display_name?: string;
color?: string;
signature?: string;
sync_window?: SyncWindow;
imap?: ServerConfig;
smtp?: ServerConfig;
password?: string;
@@ -159,6 +169,16 @@ export const api = {
quota: () => apiFetch<QuotaOut>("/settings/quota"),
folderCounts: () => apiFetch<FolderCounts>("/mailboxes/counts"),
// --- saved searches (virtual mailboxes) ---
savedSearches: () => apiFetch<SavedSearch[]>("/saved-searches"),
createSavedSearch: (name: string, query: string) =>
apiFetch<SavedSearch>("/saved-searches", {
method: "POST",
body: JSON.stringify({ name, query }),
}),
deleteSavedSearch: (id: string) =>
apiFetch<{ status: string }>(`/saved-searches/${id}`, { method: "DELETE" }),
// --- compose ---
suggestRecipients: (q: string) =>
apiFetch<RecipientSuggestion[]>(`/recipients/suggest?q=${encodeURIComponent(q)}`),
+14
View File
@@ -30,6 +30,7 @@ export interface ThreadSummary {
export interface ThreadListOut {
threads: ThreadSummary[];
cursor: string | null;
total: number;
}
export interface AttachmentOut {
@@ -55,6 +56,7 @@ export interface MessageEnvelope {
is_starred: boolean;
has_attachments: boolean;
has_remote_images: boolean;
images_allowed: boolean;
size_bytes: number;
snippet: string;
also_in_account_ids: string[];
@@ -68,6 +70,8 @@ export interface ThreadDetail {
messages: MessageEnvelope[];
}
export type SyncWindow = "all" | "3months" | "new";
export interface AccountOut {
id: string;
email: string;
@@ -75,6 +79,9 @@ export interface AccountOut {
color: string;
state: string;
signature: string;
sync_window: SyncWindow;
backfill_total: number;
backfill_done: number;
}
export interface MailboxOut {
@@ -157,6 +164,13 @@ export interface FolderCounts {
drafts: number;
}
export interface SavedSearch {
id: string;
name: string;
query: string;
pinned: boolean;
}
// --- M1: compose -------------------------------------------------------------
export interface RecipientSuggestion {
+4
View File
@@ -48,6 +48,10 @@ export function connectWs(queryClient: QueryClient): () => void {
queryClient.invalidateQueries({ queryKey: ["quota"] });
queryClient.invalidateQueries({ queryKey: ["drafts-remote"] });
queryClient.invalidateQueries({ queryKey: ["folder-counts"] });
// Backfill progress counters live on the account row (drives the banner).
if (name === "sync.progress") {
queryClient.invalidateQueries({ queryKey: ["accounts"] });
}
if (typeof payload.thread_id === "string") {
queryClient.invalidateQueries({ queryKey: ["thread", payload.thread_id] });
}
+175 -47
View File
@@ -1,4 +1,4 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { useEffect, useRef, useState } from "react";
import { Link, useMatch, useNavigate } from "react-router-dom";
@@ -9,11 +9,12 @@ import { MessageList } from "../components/list/MessageList";
import { Resizer } from "../components/Resizer";
import { ThemeToggle } from "../components/ThemeToggle";
import { EmptyIllustration } from "../components/EmptyIllustration";
import { BackfillBanner } from "../components/BackfillBanner";
import { ThreadView } from "../components/thread/ThreadView";
import { useT } from "../i18n";
import type { MessageKey } from "../i18n/en";
import { api } from "../lib/api";
import type { QuotaOut, ThreadSummary } from "../lib/types";
import type { QuotaOut, SavedSearch, ThreadSummary } from "../lib/types";
import { useKeyboardShortcuts } from "../lib/useKeyboardShortcuts";
function quotaLabel(q: QuotaOut): string {
@@ -48,6 +49,13 @@ export function MailLayout() {
const threadId = match?.params.threadId;
const [view, setView] = useState<string>("folder:inbox");
const [term, setTerm] = useState("");
// Debounced copy of the search box for search-as-you-type (avoids a request per key).
const [debouncedTerm, setDebouncedTerm] = useState("");
useEffect(() => {
const h = window.setTimeout(() => setDebouncedTerm(term), 180);
return () => window.clearTimeout(h);
}, [term]);
const searching = debouncedTerm.trim().length > 0;
const [filter, setFilter] = useState<"all" | "unread" | "pinned">("all");
const [listOrder, setListOrder] = useState<"newest" | "oldest">(() =>
localStorage.getItem("mail.listOrder") === "oldest" ? "oldest" : "newest",
@@ -90,10 +98,13 @@ export function MailLayout() {
const currentFolder = view.startsWith("folder:") ? view.slice("folder:".length) : "inbox";
const accountsQuery = useQuery({ queryKey: ["accounts"], queryFn: api.accounts });
const mailboxesQuery = useQuery({ queryKey: ["mailboxes"], queryFn: api.mailboxes });
const orderParam: "asc" | "desc" = listOrder === "newest" ? "desc" : "asc";
const threadsQuery = useQuery({
queryKey: ["threads", view],
queryFn: () => api.threads(view),
enabled: !isDrafts,
queryKey: ["threads", view, filter, orderParam, page],
queryFn: () =>
api.threads(view, { offset: page * PAGE_SIZE, limit: PAGE_SIZE, filter, order: orderParam }),
enabled: !isDrafts && !searching,
placeholderData: keepPreviousData, // keep the current page visible while the next loads
});
const draftsQuery = useQuery({
queryKey: ["drafts"],
@@ -106,33 +117,36 @@ export function MailLayout() {
enabled: isDrafts,
});
const searchQuery = useQuery({
queryKey: ["search", term],
queryFn: () => api.search(term),
enabled: term.trim().length > 0,
queryKey: ["search", debouncedTerm],
queryFn: () => api.search(debouncedTerm),
enabled: searching,
});
const savedSearchesQuery = useQuery({ queryKey: ["saved-searches"], queryFn: api.savedSearches });
const settingsQuery = useQuery({ queryKey: ["settings"], queryFn: api.settings });
const threadOrder: "asc" | "desc" = settingsQuery.data?.thread_order ?? "asc";
const quotaQuery = useQuery({ queryKey: ["quota"], queryFn: api.quota });
const countsQuery = useQuery({ queryKey: ["folder-counts"], queryFn: api.folderCounts });
const allThreads: ThreadSummary[] = term.trim()
? (searchQuery.data?.threads ?? []).map((h) => h.thread)
: (threadsQuery.data?.threads ?? []);
const filtered = allThreads.filter((th) =>
filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true,
);
// The backend returns threads newest-first; reverse for an oldest-first view.
const threads = listOrder === "newest" ? filtered : [...filtered].reverse();
// Client-side pagination of the (filtered, ordered) list.
const total = threads.length;
const maxPage = Math.max(0, Math.ceil(total / PAGE_SIZE) - 1);
const curPage = Math.min(page, maxPage);
const pageStart = curPage * PAGE_SIZE;
const pageEnd = Math.min(pageStart + PAGE_SIZE, total);
const pageThreads = threads.slice(pageStart, pageEnd);
// Reset to the first page whenever the underlying list identity changes.
useEffect(() => setPage(0), [view, filter, term, listOrder]);
// Folder/inbox views paginate server-side with an accurate grand total; search
// results (a single fetched page of hits) still paginate client-side.
let pageThreads: ThreadSummary[];
let total: number;
if (searching) {
const hits = (searchQuery.data?.threads ?? []).map((h) => h.thread);
const f = hits.filter((th) =>
filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true,
);
const ordered = listOrder === "newest" ? f : [...f].reverse();
total = ordered.length;
pageThreads = ordered.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE);
} else {
pageThreads = threadsQuery.data?.threads ?? [];
total = threadsQuery.data?.total ?? 0;
}
const pageStart = page * PAGE_SIZE;
const pageEnd = pageStart + pageThreads.length;
// Reset to the first page whenever the list identity (not the page) changes.
useEffect(() => setPage(0), [view, filter, debouncedTerm, listOrder]);
// Master "select all" state over the currently listed threads.
const listedSelectedCount = pageThreads.filter((th) => checkedIds.has(th.id)).length;
@@ -141,8 +155,9 @@ export function MailLayout() {
const toggleSelectAll = () =>
setCheckedIds(allSelected ? new Set() : new Set(pageThreads.map((th) => th.id)));
const threadsRef = useRef<ThreadSummary[]>(threads);
threadsRef.current = threads;
// Keyboard next/prev navigates within the currently visible page.
const threadsRef = useRef<ThreadSummary[]>(pageThreads);
threadsRef.current = pageThreads;
const detailQuery = useQuery({
queryKey: ["thread", threadId, threadOrder, view],
@@ -157,6 +172,27 @@ export function MailLayout() {
queryClient.invalidateQueries({ queryKey: ["folder-counts"] });
},
});
// Saved searches (virtual mailboxes, §12 M2).
const savedSearches = savedSearchesQuery.data ?? [];
const saveSearch = useMutation({
mutationFn: (name: string) => api.createSavedSearch(name, term.trim()),
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["saved-searches"] }),
});
const deleteSaved = useMutation({
mutationFn: (id: string) => api.deleteSavedSearch(id),
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["saved-searches"] }),
});
const onSaveSearch = () => {
const q = term.trim();
if (!q) return;
const name = window.prompt(t("search.saveNamePrompt"), q);
if (name && name.trim()) saveSearch.mutate(name.trim());
};
const runSavedSearch = (s: SavedSearch) => {
setTerm(s.query);
navigate("/mail");
};
// Auto-mark an opened thread read after a short dwell (4s), not instantly — and
// cancel it if you leave before then, so a quick glance doesn't mark it read.
useEffect(() => {
@@ -359,6 +395,7 @@ export function MailLayout() {
return (
<div className="flex h-screen bg-app-bg text-app-fg">
<BackfillBanner />
{/* Sidebar */}
<aside
style={{ width: collapsed ? COLLAPSED_W : sidebarW }}
@@ -435,10 +472,50 @@ export function MailLayout() {
active={view === `account:${a.id}`}
onClick={() => { setView(`account:${a.id}`); navigate("/mail"); }}
>
<span className="h-2.5 w-2.5 shrink-0 rounded-full" style={{ backgroundColor: a.color }} />
<AccountDot
color={a.color}
syncing={a.state === "backfilling" && a.backfill_total > 0}
total={a.backfill_total}
done={a.backfill_done}
/>
<span className="truncate">{a.email}</span>
</SidebarItem>
))}
{savedSearches.length > 0 && (
<>
<div className="px-3 pb-1 pt-3 text-[12px] font-semibold uppercase tracking-wide text-app-faint">
{t("search.saved")}
</div>
{savedSearches.map((s) => (
<div key={s.id} className="group/saved relative">
<SidebarItem
active={searching && debouncedTerm === s.query}
onClick={() => runSavedSearch(s)}
>
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.4" strokeLinecap="round" strokeLinejoin="round" className="shrink-0">
<circle cx="7" cy="7" r="4.5" />
<path d="M10.5 10.5L14 14" />
</svg>
<span className="flex-1 truncate">{s.name}</span>
</SidebarItem>
<button
type="button"
title={t("common.delete")}
aria-label={t("common.delete")}
onClick={(e) => {
e.stopPropagation();
deleteSaved.mutate(s.id);
}}
className="absolute right-2 top-1/2 hidden -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-fg group-hover/saved:block"
>
<svg width="13" height="13" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round">
<path d="M4 4l8 8M12 4l-8 8" />
</svg>
</button>
</div>
))}
</>
)}
</>
)}
</nav>
@@ -489,26 +566,41 @@ export function MailLayout() {
<div className="relative w-full">
<input
ref={searchRef}
className="w-full rounded-lg border border-app-border bg-app-bg px-3 py-1.5 pr-8 text-[15px] text-app-fg placeholder:text-app-faint focus:border-app-accent focus:outline-none"
className={`w-full rounded-lg border border-app-border bg-app-bg px-3 py-1.5 text-[15px] text-app-fg placeholder:text-app-faint focus:border-app-accent focus:outline-none ${
term ? "pr-14" : "pr-8"
}`}
placeholder={t("nav.search")}
value={term}
onChange={(e) => setTerm(e.target.value)}
/>
{term && (
<button
type="button"
title={t("common.clear")}
aria-label={t("common.clear")}
onClick={() => {
setTerm("");
searchRef.current?.focus();
}}
className="absolute right-1.5 top-1/2 -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-fg"
>
<svg width="14" height="14" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round">
<path d="M4 4l8 8M12 4l-8 8" />
</svg>
</button>
<>
<button
type="button"
title={t("search.save")}
aria-label={t("search.save")}
onClick={onSaveSearch}
className="absolute right-8 top-1/2 -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-accent"
>
<svg width="15" height="15" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.4" strokeLinecap="round" strokeLinejoin="round">
<path d="M4 2.5h8v11l-4-2.6-4 2.6z" />
</svg>
</button>
<button
type="button"
title={t("common.clear")}
aria-label={t("common.clear")}
onClick={() => {
setTerm("");
searchRef.current?.focus();
}}
className="absolute right-1.5 top-1/2 -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-fg"
>
<svg width="14" height="14" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round">
<path d="M4 4l8 8M12 4l-8 8" />
</svg>
</button>
</>
)}
</div>
</div>
@@ -612,14 +704,14 @@ export function MailLayout() {
</span>
<div className="-mr-1 flex items-center gap-0.5">
<PagerButton
disabled={curPage === 0}
onClick={() => setPage(curPage - 1)}
disabled={page === 0}
onClick={() => setPage(page - 1)}
label={t("pager.prev")}
dir="left"
/>
<PagerButton
disabled={pageEnd >= total}
onClick={() => setPage(curPage + 1)}
onClick={() => setPage(page + 1)}
label={t("pager.next")}
dir="right"
/>
@@ -765,6 +857,42 @@ export function MailLayout() {
);
}
// Account bullet in the sidebar. While the account is backfilling it becomes a
// spinning ring that fills with the account colour to show import progress
// (SPEC §12 M2); once live it's the plain colour-filled dot.
function AccountDot({
color,
syncing,
total,
done,
}: {
color: string;
syncing: boolean;
total: number;
done: number;
}) {
if (!syncing) {
return (
<span className="h-2.5 w-2.5 shrink-0 rounded-full" style={{ backgroundColor: color }} />
);
}
const pct = total > 0 ? Math.min(100, Math.round((done / Math.max(done, total)) * 100)) : 0;
const ring = "rgba(128,128,128,0.35)";
return (
<span
className="h-3.5 w-3.5 shrink-0 animate-spin rounded-full"
style={{
animationDuration: "1.1s",
background: `conic-gradient(${color} ${pct}%, ${ring} ${pct}% 100%)`,
// Punch out the centre to leave a ~3px ring.
WebkitMask: "radial-gradient(farthest-side, transparent calc(100% - 3px), #000 0)",
mask: "radial-gradient(farthest-side, transparent calc(100% - 3px), #000 0)",
}}
title={`${pct}%`}
/>
);
}
function FilterTab({
active,
onClick,
+21
View File
@@ -225,6 +225,10 @@ function AccountsSection() {
mutationFn: (id: string) => api.deleteAccount(id),
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["accounts"] }),
});
const downloadAll = useMutation({
mutationFn: (id: string) => api.updateAccount(id, { sync_window: "all" }),
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["accounts"] }),
});
const close = () => {
setEditing(null);
@@ -257,8 +261,25 @@ function AccountsSection() {
<span className="text-xs text-app-faint">
{t("settings.syncState", { state: a.state })}
</span>
{a.sync_window !== "all" && (
<>
<span className="text-xs text-app-faint">·</span>
<span className="text-xs text-app-faint">
{t("settings.syncedPartially")}
</span>
</>
)}
</span>
<span className="flex items-center gap-3">
{a.sync_window !== "all" && (
<button
className="text-xs text-app-accent disabled:opacity-50"
disabled={downloadAll.isPending || a.state === "backfilling"}
onClick={() => downloadAll.mutate(a.id)}
>
{t("settings.downloadAllMail")}
</button>
)}
<button
className="text-xs text-app-accent"
onClick={() => setEditing({ id: a.id, mode: "signature" })}
+25 -1
View File
@@ -4,7 +4,7 @@ import { useNavigate } from "react-router-dom";
import { useT } from "../../i18n";
import { api, ApiError } from "../../lib/api";
import type { ServerConfig } from "../../lib/types";
import type { ServerConfig, SyncWindow } from "../../lib/types";
const PALETTE = ["#2563eb", "#16a34a", "#d4762a", "#9333ea", "#dc2626", "#0891b2"];
const EMPTY: ServerConfig = { host: "", port: 993, tls: "ssl", auth: "password" };
@@ -22,6 +22,7 @@ export function AccountWizard() {
const [imap, setImap] = useState<ServerConfig>(EMPTY);
const [smtp, setSmtp] = useState<ServerConfig>({ ...EMPTY, port: 465 });
const [source, setSource] = useState<string>("");
const [syncWindow, setSyncWindow] = useState<SyncWindow>("all");
const [busy, setBusy] = useState(false);
const [tested, setTested] = useState(false);
const [error, setError] = useState("");
@@ -72,6 +73,7 @@ export function AccountWizard() {
imap,
smtp,
password,
sync_window: syncWindow,
});
await queryClient.invalidateQueries({ queryKey: ["accounts"] });
setStep(3);
@@ -180,6 +182,28 @@ export function AccountWizard() {
/>
))}
</div>
<fieldset className="rounded-lg border border-app-border p-3">
<legend className="px-1 text-xs text-app-muted">{t("wizard.syncWindow")}</legend>
<div className="space-y-1">
{(["all", "3months", "new"] as const).map((w) => (
<label key={w} className="flex cursor-pointer items-start gap-2 text-sm">
<input
type="radio"
name="sync-window"
className="mt-1"
checked={syncWindow === w}
onChange={() => setSyncWindow(w)}
/>
<span>
<span className="font-medium">{t(`wizard.syncWindow.${w}`)}</span>
<span className="block text-xs text-app-faint">
{t(`wizard.syncWindow.${w}.hint`)}
</span>
</span>
</label>
))}
</div>
</fieldset>
{tested && <p className="text-sm text-emerald-500">{t("wizard.testOk")}</p>}
<div className="flex gap-2">
{!tested ? (
+1
View File
@@ -24,6 +24,7 @@ function msg(id: string, minutes: number, unread = false): MessageEnvelope {
is_starred: false,
has_attachments: false,
has_remote_images: false,
images_allowed: false,
size_bytes: 1024,
snippet: `snippet ${id}`,
also_in_account_ids: [],
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB