diff --git a/backend/app/accounts/schemas.py b/backend/app/accounts/schemas.py index cd282f2..2b8f686 100644 --- a/backend/app/accounts/schemas.py +++ b/backend/app/accounts/schemas.py @@ -3,9 +3,13 @@ from __future__ import annotations import uuid +from typing import Literal from pydantic import BaseModel, Field +# How much history to import at account setup (SPEC §12 M2). +SyncWindow = Literal["all", "3months", "new"] + class ServerConfigIn(BaseModel): host: str @@ -56,12 +60,14 @@ class CreateAccountRequest(BaseModel): smtp: ServerConfigIn username: str | None = None password: str = Field(min_length=1) + sync_window: SyncWindow = "all" class UpdateAccountRequest(BaseModel): display_name: str | None = None color: str | None = None signature: str | None = None + sync_window: SyncWindow | None = None # Connection/credential changes require the password (re-validated + re-encrypted). imap: ServerConfigIn | None = None smtp: ServerConfigIn | None = None @@ -76,5 +82,8 @@ class AccountDetailOut(BaseModel): state: str state_detail: str signature: str + sync_window: str + backfill_total: int + backfill_done: int imap_config: dict[str, object] smtp_config: dict[str, object] diff --git a/backend/app/accounts/service.py b/backend/app/accounts/service.py index ae7d50b..9bbaa33 100644 --- a/backend/app/accounts/service.py +++ b/backend/app/accounts/service.py @@ -39,6 +39,7 @@ async def create_account( secret_ct=encrypt_secret(req.password.encode("utf-8")), discovery_src="manual", state="backfilling", + sync_window=req.sync_window, ) session.add(account) try: diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 492a530..ef003a1 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -17,6 +17,7 @@ def register_routers(app: FastAPI) -> None: from app.api.mailboxes import router as mailboxes_router from app.api.messages import router as messages_router from app.api.recipients import router as recipients_router + from app.api.saved_searches import router as saved_searches_router from app.api.search import router as search_router from app.api.settings import router as settings_router from app.api.threads import router as threads_router @@ -34,6 +35,7 @@ def register_routers(app: FastAPI) -> None: recipients_router, settings_router, mailboxes_router, + saved_searches_router, ): app.include_router(router, prefix="/api") app.include_router(ws_router, prefix="/api") diff --git a/backend/app/api/accounts.py b/backend/app/api/accounts.py index 972ca73..5756fb2 100644 --- a/backend/app/api/accounts.py +++ b/backend/app/api/accounts.py @@ -82,6 +82,9 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]: color=a.color, state=a.state, signature=a.signature, + sync_window=a.sync_window, + backfill_total=a.backfill_total, + backfill_done=a.backfill_done, ) for a in accounts ] @@ -186,6 +189,18 @@ async def update_account( if body.signature is not None: account.signature = body.signature + # Broadening the sync window (e.g. "new" -> "all", "Download all mail") means + # importing history not fetched before, so re-run the backfill (SPEC §12 M2). + # persist_message is idempotent on (mailbox_id, uid), so already-synced mail + # is skipped. Narrowing keeps existing mail — nothing to fetch. + _COVERAGE = {"new": 0, "3months": 1, "all": 2} + broaden = False + if body.sync_window is not None and body.sync_window != account.sync_window: + broaden = _COVERAGE[body.sync_window] > _COVERAGE.get(account.sync_window, 0) + account.sync_window = body.sync_window + if broaden: + account.state = "backfilling" + # Connection settings and/or password: re-validate against the servers, then # re-encrypt the secret and re-sync. The password must be supplied (§10.2: the # stored secret is only decrypted in the worker, never here). @@ -224,7 +239,7 @@ async def update_account( await db.commit() - if conn_change: + if conn_change or broaden: await enqueue_job("sync_bootstrap_account", str(account.id)) return _detail(account) @@ -248,6 +263,9 @@ def _detail(account: Account) -> AccountDetailOut: state=account.state, state_detail=account.state_detail, signature=account.signature, + sync_window=account.sync_window, + backfill_total=account.backfill_total, + backfill_done=account.backfill_done, imap_config=account.imap_config, smtp_config=account.smtp_config, ) diff --git a/backend/app/api/messages.py b/backend/app/api/messages.py index 05b9ef5..4405075 100644 --- a/backend/app/api/messages.py +++ b/backend/app/api/messages.py @@ -65,14 +65,21 @@ def _sender_email(message: Message) -> str: @router.get("/{message_id}/html", response_class=HTMLResponse) -async def message_html(message_id: uuid.UUID, user: CurrentUser, db: DbSession) -> HTMLResponse: +async def message_html( + message_id: uuid.UUID, + user: CurrentUser, + db: DbSession, + images: bool = Query(False), +) -> HTMLResponse: """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8). Re-sanitizes from the raw blob when the sender is image-approved, or when the - stored copy predates the current sanitizer version (SPEC §10.4). + stored copy predates the current sanitizer version (SPEC §10.4). ``images=1`` + is a one-time "Load images" — render remote images for this request only, + without persisting the sender as trusted. """ message = await _owned_message(db, user.id, message_id) - allow_remote = _sender_email(message) in allowed_image_senders(user) + allow_remote = images or _sender_email(message) in allowed_image_senders(user) parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id))) html_part = next((p for p in parts if p.kind == "html_sanitized"), None) diff --git a/backend/app/api/saved_searches.py b/backend/app/api/saved_searches.py new file mode 100644 index 0000000..9f31c0c --- /dev/null +++ b/backend/app/api/saved_searches.py @@ -0,0 +1,82 @@ +"""Saved searches — stored queries shown as virtual mailboxes (SPEC §12 M2).""" + +from __future__ import annotations + +import uuid +from typing import Annotated + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import raise_api_error +from app.db import get_session +from app.identity.deps import CurrentUser, require_csrf +from app.models import SavedSearch + +router = APIRouter(prefix="/saved-searches", tags=["search"], dependencies=[Depends(require_csrf)]) + +DbSession = Annotated[AsyncSession, Depends(get_session)] + + +class SavedSearchOut(BaseModel): + id: uuid.UUID + name: str + query: str + pinned: bool + + +class SavedSearchIn(BaseModel): + name: str = Field(min_length=1, max_length=80) + query: str = Field(min_length=1, max_length=500) + pinned: bool = True + + +def _out(s: SavedSearch) -> SavedSearchOut: + return SavedSearchOut(id=s.id, name=s.name, query=s.query, pinned=s.pinned) + + +@router.get("", response_model=list[SavedSearchOut]) +async def list_saved_searches(user: CurrentUser, db: DbSession) -> list[SavedSearchOut]: + rows = await db.scalars( + select(SavedSearch).where(SavedSearch.user_id == user.id).order_by(SavedSearch.name) + ) + return [_out(s) for s in rows] + + +@router.post("", response_model=SavedSearchOut) +async def create_saved_search( + body: SavedSearchIn, user: CurrentUser, db: DbSession +) -> SavedSearchOut: + name = body.name.strip() + query = body.query.strip() + if not name or not query: + raise_api_error(400, "invalid", "name and query are required") + # Upsert by (user, name): re-saving under an existing name updates its query. + existing = await db.scalar( + select(SavedSearch).where(SavedSearch.user_id == user.id, SavedSearch.name == name) + ) + if existing is not None: + existing.query = query + existing.pinned = body.pinned + saved = existing + else: + saved = SavedSearch(user_id=user.id, name=name, query=query, pinned=body.pinned) + db.add(saved) + await db.commit() + return _out(saved) + + +@router.delete("/{search_id}") +async def delete_saved_search( + search_id: uuid.UUID, user: CurrentUser, db: DbSession +) -> dict[str, str]: + saved = await db.scalar( + select(SavedSearch).where(SavedSearch.id == search_id, SavedSearch.user_id == user.id) + ) + if saved is None: + raise_api_error(404, "not_found", "saved search not found") + await db.delete(saved) + await db.commit() + return {"status": "deleted"} diff --git a/backend/app/api/schemas.py b/backend/app/api/schemas.py index d6e1e6d..5ea461f 100644 --- a/backend/app/api/schemas.py +++ b/backend/app/api/schemas.py @@ -29,6 +29,7 @@ class ThreadSummary(BaseModel): class ThreadListOut(BaseModel): threads: list[ThreadSummary] cursor: str | None = None + total: int = 0 class AttachmentOut(BaseModel): @@ -54,6 +55,8 @@ class MessageEnvelope(BaseModel): is_starred: bool has_attachments: bool has_remote_images: bool + # The sender is on the user's trusted list, so remote images load without a prompt. + images_allowed: bool = False size_bytes: int snippet: str # Other accounts this same delivery also landed in (collapsed duplicates). @@ -84,6 +87,9 @@ class AccountOut(BaseModel): color: str state: str signature: str = "" + sync_window: str = "all" + backfill_total: int = 0 + backfill_done: int = 0 class SearchThreadHit(BaseModel): diff --git a/backend/app/api/threads.py b/backend/app/api/threads.py index 78a6333..019979b 100644 --- a/backend/app/api/threads.py +++ b/backend/app/api/threads.py @@ -3,13 +3,14 @@ from __future__ import annotations import uuid +from collections.abc import Collection from typing import Annotated from fastapi import APIRouter, Depends, Query -from sqlalchemy import Text, func, literal, or_, select, tuple_ +from sqlalchemy import Text, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql.elements import ColumnElement -from app.api.cursors import decode_cursor, encode_cursor from app.api.errors import raise_api_error from app.api.schemas import ( Address, @@ -18,6 +19,7 @@ from app.api.schemas import ( ThreadListOut, ThreadSummary, ) +from app.api.settings import allowed_image_senders from app.db import get_session from app.identity.deps import CurrentUser from app.models import Mailbox, Message, Thread, User @@ -48,7 +50,8 @@ def _summary(thread: Thread, message_count: int | None = None) -> ThreadSummary: ) -def _envelope(msg: Message) -> MessageEnvelope: +def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnvelope: + sender = msg.from_addr[0].get("email", "").lower() if msg.from_addr else "" return MessageEnvelope( id=msg.id, account_id=msg.account_id, @@ -65,6 +68,7 @@ def _envelope(msg: Message) -> MessageEnvelope: is_starred="\\Flagged" in msg.flags, has_attachments=msg.has_attachments, has_remote_images=msg.has_remote_images, + images_allowed=bool(sender) and sender in allowed_senders, size_bytes=msg.size_bytes, snippet=msg.snippet, ) @@ -75,10 +79,14 @@ async def list_threads( user: CurrentUser, db: DbSession, view: str = Query("unified"), - cursor: str | None = None, limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + filter: str = Query("all"), # all | unread | pinned + order: str = Query("desc"), # desc (newest first) | asc (oldest first) ) -> ThreadListOut: - stmt = select(Thread).where(Thread.user_id == user.id) + # Collect WHERE conditions once so the page query and the total-count query + # stay in lockstep (offset pagination needs an accurate grand total). + conditions: list[ColumnElement[bool]] = [Thread.user_id == user.id] if view.startswith("account:"): # A single account's inbox: threads with a message in THIS account's @@ -97,7 +105,7 @@ async def list_threads( Mailbox.special_use == "inbox", ) ) - stmt = stmt.where(Thread.id.in_(in_account_inbox)) + conditions.append(Thread.id.in_(in_account_inbox)) elif view.startswith("folder:"): # Threads with at least one message in a mailbox of this special_use, # across all of the user's accounts (unified per-folder view). @@ -107,24 +115,28 @@ async def list_threads( .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(Message.user_id == user.id, Mailbox.special_use == special) ) - stmt = stmt.where(Thread.id.in_(in_folder)) + conditions.append(Thread.id.in_(in_folder)) elif view != "unified": raise_api_error(400, "bad_view", "unsupported view") - decoded = decode_cursor(cursor) - if decoded is not None: - cur_date, cur_id = decoded - stmt = stmt.where( - tuple_(Thread.last_message_at, Thread.id) < tuple_(literal(cur_date), literal(cur_id)) - ) - stmt = stmt.order_by(Thread.last_message_at.desc(), Thread.id.desc()).limit(limit + 1) + if filter == "unread": + conditions.append(Thread.unread_count > 0) + elif filter == "pinned": + conditions.append(Thread.pinned.is_(True)) - threads = list(await db.scalars(stmt)) - has_more = len(threads) > limit - threads = threads[:limit] - next_cursor = ( - encode_cursor(threads[-1].last_message_at, threads[-1].id) if has_more and threads else None + total = ( + await db.scalar( + select(func.count()).select_from(select(Thread.id).where(*conditions).subquery()) + ) + ) or 0 + + order_by = ( + (Thread.last_message_at.asc(), Thread.id.asc()) + if order == "asc" + else (Thread.last_message_at.desc(), Thread.id.desc()) ) + stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit) + threads = list(await db.scalars(stmt)) # Per-thread visible message count matching what get_thread shows for this view: # Trash/Spam count only their own messages; every other view excludes them. This @@ -152,7 +164,7 @@ async def list_threads( } return ThreadListOut( - threads=[_summary(t, counts.get(t.id)) for t in threads], cursor=next_cursor + threads=[_summary(t, counts.get(t.id)) for t in threads], total=total ) @@ -216,9 +228,10 @@ async def get_thread( elif m.account_id != rep.account_id and m.account_id not in also[key]: also[key].append(m.account_id) + allowed_senders = allowed_image_senders(user) envelopes = [] for m in ordered: - env = _envelope(m) + env = _envelope(m, allowed_senders) env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"] envelopes.append(env) diff --git a/backend/app/models.py b/backend/app/models.py index a442272..44c3517 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -82,6 +82,11 @@ class Account(Base): discovery_src: Mapped[str] = mapped_column(Text, nullable=False) state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'backfilling'")) state_detail: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) + # How much history to import (SPEC §12 M2): all | 3months | new. Drives the + # initial backfill; backfill_total/backfill_done feed the progress UX. + sync_window: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'all'")) + backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True)) created_at: Mapped[dt.datetime] = _now() diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py index 1f7ffc4..5224d96 100644 --- a/backend/app/providers/base.py +++ b/backend/app/providers/base.py @@ -9,7 +9,7 @@ from __future__ import annotations from collections.abc import AsyncIterator from dataclasses import dataclass -from datetime import datetime +from datetime import date, datetime from typing import Protocol @@ -71,9 +71,25 @@ class MailProvider(Protocol): async def list_mailboxes(self) -> list[RemoteMailbox]: ... def backfill( - self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200 + self, + mailbox: str, + *, + newest_first: bool = True, + batch_size: int = 200, + since: date | None = None, + before: date | None = None, ) -> AsyncIterator[list[RemoteMessage]]: - """Yields batches until the mailbox is exhausted. Headers+body in one pass.""" + """Yields batches until the mailbox is exhausted. Headers+body in one pass. + ``since``/``before`` bound the import by internal date (SPEC §12 M2).""" + ... + + async def message_count(self, mailbox: str, *, since: date | None = None) -> int: + """Number of messages to import (progress denominator); whole mailbox or + the ``since`` window.""" + ... + + async def newest_uid(self, mailbox: str) -> int: + """Current high-water UID; used by the ``new`` window to skip history.""" ... async def poll( diff --git a/backend/app/providers/imap_smtp/provider.py b/backend/app/providers/imap_smtp/provider.py index ba1660b..8c696ff 100644 --- a/backend/app/providers/imap_smtp/provider.py +++ b/backend/app/providers/imap_smtp/provider.py @@ -13,6 +13,7 @@ import contextlib import re from collections.abc import AsyncIterator from dataclasses import dataclass +from datetime import date import aiosmtplib from aioimaplib import aioimaplib @@ -42,6 +43,21 @@ _CONNECT_TIMEOUT = 20 _PER_ACCOUNT_CONN_CAP = 4 # SPEC §6.3 / §4: IDLE + transient + short-lived jobs _APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)") _COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)") +_IMAP_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") + + +def _imap_date(d: date) -> str: + """RFC 3501 search date, e.g. ``5-Jul-2026`` (locale-independent).""" + return f"{d.day}-{_IMAP_MONTHS[d.month - 1]}-{d.year}" + + +def _date_criteria(since: date | None, before: date | None) -> str: + parts: list[str] = [] + if since is not None: + parts.append(f"SINCE {_imap_date(since)}") + if before is not None: + parts.append(f"BEFORE {_imap_date(before)}") + return " ".join(parts) if parts else "ALL" @dataclass(frozen=True) @@ -184,12 +200,21 @@ class ImapSmtpProvider(MailProvider): return 0 async def backfill( - self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200 + self, + mailbox: str, + *, + newest_first: bool = True, + batch_size: int = 200, + since: date | None = None, + before: date | None = None, ) -> AsyncIterator[list[RemoteMessage]]: + """Yield message batches. ``since``/``before`` bound the import by IMAP + internal date (``SINCE`` inclusive, ``BEFORE`` exclusive) so the caller + can sync a recent window and/or the older remainder (SPEC §12 M2).""" info = await self._select(mailbox) self._uidvalidity_seen[mailbox] = info.get("uidvalidity", 0) imap = await self._connect() - search = await imap.uid_search("ALL") + search = await imap.uid_search(_date_criteria(since, before)) uids = parse_search_uids(search.lines) uids.sort(reverse=newest_first) for start in range(0, len(uids), batch_size): @@ -198,6 +223,22 @@ class ImapSmtpProvider(MailProvider): if messages: yield messages + async def message_count(self, mailbox: str, *, since: date | None = None) -> int: + """Number of messages to import for the progress denominator. Uses the + cheap ``EXISTS`` from SELECT for the whole mailbox, or a dated SEARCH.""" + info = await self._select(mailbox) + if since is None: + return int(info.get("exists", 0)) + imap = await self._connect() + search = await imap.uid_search(_date_criteria(since, None)) + return len(parse_search_uids(search.lines)) + + async def newest_uid(self, mailbox: str) -> int: + """Current high-water UID (``UIDNEXT - 1``); 0 for an empty mailbox. Used + by the ``new`` sync window to skip history and only pull future mail.""" + info = await self._select(mailbox) + return max(0, int(info.get("uidnext", 1)) - 1) + async def list_uids(self, mailbox: str) -> set[int]: """Full set of UIDs currently present in the mailbox (expunge reconciliation). diff --git a/backend/app/search/factory.py b/backend/app/search/factory.py index 66e50f0..2899abd 100644 --- a/backend/app/search/factory.py +++ b/backend/app/search/factory.py @@ -10,12 +10,14 @@ _backend: SearchBackend | None = None def get_search_backend() -> SearchBackend: - """Return the process-wide SearchBackend (``pg`` in M0-M1, ``meili`` in M2).""" + """Return the process-wide SearchBackend (``pg`` or ``meili`` per SEARCH_BACKEND).""" global _backend if _backend is None: which = get_settings().search_backend if which == "meili": - # TODO(spec): Meilisearch backend lands in M2. - raise NotImplementedError("meili backend is implemented in M2") - _backend = PgSearchBackend() + from app.search.meili import MeiliSearchBackend + + _backend = MeiliSearchBackend() + else: + _backend = PgSearchBackend() return _backend diff --git a/backend/app/search/meili.py b/backend/app/search/meili.py new file mode 100644 index 0000000..0b044e4 --- /dev/null +++ b/backend/app/search/meili.py @@ -0,0 +1,267 @@ +# INTERFACE VERSION: 1 +"""Meilisearch SearchBackend (SPEC §5.2, §8; M2). + +Typo-tolerant free-text search with highlighted snippets over a single ``messages`` +index. Every query is filtered by ``user_id`` (invariant I-7). Structured §8.2 +operators that map to filterable attributes — ``is`` / ``has`` / ``in`` / date / +size — become Meilisearch filters; the field operators (``from`` / ``to`` / ``cc`` +/ ``subject`` / ``filename``) fold into the query text, since Meilisearch 1.8 has +no substring filter (the PG backend remains available for exact operator queries). +""" + +from __future__ import annotations + +import asyncio +import base64 +import datetime as dt +import re +from typing import Any +from uuid import UUID + +from meilisearch_python_sdk import AsyncClient + +from app.config import get_settings +from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery + +_INDEX = "messages" + +# Priority order matters: subject/senders rank above the body. +_SEARCHABLE = [ + "subject", + "from_names", + "from_emails", + "to_names", + "to_emails", + "cc_names", + "cc_emails", + "body", + "attachment_names", +] +_FILTERABLE = [ + "user_id", + "account_id", + "special", + "has_attachments", + "is_unread", + "is_starred", + "date", + "size_bytes", +] +_TEXT_FIELD_OPS = {"from", "to", "cc", "subject", "filename"} +_RELATIVE_DAYS = {"d": 1, "w": 7, "m": 30, "y": 365} + + +def _emails(field: list[dict[str, str]]) -> list[str]: + return [(a.get("email") or "").lower() for a in field if a.get("email")] + + +def _names(field: list[dict[str, str]]) -> list[str]: + return [a.get("name") or "" for a in field if a.get("name")] + + +def _to_doc(doc: SearchDoc) -> dict[str, Any]: + return { + "id": str(doc.message_id), + "thread_id": str(doc.thread_id), + "user_id": str(doc.user_id), + "account_id": str(doc.account_id), + "special": doc.mailbox_special_use or "", + "subject": doc.subject, + "body": doc.body_text, + "from_names": _names(doc.from_), + "from_emails": _emails(doc.from_), + "to_names": _names(doc.to), + "to_emails": _emails(doc.to), + "cc_names": _names(doc.cc), + "cc_emails": _emails(doc.cc), + "attachment_names": doc.attachment_names, + "has_attachments": doc.has_attachments, + "is_unread": doc.is_unread, + "is_starred": doc.is_starred, + "date": doc.date, + "size_bytes": doc.size_bytes, + } + + +def _parse_date_unix(value: str) -> int | None: + try: + d = dt.datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=dt.UTC) + except ValueError: + return None + return int(d.timestamp()) + + +def _parse_relative_days(value: str) -> int | None: + m = re.fullmatch(r"(\d+)([dwmy])", value.strip().lower()) + return int(m.group(1)) * _RELATIVE_DAYS[m.group(2)] if m else None + + +def _parse_size(value: str) -> int | None: + m = re.fullmatch(r"(\d+)([km])", value.strip().lower()) + if not m: + return None + return int(m.group(1)) * (1024 if m.group(2) == "k" else 1024 * 1024) + + +def _op_filter(key: str, value: str) -> str | None: + """A single §8.2 operator -> a Meilisearch filter fragment (or None).""" + now = int(dt.datetime.now(dt.UTC).timestamp()) + if key == "in": + return f'special = "{value.lower()}"' + if key == "is": + v = value.lower() + if v == "unread": + return "is_unread = true" + if v == "read": + return "is_unread = false" + if v == "starred": + return "is_starred = true" + return None # 'answered' is not indexed in SearchDoc + if key == "has": + return "has_attachments = true" if value.lower() == "attachment" else None + if key == "before": + u = _parse_date_unix(value) + return f"date < {u}" if u is not None else None + if key == "after": + u = _parse_date_unix(value) + return f"date >= {u}" if u is not None else None + if key == "newer_than": + days = _parse_relative_days(value) + return f"date >= {now - days * 86400}" if days is not None else None + if key == "older_than": + days = _parse_relative_days(value) + return f"date < {now - days * 86400}" if days is not None else None + if key == "larger": + n = _parse_size(value) + return f"size_bytes > {n}" if n is not None else None + if key == "smaller": + n = _parse_size(value) + return f"size_bytes < {n}" if n is not None else None + return None + + +def _build_filter(groups: list[list[dict[str, Any]]], user_id: UUID) -> str: + ands = [f'user_id = "{user_id}"'] + for group in groups: + ors: list[str] = [] + for clause in group: + if clause.get("type") == "text": + continue + frag = _op_filter(str(clause.get("key")), str(clause.get("value", ""))) + if frag is None: + continue + ors.append(f"NOT ({frag})" if clause.get("negate") else frag) + if ors: + ands.append("(" + " OR ".join(ors) + ")") + return " AND ".join(ands) + + +def _query_text(groups: list[list[dict[str, Any]]]) -> str: + """Free text for Meili's `q`: text terms (negated as -term) + folded field ops.""" + terms: list[str] = [] + for group in groups: + for clause in group: + value = str(clause.get("value", "")).strip() + if not value: + continue + if clause.get("type") == "text": + terms.append(f"-{value}" if clause.get("negate") else value) + elif clause.get("key") in _TEXT_FIELD_OPS and not clause.get("negate"): + terms.append(value) + return " ".join(terms).strip() + + +class MeiliSearchBackend(SearchBackend): + def __init__(self) -> None: + cfg = get_settings() + self._client = AsyncClient(cfg.meili_url, cfg.meili_key or None) + self._index = self._client.index(_INDEX) + self._ready = False + self._lock = asyncio.Lock() + + async def _ensure(self) -> None: + """Create the index and apply settings once (idempotent).""" + if self._ready: + return + async with self._lock: + if self._ready: + return + indexes = await self._client.get_indexes() + names = {i.uid for i in indexes} if indexes else set() + if _INDEX not in names: + await self._client.create_index(_INDEX, primary_key="id") + await self._index.update_searchable_attributes(_SEARCHABLE) + await self._index.update_sortable_attributes(["date"]) + task = await self._index.update_filterable_attributes(_FILTERABLE) + await self._client.wait_for_task(task.task_uid) + self._ready = True + + async def index(self, doc: SearchDoc) -> None: + await self._ensure() + await self._index.add_documents([_to_doc(doc)]) + + async def delete(self, message_id: UUID) -> None: + await self._ensure() + await self._index.delete_document(str(message_id)) + + async def delete_user(self, user_id: UUID) -> None: + await self._ensure() + await self._index.delete_documents_by_filter(f'user_id = "{user_id}"') + + async def query( + self, q: SearchQuery, *, limit: int, cursor: str | None + ) -> tuple[list[SearchHit], str | None]: + await self._ensure() + groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else [] + assert isinstance(groups, list) + offset = _decode_cursor(cursor) + res = await self._index.search( + _query_text(groups), + filter=_build_filter(groups, q.user_id), + offset=offset, + limit=limit + 1, + sort=["date:desc"], # stable order for offset paging; typo matching still applies + attributes_to_retrieve=["id", "thread_id"], + attributes_to_highlight=["subject", "body"], + attributes_to_crop=["body"], + crop_length=30, + highlight_pre_tag="", + highlight_post_tag="", + show_ranking_score=True, + ) + rows = res.hits + has_more = len(rows) > limit + rows = rows[:limit] + hits = [_to_hit(row) for row in rows] + next_cursor = _encode_cursor(offset + limit) if has_more else None + return hits, next_cursor + + +def _to_hit(row: dict[str, Any]) -> SearchHit: + fmt = row.get("_formatted") or {} + highlights: dict[str, str] = {} + subject = fmt.get("subject") or "" + if "" in subject: + highlights["subject"] = subject + body = fmt.get("body") or "" + if "" in body: + highlights["body"] = body + return SearchHit( + message_id=UUID(row["id"]), + thread_id=UUID(row["thread_id"]), + score=float(row.get("_rankingScore") or 0.0), + highlights=highlights, + ) + + +def _encode_cursor(offset: int) -> str: + return base64.urlsafe_b64encode(str(offset).encode()).decode() + + +def _decode_cursor(cursor: str | None) -> int: + if not cursor: + return 0 + try: + return max(0, int(base64.urlsafe_b64decode(cursor.encode()).decode())) + except (ValueError, TypeError): + return 0 diff --git a/backend/app/search/pg.py b/backend/app/search/pg.py index 7c3805f..9b52bc5 100644 --- a/backend/app/search/pg.py +++ b/backend/app/search/pg.py @@ -23,6 +23,22 @@ from app.models import Attachment, Mailbox, Message from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery _TS_CONFIG = "simple" # language-neutral; avoids wrong-language stemming (M0) + + +def _prefix_tsquery(value: str) -> ColumnElement[object]: + """A prefix ``tsquery`` for search-as-you-type: ``invo`` finds ``invoice``. + + We can't tokenize in Python and match the index: Postgres' parser keeps an + email/host/URL as a *single* lexeme (``lukas.remis@gmail.com``), so splitting + on word chars would never match it. Instead we re-parse the term with the + same ``plainto_tsquery`` used at index time, then append ``:*`` to each + lexeme to make it a prefix. ``plainto_tsquery`` strips tsquery operators and + quotes, so this is injection- and error-safe on hostile input; an all-symbol + term yields the empty query (matches nothing). + """ + lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text) + prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g") + return func.to_tsquery(_TS_CONFIG, prefixed) _RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365} @@ -95,7 +111,10 @@ class PgSearchBackend(SearchBackend): def _clause_condition(self, clause: dict[str, object]) -> ColumnElement[bool] | None: value = str(clause.get("value", "")) if clause.get("type") == "text": - return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value)) + if clause.get("phrase"): + # Quoted phrase: keep exact-adjacency semantics, no prefix. + return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value)) + return Message.body_indexed.op("@@")(_prefix_tsquery(value)) key = clause.get("key") cond = self._filter_condition(str(key), value) return cond @@ -172,7 +191,7 @@ class PgSearchBackend(SearchBackend): conditions = self._build_conditions(groups) rank = ( - func.ts_rank_cd(Message.body_indexed, func.websearch_to_tsquery(_TS_CONFIG, q.text)) + func.ts_rank_cd(Message.body_indexed, _prefix_tsquery(q.text)) if q.text else literal(0.0) ) diff --git a/backend/app/sync/engine.py b/backend/app/sync/engine.py index d0e6135..f1c3772 100644 --- a/backend/app/sync/engine.py +++ b/backend/app/sync/engine.py @@ -7,6 +7,8 @@ import contextlib import logging import random import uuid +from dataclasses import dataclass, field +from datetime import date, timedelta from typing import Any from sqlalchemy import delete, select, update @@ -27,6 +29,11 @@ logger = logging.getLogger(__name__) # Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3). _PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2} +# Sync-window import strategy (SPEC §12 M2). "all" imports the most recent +# _RECENT_DAYS first (so the account is usable fast), then the older remainder. +_RECENT_DAYS = 30 +_WINDOW_DAYS = {"3months": 90} + async def _load_account(session: AsyncSession, account_id: uuid.UUID) -> Account: account = await session.get(Account, account_id) @@ -97,9 +104,8 @@ def _mailbox_order(mailboxes: list[Mailbox]) -> list[Mailbox]: return sorted(mailboxes, key=lambda m: _PRIORITY.get(m.special_use or "", 99)) -async def _emit_batch( - results: list[PersistResult], account: Account, mailbox_path: str, done: int -) -> None: +async def _index_and_notify(results: list[PersistResult], account: Account) -> None: + """Index each persisted message and publish its ``message.new`` event.""" backend = get_search_backend() bus = get_event_bus() for r in results: @@ -113,7 +119,13 @@ async def _emit_batch( "thread_id": str(r.message.thread_id), }, ) - await bus.publish( + + +async def _emit_batch( + results: list[PersistResult], account: Account, mailbox_path: str, done: int +) -> None: + await _index_and_notify(results, account) + await get_event_bus().publish( "sync.progress", { "user_id": str(account.user_id), @@ -124,14 +136,61 @@ async def _emit_batch( ) -async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox: Mailbox) -> int: - """Backfill one mailbox newest-first, indexing inline per batch (FR-30).""" +@dataclass +class _BackfillProgress: + """Account-level backfill counter: persists ``backfill_done`` and emits + ``sync.progress`` with the running total so the UI can show a progress bar.""" + + account: Account + total: int + done: int = 0 + _mailboxes: set[str] = field(default_factory=set) + + async def record(self, results: list[PersistResult], mailbox_path: str) -> None: + from app.db import session_scope + + await _index_and_notify(results, self.account) + self.done += len(results) + self._mailboxes.add(mailbox_path) + async with session_scope() as session: + await session.execute( + update(Account) + .where(Account.id == self.account.id) + .values(backfill_done=self.done) + ) + await get_event_bus().publish( + "sync.progress", + { + "user_id": str(self.account.user_id), + "account_id": str(self.account.id), + "mailbox": mailbox_path, + "done": self.done, + "total": self.total, + }, + ) + + +async def backfill_mailbox( + provider: ImapSmtpProvider, + account: Account, + mailbox: Mailbox, + progress: _BackfillProgress | None = None, + *, + since: date | None = None, + before: date | None = None, +) -> int: + """Backfill one mailbox newest-first, indexing inline per batch (FR-30). + + ``since``/``before`` bound the import by date (sync window, SPEC §12 M2). + When a ``progress`` tracker is given, batches count toward the account-level + backfill total; otherwise each batch emits a standalone ``sync.progress``. + """ from app.db import session_scope account_id, user_id, mailbox_id = account.id, account.user_id, mailbox.id special_use, path = mailbox.special_use, mailbox.path done = 0 - async for batch in provider.backfill(path, batch_size=200): + async for batch in provider.backfill(path, batch_size=200, since=since, before=before): results: list[PersistResult] = [] max_uid = 0 async with session_scope() as session: @@ -153,12 +212,41 @@ async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox .values(last_seen_uid=max_uid) ) done += len(results) - await _emit_batch(results, account, path, done) + if progress is not None: + await progress.record(results, path) + else: + await _emit_batch(results, account, path, done) return done +async def _set_backfill_counts(account_id: uuid.UUID, *, total: int, done: int) -> None: + from app.db import session_scope + + async with session_scope() as session: + await session.execute( + update(Account) + .where(Account.id == account_id) + .values(backfill_total=total, backfill_done=done) + ) + + +async def _sync_new_only(provider: ImapSmtpProvider, mailboxes: list[Mailbox]) -> None: + """``new`` window: import no history — just move each mailbox watermark to the + current tip so only mail arriving after setup is pulled (SPEC §12 M2).""" + from app.db import session_scope + + for mailbox in mailboxes: + tip = await provider.newest_uid(mailbox.path) + async with session_scope() as session: + await session.execute( + update(Mailbox) + .where(Mailbox.id == mailbox.id, Mailbox.last_seen_uid < tip) + .values(last_seen_uid=tip) + ) + + async def bootstrap_and_backfill(account_id: uuid.UUID) -> int: - """Full initial sync for an account: mailboxes + backfill; sets state=live.""" + """Full initial sync for an account: mailboxes + windowed backfill; state=live.""" from app.db import session_scope async with session_scope() as session: @@ -167,12 +255,42 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int: mailboxes = _mailbox_order(await bootstrap_mailboxes(session, provider, account)) # Detach: reuse account/mailboxes values after the session closes. account_obj = account + window = account.sync_window mailbox_list = list(mailboxes) - total = 0 try: + if window == "new": + # No backfill: set watermarks to the tip, go live immediately. + await _sync_new_only(provider, mailbox_list) + await _set_backfill_counts(account_id, total=0, done=0) + await _set_state(account_id, "live") + return 0 + + since = ( + date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None + ) + # Progress denominator: count what we're about to import up front so the + # bar has a stable total (SPEC §12 M2 backfill-progress UX). + total = 0 for mailbox in mailbox_list: - total += await backfill_mailbox(provider, account_obj, mailbox) + total += await provider.message_count(mailbox.path, since=since) + await _set_backfill_counts(account_id, total=total, done=0) + progress = _BackfillProgress(account_obj, total) + + recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS) + count = 0 + for mailbox in mailbox_list: + if since is not None: # 3months: single windowed pass + count += await backfill_mailbox( + provider, account_obj, mailbox, progress, since=since + ) + else: # all: recent window first (usable fast), then the remainder + count += await backfill_mailbox( + provider, account_obj, mailbox, progress, since=recent_cutoff + ) + count += await backfill_mailbox( + provider, account_obj, mailbox, progress, before=recent_cutoff + ) await _set_state(account_id, "live") except Exception as exc: # noqa: BLE001 - surface provider errors as account state logger.exception("backfill failed for account %s", account_id) @@ -180,7 +298,7 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int: raise finally: await provider.close() - return total + return count async def apply_delta( diff --git a/backend/app/threading_/threader.py b/backend/app/threading_/threader.py index 1857b61..0509090 100644 --- a/backend/app/threading_/threader.py +++ b/backend/app/threading_/threader.py @@ -39,6 +39,20 @@ def _recipient_emails(message: Message) -> set[str]: return emails +def _parent_refs(message: Message) -> list[str]: + """Non-synthetic parent ids this message explicitly declares (References + + In-Reply-To). A non-empty result means the message states its parentage, so it + must not be subject-merged into an unrelated conversation (§7.3).""" + refs: list[str] = list(message.references_hdr) + if message.in_reply_to: + refs.append(message.in_reply_to) + out: list[str] = [] + for r in refs: + if r and not is_synthetic(r) and r not in out: + out.append(r) + return out + + def _candidate_ids(message: Message) -> list[str]: candidates: list[str] = list(message.references_hdr) if message.in_reply_to: @@ -59,17 +73,26 @@ async def _find_by_references( user_id: uuid.UUID, message_id: uuid.UUID, candidates: list[str], + parent_refs: list[str], own_id: str | None, ) -> list[uuid.UUID]: """Threads joined by reference (SPEC §7.2.2). Matches bidirectionally: existing messages whose id is one of my candidate ids - (I reference them / duplicate id), OR existing messages that reference *my* id - (handles a late-arriving parent). Synthetic own-ids are never used to join (§7.1). + (I reference them / duplicate id), existing messages that reference *my* id + (late-arriving parent), OR existing messages that share one of my declared + parents — a sibling reply to the same original, even when neither the parent + nor a mutual reference is present. Synthetic ids are never used to join (§7.1). """ conditions: list[ColumnElement[bool]] = [] if candidates: conditions.append(Message.message_id_hdr.in_(candidates)) + if parent_refs: + # Siblings: an existing message that references, or is a reply to, one of + # my parents. This groups replies to a common (possibly absent) original + # without letting the subject fallback merge replies to *different* ones. + conditions.append(Message.references_hdr.overlap(parent_refs)) + conditions.append(Message.in_reply_to.in_(parent_refs)) if own_id is not None and not is_synthetic(own_id): # own_id = ANY(references_hdr): an existing message that references me. conditions.append(literal(own_id) == sa_any(Message.references_hdr)) @@ -191,10 +214,17 @@ async def thread_message(session: AsyncSession, message: Message) -> uuid.UUID: Returns the resulting thread id and sets ``message.thread_id``. """ candidates = _candidate_ids(message) + parent_refs = _parent_refs(message) own_id = message.message_id_hdr - thread_ids = await _find_by_references(session, message.user_id, message.id, candidates, own_id) + thread_ids = await _find_by_references( + session, message.user_id, message.id, candidates, parent_refs, own_id + ) - if not thread_ids: + # Subject fallback only for messages that declare no parentage (a broken/absent + # References chain). A message that explicitly replies to a specific parent is + # never subject-merged into an unrelated thread — e.g. two auto-replies to + # different originals that happen to share the same subject (§7.3). + if not thread_ids and not parent_refs: subject_thread = await _find_by_subject(session, message.user_id, message) if subject_thread is not None: thread_ids = [subject_thread] diff --git a/backend/migrations/versions/0012_account_sync_window.py b/backend/migrations/versions/0012_account_sync_window.py new file mode 100644 index 0000000..006b261 --- /dev/null +++ b/backend/migrations/versions/0012_account_sync_window.py @@ -0,0 +1,35 @@ +"""add accounts.sync_window + backfill progress counters + +Lets the user choose at account setup how much history to import (SPEC §12 M2): +``all`` (default; recent 30d first, then the rest), ``3months`` (windowed), or +``new`` (no backfill — only mail arriving after setup). ``backfill_total`` / +``backfill_done`` drive the initial-sync progress UX; total 0 means "nothing to +import" (the ``new`` window, or an empty account). + +Revision ID: 0012 +Revises: 0011 +Create Date: 2026-07-23 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0012" +down_revision: str | None = "0011" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("ALTER TABLE accounts ADD COLUMN sync_window text NOT NULL DEFAULT 'all'") + op.execute("ALTER TABLE accounts ADD COLUMN backfill_total integer NOT NULL DEFAULT 0") + op.execute("ALTER TABLE accounts ADD COLUMN backfill_done integer NOT NULL DEFAULT 0") + + +def downgrade() -> None: + op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS backfill_done") + op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS backfill_total") + op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS sync_window") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 39703e8..33f0f36 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "nh3~=0.2", "cryptography~=43.0", "mail-parser~=4.0", + "meilisearch-python-sdk==2.7.0", # last release that fits the pinned Meilisearch v1.8 "python-multipart~=0.0.9", # DEPENDENCY: SPEC §6.2 mandates accounts/presets.yaml (config, not code); the # stdlib has no YAML parser, so PyYAML is required to load it. diff --git a/backend/tests/api/test_api.py b/backend/tests/api/test_api.py index 98e7570..a4972c2 100644 --- a/backend/tests/api/test_api.py +++ b/backend/tests/api/test_api.py @@ -378,12 +378,13 @@ async def test_accounts_list(api: ApiCtx) -> None: assert resp.json()[0]["email"] == "me@example.com" -async def test_thread_pagination_cursor(api: ApiCtx) -> None: - first = await api.client.get("/api/threads", params={"limit": 1}) +async def test_thread_pagination_offset(api: ApiCtx) -> None: + first = await api.client.get("/api/threads", params={"limit": 1, "offset": 0}) body = first.json() assert len(body["threads"]) == 1 - assert body["cursor"] is not None - second = await api.client.get("/api/threads", params={"limit": 1, "cursor": body["cursor"]}) + assert body["total"] >= 2 # more than one page of threads exists + second = await api.client.get("/api/threads", params={"limit": 1, "offset": 1}) body2 = second.json() assert len(body2["threads"]) == 1 + assert body2["total"] == body["total"] # grand total is stable across pages assert body2["threads"][0]["id"] != body["threads"][0]["id"] diff --git a/backend/tests/api/test_saved_searches.py b/backend/tests/api/test_saved_searches.py new file mode 100644 index 0000000..76285d6 --- /dev/null +++ b/backend/tests/api/test_saved_searches.py @@ -0,0 +1,141 @@ +"""Saved searches CRUD — stored queries shown as virtual mailboxes (SPEC §12 M2).""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass + +import httpx +import pytest_asyncio +from httpx import ASGITransport +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool + +from app.config import get_settings +from app.identity.passwords import hash_password +from app.identity.sessions import create_session +from app.main import create_app +from app.models import User + +CSRF = "csrf" + + +@dataclass +class Ctx: + client: httpx.AsyncClient + user_id: uuid.UUID + + +def _hdr() -> dict[str, str]: + return {"X-CSRF": CSRF} + + +@pytest_asyncio.fixture +async def ctx() -> AsyncIterator[Ctx]: + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + sm = async_sessionmaker(engine, expire_on_commit=False) + session = sm() + user = User(handle=f"ss{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")) + session.add(user) + await session.commit() + token = await create_session(user.id, "pytest") + client = httpx.AsyncClient( + transport=ASGITransport(app=create_app()), + base_url="http://testserver", + cookies={"session": token, "csrf": CSRF}, + ) + try: + yield Ctx(client=client, user_id=user.id) + finally: + await client.aclose() + await session.execute(delete(User).where(User.id == user.id)) + await session.commit() + await session.close() + await engine.dispose() + + +async def test_saved_search_crud_and_upsert(ctx: Ctx) -> None: + # Empty to start. + resp = await ctx.client.get("/api/saved-searches") + assert resp.status_code == 200 + assert resp.json() == [] + + # Create. + resp = await ctx.client.post( + "/api/saved-searches", + json={"name": "Unread from boss", "query": "is:unread from:boss"}, + headers=_hdr(), + ) + assert resp.status_code == 200 + created = resp.json() + assert created["name"] == "Unread from boss" + assert created["query"] == "is:unread from:boss" + assert created["pinned"] is True + sid = created["id"] + + # It lists back. + resp = await ctx.client.get("/api/saved-searches") + assert [s["name"] for s in resp.json()] == ["Unread from boss"] + + # Re-saving under the same name upserts (updates query, keeps a single row). + resp = await ctx.client.post( + "/api/saved-searches", + json={"name": "Unread from boss", "query": "is:unread from:ceo"}, + headers=_hdr(), + ) + assert resp.status_code == 200 + assert resp.json()["id"] == sid # same row + assert resp.json()["query"] == "is:unread from:ceo" + + rows = (await ctx.client.get("/api/saved-searches")).json() + assert len(rows) == 1 and rows[0]["query"] == "is:unread from:ceo" + + # Delete. + resp = await ctx.client.delete(f"/api/saved-searches/{sid}", headers=_hdr()) + assert resp.status_code == 200 + assert (await ctx.client.get("/api/saved-searches")).json() == [] + + # Deleting a missing row 404s. + resp = await ctx.client.delete(f"/api/saved-searches/{uuid.uuid4()}", headers=_hdr()) + assert resp.status_code == 404 + + +async def test_saved_search_requires_csrf(ctx: Ctx) -> None: + # No CSRF header -> rejected (state-changing route). + resp = await ctx.client.post( + "/api/saved-searches", json={"name": "x", "query": "y"} + ) + assert resp.status_code == 403 + + +async def test_saved_search_is_per_user(ctx: Ctx) -> None: + await ctx.client.post( + "/api/saved-searches", + json={"name": "Mine", "query": "in:inbox"}, + headers=_hdr(), + ) + # A different user's client sees none of the above. + other = User(handle=f"ss{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")) + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + sm = async_sessionmaker(engine, expire_on_commit=False) + session = sm() + session.add(other) + await session.commit() + token = await create_session(other.id, "pytest") + try: + client = httpx.AsyncClient( + transport=ASGITransport(app=create_app()), + base_url="http://testserver", + cookies={"session": token, "csrf": CSRF}, + ) + try: + assert (await client.get("/api/saved-searches")).json() == [] + finally: + await client.aclose() + finally: + await session.execute(delete(User).where(User.id == other.id)) + await session.commit() + await session.close() + await engine.dispose() diff --git a/backend/tests/search/test_meili.py b/backend/tests/search/test_meili.py new file mode 100644 index 0000000..54040a1 --- /dev/null +++ b/backend/tests/search/test_meili.py @@ -0,0 +1,79 @@ +"""Meilisearch backend (SPEC §8, M2). Runs against the live Meili in the dev/CI +stack; skips if it is unreachable so the rest of the suite still runs locally.""" + +from __future__ import annotations + +import asyncio +import time +import uuid + +import pytest + +from app.search.base import SearchDoc +from app.search.meili import MeiliSearchBackend +from app.search.queryparser import to_search_query + + +def _doc(user_id: uuid.UUID) -> SearchDoc: + return SearchDoc( + message_id=uuid.uuid4(), + thread_id=uuid.uuid4(), + user_id=user_id, + account_id=uuid.uuid4(), + mailbox_special_use="inbox", + from_=[{"name": "Bob Vendor", "email": "bob@vendor.com"}], + to=[{"name": "Me", "email": "me@x.com"}], + cc=[], + subject="Your invoice for March", + body_text="Please find attached the invoice, amount due soon.", + attachment_names=["invoice.pdf"], + has_attachments=True, + is_unread=True, + is_starred=False, + date=int(time.time()), + size_bytes=2048, + ) + + +async def _poll(backend: MeiliSearchBackend, q: str, user_id: uuid.UUID) -> int: + """Indexing is async in Meili; retry briefly until the doc is searchable.""" + for _ in range(60): + hits, _c = await backend.query(to_search_query(q, user_id), limit=10, cursor=None) + if hits: + return len(hits) + await asyncio.sleep(0.05) + return 0 + + +async def test_meili_typo_tolerance_and_filters() -> None: + backend = MeiliSearchBackend() + user_id = uuid.uuid4() + try: + await backend.index(_doc(user_id)) + + # Acceptance (§12 M2): a typo still finds the message. + hits, _c = await backend.query( + to_search_query("invocie", user_id), limit=10, cursor=None + ) + if not hits: # wait for the async index task + assert await _poll(backend, "invocie", user_id) == 1 + hits, _c = await backend.query( + to_search_query("invocie", user_id), limit=10, cursor=None + ) + assert len(hits) == 1 + assert "" in hits[0].highlights.get("subject", "") + + async def n(query: str) -> int: + res, _c = await backend.query(to_search_query(query, user_id), limit=10, cursor=None) + return len(res) + + assert await n("is:unread invoice") == 1 + assert await n("is:read invoice") == 0 # the doc is unread + assert await n("from:vendor") == 1 # field operator folded into the query + assert await n("larger:1k invoice") == 1 + assert await n("larger:5k invoice") == 0 + assert await n("in:sent invoice") == 0 # doc is in the inbox + except (ConnectionError, OSError) as exc: # Meili not running locally + pytest.skip(f"Meilisearch unavailable: {exc}") + finally: + await backend.delete_user(user_id) diff --git a/backend/tests/search/test_search_api.py b/backend/tests/search/test_search_api.py index a76fb5d..d8cf368 100644 --- a/backend/tests/search/test_search_api.py +++ b/backend/tests/search/test_search_api.py @@ -150,6 +150,24 @@ async def test_free_text(seeded: Ctx) -> None: assert await _search(seeded, "lunch") == 1 +async def test_free_text_prefix_matches_while_typing(seeded: Ctx) -> None: + # Search-as-you-type: a partial word finds the message before it is complete. + assert await _search(seeded, "reven") == 1 # -> "revenue" + assert await _search(seeded, "lun") == 1 # -> "lunch" + # Prefix is anchored at word start, not a substring match. + assert await _search(seeded, "enue") == 0 + # Multiple partial words are ANDed as prefixes. + assert await _search(seeded, "quart reven") == 1 # "Quarterly revenue" + assert await _search(seeded, "quart lun") == 0 + + +async def test_free_text_full_email_address(seeded: Ctx) -> None: + # Postgres keeps an email as a single lexeme; searching the *whole* address + # must still match (regression: prefix-splitting on word chars broke this). + assert await _search(seeded, "bob@corp.com") == 1 + assert await _search(seeded, "bob@corp.co") == 1 # prefix while still typing + + async def test_from_operator(seeded: Ctx) -> None: assert await _search(seeded, "from:bob") == 1 assert await _search(seeded, "from:corp.com") == 3 # all three share the domain diff --git a/backend/tests/sync/test_backfill_window.py b/backend/tests/sync/test_backfill_window.py new file mode 100644 index 0000000..c55243c --- /dev/null +++ b/backend/tests/sync/test_backfill_window.py @@ -0,0 +1,160 @@ +"""Sync-window backfill strategies at account setup (SPEC §12 M2): all (recent +first, then remainder), 3months (windowed), new (no history, watermark only).""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncIterator +from datetime import UTC, date, datetime, timedelta +from email.message import EmailMessage + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool + +from app.config import get_settings +from app.identity.passwords import hash_password +from app.models import Account, Mailbox, Message, User +from app.providers.base import RemoteMailbox, RemoteMessage +from app.sync import engine as sync_engine + + +def _raw(mid: str) -> bytes: + m = EmailMessage() + m["Subject"] = "hi" + m["From"] = "me@example.com" + m["To"] = "bob@example.com" + m["Message-ID"] = mid + m["Date"] = "Mon, 09 Mar 2026 08:36:37 +0100" + m.set_content("body") + return m.as_bytes() + + +class _FakeProvider: + def __init__(self) -> None: + self.backfill_calls: list[tuple[str, date | None, date | None]] = [] + self.count_calls: list[tuple[str, date | None]] = [] + self.newest_calls: list[str] = [] + + async def list_mailboxes(self) -> list[RemoteMailbox]: + return [RemoteMailbox(path="INBOX", special_use="inbox", uidvalidity=1)] + + async def message_count(self, mailbox: str, *, since: date | None = None) -> int: + self.count_calls.append((mailbox, since)) + return 3 + + async def newest_uid(self, mailbox: str) -> int: + self.newest_calls.append(mailbox) + return 42 + + async def backfill( # type: ignore[no-untyped-def] + self, mailbox, *, newest_first=True, batch_size=200, since=None, before=None + ) -> AsyncIterator[list[RemoteMessage]]: + self.backfill_calls.append((mailbox, since, before)) + # Distinct UID per pass so recent + remainder both persist a row. + uid = 100 if since is not None else 50 + yield [ + RemoteMessage( + uid=uid, + raw=_raw(f"<{mailbox}-{uid}@x.com>"), + flags=frozenset(), + internal_date=datetime.now(UTC), + ) + ] + + async def close(self) -> None: + return None + + +async def _seed(sm: async_sessionmaker, window: str) -> tuple[uuid.UUID, uuid.UUID]: # type: ignore[type-arg] + session = sm() + user = User(handle=f"bf{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")) + session.add(user) + await session.flush() + account = Account( + user_id=user.id, + email="me@example.com", + imap_config={}, + smtp_config={}, + secret_ct=b"", + discovery_src="manual", + sync_window=window, + ) + session.add(account) + await session.commit() + aid = account.id + await session.close() + return user.id, aid + + +async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None: # type: ignore[no-untyped-def] + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + sm = async_sessionmaker(engine, expire_on_commit=False) + _uid, account_id = await _seed(sm, "new") + fake = _FakeProvider() + monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake) + try: + count = await sync_engine.bootstrap_and_backfill(account_id) + assert count == 0 + assert fake.backfill_calls == [] # no history fetched + assert fake.newest_calls == ["INBOX"] + + session = sm() + acct = await session.get(Account, account_id) + assert acct is not None and acct.state == "live" and acct.backfill_total == 0 + mb = await session.scalar(select(Mailbox).where(Mailbox.account_id == account_id)) + assert mb is not None and mb.last_seen_uid == 42 # moved to the tip + msgs = await session.scalar( + select(func.count()).select_from(Message).where(Message.mailbox_id == mb.id) + ) + assert msgs == 0 # nothing imported + await session.close() + finally: + await engine.dispose() + + +async def test_window_all_imports_recent_then_remainder(monkeypatch) -> None: # type: ignore[no-untyped-def] + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + sm = async_sessionmaker(engine, expire_on_commit=False) + _uid, account_id = await _seed(sm, "all") + fake = _FakeProvider() + monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake) + try: + await sync_engine.bootstrap_and_backfill(account_id) + + # Two passes per mailbox: recent window (since set), then the older + # remainder (before set) — the "usable fast, then fill in" strategy. + cutoff = date.today() - timedelta(days=sync_engine._RECENT_DAYS) + assert fake.backfill_calls == [("INBOX", cutoff, None), ("INBOX", None, cutoff)] + assert fake.count_calls == [("INBOX", None)] # denominator = whole mailbox + + session = sm() + acct = await session.get(Account, account_id) + assert acct is not None and acct.state == "live" + assert acct.backfill_total == 3 and acct.backfill_done == 2 + mb = await session.scalar(select(Mailbox).where(Mailbox.account_id == account_id)) + assert mb is not None + msgs = await session.scalar( + select(func.count()).select_from(Message).where(Message.mailbox_id == mb.id) + ) + assert msgs == 2 # one from each pass + await session.close() + finally: + await engine.dispose() + + +async def test_window_3months_single_dated_pass(monkeypatch) -> None: # type: ignore[no-untyped-def] + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + sm = async_sessionmaker(engine, expire_on_commit=False) + _uid, account_id = await _seed(sm, "3months") + fake = _FakeProvider() + monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake) + try: + await sync_engine.bootstrap_and_backfill(account_id) + + since = date.today() - timedelta(days=sync_engine._WINDOW_DAYS["3months"]) + assert fake.backfill_calls == [("INBOX", since, None)] # single windowed pass + assert fake.count_calls == [("INBOX", since)] # denominator counts the window + await engine.dispose() + finally: + pass diff --git a/backend/tests/threading/test_threading.py b/backend/tests/threading/test_threading.py index 2e31c53..c718de7 100644 --- a/backend/tests/threading/test_threading.py +++ b/backend/tests/threading/test_threading.py @@ -72,6 +72,48 @@ async def test_broken_client_subject_only_joins(env: ThreadingEnv) -> None: assert a.thread_id == b.thread_id +async def test_auto_replies_to_different_originals_do_not_join(env: ThreadingEnv) -> None: + # Two out-of-office auto-replies, same subject and same recipient, but each + # replying to a DIFFERENT (absent) original. They must not be subject-merged. + a = await env.add( + message_id="autoA@tvp.pl", + subject="Automatyczna odpowiedź: Silent Hill", + from_email="robert@tvp.pl", + to_email="me@gmail.com", + in_reply_to="orig1@mail.gmail.com", + references=["orig1@mail.gmail.com"], + ) + b = await env.add( + message_id="autoB@wfdif.com.pl", + subject="Automatyczna odpowiedź: Silent Hill", + from_email="dyrektor@wfdif.com.pl", + to_email="me@gmail.com", + in_reply_to="orig2@mail.gmail.com", + references=["orig2@mail.gmail.com"], + minutes=5, + ) + assert a.thread_id != b.thread_id + + +async def test_replies_to_same_absent_parent_join(env: ThreadingEnv) -> None: + # Two replies that reference the SAME (absent) original — and not each other — + # still thread together via the shared parent reference. + a = await env.add( + message_id="r1@x.com", + subject="Re: Topic", + in_reply_to="orig@x.com", + references=["orig@x.com"], + ) + b = await env.add( + message_id="r2@x.com", + subject="Re: Topic", + in_reply_to="orig@x.com", + references=["orig@x.com"], + minutes=3, + ) + assert a.thread_id == b.thread_id + + async def test_distinct_topic_same_subject_does_not_join(env: ThreadingEnv) -> None: a = await env.add( message_id="a@x.com", diff --git a/frontend/src/components/BackfillBanner.tsx b/frontend/src/components/BackfillBanner.tsx new file mode 100644 index 0000000..5db843f --- /dev/null +++ b/frontend/src/components/BackfillBanner.tsx @@ -0,0 +1,69 @@ +// Initial-sync progress UX (SPEC §12 M2). Shows a small floating card while any +// account is importing its history; reads the accounts query, which the WS +// `sync.progress` event invalidates so the bar advances live. + +import { useQuery } from "@tanstack/react-query"; +import { useEffect, useState } from "react"; + +import { useT } from "../i18n"; +import { api } from "../lib/api"; + +export function BackfillBanner() { + const t = useT(); + const accounts = useQuery({ queryKey: ["accounts"], queryFn: api.accounts }); + + const importing = (accounts.data ?? []).filter( + (a) => a.state === "backfilling" && a.backfill_total > 0, + ); + const active = importing.length > 0; + + // Dissolve the card 10s after it appears; the sidebar ring keeps showing + // progress until the sync finishes. + const [dismissed, setDismissed] = useState(false); + useEffect(() => { + if (!active) { + setDismissed(false); + return; + } + const timer = window.setTimeout(() => setDismissed(true), 10000); + return () => window.clearTimeout(timer); + }, [active]); + + if (!active) return null; + + return ( +
+ {importing.map((a) => { + const done = Math.min(a.backfill_done, a.backfill_total); + const pct = Math.round((done / a.backfill_total) * 100); + return ( +
+
+ + {a.email} +
+
+ {t("backfill.importing", { + done: done.toLocaleString(), + total: a.backfill_total.toLocaleString(), + })} +
+
+
+
+
+ ); + })} +
+ ); +} diff --git a/frontend/src/components/thread/MessageBody.tsx b/frontend/src/components/thread/MessageBody.tsx index 466b221..0738beb 100644 --- a/frontend/src/components/thread/MessageBody.tsx +++ b/frontend/src/components/thread/MessageBody.tsx @@ -15,7 +15,9 @@ import { Attachments } from "./Attachments"; export function MessageBody({ message }: { message: MessageEnvelope }) { const t = useT(); const queryClient = useQueryClient(); - const [allowed, setAllowed] = useState(false); + // Seed from the server's trusted-sender flag so an already-approved sender + // loads images and shows no "blocked" prompt on reopen. + const [allowed, setAllowed] = useState(message.images_allowed); const frameRef = useRef(null); const observerRef = useRef(null); const [height, setHeight] = useState(80); @@ -23,7 +25,10 @@ export function MessageBody({ message }: { message: MessageEnvelope }) { const { data, isLoading, isError } = useQuery({ queryKey: ["message-html", message.id, allowed], queryFn: async () => { - const resp = await fetch(`/api/messages/${message.id}/html`, { credentials: "include" }); + // ?images=1 asks the server to render remote images for this one request + // (the "Load images" one-off); "Always from this sender" persists separately. + const url = `/api/messages/${message.id}/html${allowed ? "?images=1" : ""}`; + const resp = await fetch(url, { credentials: "include" }); if (!resp.ok) throw new Error("failed to load message"); return resp.text(); }, @@ -89,6 +94,11 @@ export function MessageBody({ message }: { message: MessageEnvelope }) { if (persist) { await api.allowRemoteImages(message.id); queryClient.invalidateQueries({ queryKey: ["message-html", message.id] }); + // Refresh thread detail so the sender's images_allowed flag reflects the + // new trust on reopen (suppresses the prompt next time). + if (message.thread_id) { + queryClient.invalidateQueries({ queryKey: ["thread", message.thread_id] }); + } } setAllowed(true); }; diff --git a/frontend/src/i18n/en.ts b/frontend/src/i18n/en.ts index 54a8956..42c1e26 100644 --- a/frontend/src/i18n/en.ts +++ b/frontend/src/i18n/en.ts @@ -49,6 +49,9 @@ export const en = { "nav.spam": "Spam", "nav.trash": "Trash", "nav.search": "Search mail…", + "search.save": "Save this search", + "search.saved": "Saved searches", + "search.saveNamePrompt": "Name this saved search:", "nav.storageQuota": "Storage used", "nav.refresh": "Refresh all folders", @@ -176,11 +179,22 @@ export const en = { "wizard.startReading": "Start reading now", "wizard.manualConfig": "Enter settings manually", "wizard.displayName": "Display name", + "wizard.syncWindow": "How much mail to sync", + "wizard.syncWindow.all": "All mail", + "wizard.syncWindow.all.hint": "Imports the last 30 days first, then the rest in the background.", + "wizard.syncWindow.3months": "Last 3 months only", + "wizard.syncWindow.3months.hint": "Older mail stays on the server until you download it.", + "wizard.syncWindow.new": "New mail only", + "wizard.syncWindow.new.hint": + "Nothing from before now — download older mail anytime from account settings.", "settings.title": "Settings", "settings.profile": "Profile", "settings.security": "Security", "settings.accounts": "Accounts", + "settings.downloadAllMail": "Sync remaining mail", + "settings.syncedPartially": "Synced partially", + "backfill.importing": "Importing messages… {done} of {total}", "settings.appearance": "Appearance", "settings.handle": "Handle", "settings.recoveryEmail": "Recovery email", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index cac0a03..a8df9c2 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -13,10 +13,12 @@ import type { QuotaOut, RecipientSuggestion, RemoteDraftOut, + SavedSearch, SearchOut, SendResponse, ServerConfig, SessionInfo, + SyncWindow, ThreadDetail, ThreadListOut, TotpSetup, @@ -90,9 +92,15 @@ export const api = { mailboxes: () => apiFetch("/mailboxes"), syncNow: () => apiFetch<{ status: string }>("/accounts/sync", { method: "POST" }), - threads: (view = "unified", cursor?: string) => { + threads: ( + view = "unified", + opts?: { offset?: number; limit?: number; filter?: string; order?: "asc" | "desc" }, + ) => { const params = new URLSearchParams({ view }); - if (cursor) params.set("cursor", cursor); + if (opts?.offset) params.set("offset", String(opts.offset)); + if (opts?.limit) params.set("limit", String(opts.limit)); + if (opts?.filter && opts.filter !== "all") params.set("filter", opts.filter); + if (opts?.order) params.set("order", opts.order); return apiFetch(`/threads?${params.toString()}`); }, thread: (id: string, sort: "asc" | "desc" = "asc", view = "unified") => @@ -121,6 +129,7 @@ export const api = { imap: ServerConfig; smtp: ServerConfig; password: string; + sync_window?: SyncWindow; }) => apiFetch("/accounts", { method: "POST", body: JSON.stringify(input) }), accountDetail: (id: string) => apiFetch(`/accounts/${id}`), updateAccount: ( @@ -129,6 +138,7 @@ export const api = { display_name?: string; color?: string; signature?: string; + sync_window?: SyncWindow; imap?: ServerConfig; smtp?: ServerConfig; password?: string; @@ -159,6 +169,16 @@ export const api = { quota: () => apiFetch("/settings/quota"), folderCounts: () => apiFetch("/mailboxes/counts"), + // --- saved searches (virtual mailboxes) --- + savedSearches: () => apiFetch("/saved-searches"), + createSavedSearch: (name: string, query: string) => + apiFetch("/saved-searches", { + method: "POST", + body: JSON.stringify({ name, query }), + }), + deleteSavedSearch: (id: string) => + apiFetch<{ status: string }>(`/saved-searches/${id}`, { method: "DELETE" }), + // --- compose --- suggestRecipients: (q: string) => apiFetch(`/recipients/suggest?q=${encodeURIComponent(q)}`), diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 29894d6..f508054 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -30,6 +30,7 @@ export interface ThreadSummary { export interface ThreadListOut { threads: ThreadSummary[]; cursor: string | null; + total: number; } export interface AttachmentOut { @@ -55,6 +56,7 @@ export interface MessageEnvelope { is_starred: boolean; has_attachments: boolean; has_remote_images: boolean; + images_allowed: boolean; size_bytes: number; snippet: string; also_in_account_ids: string[]; @@ -68,6 +70,8 @@ export interface ThreadDetail { messages: MessageEnvelope[]; } +export type SyncWindow = "all" | "3months" | "new"; + export interface AccountOut { id: string; email: string; @@ -75,6 +79,9 @@ export interface AccountOut { color: string; state: string; signature: string; + sync_window: SyncWindow; + backfill_total: number; + backfill_done: number; } export interface MailboxOut { @@ -157,6 +164,13 @@ export interface FolderCounts { drafts: number; } +export interface SavedSearch { + id: string; + name: string; + query: string; + pinned: boolean; +} + // --- M1: compose ------------------------------------------------------------- export interface RecipientSuggestion { diff --git a/frontend/src/lib/ws.ts b/frontend/src/lib/ws.ts index 7dc4edc..2a828ed 100644 --- a/frontend/src/lib/ws.ts +++ b/frontend/src/lib/ws.ts @@ -48,6 +48,10 @@ export function connectWs(queryClient: QueryClient): () => void { queryClient.invalidateQueries({ queryKey: ["quota"] }); queryClient.invalidateQueries({ queryKey: ["drafts-remote"] }); queryClient.invalidateQueries({ queryKey: ["folder-counts"] }); + // Backfill progress counters live on the account row (drives the banner). + if (name === "sync.progress") { + queryClient.invalidateQueries({ queryKey: ["accounts"] }); + } if (typeof payload.thread_id === "string") { queryClient.invalidateQueries({ queryKey: ["thread", payload.thread_id] }); } diff --git a/frontend/src/routes/MailLayout.tsx b/frontend/src/routes/MailLayout.tsx index 0aee219..495a3d2 100644 --- a/frontend/src/routes/MailLayout.tsx +++ b/frontend/src/routes/MailLayout.tsx @@ -1,4 +1,4 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect, useRef, useState } from "react"; import { Link, useMatch, useNavigate } from "react-router-dom"; @@ -9,11 +9,12 @@ import { MessageList } from "../components/list/MessageList"; import { Resizer } from "../components/Resizer"; import { ThemeToggle } from "../components/ThemeToggle"; import { EmptyIllustration } from "../components/EmptyIllustration"; +import { BackfillBanner } from "../components/BackfillBanner"; import { ThreadView } from "../components/thread/ThreadView"; import { useT } from "../i18n"; import type { MessageKey } from "../i18n/en"; import { api } from "../lib/api"; -import type { QuotaOut, ThreadSummary } from "../lib/types"; +import type { QuotaOut, SavedSearch, ThreadSummary } from "../lib/types"; import { useKeyboardShortcuts } from "../lib/useKeyboardShortcuts"; function quotaLabel(q: QuotaOut): string { @@ -48,6 +49,13 @@ export function MailLayout() { const threadId = match?.params.threadId; const [view, setView] = useState("folder:inbox"); const [term, setTerm] = useState(""); + // Debounced copy of the search box for search-as-you-type (avoids a request per key). + const [debouncedTerm, setDebouncedTerm] = useState(""); + useEffect(() => { + const h = window.setTimeout(() => setDebouncedTerm(term), 180); + return () => window.clearTimeout(h); + }, [term]); + const searching = debouncedTerm.trim().length > 0; const [filter, setFilter] = useState<"all" | "unread" | "pinned">("all"); const [listOrder, setListOrder] = useState<"newest" | "oldest">(() => localStorage.getItem("mail.listOrder") === "oldest" ? "oldest" : "newest", @@ -90,10 +98,13 @@ export function MailLayout() { const currentFolder = view.startsWith("folder:") ? view.slice("folder:".length) : "inbox"; const accountsQuery = useQuery({ queryKey: ["accounts"], queryFn: api.accounts }); const mailboxesQuery = useQuery({ queryKey: ["mailboxes"], queryFn: api.mailboxes }); + const orderParam: "asc" | "desc" = listOrder === "newest" ? "desc" : "asc"; const threadsQuery = useQuery({ - queryKey: ["threads", view], - queryFn: () => api.threads(view), - enabled: !isDrafts, + queryKey: ["threads", view, filter, orderParam, page], + queryFn: () => + api.threads(view, { offset: page * PAGE_SIZE, limit: PAGE_SIZE, filter, order: orderParam }), + enabled: !isDrafts && !searching, + placeholderData: keepPreviousData, // keep the current page visible while the next loads }); const draftsQuery = useQuery({ queryKey: ["drafts"], @@ -106,33 +117,36 @@ export function MailLayout() { enabled: isDrafts, }); const searchQuery = useQuery({ - queryKey: ["search", term], - queryFn: () => api.search(term), - enabled: term.trim().length > 0, + queryKey: ["search", debouncedTerm], + queryFn: () => api.search(debouncedTerm), + enabled: searching, }); + const savedSearchesQuery = useQuery({ queryKey: ["saved-searches"], queryFn: api.savedSearches }); const settingsQuery = useQuery({ queryKey: ["settings"], queryFn: api.settings }); const threadOrder: "asc" | "desc" = settingsQuery.data?.thread_order ?? "asc"; const quotaQuery = useQuery({ queryKey: ["quota"], queryFn: api.quota }); const countsQuery = useQuery({ queryKey: ["folder-counts"], queryFn: api.folderCounts }); - const allThreads: ThreadSummary[] = term.trim() - ? (searchQuery.data?.threads ?? []).map((h) => h.thread) - : (threadsQuery.data?.threads ?? []); - const filtered = allThreads.filter((th) => - filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true, - ); - // The backend returns threads newest-first; reverse for an oldest-first view. - const threads = listOrder === "newest" ? filtered : [...filtered].reverse(); - - // Client-side pagination of the (filtered, ordered) list. - const total = threads.length; - const maxPage = Math.max(0, Math.ceil(total / PAGE_SIZE) - 1); - const curPage = Math.min(page, maxPage); - const pageStart = curPage * PAGE_SIZE; - const pageEnd = Math.min(pageStart + PAGE_SIZE, total); - const pageThreads = threads.slice(pageStart, pageEnd); - // Reset to the first page whenever the underlying list identity changes. - useEffect(() => setPage(0), [view, filter, term, listOrder]); + // Folder/inbox views paginate server-side with an accurate grand total; search + // results (a single fetched page of hits) still paginate client-side. + let pageThreads: ThreadSummary[]; + let total: number; + if (searching) { + const hits = (searchQuery.data?.threads ?? []).map((h) => h.thread); + const f = hits.filter((th) => + filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true, + ); + const ordered = listOrder === "newest" ? f : [...f].reverse(); + total = ordered.length; + pageThreads = ordered.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE); + } else { + pageThreads = threadsQuery.data?.threads ?? []; + total = threadsQuery.data?.total ?? 0; + } + const pageStart = page * PAGE_SIZE; + const pageEnd = pageStart + pageThreads.length; + // Reset to the first page whenever the list identity (not the page) changes. + useEffect(() => setPage(0), [view, filter, debouncedTerm, listOrder]); // Master "select all" state over the currently listed threads. const listedSelectedCount = pageThreads.filter((th) => checkedIds.has(th.id)).length; @@ -141,8 +155,9 @@ export function MailLayout() { const toggleSelectAll = () => setCheckedIds(allSelected ? new Set() : new Set(pageThreads.map((th) => th.id))); - const threadsRef = useRef(threads); - threadsRef.current = threads; + // Keyboard next/prev navigates within the currently visible page. + const threadsRef = useRef(pageThreads); + threadsRef.current = pageThreads; const detailQuery = useQuery({ queryKey: ["thread", threadId, threadOrder, view], @@ -157,6 +172,27 @@ export function MailLayout() { queryClient.invalidateQueries({ queryKey: ["folder-counts"] }); }, }); + // Saved searches (virtual mailboxes, §12 M2). + const savedSearches = savedSearchesQuery.data ?? []; + const saveSearch = useMutation({ + mutationFn: (name: string) => api.createSavedSearch(name, term.trim()), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ["saved-searches"] }), + }); + const deleteSaved = useMutation({ + mutationFn: (id: string) => api.deleteSavedSearch(id), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ["saved-searches"] }), + }); + const onSaveSearch = () => { + const q = term.trim(); + if (!q) return; + const name = window.prompt(t("search.saveNamePrompt"), q); + if (name && name.trim()) saveSearch.mutate(name.trim()); + }; + const runSavedSearch = (s: SavedSearch) => { + setTerm(s.query); + navigate("/mail"); + }; + // Auto-mark an opened thread read after a short dwell (4s), not instantly — and // cancel it if you leave before then, so a quick glance doesn't mark it read. useEffect(() => { @@ -359,6 +395,7 @@ export function MailLayout() { return (
+ {/* Sidebar */}
@@ -612,14 +704,14 @@ export function MailLayout() {
setPage(curPage - 1)} + disabled={page === 0} + onClick={() => setPage(page - 1)} label={t("pager.prev")} dir="left" /> = total} - onClick={() => setPage(curPage + 1)} + onClick={() => setPage(page + 1)} label={t("pager.next")} dir="right" /> @@ -765,6 +857,42 @@ export function MailLayout() { ); } +// Account bullet in the sidebar. While the account is backfilling it becomes a +// spinning ring that fills with the account colour to show import progress +// (SPEC §12 M2); once live it's the plain colour-filled dot. +function AccountDot({ + color, + syncing, + total, + done, +}: { + color: string; + syncing: boolean; + total: number; + done: number; +}) { + if (!syncing) { + return ( + + ); + } + const pct = total > 0 ? Math.min(100, Math.round((done / Math.max(done, total)) * 100)) : 0; + const ring = "rgba(128,128,128,0.35)"; + return ( + + ); +} + function FilterTab({ active, onClick, diff --git a/frontend/src/routes/settings/Settings.tsx b/frontend/src/routes/settings/Settings.tsx index 24f5af0..cb6c051 100644 --- a/frontend/src/routes/settings/Settings.tsx +++ b/frontend/src/routes/settings/Settings.tsx @@ -225,6 +225,10 @@ function AccountsSection() { mutationFn: (id: string) => api.deleteAccount(id), onSuccess: () => queryClient.invalidateQueries({ queryKey: ["accounts"] }), }); + const downloadAll = useMutation({ + mutationFn: (id: string) => api.updateAccount(id, { sync_window: "all" }), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ["accounts"] }), + }); const close = () => { setEditing(null); @@ -257,8 +261,25 @@ function AccountsSection() { {t("settings.syncState", { state: a.state })} + {a.sync_window !== "all" && ( + <> + · + + {t("settings.syncedPartially")} + + + )} + {a.sync_window !== "all" && ( + + )}
+
+ {t("wizard.syncWindow")} +
+ {(["all", "3months", "new"] as const).map((w) => ( + + ))} +
+
{tested &&

{t("wizard.testOk")}

}
{!tested ? ( diff --git a/frontend/src/test/ThreadView.test.tsx b/frontend/src/test/ThreadView.test.tsx index 31e6953..8b72c00 100644 --- a/frontend/src/test/ThreadView.test.tsx +++ b/frontend/src/test/ThreadView.test.tsx @@ -24,6 +24,7 @@ function msg(id: string, minutes: number, unread = false): MessageEnvelope { is_starred: false, has_attachments: false, has_remote_images: false, + images_allowed: false, size_bytes: 1024, snippet: `snippet ${id}`, also_in_account_ids: [], diff --git a/screenshots/PDF_preview.png b/screenshots/PDF_preview.png new file mode 100644 index 0000000..e6bb017 Binary files /dev/null and b/screenshots/PDF_preview.png differ diff --git a/screenshots/image_preview.png b/screenshots/image_preview.png new file mode 100644 index 0000000..989dc97 Binary files /dev/null and b/screenshots/image_preview.png differ diff --git a/screenshots/progress-circles.jpg b/screenshots/progress-circles.jpg new file mode 100644 index 0000000..0454fd3 Binary files /dev/null and b/screenshots/progress-circles.jpg differ