M2: Meili search + saved searches, sync windows + backfill progress, threading & image fixes
Search: - Meilisearch backend (app/search/meili.py) + factory wiring; pinned SDK 2.7 - Search-as-you-type: debounced query + Postgres prefix matching, built from the DB tokenizer so full email addresses match (email stays one lexeme) - Saved searches API + sidebar virtual mailboxes; save affordance in the box Account sync windows (SPEC §12 M2): - Choose at setup: All (recent 30d first, then remainder), Last 3 months, New only - Per-account sync_window + "Sync remaining mail" from settings (idempotent re-run) - Provider windowed backfill (SINCE/BEFORE), message_count, newest_uid - migration 0012: accounts.sync_window + backfill_total/backfill_done Backfill progress UX: - Floating progress card (auto-dissolves after 10s) + spinning sidebar ring that fills with the account colour; driven by sync.progress over WS Thread list pagination: - Server-side offset + accurate grand total, with filter/order in SQL; working prev/next across all synced mail Reading panel: - "Load images" one-off now actually renders images (?images=1) without trusting the sender; envelope carries images_allowed so a trusted sender shows no prompt Threading: - Subject fallback no longer merges messages that declare an explicit (different) parent; true siblings still group via a shared-parent reference match. Fixes unrelated auto-replies with the same subject landing in one thread Backend 257 tests green, ruff + mypy --strict clean; frontend tsc/eslint/vitest clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
@@ -3,9 +3,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# How much history to import at account setup (SPEC §12 M2).
|
||||
SyncWindow = Literal["all", "3months", "new"]
|
||||
|
||||
|
||||
class ServerConfigIn(BaseModel):
|
||||
host: str
|
||||
@@ -56,12 +60,14 @@ class CreateAccountRequest(BaseModel):
|
||||
smtp: ServerConfigIn
|
||||
username: str | None = None
|
||||
password: str = Field(min_length=1)
|
||||
sync_window: SyncWindow = "all"
|
||||
|
||||
|
||||
class UpdateAccountRequest(BaseModel):
|
||||
display_name: str | None = None
|
||||
color: str | None = None
|
||||
signature: str | None = None
|
||||
sync_window: SyncWindow | None = None
|
||||
# Connection/credential changes require the password (re-validated + re-encrypted).
|
||||
imap: ServerConfigIn | None = None
|
||||
smtp: ServerConfigIn | None = None
|
||||
@@ -76,5 +82,8 @@ class AccountDetailOut(BaseModel):
|
||||
state: str
|
||||
state_detail: str
|
||||
signature: str
|
||||
sync_window: str
|
||||
backfill_total: int
|
||||
backfill_done: int
|
||||
imap_config: dict[str, object]
|
||||
smtp_config: dict[str, object]
|
||||
|
||||
@@ -39,6 +39,7 @@ async def create_account(
|
||||
secret_ct=encrypt_secret(req.password.encode("utf-8")),
|
||||
discovery_src="manual",
|
||||
state="backfilling",
|
||||
sync_window=req.sync_window,
|
||||
)
|
||||
session.add(account)
|
||||
try:
|
||||
|
||||
@@ -17,6 +17,7 @@ def register_routers(app: FastAPI) -> None:
|
||||
from app.api.mailboxes import router as mailboxes_router
|
||||
from app.api.messages import router as messages_router
|
||||
from app.api.recipients import router as recipients_router
|
||||
from app.api.saved_searches import router as saved_searches_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.threads import router as threads_router
|
||||
@@ -34,6 +35,7 @@ def register_routers(app: FastAPI) -> None:
|
||||
recipients_router,
|
||||
settings_router,
|
||||
mailboxes_router,
|
||||
saved_searches_router,
|
||||
):
|
||||
app.include_router(router, prefix="/api")
|
||||
app.include_router(ws_router, prefix="/api")
|
||||
|
||||
@@ -82,6 +82,9 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
|
||||
color=a.color,
|
||||
state=a.state,
|
||||
signature=a.signature,
|
||||
sync_window=a.sync_window,
|
||||
backfill_total=a.backfill_total,
|
||||
backfill_done=a.backfill_done,
|
||||
)
|
||||
for a in accounts
|
||||
]
|
||||
@@ -186,6 +189,18 @@ async def update_account(
|
||||
if body.signature is not None:
|
||||
account.signature = body.signature
|
||||
|
||||
# Broadening the sync window (e.g. "new" -> "all", "Download all mail") means
|
||||
# importing history not fetched before, so re-run the backfill (SPEC §12 M2).
|
||||
# persist_message is idempotent on (mailbox_id, uid), so already-synced mail
|
||||
# is skipped. Narrowing keeps existing mail — nothing to fetch.
|
||||
_COVERAGE = {"new": 0, "3months": 1, "all": 2}
|
||||
broaden = False
|
||||
if body.sync_window is not None and body.sync_window != account.sync_window:
|
||||
broaden = _COVERAGE[body.sync_window] > _COVERAGE.get(account.sync_window, 0)
|
||||
account.sync_window = body.sync_window
|
||||
if broaden:
|
||||
account.state = "backfilling"
|
||||
|
||||
# Connection settings and/or password: re-validate against the servers, then
|
||||
# re-encrypt the secret and re-sync. The password must be supplied (§10.2: the
|
||||
# stored secret is only decrypted in the worker, never here).
|
||||
@@ -224,7 +239,7 @@ async def update_account(
|
||||
|
||||
await db.commit()
|
||||
|
||||
if conn_change:
|
||||
if conn_change or broaden:
|
||||
await enqueue_job("sync_bootstrap_account", str(account.id))
|
||||
|
||||
return _detail(account)
|
||||
@@ -248,6 +263,9 @@ def _detail(account: Account) -> AccountDetailOut:
|
||||
state=account.state,
|
||||
state_detail=account.state_detail,
|
||||
signature=account.signature,
|
||||
sync_window=account.sync_window,
|
||||
backfill_total=account.backfill_total,
|
||||
backfill_done=account.backfill_done,
|
||||
imap_config=account.imap_config,
|
||||
smtp_config=account.smtp_config,
|
||||
)
|
||||
|
||||
@@ -65,14 +65,21 @@ def _sender_email(message: Message) -> str:
|
||||
|
||||
|
||||
@router.get("/{message_id}/html", response_class=HTMLResponse)
|
||||
async def message_html(message_id: uuid.UUID, user: CurrentUser, db: DbSession) -> HTMLResponse:
|
||||
async def message_html(
|
||||
message_id: uuid.UUID,
|
||||
user: CurrentUser,
|
||||
db: DbSession,
|
||||
images: bool = Query(False),
|
||||
) -> HTMLResponse:
|
||||
"""Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
|
||||
|
||||
Re-sanitizes from the raw blob when the sender is image-approved, or when the
|
||||
stored copy predates the current sanitizer version (SPEC §10.4).
|
||||
stored copy predates the current sanitizer version (SPEC §10.4). ``images=1``
|
||||
is a one-time "Load images" — render remote images for this request only,
|
||||
without persisting the sender as trusted.
|
||||
"""
|
||||
message = await _owned_message(db, user.id, message_id)
|
||||
allow_remote = _sender_email(message) in allowed_image_senders(user)
|
||||
allow_remote = images or _sender_email(message) in allowed_image_senders(user)
|
||||
|
||||
parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
|
||||
html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Saved searches — stored queries shown as virtual mailboxes (SPEC §12 M2)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.errors import raise_api_error
|
||||
from app.db import get_session
|
||||
from app.identity.deps import CurrentUser, require_csrf
|
||||
from app.models import SavedSearch
|
||||
|
||||
router = APIRouter(prefix="/saved-searches", tags=["search"], dependencies=[Depends(require_csrf)])
|
||||
|
||||
DbSession = Annotated[AsyncSession, Depends(get_session)]
|
||||
|
||||
|
||||
class SavedSearchOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
query: str
|
||||
pinned: bool
|
||||
|
||||
|
||||
class SavedSearchIn(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=80)
|
||||
query: str = Field(min_length=1, max_length=500)
|
||||
pinned: bool = True
|
||||
|
||||
|
||||
def _out(s: SavedSearch) -> SavedSearchOut:
|
||||
return SavedSearchOut(id=s.id, name=s.name, query=s.query, pinned=s.pinned)
|
||||
|
||||
|
||||
@router.get("", response_model=list[SavedSearchOut])
|
||||
async def list_saved_searches(user: CurrentUser, db: DbSession) -> list[SavedSearchOut]:
|
||||
rows = await db.scalars(
|
||||
select(SavedSearch).where(SavedSearch.user_id == user.id).order_by(SavedSearch.name)
|
||||
)
|
||||
return [_out(s) for s in rows]
|
||||
|
||||
|
||||
@router.post("", response_model=SavedSearchOut)
|
||||
async def create_saved_search(
|
||||
body: SavedSearchIn, user: CurrentUser, db: DbSession
|
||||
) -> SavedSearchOut:
|
||||
name = body.name.strip()
|
||||
query = body.query.strip()
|
||||
if not name or not query:
|
||||
raise_api_error(400, "invalid", "name and query are required")
|
||||
# Upsert by (user, name): re-saving under an existing name updates its query.
|
||||
existing = await db.scalar(
|
||||
select(SavedSearch).where(SavedSearch.user_id == user.id, SavedSearch.name == name)
|
||||
)
|
||||
if existing is not None:
|
||||
existing.query = query
|
||||
existing.pinned = body.pinned
|
||||
saved = existing
|
||||
else:
|
||||
saved = SavedSearch(user_id=user.id, name=name, query=query, pinned=body.pinned)
|
||||
db.add(saved)
|
||||
await db.commit()
|
||||
return _out(saved)
|
||||
|
||||
|
||||
@router.delete("/{search_id}")
|
||||
async def delete_saved_search(
|
||||
search_id: uuid.UUID, user: CurrentUser, db: DbSession
|
||||
) -> dict[str, str]:
|
||||
saved = await db.scalar(
|
||||
select(SavedSearch).where(SavedSearch.id == search_id, SavedSearch.user_id == user.id)
|
||||
)
|
||||
if saved is None:
|
||||
raise_api_error(404, "not_found", "saved search not found")
|
||||
await db.delete(saved)
|
||||
await db.commit()
|
||||
return {"status": "deleted"}
|
||||
@@ -29,6 +29,7 @@ class ThreadSummary(BaseModel):
|
||||
class ThreadListOut(BaseModel):
|
||||
threads: list[ThreadSummary]
|
||||
cursor: str | None = None
|
||||
total: int = 0
|
||||
|
||||
|
||||
class AttachmentOut(BaseModel):
|
||||
@@ -54,6 +55,8 @@ class MessageEnvelope(BaseModel):
|
||||
is_starred: bool
|
||||
has_attachments: bool
|
||||
has_remote_images: bool
|
||||
# The sender is on the user's trusted list, so remote images load without a prompt.
|
||||
images_allowed: bool = False
|
||||
size_bytes: int
|
||||
snippet: str
|
||||
# Other accounts this same delivery also landed in (collapsed duplicates).
|
||||
@@ -84,6 +87,9 @@ class AccountOut(BaseModel):
|
||||
color: str
|
||||
state: str
|
||||
signature: str = ""
|
||||
sync_window: str = "all"
|
||||
backfill_total: int = 0
|
||||
backfill_done: int = 0
|
||||
|
||||
|
||||
class SearchThreadHit(BaseModel):
|
||||
|
||||
+34
-21
@@ -3,13 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Collection
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import Text, func, literal, or_, select, tuple_
|
||||
from sqlalchemy import Text, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from app.api.cursors import decode_cursor, encode_cursor
|
||||
from app.api.errors import raise_api_error
|
||||
from app.api.schemas import (
|
||||
Address,
|
||||
@@ -18,6 +19,7 @@ from app.api.schemas import (
|
||||
ThreadListOut,
|
||||
ThreadSummary,
|
||||
)
|
||||
from app.api.settings import allowed_image_senders
|
||||
from app.db import get_session
|
||||
from app.identity.deps import CurrentUser
|
||||
from app.models import Mailbox, Message, Thread, User
|
||||
@@ -48,7 +50,8 @@ def _summary(thread: Thread, message_count: int | None = None) -> ThreadSummary:
|
||||
)
|
||||
|
||||
|
||||
def _envelope(msg: Message) -> MessageEnvelope:
|
||||
def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnvelope:
|
||||
sender = msg.from_addr[0].get("email", "").lower() if msg.from_addr else ""
|
||||
return MessageEnvelope(
|
||||
id=msg.id,
|
||||
account_id=msg.account_id,
|
||||
@@ -65,6 +68,7 @@ def _envelope(msg: Message) -> MessageEnvelope:
|
||||
is_starred="\\Flagged" in msg.flags,
|
||||
has_attachments=msg.has_attachments,
|
||||
has_remote_images=msg.has_remote_images,
|
||||
images_allowed=bool(sender) and sender in allowed_senders,
|
||||
size_bytes=msg.size_bytes,
|
||||
snippet=msg.snippet,
|
||||
)
|
||||
@@ -75,10 +79,14 @@ async def list_threads(
|
||||
user: CurrentUser,
|
||||
db: DbSession,
|
||||
view: str = Query("unified"),
|
||||
cursor: str | None = None,
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
offset: int = Query(0, ge=0),
|
||||
filter: str = Query("all"), # all | unread | pinned
|
||||
order: str = Query("desc"), # desc (newest first) | asc (oldest first)
|
||||
) -> ThreadListOut:
|
||||
stmt = select(Thread).where(Thread.user_id == user.id)
|
||||
# Collect WHERE conditions once so the page query and the total-count query
|
||||
# stay in lockstep (offset pagination needs an accurate grand total).
|
||||
conditions: list[ColumnElement[bool]] = [Thread.user_id == user.id]
|
||||
|
||||
if view.startswith("account:"):
|
||||
# A single account's inbox: threads with a message in THIS account's
|
||||
@@ -97,7 +105,7 @@ async def list_threads(
|
||||
Mailbox.special_use == "inbox",
|
||||
)
|
||||
)
|
||||
stmt = stmt.where(Thread.id.in_(in_account_inbox))
|
||||
conditions.append(Thread.id.in_(in_account_inbox))
|
||||
elif view.startswith("folder:"):
|
||||
# Threads with at least one message in a mailbox of this special_use,
|
||||
# across all of the user's accounts (unified per-folder view).
|
||||
@@ -107,24 +115,28 @@ async def list_threads(
|
||||
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||||
.where(Message.user_id == user.id, Mailbox.special_use == special)
|
||||
)
|
||||
stmt = stmt.where(Thread.id.in_(in_folder))
|
||||
conditions.append(Thread.id.in_(in_folder))
|
||||
elif view != "unified":
|
||||
raise_api_error(400, "bad_view", "unsupported view")
|
||||
|
||||
decoded = decode_cursor(cursor)
|
||||
if decoded is not None:
|
||||
cur_date, cur_id = decoded
|
||||
stmt = stmt.where(
|
||||
tuple_(Thread.last_message_at, Thread.id) < tuple_(literal(cur_date), literal(cur_id))
|
||||
)
|
||||
stmt = stmt.order_by(Thread.last_message_at.desc(), Thread.id.desc()).limit(limit + 1)
|
||||
if filter == "unread":
|
||||
conditions.append(Thread.unread_count > 0)
|
||||
elif filter == "pinned":
|
||||
conditions.append(Thread.pinned.is_(True))
|
||||
|
||||
threads = list(await db.scalars(stmt))
|
||||
has_more = len(threads) > limit
|
||||
threads = threads[:limit]
|
||||
next_cursor = (
|
||||
encode_cursor(threads[-1].last_message_at, threads[-1].id) if has_more and threads else None
|
||||
total = (
|
||||
await db.scalar(
|
||||
select(func.count()).select_from(select(Thread.id).where(*conditions).subquery())
|
||||
)
|
||||
) or 0
|
||||
|
||||
order_by = (
|
||||
(Thread.last_message_at.asc(), Thread.id.asc())
|
||||
if order == "asc"
|
||||
else (Thread.last_message_at.desc(), Thread.id.desc())
|
||||
)
|
||||
stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit)
|
||||
threads = list(await db.scalars(stmt))
|
||||
|
||||
# Per-thread visible message count matching what get_thread shows for this view:
|
||||
# Trash/Spam count only their own messages; every other view excludes them. This
|
||||
@@ -152,7 +164,7 @@ async def list_threads(
|
||||
}
|
||||
|
||||
return ThreadListOut(
|
||||
threads=[_summary(t, counts.get(t.id)) for t in threads], cursor=next_cursor
|
||||
threads=[_summary(t, counts.get(t.id)) for t in threads], total=total
|
||||
)
|
||||
|
||||
|
||||
@@ -216,9 +228,10 @@ async def get_thread(
|
||||
elif m.account_id != rep.account_id and m.account_id not in also[key]:
|
||||
also[key].append(m.account_id)
|
||||
|
||||
allowed_senders = allowed_image_senders(user)
|
||||
envelopes = []
|
||||
for m in ordered:
|
||||
env = _envelope(m)
|
||||
env = _envelope(m, allowed_senders)
|
||||
env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"]
|
||||
envelopes.append(env)
|
||||
|
||||
|
||||
@@ -82,6 +82,11 @@ class Account(Base):
|
||||
discovery_src: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
state: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'backfilling'"))
|
||||
state_detail: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''"))
|
||||
# How much history to import (SPEC §12 M2): all | 3months | new. Drives the
|
||||
# initial backfill; backfill_total/backfill_done feed the progress UX.
|
||||
sync_window: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'all'"))
|
||||
backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
|
||||
backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
|
||||
last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))
|
||||
created_at: Mapped[dt.datetime] = _now()
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
@@ -71,9 +71,25 @@ class MailProvider(Protocol):
|
||||
async def list_mailboxes(self) -> list[RemoteMailbox]: ...
|
||||
|
||||
def backfill(
|
||||
self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
|
||||
self,
|
||||
mailbox: str,
|
||||
*,
|
||||
newest_first: bool = True,
|
||||
batch_size: int = 200,
|
||||
since: date | None = None,
|
||||
before: date | None = None,
|
||||
) -> AsyncIterator[list[RemoteMessage]]:
|
||||
"""Yields batches until the mailbox is exhausted. Headers+body in one pass."""
|
||||
"""Yields batches until the mailbox is exhausted. Headers+body in one pass.
|
||||
``since``/``before`` bound the import by internal date (SPEC §12 M2)."""
|
||||
...
|
||||
|
||||
async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
|
||||
"""Number of messages to import (progress denominator); whole mailbox or
|
||||
the ``since`` window."""
|
||||
...
|
||||
|
||||
async def newest_uid(self, mailbox: str) -> int:
|
||||
"""Current high-water UID; used by the ``new`` window to skip history."""
|
||||
...
|
||||
|
||||
async def poll(
|
||||
|
||||
@@ -13,6 +13,7 @@ import contextlib
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
|
||||
import aiosmtplib
|
||||
from aioimaplib import aioimaplib
|
||||
@@ -42,6 +43,21 @@ _CONNECT_TIMEOUT = 20
|
||||
_PER_ACCOUNT_CONN_CAP = 4 # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
|
||||
_APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
|
||||
_COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")
|
||||
_IMAP_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
|
||||
|
||||
|
||||
def _imap_date(d: date) -> str:
|
||||
"""RFC 3501 search date, e.g. ``5-Jul-2026`` (locale-independent)."""
|
||||
return f"{d.day}-{_IMAP_MONTHS[d.month - 1]}-{d.year}"
|
||||
|
||||
|
||||
def _date_criteria(since: date | None, before: date | None) -> str:
|
||||
parts: list[str] = []
|
||||
if since is not None:
|
||||
parts.append(f"SINCE {_imap_date(since)}")
|
||||
if before is not None:
|
||||
parts.append(f"BEFORE {_imap_date(before)}")
|
||||
return " ".join(parts) if parts else "ALL"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -184,12 +200,21 @@ class ImapSmtpProvider(MailProvider):
|
||||
return 0
|
||||
|
||||
async def backfill(
|
||||
self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
|
||||
self,
|
||||
mailbox: str,
|
||||
*,
|
||||
newest_first: bool = True,
|
||||
batch_size: int = 200,
|
||||
since: date | None = None,
|
||||
before: date | None = None,
|
||||
) -> AsyncIterator[list[RemoteMessage]]:
|
||||
"""Yield message batches. ``since``/``before`` bound the import by IMAP
|
||||
internal date (``SINCE`` inclusive, ``BEFORE`` exclusive) so the caller
|
||||
can sync a recent window and/or the older remainder (SPEC §12 M2)."""
|
||||
info = await self._select(mailbox)
|
||||
self._uidvalidity_seen[mailbox] = info.get("uidvalidity", 0)
|
||||
imap = await self._connect()
|
||||
search = await imap.uid_search("ALL")
|
||||
search = await imap.uid_search(_date_criteria(since, before))
|
||||
uids = parse_search_uids(search.lines)
|
||||
uids.sort(reverse=newest_first)
|
||||
for start in range(0, len(uids), batch_size):
|
||||
@@ -198,6 +223,22 @@ class ImapSmtpProvider(MailProvider):
|
||||
if messages:
|
||||
yield messages
|
||||
|
||||
async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
|
||||
"""Number of messages to import for the progress denominator. Uses the
|
||||
cheap ``EXISTS`` from SELECT for the whole mailbox, or a dated SEARCH."""
|
||||
info = await self._select(mailbox)
|
||||
if since is None:
|
||||
return int(info.get("exists", 0))
|
||||
imap = await self._connect()
|
||||
search = await imap.uid_search(_date_criteria(since, None))
|
||||
return len(parse_search_uids(search.lines))
|
||||
|
||||
async def newest_uid(self, mailbox: str) -> int:
|
||||
"""Current high-water UID (``UIDNEXT - 1``); 0 for an empty mailbox. Used
|
||||
by the ``new`` sync window to skip history and only pull future mail."""
|
||||
info = await self._select(mailbox)
|
||||
return max(0, int(info.get("uidnext", 1)) - 1)
|
||||
|
||||
async def list_uids(self, mailbox: str) -> set[int]:
|
||||
"""Full set of UIDs currently present in the mailbox (expunge reconciliation).
|
||||
|
||||
|
||||
@@ -10,12 +10,14 @@ _backend: SearchBackend | None = None
|
||||
|
||||
|
||||
def get_search_backend() -> SearchBackend:
|
||||
"""Return the process-wide SearchBackend (``pg`` in M0-M1, ``meili`` in M2)."""
|
||||
"""Return the process-wide SearchBackend (``pg`` or ``meili`` per SEARCH_BACKEND)."""
|
||||
global _backend
|
||||
if _backend is None:
|
||||
which = get_settings().search_backend
|
||||
if which == "meili":
|
||||
# TODO(spec): Meilisearch backend lands in M2.
|
||||
raise NotImplementedError("meili backend is implemented in M2")
|
||||
_backend = PgSearchBackend()
|
||||
from app.search.meili import MeiliSearchBackend
|
||||
|
||||
_backend = MeiliSearchBackend()
|
||||
else:
|
||||
_backend = PgSearchBackend()
|
||||
return _backend
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
# INTERFACE VERSION: 1
|
||||
"""Meilisearch SearchBackend (SPEC §5.2, §8; M2).
|
||||
|
||||
Typo-tolerant free-text search with highlighted snippets over a single ``messages``
|
||||
index. Every query is filtered by ``user_id`` (invariant I-7). Structured §8.2
|
||||
operators that map to filterable attributes — ``is`` / ``has`` / ``in`` / date /
|
||||
size — become Meilisearch filters; the field operators (``from`` / ``to`` / ``cc``
|
||||
/ ``subject`` / ``filename``) fold into the query text, since Meilisearch 1.8 has
|
||||
no substring filter (the PG backend remains available for exact operator queries).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import datetime as dt
|
||||
import re
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from meilisearch_python_sdk import AsyncClient
|
||||
|
||||
from app.config import get_settings
|
||||
from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
|
||||
|
||||
_INDEX = "messages"
|
||||
|
||||
# Priority order matters: subject/senders rank above the body.
|
||||
_SEARCHABLE = [
|
||||
"subject",
|
||||
"from_names",
|
||||
"from_emails",
|
||||
"to_names",
|
||||
"to_emails",
|
||||
"cc_names",
|
||||
"cc_emails",
|
||||
"body",
|
||||
"attachment_names",
|
||||
]
|
||||
_FILTERABLE = [
|
||||
"user_id",
|
||||
"account_id",
|
||||
"special",
|
||||
"has_attachments",
|
||||
"is_unread",
|
||||
"is_starred",
|
||||
"date",
|
||||
"size_bytes",
|
||||
]
|
||||
_TEXT_FIELD_OPS = {"from", "to", "cc", "subject", "filename"}
|
||||
_RELATIVE_DAYS = {"d": 1, "w": 7, "m": 30, "y": 365}
|
||||
|
||||
|
||||
def _emails(field: list[dict[str, str]]) -> list[str]:
|
||||
return [(a.get("email") or "").lower() for a in field if a.get("email")]
|
||||
|
||||
|
||||
def _names(field: list[dict[str, str]]) -> list[str]:
|
||||
return [a.get("name") or "" for a in field if a.get("name")]
|
||||
|
||||
|
||||
def _to_doc(doc: SearchDoc) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(doc.message_id),
|
||||
"thread_id": str(doc.thread_id),
|
||||
"user_id": str(doc.user_id),
|
||||
"account_id": str(doc.account_id),
|
||||
"special": doc.mailbox_special_use or "",
|
||||
"subject": doc.subject,
|
||||
"body": doc.body_text,
|
||||
"from_names": _names(doc.from_),
|
||||
"from_emails": _emails(doc.from_),
|
||||
"to_names": _names(doc.to),
|
||||
"to_emails": _emails(doc.to),
|
||||
"cc_names": _names(doc.cc),
|
||||
"cc_emails": _emails(doc.cc),
|
||||
"attachment_names": doc.attachment_names,
|
||||
"has_attachments": doc.has_attachments,
|
||||
"is_unread": doc.is_unread,
|
||||
"is_starred": doc.is_starred,
|
||||
"date": doc.date,
|
||||
"size_bytes": doc.size_bytes,
|
||||
}
|
||||
|
||||
|
||||
def _parse_date_unix(value: str) -> int | None:
|
||||
try:
|
||||
d = dt.datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=dt.UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
return int(d.timestamp())
|
||||
|
||||
|
||||
def _parse_relative_days(value: str) -> int | None:
|
||||
m = re.fullmatch(r"(\d+)([dwmy])", value.strip().lower())
|
||||
return int(m.group(1)) * _RELATIVE_DAYS[m.group(2)] if m else None
|
||||
|
||||
|
||||
def _parse_size(value: str) -> int | None:
|
||||
m = re.fullmatch(r"(\d+)([km])", value.strip().lower())
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) * (1024 if m.group(2) == "k" else 1024 * 1024)
|
||||
|
||||
|
||||
def _op_filter(key: str, value: str) -> str | None:
|
||||
"""A single §8.2 operator -> a Meilisearch filter fragment (or None)."""
|
||||
now = int(dt.datetime.now(dt.UTC).timestamp())
|
||||
if key == "in":
|
||||
return f'special = "{value.lower()}"'
|
||||
if key == "is":
|
||||
v = value.lower()
|
||||
if v == "unread":
|
||||
return "is_unread = true"
|
||||
if v == "read":
|
||||
return "is_unread = false"
|
||||
if v == "starred":
|
||||
return "is_starred = true"
|
||||
return None # 'answered' is not indexed in SearchDoc
|
||||
if key == "has":
|
||||
return "has_attachments = true" if value.lower() == "attachment" else None
|
||||
if key == "before":
|
||||
u = _parse_date_unix(value)
|
||||
return f"date < {u}" if u is not None else None
|
||||
if key == "after":
|
||||
u = _parse_date_unix(value)
|
||||
return f"date >= {u}" if u is not None else None
|
||||
if key == "newer_than":
|
||||
days = _parse_relative_days(value)
|
||||
return f"date >= {now - days * 86400}" if days is not None else None
|
||||
if key == "older_than":
|
||||
days = _parse_relative_days(value)
|
||||
return f"date < {now - days * 86400}" if days is not None else None
|
||||
if key == "larger":
|
||||
n = _parse_size(value)
|
||||
return f"size_bytes > {n}" if n is not None else None
|
||||
if key == "smaller":
|
||||
n = _parse_size(value)
|
||||
return f"size_bytes < {n}" if n is not None else None
|
||||
return None
|
||||
|
||||
|
||||
def _build_filter(groups: list[list[dict[str, Any]]], user_id: UUID) -> str:
|
||||
ands = [f'user_id = "{user_id}"']
|
||||
for group in groups:
|
||||
ors: list[str] = []
|
||||
for clause in group:
|
||||
if clause.get("type") == "text":
|
||||
continue
|
||||
frag = _op_filter(str(clause.get("key")), str(clause.get("value", "")))
|
||||
if frag is None:
|
||||
continue
|
||||
ors.append(f"NOT ({frag})" if clause.get("negate") else frag)
|
||||
if ors:
|
||||
ands.append("(" + " OR ".join(ors) + ")")
|
||||
return " AND ".join(ands)
|
||||
|
||||
|
||||
def _query_text(groups: list[list[dict[str, Any]]]) -> str:
|
||||
"""Free text for Meili's `q`: text terms (negated as -term) + folded field ops."""
|
||||
terms: list[str] = []
|
||||
for group in groups:
|
||||
for clause in group:
|
||||
value = str(clause.get("value", "")).strip()
|
||||
if not value:
|
||||
continue
|
||||
if clause.get("type") == "text":
|
||||
terms.append(f"-{value}" if clause.get("negate") else value)
|
||||
elif clause.get("key") in _TEXT_FIELD_OPS and not clause.get("negate"):
|
||||
terms.append(value)
|
||||
return " ".join(terms).strip()
|
||||
|
||||
|
||||
class MeiliSearchBackend(SearchBackend):
|
||||
def __init__(self) -> None:
|
||||
cfg = get_settings()
|
||||
self._client = AsyncClient(cfg.meili_url, cfg.meili_key or None)
|
||||
self._index = self._client.index(_INDEX)
|
||||
self._ready = False
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def _ensure(self) -> None:
|
||||
"""Create the index and apply settings once (idempotent)."""
|
||||
if self._ready:
|
||||
return
|
||||
async with self._lock:
|
||||
if self._ready:
|
||||
return
|
||||
indexes = await self._client.get_indexes()
|
||||
names = {i.uid for i in indexes} if indexes else set()
|
||||
if _INDEX not in names:
|
||||
await self._client.create_index(_INDEX, primary_key="id")
|
||||
await self._index.update_searchable_attributes(_SEARCHABLE)
|
||||
await self._index.update_sortable_attributes(["date"])
|
||||
task = await self._index.update_filterable_attributes(_FILTERABLE)
|
||||
await self._client.wait_for_task(task.task_uid)
|
||||
self._ready = True
|
||||
|
||||
async def index(self, doc: SearchDoc) -> None:
|
||||
await self._ensure()
|
||||
await self._index.add_documents([_to_doc(doc)])
|
||||
|
||||
async def delete(self, message_id: UUID) -> None:
|
||||
await self._ensure()
|
||||
await self._index.delete_document(str(message_id))
|
||||
|
||||
async def delete_user(self, user_id: UUID) -> None:
|
||||
await self._ensure()
|
||||
await self._index.delete_documents_by_filter(f'user_id = "{user_id}"')
|
||||
|
||||
async def query(
|
||||
self, q: SearchQuery, *, limit: int, cursor: str | None
|
||||
) -> tuple[list[SearchHit], str | None]:
|
||||
await self._ensure()
|
||||
groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
|
||||
assert isinstance(groups, list)
|
||||
offset = _decode_cursor(cursor)
|
||||
res = await self._index.search(
|
||||
_query_text(groups),
|
||||
filter=_build_filter(groups, q.user_id),
|
||||
offset=offset,
|
||||
limit=limit + 1,
|
||||
sort=["date:desc"], # stable order for offset paging; typo matching still applies
|
||||
attributes_to_retrieve=["id", "thread_id"],
|
||||
attributes_to_highlight=["subject", "body"],
|
||||
attributes_to_crop=["body"],
|
||||
crop_length=30,
|
||||
highlight_pre_tag="<mark>",
|
||||
highlight_post_tag="</mark>",
|
||||
show_ranking_score=True,
|
||||
)
|
||||
rows = res.hits
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
hits = [_to_hit(row) for row in rows]
|
||||
next_cursor = _encode_cursor(offset + limit) if has_more else None
|
||||
return hits, next_cursor
|
||||
|
||||
|
||||
def _to_hit(row: dict[str, Any]) -> SearchHit:
|
||||
fmt = row.get("_formatted") or {}
|
||||
highlights: dict[str, str] = {}
|
||||
subject = fmt.get("subject") or ""
|
||||
if "<mark>" in subject:
|
||||
highlights["subject"] = subject
|
||||
body = fmt.get("body") or ""
|
||||
if "<mark>" in body:
|
||||
highlights["body"] = body
|
||||
return SearchHit(
|
||||
message_id=UUID(row["id"]),
|
||||
thread_id=UUID(row["thread_id"]),
|
||||
score=float(row.get("_rankingScore") or 0.0),
|
||||
highlights=highlights,
|
||||
)
|
||||
|
||||
|
||||
def _encode_cursor(offset: int) -> str:
|
||||
return base64.urlsafe_b64encode(str(offset).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str | None) -> int:
|
||||
if not cursor:
|
||||
return 0
|
||||
try:
|
||||
return max(0, int(base64.urlsafe_b64decode(cursor.encode()).decode()))
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
@@ -23,6 +23,22 @@ from app.models import Attachment, Mailbox, Message
|
||||
from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery
|
||||
|
||||
_TS_CONFIG = "simple" # language-neutral; avoids wrong-language stemming (M0)
|
||||
|
||||
|
||||
def _prefix_tsquery(value: str) -> ColumnElement[object]:
|
||||
"""A prefix ``tsquery`` for search-as-you-type: ``invo`` finds ``invoice``.
|
||||
|
||||
We can't tokenize in Python and match the index: Postgres' parser keeps an
|
||||
email/host/URL as a *single* lexeme (``lukas.remis@gmail.com``), so splitting
|
||||
on word chars would never match it. Instead we re-parse the term with the
|
||||
same ``plainto_tsquery`` used at index time, then append ``:*`` to each
|
||||
lexeme to make it a prefix. ``plainto_tsquery`` strips tsquery operators and
|
||||
quotes, so this is injection- and error-safe on hostile input; an all-symbol
|
||||
term yields the empty query (matches nothing).
|
||||
"""
|
||||
lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text)
|
||||
prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g")
|
||||
return func.to_tsquery(_TS_CONFIG, prefixed)
|
||||
_RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}
|
||||
|
||||
|
||||
@@ -95,7 +111,10 @@ class PgSearchBackend(SearchBackend):
|
||||
def _clause_condition(self, clause: dict[str, object]) -> ColumnElement[bool] | None:
|
||||
value = str(clause.get("value", ""))
|
||||
if clause.get("type") == "text":
|
||||
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
|
||||
if clause.get("phrase"):
|
||||
# Quoted phrase: keep exact-adjacency semantics, no prefix.
|
||||
return Message.body_indexed.op("@@")(func.websearch_to_tsquery(_TS_CONFIG, value))
|
||||
return Message.body_indexed.op("@@")(_prefix_tsquery(value))
|
||||
key = clause.get("key")
|
||||
cond = self._filter_condition(str(key), value)
|
||||
return cond
|
||||
@@ -172,7 +191,7 @@ class PgSearchBackend(SearchBackend):
|
||||
conditions = self._build_conditions(groups)
|
||||
|
||||
rank = (
|
||||
func.ts_rank_cd(Message.body_indexed, func.websearch_to_tsquery(_TS_CONFIG, q.text))
|
||||
func.ts_rank_cd(Message.body_indexed, _prefix_tsquery(q.text))
|
||||
if q.text
|
||||
else literal(0.0)
|
||||
)
|
||||
|
||||
+130
-12
@@ -7,6 +7,8 @@ import contextlib
|
||||
import logging
|
||||
import random
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
@@ -27,6 +29,11 @@ logger = logging.getLogger(__name__)
|
||||
# Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3).
|
||||
_PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2}
|
||||
|
||||
# Sync-window import strategy (SPEC §12 M2). "all" imports the most recent
|
||||
# _RECENT_DAYS first (so the account is usable fast), then the older remainder.
|
||||
_RECENT_DAYS = 30
|
||||
_WINDOW_DAYS = {"3months": 90}
|
||||
|
||||
|
||||
async def _load_account(session: AsyncSession, account_id: uuid.UUID) -> Account:
|
||||
account = await session.get(Account, account_id)
|
||||
@@ -97,9 +104,8 @@ def _mailbox_order(mailboxes: list[Mailbox]) -> list[Mailbox]:
|
||||
return sorted(mailboxes, key=lambda m: _PRIORITY.get(m.special_use or "", 99))
|
||||
|
||||
|
||||
async def _emit_batch(
|
||||
results: list[PersistResult], account: Account, mailbox_path: str, done: int
|
||||
) -> None:
|
||||
async def _index_and_notify(results: list[PersistResult], account: Account) -> None:
|
||||
"""Index each persisted message and publish its ``message.new`` event."""
|
||||
backend = get_search_backend()
|
||||
bus = get_event_bus()
|
||||
for r in results:
|
||||
@@ -113,7 +119,13 @@ async def _emit_batch(
|
||||
"thread_id": str(r.message.thread_id),
|
||||
},
|
||||
)
|
||||
await bus.publish(
|
||||
|
||||
|
||||
async def _emit_batch(
|
||||
results: list[PersistResult], account: Account, mailbox_path: str, done: int
|
||||
) -> None:
|
||||
await _index_and_notify(results, account)
|
||||
await get_event_bus().publish(
|
||||
"sync.progress",
|
||||
{
|
||||
"user_id": str(account.user_id),
|
||||
@@ -124,14 +136,61 @@ async def _emit_batch(
|
||||
)
|
||||
|
||||
|
||||
async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox: Mailbox) -> int:
|
||||
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30)."""
|
||||
@dataclass
|
||||
class _BackfillProgress:
|
||||
"""Account-level backfill counter: persists ``backfill_done`` and emits
|
||||
``sync.progress`` with the running total so the UI can show a progress bar."""
|
||||
|
||||
account: Account
|
||||
total: int
|
||||
done: int = 0
|
||||
_mailboxes: set[str] = field(default_factory=set)
|
||||
|
||||
async def record(self, results: list[PersistResult], mailbox_path: str) -> None:
|
||||
from app.db import session_scope
|
||||
|
||||
await _index_and_notify(results, self.account)
|
||||
self.done += len(results)
|
||||
self._mailboxes.add(mailbox_path)
|
||||
async with session_scope() as session:
|
||||
await session.execute(
|
||||
update(Account)
|
||||
.where(Account.id == self.account.id)
|
||||
.values(backfill_done=self.done)
|
||||
)
|
||||
await get_event_bus().publish(
|
||||
"sync.progress",
|
||||
{
|
||||
"user_id": str(self.account.user_id),
|
||||
"account_id": str(self.account.id),
|
||||
"mailbox": mailbox_path,
|
||||
"done": self.done,
|
||||
"total": self.total,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def backfill_mailbox(
|
||||
provider: ImapSmtpProvider,
|
||||
account: Account,
|
||||
mailbox: Mailbox,
|
||||
progress: _BackfillProgress | None = None,
|
||||
*,
|
||||
since: date | None = None,
|
||||
before: date | None = None,
|
||||
) -> int:
|
||||
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30).
|
||||
|
||||
``since``/``before`` bound the import by date (sync window, SPEC §12 M2).
|
||||
When a ``progress`` tracker is given, batches count toward the account-level
|
||||
backfill total; otherwise each batch emits a standalone ``sync.progress``.
|
||||
"""
|
||||
from app.db import session_scope
|
||||
|
||||
account_id, user_id, mailbox_id = account.id, account.user_id, mailbox.id
|
||||
special_use, path = mailbox.special_use, mailbox.path
|
||||
done = 0
|
||||
async for batch in provider.backfill(path, batch_size=200):
|
||||
async for batch in provider.backfill(path, batch_size=200, since=since, before=before):
|
||||
results: list[PersistResult] = []
|
||||
max_uid = 0
|
||||
async with session_scope() as session:
|
||||
@@ -153,12 +212,41 @@ async def backfill_mailbox(provider: ImapSmtpProvider, account: Account, mailbox
|
||||
.values(last_seen_uid=max_uid)
|
||||
)
|
||||
done += len(results)
|
||||
await _emit_batch(results, account, path, done)
|
||||
if progress is not None:
|
||||
await progress.record(results, path)
|
||||
else:
|
||||
await _emit_batch(results, account, path, done)
|
||||
return done
|
||||
|
||||
|
||||
async def _set_backfill_counts(account_id: uuid.UUID, *, total: int, done: int) -> None:
|
||||
from app.db import session_scope
|
||||
|
||||
async with session_scope() as session:
|
||||
await session.execute(
|
||||
update(Account)
|
||||
.where(Account.id == account_id)
|
||||
.values(backfill_total=total, backfill_done=done)
|
||||
)
|
||||
|
||||
|
||||
async def _sync_new_only(provider: ImapSmtpProvider, mailboxes: list[Mailbox]) -> None:
|
||||
"""``new`` window: import no history — just move each mailbox watermark to the
|
||||
current tip so only mail arriving after setup is pulled (SPEC §12 M2)."""
|
||||
from app.db import session_scope
|
||||
|
||||
for mailbox in mailboxes:
|
||||
tip = await provider.newest_uid(mailbox.path)
|
||||
async with session_scope() as session:
|
||||
await session.execute(
|
||||
update(Mailbox)
|
||||
.where(Mailbox.id == mailbox.id, Mailbox.last_seen_uid < tip)
|
||||
.values(last_seen_uid=tip)
|
||||
)
|
||||
|
||||
|
||||
async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
|
||||
"""Full initial sync for an account: mailboxes + backfill; sets state=live."""
|
||||
"""Full initial sync for an account: mailboxes + windowed backfill; state=live."""
|
||||
from app.db import session_scope
|
||||
|
||||
async with session_scope() as session:
|
||||
@@ -167,12 +255,42 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
|
||||
mailboxes = _mailbox_order(await bootstrap_mailboxes(session, provider, account))
|
||||
# Detach: reuse account/mailboxes values after the session closes.
|
||||
account_obj = account
|
||||
window = account.sync_window
|
||||
mailbox_list = list(mailboxes)
|
||||
|
||||
total = 0
|
||||
try:
|
||||
if window == "new":
|
||||
# No backfill: set watermarks to the tip, go live immediately.
|
||||
await _sync_new_only(provider, mailbox_list)
|
||||
await _set_backfill_counts(account_id, total=0, done=0)
|
||||
await _set_state(account_id, "live")
|
||||
return 0
|
||||
|
||||
since = (
|
||||
date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None
|
||||
)
|
||||
# Progress denominator: count what we're about to import up front so the
|
||||
# bar has a stable total (SPEC §12 M2 backfill-progress UX).
|
||||
total = 0
|
||||
for mailbox in mailbox_list:
|
||||
total += await backfill_mailbox(provider, account_obj, mailbox)
|
||||
total += await provider.message_count(mailbox.path, since=since)
|
||||
await _set_backfill_counts(account_id, total=total, done=0)
|
||||
progress = _BackfillProgress(account_obj, total)
|
||||
|
||||
recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS)
|
||||
count = 0
|
||||
for mailbox in mailbox_list:
|
||||
if since is not None: # 3months: single windowed pass
|
||||
count += await backfill_mailbox(
|
||||
provider, account_obj, mailbox, progress, since=since
|
||||
)
|
||||
else: # all: recent window first (usable fast), then the remainder
|
||||
count += await backfill_mailbox(
|
||||
provider, account_obj, mailbox, progress, since=recent_cutoff
|
||||
)
|
||||
count += await backfill_mailbox(
|
||||
provider, account_obj, mailbox, progress, before=recent_cutoff
|
||||
)
|
||||
await _set_state(account_id, "live")
|
||||
except Exception as exc: # noqa: BLE001 - surface provider errors as account state
|
||||
logger.exception("backfill failed for account %s", account_id)
|
||||
@@ -180,7 +298,7 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
|
||||
raise
|
||||
finally:
|
||||
await provider.close()
|
||||
return total
|
||||
return count
|
||||
|
||||
|
||||
async def apply_delta(
|
||||
|
||||
@@ -39,6 +39,20 @@ def _recipient_emails(message: Message) -> set[str]:
|
||||
return emails
|
||||
|
||||
|
||||
def _parent_refs(message: Message) -> list[str]:
|
||||
"""Non-synthetic parent ids this message explicitly declares (References +
|
||||
In-Reply-To). A non-empty result means the message states its parentage, so it
|
||||
must not be subject-merged into an unrelated conversation (§7.3)."""
|
||||
refs: list[str] = list(message.references_hdr)
|
||||
if message.in_reply_to:
|
||||
refs.append(message.in_reply_to)
|
||||
out: list[str] = []
|
||||
for r in refs:
|
||||
if r and not is_synthetic(r) and r not in out:
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def _candidate_ids(message: Message) -> list[str]:
|
||||
candidates: list[str] = list(message.references_hdr)
|
||||
if message.in_reply_to:
|
||||
@@ -59,17 +73,26 @@ async def _find_by_references(
|
||||
user_id: uuid.UUID,
|
||||
message_id: uuid.UUID,
|
||||
candidates: list[str],
|
||||
parent_refs: list[str],
|
||||
own_id: str | None,
|
||||
) -> list[uuid.UUID]:
|
||||
"""Threads joined by reference (SPEC §7.2.2).
|
||||
|
||||
Matches bidirectionally: existing messages whose id is one of my candidate ids
|
||||
(I reference them / duplicate id), OR existing messages that reference *my* id
|
||||
(handles a late-arriving parent). Synthetic own-ids are never used to join (§7.1).
|
||||
(I reference them / duplicate id), existing messages that reference *my* id
|
||||
(late-arriving parent), OR existing messages that share one of my declared
|
||||
parents — a sibling reply to the same original, even when neither the parent
|
||||
nor a mutual reference is present. Synthetic ids are never used to join (§7.1).
|
||||
"""
|
||||
conditions: list[ColumnElement[bool]] = []
|
||||
if candidates:
|
||||
conditions.append(Message.message_id_hdr.in_(candidates))
|
||||
if parent_refs:
|
||||
# Siblings: an existing message that references, or is a reply to, one of
|
||||
# my parents. This groups replies to a common (possibly absent) original
|
||||
# without letting the subject fallback merge replies to *different* ones.
|
||||
conditions.append(Message.references_hdr.overlap(parent_refs))
|
||||
conditions.append(Message.in_reply_to.in_(parent_refs))
|
||||
if own_id is not None and not is_synthetic(own_id):
|
||||
# own_id = ANY(references_hdr): an existing message that references me.
|
||||
conditions.append(literal(own_id) == sa_any(Message.references_hdr))
|
||||
@@ -191,10 +214,17 @@ async def thread_message(session: AsyncSession, message: Message) -> uuid.UUID:
|
||||
Returns the resulting thread id and sets ``message.thread_id``.
|
||||
"""
|
||||
candidates = _candidate_ids(message)
|
||||
parent_refs = _parent_refs(message)
|
||||
own_id = message.message_id_hdr
|
||||
thread_ids = await _find_by_references(session, message.user_id, message.id, candidates, own_id)
|
||||
thread_ids = await _find_by_references(
|
||||
session, message.user_id, message.id, candidates, parent_refs, own_id
|
||||
)
|
||||
|
||||
if not thread_ids:
|
||||
# Subject fallback only for messages that declare no parentage (a broken/absent
|
||||
# References chain). A message that explicitly replies to a specific parent is
|
||||
# never subject-merged into an unrelated thread — e.g. two auto-replies to
|
||||
# different originals that happen to share the same subject (§7.3).
|
||||
if not thread_ids and not parent_refs:
|
||||
subject_thread = await _find_by_subject(session, message.user_id, message)
|
||||
if subject_thread is not None:
|
||||
thread_ids = [subject_thread]
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""add accounts.sync_window + backfill progress counters
|
||||
|
||||
Lets the user choose at account setup how much history to import (SPEC §12 M2):
|
||||
``all`` (default; recent 30d first, then the rest), ``3months`` (windowed), or
|
||||
``new`` (no backfill — only mail arriving after setup). ``backfill_total`` /
|
||||
``backfill_done`` drive the initial-sync progress UX; total 0 means "nothing to
|
||||
import" (the ``new`` window, or an empty account).
|
||||
|
||||
Revision ID: 0012
|
||||
Revises: 0011
|
||||
Create Date: 2026-07-23
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0012"
|
||||
down_revision: str | None = "0011"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("ALTER TABLE accounts ADD COLUMN sync_window text NOT NULL DEFAULT 'all'")
|
||||
op.execute("ALTER TABLE accounts ADD COLUMN backfill_total integer NOT NULL DEFAULT 0")
|
||||
op.execute("ALTER TABLE accounts ADD COLUMN backfill_done integer NOT NULL DEFAULT 0")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS backfill_done")
|
||||
op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS backfill_total")
|
||||
op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS sync_window")
|
||||
@@ -28,6 +28,7 @@ dependencies = [
|
||||
"nh3~=0.2",
|
||||
"cryptography~=43.0",
|
||||
"mail-parser~=4.0",
|
||||
"meilisearch-python-sdk==2.7.0", # last release that fits the pinned Meilisearch v1.8
|
||||
"python-multipart~=0.0.9",
|
||||
# DEPENDENCY: SPEC §6.2 mandates accounts/presets.yaml (config, not code); the
|
||||
# stdlib has no YAML parser, so PyYAML is required to load it.
|
||||
|
||||
@@ -378,12 +378,13 @@ async def test_accounts_list(api: ApiCtx) -> None:
|
||||
assert resp.json()[0]["email"] == "me@example.com"
|
||||
|
||||
|
||||
async def test_thread_pagination_cursor(api: ApiCtx) -> None:
|
||||
first = await api.client.get("/api/threads", params={"limit": 1})
|
||||
async def test_thread_pagination_offset(api: ApiCtx) -> None:
|
||||
first = await api.client.get("/api/threads", params={"limit": 1, "offset": 0})
|
||||
body = first.json()
|
||||
assert len(body["threads"]) == 1
|
||||
assert body["cursor"] is not None
|
||||
second = await api.client.get("/api/threads", params={"limit": 1, "cursor": body["cursor"]})
|
||||
assert body["total"] >= 2 # more than one page of threads exists
|
||||
second = await api.client.get("/api/threads", params={"limit": 1, "offset": 1})
|
||||
body2 = second.json()
|
||||
assert len(body2["threads"]) == 1
|
||||
assert body2["total"] == body["total"] # grand total is stable across pages
|
||||
assert body2["threads"][0]["id"] != body["threads"][0]["id"]
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Saved searches CRUD — stored queries shown as virtual mailboxes (SPEC §12 M2)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.config import get_settings
|
||||
from app.identity.passwords import hash_password
|
||||
from app.identity.sessions import create_session
|
||||
from app.main import create_app
|
||||
from app.models import User
|
||||
|
||||
CSRF = "csrf"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Ctx:
|
||||
client: httpx.AsyncClient
|
||||
user_id: uuid.UUID
|
||||
|
||||
|
||||
def _hdr() -> dict[str, str]:
|
||||
return {"X-CSRF": CSRF}
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def ctx() -> AsyncIterator[Ctx]:
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
session = sm()
|
||||
user = User(handle=f"ss{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
token = await create_session(user.id, "pytest")
|
||||
client = httpx.AsyncClient(
|
||||
transport=ASGITransport(app=create_app()),
|
||||
base_url="http://testserver",
|
||||
cookies={"session": token, "csrf": CSRF},
|
||||
)
|
||||
try:
|
||||
yield Ctx(client=client, user_id=user.id)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await session.execute(delete(User).where(User.id == user.id))
|
||||
await session.commit()
|
||||
await session.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_saved_search_crud_and_upsert(ctx: Ctx) -> None:
|
||||
# Empty to start.
|
||||
resp = await ctx.client.get("/api/saved-searches")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == []
|
||||
|
||||
# Create.
|
||||
resp = await ctx.client.post(
|
||||
"/api/saved-searches",
|
||||
json={"name": "Unread from boss", "query": "is:unread from:boss"},
|
||||
headers=_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
created = resp.json()
|
||||
assert created["name"] == "Unread from boss"
|
||||
assert created["query"] == "is:unread from:boss"
|
||||
assert created["pinned"] is True
|
||||
sid = created["id"]
|
||||
|
||||
# It lists back.
|
||||
resp = await ctx.client.get("/api/saved-searches")
|
||||
assert [s["name"] for s in resp.json()] == ["Unread from boss"]
|
||||
|
||||
# Re-saving under the same name upserts (updates query, keeps a single row).
|
||||
resp = await ctx.client.post(
|
||||
"/api/saved-searches",
|
||||
json={"name": "Unread from boss", "query": "is:unread from:ceo"},
|
||||
headers=_hdr(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["id"] == sid # same row
|
||||
assert resp.json()["query"] == "is:unread from:ceo"
|
||||
|
||||
rows = (await ctx.client.get("/api/saved-searches")).json()
|
||||
assert len(rows) == 1 and rows[0]["query"] == "is:unread from:ceo"
|
||||
|
||||
# Delete.
|
||||
resp = await ctx.client.delete(f"/api/saved-searches/{sid}", headers=_hdr())
|
||||
assert resp.status_code == 200
|
||||
assert (await ctx.client.get("/api/saved-searches")).json() == []
|
||||
|
||||
# Deleting a missing row 404s.
|
||||
resp = await ctx.client.delete(f"/api/saved-searches/{uuid.uuid4()}", headers=_hdr())
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
async def test_saved_search_requires_csrf(ctx: Ctx) -> None:
|
||||
# No CSRF header -> rejected (state-changing route).
|
||||
resp = await ctx.client.post(
|
||||
"/api/saved-searches", json={"name": "x", "query": "y"}
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
async def test_saved_search_is_per_user(ctx: Ctx) -> None:
|
||||
await ctx.client.post(
|
||||
"/api/saved-searches",
|
||||
json={"name": "Mine", "query": "in:inbox"},
|
||||
headers=_hdr(),
|
||||
)
|
||||
# A different user's client sees none of the above.
|
||||
other = User(handle=f"ss{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
session = sm()
|
||||
session.add(other)
|
||||
await session.commit()
|
||||
token = await create_session(other.id, "pytest")
|
||||
try:
|
||||
client = httpx.AsyncClient(
|
||||
transport=ASGITransport(app=create_app()),
|
||||
base_url="http://testserver",
|
||||
cookies={"session": token, "csrf": CSRF},
|
||||
)
|
||||
try:
|
||||
assert (await client.get("/api/saved-searches")).json() == []
|
||||
finally:
|
||||
await client.aclose()
|
||||
finally:
|
||||
await session.execute(delete(User).where(User.id == other.id))
|
||||
await session.commit()
|
||||
await session.close()
|
||||
await engine.dispose()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Meilisearch backend (SPEC §8, M2). Runs against the live Meili in the dev/CI
|
||||
stack; skips if it is unreachable so the rest of the suite still runs locally."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from app.search.base import SearchDoc
|
||||
from app.search.meili import MeiliSearchBackend
|
||||
from app.search.queryparser import to_search_query
|
||||
|
||||
|
||||
def _doc(user_id: uuid.UUID) -> SearchDoc:
|
||||
return SearchDoc(
|
||||
message_id=uuid.uuid4(),
|
||||
thread_id=uuid.uuid4(),
|
||||
user_id=user_id,
|
||||
account_id=uuid.uuid4(),
|
||||
mailbox_special_use="inbox",
|
||||
from_=[{"name": "Bob Vendor", "email": "bob@vendor.com"}],
|
||||
to=[{"name": "Me", "email": "me@x.com"}],
|
||||
cc=[],
|
||||
subject="Your invoice for March",
|
||||
body_text="Please find attached the invoice, amount due soon.",
|
||||
attachment_names=["invoice.pdf"],
|
||||
has_attachments=True,
|
||||
is_unread=True,
|
||||
is_starred=False,
|
||||
date=int(time.time()),
|
||||
size_bytes=2048,
|
||||
)
|
||||
|
||||
|
||||
async def _poll(backend: MeiliSearchBackend, q: str, user_id: uuid.UUID) -> int:
|
||||
"""Indexing is async in Meili; retry briefly until the doc is searchable."""
|
||||
for _ in range(60):
|
||||
hits, _c = await backend.query(to_search_query(q, user_id), limit=10, cursor=None)
|
||||
if hits:
|
||||
return len(hits)
|
||||
await asyncio.sleep(0.05)
|
||||
return 0
|
||||
|
||||
|
||||
async def test_meili_typo_tolerance_and_filters() -> None:
|
||||
backend = MeiliSearchBackend()
|
||||
user_id = uuid.uuid4()
|
||||
try:
|
||||
await backend.index(_doc(user_id))
|
||||
|
||||
# Acceptance (§12 M2): a typo still finds the message.
|
||||
hits, _c = await backend.query(
|
||||
to_search_query("invocie", user_id), limit=10, cursor=None
|
||||
)
|
||||
if not hits: # wait for the async index task
|
||||
assert await _poll(backend, "invocie", user_id) == 1
|
||||
hits, _c = await backend.query(
|
||||
to_search_query("invocie", user_id), limit=10, cursor=None
|
||||
)
|
||||
assert len(hits) == 1
|
||||
assert "<mark>" in hits[0].highlights.get("subject", "")
|
||||
|
||||
async def n(query: str) -> int:
|
||||
res, _c = await backend.query(to_search_query(query, user_id), limit=10, cursor=None)
|
||||
return len(res)
|
||||
|
||||
assert await n("is:unread invoice") == 1
|
||||
assert await n("is:read invoice") == 0 # the doc is unread
|
||||
assert await n("from:vendor") == 1 # field operator folded into the query
|
||||
assert await n("larger:1k invoice") == 1
|
||||
assert await n("larger:5k invoice") == 0
|
||||
assert await n("in:sent invoice") == 0 # doc is in the inbox
|
||||
except (ConnectionError, OSError) as exc: # Meili not running locally
|
||||
pytest.skip(f"Meilisearch unavailable: {exc}")
|
||||
finally:
|
||||
await backend.delete_user(user_id)
|
||||
@@ -150,6 +150,24 @@ async def test_free_text(seeded: Ctx) -> None:
|
||||
assert await _search(seeded, "lunch") == 1
|
||||
|
||||
|
||||
async def test_free_text_prefix_matches_while_typing(seeded: Ctx) -> None:
|
||||
# Search-as-you-type: a partial word finds the message before it is complete.
|
||||
assert await _search(seeded, "reven") == 1 # -> "revenue"
|
||||
assert await _search(seeded, "lun") == 1 # -> "lunch"
|
||||
# Prefix is anchored at word start, not a substring match.
|
||||
assert await _search(seeded, "enue") == 0
|
||||
# Multiple partial words are ANDed as prefixes.
|
||||
assert await _search(seeded, "quart reven") == 1 # "Quarterly revenue"
|
||||
assert await _search(seeded, "quart lun") == 0
|
||||
|
||||
|
||||
async def test_free_text_full_email_address(seeded: Ctx) -> None:
|
||||
# Postgres keeps an email as a single lexeme; searching the *whole* address
|
||||
# must still match (regression: prefix-splitting on word chars broke this).
|
||||
assert await _search(seeded, "bob@corp.com") == 1
|
||||
assert await _search(seeded, "bob@corp.co") == 1 # prefix while still typing
|
||||
|
||||
|
||||
async def test_from_operator(seeded: Ctx) -> None:
|
||||
assert await _search(seeded, "from:bob") == 1
|
||||
assert await _search(seeded, "from:corp.com") == 3 # all three share the domain
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Sync-window backfill strategies at account setup (SPEC §12 M2): all (recent
|
||||
first, then remainder), 3months (windowed), new (no history, watermark only)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from email.message import EmailMessage
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.config import get_settings
|
||||
from app.identity.passwords import hash_password
|
||||
from app.models import Account, Mailbox, Message, User
|
||||
from app.providers.base import RemoteMailbox, RemoteMessage
|
||||
from app.sync import engine as sync_engine
|
||||
|
||||
|
||||
def _raw(mid: str) -> bytes:
|
||||
m = EmailMessage()
|
||||
m["Subject"] = "hi"
|
||||
m["From"] = "me@example.com"
|
||||
m["To"] = "bob@example.com"
|
||||
m["Message-ID"] = mid
|
||||
m["Date"] = "Mon, 09 Mar 2026 08:36:37 +0100"
|
||||
m.set_content("body")
|
||||
return m.as_bytes()
|
||||
|
||||
|
||||
class _FakeProvider:
|
||||
def __init__(self) -> None:
|
||||
self.backfill_calls: list[tuple[str, date | None, date | None]] = []
|
||||
self.count_calls: list[tuple[str, date | None]] = []
|
||||
self.newest_calls: list[str] = []
|
||||
|
||||
async def list_mailboxes(self) -> list[RemoteMailbox]:
|
||||
return [RemoteMailbox(path="INBOX", special_use="inbox", uidvalidity=1)]
|
||||
|
||||
async def message_count(self, mailbox: str, *, since: date | None = None) -> int:
|
||||
self.count_calls.append((mailbox, since))
|
||||
return 3
|
||||
|
||||
async def newest_uid(self, mailbox: str) -> int:
|
||||
self.newest_calls.append(mailbox)
|
||||
return 42
|
||||
|
||||
async def backfill( # type: ignore[no-untyped-def]
|
||||
self, mailbox, *, newest_first=True, batch_size=200, since=None, before=None
|
||||
) -> AsyncIterator[list[RemoteMessage]]:
|
||||
self.backfill_calls.append((mailbox, since, before))
|
||||
# Distinct UID per pass so recent + remainder both persist a row.
|
||||
uid = 100 if since is not None else 50
|
||||
yield [
|
||||
RemoteMessage(
|
||||
uid=uid,
|
||||
raw=_raw(f"<{mailbox}-{uid}@x.com>"),
|
||||
flags=frozenset(),
|
||||
internal_date=datetime.now(UTC),
|
||||
)
|
||||
]
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def _seed(sm: async_sessionmaker, window: str) -> tuple[uuid.UUID, uuid.UUID]: # type: ignore[type-arg]
|
||||
session = sm()
|
||||
user = User(handle=f"bf{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw"))
|
||||
session.add(user)
|
||||
await session.flush()
|
||||
account = Account(
|
||||
user_id=user.id,
|
||||
email="me@example.com",
|
||||
imap_config={},
|
||||
smtp_config={},
|
||||
secret_ct=b"",
|
||||
discovery_src="manual",
|
||||
sync_window=window,
|
||||
)
|
||||
session.add(account)
|
||||
await session.commit()
|
||||
aid = account.id
|
||||
await session.close()
|
||||
return user.id, aid
|
||||
|
||||
|
||||
async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
_uid, account_id = await _seed(sm, "new")
|
||||
fake = _FakeProvider()
|
||||
monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake)
|
||||
try:
|
||||
count = await sync_engine.bootstrap_and_backfill(account_id)
|
||||
assert count == 0
|
||||
assert fake.backfill_calls == [] # no history fetched
|
||||
assert fake.newest_calls == ["INBOX"]
|
||||
|
||||
session = sm()
|
||||
acct = await session.get(Account, account_id)
|
||||
assert acct is not None and acct.state == "live" and acct.backfill_total == 0
|
||||
mb = await session.scalar(select(Mailbox).where(Mailbox.account_id == account_id))
|
||||
assert mb is not None and mb.last_seen_uid == 42 # moved to the tip
|
||||
msgs = await session.scalar(
|
||||
select(func.count()).select_from(Message).where(Message.mailbox_id == mb.id)
|
||||
)
|
||||
assert msgs == 0 # nothing imported
|
||||
await session.close()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_window_all_imports_recent_then_remainder(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
_uid, account_id = await _seed(sm, "all")
|
||||
fake = _FakeProvider()
|
||||
monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake)
|
||||
try:
|
||||
await sync_engine.bootstrap_and_backfill(account_id)
|
||||
|
||||
# Two passes per mailbox: recent window (since set), then the older
|
||||
# remainder (before set) — the "usable fast, then fill in" strategy.
|
||||
cutoff = date.today() - timedelta(days=sync_engine._RECENT_DAYS)
|
||||
assert fake.backfill_calls == [("INBOX", cutoff, None), ("INBOX", None, cutoff)]
|
||||
assert fake.count_calls == [("INBOX", None)] # denominator = whole mailbox
|
||||
|
||||
session = sm()
|
||||
acct = await session.get(Account, account_id)
|
||||
assert acct is not None and acct.state == "live"
|
||||
assert acct.backfill_total == 3 and acct.backfill_done == 2
|
||||
mb = await session.scalar(select(Mailbox).where(Mailbox.account_id == account_id))
|
||||
assert mb is not None
|
||||
msgs = await session.scalar(
|
||||
select(func.count()).select_from(Message).where(Message.mailbox_id == mb.id)
|
||||
)
|
||||
assert msgs == 2 # one from each pass
|
||||
await session.close()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_window_3months_single_dated_pass(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
_uid, account_id = await _seed(sm, "3months")
|
||||
fake = _FakeProvider()
|
||||
monkeypatch.setattr(sync_engine, "build_provider", lambda _a: fake)
|
||||
try:
|
||||
await sync_engine.bootstrap_and_backfill(account_id)
|
||||
|
||||
since = date.today() - timedelta(days=sync_engine._WINDOW_DAYS["3months"])
|
||||
assert fake.backfill_calls == [("INBOX", since, None)] # single windowed pass
|
||||
assert fake.count_calls == [("INBOX", since)] # denominator counts the window
|
||||
await engine.dispose()
|
||||
finally:
|
||||
pass
|
||||
@@ -72,6 +72,48 @@ async def test_broken_client_subject_only_joins(env: ThreadingEnv) -> None:
|
||||
assert a.thread_id == b.thread_id
|
||||
|
||||
|
||||
async def test_auto_replies_to_different_originals_do_not_join(env: ThreadingEnv) -> None:
|
||||
# Two out-of-office auto-replies, same subject and same recipient, but each
|
||||
# replying to a DIFFERENT (absent) original. They must not be subject-merged.
|
||||
a = await env.add(
|
||||
message_id="autoA@tvp.pl",
|
||||
subject="Automatyczna odpowiedź: Silent Hill",
|
||||
from_email="robert@tvp.pl",
|
||||
to_email="me@gmail.com",
|
||||
in_reply_to="orig1@mail.gmail.com",
|
||||
references=["orig1@mail.gmail.com"],
|
||||
)
|
||||
b = await env.add(
|
||||
message_id="autoB@wfdif.com.pl",
|
||||
subject="Automatyczna odpowiedź: Silent Hill",
|
||||
from_email="dyrektor@wfdif.com.pl",
|
||||
to_email="me@gmail.com",
|
||||
in_reply_to="orig2@mail.gmail.com",
|
||||
references=["orig2@mail.gmail.com"],
|
||||
minutes=5,
|
||||
)
|
||||
assert a.thread_id != b.thread_id
|
||||
|
||||
|
||||
async def test_replies_to_same_absent_parent_join(env: ThreadingEnv) -> None:
|
||||
# Two replies that reference the SAME (absent) original — and not each other —
|
||||
# still thread together via the shared parent reference.
|
||||
a = await env.add(
|
||||
message_id="r1@x.com",
|
||||
subject="Re: Topic",
|
||||
in_reply_to="orig@x.com",
|
||||
references=["orig@x.com"],
|
||||
)
|
||||
b = await env.add(
|
||||
message_id="r2@x.com",
|
||||
subject="Re: Topic",
|
||||
in_reply_to="orig@x.com",
|
||||
references=["orig@x.com"],
|
||||
minutes=3,
|
||||
)
|
||||
assert a.thread_id == b.thread_id
|
||||
|
||||
|
||||
async def test_distinct_topic_same_subject_does_not_join(env: ThreadingEnv) -> None:
|
||||
a = await env.add(
|
||||
message_id="a@x.com",
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
// Initial-sync progress UX (SPEC §12 M2). Shows a small floating card while any
|
||||
// account is importing its history; reads the accounts query, which the WS
|
||||
// `sync.progress` event invalidates so the bar advances live.
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import { useT } from "../i18n";
|
||||
import { api } from "../lib/api";
|
||||
|
||||
export function BackfillBanner() {
|
||||
const t = useT();
|
||||
const accounts = useQuery({ queryKey: ["accounts"], queryFn: api.accounts });
|
||||
|
||||
const importing = (accounts.data ?? []).filter(
|
||||
(a) => a.state === "backfilling" && a.backfill_total > 0,
|
||||
);
|
||||
const active = importing.length > 0;
|
||||
|
||||
// Dissolve the card 10s after it appears; the sidebar ring keeps showing
|
||||
// progress until the sync finishes.
|
||||
const [dismissed, setDismissed] = useState(false);
|
||||
useEffect(() => {
|
||||
if (!active) {
|
||||
setDismissed(false);
|
||||
return;
|
||||
}
|
||||
const timer = window.setTimeout(() => setDismissed(true), 10000);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [active]);
|
||||
|
||||
if (!active) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`fixed bottom-4 left-4 z-40 w-72 space-y-2 transition-opacity duration-700 ${
|
||||
dismissed ? "pointer-events-none opacity-0" : "opacity-100"
|
||||
}`}
|
||||
>
|
||||
{importing.map((a) => {
|
||||
const done = Math.min(a.backfill_done, a.backfill_total);
|
||||
const pct = Math.round((done / a.backfill_total) * 100);
|
||||
return (
|
||||
<div
|
||||
key={a.id}
|
||||
className="rounded-lg border border-app-border bg-app-surface p-3 shadow-lg"
|
||||
>
|
||||
<div className="mb-1 flex items-center gap-2">
|
||||
<span className="h-2.5 w-2.5 shrink-0 rounded-full" style={{ backgroundColor: a.color }} />
|
||||
<span className="truncate text-xs font-medium text-app-fg">{a.email}</span>
|
||||
</div>
|
||||
<div className="mb-1.5 text-xs text-app-muted">
|
||||
{t("backfill.importing", {
|
||||
done: done.toLocaleString(),
|
||||
total: a.backfill_total.toLocaleString(),
|
||||
})}
|
||||
</div>
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-app-hover">
|
||||
<div
|
||||
className="h-full rounded-full bg-app-accent transition-all duration-500"
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -15,7 +15,9 @@ import { Attachments } from "./Attachments";
|
||||
export function MessageBody({ message }: { message: MessageEnvelope }) {
|
||||
const t = useT();
|
||||
const queryClient = useQueryClient();
|
||||
const [allowed, setAllowed] = useState(false);
|
||||
// Seed from the server's trusted-sender flag so an already-approved sender
|
||||
// loads images and shows no "blocked" prompt on reopen.
|
||||
const [allowed, setAllowed] = useState(message.images_allowed);
|
||||
const frameRef = useRef<HTMLIFrameElement>(null);
|
||||
const observerRef = useRef<ResizeObserver | null>(null);
|
||||
const [height, setHeight] = useState(80);
|
||||
@@ -23,7 +25,10 @@ export function MessageBody({ message }: { message: MessageEnvelope }) {
|
||||
const { data, isLoading, isError } = useQuery({
|
||||
queryKey: ["message-html", message.id, allowed],
|
||||
queryFn: async () => {
|
||||
const resp = await fetch(`/api/messages/${message.id}/html`, { credentials: "include" });
|
||||
// ?images=1 asks the server to render remote images for this one request
|
||||
// (the "Load images" one-off); "Always from this sender" persists separately.
|
||||
const url = `/api/messages/${message.id}/html${allowed ? "?images=1" : ""}`;
|
||||
const resp = await fetch(url, { credentials: "include" });
|
||||
if (!resp.ok) throw new Error("failed to load message");
|
||||
return resp.text();
|
||||
},
|
||||
@@ -89,6 +94,11 @@ export function MessageBody({ message }: { message: MessageEnvelope }) {
|
||||
if (persist) {
|
||||
await api.allowRemoteImages(message.id);
|
||||
queryClient.invalidateQueries({ queryKey: ["message-html", message.id] });
|
||||
// Refresh thread detail so the sender's images_allowed flag reflects the
|
||||
// new trust on reopen (suppresses the prompt next time).
|
||||
if (message.thread_id) {
|
||||
queryClient.invalidateQueries({ queryKey: ["thread", message.thread_id] });
|
||||
}
|
||||
}
|
||||
setAllowed(true);
|
||||
};
|
||||
|
||||
@@ -49,6 +49,9 @@ export const en = {
|
||||
"nav.spam": "Spam",
|
||||
"nav.trash": "Trash",
|
||||
"nav.search": "Search mail…",
|
||||
"search.save": "Save this search",
|
||||
"search.saved": "Saved searches",
|
||||
"search.saveNamePrompt": "Name this saved search:",
|
||||
"nav.storageQuota": "Storage used",
|
||||
"nav.refresh": "Refresh all folders",
|
||||
|
||||
@@ -176,11 +179,22 @@ export const en = {
|
||||
"wizard.startReading": "Start reading now",
|
||||
"wizard.manualConfig": "Enter settings manually",
|
||||
"wizard.displayName": "Display name",
|
||||
"wizard.syncWindow": "How much mail to sync",
|
||||
"wizard.syncWindow.all": "All mail",
|
||||
"wizard.syncWindow.all.hint": "Imports the last 30 days first, then the rest in the background.",
|
||||
"wizard.syncWindow.3months": "Last 3 months only",
|
||||
"wizard.syncWindow.3months.hint": "Older mail stays on the server until you download it.",
|
||||
"wizard.syncWindow.new": "New mail only",
|
||||
"wizard.syncWindow.new.hint":
|
||||
"Nothing from before now — download older mail anytime from account settings.",
|
||||
|
||||
"settings.title": "Settings",
|
||||
"settings.profile": "Profile",
|
||||
"settings.security": "Security",
|
||||
"settings.accounts": "Accounts",
|
||||
"settings.downloadAllMail": "Sync remaining mail",
|
||||
"settings.syncedPartially": "Synced partially",
|
||||
"backfill.importing": "Importing messages… {done} of {total}",
|
||||
"settings.appearance": "Appearance",
|
||||
"settings.handle": "Handle",
|
||||
"settings.recoveryEmail": "Recovery email",
|
||||
|
||||
+22
-2
@@ -13,10 +13,12 @@ import type {
|
||||
QuotaOut,
|
||||
RecipientSuggestion,
|
||||
RemoteDraftOut,
|
||||
SavedSearch,
|
||||
SearchOut,
|
||||
SendResponse,
|
||||
ServerConfig,
|
||||
SessionInfo,
|
||||
SyncWindow,
|
||||
ThreadDetail,
|
||||
ThreadListOut,
|
||||
TotpSetup,
|
||||
@@ -90,9 +92,15 @@ export const api = {
|
||||
mailboxes: () => apiFetch<MailboxOut[]>("/mailboxes"),
|
||||
syncNow: () => apiFetch<{ status: string }>("/accounts/sync", { method: "POST" }),
|
||||
|
||||
threads: (view = "unified", cursor?: string) => {
|
||||
threads: (
|
||||
view = "unified",
|
||||
opts?: { offset?: number; limit?: number; filter?: string; order?: "asc" | "desc" },
|
||||
) => {
|
||||
const params = new URLSearchParams({ view });
|
||||
if (cursor) params.set("cursor", cursor);
|
||||
if (opts?.offset) params.set("offset", String(opts.offset));
|
||||
if (opts?.limit) params.set("limit", String(opts.limit));
|
||||
if (opts?.filter && opts.filter !== "all") params.set("filter", opts.filter);
|
||||
if (opts?.order) params.set("order", opts.order);
|
||||
return apiFetch<ThreadListOut>(`/threads?${params.toString()}`);
|
||||
},
|
||||
thread: (id: string, sort: "asc" | "desc" = "asc", view = "unified") =>
|
||||
@@ -121,6 +129,7 @@ export const api = {
|
||||
imap: ServerConfig;
|
||||
smtp: ServerConfig;
|
||||
password: string;
|
||||
sync_window?: SyncWindow;
|
||||
}) => apiFetch<AccountDetail>("/accounts", { method: "POST", body: JSON.stringify(input) }),
|
||||
accountDetail: (id: string) => apiFetch<AccountDetail>(`/accounts/${id}`),
|
||||
updateAccount: (
|
||||
@@ -129,6 +138,7 @@ export const api = {
|
||||
display_name?: string;
|
||||
color?: string;
|
||||
signature?: string;
|
||||
sync_window?: SyncWindow;
|
||||
imap?: ServerConfig;
|
||||
smtp?: ServerConfig;
|
||||
password?: string;
|
||||
@@ -159,6 +169,16 @@ export const api = {
|
||||
quota: () => apiFetch<QuotaOut>("/settings/quota"),
|
||||
folderCounts: () => apiFetch<FolderCounts>("/mailboxes/counts"),
|
||||
|
||||
// --- saved searches (virtual mailboxes) ---
|
||||
savedSearches: () => apiFetch<SavedSearch[]>("/saved-searches"),
|
||||
createSavedSearch: (name: string, query: string) =>
|
||||
apiFetch<SavedSearch>("/saved-searches", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ name, query }),
|
||||
}),
|
||||
deleteSavedSearch: (id: string) =>
|
||||
apiFetch<{ status: string }>(`/saved-searches/${id}`, { method: "DELETE" }),
|
||||
|
||||
// --- compose ---
|
||||
suggestRecipients: (q: string) =>
|
||||
apiFetch<RecipientSuggestion[]>(`/recipients/suggest?q=${encodeURIComponent(q)}`),
|
||||
|
||||
@@ -30,6 +30,7 @@ export interface ThreadSummary {
|
||||
export interface ThreadListOut {
|
||||
threads: ThreadSummary[];
|
||||
cursor: string | null;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface AttachmentOut {
|
||||
@@ -55,6 +56,7 @@ export interface MessageEnvelope {
|
||||
is_starred: boolean;
|
||||
has_attachments: boolean;
|
||||
has_remote_images: boolean;
|
||||
images_allowed: boolean;
|
||||
size_bytes: number;
|
||||
snippet: string;
|
||||
also_in_account_ids: string[];
|
||||
@@ -68,6 +70,8 @@ export interface ThreadDetail {
|
||||
messages: MessageEnvelope[];
|
||||
}
|
||||
|
||||
export type SyncWindow = "all" | "3months" | "new";
|
||||
|
||||
export interface AccountOut {
|
||||
id: string;
|
||||
email: string;
|
||||
@@ -75,6 +79,9 @@ export interface AccountOut {
|
||||
color: string;
|
||||
state: string;
|
||||
signature: string;
|
||||
sync_window: SyncWindow;
|
||||
backfill_total: number;
|
||||
backfill_done: number;
|
||||
}
|
||||
|
||||
export interface MailboxOut {
|
||||
@@ -157,6 +164,13 @@ export interface FolderCounts {
|
||||
drafts: number;
|
||||
}
|
||||
|
||||
export interface SavedSearch {
|
||||
id: string;
|
||||
name: string;
|
||||
query: string;
|
||||
pinned: boolean;
|
||||
}
|
||||
|
||||
// --- M1: compose -------------------------------------------------------------
|
||||
|
||||
export interface RecipientSuggestion {
|
||||
|
||||
@@ -48,6 +48,10 @@ export function connectWs(queryClient: QueryClient): () => void {
|
||||
queryClient.invalidateQueries({ queryKey: ["quota"] });
|
||||
queryClient.invalidateQueries({ queryKey: ["drafts-remote"] });
|
||||
queryClient.invalidateQueries({ queryKey: ["folder-counts"] });
|
||||
// Backfill progress counters live on the account row (drives the banner).
|
||||
if (name === "sync.progress") {
|
||||
queryClient.invalidateQueries({ queryKey: ["accounts"] });
|
||||
}
|
||||
if (typeof payload.thread_id === "string") {
|
||||
queryClient.invalidateQueries({ queryKey: ["thread", payload.thread_id] });
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Link, useMatch, useNavigate } from "react-router-dom";
|
||||
|
||||
@@ -9,11 +9,12 @@ import { MessageList } from "../components/list/MessageList";
|
||||
import { Resizer } from "../components/Resizer";
|
||||
import { ThemeToggle } from "../components/ThemeToggle";
|
||||
import { EmptyIllustration } from "../components/EmptyIllustration";
|
||||
import { BackfillBanner } from "../components/BackfillBanner";
|
||||
import { ThreadView } from "../components/thread/ThreadView";
|
||||
import { useT } from "../i18n";
|
||||
import type { MessageKey } from "../i18n/en";
|
||||
import { api } from "../lib/api";
|
||||
import type { QuotaOut, ThreadSummary } from "../lib/types";
|
||||
import type { QuotaOut, SavedSearch, ThreadSummary } from "../lib/types";
|
||||
import { useKeyboardShortcuts } from "../lib/useKeyboardShortcuts";
|
||||
|
||||
function quotaLabel(q: QuotaOut): string {
|
||||
@@ -48,6 +49,13 @@ export function MailLayout() {
|
||||
const threadId = match?.params.threadId;
|
||||
const [view, setView] = useState<string>("folder:inbox");
|
||||
const [term, setTerm] = useState("");
|
||||
// Debounced copy of the search box for search-as-you-type (avoids a request per key).
|
||||
const [debouncedTerm, setDebouncedTerm] = useState("");
|
||||
useEffect(() => {
|
||||
const h = window.setTimeout(() => setDebouncedTerm(term), 180);
|
||||
return () => window.clearTimeout(h);
|
||||
}, [term]);
|
||||
const searching = debouncedTerm.trim().length > 0;
|
||||
const [filter, setFilter] = useState<"all" | "unread" | "pinned">("all");
|
||||
const [listOrder, setListOrder] = useState<"newest" | "oldest">(() =>
|
||||
localStorage.getItem("mail.listOrder") === "oldest" ? "oldest" : "newest",
|
||||
@@ -90,10 +98,13 @@ export function MailLayout() {
|
||||
const currentFolder = view.startsWith("folder:") ? view.slice("folder:".length) : "inbox";
|
||||
const accountsQuery = useQuery({ queryKey: ["accounts"], queryFn: api.accounts });
|
||||
const mailboxesQuery = useQuery({ queryKey: ["mailboxes"], queryFn: api.mailboxes });
|
||||
const orderParam: "asc" | "desc" = listOrder === "newest" ? "desc" : "asc";
|
||||
const threadsQuery = useQuery({
|
||||
queryKey: ["threads", view],
|
||||
queryFn: () => api.threads(view),
|
||||
enabled: !isDrafts,
|
||||
queryKey: ["threads", view, filter, orderParam, page],
|
||||
queryFn: () =>
|
||||
api.threads(view, { offset: page * PAGE_SIZE, limit: PAGE_SIZE, filter, order: orderParam }),
|
||||
enabled: !isDrafts && !searching,
|
||||
placeholderData: keepPreviousData, // keep the current page visible while the next loads
|
||||
});
|
||||
const draftsQuery = useQuery({
|
||||
queryKey: ["drafts"],
|
||||
@@ -106,33 +117,36 @@ export function MailLayout() {
|
||||
enabled: isDrafts,
|
||||
});
|
||||
const searchQuery = useQuery({
|
||||
queryKey: ["search", term],
|
||||
queryFn: () => api.search(term),
|
||||
enabled: term.trim().length > 0,
|
||||
queryKey: ["search", debouncedTerm],
|
||||
queryFn: () => api.search(debouncedTerm),
|
||||
enabled: searching,
|
||||
});
|
||||
const savedSearchesQuery = useQuery({ queryKey: ["saved-searches"], queryFn: api.savedSearches });
|
||||
const settingsQuery = useQuery({ queryKey: ["settings"], queryFn: api.settings });
|
||||
const threadOrder: "asc" | "desc" = settingsQuery.data?.thread_order ?? "asc";
|
||||
const quotaQuery = useQuery({ queryKey: ["quota"], queryFn: api.quota });
|
||||
const countsQuery = useQuery({ queryKey: ["folder-counts"], queryFn: api.folderCounts });
|
||||
|
||||
const allThreads: ThreadSummary[] = term.trim()
|
||||
? (searchQuery.data?.threads ?? []).map((h) => h.thread)
|
||||
: (threadsQuery.data?.threads ?? []);
|
||||
const filtered = allThreads.filter((th) =>
|
||||
filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true,
|
||||
);
|
||||
// The backend returns threads newest-first; reverse for an oldest-first view.
|
||||
const threads = listOrder === "newest" ? filtered : [...filtered].reverse();
|
||||
|
||||
// Client-side pagination of the (filtered, ordered) list.
|
||||
const total = threads.length;
|
||||
const maxPage = Math.max(0, Math.ceil(total / PAGE_SIZE) - 1);
|
||||
const curPage = Math.min(page, maxPage);
|
||||
const pageStart = curPage * PAGE_SIZE;
|
||||
const pageEnd = Math.min(pageStart + PAGE_SIZE, total);
|
||||
const pageThreads = threads.slice(pageStart, pageEnd);
|
||||
// Reset to the first page whenever the underlying list identity changes.
|
||||
useEffect(() => setPage(0), [view, filter, term, listOrder]);
|
||||
// Folder/inbox views paginate server-side with an accurate grand total; search
|
||||
// results (a single fetched page of hits) still paginate client-side.
|
||||
let pageThreads: ThreadSummary[];
|
||||
let total: number;
|
||||
if (searching) {
|
||||
const hits = (searchQuery.data?.threads ?? []).map((h) => h.thread);
|
||||
const f = hits.filter((th) =>
|
||||
filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true,
|
||||
);
|
||||
const ordered = listOrder === "newest" ? f : [...f].reverse();
|
||||
total = ordered.length;
|
||||
pageThreads = ordered.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE);
|
||||
} else {
|
||||
pageThreads = threadsQuery.data?.threads ?? [];
|
||||
total = threadsQuery.data?.total ?? 0;
|
||||
}
|
||||
const pageStart = page * PAGE_SIZE;
|
||||
const pageEnd = pageStart + pageThreads.length;
|
||||
// Reset to the first page whenever the list identity (not the page) changes.
|
||||
useEffect(() => setPage(0), [view, filter, debouncedTerm, listOrder]);
|
||||
|
||||
// Master "select all" state over the currently listed threads.
|
||||
const listedSelectedCount = pageThreads.filter((th) => checkedIds.has(th.id)).length;
|
||||
@@ -141,8 +155,9 @@ export function MailLayout() {
|
||||
const toggleSelectAll = () =>
|
||||
setCheckedIds(allSelected ? new Set() : new Set(pageThreads.map((th) => th.id)));
|
||||
|
||||
const threadsRef = useRef<ThreadSummary[]>(threads);
|
||||
threadsRef.current = threads;
|
||||
// Keyboard next/prev navigates within the currently visible page.
|
||||
const threadsRef = useRef<ThreadSummary[]>(pageThreads);
|
||||
threadsRef.current = pageThreads;
|
||||
|
||||
const detailQuery = useQuery({
|
||||
queryKey: ["thread", threadId, threadOrder, view],
|
||||
@@ -157,6 +172,27 @@ export function MailLayout() {
|
||||
queryClient.invalidateQueries({ queryKey: ["folder-counts"] });
|
||||
},
|
||||
});
|
||||
// Saved searches (virtual mailboxes, §12 M2).
|
||||
const savedSearches = savedSearchesQuery.data ?? [];
|
||||
const saveSearch = useMutation({
|
||||
mutationFn: (name: string) => api.createSavedSearch(name, term.trim()),
|
||||
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["saved-searches"] }),
|
||||
});
|
||||
const deleteSaved = useMutation({
|
||||
mutationFn: (id: string) => api.deleteSavedSearch(id),
|
||||
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["saved-searches"] }),
|
||||
});
|
||||
const onSaveSearch = () => {
|
||||
const q = term.trim();
|
||||
if (!q) return;
|
||||
const name = window.prompt(t("search.saveNamePrompt"), q);
|
||||
if (name && name.trim()) saveSearch.mutate(name.trim());
|
||||
};
|
||||
const runSavedSearch = (s: SavedSearch) => {
|
||||
setTerm(s.query);
|
||||
navigate("/mail");
|
||||
};
|
||||
|
||||
// Auto-mark an opened thread read after a short dwell (4s), not instantly — and
|
||||
// cancel it if you leave before then, so a quick glance doesn't mark it read.
|
||||
useEffect(() => {
|
||||
@@ -359,6 +395,7 @@ export function MailLayout() {
|
||||
|
||||
return (
|
||||
<div className="flex h-screen bg-app-bg text-app-fg">
|
||||
<BackfillBanner />
|
||||
{/* Sidebar */}
|
||||
<aside
|
||||
style={{ width: collapsed ? COLLAPSED_W : sidebarW }}
|
||||
@@ -435,10 +472,50 @@ export function MailLayout() {
|
||||
active={view === `account:${a.id}`}
|
||||
onClick={() => { setView(`account:${a.id}`); navigate("/mail"); }}
|
||||
>
|
||||
<span className="h-2.5 w-2.5 shrink-0 rounded-full" style={{ backgroundColor: a.color }} />
|
||||
<AccountDot
|
||||
color={a.color}
|
||||
syncing={a.state === "backfilling" && a.backfill_total > 0}
|
||||
total={a.backfill_total}
|
||||
done={a.backfill_done}
|
||||
/>
|
||||
<span className="truncate">{a.email}</span>
|
||||
</SidebarItem>
|
||||
))}
|
||||
{savedSearches.length > 0 && (
|
||||
<>
|
||||
<div className="px-3 pb-1 pt-3 text-[12px] font-semibold uppercase tracking-wide text-app-faint">
|
||||
{t("search.saved")}
|
||||
</div>
|
||||
{savedSearches.map((s) => (
|
||||
<div key={s.id} className="group/saved relative">
|
||||
<SidebarItem
|
||||
active={searching && debouncedTerm === s.query}
|
||||
onClick={() => runSavedSearch(s)}
|
||||
>
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.4" strokeLinecap="round" strokeLinejoin="round" className="shrink-0">
|
||||
<circle cx="7" cy="7" r="4.5" />
|
||||
<path d="M10.5 10.5L14 14" />
|
||||
</svg>
|
||||
<span className="flex-1 truncate">{s.name}</span>
|
||||
</SidebarItem>
|
||||
<button
|
||||
type="button"
|
||||
title={t("common.delete")}
|
||||
aria-label={t("common.delete")}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
deleteSaved.mutate(s.id);
|
||||
}}
|
||||
className="absolute right-2 top-1/2 hidden -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-fg group-hover/saved:block"
|
||||
>
|
||||
<svg width="13" height="13" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round">
|
||||
<path d="M4 4l8 8M12 4l-8 8" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</nav>
|
||||
@@ -489,26 +566,41 @@ export function MailLayout() {
|
||||
<div className="relative w-full">
|
||||
<input
|
||||
ref={searchRef}
|
||||
className="w-full rounded-lg border border-app-border bg-app-bg px-3 py-1.5 pr-8 text-[15px] text-app-fg placeholder:text-app-faint focus:border-app-accent focus:outline-none"
|
||||
className={`w-full rounded-lg border border-app-border bg-app-bg px-3 py-1.5 text-[15px] text-app-fg placeholder:text-app-faint focus:border-app-accent focus:outline-none ${
|
||||
term ? "pr-14" : "pr-8"
|
||||
}`}
|
||||
placeholder={t("nav.search")}
|
||||
value={term}
|
||||
onChange={(e) => setTerm(e.target.value)}
|
||||
/>
|
||||
{term && (
|
||||
<button
|
||||
type="button"
|
||||
title={t("common.clear")}
|
||||
aria-label={t("common.clear")}
|
||||
onClick={() => {
|
||||
setTerm("");
|
||||
searchRef.current?.focus();
|
||||
}}
|
||||
className="absolute right-1.5 top-1/2 -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-fg"
|
||||
>
|
||||
<svg width="14" height="14" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round">
|
||||
<path d="M4 4l8 8M12 4l-8 8" />
|
||||
</svg>
|
||||
</button>
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
title={t("search.save")}
|
||||
aria-label={t("search.save")}
|
||||
onClick={onSaveSearch}
|
||||
className="absolute right-8 top-1/2 -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-accent"
|
||||
>
|
||||
<svg width="15" height="15" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.4" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M4 2.5h8v11l-4-2.6-4 2.6z" />
|
||||
</svg>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
title={t("common.clear")}
|
||||
aria-label={t("common.clear")}
|
||||
onClick={() => {
|
||||
setTerm("");
|
||||
searchRef.current?.focus();
|
||||
}}
|
||||
className="absolute right-1.5 top-1/2 -translate-y-1/2 rounded p-1 text-app-faint hover:bg-app-hover hover:text-app-fg"
|
||||
>
|
||||
<svg width="14" height="14" viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round">
|
||||
<path d="M4 4l8 8M12 4l-8 8" />
|
||||
</svg>
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
@@ -612,14 +704,14 @@ export function MailLayout() {
|
||||
</span>
|
||||
<div className="-mr-1 flex items-center gap-0.5">
|
||||
<PagerButton
|
||||
disabled={curPage === 0}
|
||||
onClick={() => setPage(curPage - 1)}
|
||||
disabled={page === 0}
|
||||
onClick={() => setPage(page - 1)}
|
||||
label={t("pager.prev")}
|
||||
dir="left"
|
||||
/>
|
||||
<PagerButton
|
||||
disabled={pageEnd >= total}
|
||||
onClick={() => setPage(curPage + 1)}
|
||||
onClick={() => setPage(page + 1)}
|
||||
label={t("pager.next")}
|
||||
dir="right"
|
||||
/>
|
||||
@@ -765,6 +857,42 @@ export function MailLayout() {
|
||||
);
|
||||
}
|
||||
|
||||
// Account bullet in the sidebar. While the account is backfilling it becomes a
|
||||
// spinning ring that fills with the account colour to show import progress
|
||||
// (SPEC §12 M2); once live it's the plain colour-filled dot.
|
||||
function AccountDot({
|
||||
color,
|
||||
syncing,
|
||||
total,
|
||||
done,
|
||||
}: {
|
||||
color: string;
|
||||
syncing: boolean;
|
||||
total: number;
|
||||
done: number;
|
||||
}) {
|
||||
if (!syncing) {
|
||||
return (
|
||||
<span className="h-2.5 w-2.5 shrink-0 rounded-full" style={{ backgroundColor: color }} />
|
||||
);
|
||||
}
|
||||
const pct = total > 0 ? Math.min(100, Math.round((done / Math.max(done, total)) * 100)) : 0;
|
||||
const ring = "rgba(128,128,128,0.35)";
|
||||
return (
|
||||
<span
|
||||
className="h-3.5 w-3.5 shrink-0 animate-spin rounded-full"
|
||||
style={{
|
||||
animationDuration: "1.1s",
|
||||
background: `conic-gradient(${color} ${pct}%, ${ring} ${pct}% 100%)`,
|
||||
// Punch out the centre to leave a ~3px ring.
|
||||
WebkitMask: "radial-gradient(farthest-side, transparent calc(100% - 3px), #000 0)",
|
||||
mask: "radial-gradient(farthest-side, transparent calc(100% - 3px), #000 0)",
|
||||
}}
|
||||
title={`${pct}%`}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function FilterTab({
|
||||
active,
|
||||
onClick,
|
||||
|
||||
@@ -225,6 +225,10 @@ function AccountsSection() {
|
||||
mutationFn: (id: string) => api.deleteAccount(id),
|
||||
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["accounts"] }),
|
||||
});
|
||||
const downloadAll = useMutation({
|
||||
mutationFn: (id: string) => api.updateAccount(id, { sync_window: "all" }),
|
||||
onSuccess: () => queryClient.invalidateQueries({ queryKey: ["accounts"] }),
|
||||
});
|
||||
|
||||
const close = () => {
|
||||
setEditing(null);
|
||||
@@ -257,8 +261,25 @@ function AccountsSection() {
|
||||
<span className="text-xs text-app-faint">
|
||||
{t("settings.syncState", { state: a.state })}
|
||||
</span>
|
||||
{a.sync_window !== "all" && (
|
||||
<>
|
||||
<span className="text-xs text-app-faint">·</span>
|
||||
<span className="text-xs text-app-faint">
|
||||
{t("settings.syncedPartially")}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
<span className="flex items-center gap-3">
|
||||
{a.sync_window !== "all" && (
|
||||
<button
|
||||
className="text-xs text-app-accent disabled:opacity-50"
|
||||
disabled={downloadAll.isPending || a.state === "backfilling"}
|
||||
onClick={() => downloadAll.mutate(a.id)}
|
||||
>
|
||||
{t("settings.downloadAllMail")}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
className="text-xs text-app-accent"
|
||||
onClick={() => setEditing({ id: a.id, mode: "signature" })}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { useNavigate } from "react-router-dom";
|
||||
|
||||
import { useT } from "../../i18n";
|
||||
import { api, ApiError } from "../../lib/api";
|
||||
import type { ServerConfig } from "../../lib/types";
|
||||
import type { ServerConfig, SyncWindow } from "../../lib/types";
|
||||
|
||||
const PALETTE = ["#2563eb", "#16a34a", "#d4762a", "#9333ea", "#dc2626", "#0891b2"];
|
||||
const EMPTY: ServerConfig = { host: "", port: 993, tls: "ssl", auth: "password" };
|
||||
@@ -22,6 +22,7 @@ export function AccountWizard() {
|
||||
const [imap, setImap] = useState<ServerConfig>(EMPTY);
|
||||
const [smtp, setSmtp] = useState<ServerConfig>({ ...EMPTY, port: 465 });
|
||||
const [source, setSource] = useState<string>("");
|
||||
const [syncWindow, setSyncWindow] = useState<SyncWindow>("all");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [tested, setTested] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
@@ -72,6 +73,7 @@ export function AccountWizard() {
|
||||
imap,
|
||||
smtp,
|
||||
password,
|
||||
sync_window: syncWindow,
|
||||
});
|
||||
await queryClient.invalidateQueries({ queryKey: ["accounts"] });
|
||||
setStep(3);
|
||||
@@ -180,6 +182,28 @@ export function AccountWizard() {
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<fieldset className="rounded-lg border border-app-border p-3">
|
||||
<legend className="px-1 text-xs text-app-muted">{t("wizard.syncWindow")}</legend>
|
||||
<div className="space-y-1">
|
||||
{(["all", "3months", "new"] as const).map((w) => (
|
||||
<label key={w} className="flex cursor-pointer items-start gap-2 text-sm">
|
||||
<input
|
||||
type="radio"
|
||||
name="sync-window"
|
||||
className="mt-1"
|
||||
checked={syncWindow === w}
|
||||
onChange={() => setSyncWindow(w)}
|
||||
/>
|
||||
<span>
|
||||
<span className="font-medium">{t(`wizard.syncWindow.${w}`)}</span>
|
||||
<span className="block text-xs text-app-faint">
|
||||
{t(`wizard.syncWindow.${w}.hint`)}
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
{tested && <p className="text-sm text-emerald-500">{t("wizard.testOk")}</p>}
|
||||
<div className="flex gap-2">
|
||||
{!tested ? (
|
||||
|
||||
@@ -24,6 +24,7 @@ function msg(id: string, minutes: number, unread = false): MessageEnvelope {
|
||||
is_starred: false,
|
||||
has_attachments: false,
|
||||
has_remote_images: false,
|
||||
images_allowed: false,
|
||||
size_bytes: 1024,
|
||||
snippet: `snippet ${id}`,
|
||||
also_in_account_ids: [],
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.7 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 KiB |
Reference in New Issue
Block a user