diff --git a/backend/app/accounts/discovery.py b/backend/app/accounts/discovery.py index 67e7525..8c5ef16 100644 --- a/backend/app/accounts/discovery.py +++ b/backend/app/accounts/discovery.py @@ -49,6 +49,8 @@ def _help_block(raw: object) -> HelpBlock | None: app_name_hint=str(raw.get("app_name_hint", "")), steps=tuple(str(s) for s in steps) if isinstance(steps, list) else (), docs_url=str(raw.get("docs_url", "")), + setup_link=str(raw.get("setup_link", "")), + button_after_step=int(raw.get("button_after_step", 0)), ) diff --git a/backend/app/accounts/presets.yaml b/backend/app/accounts/presets.yaml index 4268fa7..717f1ce 100644 --- a/backend/app/accounts/presets.yaml +++ b/backend/app/accounts/presets.yaml @@ -25,12 +25,15 @@ providers: help: help_id: gmail deep_link: https://myaccount.google.com/apppasswords + setup_link: https://myaccount.google.com/signinoptions/two-step-verification app_name_hint: "ultimo.email" + button_after_step: 2 docs_url: https://support.google.com/mail/answer/185833 steps: - wizard.help.gmail.step1 - wizard.help.gmail.step2 - wizard.help.gmail.step3 + - wizard.help.gmail.step4 microsoft: domains: [outlook.com, hotmail.com, live.com, msn.com] diff --git a/backend/app/accounts/schemas.py b/backend/app/accounts/schemas.py index 0ca062f..24bdce2 100644 --- a/backend/app/accounts/schemas.py +++ b/backend/app/accounts/schemas.py @@ -28,6 +28,8 @@ class HelpBlockOut(BaseModel): app_name_hint: str = "" steps: list[str] = [] docs_url: str = "" + setup_link: str = "" + button_after_step: int = 0 class CandidateOut(BaseModel): @@ -73,6 +75,7 @@ class CreateAccountRequest(BaseModel): username: str | None = None password: str = Field(min_length=1) sync_window: SyncWindow = "all" + sync_spam: bool = False class UpdateAccountRequest(BaseModel): diff --git a/backend/app/accounts/service.py b/backend/app/accounts/service.py index 9bbaa33..cba827c 100644 --- a/backend/app/accounts/service.py +++ b/backend/app/accounts/service.py @@ -4,14 +4,14 @@ from __future__ import annotations import uuid -from sqlalchemy import delete, select +from sqlalchemy import delete, func, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.accounts.schemas import CreateAccountRequest from app.accounts.types import ServerConfig from app.indexer import blobstore -from app.models import Account, Attachment, Message +from app.models import Account, Attachment, Message, Thread from app.security.crypto import encrypt_secret @@ -40,6 +40,7 @@ async def create_account( discovery_src="manual", state="backfilling", sync_window=req.sync_window, + sync_spam=req.sync_spam, ) session.add(account) try: @@ -81,6 +82,23 @@ async def delete_account( await session.execute(delete(Account).where(Account.id == account_id)) await session.flush() + # Threads are user-scoped and can span accounts (Thread.account_ids), so the + # account cascade only removes its messages — not the threads. Clean up: drop + # threads left with no messages (else they linger as empty rows the unified list + # and label views still show but can't open), and de-list the account from + # threads that survive on another account's messages. + await session.execute( + delete(Thread).where( + Thread.user_id == user_id, + ~select(Message.id).where(Message.thread_id == Thread.id).exists(), + ) + ) + await session.execute( + update(Thread) + .where(Thread.user_id == user_id, Thread.account_ids.contains([account_id])) + .values(account_ids=func.array_remove(Thread.account_ids, account_id)) + ) + # Deregister from sync:live and drop any lease so the owning sync worker # stops the account's runner on its next renewal (refactor §2 fencing). from app.events import get_event_bus diff --git a/backend/app/accounts/types.py b/backend/app/accounts/types.py index b3513e9..9f49a8e 100644 --- a/backend/app/accounts/types.py +++ b/backend/app/accounts/types.py @@ -22,6 +22,8 @@ class HelpBlock: app_name_hint: str # suggested app-password label steps: tuple[str, ...] # i18n message keys docs_url: str = "" + setup_link: str = "" # optional 2FA/security page linked from the first step + button_after_step: int = 0 # 1-based step to place the app-password button after; 0 = top @dataclass(frozen=True) diff --git a/backend/app/api/accounts.py b/backend/app/api/accounts.py index ba3fcab..f308f61 100644 --- a/backend/app/api/accounts.py +++ b/backend/app/api/accounts.py @@ -72,6 +72,8 @@ def _candidate_out(c: CandidateConfig) -> CandidateOut: app_name_hint=c.help.app_name_hint, steps=list(c.help.steps), docs_url=c.help.docs_url, + setup_link=c.help.setup_link, + button_after_step=c.help.button_after_step, ) if c.help else None diff --git a/backend/app/indexer/indexer.py b/backend/app/indexer/indexer.py index 37b3e38..3443d23 100644 --- a/backend/app/indexer/indexer.py +++ b/backend/app/indexer/indexer.py @@ -36,6 +36,7 @@ class PersistResult: message: Message search_doc: SearchDoc has_remote_images: bool + gmail_labels: tuple[str, ...] = () def _epoch(value: datetime) -> int: @@ -140,8 +141,13 @@ async def persist_message( mailbox_id: uuid.UUID, mailbox_special_use: str | None, remote: RemoteMessage, + dedupe_account_scope: bool = False, ) -> PersistResult | None: - """Persist one message. Returns None if it already exists (idempotent).""" + """Persist one message. Returns None if it already exists (idempotent). + + ``dedupe_account_scope`` skips a message already stored under a *different* + mailbox of the same account (same Message-ID) — used for Gmail's All Mail pass so + a message already synced from INBOX/Sent isn't duplicated (SPEC: Gmail sync).""" existing = await session.scalar( select(Message.id).where(Message.mailbox_id == mailbox_id, Message.uid == remote.uid) ) @@ -154,6 +160,18 @@ async def persist_message( message_id_hdr = await _resolve_message_id( session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw ) + if dedupe_account_scope and message_id_hdr is not None: + dup = await session.scalar( + select(Message.id) + .where( + Message.account_id == account_id, + Message.message_id_hdr == message_id_hdr, + Message.mailbox_id != mailbox_id, + ) + .limit(1) + ) + if dup is not None: + return None message = Message( id=uuid.uuid4(), user_id=user_id, @@ -223,4 +241,9 @@ async def persist_message( await session.flush() doc = _build_search_doc(message, parsed, mailbox_special_use) - return PersistResult(message=message, search_doc=doc, has_remote_images=has_remote_images) + return PersistResult( + message=message, + search_doc=doc, + has_remote_images=has_remote_images, + gmail_labels=remote.gmail_labels, + ) diff --git a/backend/app/indexer/mime.py b/backend/app/indexer/mime.py index 43db8c4..a269cd5 100644 --- a/backend/app/indexer/mime.py +++ b/backend/app/indexer/mime.py @@ -8,6 +8,7 @@ from __future__ import annotations import email import os +import re from dataclasses import dataclass, field from datetime import datetime from email.headerregistry import AddressHeader @@ -20,6 +21,19 @@ from app.threading_.normalize import normalize_message_id, normalize_reference_l _SNIPPET_LEN = 160 +# Undecodable header/body bytes come out of the stdlib parser as lone UTF-16 +# surrogates (U+D800–U+DFFF). Those are invalid UTF-8, so Postgres rejects them in +# text/JSONB columns — scrub them to U+FFFD before anything is stored. +_SURROGATE_RE = re.compile("[\ud800-\udfff]") + + +def _scrub(value: str) -> str: + return _SURROGATE_RE.sub("�", value) if value else value + + +def _scrub_addrs(addrs: list[dict[str, str]]) -> list[dict[str, str]]: + return [{k: _scrub(v) for k, v in addr.items()} for addr in addrs] + @dataclass(frozen=True) class ParsedAttachment: @@ -112,7 +126,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]: inline = cid is not None and disposition != "attachment" out.append( ParsedAttachment( - filename=filename or "attachment", + filename=_scrub(filename or "attachment"), mime_type=part.get_content_type(), content_id=cid.strip("<>") if (inline and cid is not None) else None, payload=payload, @@ -166,13 +180,13 @@ def _parse_email(msg: EmailMessage) -> ParsedEmail: message_id=message_id, in_reply_to=in_reply_to, references=references, - subject=subject, - from_addr=_addresses(msg, "From"), - to_addr=_addresses(msg, "To"), - cc_addr=_addresses(msg, "Cc"), - bcc_addr=_addresses(msg, "Bcc"), + subject=_scrub(subject), + from_addr=_scrub_addrs(_addresses(msg, "From")), + to_addr=_scrub_addrs(_addresses(msg, "To")), + cc_addr=_scrub_addrs(_addresses(msg, "Cc")), + bcc_addr=_scrub_addrs(_addresses(msg, "Bcc")), date=date, - text_body=text_body or "", - html_body=html_body, + text_body=_scrub(text_body or ""), + html_body=_scrub(html_body) if html_body is not None else None, attachments=_extract_attachments(msg), ) diff --git a/backend/app/models.py b/backend/app/models.py index 68cd7f9..7d7f79e 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -86,6 +86,8 @@ class Account(Base): # How much history to import (SPEC §12 M2): all | 3months | new. Drives the # initial backfill; backfill_total/backfill_done feed the progress UX. sync_window: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'all'")) + # Whether the Spam/Junk folder is synced (opt-in; SPEC §6.3 folder selection). + sync_spam: Mapped[bool] = mapped_column(nullable=False, server_default=text("false")) backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True)) diff --git a/backend/app/plugins/actions.py b/backend/app/plugins/actions.py index c2e13dd..7cee6a7 100644 --- a/backend/app/plugins/actions.py +++ b/backend/app/plugins/actions.py @@ -22,7 +22,7 @@ from sqlalchemy import select from app.db import session_scope from app.events import get_event_bus from app.indexer import blobstore -from app.models import Message +from app.models import Message, Thread class PluginMessages: @@ -44,6 +44,29 @@ class PluginMessages: return None return message_from_bytes(raw).get(name) + async def threads_of( + self, user_id: uuid.UUID, message_ids: Iterable[str] + ) -> dict[str, str | None]: + """Map still-existing message ids to their current thread id (None if + unthreaded). Deleted messages are omitted, so a plugin can prune stale + per-message references and re-home labels onto the live thread after a + JWZ thread merge.""" + ids: list[uuid.UUID] = [] + for mid in message_ids: + try: + ids.append(uuid.UUID(str(mid))) + except (ValueError, AttributeError, TypeError): + continue + if not ids: + return {} + async with session_scope() as session: + rows = await session.execute( + select(Message.id, Message.thread_id).where( + Message.user_id == user_id, Message.id.in_(ids) + ) + ) + return {str(mid): (str(tid) if tid else None) for mid, tid in rows} + async def envelope(self, user_id: uuid.UUID, message_id: uuid.UUID) -> dict[str, Any] | None: async with session_scope() as session: msg = await session.scalar( @@ -66,6 +89,29 @@ class PluginMessages: } +class PluginThreads: + """Read-only, user-scoped thread queries (e.g. self-healing stale references).""" + + async def existing(self, user_id: uuid.UUID, thread_ids: Iterable[str]) -> set[str]: + """Return the subset of ``thread_ids`` that still exist for the user. + + Lets a plugin prune references to threads deleted out from under it (e.g. an + account removal drops its threads, orphaning label aggregates keyed by id).""" + ids: list[uuid.UUID] = [] + for tid in thread_ids: + try: + ids.append(uuid.UUID(str(tid))) + except (ValueError, AttributeError, TypeError): + continue + if not ids: + return set() + async with session_scope() as session: + rows = await session.scalars( + select(Thread.id).where(Thread.user_id == user_id, Thread.id.in_(ids)) + ) + return {str(r) for r in rows} + + class PluginActions: """User-scoped actions a plugin can take on a message.""" diff --git a/backend/app/plugins/sdk.py b/backend/app/plugins/sdk.py index b4692b4..0dbda0b 100644 --- a/backend/app/plugins/sdk.py +++ b/backend/app/plugins/sdk.py @@ -28,7 +28,7 @@ from app.identity.deps import current_user, require_csrf from app.indexer import blobstore from app.models import User from app.outbox import service as outbox_service -from app.plugins.actions import PluginActions, PluginMessages +from app.plugins.actions import PluginActions, PluginMessages, PluginThreads from app.plugins.db import plugin_session from app.plugins.http import PluginHttp from app.plugins.kv import PluginKvStore @@ -151,6 +151,11 @@ class PluginSdk: """User-scoped message actions (``set_flags``) — e.g. for filter rules.""" return PluginActions() + @property + def threads(self) -> PluginThreads: + """Read-only thread queries (e.g. pruning stale references).""" + return PluginThreads() + # --- outbound HTTP ------------------------------------------------------ async def http_get(self, url: str) -> str | None: """GET ``url`` through the §10.6 SSRF guard (blocks private IPs/oversize).""" diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py index 5224d96..4a57bed 100644 --- a/backend/app/providers/base.py +++ b/backend/app/providers/base.py @@ -46,6 +46,7 @@ class RemoteMessage: raw: bytes # full RFC822 flags: frozenset[str] internal_date: datetime + gmail_labels: tuple[str, ...] = () # custom Gmail labels (Gmail accounts only) @dataclass(frozen=True) diff --git a/backend/app/providers/imap_smtp/parsing.py b/backend/app/providers/imap_smtp/parsing.py index 99a97ee..df20271 100644 --- a/backend/app/providers/imap_smtp/parsing.py +++ b/backend/app/providers/imap_smtp/parsing.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from datetime import datetime from imapclient.imap_utf7 import decode as imap_utf7_decode +from imapclient.imap_utf7 import encode as imap_utf7_encode # --- SPECIAL-USE mapping (SPEC §4 special_use values) ----------------------- @@ -27,6 +28,7 @@ _SPECIAL_USE_FLAGS = { _LIST_RE = re.compile(rb'^\(([^)]*)\)\s+(?:"([^"]*)"|NIL)\s+(.+)$') _UID_RE = re.compile(rb"UID\s+(\d+)") _FLAGS_RE = re.compile(rb"FLAGS\s+\(([^)]*)\)") +_GM_LABELS_RE = re.compile(rb"X-GM-LABELS\s+\(([^)]*)\)") _INTERNALDATE_RE = re.compile(rb'INTERNALDATE\s+"([^"]+)"') _LITERAL_RE = re.compile(rb"\{(\d+)\}\s*$") _FETCH_HEADER_RE = re.compile(rb"^\d+\s+FETCH\s+\(", re.IGNORECASE) @@ -44,6 +46,7 @@ class ParsedFetch: flags: frozenset[str] internal_date: datetime | None raw: bytes | None # None when the fetch did not request the body + gmail_labels: tuple[str, ...] = () # custom Gmail labels (X-GM-LABELS), system ones dropped def _decode_mailbox_name(raw: bytes) -> str: @@ -54,12 +57,29 @@ def _decode_mailbox_name(raw: bytes) -> str: return str(imap_utf7_decode(bytes(name))) +def encode_mailbox_name(name: str) -> str: + """Modified UTF-7 wire form of a mailbox name (inverse of ``_decode_mailbox_name``). + + Mailbox names are stored/displayed as Unicode but IMAP commands + (SELECT/STATUS/APPEND) require them mUTF-7 encoded. ASCII names are unchanged; + non-ASCII names — e.g. Gmail's localized ``[Gmail]/Wysłane`` — get encoded so the + server can find them.""" + return str(imap_utf7_encode(name).decode("ascii")) + + def parse_list_line(line: bytes) -> ParsedMailbox | None: - """Parse one LIST response line into a :class:`ParsedMailbox` (or None).""" + """Parse one LIST response line into a :class:`ParsedMailbox` (or None). + + Non-selectable mailboxes are skipped (return None): ``\\Noselect`` / + ``\\NonExistent`` names are pure hierarchy containers that hold no messages and + cannot be SELECTed — e.g. Gmail's ``[Gmail]`` root or a parent label like + ``Serial numbers`` that only groups child labels.""" m = _LIST_RE.match(line.strip()) if not m: return None flags = m.group(1).lower().split() + if b"\\noselect" in flags or b"\\nonexistent" in flags: + return None name = _decode_mailbox_name(m.group(3)) special: str | None = None for flag in flags: @@ -100,6 +120,57 @@ def parse_fetch_header(line: bytes) -> tuple[int | None, frozenset[str], datetim return uid, flags, internal +def _tokenize_gm_labels(blob: bytes) -> list[bytes]: + """Split an ``X-GM-LABELS (...)`` body into raw tokens, keeping quoted strings + (which may contain spaces) whole. Escaped chars inside quotes are unescaped.""" + tokens: list[bytes] = [] + i, n = 0, len(blob) + while i < n: + ch = blob[i : i + 1] + if ch in (b" ", b"\t"): + i += 1 + continue + if ch == b'"': + j = i + 1 + buf = bytearray() + while j < n and blob[j : j + 1] != b'"': + if blob[j : j + 1] == b"\\" and j + 1 < n: + buf += blob[j + 1 : j + 2] + j += 2 + continue + buf += blob[j : j + 1] + j += 1 + tokens.append(bytes(buf)) + i = j + 1 + else: + j = i + while j < n and blob[j : j + 1] not in (b" ", b"\t"): + j += 1 + tokens.append(blob[i:j]) + i = j + return tokens + + +def parse_gm_labels(header: bytes) -> tuple[str, ...]: + """Custom Gmail labels from a FETCH header's ``X-GM-LABELS`` (SPEC: Gmail sync). + + System labels (``\\Inbox``, ``\\Sent``, ``\\Important``, ``\\Starred``, …) are + dropped — only user labels become chips. Names are modified-UTF-7 decoded, like + mailbox names, so e.g. ``Zg&AAI...`` → the real Unicode label.""" + m = _GM_LABELS_RE.search(header) + if not m: + return () + out: list[str] = [] + for tok in _tokenize_gm_labels(m.group(1)): + if not tok or tok.startswith(b"\\"): # system label + continue + try: + out.append(str(imap_utf7_decode(tok))) + except Exception: # noqa: BLE001 - fall back to a best-effort ASCII decode + out.append(tok.decode("utf-8", "ignore")) + return tuple(out) + + def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]: """Walk a full FETCH response, returning one :class:`ParsedFetch` per message. @@ -131,7 +202,15 @@ def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]: else: i += 1 if uid is not None: - results.append(ParsedFetch(uid=uid, flags=flags, internal_date=internal, raw=raw)) + results.append( + ParsedFetch( + uid=uid, + flags=flags, + internal_date=internal, + raw=raw, + gmail_labels=parse_gm_labels(header), + ) + ) return results diff --git a/backend/app/providers/imap_smtp/provider.py b/backend/app/providers/imap_smtp/provider.py index d3df7e3..71e3409 100644 --- a/backend/app/providers/imap_smtp/provider.py +++ b/backend/app/providers/imap_smtp/provider.py @@ -31,6 +31,7 @@ from app.providers.base import ( RemoteMessage, ) from app.providers.imap_smtp.parsing import ( + encode_mailbox_name, parse_fetch_lines, parse_list_line, parse_search_uids, @@ -39,6 +40,9 @@ from app.providers.imap_smtp.parsing import ( ) _FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])" +# Gmail (X-GM-EXT-1): also pull each message's labels so custom labels can map to +# chips without visiting every label folder (SPEC: Gmail sync / no-duplication). +_FETCH_ITEMS_GM = "(UID FLAGS INTERNALDATE RFC822.SIZE X-GM-LABELS BODY.PEEK[])" _IDLE_TIMEOUT = 25 * 60 # SPEC §5.1: re-issue IDLE at least every 25 min _CONNECT_TIMEOUT = 20 _PER_ACCOUNT_CONN_CAP = 4 # SPEC §6.3 / §4: IDLE + transient + short-lived jobs @@ -81,6 +85,13 @@ class ImapSmtpProvider(MailProvider): self._uidvalidity_seen: dict[str, int] = {} self._conn_slots: list[asyncio.Semaphore] = [] self._qresync_enabled = False + self._gm_ext = False + + @property + def gmail_labels_supported(self) -> bool: + """True when the server advertises ``X-GM-EXT-1`` (Gmail), so FETCH can pull + ``X-GM-LABELS`` and the engine can use the no-duplication Gmail sync path.""" + return self._gm_ext @property def qresync_enabled(self) -> bool: @@ -148,6 +159,8 @@ class ImapSmtpProvider(MailProvider): self._release_conn_slots() # never hold a slot without an open socket raise self._imap = imap + caps = {c.upper() for c in (imap.protocol.capabilities or set())} + self._gm_ext = "X-GM-EXT-1" in caps await self._maybe_enable_qresync(imap) return imap @@ -165,9 +178,16 @@ class ImapSmtpProvider(MailProvider): except Exception: # noqa: BLE001 - degrade gracefully to the reconcile path self._qresync_enabled = False + @staticmethod + def _wire(mailbox: str) -> str: + """Server wire form of a mailbox name: mUTF-7 encoded and quoted. Handles + non-ASCII names (Gmail's localized ``[Gmail]/Wysłane``) and names with + spaces (``[Gmail]/All Mail``); ASCII names are unchanged bar the quotes.""" + return str(aioimaplib.quoted(encode_mailbox_name(mailbox))) + async def _select(self, mailbox: str) -> dict[str, int]: imap = await self._connect() - resp = await imap.select(mailbox) + resp = await imap.select(self._wire(mailbox)) if resp.result != "OK": raise ProviderConnectError(f"cannot select {mailbox!r}") self._selected = mailbox @@ -212,7 +232,7 @@ class ImapSmtpProvider(MailProvider): async def _mailbox_uidvalidity(self, path: str) -> int: imap = await self._connect() try: - resp = await imap.status(path, "(UIDVALIDITY)") + resp = await imap.status(self._wire(path), "(UIDVALIDITY)") except Exception: return 0 if resp.result != "OK": @@ -280,7 +300,8 @@ class ImapSmtpProvider(MailProvider): if not uids: return [] uid_set = ",".join(str(u) for u in uids) - resp = await imap.uid("fetch", uid_set, _FETCH_ITEMS) + items = _FETCH_ITEMS_GM if self._gm_ext else _FETCH_ITEMS + resp = await imap.uid("fetch", uid_set, items) if resp.result != "OK": raise ProviderConnectError("FETCH failed") out: list[RemoteMessage] = [] @@ -289,7 +310,11 @@ class ImapSmtpProvider(MailProvider): continue out.append( RemoteMessage( - uid=pf.uid, raw=pf.raw, flags=pf.flags, internal_date=pf.internal_date + uid=pf.uid, + raw=pf.raw, + flags=pf.flags, + internal_date=pf.internal_date, + gmail_labels=pf.gmail_labels, ) ) return out @@ -355,7 +380,7 @@ class ImapSmtpProvider(MailProvider): proto = imap.protocol param = f"(QRESYNC ({uidvalidity} {modseq}))" cmd = aioimaplib.Command( - "SELECT", proto.new_tag(), aioimaplib.quoted(mailbox), param, loop=proto.loop + "SELECT", proto.new_tag(), self._wire(mailbox), param, loop=proto.loop ) return await proto.execute(cmd) @@ -419,7 +444,7 @@ class ImapSmtpProvider(MailProvider): async def append(self, mailbox: str, raw: bytes, flags: frozenset[str]) -> int: imap = await self._connect() flag_str = f"({' '.join(sorted(flags))})" if flags else None - resp = await imap.append(raw, mailbox=mailbox, flags=flag_str) + resp = await imap.append(raw, mailbox=self._wire(mailbox), flags=flag_str) if resp.result != "OK": raise ProviderConnectError(f"APPEND to {mailbox} failed") for line in resp.lines: diff --git a/backend/app/sync/engine.py b/backend/app/sync/engine.py index 8d20d0e..3401e1f 100644 --- a/backend/app/sync/engine.py +++ b/backend/app/sync/engine.py @@ -27,7 +27,15 @@ from app.sync.provider_factory import build_provider logger = logging.getLogger(__name__) # Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3). +# archive (Gmail All Mail) sorts last so its dedupe pass runs after the system folders. _PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2} +# Gmail sync (SPEC: Gmail sync): the only folders backfilled — All Mail (archive) is +# the complete store, the rest are the system folders needed for their views. Every +# custom-label folder is skipped; labels come from X-GM-LABELS instead. +_GMAIL_SYNC_FOLDERS = {"inbox", "sent", "drafts", "trash", "junk", "archive"} +# Progress denominator for Gmail: All Mail already contains INBOX/Sent/Drafts/labels, +# so only these three are counted (Trash/Spam sit outside All Mail). +_GMAIL_DENOM_FOLDERS = {"archive", "trash", "junk"} # Sync-window import strategy (SPEC §12 M2). "all" imports the most recent # _RECENT_DAYS first (so the account is usable fast), then the older remainder. @@ -110,15 +118,17 @@ async def _index_and_notify(results: list[PersistResult], account: Account) -> N bus = get_event_bus() for r in results: await backend.index(r.search_doc) - await bus.publish( - "message.new", - { - "user_id": str(account.user_id), - "account_id": str(account.id), - "message_id": str(r.message.id), - "thread_id": str(r.message.thread_id), - }, - ) + payload: dict[str, object] = { + "user_id": str(account.user_id), + "account_id": str(account.id), + "message_id": str(r.message.id), + "thread_id": str(r.message.thread_id), + } + # Gmail custom labels ride along so the labels plugin can map them to chips + # (SPEC: Gmail sync). Only present on Gmail accounts with labelled messages. + if r.gmail_labels: + payload["gmail_labels"] = list(r.gmail_labels) + await bus.publish("message.new", payload) async def _emit_batch( @@ -178,12 +188,15 @@ async def backfill_mailbox( *, since: date | None = None, before: date | None = None, + dedupe_account_scope: bool = False, ) -> int: """Backfill one mailbox newest-first, indexing inline per batch (FR-30). ``since``/``before`` bound the import by date (sync window, SPEC §12 M2). When a ``progress`` tracker is given, batches count toward the account-level backfill total; otherwise each batch emits a standalone ``sync.progress``. + ``dedupe_account_scope`` drops messages already synced under another mailbox of + the account (Gmail All Mail pass — SPEC: Gmail sync / no-duplication). """ from app.db import session_scope @@ -202,6 +215,7 @@ async def backfill_mailbox( mailbox_id=mailbox_id, mailbox_special_use=special_use, remote=remote, + dedupe_account_scope=dedupe_account_scope, ) if r is not None: results.append(r) @@ -257,6 +271,16 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int: account_obj = account window = account.sync_window mailbox_list = list(mailboxes) + # Gmail: sync only the system folders + All Mail and read each message's + # labels (X-GM-LABELS) instead of visiting every label folder. This avoids + # the INBOX/All-Mail/label duplication and lets labels map to chips lazily, + # tied to the sync window (SPEC: Gmail sync). + gmail = provider.gmail_labels_supported + if gmail: + mailbox_list = [m for m in mailbox_list if m.special_use in _GMAIL_SYNC_FOLDERS] + # Spam is opt-in: skip the Junk folder unless the account enabled it (§6.3). + if not account.sync_spam: + mailbox_list = [m for m in mailbox_list if m.special_use != "junk"] try: if window == "new": @@ -270,9 +294,13 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int: date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None ) # Progress denominator: count what we're about to import up front so the - # bar has a stable total (SPEC §12 M2 backfill-progress UX). + # bar has a stable total (SPEC §12 M2 backfill-progress UX). For Gmail only + # All Mail + Trash + Spam are counted — INBOX/Sent/Drafts are subsets of All + # Mail, so counting them too would inflate the total past the deduped `done`. total = 0 for mailbox in mailbox_list: + if gmail and mailbox.special_use not in _GMAIL_DENOM_FOLDERS: + continue total += await provider.message_count(mailbox.path, since=since) await _set_backfill_counts(account_id, total=total, done=0) progress = _BackfillProgress(account_obj, total) @@ -280,16 +308,22 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int: recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS) count = 0 for mailbox in mailbox_list: + # Dedupe the Gmail All Mail pass against messages already stored under a + # system folder (INBOX/Sent/…), which the priority order backfills first. + dedupe = gmail and mailbox.special_use == "archive" if since is not None: # 3months: single windowed pass count += await backfill_mailbox( - provider, account_obj, mailbox, progress, since=since + provider, account_obj, mailbox, progress, since=since, + dedupe_account_scope=dedupe, ) else: # all: recent window first (usable fast), then the remainder count += await backfill_mailbox( - provider, account_obj, mailbox, progress, since=recent_cutoff + provider, account_obj, mailbox, progress, since=recent_cutoff, + dedupe_account_scope=dedupe, ) count += await backfill_mailbox( - provider, account_obj, mailbox, progress, before=recent_cutoff + provider, account_obj, mailbox, progress, before=recent_cutoff, + dedupe_account_scope=dedupe, ) await _set_state(account_id, "live") except Exception as exc: # noqa: BLE001 - surface provider errors as account state @@ -490,7 +524,8 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No ``include_inbox`` also polls INBOX (used by the manual Refresh); the periodic secondary loop leaves INBOX to its dedicated IDLE loop. Custom (non special-use) folders are not polled. Drafts are polled so drafts made in other clients appear - (persisted un-threaded). + (persisted un-threaded). ``archive`` (Gmail All Mail) is never live-polled — it is + huge and cold, and backfill already covers it. """ from app.db import session_scope @@ -498,10 +533,14 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No account = await _load_account(session, account_id) account_obj = account stmt = select(Mailbox.id).where( - Mailbox.account_id == account_id, Mailbox.special_use.isnot(None) + Mailbox.account_id == account_id, + Mailbox.special_use.isnot(None), + Mailbox.special_use != "archive", ) if not include_inbox: stmt = stmt.where(Mailbox.special_use != "inbox") + if not account.sync_spam: # Spam is opt-in (§6.3) + stmt = stmt.where(Mailbox.special_use != "junk") mailbox_ids = list(await session.scalars(stmt)) if not mailbox_ids: return diff --git a/backend/migrations/versions/0015_account_sync_spam.py b/backend/migrations/versions/0015_account_sync_spam.py new file mode 100644 index 0000000..0ec2dfc --- /dev/null +++ b/backend/migrations/versions/0015_account_sync_spam.py @@ -0,0 +1,28 @@ +"""add accounts.sync_spam + +Per-account toggle for whether the Spam/Junk folder is synced. Off by default — +most users don't want spam mirrored into the app; they can opt in at setup. + +Revision ID: 0015 +Revises: 0014 +Create Date: 2026-08-05 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0015" +down_revision: str | None = "0014" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("ALTER TABLE accounts ADD COLUMN sync_spam boolean NOT NULL DEFAULT false") + + +def downgrade() -> None: + op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS sync_spam") diff --git a/backend/tests/accounts/test_account_api.py b/backend/tests/accounts/test_account_api.py index 0219f59..cb1ec44 100644 --- a/backend/tests/accounts/test_account_api.py +++ b/backend/tests/accounts/test_account_api.py @@ -19,7 +19,7 @@ from app.config import get_settings from app.identity.passwords import hash_password from app.identity.sessions import create_session from app.main import create_app -from app.models import Account, User +from app.models import Account, Thread, User CSRF = "test-csrf-token" GOOD_CFG = { @@ -281,3 +281,46 @@ async def test_delete_account(ctx: Ctx, monkeypatch) -> None: # type: ignore[no gone = await ctx.session.scalar(select(Account).where(Account.id == uuid.UUID(account_id))) assert gone is None + + +async def test_delete_account_prunes_orphaned_threads(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def] + """Deleting an account removes threads left with no messages (else the unified + list / label views show empty rows that can't be opened).""" + import datetime as dt + + async def fake_test(**_kw: object) -> None: + return None + + class FakePool: + async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None: + return None + + async def fake_pool() -> FakePool: + return FakePool() + + monkeypatch.setattr(accounts_api, "test_account", fake_test) + monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool) + + created = await ctx.client.post( + "/api/accounts", + json={**GOOD_CFG, "email": "orphan@example.com", "password": "secret"}, + headers=_hdr(), + ) + account_id = uuid.UUID(created.json()["id"]) + # An empty thread tagged to this account (message-less, as after a cascade delete). + thread_id = uuid.uuid4() + ctx.session.add( + Thread( + id=thread_id, + user_id=ctx.user_id, + subject_norm="orphan", + last_message_at=dt.datetime.now(dt.UTC), + account_ids=[account_id], + ) + ) + await ctx.session.commit() + ctx.session.expunge_all() # force the assertion query to hit the DB + + resp = await ctx.client.request("DELETE", f"/api/accounts/{account_id}", headers=_hdr()) + assert resp.status_code == 200 + assert await ctx.session.scalar(select(Thread).where(Thread.id == thread_id)) is None diff --git a/backend/tests/indexer/test_mime.py b/backend/tests/indexer/test_mime.py index 9d0552b..8a1b51f 100644 --- a/backend/tests/indexer/test_mime.py +++ b/backend/tests/indexer/test_mime.py @@ -49,6 +49,17 @@ def test_parse_basic_headers_and_body() -> None: assert parsed.snippet == "plain body" +def test_undecodable_header_bytes_scrubbed_to_valid_utf8() -> None: + # A raw 8-bit byte the parser can't decode surfaces as a lone surrogate; it must + # be scrubbed so the value is valid UTF-8 for Postgres text/JSONB storage. + raw = _build(subject="Fakt\udce9ra", from_addr="B\udce9ta ") + parsed = parse_message(raw) + parsed.subject.encode("utf-8") # would raise on a lone surrogate + parsed.from_addr[0]["name"].encode("utf-8") + assert "\udce9" not in parsed.subject + assert "�" in parsed.subject + + def test_references_include_in_reply_to_in_order() -> None: parsed = parse_message(_build(references=" ", in_reply_to="")) # References first, then In-Reply-To, deduped diff --git a/backend/tests/providers/test_parsing.py b/backend/tests/providers/test_parsing.py index 5d5f2ed..0887a6c 100644 --- a/backend/tests/providers/test_parsing.py +++ b/backend/tests/providers/test_parsing.py @@ -37,6 +37,17 @@ def test_parse_list_plain_folder_has_no_special_use() -> None: assert parsed.special_use is None +def test_parse_list_skips_noselect_containers() -> None: + # \Noselect / \NonExistent hierarchy nodes can't be SELECTed (Gmail's [Gmail] + # root, a parent label like "Serial numbers " that only groups children). + assert parse_list_line(rb'(\HasChildren \Noselect) "/" "[Gmail]"') is None + assert parse_list_line(rb'(\HasChildren \Noselect) "/" "Serial numbers "') is None + assert parse_list_line(rb'(\NonExistent \HasChildren) "/" "Parent"') is None + # A selectable child of that container is still parsed. + child = parse_list_line(rb'(\HasNoChildren) "/" "Serial numbers / licencje"') + assert child is not None and child.path == "Serial numbers / licencje" + + def test_parse_list_utf7_name() -> None: # Modified UTF-7: "&AOk-" decodes to "é" -> folder "Réponses" parsed = parse_list_line(rb'(\HasNoChildren) "/" "R&AOk-ponses"') @@ -44,6 +55,35 @@ def test_parse_list_utf7_name() -> None: assert parsed.path == "Réponses" +def test_parse_gm_labels() -> None: + from app.providers.imap_smtp.parsing import parse_gm_labels + + # System labels (backslash-prefixed) are dropped; custom labels kept + decoded. + header = rb'1 FETCH (UID 5 X-GM-LABELS (\Inbox \Important "Konkurs" "Projekty/2026") FLAGS ())' + assert parse_gm_labels(header) == ("Konkurs", "Projekty/2026") + # Unquoted single-word custom label. + assert parse_gm_labels(rb"1 FETCH (UID 5 X-GM-LABELS (\Sent PUP))") == ("PUP",) + # Modified UTF-7 decoding of a non-ASCII label ("Zg&AUI-oszenia" -> "Zgłoszenia"). + assert parse_gm_labels(rb'1 FETCH (X-GM-LABELS ("Zg&AUI-oszenia"))') == ("Zgłoszenia",) + # No X-GM-LABELS at all. + assert parse_gm_labels(rb"1 FETCH (UID 5 FLAGS (\Seen))") == () + # Only system labels -> empty. + assert parse_gm_labels(rb"1 FETCH (X-GM-LABELS (\Inbox \Starred))") == () + + +def test_parse_fetch_lines_carries_gm_labels() -> None: + lines = [ + rb'1 FETCH (UID 9 X-GM-LABELS (\Inbox "Work") FLAGS (\Seen) ' + rb'INTERNALDATE "27-May-2026 19:08:09 +0200" RFC822.SIZE 3 BODY[] {3}', + b"abc", + b")", + ] + out = parse_fetch_lines(lines) + assert len(out) == 1 + assert out[0].gmail_labels == ("Work",) + assert out[0].raw == b"abc" + + def test_parse_internaldate() -> None: dt = parse_internaldate("27-May-2026 19:08:09 +0200") assert dt == datetime.fromisoformat("2026-05-27T19:08:09+02:00") diff --git a/backend/tests/sync/test_backfill_window.py b/backend/tests/sync/test_backfill_window.py index c55243c..cbed98f 100644 --- a/backend/tests/sync/test_backfill_window.py +++ b/backend/tests/sync/test_backfill_window.py @@ -31,6 +31,8 @@ def _raw(mid: str) -> bytes: class _FakeProvider: + gmail_labels_supported = False + def __init__(self) -> None: self.backfill_calls: list[tuple[str, date | None, date | None]] = [] self.count_calls: list[tuple[str, date | None]] = [] diff --git a/frontend/icons/ultimo-logo-signet.svg b/frontend/icons/ultimo-logo-signet.svg new file mode 100644 index 0000000..f5ff865 --- /dev/null +++ b/frontend/icons/ultimo-logo-signet.svg @@ -0,0 +1,34 @@ + + + + diff --git a/frontend/icons/ultimo-logo.svg b/frontend/icons/ultimo-logo.svg new file mode 100644 index 0000000..4d24abe --- /dev/null +++ b/frontend/icons/ultimo-logo.svg @@ -0,0 +1,38 @@ + + + + diff --git a/frontend/public/ultimo-logo-signet.svg b/frontend/public/ultimo-logo-signet.svg new file mode 100644 index 0000000..f5ff865 --- /dev/null +++ b/frontend/public/ultimo-logo-signet.svg @@ -0,0 +1,34 @@ + + + + diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index d418060..4930104 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,7 +10,6 @@ import { Login } from "./routes/Login"; import { MailLayout } from "./routes/MailLayout"; import { Register } from "./routes/Register"; import { Settings } from "./routes/settings/Settings"; -import { AccountWizard } from "./routes/wizard/AccountWizard"; function RequireAuth({ children }: { children: JSX.Element }) { const { data, isLoading, isError } = useMe(); @@ -64,7 +63,7 @@ export function App() { path="/settings/accounts/new" element={ - + } /> diff --git a/frontend/src/components/BackfillBanner.tsx b/frontend/src/components/BackfillBanner.tsx index 5db843f..5ad907d 100644 --- a/frontend/src/components/BackfillBanner.tsx +++ b/frontend/src/components/BackfillBanner.tsx @@ -13,7 +13,9 @@ export function BackfillBanner() { const accounts = useQuery({ queryKey: ["accounts"], queryFn: api.accounts }); const importing = (accounts.data ?? []).filter( - (a) => a.state === "backfilling" && a.backfill_total > 0, + // Also require done < total so a completed account whose state event was missed + // (e.g. a dropped WS) never leaves a stuck "X of X" banner. + (a) => a.state === "backfilling" && a.backfill_total > 0 && a.backfill_done < a.backfill_total, ); const active = importing.length > 0; diff --git a/frontend/src/i18n/en.ts b/frontend/src/i18n/en.ts index 6a8e200..a8cbb75 100644 --- a/frontend/src/i18n/en.ts +++ b/frontend/src/i18n/en.ts @@ -239,6 +239,7 @@ export const en = { "wizard.creating": "Adding…", "wizard.created": "Account added — syncing your mail.", "wizard.startReading": "Start reading now", + "wizard.backToAccounts": "Back to accounts", "wizard.manualConfig": "Enter settings manually", "wizard.displayName": "Display name", "wizard.syncWindow": "How much mail to sync", @@ -249,6 +250,8 @@ export const en = { "wizard.syncWindow.new": "New mail only", "wizard.syncWindow.new.hint": "Nothing from before now — download older mail anytime from account settings.", + "wizard.syncSpam": "Sync spam?", + "wizard.syncSpam.hint": "Also import the Spam/Junk folder. Off by default.", // Provider credential-helper panel (app passwords, SPEC §6.2.2). "wizard.appPassword.title": "Create an app password", @@ -272,8 +275,9 @@ export const en = { "wizard.appPassword.oauthInstead": "Sign in with Google instead", // Gmail steps. "wizard.help.gmail.step1": "Turn on 2-Step Verification for your Google account if you haven't.", - "wizard.help.gmail.step2": "On the App passwords page, name the password (e.g. \"ultimo.email\") and create it.", - "wizard.help.gmail.step3": "Copy the 16-character password and paste it below.", + "wizard.help.gmail.step2": "Click the button to generate an app password.", + "wizard.help.gmail.step3": "On the App passwords page, name the password (e.g. \"ultimo.email\") and create it.", + "wizard.help.gmail.step4": "Copy the 16-character password and paste it below.", // Outlook / Microsoft personal steps. "wizard.help.outlook.step1": "Turn on two-step verification in your Microsoft account security settings.", "wizard.help.outlook.step2": "Under Advanced security options, create a new app password.", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 2c38fbe..8dbc023 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -133,6 +133,7 @@ export const api = { smtp: ServerConfig; password: string; sync_window?: SyncWindow; + sync_spam?: boolean; }) => apiFetch("/accounts", { method: "POST", body: JSON.stringify(input) }), accountDetail: (id: string) => apiFetch(`/accounts/${id}`), updateAccount: ( diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 35c75eb..62772da 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -116,6 +116,8 @@ export interface HelpBlock { app_name_hint: string; steps: string[]; // i18n message keys docs_url: string; + setup_link: string; // optional 2FA page linked from the first step + button_after_step: number; // 1-based step to place the app-password button after; 0 = top } export interface CandidateConfig { diff --git a/frontend/src/lib/ws.ts b/frontend/src/lib/ws.ts index 213855f..18fc1f5 100644 --- a/frontend/src/lib/ws.ts +++ b/frontend/src/lib/ws.ts @@ -37,15 +37,29 @@ export function subscribeWs(listener: WsListener): () => void { export function connectWs(queryClient: QueryClient): () => void { const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const socket = new WebSocket(`${proto}://${window.location.host}/api/ws`); + const url = `${proto}://${window.location.host}/api/ws`; - const ping = window.setInterval(() => { - if (socket.readyState === WebSocket.OPEN) { - socket.send(JSON.stringify({ type: "ping" })); + let socket: WebSocket | null = null; + let ping: number | undefined; + let reconnectTimer: number | undefined; + let attempts = 0; + let stopped = false; + + // Refetch the live-updated lists — used on every (re)connect so a dropped socket + // (server restart, sleep, network blip) self-heals and catches up on missed events. + const resync = () => { + for (const key of [ + ["threads"], + ["accounts"], + ["folder-counts"], + ["quota"], + ["drafts-remote"], + ]) { + queryClient.invalidateQueries({ queryKey: key }); } - }, 30000); + }; - socket.onmessage = (event) => { + const handleMessage = (event: MessageEvent) => { let parsed: unknown; try { parsed = JSON.parse(event.data); @@ -87,7 +101,15 @@ export function connectWs(queryClient: QueryClient): () => void { queryClient.invalidateQueries({ queryKey: ["folder-counts"] }); } if (name === "account.state_changed") { + // Adding/deleting an account changes threads and (Gmail) labels wholesale — + // refetch the list, chips, and sidebar so nothing stale lingers. queryClient.invalidateQueries({ queryKey: ["accounts"] }); + queryClient.invalidateQueries({ queryKey: ["threads"] }); + queryClient.invalidateQueries({ queryKey: ["thread-labels"] }); + queryClient.invalidateQueries({ queryKey: ["message-labels"] }); + queryClient.invalidateQueries({ queryKey: ["labels-all"] }); + queryClient.invalidateQueries({ queryKey: ["folder-counts"] }); + window.dispatchEvent(new Event("webmail:labels-changed")); } if (name === "message.sent") { // The sent message's draft is deleted server-side; refresh the Drafts list. @@ -96,8 +118,35 @@ export function connectWs(queryClient: QueryClient): () => void { } }; + const connect = () => { + socket = new WebSocket(url); + socket.onopen = () => { + attempts = 0; + ping = window.setInterval(() => { + if (socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "ping" })); + } + }, 30000); + resync(); // catch up on anything published while we were disconnected + }; + socket.onmessage = handleMessage; + socket.onclose = () => { + if (ping !== undefined) window.clearInterval(ping); + if (stopped) return; + // Exponential backoff, capped at 30s, so a downed server is retried politely. + const delay = Math.min(30000, 1000 * 2 ** attempts); + attempts += 1; + reconnectTimer = window.setTimeout(connect, delay); + }; + socket.onerror = () => socket?.close(); // route errors through onclose -> reconnect + }; + + connect(); + return () => { - window.clearInterval(ping); - socket.close(); + stopped = true; + if (ping !== undefined) window.clearInterval(ping); + if (reconnectTimer !== undefined) window.clearTimeout(reconnectTimer); + socket?.close(); }; } diff --git a/frontend/src/plugins/plugin-ui.css b/frontend/src/plugins/plugin-ui.css index 7770c37..edf5fb3 100644 --- a/frontend/src/plugins/plugin-ui.css +++ b/frontend/src/plugins/plugin-ui.css @@ -148,7 +148,25 @@ @apply flex items-center justify-between pt-3 px-3 pb-1; } .wm-side-switch { - @apply relative; + @apply relative flex items-center gap-2; +} +/* Ultimo signet in front of the app-switcher menu (Calendar / Contacts headers). */ +.wm-side-switch::before { + content: ""; + flex: 0 0 auto; + width: 28px; + height: 28px; + background: url("/ultimo-logo-signet.svg") center / contain no-repeat; +} +/* And at the top of the collapsed rail, above the expand/theme buttons. */ +.cal-rail::before, +.ct-rail::before { + content: ""; + flex: 0 0 auto; + width: 30px; + height: 30px; + margin: 10px auto 6px; + background: url("/ultimo-logo-signet.svg") center / contain no-repeat; } .wm-side-switch > button { @apply flex items-center gap-1.5 rounded px-1.5 py-1 text-[15px] font-semibold text-app-fg hover:bg-app-hover; diff --git a/frontend/src/routes/MailLayout.tsx b/frontend/src/routes/MailLayout.tsx index f1f6ed2..2171f2c 100644 --- a/frontend/src/routes/MailLayout.tsx +++ b/frontend/src/routes/MailLayout.tsx @@ -2,6 +2,7 @@ import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tansta import { useEffect, useRef, useState } from "react"; import { Link, useMatch, useNavigate, useSearchParams } from "react-router-dom"; +import signetLogo from "../../icons/ultimo-logo-signet.svg"; import { AppSwitcher } from "../components/AppSwitcher"; import { Composer, type ComposerInit, draftToInit } from "../components/compose/Composer"; import { DraftsList } from "../components/list/DraftsList"; @@ -512,7 +513,14 @@ export function MailLayout() { collapsed ? "flex-col items-center gap-1 px-2" : "items-center justify-between px-3" }`} > - {!collapsed && } + {collapsed ? ( + Ultimo + ) : ( +
+ Ultimo + +
+ )}
+
+ + {embedded && onDone && ( + + )} +
)} @@ -311,12 +339,19 @@ export function AccountWizard() { {step === 3 && (

{t("wizard.created")}

- +
+ + {embedded && onDone && ( + + )} +
)} diff --git a/frontend/src/routes/wizard/ProviderCredentialPanel.tsx b/frontend/src/routes/wizard/ProviderCredentialPanel.tsx index 72f6c43..9067949 100644 --- a/frontend/src/routes/wizard/ProviderCredentialPanel.tsx +++ b/frontend/src/routes/wizard/ProviderCredentialPanel.tsx @@ -94,22 +94,31 @@ export function ProviderCredentialPanel({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [value]); - const openDeepLink = () => { - // Must fire from this click (a direct user gesture) so popup blockers allow it. - // Sized features open a popup window rather than a new tab. + // Open a provider page in a popup window (not a tab). Must fire from a direct + // user gesture (this click) so popup blockers allow it. + const openPopup = (url: string, name: string) => { const w = 720; const h = 680; - // Position the popup at the right (3/3) of the screen, pulled 200px back from - // the edge so it doesn't touch it; clamped to stay on-screen. + // Position at the right (3/3) of the screen, pulled 200px back from the edge. const left = Math.max(0, window.screen.width - w - 200); const top = window.screenY + Math.max(0, (window.outerHeight - h) / 2); window.open( - help.deep_link, - `${help.help_id}-app-passwords`, + url, + `${help.help_id}-${name}`, `popup,noopener,noreferrer,width=${w},height=${h},left=${left},top=${top}`, ); }; + const appPasswordButton = ( + + ); + return (
@@ -117,24 +126,27 @@ export function ProviderCredentialPanel({

{t("wizard.appPassword.intro")}

- + {/* Button above the list only when no step placement is configured. */} + {help.button_after_step === 0 && appPasswordButton} -
    - {help.steps.map((key) => ( -
  1. {t(key as MessageKey)}
    2. +
      + {help.steps.map((key, i) => ( +
    1. + {i === 0 && help.setup_link ? ( + + ) : ( + t(key as MessageKey) + )} + {help.button_after_step === i + 1 &&
      {appPasswordButton}
      } +
      2. ))}
    - {help.app_name_hint && ( -

    - {t("wizard.appPassword.appNameHint", { hint: help.app_name_hint })} -

    - )}
    [Gmail]`` chip using the same KV layout as the rest of the feature: a +per-message key, a ``t:`` thread aggregate, and the ``_labels`` known set. +Names are merged (union), never overwritten, so app-native labels are preserved. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from app.plugins.sdk import sdk + +logger = logging.getLogger("plg.labels") + +_ALL_KEY = "_labels" +_GMAIL_SUFFIX = " [Gmail]" + + +async def _merge(kv: Any, key: str, names: set[str]) -> None: + """Union ``names`` into the list stored at ``key`` (no-op if already present).""" + current = set(await kv.get(key, [])) + if names <= current: + return + await kv.set(key, sorted(current | names)) + + +async def on_message_new(payload: dict[str, Any]) -> None: + labels = payload.get("gmail_labels") + user_raw, message_id = payload.get("user_id"), payload.get("message_id") + thread_id = payload.get("thread_id") + if not labels or not isinstance(user_raw, str) or not isinstance(message_id, str): + return + names = {f"{str(name).strip()}{_GMAIL_SUFFIX}" for name in labels if str(name).strip()} + if not names: + return + + s = sdk("labels") + kv = s.kv(uuid.UUID(user_raw)) + await _merge(kv, message_id, names) # per-message labels + if isinstance(thread_id, str): + await _merge(kv, f"t:{thread_id}", names) # thread aggregate (list view chips) + + # Known-label set (sidebar). When it actually grows, push plugin.labels.changed + # over the WS so the sidebar list refreshes live during a sync (SPEC §9.2/§11.3). + known = set(await kv.get(_ALL_KEY, [])) + fresh = names - known + if fresh: + await kv.set(_ALL_KEY, sorted(known | names)) + await s.publish("changed", {"user_id": user_raw}) + logger.info("labels: %d new Gmail label(s) from message %s", len(fresh), message_id) diff --git a/plugins/webmail_plugin_labels/webmail_plugin_labels/router.py b/plugins/webmail_plugin_labels/webmail_plugin_labels/router.py index e016521..e7f9d07 100644 --- a/plugins/webmail_plugin_labels/webmail_plugin_labels/router.py +++ b/plugins/webmail_plugin_labels/webmail_plugin_labels/router.py @@ -80,26 +80,76 @@ async def set_thread_labels( @router.post("/threads/labels") async def batch_thread_labels(body: IdsIn, user_id: CurrentUserId) -> dict[str, list[str]]: + """Aggregated labels for a set of threads (list-view chips). Reconciles first so + the chips reflect live messages — stale ones from deleted mail are dropped.""" + live = await _reconcile(user_id) + return {tid: live[tid] for tid in body.ids if live.get(tid)} + + +_GMAIL_SUFFIX = " [Gmail]" + + +async def _reconcile(user_id: object) -> dict[str, list[str]]: + """Self-heal label state from the per-message keys (the source of truth) and + return the live ``thread_id -> labels`` map. + + Per-message keys (````) are authoritative; the ``t:`` + aggregates and the ``_labels`` set are derived. Messages get deleted (account + removal) and re-threaded (JWZ merges) out from under us, which orphans thread + aggregates — so rebuild them here: drop labels of deleted messages, regroup the + survivors by their *current* thread, rewrite changed/stale ``t:`` keys, and + rebuild ``_labels`` from labels on live messages. User-created labels (no + ``… [Gmail]`` suffix) are preserved even when unused; orphaned Gmail labels drop.""" kv = _SDK.kv(user_id) - out = {tid: await kv.get(f"t:{tid}", []) for tid in body.ids} - return {tid: labels for tid, labels in out.items() if labels} + items = await kv.items() + msg_keys = { + k: v + for k, v in items.items() + if k != _ALL_KEY and not k.startswith("t:") and isinstance(v, list) + } + homes = await _SDK.messages.threads_of(user_id, list(msg_keys)) + + thread_labels: dict[str, set[str]] = {} + live_labels: set[str] = set() + for message_id, labels in msg_keys.items(): + if message_id not in homes: # message deleted -> drop its orphaned key + await kv.delete(message_id) + continue + live_labels.update(labels) + thread_id = homes[message_id] + if thread_id is not None: + thread_labels.setdefault(thread_id, set()).update(labels) + + # Rewrite only the thread aggregates that changed; delete ones with no messages. + old_thread_keys = {k[2:] for k in items if k.startswith("t:")} + result: dict[str, list[str]] = {} + for thread_id, names in thread_labels.items(): + value = sorted(names) + result[thread_id] = value + if items.get(f"t:{thread_id}") != value: + await kv.set(f"t:{thread_id}", value) + for thread_id in old_thread_keys - set(thread_labels): + await kv.delete(f"t:{thread_id}") + + known = set(items.get(_ALL_KEY, []) or []) + new_known = {n for n in known if not n.endswith(_GMAIL_SUFFIX)} | live_labels + if new_known != known: + await kv.set(_ALL_KEY, sorted(new_known)) + return result @router.get("/labels", response_model=list[str]) async def all_labels(user_id: CurrentUserId) -> list[str]: + await _reconcile(user_id) # self-heal from live messages, then return the known set return await _SDK.kv(user_id).get(_ALL_KEY, []) @router.get("/labels/{name}/threads", response_model=list[str]) async def label_threads(name: str, user_id: CurrentUserId) -> list[str]: - """Thread ids whose aggregate labels include ``name`` — powers the label view - in the mail list. Scans the per-thread label keys (``t:``).""" - items = await _SDK.kv(user_id).items() - return [ - key[2:] - for key, labels in items.items() - if key.startswith("t:") and isinstance(labels, list) and name in labels - ] + """Thread ids whose aggregate labels include ``name`` — powers the label view in + the mail list. Reconciles first, so it reflects live messages' current threads.""" + live = await _reconcile(user_id) + return [tid for tid, labels in live.items() if name in labels] @router.post("/labels", dependencies=[Depends(require_csrf)]) diff --git a/plugins/webmail_plugin_labels/webmail_plugin_labels/ui/sidebar.js b/plugins/webmail_plugin_labels/webmail_plugin_labels/ui/sidebar.js index ef4d61f..2ada133 100644 --- a/plugins/webmail_plugin_labels/webmail_plugin_labels/ui/sidebar.js +++ b/plugins/webmail_plugin_labels/webmail_plugin_labels/ui/sidebar.js @@ -128,8 +128,18 @@ export default function mount(el, ctx) { refresh(); window.addEventListener("webmail:labels-changed", refresh); + // Live update while syncing: the backend pushes plugin.labels.changed over the WS + // whenever a new Gmail label is mapped. Re-dispatch the local event so this list + // (and Mail's chip queries) refresh together. + const unsubscribe = + ctx.events && ctx.events.subscribe + ? ctx.events.subscribe("plugin.labels.changed", () => + window.dispatchEvent(new Event("webmail:labels-changed")), + ) + : null; return () => { window.removeEventListener("webmail:labels-changed", refresh); + if (unsubscribe) unsubscribe(); el.innerHTML = ""; }; }