Gmail label sync + no-dup backfill, live label/UI updates, account-delete cleanup, Sync-SPAM toggle, sidebar logo

Gmail sync (SPEC: Gmail sync):
- Provider fetches X-GM-LABELS when X-GM-EXT-1 is advertised; parses custom labels
  (system labels dropped, mUTF-7 decoded) onto RemoteMessage.
- Engine syncs only system folders + All Mail for Gmail, skips every label folder,
  and dedupes the All Mail pass by Message-ID — kills the INBOX/All-Mail duplication.
- Custom labels ride the message.new event; the labels plugin subscribes and maps
  them to chips as "<name> [Gmail]" (per-message + thread aggregate + known set),
  so labels fill in gradually with the sync window. Non-ASCII labels decode right.

Labels self-healing (message-driven reconcile):
- Per-message keys are the source of truth; t: aggregates and _labels are derived.
  Reconcile drops labels of deleted messages, regroups survivors by their current
  thread (fixing JWZ thread merges), rewrites changed t: keys, and rebuilds _labels
  from live messages — keeping user labels, dropping orphaned Gmail ones.
- New SDK reads: sdk.threads.existing / sdk.messages.threads_of. Sidebar refreshes
  live via a plugin.labels.changed WS event on new labels.

Account deletion: prune threads left empty after the cascade (they were user-scoped,
so only messages cascaded) + de-list the account from surviving cross-account threads.

Sync SPAM: opt-in per-account sync_spam (migration 0015, default off) — the engine
skips the Junk folder in backfill and poll unless enabled; wizard checkbox added.

Robustness / UX:
- WebSocket auto-reconnects with backoff and resyncs on (re)connect, and account
  add/delete invalidates threads/labels — no more hard refresh for live updates.
- Backfill banner + sidebar ring also require done < total, so a completed account
  can't leave a stuck "X of X".
- Scrub lone UTF-16 surrogates from parsed headers/bodies so bad messages no longer
  crash the backfill on Postgres text/JSONB.
- Ultimo signet logo in the Mail/Calendar/Contacts sidebar header (and at the top of
  the collapsed rail), via the shared plugin-ui.css + MailLayout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA

backend/app/accounts/discovery.py

@@ -49,6 +49,8 @@ def _help_block(raw: object) -> HelpBlock | None:
         app_name_hint=str(raw.get("app_name_hint", "")),
         steps=tuple(str(s) for s in steps) if isinstance(steps, list) else (),
         docs_url=str(raw.get("docs_url", "")),
+        setup_link=str(raw.get("setup_link", "")),
+        button_after_step=int(raw.get("button_after_step", 0)),
     )
 
 

backend/app/accounts/presets.yaml

@@ -25,12 +25,15 @@ providers:
     help:
       help_id: gmail
       deep_link: https://myaccount.google.com/apppasswords
+      setup_link: https://myaccount.google.com/signinoptions/two-step-verification
       app_name_hint: "ultimo.email"
+      button_after_step: 2
       docs_url: https://support.google.com/mail/answer/185833
       steps:
         - wizard.help.gmail.step1
         - wizard.help.gmail.step2
         - wizard.help.gmail.step3
+        - wizard.help.gmail.step4
 
   microsoft:
     domains: [outlook.com, hotmail.com, live.com, msn.com]

backend/app/accounts/schemas.py

@@ -28,6 +28,8 @@ class HelpBlockOut(BaseModel):
     app_name_hint: str = ""
     steps: list[str] = []
     docs_url: str = ""
+    setup_link: str = ""
+    button_after_step: int = 0
 
 
 class CandidateOut(BaseModel):
@@ -73,6 +75,7 @@ class CreateAccountRequest(BaseModel):
     username: str | None = None
     password: str = Field(min_length=1)
     sync_window: SyncWindow = "all"
+    sync_spam: bool = False
 
 
 class UpdateAccountRequest(BaseModel):

backend/app/accounts/service.py

@@ -4,14 +4,14 @@ from __future__ import annotations
 
 import uuid
 
-from sqlalchemy import delete, select
+from sqlalchemy import delete, func, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.accounts.schemas import CreateAccountRequest
 from app.accounts.types import ServerConfig
 from app.indexer import blobstore
-from app.models import Account, Attachment, Message
+from app.models import Account, Attachment, Message, Thread
 from app.security.crypto import encrypt_secret
 
 
@@ -40,6 +40,7 @@ async def create_account(
         discovery_src="manual",
         state="backfilling",
         sync_window=req.sync_window,
+        sync_spam=req.sync_spam,
     )
     session.add(account)
     try:
@@ -81,6 +82,23 @@ async def delete_account(
     await session.execute(delete(Account).where(Account.id == account_id))
     await session.flush()
 
+    # Threads are user-scoped and can span accounts (Thread.account_ids), so the
+    # account cascade only removes its messages — not the threads. Clean up: drop
+    # threads left with no messages (else they linger as empty rows the unified list
+    # and label views still show but can't open), and de-list the account from
+    # threads that survive on another account's messages.
+    await session.execute(
+        delete(Thread).where(
+            Thread.user_id == user_id,
+            ~select(Message.id).where(Message.thread_id == Thread.id).exists(),
+        )
+    )
+    await session.execute(
+        update(Thread)
+        .where(Thread.user_id == user_id, Thread.account_ids.contains([account_id]))
+        .values(account_ids=func.array_remove(Thread.account_ids, account_id))
+    )
+
     # Deregister from sync:live and drop any lease so the owning sync worker
     # stops the account's runner on its next renewal (refactor §2 fencing).
     from app.events import get_event_bus

backend/app/accounts/types.py

@@ -22,6 +22,8 @@ class HelpBlock:
     app_name_hint: str  # suggested app-password label
     steps: tuple[str, ...]  # i18n message keys
     docs_url: str = ""
+    setup_link: str = ""  # optional 2FA/security page linked from the first step
+    button_after_step: int = 0  # 1-based step to place the app-password button after; 0 = top
 
 
 @dataclass(frozen=True)

backend/app/api/accounts.py

@@ -72,6 +72,8 @@ def _candidate_out(c: CandidateConfig) -> CandidateOut:
                 app_name_hint=c.help.app_name_hint,
                 steps=list(c.help.steps),
                 docs_url=c.help.docs_url,
+                setup_link=c.help.setup_link,
+                button_after_step=c.help.button_after_step,
             )
             if c.help
             else None

backend/app/indexer/indexer.py

@@ -36,6 +36,7 @@ class PersistResult:
     message: Message
     search_doc: SearchDoc
     has_remote_images: bool
+    gmail_labels: tuple[str, ...] = ()
 
 
 def _epoch(value: datetime) -> int:
@@ -140,8 +141,13 @@ async def persist_message(
     mailbox_id: uuid.UUID,
     mailbox_special_use: str | None,
     remote: RemoteMessage,
+    dedupe_account_scope: bool = False,
 ) -> PersistResult | None:
-    """Persist one message. Returns None if it already exists (idempotent)."""
+    """Persist one message. Returns None if it already exists (idempotent).
+
+    ``dedupe_account_scope`` skips a message already stored under a *different*
+    mailbox of the same account (same Message-ID) — used for Gmail's All Mail pass so
+    a message already synced from INBOX/Sent isn't duplicated (SPEC: Gmail sync)."""
     existing = await session.scalar(
         select(Message.id).where(Message.mailbox_id == mailbox_id, Message.uid == remote.uid)
     )
@@ -154,6 +160,18 @@ async def persist_message(
     message_id_hdr = await _resolve_message_id(
         session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw
     )
+    if dedupe_account_scope and message_id_hdr is not None:
+        dup = await session.scalar(
+            select(Message.id)
+            .where(
+                Message.account_id == account_id,
+                Message.message_id_hdr == message_id_hdr,
+                Message.mailbox_id != mailbox_id,
+            )
+            .limit(1)
+        )
+        if dup is not None:
+            return None
     message = Message(
         id=uuid.uuid4(),
         user_id=user_id,
@@ -223,4 +241,9 @@ async def persist_message(
 
     await session.flush()
     doc = _build_search_doc(message, parsed, mailbox_special_use)
-    return PersistResult(message=message, search_doc=doc, has_remote_images=has_remote_images)
+    return PersistResult(
+        message=message,
+        search_doc=doc,
+        has_remote_images=has_remote_images,
+        gmail_labels=remote.gmail_labels,
+    )

backend/app/indexer/mime.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import email
 import os
+import re
 from dataclasses import dataclass, field
 from datetime import datetime
 from email.headerregistry import AddressHeader
@@ -20,6 +21,19 @@ from app.threading_.normalize import normalize_message_id, normalize_reference_l
 
 _SNIPPET_LEN = 160
 
+# Undecodable header/body bytes come out of the stdlib parser as lone UTF-16
+# surrogates (U+D800–U+DFFF). Those are invalid UTF-8, so Postgres rejects them in
+# text/JSONB columns — scrub them to U+FFFD before anything is stored.
+_SURROGATE_RE = re.compile("[\ud800-\udfff]")
+
+
+def _scrub(value: str) -> str:
+    return _SURROGATE_RE.sub("�", value) if value else value
+
+
+def _scrub_addrs(addrs: list[dict[str, str]]) -> list[dict[str, str]]:
+    return [{k: _scrub(v) for k, v in addr.items()} for addr in addrs]
+
 
 @dataclass(frozen=True)
 class ParsedAttachment:
@@ -112,7 +126,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
         inline = cid is not None and disposition != "attachment"
         out.append(
             ParsedAttachment(
-                filename=filename or "attachment",
+                filename=_scrub(filename or "attachment"),
                 mime_type=part.get_content_type(),
                 content_id=cid.strip("<>") if (inline and cid is not None) else None,
                 payload=payload,
@@ -166,13 +180,13 @@ def _parse_email(msg: EmailMessage) -> ParsedEmail:
         message_id=message_id,
         in_reply_to=in_reply_to,
         references=references,
-        subject=subject,
-        from_addr=_addresses(msg, "From"),
-        to_addr=_addresses(msg, "To"),
-        cc_addr=_addresses(msg, "Cc"),
-        bcc_addr=_addresses(msg, "Bcc"),
+        subject=_scrub(subject),
+        from_addr=_scrub_addrs(_addresses(msg, "From")),
+        to_addr=_scrub_addrs(_addresses(msg, "To")),
+        cc_addr=_scrub_addrs(_addresses(msg, "Cc")),
+        bcc_addr=_scrub_addrs(_addresses(msg, "Bcc")),
         date=date,
-        text_body=text_body or "",
-        html_body=html_body,
+        text_body=_scrub(text_body or ""),
+        html_body=_scrub(html_body) if html_body is not None else None,
         attachments=_extract_attachments(msg),
     )

backend/app/models.py

@@ -86,6 +86,8 @@ class Account(Base):
     # How much history to import (SPEC §12 M2): all | 3months | new. Drives the
     # initial backfill; backfill_total/backfill_done feed the progress UX.
     sync_window: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("'all'"))
+    # Whether the Spam/Junk folder is synced (opt-in; SPEC §6.3 folder selection).
+    sync_spam: Mapped[bool] = mapped_column(nullable=False, server_default=text("false"))
     backfill_total: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     backfill_done: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     last_sync_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True))

backend/app/plugins/actions.py

@@ -22,7 +22,7 @@ from sqlalchemy import select
 from app.db import session_scope
 from app.events import get_event_bus
 from app.indexer import blobstore
-from app.models import Message
+from app.models import Message, Thread
 
 
 class PluginMessages:
@@ -44,6 +44,29 @@ class PluginMessages:
             return None
         return message_from_bytes(raw).get(name)
 
+    async def threads_of(
+        self, user_id: uuid.UUID, message_ids: Iterable[str]
+    ) -> dict[str, str | None]:
+        """Map still-existing message ids to their current thread id (None if
+        unthreaded). Deleted messages are omitted, so a plugin can prune stale
+        per-message references and re-home labels onto the live thread after a
+        JWZ thread merge."""
+        ids: list[uuid.UUID] = []
+        for mid in message_ids:
+            try:
+                ids.append(uuid.UUID(str(mid)))
+            except (ValueError, AttributeError, TypeError):
+                continue
+        if not ids:
+            return {}
+        async with session_scope() as session:
+            rows = await session.execute(
+                select(Message.id, Message.thread_id).where(
+                    Message.user_id == user_id, Message.id.in_(ids)
+                )
+            )
+            return {str(mid): (str(tid) if tid else None) for mid, tid in rows}
+
     async def envelope(self, user_id: uuid.UUID, message_id: uuid.UUID) -> dict[str, Any] | None:
         async with session_scope() as session:
             msg = await session.scalar(
@@ -66,6 +89,29 @@ class PluginMessages:
             }
 
 
+class PluginThreads:
+    """Read-only, user-scoped thread queries (e.g. self-healing stale references)."""
+
+    async def existing(self, user_id: uuid.UUID, thread_ids: Iterable[str]) -> set[str]:
+        """Return the subset of ``thread_ids`` that still exist for the user.
+
+        Lets a plugin prune references to threads deleted out from under it (e.g. an
+        account removal drops its threads, orphaning label aggregates keyed by id)."""
+        ids: list[uuid.UUID] = []
+        for tid in thread_ids:
+            try:
+                ids.append(uuid.UUID(str(tid)))
+            except (ValueError, AttributeError, TypeError):
+                continue
+        if not ids:
+            return set()
+        async with session_scope() as session:
+            rows = await session.scalars(
+                select(Thread.id).where(Thread.user_id == user_id, Thread.id.in_(ids))
+            )
+            return {str(r) for r in rows}
+
+
 class PluginActions:
     """User-scoped actions a plugin can take on a message."""
 

backend/app/plugins/sdk.py

@@ -28,7 +28,7 @@ from app.identity.deps import current_user, require_csrf
 from app.indexer import blobstore
 from app.models import User
 from app.outbox import service as outbox_service
-from app.plugins.actions import PluginActions, PluginMessages
+from app.plugins.actions import PluginActions, PluginMessages, PluginThreads
 from app.plugins.db import plugin_session
 from app.plugins.http import PluginHttp
 from app.plugins.kv import PluginKvStore
@@ -151,6 +151,11 @@ class PluginSdk:
         """User-scoped message actions (``set_flags``) — e.g. for filter rules."""
         return PluginActions()
 
+    @property
+    def threads(self) -> PluginThreads:
+        """Read-only thread queries (e.g. pruning stale references)."""
+        return PluginThreads()
+
     # --- outbound HTTP ------------------------------------------------------
     async def http_get(self, url: str) -> str | None:
         """GET ``url`` through the §10.6 SSRF guard (blocks private IPs/oversize)."""

backend/app/providers/base.py

@@ -46,6 +46,7 @@ class RemoteMessage:
     raw: bytes  # full RFC822
     flags: frozenset[str]
     internal_date: datetime
+    gmail_labels: tuple[str, ...] = ()  # custom Gmail labels (Gmail accounts only)
 
 
 @dataclass(frozen=True)

backend/app/providers/imap_smtp/parsing.py

@@ -12,6 +12,7 @@ from dataclasses import dataclass
 from datetime import datetime
 
 from imapclient.imap_utf7 import decode as imap_utf7_decode
+from imapclient.imap_utf7 import encode as imap_utf7_encode
 
 # --- SPECIAL-USE mapping (SPEC §4 special_use values) -----------------------
 
@@ -27,6 +28,7 @@ _SPECIAL_USE_FLAGS = {
 _LIST_RE = re.compile(rb'^\(([^)]*)\)\s+(?:"([^"]*)"|NIL)\s+(.+)$')
 _UID_RE = re.compile(rb"UID\s+(\d+)")
 _FLAGS_RE = re.compile(rb"FLAGS\s+\(([^)]*)\)")
+_GM_LABELS_RE = re.compile(rb"X-GM-LABELS\s+\(([^)]*)\)")
 _INTERNALDATE_RE = re.compile(rb'INTERNALDATE\s+"([^"]+)"')
 _LITERAL_RE = re.compile(rb"\{(\d+)\}\s*$")
 _FETCH_HEADER_RE = re.compile(rb"^\d+\s+FETCH\s+\(", re.IGNORECASE)
@@ -44,6 +46,7 @@ class ParsedFetch:
     flags: frozenset[str]
     internal_date: datetime | None
     raw: bytes | None  # None when the fetch did not request the body
+    gmail_labels: tuple[str, ...] = ()  # custom Gmail labels (X-GM-LABELS), system ones dropped
 
 
 def _decode_mailbox_name(raw: bytes) -> str:
@@ -54,12 +57,29 @@ def _decode_mailbox_name(raw: bytes) -> str:
     return str(imap_utf7_decode(bytes(name)))
 
 
+def encode_mailbox_name(name: str) -> str:
+    """Modified UTF-7 wire form of a mailbox name (inverse of ``_decode_mailbox_name``).
+
+    Mailbox names are stored/displayed as Unicode but IMAP commands
+    (SELECT/STATUS/APPEND) require them mUTF-7 encoded. ASCII names are unchanged;
+    non-ASCII names — e.g. Gmail's localized ``[Gmail]/Wysłane`` — get encoded so the
+    server can find them."""
+    return str(imap_utf7_encode(name).decode("ascii"))
+
+
 def parse_list_line(line: bytes) -> ParsedMailbox | None:
-    """Parse one LIST response line into a :class:`ParsedMailbox` (or None)."""
+    """Parse one LIST response line into a :class:`ParsedMailbox` (or None).
+
+    Non-selectable mailboxes are skipped (return None): ``\\Noselect`` /
+    ``\\NonExistent`` names are pure hierarchy containers that hold no messages and
+    cannot be SELECTed — e.g. Gmail's ``[Gmail]`` root or a parent label like
+    ``Serial numbers`` that only groups child labels."""
     m = _LIST_RE.match(line.strip())
     if not m:
         return None
     flags = m.group(1).lower().split()
+    if b"\\noselect" in flags or b"\\nonexistent" in flags:
+        return None
     name = _decode_mailbox_name(m.group(3))
     special: str | None = None
     for flag in flags:
@@ -100,6 +120,57 @@ def parse_fetch_header(line: bytes) -> tuple[int | None, frozenset[str], datetim
     return uid, flags, internal
 
 
+def _tokenize_gm_labels(blob: bytes) -> list[bytes]:
+    """Split an ``X-GM-LABELS (...)`` body into raw tokens, keeping quoted strings
+    (which may contain spaces) whole. Escaped chars inside quotes are unescaped."""
+    tokens: list[bytes] = []
+    i, n = 0, len(blob)
+    while i < n:
+        ch = blob[i : i + 1]
+        if ch in (b" ", b"\t"):
+            i += 1
+            continue
+        if ch == b'"':
+            j = i + 1
+            buf = bytearray()
+            while j < n and blob[j : j + 1] != b'"':
+                if blob[j : j + 1] == b"\\" and j + 1 < n:
+                    buf += blob[j + 1 : j + 2]
+                    j += 2
+                    continue
+                buf += blob[j : j + 1]
+                j += 1
+            tokens.append(bytes(buf))
+            i = j + 1
+        else:
+            j = i
+            while j < n and blob[j : j + 1] not in (b" ", b"\t"):
+                j += 1
+            tokens.append(blob[i:j])
+            i = j
+    return tokens
+
+
+def parse_gm_labels(header: bytes) -> tuple[str, ...]:
+    """Custom Gmail labels from a FETCH header's ``X-GM-LABELS`` (SPEC: Gmail sync).
+
+    System labels (``\\Inbox``, ``\\Sent``, ``\\Important``, ``\\Starred``, …) are
+    dropped — only user labels become chips. Names are modified-UTF-7 decoded, like
+    mailbox names, so e.g. ``Zg&AAI...`` → the real Unicode label."""
+    m = _GM_LABELS_RE.search(header)
+    if not m:
+        return ()
+    out: list[str] = []
+    for tok in _tokenize_gm_labels(m.group(1)):
+        if not tok or tok.startswith(b"\\"):  # system label
+            continue
+        try:
+            out.append(str(imap_utf7_decode(tok)))
+        except Exception:  # noqa: BLE001 - fall back to a best-effort ASCII decode
+            out.append(tok.decode("utf-8", "ignore"))
+    return tuple(out)
+
+
 def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
     """Walk a full FETCH response, returning one :class:`ParsedFetch` per message.
 
@@ -131,7 +202,15 @@ def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
         else:
             i += 1
         if uid is not None:
-            results.append(ParsedFetch(uid=uid, flags=flags, internal_date=internal, raw=raw))
+            results.append(
+                ParsedFetch(
+                    uid=uid,
+                    flags=flags,
+                    internal_date=internal,
+                    raw=raw,
+                    gmail_labels=parse_gm_labels(header),
+                )
+            )
     return results
 
 

backend/app/providers/imap_smtp/provider.py

@@ -31,6 +31,7 @@ from app.providers.base import (
     RemoteMessage,
 )
 from app.providers.imap_smtp.parsing import (
+    encode_mailbox_name,
     parse_fetch_lines,
     parse_list_line,
     parse_search_uids,
@@ -39,6 +40,9 @@ from app.providers.imap_smtp.parsing import (
 )
 
 _FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
+# Gmail (X-GM-EXT-1): also pull each message's labels so custom labels can map to
+# chips without visiting every label folder (SPEC: Gmail sync / no-duplication).
+_FETCH_ITEMS_GM = "(UID FLAGS INTERNALDATE RFC822.SIZE X-GM-LABELS BODY.PEEK[])"
 _IDLE_TIMEOUT = 25 * 60  # SPEC §5.1: re-issue IDLE at least every 25 min
 _CONNECT_TIMEOUT = 20
 _PER_ACCOUNT_CONN_CAP = 4  # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
@@ -81,6 +85,13 @@ class ImapSmtpProvider(MailProvider):
         self._uidvalidity_seen: dict[str, int] = {}
         self._conn_slots: list[asyncio.Semaphore] = []
         self._qresync_enabled = False
+        self._gm_ext = False
+
+    @property
+    def gmail_labels_supported(self) -> bool:
+        """True when the server advertises ``X-GM-EXT-1`` (Gmail), so FETCH can pull
+        ``X-GM-LABELS`` and the engine can use the no-duplication Gmail sync path."""
+        return self._gm_ext
 
     @property
     def qresync_enabled(self) -> bool:
@@ -148,6 +159,8 @@ class ImapSmtpProvider(MailProvider):
             self._release_conn_slots()  # never hold a slot without an open socket
             raise
         self._imap = imap
+        caps = {c.upper() for c in (imap.protocol.capabilities or set())}
+        self._gm_ext = "X-GM-EXT-1" in caps
         await self._maybe_enable_qresync(imap)
         return imap
 
@@ -165,9 +178,16 @@ class ImapSmtpProvider(MailProvider):
         except Exception:  # noqa: BLE001 - degrade gracefully to the reconcile path
             self._qresync_enabled = False
 
+    @staticmethod
+    def _wire(mailbox: str) -> str:
+        """Server wire form of a mailbox name: mUTF-7 encoded and quoted. Handles
+        non-ASCII names (Gmail's localized ``[Gmail]/Wysłane``) and names with
+        spaces (``[Gmail]/All Mail``); ASCII names are unchanged bar the quotes."""
+        return str(aioimaplib.quoted(encode_mailbox_name(mailbox)))
+
     async def _select(self, mailbox: str) -> dict[str, int]:
         imap = await self._connect()
-        resp = await imap.select(mailbox)
+        resp = await imap.select(self._wire(mailbox))
         if resp.result != "OK":
             raise ProviderConnectError(f"cannot select {mailbox!r}")
         self._selected = mailbox
@@ -212,7 +232,7 @@ class ImapSmtpProvider(MailProvider):
     async def _mailbox_uidvalidity(self, path: str) -> int:
         imap = await self._connect()
         try:
-            resp = await imap.status(path, "(UIDVALIDITY)")
+            resp = await imap.status(self._wire(path), "(UIDVALIDITY)")
         except Exception:
             return 0
         if resp.result != "OK":
@@ -280,7 +300,8 @@ class ImapSmtpProvider(MailProvider):
         if not uids:
             return []
         uid_set = ",".join(str(u) for u in uids)
-        resp = await imap.uid("fetch", uid_set, _FETCH_ITEMS)
+        items = _FETCH_ITEMS_GM if self._gm_ext else _FETCH_ITEMS
+        resp = await imap.uid("fetch", uid_set, items)
         if resp.result != "OK":
             raise ProviderConnectError("FETCH failed")
         out: list[RemoteMessage] = []
@@ -289,7 +310,11 @@ class ImapSmtpProvider(MailProvider):
                 continue
             out.append(
                 RemoteMessage(
-                    uid=pf.uid, raw=pf.raw, flags=pf.flags, internal_date=pf.internal_date
+                    uid=pf.uid,
+                    raw=pf.raw,
+                    flags=pf.flags,
+                    internal_date=pf.internal_date,
+                    gmail_labels=pf.gmail_labels,
                 )
             )
         return out
@@ -355,7 +380,7 @@ class ImapSmtpProvider(MailProvider):
         proto = imap.protocol
         param = f"(QRESYNC ({uidvalidity} {modseq}))"
         cmd = aioimaplib.Command(
-            "SELECT", proto.new_tag(), aioimaplib.quoted(mailbox), param, loop=proto.loop
+            "SELECT", proto.new_tag(), self._wire(mailbox), param, loop=proto.loop
         )
         return await proto.execute(cmd)
 
@@ -419,7 +444,7 @@ class ImapSmtpProvider(MailProvider):
     async def append(self, mailbox: str, raw: bytes, flags: frozenset[str]) -> int:
         imap = await self._connect()
         flag_str = f"({' '.join(sorted(flags))})" if flags else None
-        resp = await imap.append(raw, mailbox=mailbox, flags=flag_str)
+        resp = await imap.append(raw, mailbox=self._wire(mailbox), flags=flag_str)
         if resp.result != "OK":
             raise ProviderConnectError(f"APPEND to {mailbox} failed")
         for line in resp.lines:

backend/app/sync/engine.py

@@ -27,7 +27,15 @@ from app.sync.provider_factory import build_provider
 logger = logging.getLogger(__name__)
 
 # Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3).
+# archive (Gmail All Mail) sorts last so its dedupe pass runs after the system folders.
 _PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2}
+# Gmail sync (SPEC: Gmail sync): the only folders backfilled — All Mail (archive) is
+# the complete store, the rest are the system folders needed for their views. Every
+# custom-label folder is skipped; labels come from X-GM-LABELS instead.
+_GMAIL_SYNC_FOLDERS = {"inbox", "sent", "drafts", "trash", "junk", "archive"}
+# Progress denominator for Gmail: All Mail already contains INBOX/Sent/Drafts/labels,
+# so only these three are counted (Trash/Spam sit outside All Mail).
+_GMAIL_DENOM_FOLDERS = {"archive", "trash", "junk"}
 
 # Sync-window import strategy (SPEC §12 M2). "all" imports the most recent
 # _RECENT_DAYS first (so the account is usable fast), then the older remainder.
@@ -110,15 +118,17 @@ async def _index_and_notify(results: list[PersistResult], account: Account) -> N
     bus = get_event_bus()
     for r in results:
         await backend.index(r.search_doc)
-        await bus.publish(
-            "message.new",
-            {
-                "user_id": str(account.user_id),
-                "account_id": str(account.id),
-                "message_id": str(r.message.id),
-                "thread_id": str(r.message.thread_id),
-            },
-        )
+        payload: dict[str, object] = {
+            "user_id": str(account.user_id),
+            "account_id": str(account.id),
+            "message_id": str(r.message.id),
+            "thread_id": str(r.message.thread_id),
+        }
+        # Gmail custom labels ride along so the labels plugin can map them to chips
+        # (SPEC: Gmail sync). Only present on Gmail accounts with labelled messages.
+        if r.gmail_labels:
+            payload["gmail_labels"] = list(r.gmail_labels)
+        await bus.publish("message.new", payload)
 
 
 async def _emit_batch(
@@ -178,12 +188,15 @@ async def backfill_mailbox(
     *,
     since: date | None = None,
     before: date | None = None,
+    dedupe_account_scope: bool = False,
 ) -> int:
     """Backfill one mailbox newest-first, indexing inline per batch (FR-30).
 
     ``since``/``before`` bound the import by date (sync window, SPEC §12 M2).
     When a ``progress`` tracker is given, batches count toward the account-level
     backfill total; otherwise each batch emits a standalone ``sync.progress``.
+    ``dedupe_account_scope`` drops messages already synced under another mailbox of
+    the account (Gmail All Mail pass — SPEC: Gmail sync / no-duplication).
     """
     from app.db import session_scope
 
@@ -202,6 +215,7 @@ async def backfill_mailbox(
                     mailbox_id=mailbox_id,
                     mailbox_special_use=special_use,
                     remote=remote,
+                    dedupe_account_scope=dedupe_account_scope,
                 )
                 if r is not None:
                     results.append(r)
@@ -257,6 +271,16 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
         account_obj = account
         window = account.sync_window
         mailbox_list = list(mailboxes)
+        # Gmail: sync only the system folders + All Mail and read each message's
+        # labels (X-GM-LABELS) instead of visiting every label folder. This avoids
+        # the INBOX/All-Mail/label duplication and lets labels map to chips lazily,
+        # tied to the sync window (SPEC: Gmail sync).
+        gmail = provider.gmail_labels_supported
+        if gmail:
+            mailbox_list = [m for m in mailbox_list if m.special_use in _GMAIL_SYNC_FOLDERS]
+        # Spam is opt-in: skip the Junk folder unless the account enabled it (§6.3).
+        if not account.sync_spam:
+            mailbox_list = [m for m in mailbox_list if m.special_use != "junk"]
 
     try:
         if window == "new":
@@ -270,9 +294,13 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
             date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None
         )
         # Progress denominator: count what we're about to import up front so the
-        # bar has a stable total (SPEC §12 M2 backfill-progress UX).
+        # bar has a stable total (SPEC §12 M2 backfill-progress UX). For Gmail only
+        # All Mail + Trash + Spam are counted — INBOX/Sent/Drafts are subsets of All
+        # Mail, so counting them too would inflate the total past the deduped `done`.
         total = 0
         for mailbox in mailbox_list:
+            if gmail and mailbox.special_use not in _GMAIL_DENOM_FOLDERS:
+                continue
             total += await provider.message_count(mailbox.path, since=since)
         await _set_backfill_counts(account_id, total=total, done=0)
         progress = _BackfillProgress(account_obj, total)
@@ -280,16 +308,22 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
         recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS)
         count = 0
         for mailbox in mailbox_list:
+            # Dedupe the Gmail All Mail pass against messages already stored under a
+            # system folder (INBOX/Sent/…), which the priority order backfills first.
+            dedupe = gmail and mailbox.special_use == "archive"
             if since is not None:  # 3months: single windowed pass
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, since=since
+                    provider, account_obj, mailbox, progress, since=since,
+                    dedupe_account_scope=dedupe,
                 )
             else:  # all: recent window first (usable fast), then the remainder
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, since=recent_cutoff
+                    provider, account_obj, mailbox, progress, since=recent_cutoff,
+                    dedupe_account_scope=dedupe,
                 )
                 count += await backfill_mailbox(
-                    provider, account_obj, mailbox, progress, before=recent_cutoff
+                    provider, account_obj, mailbox, progress, before=recent_cutoff,
+                    dedupe_account_scope=dedupe,
                 )
         await _set_state(account_id, "live")
     except Exception as exc:  # noqa: BLE001 - surface provider errors as account state
@@ -490,7 +524,8 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No
     ``include_inbox`` also polls INBOX (used by the manual Refresh); the periodic
     secondary loop leaves INBOX to its dedicated IDLE loop. Custom (non special-use)
     folders are not polled. Drafts are polled so drafts made in other clients appear
-    (persisted un-threaded).
+    (persisted un-threaded). ``archive`` (Gmail All Mail) is never live-polled — it is
+    huge and cold, and backfill already covers it.
     """
     from app.db import session_scope
 
@@ -498,10 +533,14 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No
         account = await _load_account(session, account_id)
         account_obj = account
         stmt = select(Mailbox.id).where(
-            Mailbox.account_id == account_id, Mailbox.special_use.isnot(None)
+            Mailbox.account_id == account_id,
+            Mailbox.special_use.isnot(None),
+            Mailbox.special_use != "archive",
         )
         if not include_inbox:
             stmt = stmt.where(Mailbox.special_use != "inbox")
+        if not account.sync_spam:  # Spam is opt-in (§6.3)
+            stmt = stmt.where(Mailbox.special_use != "junk")
         mailbox_ids = list(await session.scalars(stmt))
     if not mailbox_ids:
         return

backend/migrations/versions/0015_account_sync_spam.py

@@ -0,0 +1,28 @@
+"""add accounts.sync_spam
+
+Per-account toggle for whether the Spam/Junk folder is synced. Off by default —
+most users don't want spam mirrored into the app; they can opt in at setup.
+
+Revision ID: 0015
+Revises: 0014
+Create Date: 2026-08-05
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0015"
+down_revision: str | None = "0014"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE accounts ADD COLUMN sync_spam boolean NOT NULL DEFAULT false")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE accounts DROP COLUMN IF EXISTS sync_spam")

backend/tests/accounts/test_account_api.py

@@ -19,7 +19,7 @@ from app.config import get_settings
 from app.identity.passwords import hash_password
 from app.identity.sessions import create_session
 from app.main import create_app
-from app.models import Account, User
+from app.models import Account, Thread, User
 
 CSRF = "test-csrf-token"
 GOOD_CFG = {
@@ -281,3 +281,46 @@ async def test_delete_account(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no
 
     gone = await ctx.session.scalar(select(Account).where(Account.id == uuid.UUID(account_id)))
     assert gone is None
+
+
+async def test_delete_account_prunes_orphaned_threads(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    """Deleting an account removes threads left with no messages (else the unified
+    list / label views show empty rows that can't be opened)."""
+    import datetime as dt
+
+    async def fake_test(**_kw: object) -> None:
+        return None
+
+    class FakePool:
+        async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
+            return None
+
+    async def fake_pool() -> FakePool:
+        return FakePool()
+
+    monkeypatch.setattr(accounts_api, "test_account", fake_test)
+    monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
+
+    created = await ctx.client.post(
+        "/api/accounts",
+        json={**GOOD_CFG, "email": "orphan@example.com", "password": "secret"},
+        headers=_hdr(),
+    )
+    account_id = uuid.UUID(created.json()["id"])
+    # An empty thread tagged to this account (message-less, as after a cascade delete).
+    thread_id = uuid.uuid4()
+    ctx.session.add(
+        Thread(
+            id=thread_id,
+            user_id=ctx.user_id,
+            subject_norm="orphan",
+            last_message_at=dt.datetime.now(dt.UTC),
+            account_ids=[account_id],
+        )
+    )
+    await ctx.session.commit()
+    ctx.session.expunge_all()  # force the assertion query to hit the DB
+
+    resp = await ctx.client.request("DELETE", f"/api/accounts/{account_id}", headers=_hdr())
+    assert resp.status_code == 200
+    assert await ctx.session.scalar(select(Thread).where(Thread.id == thread_id)) is None

backend/tests/indexer/test_mime.py

@@ -49,6 +49,17 @@ def test_parse_basic_headers_and_body() -> None:
     assert parsed.snippet == "plain body"
 
 
+def test_undecodable_header_bytes_scrubbed_to_valid_utf8() -> None:
+    # A raw 8-bit byte the parser can't decode surfaces as a lone surrogate; it must
+    # be scrubbed so the value is valid UTF-8 for Postgres text/JSONB storage.
+    raw = _build(subject="Fakt\udce9ra", from_addr="B\udce9ta <b@example.com>")
+    parsed = parse_message(raw)
+    parsed.subject.encode("utf-8")  # would raise on a lone surrogate
+    parsed.from_addr[0]["name"].encode("utf-8")
+    assert "\udce9" not in parsed.subject
+    assert "�" in parsed.subject
+
+
 def test_references_include_in_reply_to_in_order() -> None:
     parsed = parse_message(_build(references="<a@x.com> <b@x.com>", in_reply_to="<b@x.com>"))
     # References first, then In-Reply-To, deduped

backend/tests/providers/test_parsing.py

@@ -37,6 +37,17 @@ def test_parse_list_plain_folder_has_no_special_use() -> None:
     assert parsed.special_use is None
 
 
+def test_parse_list_skips_noselect_containers() -> None:
+    # \Noselect / \NonExistent hierarchy nodes can't be SELECTed (Gmail's [Gmail]
+    # root, a parent label like "Serial numbers " that only groups children).
+    assert parse_list_line(rb'(\HasChildren \Noselect) "/" "[Gmail]"') is None
+    assert parse_list_line(rb'(\HasChildren \Noselect) "/" "Serial numbers "') is None
+    assert parse_list_line(rb'(\NonExistent \HasChildren) "/" "Parent"') is None
+    # A selectable child of that container is still parsed.
+    child = parse_list_line(rb'(\HasNoChildren) "/" "Serial numbers / licencje"')
+    assert child is not None and child.path == "Serial numbers / licencje"
+
+
 def test_parse_list_utf7_name() -> None:
     # Modified UTF-7: "&AOk-" decodes to "é" -> folder "Réponses"
     parsed = parse_list_line(rb'(\HasNoChildren) "/" "R&AOk-ponses"')
@@ -44,6 +55,35 @@ def test_parse_list_utf7_name() -> None:
     assert parsed.path == "Réponses"
 
 
+def test_parse_gm_labels() -> None:
+    from app.providers.imap_smtp.parsing import parse_gm_labels
+
+    # System labels (backslash-prefixed) are dropped; custom labels kept + decoded.
+    header = rb'1 FETCH (UID 5 X-GM-LABELS (\Inbox \Important "Konkurs" "Projekty/2026") FLAGS ())'
+    assert parse_gm_labels(header) == ("Konkurs", "Projekty/2026")
+    # Unquoted single-word custom label.
+    assert parse_gm_labels(rb"1 FETCH (UID 5 X-GM-LABELS (\Sent PUP))") == ("PUP",)
+    # Modified UTF-7 decoding of a non-ASCII label ("Zg&AUI-oszenia" -> "Zgłoszenia").
+    assert parse_gm_labels(rb'1 FETCH (X-GM-LABELS ("Zg&AUI-oszenia"))') == ("Zgłoszenia",)
+    # No X-GM-LABELS at all.
+    assert parse_gm_labels(rb"1 FETCH (UID 5 FLAGS (\Seen))") == ()
+    # Only system labels -> empty.
+    assert parse_gm_labels(rb"1 FETCH (X-GM-LABELS (\Inbox \Starred))") == ()
+
+
+def test_parse_fetch_lines_carries_gm_labels() -> None:
+    lines = [
+        rb'1 FETCH (UID 9 X-GM-LABELS (\Inbox "Work") FLAGS (\Seen) '
+        rb'INTERNALDATE "27-May-2026 19:08:09 +0200" RFC822.SIZE 3 BODY[] {3}',
+        b"abc",
+        b")",
+    ]
+    out = parse_fetch_lines(lines)
+    assert len(out) == 1
+    assert out[0].gmail_labels == ("Work",)
+    assert out[0].raw == b"abc"
+
+
 def test_parse_internaldate() -> None:
     dt = parse_internaldate("27-May-2026 19:08:09 +0200")
     assert dt == datetime.fromisoformat("2026-05-27T19:08:09+02:00")

backend/tests/sync/test_backfill_window.py

@@ -31,6 +31,8 @@ def _raw(mid: str) -> bytes:
 
 
 class _FakeProvider:
+    gmail_labels_supported = False
+
     def __init__(self) -> None:
         self.backfill_calls: list[tuple[str, date | None, date | None]] = []
         self.count_calls: list[tuple[str, date | None]] = []