Gmail label sync + no-dup backfill, live label/UI updates, account-delete cleanup, Sync-SPAM toggle, sidebar logo

Gmail sync (SPEC: Gmail sync): - Provider fetches X-GM-LABELS when X-GM-EXT-1 is advertised; parses custom labels (system labels dropped, mUTF-7 decoded) onto RemoteMessage. - Engine syncs only system folders + All Mail for Gmail, skips every label folder, and dedupes the All Mail pass by Message-ID — kills the INBOX/All-Mail duplication. - Custom labels ride the message.new event; the labels plugin subscribes and maps them to chips as "<name> [Gmail]" (per-message + thread aggregate + known set), so labels fill in gradually with the sync window. Non-ASCII labels decode right. Labels self-healing (message-driven reconcile): - Per-message keys are the source of truth; t: aggregates and _labels are derived. Reconcile drops labels of deleted messages, regroups survivors by their current thread (fixing JWZ thread merges), rewrites changed t: keys, and rebuilds _labels from live messages — keeping user labels, dropping orphaned Gmail ones. - New SDK reads: sdk.threads.existing / sdk.messages.threads_of. Sidebar refreshes live via a plugin.labels.changed WS event on new labels. Account deletion: prune threads left empty after the cascade (they were user-scoped, so only messages cascaded) + de-list the account from surviving cross-account threads. Sync SPAM: opt-in per-account sync_spam (migration 0015, default off) — the engine skips the Junk folder in backfill and poll unless enabled; wizard checkbox added. Robustness / UX: - WebSocket auto-reconnects with backoff and resyncs on (re)connect, and account add/delete invalidates threads/labels — no more hard refresh for live updates. - Backfill banner + sidebar ring also require done < total, so a completed account can't leave a stuck "X of X". - Scrub lone UTF-16 surrogates from parsed headers/bodies so bad messages no longer crash the backfill on Postgres text/JSONB. - Ultimo signet logo in the Mail/Calendar/Contacts sidebar header (and at the top of the collapsed rail), via the shared plugin-ui.css + MailLayout. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
2026-08-05 18:04:49 +02:00
parent e5a87b37b1
commit 4c200ab94a
38 changed files with 863 additions and 106 deletions
@@ -12,6 +12,7 @@ from dataclasses import dataclass
 from datetime import datetime

 from imapclient.imap_utf7 import decode as imap_utf7_decode
+from imapclient.imap_utf7 import encode as imap_utf7_encode

 # --- SPECIAL-USE mapping (SPEC §4 special_use values) -----------------------

@@ -27,6 +28,7 @@ _SPECIAL_USE_FLAGS = {
 _LIST_RE = re.compile(rb'^\(([^)]*)\)\s+(?:"([^"]*)"|NIL)\s+(.+)$')
 _UID_RE = re.compile(rb"UID\s+(\d+)")
 _FLAGS_RE = re.compile(rb"FLAGS\s+\(([^)]*)\)")
+_GM_LABELS_RE = re.compile(rb"X-GM-LABELS\s+\(([^)]*)\)")
 _INTERNALDATE_RE = re.compile(rb'INTERNALDATE\s+"([^"]+)"')
 _LITERAL_RE = re.compile(rb"\{(\d+)\}\s*$")
 _FETCH_HEADER_RE = re.compile(rb"^\d+\s+FETCH\s+\(", re.IGNORECASE)
@@ -44,6 +46,7 @@ class ParsedFetch:
    flags: frozenset[str]
    internal_date: datetime | None
    raw: bytes | None  # None when the fetch did not request the body
+    gmail_labels: tuple[str, ...] = ()  # custom Gmail labels (X-GM-LABELS), system ones dropped


 def _decode_mailbox_name(raw: bytes) -> str:
@@ -54,12 +57,29 @@ def _decode_mailbox_name(raw: bytes) -> str:
    return str(imap_utf7_decode(bytes(name)))


+def encode_mailbox_name(name: str) -> str:
+    """Modified UTF-7 wire form of a mailbox name (inverse of ``_decode_mailbox_name``).
+
+    Mailbox names are stored/displayed as Unicode but IMAP commands
+    (SELECT/STATUS/APPEND) require them mUTF-7 encoded. ASCII names are unchanged;
+    non-ASCII names — e.g. Gmail's localized ``[Gmail]/Wysłane`` — get encoded so the
+    server can find them."""
+    return str(imap_utf7_encode(name).decode("ascii"))
+
+
 def parse_list_line(line: bytes) -> ParsedMailbox | None:
-    """Parse one LIST response line into a :class:`ParsedMailbox` (or None)."""
+    """Parse one LIST response line into a :class:`ParsedMailbox` (or None).
+
+    Non-selectable mailboxes are skipped (return None): ``\\Noselect`` /
+    ``\\NonExistent`` names are pure hierarchy containers that hold no messages and
+    cannot be SELECTed — e.g. Gmail's ``[Gmail]`` root or a parent label like
+    ``Serial numbers`` that only groups child labels."""
    m = _LIST_RE.match(line.strip())
    if not m:
        return None
    flags = m.group(1).lower().split()
+    if b"\\noselect" in flags or b"\\nonexistent" in flags:
+        return None
    name = _decode_mailbox_name(m.group(3))
    special: str | None = None
    for flag in flags:
@@ -100,6 +120,57 @@ def parse_fetch_header(line: bytes) -> tuple[int | None, frozenset[str], datetim
    return uid, flags, internal


+def _tokenize_gm_labels(blob: bytes) -> list[bytes]:
+    """Split an ``X-GM-LABELS (...)`` body into raw tokens, keeping quoted strings
+    (which may contain spaces) whole. Escaped chars inside quotes are unescaped."""
+    tokens: list[bytes] = []
+    i, n = 0, len(blob)
+    while i < n:
+        ch = blob[i : i + 1]
+        if ch in (b" ", b"\t"):
+            i += 1
+            continue
+        if ch == b'"':
+            j = i + 1
+            buf = bytearray()
+            while j < n and blob[j : j + 1] != b'"':
+                if blob[j : j + 1] == b"\\" and j + 1 < n:
+                    buf += blob[j + 1 : j + 2]
+                    j += 2
+                    continue
+                buf += blob[j : j + 1]
+                j += 1
+            tokens.append(bytes(buf))
+            i = j + 1
+        else:
+            j = i
+            while j < n and blob[j : j + 1] not in (b" ", b"\t"):
+                j += 1
+            tokens.append(blob[i:j])
+            i = j
+    return tokens
+
+
+def parse_gm_labels(header: bytes) -> tuple[str, ...]:
+    """Custom Gmail labels from a FETCH header's ``X-GM-LABELS`` (SPEC: Gmail sync).
+
+    System labels (``\\Inbox``, ``\\Sent``, ``\\Important``, ``\\Starred``, …) are
+    dropped — only user labels become chips. Names are modified-UTF-7 decoded, like
+    mailbox names, so e.g. ``Zg&AAI...`` → the real Unicode label."""
+    m = _GM_LABELS_RE.search(header)
+    if not m:
+        return ()
+    out: list[str] = []
+    for tok in _tokenize_gm_labels(m.group(1)):
+        if not tok or tok.startswith(b"\\"):  # system label
+            continue
+        try:
+            out.append(str(imap_utf7_decode(tok)))
+        except Exception:  # noqa: BLE001 - fall back to a best-effort ASCII decode
+            out.append(tok.decode("utf-8", "ignore"))
+    return tuple(out)
+
+
 def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
    """Walk a full FETCH response, returning one :class:`ParsedFetch` per message.

@@ -131,7 +202,15 @@ def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
        else:
            i += 1
        if uid is not None:
-            results.append(ParsedFetch(uid=uid, flags=flags, internal_date=internal, raw=raw))
+            results.append(
+                ParsedFetch(
+                    uid=uid,
+                    flags=flags,
+                    internal_date=internal,
+                    raw=raw,
+                    gmail_labels=parse_gm_labels(header),
+                )
+            )
    return results