Gmail label sync + no-dup backfill, live label/UI updates, account-delete cleanup, Sync-SPAM toggle, sidebar logo

Gmail sync (SPEC: Gmail sync):
- Provider fetches X-GM-LABELS when X-GM-EXT-1 is advertised; parses custom labels
  (system labels dropped, mUTF-7 decoded) onto RemoteMessage.
- Engine syncs only system folders + All Mail for Gmail, skips every label folder,
  and dedupes the All Mail pass by Message-ID — kills the INBOX/All-Mail duplication.
- Custom labels ride the message.new event; the labels plugin subscribes and maps
  them to chips as "<name> [Gmail]" (per-message + thread aggregate + known set),
  so labels fill in gradually with the sync window. Non-ASCII labels decode right.

Labels self-healing (message-driven reconcile):
- Per-message keys are the source of truth; t: aggregates and _labels are derived.
  Reconcile drops labels of deleted messages, regroups survivors by their current
  thread (fixing JWZ thread merges), rewrites changed t: keys, and rebuilds _labels
  from live messages — keeping user labels, dropping orphaned Gmail ones.
- New SDK reads: sdk.threads.existing / sdk.messages.threads_of. Sidebar refreshes
  live via a plugin.labels.changed WS event on new labels.

Account deletion: prune threads left empty after the cascade (they were user-scoped,
so only messages cascaded) + de-list the account from surviving cross-account threads.

Sync SPAM: opt-in per-account sync_spam (migration 0015, default off) — the engine
skips the Junk folder in backfill and poll unless enabled; wizard checkbox added.

Robustness / UX:
- WebSocket auto-reconnects with backoff and resyncs on (re)connect, and account
  add/delete invalidates threads/labels — no more hard refresh for live updates.
- Backfill banner + sidebar ring also require done < total, so a completed account
  can't leave a stuck "X of X".
- Scrub lone UTF-16 surrogates from parsed headers/bodies so bad messages no longer
  crash the backfill on Postgres text/JSONB.
- Ultimo signet logo in the Mail/Calendar/Contacts sidebar header (and at the top of
  the collapsed rail), via the shared plugin-ui.css + MailLayout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA

plugins/webmail_plugin_labels/webmail_plugin_labels/__init__.py

@@ -9,11 +9,13 @@ own Actions menu + label chips (which read the plugin's API directly); only the
 from __future__ import annotations
 
 from app.plugins.sdk import Plugin, Slot, UiManifest
+from webmail_plugin_labels.hook import on_message_new
 from webmail_plugin_labels.router import router
 
 plugin = Plugin(
     name="labels",
     version="0.1.0",
+    hooks={"message.new": on_message_new},
     router=router,
     ui_manifest=UiManifest(
         slots=[

plugins/webmail_plugin_labels/webmail_plugin_labels/hook.py

@@ -0,0 +1,55 @@
+"""message.new hook: map a message's Gmail labels to label chips (SPEC: Gmail sync).
+
+Runs in the sync worker wherever ``message.new`` is published. When the core carries
+``gmail_labels`` on the event (Gmail accounts only), each custom Gmail label is stored
+as a ``<name> [Gmail]`` chip using the same KV layout as the rest of the feature: a
+per-message key, a ``t:<thread_id>`` thread aggregate, and the ``_labels`` known set.
+Names are merged (union), never overwritten, so app-native labels are preserved.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from app.plugins.sdk import sdk
+
+logger = logging.getLogger("plg.labels")
+
+_ALL_KEY = "_labels"
+_GMAIL_SUFFIX = " [Gmail]"
+
+
+async def _merge(kv: Any, key: str, names: set[str]) -> None:
+    """Union ``names`` into the list stored at ``key`` (no-op if already present)."""
+    current = set(await kv.get(key, []))
+    if names <= current:
+        return
+    await kv.set(key, sorted(current | names))
+
+
+async def on_message_new(payload: dict[str, Any]) -> None:
+    labels = payload.get("gmail_labels")
+    user_raw, message_id = payload.get("user_id"), payload.get("message_id")
+    thread_id = payload.get("thread_id")
+    if not labels or not isinstance(user_raw, str) or not isinstance(message_id, str):
+        return
+    names = {f"{str(name).strip()}{_GMAIL_SUFFIX}" for name in labels if str(name).strip()}
+    if not names:
+        return
+
+    s = sdk("labels")
+    kv = s.kv(uuid.UUID(user_raw))
+    await _merge(kv, message_id, names)  # per-message labels
+    if isinstance(thread_id, str):
+        await _merge(kv, f"t:{thread_id}", names)  # thread aggregate (list view chips)
+
+    # Known-label set (sidebar). When it actually grows, push plugin.labels.changed
+    # over the WS so the sidebar list refreshes live during a sync (SPEC §9.2/§11.3).
+    known = set(await kv.get(_ALL_KEY, []))
+    fresh = names - known
+    if fresh:
+        await kv.set(_ALL_KEY, sorted(known | names))
+        await s.publish("changed", {"user_id": user_raw})
+        logger.info("labels: %d new Gmail label(s) from message %s", len(fresh), message_id)

plugins/webmail_plugin_labels/webmail_plugin_labels/router.py

@@ -80,26 +80,76 @@ async def set_thread_labels(
 
 @router.post("/threads/labels")
 async def batch_thread_labels(body: IdsIn, user_id: CurrentUserId) -> dict[str, list[str]]:
+    """Aggregated labels for a set of threads (list-view chips). Reconciles first so
+    the chips reflect live messages — stale ones from deleted mail are dropped."""
+    live = await _reconcile(user_id)
+    return {tid: live[tid] for tid in body.ids if live.get(tid)}
+
+
+_GMAIL_SUFFIX = " [Gmail]"
+
+
+async def _reconcile(user_id: object) -> dict[str, list[str]]:
+    """Self-heal label state from the per-message keys (the source of truth) and
+    return the live ``thread_id -> labels`` map.
+
+    Per-message keys (``<message_id>``) are authoritative; the ``t:<thread_id>``
+    aggregates and the ``_labels`` set are derived. Messages get deleted (account
+    removal) and re-threaded (JWZ merges) out from under us, which orphans thread
+    aggregates — so rebuild them here: drop labels of deleted messages, regroup the
+    survivors by their *current* thread, rewrite changed/stale ``t:`` keys, and
+    rebuild ``_labels`` from labels on live messages. User-created labels (no
+    ``… [Gmail]`` suffix) are preserved even when unused; orphaned Gmail labels drop."""
     kv = _SDK.kv(user_id)
-    out = {tid: await kv.get(f"t:{tid}", []) for tid in body.ids}
-    return {tid: labels for tid, labels in out.items() if labels}
+    items = await kv.items()
+    msg_keys = {
+        k: v
+        for k, v in items.items()
+        if k != _ALL_KEY and not k.startswith("t:") and isinstance(v, list)
+    }
+    homes = await _SDK.messages.threads_of(user_id, list(msg_keys))
+
+    thread_labels: dict[str, set[str]] = {}
+    live_labels: set[str] = set()
+    for message_id, labels in msg_keys.items():
+        if message_id not in homes:  # message deleted -> drop its orphaned key
+            await kv.delete(message_id)
+            continue
+        live_labels.update(labels)
+        thread_id = homes[message_id]
+        if thread_id is not None:
+            thread_labels.setdefault(thread_id, set()).update(labels)
+
+    # Rewrite only the thread aggregates that changed; delete ones with no messages.
+    old_thread_keys = {k[2:] for k in items if k.startswith("t:")}
+    result: dict[str, list[str]] = {}
+    for thread_id, names in thread_labels.items():
+        value = sorted(names)
+        result[thread_id] = value
+        if items.get(f"t:{thread_id}") != value:
+            await kv.set(f"t:{thread_id}", value)
+    for thread_id in old_thread_keys - set(thread_labels):
+        await kv.delete(f"t:{thread_id}")
+
+    known = set(items.get(_ALL_KEY, []) or [])
+    new_known = {n for n in known if not n.endswith(_GMAIL_SUFFIX)} | live_labels
+    if new_known != known:
+        await kv.set(_ALL_KEY, sorted(new_known))
+    return result
 
 
 @router.get("/labels", response_model=list[str])
 async def all_labels(user_id: CurrentUserId) -> list[str]:
+    await _reconcile(user_id)  # self-heal from live messages, then return the known set
     return await _SDK.kv(user_id).get(_ALL_KEY, [])
 
 
 @router.get("/labels/{name}/threads", response_model=list[str])
 async def label_threads(name: str, user_id: CurrentUserId) -> list[str]:
-    """Thread ids whose aggregate labels include ``name`` — powers the label view
-    in the mail list. Scans the per-thread label keys (``t:<thread_id>``)."""
-    items = await _SDK.kv(user_id).items()
-    return [
-        key[2:]
-        for key, labels in items.items()
-        if key.startswith("t:") and isinstance(labels, list) and name in labels
-    ]
+    """Thread ids whose aggregate labels include ``name`` — powers the label view in
+    the mail list. Reconciles first, so it reflects live messages' current threads."""
+    live = await _reconcile(user_id)
+    return [tid for tid, labels in live.items() if name in labels]
 
 
 @router.post("/labels", dependencies=[Depends(require_csrf)])

plugins/webmail_plugin_labels/webmail_plugin_labels/ui/sidebar.js

@@ -128,8 +128,18 @@ export default function mount(el, ctx) {
 
   refresh();
   window.addEventListener("webmail:labels-changed", refresh);
+  // Live update while syncing: the backend pushes plugin.labels.changed over the WS
+  // whenever a new Gmail label is mapped. Re-dispatch the local event so this list
+  // (and Mail's chip queries) refresh together.
+  const unsubscribe =
+    ctx.events && ctx.events.subscribe
+      ? ctx.events.subscribe("plugin.labels.changed", () =>
+          window.dispatchEvent(new Event("webmail:labels-changed")),
+        )
+      : null;
   return () => {
     window.removeEventListener("webmail:labels-changed", refresh);
+    if (unsubscribe) unsubscribe();
     el.innerHTML = "";
   };
 }