Gmail label sync + no-dup backfill, live label/UI updates, account-delete cleanup, Sync-SPAM toggle, sidebar logo
Gmail sync (SPEC: Gmail sync): - Provider fetches X-GM-LABELS when X-GM-EXT-1 is advertised; parses custom labels (system labels dropped, mUTF-7 decoded) onto RemoteMessage. - Engine syncs only system folders + All Mail for Gmail, skips every label folder, and dedupes the All Mail pass by Message-ID — kills the INBOX/All-Mail duplication. - Custom labels ride the message.new event; the labels plugin subscribes and maps them to chips as "<name> [Gmail]" (per-message + thread aggregate + known set), so labels fill in gradually with the sync window. Non-ASCII labels decode right. Labels self-healing (message-driven reconcile): - Per-message keys are the source of truth; t: aggregates and _labels are derived. Reconcile drops labels of deleted messages, regroups survivors by their current thread (fixing JWZ thread merges), rewrites changed t: keys, and rebuilds _labels from live messages — keeping user labels, dropping orphaned Gmail ones. - New SDK reads: sdk.threads.existing / sdk.messages.threads_of. Sidebar refreshes live via a plugin.labels.changed WS event on new labels. Account deletion: prune threads left empty after the cascade (they were user-scoped, so only messages cascaded) + de-list the account from surviving cross-account threads. Sync SPAM: opt-in per-account sync_spam (migration 0015, default off) — the engine skips the Junk folder in backfill and poll unless enabled; wizard checkbox added. Robustness / UX: - WebSocket auto-reconnects with backoff and resyncs on (re)connect, and account add/delete invalidates threads/labels — no more hard refresh for live updates. - Backfill banner + sidebar ring also require done < total, so a completed account can't leave a stuck "X of X". - Scrub lone UTF-16 surrogates from parsed headers/bodies so bad messages no longer crash the backfill on Postgres text/JSONB. - Ultimo signet logo in the Mail/Calendar/Contacts sidebar header (and at the top of the collapsed rail), via the shared plugin-ui.css + MailLayout. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
Claude Opus 4.8
+54
-15
@@ -27,7 +27,15 @@ from app.sync.provider_factory import build_provider
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Backfill priority: inbox, then sent/drafts, then everything else (SPEC §6.3).
|
||||
# archive (Gmail All Mail) sorts last so its dedupe pass runs after the system folders.
|
||||
_PRIORITY = {"inbox": 0, "sent": 1, "drafts": 2}
|
||||
# Gmail sync (SPEC: Gmail sync): the only folders backfilled — All Mail (archive) is
|
||||
# the complete store, the rest are the system folders needed for their views. Every
|
||||
# custom-label folder is skipped; labels come from X-GM-LABELS instead.
|
||||
_GMAIL_SYNC_FOLDERS = {"inbox", "sent", "drafts", "trash", "junk", "archive"}
|
||||
# Progress denominator for Gmail: All Mail already contains INBOX/Sent/Drafts/labels,
|
||||
# so only these three are counted (Trash/Spam sit outside All Mail).
|
||||
_GMAIL_DENOM_FOLDERS = {"archive", "trash", "junk"}
|
||||
|
||||
# Sync-window import strategy (SPEC §12 M2). "all" imports the most recent
|
||||
# _RECENT_DAYS first (so the account is usable fast), then the older remainder.
|
||||
@@ -110,15 +118,17 @@ async def _index_and_notify(results: list[PersistResult], account: Account) -> N
|
||||
bus = get_event_bus()
|
||||
for r in results:
|
||||
await backend.index(r.search_doc)
|
||||
await bus.publish(
|
||||
"message.new",
|
||||
{
|
||||
"user_id": str(account.user_id),
|
||||
"account_id": str(account.id),
|
||||
"message_id": str(r.message.id),
|
||||
"thread_id": str(r.message.thread_id),
|
||||
},
|
||||
)
|
||||
payload: dict[str, object] = {
|
||||
"user_id": str(account.user_id),
|
||||
"account_id": str(account.id),
|
||||
"message_id": str(r.message.id),
|
||||
"thread_id": str(r.message.thread_id),
|
||||
}
|
||||
# Gmail custom labels ride along so the labels plugin can map them to chips
|
||||
# (SPEC: Gmail sync). Only present on Gmail accounts with labelled messages.
|
||||
if r.gmail_labels:
|
||||
payload["gmail_labels"] = list(r.gmail_labels)
|
||||
await bus.publish("message.new", payload)
|
||||
|
||||
|
||||
async def _emit_batch(
|
||||
@@ -178,12 +188,15 @@ async def backfill_mailbox(
|
||||
*,
|
||||
since: date | None = None,
|
||||
before: date | None = None,
|
||||
dedupe_account_scope: bool = False,
|
||||
) -> int:
|
||||
"""Backfill one mailbox newest-first, indexing inline per batch (FR-30).
|
||||
|
||||
``since``/``before`` bound the import by date (sync window, SPEC §12 M2).
|
||||
When a ``progress`` tracker is given, batches count toward the account-level
|
||||
backfill total; otherwise each batch emits a standalone ``sync.progress``.
|
||||
``dedupe_account_scope`` drops messages already synced under another mailbox of
|
||||
the account (Gmail All Mail pass — SPEC: Gmail sync / no-duplication).
|
||||
"""
|
||||
from app.db import session_scope
|
||||
|
||||
@@ -202,6 +215,7 @@ async def backfill_mailbox(
|
||||
mailbox_id=mailbox_id,
|
||||
mailbox_special_use=special_use,
|
||||
remote=remote,
|
||||
dedupe_account_scope=dedupe_account_scope,
|
||||
)
|
||||
if r is not None:
|
||||
results.append(r)
|
||||
@@ -257,6 +271,16 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
|
||||
account_obj = account
|
||||
window = account.sync_window
|
||||
mailbox_list = list(mailboxes)
|
||||
# Gmail: sync only the system folders + All Mail and read each message's
|
||||
# labels (X-GM-LABELS) instead of visiting every label folder. This avoids
|
||||
# the INBOX/All-Mail/label duplication and lets labels map to chips lazily,
|
||||
# tied to the sync window (SPEC: Gmail sync).
|
||||
gmail = provider.gmail_labels_supported
|
||||
if gmail:
|
||||
mailbox_list = [m for m in mailbox_list if m.special_use in _GMAIL_SYNC_FOLDERS]
|
||||
# Spam is opt-in: skip the Junk folder unless the account enabled it (§6.3).
|
||||
if not account.sync_spam:
|
||||
mailbox_list = [m for m in mailbox_list if m.special_use != "junk"]
|
||||
|
||||
try:
|
||||
if window == "new":
|
||||
@@ -270,9 +294,13 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
|
||||
date.today() - timedelta(days=_WINDOW_DAYS[window]) if window in _WINDOW_DAYS else None
|
||||
)
|
||||
# Progress denominator: count what we're about to import up front so the
|
||||
# bar has a stable total (SPEC §12 M2 backfill-progress UX).
|
||||
# bar has a stable total (SPEC §12 M2 backfill-progress UX). For Gmail only
|
||||
# All Mail + Trash + Spam are counted — INBOX/Sent/Drafts are subsets of All
|
||||
# Mail, so counting them too would inflate the total past the deduped `done`.
|
||||
total = 0
|
||||
for mailbox in mailbox_list:
|
||||
if gmail and mailbox.special_use not in _GMAIL_DENOM_FOLDERS:
|
||||
continue
|
||||
total += await provider.message_count(mailbox.path, since=since)
|
||||
await _set_backfill_counts(account_id, total=total, done=0)
|
||||
progress = _BackfillProgress(account_obj, total)
|
||||
@@ -280,16 +308,22 @@ async def bootstrap_and_backfill(account_id: uuid.UUID) -> int:
|
||||
recent_cutoff = date.today() - timedelta(days=_RECENT_DAYS)
|
||||
count = 0
|
||||
for mailbox in mailbox_list:
|
||||
# Dedupe the Gmail All Mail pass against messages already stored under a
|
||||
# system folder (INBOX/Sent/…), which the priority order backfills first.
|
||||
dedupe = gmail and mailbox.special_use == "archive"
|
||||
if since is not None: # 3months: single windowed pass
|
||||
count += await backfill_mailbox(
|
||||
provider, account_obj, mailbox, progress, since=since
|
||||
provider, account_obj, mailbox, progress, since=since,
|
||||
dedupe_account_scope=dedupe,
|
||||
)
|
||||
else: # all: recent window first (usable fast), then the remainder
|
||||
count += await backfill_mailbox(
|
||||
provider, account_obj, mailbox, progress, since=recent_cutoff
|
||||
provider, account_obj, mailbox, progress, since=recent_cutoff,
|
||||
dedupe_account_scope=dedupe,
|
||||
)
|
||||
count += await backfill_mailbox(
|
||||
provider, account_obj, mailbox, progress, before=recent_cutoff
|
||||
provider, account_obj, mailbox, progress, before=recent_cutoff,
|
||||
dedupe_account_scope=dedupe,
|
||||
)
|
||||
await _set_state(account_id, "live")
|
||||
except Exception as exc: # noqa: BLE001 - surface provider errors as account state
|
||||
@@ -490,7 +524,8 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No
|
||||
``include_inbox`` also polls INBOX (used by the manual Refresh); the periodic
|
||||
secondary loop leaves INBOX to its dedicated IDLE loop. Custom (non special-use)
|
||||
folders are not polled. Drafts are polled so drafts made in other clients appear
|
||||
(persisted un-threaded).
|
||||
(persisted un-threaded). ``archive`` (Gmail All Mail) is never live-polled — it is
|
||||
huge and cold, and backfill already covers it.
|
||||
"""
|
||||
from app.db import session_scope
|
||||
|
||||
@@ -498,10 +533,14 @@ async def poll_folders_once(account_id: uuid.UUID, *, include_inbox: bool) -> No
|
||||
account = await _load_account(session, account_id)
|
||||
account_obj = account
|
||||
stmt = select(Mailbox.id).where(
|
||||
Mailbox.account_id == account_id, Mailbox.special_use.isnot(None)
|
||||
Mailbox.account_id == account_id,
|
||||
Mailbox.special_use.isnot(None),
|
||||
Mailbox.special_use != "archive",
|
||||
)
|
||||
if not include_inbox:
|
||||
stmt = stmt.where(Mailbox.special_use != "inbox")
|
||||
if not account.sync_spam: # Spam is opt-in (§6.3)
|
||||
stmt = stmt.where(Mailbox.special_use != "junk")
|
||||
mailbox_ids = list(await session.scalars(stmt))
|
||||
if not mailbox_ids:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user