Gmail label sync + no-dup backfill, live label/UI updates, account-delete cleanup, Sync-SPAM toggle, sidebar logo
Gmail sync (SPEC: Gmail sync): - Provider fetches X-GM-LABELS when X-GM-EXT-1 is advertised; parses custom labels (system labels dropped, mUTF-7 decoded) onto RemoteMessage. - Engine syncs only system folders + All Mail for Gmail, skips every label folder, and dedupes the All Mail pass by Message-ID — kills the INBOX/All-Mail duplication. - Custom labels ride the message.new event; the labels plugin subscribes and maps them to chips as "<name> [Gmail]" (per-message + thread aggregate + known set), so labels fill in gradually with the sync window. Non-ASCII labels decode right. Labels self-healing (message-driven reconcile): - Per-message keys are the source of truth; t: aggregates and _labels are derived. Reconcile drops labels of deleted messages, regroups survivors by their current thread (fixing JWZ thread merges), rewrites changed t: keys, and rebuilds _labels from live messages — keeping user labels, dropping orphaned Gmail ones. - New SDK reads: sdk.threads.existing / sdk.messages.threads_of. Sidebar refreshes live via a plugin.labels.changed WS event on new labels. Account deletion: prune threads left empty after the cascade (they were user-scoped, so only messages cascaded) + de-list the account from surviving cross-account threads. Sync SPAM: opt-in per-account sync_spam (migration 0015, default off) — the engine skips the Junk folder in backfill and poll unless enabled; wizard checkbox added. Robustness / UX: - WebSocket auto-reconnects with backoff and resyncs on (re)connect, and account add/delete invalidates threads/labels — no more hard refresh for live updates. - Backfill banner + sidebar ring also require done < total, so a completed account can't leave a stuck "X of X". - Scrub lone UTF-16 surrogates from parsed headers/bodies so bad messages no longer crash the backfill on Postgres text/JSONB. - Ultimo signet logo in the Mail/Calendar/Contacts sidebar header (and at the top of the collapsed rail), via the shared plugin-ui.css + MailLayout. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U72FZiF1W1pb8XxyWBTREA
This commit is contained in:
Claude Opus 4.8
@@ -46,6 +46,7 @@ class RemoteMessage:
|
||||
raw: bytes # full RFC822
|
||||
flags: frozenset[str]
|
||||
internal_date: datetime
|
||||
gmail_labels: tuple[str, ...] = () # custom Gmail labels (Gmail accounts only)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -12,6 +12,7 @@ from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from imapclient.imap_utf7 import decode as imap_utf7_decode
|
||||
from imapclient.imap_utf7 import encode as imap_utf7_encode
|
||||
|
||||
# --- SPECIAL-USE mapping (SPEC §4 special_use values) -----------------------
|
||||
|
||||
@@ -27,6 +28,7 @@ _SPECIAL_USE_FLAGS = {
|
||||
_LIST_RE = re.compile(rb'^\(([^)]*)\)\s+(?:"([^"]*)"|NIL)\s+(.+)$')
|
||||
_UID_RE = re.compile(rb"UID\s+(\d+)")
|
||||
_FLAGS_RE = re.compile(rb"FLAGS\s+\(([^)]*)\)")
|
||||
_GM_LABELS_RE = re.compile(rb"X-GM-LABELS\s+\(([^)]*)\)")
|
||||
_INTERNALDATE_RE = re.compile(rb'INTERNALDATE\s+"([^"]+)"')
|
||||
_LITERAL_RE = re.compile(rb"\{(\d+)\}\s*$")
|
||||
_FETCH_HEADER_RE = re.compile(rb"^\d+\s+FETCH\s+\(", re.IGNORECASE)
|
||||
@@ -44,6 +46,7 @@ class ParsedFetch:
|
||||
flags: frozenset[str]
|
||||
internal_date: datetime | None
|
||||
raw: bytes | None # None when the fetch did not request the body
|
||||
gmail_labels: tuple[str, ...] = () # custom Gmail labels (X-GM-LABELS), system ones dropped
|
||||
|
||||
|
||||
def _decode_mailbox_name(raw: bytes) -> str:
|
||||
@@ -54,12 +57,29 @@ def _decode_mailbox_name(raw: bytes) -> str:
|
||||
return str(imap_utf7_decode(bytes(name)))
|
||||
|
||||
|
||||
def encode_mailbox_name(name: str) -> str:
|
||||
"""Modified UTF-7 wire form of a mailbox name (inverse of ``_decode_mailbox_name``).
|
||||
|
||||
Mailbox names are stored/displayed as Unicode but IMAP commands
|
||||
(SELECT/STATUS/APPEND) require them mUTF-7 encoded. ASCII names are unchanged;
|
||||
non-ASCII names — e.g. Gmail's localized ``[Gmail]/Wysłane`` — get encoded so the
|
||||
server can find them."""
|
||||
return str(imap_utf7_encode(name).decode("ascii"))
|
||||
|
||||
|
||||
def parse_list_line(line: bytes) -> ParsedMailbox | None:
|
||||
"""Parse one LIST response line into a :class:`ParsedMailbox` (or None)."""
|
||||
"""Parse one LIST response line into a :class:`ParsedMailbox` (or None).
|
||||
|
||||
Non-selectable mailboxes are skipped (return None): ``\\Noselect`` /
|
||||
``\\NonExistent`` names are pure hierarchy containers that hold no messages and
|
||||
cannot be SELECTed — e.g. Gmail's ``[Gmail]`` root or a parent label like
|
||||
``Serial numbers`` that only groups child labels."""
|
||||
m = _LIST_RE.match(line.strip())
|
||||
if not m:
|
||||
return None
|
||||
flags = m.group(1).lower().split()
|
||||
if b"\\noselect" in flags or b"\\nonexistent" in flags:
|
||||
return None
|
||||
name = _decode_mailbox_name(m.group(3))
|
||||
special: str | None = None
|
||||
for flag in flags:
|
||||
@@ -100,6 +120,57 @@ def parse_fetch_header(line: bytes) -> tuple[int | None, frozenset[str], datetim
|
||||
return uid, flags, internal
|
||||
|
||||
|
||||
def _tokenize_gm_labels(blob: bytes) -> list[bytes]:
|
||||
"""Split an ``X-GM-LABELS (...)`` body into raw tokens, keeping quoted strings
|
||||
(which may contain spaces) whole. Escaped chars inside quotes are unescaped."""
|
||||
tokens: list[bytes] = []
|
||||
i, n = 0, len(blob)
|
||||
while i < n:
|
||||
ch = blob[i : i + 1]
|
||||
if ch in (b" ", b"\t"):
|
||||
i += 1
|
||||
continue
|
||||
if ch == b'"':
|
||||
j = i + 1
|
||||
buf = bytearray()
|
||||
while j < n and blob[j : j + 1] != b'"':
|
||||
if blob[j : j + 1] == b"\\" and j + 1 < n:
|
||||
buf += blob[j + 1 : j + 2]
|
||||
j += 2
|
||||
continue
|
||||
buf += blob[j : j + 1]
|
||||
j += 1
|
||||
tokens.append(bytes(buf))
|
||||
i = j + 1
|
||||
else:
|
||||
j = i
|
||||
while j < n and blob[j : j + 1] not in (b" ", b"\t"):
|
||||
j += 1
|
||||
tokens.append(blob[i:j])
|
||||
i = j
|
||||
return tokens
|
||||
|
||||
|
||||
def parse_gm_labels(header: bytes) -> tuple[str, ...]:
|
||||
"""Custom Gmail labels from a FETCH header's ``X-GM-LABELS`` (SPEC: Gmail sync).
|
||||
|
||||
System labels (``\\Inbox``, ``\\Sent``, ``\\Important``, ``\\Starred``, …) are
|
||||
dropped — only user labels become chips. Names are modified-UTF-7 decoded, like
|
||||
mailbox names, so e.g. ``Zg&AAI...`` → the real Unicode label."""
|
||||
m = _GM_LABELS_RE.search(header)
|
||||
if not m:
|
||||
return ()
|
||||
out: list[str] = []
|
||||
for tok in _tokenize_gm_labels(m.group(1)):
|
||||
if not tok or tok.startswith(b"\\"): # system label
|
||||
continue
|
||||
try:
|
||||
out.append(str(imap_utf7_decode(tok)))
|
||||
except Exception: # noqa: BLE001 - fall back to a best-effort ASCII decode
|
||||
out.append(tok.decode("utf-8", "ignore"))
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
|
||||
"""Walk a full FETCH response, returning one :class:`ParsedFetch` per message.
|
||||
|
||||
@@ -131,7 +202,15 @@ def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
|
||||
else:
|
||||
i += 1
|
||||
if uid is not None:
|
||||
results.append(ParsedFetch(uid=uid, flags=flags, internal_date=internal, raw=raw))
|
||||
results.append(
|
||||
ParsedFetch(
|
||||
uid=uid,
|
||||
flags=flags,
|
||||
internal_date=internal,
|
||||
raw=raw,
|
||||
gmail_labels=parse_gm_labels(header),
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ from app.providers.base import (
|
||||
RemoteMessage,
|
||||
)
|
||||
from app.providers.imap_smtp.parsing import (
|
||||
encode_mailbox_name,
|
||||
parse_fetch_lines,
|
||||
parse_list_line,
|
||||
parse_search_uids,
|
||||
@@ -39,6 +40,9 @@ from app.providers.imap_smtp.parsing import (
|
||||
)
|
||||
|
||||
_FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
|
||||
# Gmail (X-GM-EXT-1): also pull each message's labels so custom labels can map to
|
||||
# chips without visiting every label folder (SPEC: Gmail sync / no-duplication).
|
||||
_FETCH_ITEMS_GM = "(UID FLAGS INTERNALDATE RFC822.SIZE X-GM-LABELS BODY.PEEK[])"
|
||||
_IDLE_TIMEOUT = 25 * 60 # SPEC §5.1: re-issue IDLE at least every 25 min
|
||||
_CONNECT_TIMEOUT = 20
|
||||
_PER_ACCOUNT_CONN_CAP = 4 # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
|
||||
@@ -81,6 +85,13 @@ class ImapSmtpProvider(MailProvider):
|
||||
self._uidvalidity_seen: dict[str, int] = {}
|
||||
self._conn_slots: list[asyncio.Semaphore] = []
|
||||
self._qresync_enabled = False
|
||||
self._gm_ext = False
|
||||
|
||||
@property
|
||||
def gmail_labels_supported(self) -> bool:
|
||||
"""True when the server advertises ``X-GM-EXT-1`` (Gmail), so FETCH can pull
|
||||
``X-GM-LABELS`` and the engine can use the no-duplication Gmail sync path."""
|
||||
return self._gm_ext
|
||||
|
||||
@property
|
||||
def qresync_enabled(self) -> bool:
|
||||
@@ -148,6 +159,8 @@ class ImapSmtpProvider(MailProvider):
|
||||
self._release_conn_slots() # never hold a slot without an open socket
|
||||
raise
|
||||
self._imap = imap
|
||||
caps = {c.upper() for c in (imap.protocol.capabilities or set())}
|
||||
self._gm_ext = "X-GM-EXT-1" in caps
|
||||
await self._maybe_enable_qresync(imap)
|
||||
return imap
|
||||
|
||||
@@ -165,9 +178,16 @@ class ImapSmtpProvider(MailProvider):
|
||||
except Exception: # noqa: BLE001 - degrade gracefully to the reconcile path
|
||||
self._qresync_enabled = False
|
||||
|
||||
@staticmethod
|
||||
def _wire(mailbox: str) -> str:
|
||||
"""Server wire form of a mailbox name: mUTF-7 encoded and quoted. Handles
|
||||
non-ASCII names (Gmail's localized ``[Gmail]/Wysłane``) and names with
|
||||
spaces (``[Gmail]/All Mail``); ASCII names are unchanged bar the quotes."""
|
||||
return str(aioimaplib.quoted(encode_mailbox_name(mailbox)))
|
||||
|
||||
async def _select(self, mailbox: str) -> dict[str, int]:
|
||||
imap = await self._connect()
|
||||
resp = await imap.select(mailbox)
|
||||
resp = await imap.select(self._wire(mailbox))
|
||||
if resp.result != "OK":
|
||||
raise ProviderConnectError(f"cannot select {mailbox!r}")
|
||||
self._selected = mailbox
|
||||
@@ -212,7 +232,7 @@ class ImapSmtpProvider(MailProvider):
|
||||
async def _mailbox_uidvalidity(self, path: str) -> int:
|
||||
imap = await self._connect()
|
||||
try:
|
||||
resp = await imap.status(path, "(UIDVALIDITY)")
|
||||
resp = await imap.status(self._wire(path), "(UIDVALIDITY)")
|
||||
except Exception:
|
||||
return 0
|
||||
if resp.result != "OK":
|
||||
@@ -280,7 +300,8 @@ class ImapSmtpProvider(MailProvider):
|
||||
if not uids:
|
||||
return []
|
||||
uid_set = ",".join(str(u) for u in uids)
|
||||
resp = await imap.uid("fetch", uid_set, _FETCH_ITEMS)
|
||||
items = _FETCH_ITEMS_GM if self._gm_ext else _FETCH_ITEMS
|
||||
resp = await imap.uid("fetch", uid_set, items)
|
||||
if resp.result != "OK":
|
||||
raise ProviderConnectError("FETCH failed")
|
||||
out: list[RemoteMessage] = []
|
||||
@@ -289,7 +310,11 @@ class ImapSmtpProvider(MailProvider):
|
||||
continue
|
||||
out.append(
|
||||
RemoteMessage(
|
||||
uid=pf.uid, raw=pf.raw, flags=pf.flags, internal_date=pf.internal_date
|
||||
uid=pf.uid,
|
||||
raw=pf.raw,
|
||||
flags=pf.flags,
|
||||
internal_date=pf.internal_date,
|
||||
gmail_labels=pf.gmail_labels,
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -355,7 +380,7 @@ class ImapSmtpProvider(MailProvider):
|
||||
proto = imap.protocol
|
||||
param = f"(QRESYNC ({uidvalidity} {modseq}))"
|
||||
cmd = aioimaplib.Command(
|
||||
"SELECT", proto.new_tag(), aioimaplib.quoted(mailbox), param, loop=proto.loop
|
||||
"SELECT", proto.new_tag(), self._wire(mailbox), param, loop=proto.loop
|
||||
)
|
||||
return await proto.execute(cmd)
|
||||
|
||||
@@ -419,7 +444,7 @@ class ImapSmtpProvider(MailProvider):
|
||||
async def append(self, mailbox: str, raw: bytes, flags: frozenset[str]) -> int:
|
||||
imap = await self._connect()
|
||||
flag_str = f"({' '.join(sorted(flags))})" if flags else None
|
||||
resp = await imap.append(raw, mailbox=mailbox, flags=flag_str)
|
||||
resp = await imap.append(raw, mailbox=self._wire(mailbox), flags=flag_str)
|
||||
if resp.result != "OK":
|
||||
raise ProviderConnectError(f"APPEND to {mailbox} failed")
|
||||
for line in resp.lines:
|
||||
|
||||
Reference in New Issue
Block a user