sync: QRESYNC VANISHED expunge detection (SPEC §6.3 M2 fast resync)
Server-side deletions now propagate to the DB. Previously poll() detected new messages and CONDSTORE flag deltas but never expunges (the old TODO), so mail deleted in another client lingered as phantoms outside the small Drafts reconcile. - provider: enable QRESYNC when advertised (_maybe_enable_qresync + qresync_enabled property). poll() harvests VANISHED (expunged UIDs) and changed FLAGS in one pass via a raw `SELECT <mbox> (QRESYNC (uidvalidity modseq))` — issued as a low-level SELECT because aioimaplib only surfaces untagged VANISHED lines on a sync command, not on an async UID FETCH. CONDSTORE-only servers (e.g. Gmail) keep the CHANGEDSINCE flag delta and fall back to the engine reconcile. - parsing: parse_vanished() for `VANISHED [(EARLIER)] <uid-set>` with range expansion. - engine: apply_delta now drops expunged rows from the search index, un-links any local draft mirroring a vanished UID, and publishes message.deleted for the UI. - tests: parse_vanished unit cases + apply_delta expunge (removal, draft un-link, modseq advance). Verified live: the worker issues `SELECT "INBOX" (QRESYNC (uidvalidity modseq))` against QRESYNC-capable accounts. ruff + mypy --strict clean; 296 tests pass. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Rcj3TpydMTzTSmU7kunzf8
This commit is contained in:
Claude Opus 4.8
@@ -135,6 +135,41 @@ def parse_fetch_lines(lines: list[bytes | bytearray]) -> list[ParsedFetch]:
|
||||
return results
|
||||
|
||||
|
||||
_VANISHED_RE = re.compile(rb"^VANISHED\s+(?:\(EARLIER\)\s+)?(\S+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _expand_uid_set(token: bytes) -> list[int]:
|
||||
"""Expand an IMAP uid-set (``1,3:5,9``) to a flat list of ints (``*`` skipped)."""
|
||||
out: list[int] = []
|
||||
for part in token.split(b","):
|
||||
part = part.strip()
|
||||
lo_b, sep, hi_b = part.partition(b":")
|
||||
if sep:
|
||||
if lo_b.isdigit() and hi_b.isdigit():
|
||||
lo, hi = int(lo_b), int(hi_b)
|
||||
out.extend(range(min(lo, hi), max(lo, hi) + 1))
|
||||
elif part.isdigit():
|
||||
out.append(int(part))
|
||||
return out
|
||||
|
||||
|
||||
def parse_vanished(lines: list[bytes | bytearray]) -> list[int]:
|
||||
"""UIDs from QRESYNC ``VANISHED [(EARLIER)] <uid-set>`` responses (RFC 7162).
|
||||
|
||||
aioimaplib strips the leading ``* `` before buffering the line, so we match on
|
||||
the bare ``VANISHED`` token. Both the SELECT-time ``(EARLIER)`` form and the
|
||||
live-expunge form are accepted.
|
||||
"""
|
||||
out: list[int] = []
|
||||
for line in lines:
|
||||
if not isinstance(line, (bytes, bytearray)):
|
||||
continue
|
||||
m = _VANISHED_RE.match(bytes(line).strip())
|
||||
if m:
|
||||
out.extend(_expand_uid_set(m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
def parse_search_uids(lines: list[bytes | bytearray]) -> list[int]:
|
||||
"""Parse a UID SEARCH response's first line into a list of ints."""
|
||||
if not lines:
|
||||
|
||||
@@ -35,6 +35,7 @@ from app.providers.imap_smtp.parsing import (
|
||||
parse_list_line,
|
||||
parse_search_uids,
|
||||
parse_select_untagged,
|
||||
parse_vanished,
|
||||
)
|
||||
|
||||
_FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
|
||||
@@ -79,6 +80,15 @@ class ImapSmtpProvider(MailProvider):
|
||||
self._selected: str | None = None
|
||||
self._uidvalidity_seen: dict[str, int] = {}
|
||||
self._conn_slots: list[asyncio.Semaphore] = []
|
||||
self._qresync_enabled = False
|
||||
|
||||
@property
|
||||
def qresync_enabled(self) -> bool:
|
||||
"""True once ``ENABLE QRESYNC`` succeeded on this connection (RFC 7162).
|
||||
|
||||
When set, :meth:`poll` reports server-side expunges via VANISHED; otherwise
|
||||
deletion detection falls back to the engine's full-UID reconcile."""
|
||||
return self._qresync_enabled
|
||||
|
||||
# --- connection lifecycle ------------------------------------------------
|
||||
|
||||
@@ -138,8 +148,23 @@ class ImapSmtpProvider(MailProvider):
|
||||
self._release_conn_slots() # never hold a slot without an open socket
|
||||
raise
|
||||
self._imap = imap
|
||||
await self._maybe_enable_qresync(imap)
|
||||
return imap
|
||||
|
||||
async def _maybe_enable_qresync(self, imap: aioimaplib.IMAP4) -> None:
|
||||
"""Enable QRESYNC when advertised so SELECT reports VANISHED (RFC 7162).
|
||||
|
||||
Best-effort: servers without QRESYNC (e.g. Gmail, which is CONDSTORE-only)
|
||||
simply keep ``_qresync_enabled`` False and use the engine's reconcile path."""
|
||||
caps = {c.upper() for c in (imap.protocol.capabilities or set())}
|
||||
if "QRESYNC" not in caps:
|
||||
return
|
||||
try:
|
||||
resp = await imap.enable("QRESYNC")
|
||||
self._qresync_enabled = resp.result == "OK"
|
||||
except Exception: # noqa: BLE001 - degrade gracefully to the reconcile path
|
||||
self._qresync_enabled = False
|
||||
|
||||
async def _select(self, mailbox: str) -> dict[str, int]:
|
||||
imap = await self._connect()
|
||||
resp = await imap.select(mailbox)
|
||||
@@ -287,26 +312,53 @@ class ImapSmtpProvider(MailProvider):
|
||||
search = await imap.uid_search(f"UID {last_seen_uid + 1}:*")
|
||||
candidate_uids = [u for u in parse_search_uids(search.lines) if u > last_seen_uid]
|
||||
new_messages = await self._fetch_messages(imap, sorted(candidate_uids))
|
||||
new_uid_set = {m.uid for m in new_messages}
|
||||
|
||||
# Flag changes via CONDSTORE when we have a modseq baseline.
|
||||
flag_changes: list[FlagChange] = []
|
||||
if highestmodseq is not None:
|
||||
expunged_uids: list[int] = []
|
||||
if highestmodseq is not None and self._qresync_enabled:
|
||||
# QRESYNC fast resync (RFC 7162): a re-SELECT with (QRESYNC (uidvalidity
|
||||
# modseq)) makes the server report, in one pass, the FLAGS changed and the
|
||||
# UIDs VANISHED since ``modseq``. VANISHED is only surfaced on a *sync*
|
||||
# command, so we harvest both from the SELECT response here.
|
||||
qr = await self._qresync_select(mailbox, current_uidvalidity, highestmodseq)
|
||||
expunged_uids = [u for u in parse_vanished(qr.lines) if u not in new_uid_set]
|
||||
for pf in parse_fetch_lines(qr.lines):
|
||||
if pf.uid not in new_uid_set:
|
||||
flag_changes.append(FlagChange(uid=pf.uid, flags=pf.flags))
|
||||
elif highestmodseq is not None:
|
||||
# CONDSTORE-only server (e.g. Gmail): flag delta via CHANGEDSINCE. There
|
||||
# is no VANISHED here — the engine reconciles expunges by full-UID diff.
|
||||
fresp = await imap.uid("fetch", "1:*", f"(UID FLAGS) (CHANGEDSINCE {highestmodseq})")
|
||||
if fresp.result == "OK":
|
||||
new_uid_set = {m.uid for m in new_messages}
|
||||
for pf in parse_fetch_lines(fresp.lines):
|
||||
if pf.uid not in new_uid_set:
|
||||
flag_changes.append(FlagChange(uid=pf.uid, flags=pf.flags))
|
||||
|
||||
# TODO(spec): expunge detection needs QRESYNC/VANISHED (M2 hardening).
|
||||
return MailboxDelta(
|
||||
new=new_messages,
|
||||
flag_changes=flag_changes,
|
||||
expunged_uids=[],
|
||||
expunged_uids=expunged_uids,
|
||||
new_uidvalidity=None,
|
||||
new_highestmodseq=info.get("highestmodseq"),
|
||||
)
|
||||
|
||||
async def _qresync_select(
|
||||
self, mailbox: str, uidvalidity: int, modseq: int
|
||||
) -> aioimaplib.Response:
|
||||
"""Raw ``SELECT <mailbox> (QRESYNC (uidvalidity modseq))`` (RFC 7162).
|
||||
|
||||
Issued as a low-level command because aioimaplib's high-level ``select``
|
||||
takes no parameters, and because SELECT is a *sync* command — the only kind
|
||||
whose untagged responses (VANISHED, changed FETCH) aioimaplib buffers."""
|
||||
imap = await self._connect()
|
||||
proto = imap.protocol
|
||||
param = f"(QRESYNC ({uidvalidity} {modseq}))"
|
||||
cmd = aioimaplib.Command(
|
||||
"SELECT", proto.new_tag(), aioimaplib.quoted(mailbox), param, loop=proto.loop
|
||||
)
|
||||
return await proto.execute(cmd)
|
||||
|
||||
async def idle(self, mailbox: str) -> None:
|
||||
imap = await self._connect()
|
||||
if "IDLE" not in (imap.protocol.capabilities or set()):
|
||||
|
||||
Reference in New Issue
Block a user