sync-v2 step 3: lease-owned account claimer (replaces startup supervisor)

Ownership of an account now moves through a Redis lease so any number of sync workers can run against the same account set with no double-IDLE. - leases.py: sync:live registry + sync:lease:{id} (SET NX EX) with owner-checked Lua renew/release; mark_live / mark_not_live helpers. - coordinator.py: SyncCoordinator claims from sync:live up to MAX_ACCOUNTS_PER_ WORKER, renews every LEASE_RENEW (fences the runner on lease loss), reconciles sync:live against Postgres, heartbeats a worker key, and rebalances when it holds well above a fair share. - supervisor start/stop now drive a coordinator; engine._set_state and delete_account maintain sync:live membership (+ lease drop = fencing teardown). - config: MAX_ACCOUNTS_PER_WORKER, LEASE_TTL, LEASE_RENEW. - tests: acceptance 1-3 (single ownership, crash failover, create/delete lifecycle) driving the real coordinator + Redis with faked IMAP loops. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-18 20:34:22 +02:00
parent 23704f323c
commit bcf11fcecb
7 changed files with 549 additions and 30 deletions
@@ -0,0 +1,226 @@
+"""Sync-engine v2 acceptance tests 1-3 (refactor §10).
+
+These drive the real :class:`SyncCoordinator` against the test Redis with the
+per-account IMAP loops replaced by fakes, so the lease protocol — single
+ownership, crash failover, and create/delete lifecycle — is exercised without a
+live mail server. Intervals are shortened so the tests run in seconds.
+
+  1. Two workers, N accounts: every account owned by exactly one worker, exactly
+     one live (IDLE) connection each.
+  2. Kill a worker: its orphaned accounts are re-owned by the survivor once the
+     leases expire, still one connection each (no double ownership).
+  3. A newly-live account is claimed without a restart; a removed account's
+     connection is torn down (lease fencing), with no further ownership.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from collections import defaultdict
+from collections.abc import AsyncIterator, Callable
+
+import pytest_asyncio
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+from app.identity.passwords import hash_password
+from app.models import Account, User
+from app.redis_client import get_redis
+from app.sync import coordinator as coordinator_mod
+from app.sync import leases, runner
+from app.sync.coordinator import SyncCoordinator
+
+# --- fakes: replace the real IMAP loops with connection-counting stand-ins ----
+
+
+@pytest_asyncio.fixture
+async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]:  # type: ignore[no-untyped-def]
+    """Patch the runner's loops; yield a live-connection count per account."""
+    conns: dict[uuid.UUID, int] = defaultdict(int)
+
+    async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None:
+        conns[account_id] += 1  # one IDLE connection while the loop is alive
+        try:
+            await stop.wait()
+        finally:
+            conns[account_id] -= 1
+
+    async def fake_secondary(account_id: uuid.UUID, stop: asyncio.Event | None = None) -> None:
+        assert stop is not None
+        await stop.wait()
+
+    monkeypatch.setattr(runner, "live_loop", fake_live_loop)
+    monkeypatch.setattr(runner, "secondary_poll_loop", fake_secondary)
+    yield conns
+
+
+@pytest_asyncio.fixture
+async def make_accounts() -> AsyncIterator[Callable[[int, str], object]]:
+    """Factory creating N live accounts; cleans up its users afterwards."""
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    user_ids: list[uuid.UUID] = []
+
+    async def _make(n: int, state: str = "live") -> list[uuid.UUID]:
+        async with sm() as session:
+            user = User(
+                handle=f"acc{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")
+            )
+            session.add(user)
+            await session.flush()
+            user_ids.append(user.id)
+            ids: list[uuid.UUID] = []
+            for _ in range(n):
+                account = Account(
+                    user_id=user.id,
+                    email=f"{uuid.uuid4().hex[:10]}@example.com",
+                    display_name="Acct",
+                    imap_config={},
+                    smtp_config={},
+                    secret_ct=b"",
+                    discovery_src="manual",
+                    state=state,
+                )
+                session.add(account)
+                await session.flush()
+                ids.append(account.id)
+            await session.commit()
+            return ids
+
+    try:
+        yield _make
+    finally:
+        async with sm() as session:
+            for uid in user_ids:
+                await session.execute(delete(User).where(User.id == uid))
+            await session.commit()
+        await engine.dispose()
+
+
+async def _no_reconcile() -> None:
+    """Stub: keep the shared dev DB's real live accounts out of the test."""
+    return None
+
+
+def _fast(c: SyncCoordinator) -> SyncCoordinator:
+    """Compress cadences and drive sync:live explicitly (no DB reconcile).
+
+    The tests share the dev Postgres, which holds real live accounts; reconciling
+    from it would pull them into the test. We seed ``sync:live`` ourselves instead.
+    """
+    c.claim_interval = 0.05
+    c.claim_jitter = 0.0
+    c.lease_renew = 0.3
+    c.lease_ttl = 2  # short so a crashed worker's leases expire quickly
+    c.reconcile_interval = 1000.0
+    c.heartbeat_refresh = 1000.0
+    c.rebalance_interval = 1000.0
+    c._reconcile_once = _no_reconcile  # type: ignore[method-assign]
+    return c
+
+
+async def _seed(ids: set[uuid.UUID]) -> None:
+    for account_id in ids:
+        await leases.mark_live(get_redis(), account_id)
+
+
+async def _until(pred: Callable[[], bool], timeout_s: float = 6.0) -> None:
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + timeout_s
+    while loop.time() < deadline:
+        if pred():
+            return
+        await asyncio.sleep(0.05)
+    assert pred(), "condition not met within timeout"
+
+
+# --- acceptance 1: single ownership, one connection per account ---------------
+
+
+async def test_acceptance_1_single_ownership(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    ids = set(await make_accounts(20))
+    await _seed(ids)
+    c1 = _fast(SyncCoordinator("w1"))
+    c2 = _fast(SyncCoordinator("w2"))
+    c1.rebalance_interval = c2.rebalance_interval = 0.3  # let the load even out
+    # Start concurrently so the two workers race for the leases.
+    await asyncio.gather(c1.start(), c2.start())
+    try:
+        # Both workers end up holding accounts (via the initial race + rebalance),
+        # together covering all 20 with no account owned twice.
+        await _until(lambda: c1.owned | c2.owned == ids and bool(c1.owned and c2.owned))
+        assert c1.owned & c2.owned == set()
+        # At steady state each account has exactly one live IDLE connection.
+        await _until(lambda: all(connections[a] == 1 for a in ids))
+    finally:
+        await c1.stop()
+        await c2.stop()
+    await _until(lambda: all(connections[a] == 0 for a in ids))
+
+
+# --- acceptance 2: crash failover --------------------------------------------
+
+
+async def test_acceptance_2_failover(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    ids = set(await make_accounts(10))
+    await _seed(ids)
+    c1 = _fast(SyncCoordinator("w1"))
+    await c1.start()
+    try:
+        await _until(lambda: c1.owned == ids)  # sole worker owns everything
+
+        c2 = _fast(SyncCoordinator("w2"))
+        await c2.start()
+
+        # Simulate `kill -9` on w1: connections drop and renewals stop, but the
+        # leases are NOT released — they must expire before w2 can take over.
+        for task in c1._tasks:
+            task.cancel()
+        await asyncio.gather(*c1._tasks, return_exceptions=True)
+        c1._tasks = []
+        for r in list(c1._runners.values()):
+            await r.stop()
+        c1._runners.clear()
+
+        try:
+            # w2 re-owns every orphan once the (short) leases expire.
+            await _until(lambda: c2.owned == ids, timeout_s=8.0)
+            assert all(connections[a] == 1 for a in ids)  # one connection each, no dupes
+        finally:
+            await c2.stop()
+    finally:
+        # w1 already crashed; clear any residual leases it left behind.
+        await get_redis().flushdb()
+    await _until(lambda: all(connections[a] == 0 for a in ids))
+
+
+# --- acceptance 3: create / delete lifecycle without restart ------------------
+
+
+async def test_acceptance_3_create_delete(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    c = _fast(SyncCoordinator("w1"))
+    await c.start()  # starts with no live accounts
+    try:
+        assert c.owned == set()
+
+        # A newly-live account (as the bootstrap job's _set_state('live') would do):
+        # register it in sync:live; the running coordinator must claim it, no restart.
+        (aid,) = await make_accounts(1)
+        await leases.mark_live(get_redis(), aid)
+        await _until(lambda: aid in c.owned)
+        assert connections[aid] == 1
+
+        # Removing it (as delete_account does) fences the runner: lease gone ->
+        # next renewal fails -> connection torn down, no further ownership.
+        await leases.mark_not_live(get_redis(), aid)
+        await _until(lambda: aid not in c.owned)
+        await _until(lambda: connections[aid] == 0)
+    finally:
+        await c.stop()
+
+
+def test_worker_id_is_unique() -> None:
+    assert coordinator_mod._new_worker_id() != coordinator_mod._new_worker_id()