"""Sync-engine v2 acceptance tests 1-3 (refactor ยง10). These drive the real :class:`SyncCoordinator` against the test Redis with the per-account IMAP loops replaced by fakes, so the lease protocol โ€” single ownership, crash failover, and create/delete lifecycle โ€” is exercised without a live mail server. Intervals are shortened so the tests run in seconds. 1. Two workers, N accounts: every account owned by exactly one worker, exactly one live (IDLE) connection each. 2. Kill a worker: its orphaned accounts are re-owned by the survivor once the leases expire, still one connection each (no double ownership). 3. A newly-live account is claimed without a restart; a removed account's connection is torn down (lease fencing), with no further ownership. """ from __future__ import annotations import asyncio import uuid from collections import defaultdict from collections.abc import AsyncIterator, Callable import pytest_asyncio from sqlalchemy import delete from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from sqlalchemy.pool import NullPool from app.config import get_settings from app.identity.passwords import hash_password from app.models import Account, User from app.redis_client import get_redis from app.sync import coordinator as coordinator_mod from app.sync import leases, runner from app.sync.coordinator import SyncCoordinator # --- fakes: replace the real IMAP loops with connection-counting stand-ins ---- @pytest_asyncio.fixture async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]: # type: ignore[no-untyped-def] """Patch the runner's loops; yield a live-connection count per account.""" conns: dict[uuid.UUID, int] = defaultdict(int) async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None: conns[account_id] += 1 # one IDLE connection while the loop is alive try: await stop.wait() finally: conns[account_id] -= 1 async def fake_secondary(account_id: uuid.UUID, stop: asyncio.Event | None = None) -> None: assert stop is not None await stop.wait() monkeypatch.setattr(runner, "live_loop", fake_live_loop) monkeypatch.setattr(runner, "secondary_poll_loop", fake_secondary) yield conns @pytest_asyncio.fixture async def make_accounts() -> AsyncIterator[Callable[[int, str], object]]: """Factory creating N live accounts; cleans up its users afterwards.""" engine = create_async_engine(get_settings().database_url, poolclass=NullPool) sm = async_sessionmaker(engine, expire_on_commit=False) user_ids: list[uuid.UUID] = [] async def _make(n: int, state: str = "live") -> list[uuid.UUID]: async with sm() as session: user = User( handle=f"acc{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw") ) session.add(user) await session.flush() user_ids.append(user.id) ids: list[uuid.UUID] = [] for _ in range(n): account = Account( user_id=user.id, email=f"{uuid.uuid4().hex[:10]}@example.com", display_name="Acct", imap_config={}, smtp_config={}, secret_ct=b"", discovery_src="manual", state=state, ) session.add(account) await session.flush() ids.append(account.id) await session.commit() return ids try: yield _make finally: async with sm() as session: for uid in user_ids: await session.execute(delete(User).where(User.id == uid)) await session.commit() await engine.dispose() async def _no_reconcile() -> None: """Stub: keep the shared dev DB's real live accounts out of the test.""" return None def _fast(c: SyncCoordinator) -> SyncCoordinator: """Compress cadences and drive sync:live explicitly (no DB reconcile). The tests share the dev Postgres, which holds real live accounts; reconciling from it would pull them into the test. We seed ``sync:live`` ourselves instead. """ c.claim_interval = 0.05 c.claim_jitter = 0.0 c.lease_renew = 0.3 c.lease_ttl = 2 # short so a crashed worker's leases expire quickly c.reconcile_interval = 1000.0 c.heartbeat_refresh = 1000.0 c.rebalance_interval = 1000.0 c._reconcile_once = _no_reconcile # type: ignore[method-assign] return c async def _seed(ids: set[uuid.UUID]) -> None: for account_id in ids: await leases.mark_live(get_redis(), account_id) async def _until(pred: Callable[[], bool], timeout_s: float = 6.0) -> None: loop = asyncio.get_running_loop() deadline = loop.time() + timeout_s while loop.time() < deadline: if pred(): return await asyncio.sleep(0.05) assert pred(), "condition not met within timeout" # --- acceptance 1: single ownership, one connection per account --------------- async def test_acceptance_1_single_ownership(connections, make_accounts) -> None: # type: ignore[no-untyped-def] ids = set(await make_accounts(20)) await _seed(ids) c1 = _fast(SyncCoordinator("w1")) c2 = _fast(SyncCoordinator("w2")) c1.rebalance_interval = c2.rebalance_interval = 0.3 # let the load even out # Start concurrently so the two workers race for the leases. await asyncio.gather(c1.start(), c2.start()) try: # Both workers end up holding accounts (via the initial race + rebalance), # together covering all 20 with no account owned twice. await _until(lambda: c1.owned | c2.owned == ids and bool(c1.owned and c2.owned)) assert c1.owned & c2.owned == set() # At steady state each account has exactly one live IDLE connection. await _until(lambda: all(connections[a] == 1 for a in ids)) finally: await c1.stop() await c2.stop() await _until(lambda: all(connections[a] == 0 for a in ids)) # --- acceptance 2: crash failover -------------------------------------------- async def test_acceptance_2_failover(connections, make_accounts) -> None: # type: ignore[no-untyped-def] ids = set(await make_accounts(10)) await _seed(ids) c1 = _fast(SyncCoordinator("w1")) await c1.start() try: await _until(lambda: c1.owned == ids) # sole worker owns everything c2 = _fast(SyncCoordinator("w2")) await c2.start() # Simulate `kill -9` on w1: connections drop and renewals stop, but the # leases are NOT released โ€” they must expire before w2 can take over. for task in c1._tasks: task.cancel() await asyncio.gather(*c1._tasks, return_exceptions=True) c1._tasks = [] for r in list(c1._runners.values()): await r.stop() c1._runners.clear() try: # w2 re-owns every orphan once the (short) leases expire. await _until(lambda: c2.owned == ids, timeout_s=8.0) assert all(connections[a] == 1 for a in ids) # one connection each, no dupes finally: await c2.stop() finally: # w1 already crashed; clear any residual leases it left behind. await get_redis().flushdb() await _until(lambda: all(connections[a] == 0 for a in ids)) # --- acceptance 3: create / delete lifecycle without restart ------------------ async def test_acceptance_3_create_delete(connections, make_accounts) -> None: # type: ignore[no-untyped-def] c = _fast(SyncCoordinator("w1")) await c.start() # starts with no live accounts try: assert c.owned == set() # A newly-live account (as the bootstrap job's _set_state('live') would do): # register it in sync:live; the running coordinator must claim it, no restart. (aid,) = await make_accounts(1) await leases.mark_live(get_redis(), aid) await _until(lambda: aid in c.owned) assert connections[aid] == 1 # Removing it (as delete_account does) fences the runner: lease gone -> # next renewal fails -> connection torn down, no further ownership. await leases.mark_not_live(get_redis(), aid) await _until(lambda: aid not in c.owned) await _until(lambda: connections[aid] == 0) finally: await c.stop() def test_worker_id_is_unique() -> None: assert coordinator_mod._new_worker_id() != coordinator_mod._new_worker_id()