webmail-client/backend/tests/sync/test_acceptance.py

"""Sync-engine v2 acceptance tests 1-3 (refactor §10).

These drive the real :class:`SyncCoordinator` against the test Redis with the
per-account IMAP loops replaced by fakes, so the lease protocol — single
ownership, crash failover, and create/delete lifecycle — is exercised without a
live mail server. Intervals are shortened so the tests run in seconds.

  1. Two workers, N accounts: every account owned by exactly one worker, exactly
     one live (IDLE) connection each.
  2. Kill a worker: its orphaned accounts are re-owned by the survivor once the
     leases expire, still one connection each (no double ownership).
  3. A newly-live account is claimed without a restart; a removed account's
     connection is torn down (lease fencing), with no further ownership.
"""

from __future__ import annotations

import asyncio
import uuid
from collections import defaultdict
from collections.abc import AsyncIterator, Callable

import pytest_asyncio
from sqlalchemy import delete
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool

from app.config import get_settings
from app.events import get_event_bus
from app.identity.passwords import hash_password
from app.models import Account, User
from app.redis_client import get_redis
from app.sync import coordinator as coordinator_mod
from app.sync import leases, runner
from app.sync.coordinator import SyncCoordinator

# --- fakes: replace the real IMAP loops with connection-counting stand-ins ----


@pytest_asyncio.fixture
async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]:  # type: ignore[no-untyped-def]
    """Patch the runner's loops; yield a live-connection count per account."""
    conns: dict[uuid.UUID, int] = defaultdict(int)

    async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None:
        conns[account_id] += 1  # one IDLE connection while the loop is alive
        try:
            await stop.wait()
        finally:
            conns[account_id] -= 1

    async def fake_secondary(account_id: uuid.UUID, stop: asyncio.Event | None = None) -> None:
        assert stop is not None
        await stop.wait()

    monkeypatch.setattr(runner, "live_loop", fake_live_loop)
    monkeypatch.setattr(runner, "secondary_poll_loop", fake_secondary)
    yield conns


@pytest_asyncio.fixture
async def make_accounts() -> AsyncIterator[Callable[[int, str], object]]:
    """Factory creating N live accounts; cleans up its users afterwards."""
    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
    sm = async_sessionmaker(engine, expire_on_commit=False)
    user_ids: list[uuid.UUID] = []

    async def _make(n: int, state: str = "live") -> list[uuid.UUID]:
        async with sm() as session:
            user = User(
                handle=f"acc{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")
            )
            session.add(user)
            await session.flush()
            user_ids.append(user.id)
            ids: list[uuid.UUID] = []
            for _ in range(n):
                account = Account(
                    user_id=user.id,
                    email=f"{uuid.uuid4().hex[:10]}@example.com",
                    display_name="Acct",
                    imap_config={},
                    smtp_config={},
                    secret_ct=b"",
                    discovery_src="manual",
                    state=state,
                )
                session.add(account)
                await session.flush()
                ids.append(account.id)
            await session.commit()
            return ids

    try:
        yield _make
    finally:
        async with sm() as session:
            for uid in user_ids:
                await session.execute(delete(User).where(User.id == uid))
            await session.commit()
        await engine.dispose()


async def _no_reconcile() -> None:
    """Stub: keep the shared dev DB's real live accounts out of the test."""
    return None


def _fast(c: SyncCoordinator) -> SyncCoordinator:
    """Compress cadences and drive sync:live explicitly (no DB reconcile).

    The tests share the dev Postgres, which holds real live accounts; reconciling
    from it would pull them into the test. We seed ``sync:live`` ourselves instead.
    """
    c.claim_interval = 0.05
    c.claim_jitter = 0.0
    c.lease_renew = 0.3
    c.lease_ttl = 2  # short so a crashed worker's leases expire quickly
    c.reconcile_interval = 1000.0
    c.heartbeat_refresh = 1000.0
    c.rebalance_interval = 1000.0
    c._reconcile_once = _no_reconcile  # type: ignore[method-assign]
    return c


async def _seed(ids: set[uuid.UUID]) -> None:
    for account_id in ids:
        await leases.mark_live(get_redis(), account_id)


async def _until(pred: Callable[[], bool], timeout_s: float = 6.0) -> None:
    loop = asyncio.get_running_loop()
    deadline = loop.time() + timeout_s
    while loop.time() < deadline:
        if pred():
            return
        await asyncio.sleep(0.05)
    assert pred(), "condition not met within timeout"


# --- acceptance 1: single ownership, one connection per account ---------------


async def test_acceptance_1_single_ownership(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
    ids = set(await make_accounts(20))
    await _seed(ids)
    c1 = _fast(SyncCoordinator("w1"))
    c2 = _fast(SyncCoordinator("w2"))
    c1.rebalance_interval = c2.rebalance_interval = 0.3  # let the load even out
    # Start concurrently so the two workers race for the leases.
    await asyncio.gather(c1.start(), c2.start())
    try:
        # Both workers end up holding accounts (via the initial race + rebalance),
        # together covering all 20 with no account owned twice.
        await _until(lambda: c1.owned | c2.owned == ids and bool(c1.owned and c2.owned))
        assert c1.owned & c2.owned == set()
        # At steady state each account has exactly one live IDLE connection.
        await _until(lambda: all(connections[a] == 1 for a in ids))
    finally:
        await c1.stop()
        await c2.stop()
    await _until(lambda: all(connections[a] == 0 for a in ids))


# --- acceptance 2: crash failover --------------------------------------------


async def test_acceptance_2_failover(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
    ids = set(await make_accounts(10))
    await _seed(ids)
    c1 = _fast(SyncCoordinator("w1"))
    await c1.start()
    try:
        await _until(lambda: c1.owned == ids)  # sole worker owns everything

        c2 = _fast(SyncCoordinator("w2"))
        await c2.start()

        # Simulate `kill -9` on w1: connections drop and renewals stop, but the
        # leases are NOT released — they must expire before w2 can take over.
        for task in c1._tasks:
            task.cancel()
        await asyncio.gather(*c1._tasks, return_exceptions=True)
        c1._tasks = []
        for r in list(c1._runners.values()):
            await r.stop()
        c1._runners.clear()

        try:
            # w2 re-owns every orphan once the (short) leases expire.
            await _until(lambda: c2.owned == ids, timeout_s=8.0)
            assert all(connections[a] == 1 for a in ids)  # one connection each, no dupes
        finally:
            await c2.stop()
    finally:
        # w1 already crashed; clear any residual leases it left behind.
        await get_redis().flushdb()
    await _until(lambda: all(connections[a] == 0 for a in ids))


# --- acceptance 3: create / delete lifecycle without restart ------------------


async def test_acceptance_3_create_delete(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
    c = _fast(SyncCoordinator("w1"))
    await c.start()  # starts with no live accounts
    try:
        assert c.owned == set()

        # A newly-live account (as the bootstrap job's _set_state('live') would do):
        # register it in sync:live; the running coordinator must claim it, no restart.
        (aid,) = await make_accounts(1)
        await leases.mark_live(get_redis(), aid)
        await _until(lambda: aid in c.owned)
        assert connections[aid] == 1

        # Removing it (as delete_account does) fences the runner: lease gone ->
        # next renewal fails -> connection torn down, no further ownership.
        await leases.mark_not_live(get_redis(), aid)
        await _until(lambda: aid not in c.owned)
        await _until(lambda: connections[aid] == 0)
    finally:
        await c.stop()


async def test_event_driven_teardown(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
    """§3: an account.state_changed event tears the runner down at once.

    Leases are kept long (no fencing) and the verify loop slow, so the only path
    that can stop the runner in time is the cross-process event subscriber.
    """
    (aid,) = await make_accounts(1)
    user_id = uuid.uuid4()
    await _seed({aid})
    c = _fast(SyncCoordinator("w1"))
    c.lease_ttl = 60  # lease stays valid: fencing cannot explain the teardown
    c.lease_renew = 30
    await c.start()
    try:
        await _until(lambda: aid in c.owned)
        assert connections[aid] == 1

        # As delete_account does: drop it from sync:live (so it isn't re-claimed),
        # then emit the event. The lease is left intact, so only the event — not
        # fencing — can explain the teardown.
        await get_redis().srem(leases.LIVE_SET, str(aid))
        await get_event_bus().publish(
            "account.state_changed",
            {"user_id": str(user_id), "account_id": str(aid), "state": "deleted"},
        )
        await _until(lambda: aid not in c.owned, timeout_s=3.0)
        await _until(lambda: connections[aid] == 0, timeout_s=3.0)
    finally:
        await c.stop()


def test_worker_id_is_unique() -> None:
    assert coordinator_mod._new_worker_id() != coordinator_mod._new_worker_id()