lukasandClaude Opus 4.8
245672d3f2
- Coordinator subscribes to account.state_changed over Redis pub/sub: going live marks the account claimable at once; disabled/deleted/error tears the owning runner down immediately (≤30s) instead of waiting for the lease TTL. - delete_account now publishes account.state_changed:deleted after dropping the account from sync:live, so teardown is event-driven, not just fencing. - AccountRunner gains a verify loop (defense in depth): it periodically confirms the account is still live and flags itself dead if not; the coordinator reaps dead runners and releases their lease. Zombie loops are structurally impossible. - New core event user.active (SPEC §8.3), published on login and on every WS connect; consumed for tier promotion in step 5. - Acceptance: added event-driven teardown test (isolated from fencing/verify). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
"""Per-account sync runner (refactor §4).
|
|
|
|
An ``AccountRunner`` owns the long-lived sync loops for a single account and
|
|
exposes a small ``start`` / ``stop`` lifecycle. This is a pure extraction of the
|
|
loops that previously lived inline in the supervisor — behavior is unchanged
|
|
(INBOX IDLE + secondary CONDSTORE poll, each self-healing with backoff).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import logging
|
|
import uuid
|
|
from collections.abc import Awaitable, Callable
|
|
|
|
from sqlalchemy import select
|
|
|
|
from app.db import session_scope
|
|
from app.models import Account
|
|
from app.sync.engine import live_loop, secondary_poll_loop
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_RECONNECT_BACKOFF = 30.0
|
|
_VERIFY_INTERVAL = 60.0
|
|
|
|
|
|
class AccountRunner:
|
|
"""Owns the sync loops for one account until stopped."""
|
|
|
|
def __init__(self, account_id: uuid.UUID) -> None:
|
|
self.account_id = account_id
|
|
self._stop = asyncio.Event()
|
|
self._tasks: list[asyncio.Task[None]] = []
|
|
# Set when the account is found no longer live (§3 defense in depth); the
|
|
# coordinator reaps such runners and releases their lease.
|
|
self.dead = asyncio.Event()
|
|
self.verify_interval = _VERIFY_INTERVAL
|
|
|
|
def start(self) -> None:
|
|
"""Launch the account's sync loops (idempotent-ish: call once)."""
|
|
self._stop.clear()
|
|
self._tasks = [
|
|
asyncio.create_task(self._supervise(self._run_live, "live")),
|
|
asyncio.create_task(self._supervise(self._run_secondary, "secondary")),
|
|
asyncio.create_task(self._verify_loop()),
|
|
]
|
|
|
|
def running(self) -> bool:
|
|
return any(not t.done() for t in self._tasks)
|
|
|
|
async def stop(self) -> None:
|
|
"""Signal and cancel the loops, waiting for teardown."""
|
|
self._stop.set()
|
|
for task in self._tasks:
|
|
task.cancel()
|
|
if self._tasks:
|
|
await asyncio.gather(*self._tasks, return_exceptions=True)
|
|
self._tasks = []
|
|
|
|
async def _run_live(self) -> None:
|
|
await live_loop(self.account_id, "INBOX", self._stop)
|
|
|
|
async def _run_secondary(self) -> None:
|
|
await secondary_poll_loop(self.account_id, self._stop)
|
|
|
|
async def _verify_loop(self) -> None:
|
|
"""Periodically confirm the account is still live; flag ``dead`` if not.
|
|
|
|
This makes zombie loops structurally impossible even if a lifecycle event
|
|
is lost: an account that was disabled/deleted stops being synced within a
|
|
verify cycle regardless of the lease/event paths.
|
|
"""
|
|
while not self._stop.is_set():
|
|
with contextlib.suppress(TimeoutError):
|
|
await asyncio.wait_for(self._stop.wait(), timeout=self.verify_interval)
|
|
if self._stop.is_set():
|
|
return
|
|
if not await self._account_is_live():
|
|
logger.info("account %s no longer live; flagging runner dead", self.account_id)
|
|
self.dead.set()
|
|
return
|
|
|
|
async def _account_is_live(self) -> bool:
|
|
async with session_scope() as session:
|
|
state = await session.scalar(select(Account.state).where(Account.id == self.account_id))
|
|
return state == "live"
|
|
|
|
async def _supervise(self, run: Callable[[], Awaitable[None]], label: str) -> None:
|
|
"""Run a loop, restarting it (with backoff) until stopped."""
|
|
while not self._stop.is_set():
|
|
try:
|
|
await run()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.exception("%s loop for account %s crashed; retrying", label, self.account_id)
|
|
with contextlib.suppress(TimeoutError):
|
|
await asyncio.wait_for(self._stop.wait(), timeout=_RECONNECT_BACKOFF)
|