sync-v2 step 4: event-driven account lifecycle (§3)

- Coordinator subscribes to account.state_changed over Redis pub/sub: going
  live marks the account claimable at once; disabled/deleted/error tears the
  owning runner down immediately (≤30s) instead of waiting for the lease TTL.
- delete_account now publishes account.state_changed:deleted after dropping the
  account from sync:live, so teardown is event-driven, not just fencing.
- AccountRunner gains a verify loop (defense in depth): it periodically confirms
  the account is still live and flags itself dead if not; the coordinator reaps
  dead runners and releases their lease. Zombie loops are structurally impossible.
- New core event user.active (SPEC §8.3), published on login and on every WS
  connect; consumed for tier promotion in step 5.
- Acceptance: added event-driven teardown test (isolated from fencing/verify).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-19 06:49:48 +02:00
co-authored by Claude Opus 4.8
parent bcf11fcecb
commit 245672d3f2
7 changed files with 127 additions and 0 deletions
+47
View File
@@ -16,6 +16,7 @@ account because claims are ``SET NX`` and renew/release are owner-checked.
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import os
@@ -29,6 +30,7 @@ from sqlalchemy import select
from app.config import get_settings
from app.db import session_scope
from app.events import channel_for
from app.models import Account
from app.redis_client import aw, get_redis
from app.sync import leases
@@ -78,6 +80,7 @@ class SyncCoordinator:
asyncio.create_task(self._loop(self._reconcile_once, self.reconcile_interval)),
asyncio.create_task(self._loop(self._heartbeat_once, self.heartbeat_refresh)),
asyncio.create_task(self._loop(self._rebalance_once, self.rebalance_interval)),
asyncio.create_task(self._event_subscriber()),
]
logger.info("sync coordinator %s started", self.worker_id)
@@ -132,6 +135,14 @@ class SyncCoordinator:
async def _renew_once(self) -> None:
for account_id in list(self._runners):
runner = self._runners.get(account_id)
if runner is None:
continue
# Reap a runner that self-verified it is no longer live (§3).
if runner.dead.is_set():
logger.info("reaping dead runner for account %s", account_id)
await self._release(account_id)
continue
if not await self._scripts.renew(account_id, self.worker_id, self.lease_ttl):
logger.warning("lost lease for account %s; fencing runner", account_id)
await self._drop(account_id)
@@ -203,3 +214,39 @@ class SyncCoordinator:
for account_id in list(self._runners)[-excess:]:
await self._release(account_id)
logger.info("worker %s shed %d account(s) to rebalance", self.worker_id, excess)
# --- event-driven lifecycle (§3) -----------------------------------------
async def _event_subscriber(self) -> None:
"""React to ``account.state_changed`` across processes (Redis pub/sub).
Going live makes an account claimable at once; going away tears the
runner down immediately (≤30s from the API call) instead of waiting for
the lease to lapse.
"""
pubsub = self._redis.pubsub()
await pubsub.subscribe(channel_for("account.state_changed"))
try:
async for message in pubsub.listen():
if self._stop.is_set():
break
if message.get("type") != "message":
continue
try:
payload = json.loads(message["data"]).get("payload", {})
except (ValueError, TypeError):
continue
await self._on_state_changed(payload)
finally:
with contextlib.suppress(Exception):
await pubsub.aclose() # type: ignore[no-untyped-call]
async def _on_state_changed(self, payload: dict[str, object]) -> None:
raw = payload.get("account_id")
if not isinstance(raw, str):
return
account_id = uuid.UUID(raw)
if payload.get("state") == "live":
await leases.mark_live(self._redis, account_id) # claimable next cycle
elif account_id in self._runners:
await self._release(account_id) # disabled/deleted/error -> stop now