sync-v2 step 7: observability + graceful shutdown (§7)

- Heartbeat JSON now carries {owned, sockets_open, tiers:{hot,warm,cold}, ts} per
  §2 so peers and operators can see each worker's load and socket footprint.
- run.py runs the sync coordinator directly (not via a placeholder arq worker) so
  SIGINT/SIGTERM can stop the claimer, cancel runners, and release every lease
  explicitly within a 10s drain budget — peers adopt the accounts immediately
  instead of waiting out the lease TTL. Removed the sync:control-queue hack and
  the supervisor indirection (start/stop_supervisor, sync_on_startup/shutdown).
- Acceptance: graceful-shutdown test proves a peer adopts released accounts within
  seconds under a 30s TTL (i.e. via release, not expiry).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/workers/run.py

@@ -1,12 +1,16 @@
-"""Worker entrypoint (refactor §1).
+"""Worker entrypoint (refactor §1, §7).
 
 One image, two roles selected by ``WORKER_ROLE``:
 
 - ``jobs`` — arq consumer for the ``interactive`` and ``bulk`` queues (both in one
   process, per-queue concurrency/timeout).
-- ``sync`` — runs the account-sync supervisor (long-lived per-account loops).
+- ``sync`` — runs the lease-owned account-sync coordinator (§2) directly.
 - ``all``  — both, for single-process development (the default).
 
+Graceful shutdown (SIGINT/SIGTERM): stop the coordinator first — it stops the
+claimer, cancels the runners, and releases every lease so a peer adopts the
+accounts immediately — then drain the arq consumers. Target < 10 s.
+
 Run with: ``python -m workers.run``.
 """
 
@@ -21,77 +25,61 @@ from arq.worker import Worker
 
 from app.config import get_settings
 from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, redis_settings
-from workers.worker import (
-    BULK_FUNCTIONS,
-    INTERACTIVE_FUNCTIONS,
-    sync_on_shutdown,
-    sync_on_startup,
-)
+from app.sync.coordinator import SyncCoordinator
+from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS
 
 logger = logging.getLogger(__name__)
 
-# arq requires a non-empty function set; the sync worker consumes a queue nothing
-# is ever enqueued to, so it just keeps the event loop alive for the supervisor.
-_SYNC_CONTROL_QUEUE = "sync:control"
+_SHUTDOWN_DRAIN_S = 10.0  # §7: release leases + drain within this budget
 
 
-async def _noop(ctx: dict[str, Any]) -> None:
-    return None
-
-
-def _worker(
-    functions: list[Any],
-    queue_name: str,
-    max_jobs: int,
-    job_timeout: int,
-    on_startup: Any = None,
-    on_shutdown: Any = None,
-) -> Worker:
+def _worker(functions: list[Any], queue_name: str, max_jobs: int, job_timeout: int) -> Worker:
     return Worker(
         functions=functions,
         queue_name=queue_name,
         redis_settings=redis_settings(),
         max_jobs=max_jobs,
         job_timeout=job_timeout,
-        on_startup=on_startup,
-        on_shutdown=on_shutdown,
         handle_signals=False,  # the runner owns signal handling (graceful shutdown)
     )
 
 
-def build_workers(role: str) -> list[Worker]:
-    """Construct the arq Worker instances for the given role."""
+def build_arq_workers(role: str) -> list[Worker]:
+    """Construct the arq Worker instances for the given role (jobs only)."""
     s = get_settings()
-    workers: list[Worker] = []
-    if role in ("sync", "all"):
-        workers.append(
-            _worker([_noop], _SYNC_CONTROL_QUEUE, 1, 3600, sync_on_startup, sync_on_shutdown)
-        )
-    if role in ("jobs", "all"):
-        workers.append(
-            _worker(INTERACTIVE_FUNCTIONS, QUEUE_INTERACTIVE, s.interactive_concurrency, 120)
-        )
-        workers.append(_worker(BULK_FUNCTIONS, QUEUE_BULK, s.bulk_concurrency, 3600))
-    return workers
+    if role not in ("jobs", "all"):
+        return []
+    return [
+        _worker(INTERACTIVE_FUNCTIONS, QUEUE_INTERACTIVE, s.interactive_concurrency, 120),
+        _worker(BULK_FUNCTIONS, QUEUE_BULK, s.bulk_concurrency, 3600),
+    ]
 
 
 async def _run() -> None:
     logging.basicConfig(level=get_settings().log_level)
     role = get_settings().worker_role
-    workers = build_workers(role)
-    logger.info("worker starting: role=%s (%d arq worker(s))", role, len(workers))
 
-    tasks = [asyncio.create_task(w.async_run()) for w in workers]
+    coordinator = SyncCoordinator() if role in ("sync", "all") else None
+    if coordinator is not None:
+        await coordinator.start()
+    arq_tasks = [asyncio.create_task(w.async_run()) for w in build_arq_workers(role)]
+    logger.info("worker starting: role=%s (%d arq worker(s))", role, len(arq_tasks))
+
     stop = asyncio.Event()
     loop = asyncio.get_running_loop()
     for sig in (signal.SIGINT, signal.SIGTERM):
         loop.add_signal_handler(sig, stop.set)
-
     await stop.wait()
-    logger.info("worker stopping: cancelling %d worker(s)", len(tasks))
-    for task in tasks:
-        task.cancel()  # arq runs each worker's on_shutdown in its finally
-    await asyncio.gather(*tasks, return_exceptions=True)
+
+    logger.info("worker stopping: releasing leases + draining")
+    if coordinator is not None:
+        try:
+            await asyncio.wait_for(coordinator.stop(), timeout=_SHUTDOWN_DRAIN_S)
+        except TimeoutError:
+            logger.warning("coordinator stop exceeded %.0fs drain budget", _SHUTDOWN_DRAIN_S)
+    for task in arq_tasks:
+        task.cancel()
+    await asyncio.gather(*arq_tasks, return_exceptions=True)
 
     from app.db import dispose_engine
     from app.queue import close_arq_pool