lukasandClaude Opus 4.8
6d83f7cde8
- Heartbeat JSON now carries {owned, sockets_open, tiers:{hot,warm,cold}, ts} per
§2 so peers and operators can see each worker's load and socket footprint.
- run.py runs the sync coordinator directly (not via a placeholder arq worker) so
SIGINT/SIGTERM can stop the claimer, cancel runners, and release every lease
explicitly within a 10s drain budget — peers adopt the accounts immediately
instead of waiting out the lease TTL. Removed the sync:control-queue hack and
the supervisor indirection (start/stop_supervisor, sync_on_startup/shutdown).
- Acceptance: graceful-shutdown test proves a peer adopts released accounts within
seconds under a 30s TTL (i.e. via release, not expiry).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
100 lines
3.1 KiB
Python
100 lines
3.1 KiB
Python
"""Worker entrypoint (refactor §1, §7).
|
|
|
|
One image, two roles selected by ``WORKER_ROLE``:
|
|
|
|
- ``jobs`` — arq consumer for the ``interactive`` and ``bulk`` queues (both in one
|
|
process, per-queue concurrency/timeout).
|
|
- ``sync`` — runs the lease-owned account-sync coordinator (§2) directly.
|
|
- ``all`` — both, for single-process development (the default).
|
|
|
|
Graceful shutdown (SIGINT/SIGTERM): stop the coordinator first — it stops the
|
|
claimer, cancels the runners, and releases every lease so a peer adopts the
|
|
accounts immediately — then drain the arq consumers. Target < 10 s.
|
|
|
|
Run with: ``python -m workers.run``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import signal
|
|
from typing import Any
|
|
|
|
from arq.worker import Worker
|
|
|
|
from app.config import get_settings
|
|
from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, redis_settings
|
|
from app.sync.coordinator import SyncCoordinator
|
|
from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_SHUTDOWN_DRAIN_S = 10.0 # §7: release leases + drain within this budget
|
|
|
|
|
|
def _worker(functions: list[Any], queue_name: str, max_jobs: int, job_timeout: int) -> Worker:
|
|
return Worker(
|
|
functions=functions,
|
|
queue_name=queue_name,
|
|
redis_settings=redis_settings(),
|
|
max_jobs=max_jobs,
|
|
job_timeout=job_timeout,
|
|
handle_signals=False, # the runner owns signal handling (graceful shutdown)
|
|
)
|
|
|
|
|
|
def build_arq_workers(role: str) -> list[Worker]:
|
|
"""Construct the arq Worker instances for the given role (jobs only)."""
|
|
s = get_settings()
|
|
if role not in ("jobs", "all"):
|
|
return []
|
|
return [
|
|
_worker(INTERACTIVE_FUNCTIONS, QUEUE_INTERACTIVE, s.interactive_concurrency, 120),
|
|
_worker(BULK_FUNCTIONS, QUEUE_BULK, s.bulk_concurrency, 3600),
|
|
]
|
|
|
|
|
|
async def _run() -> None:
|
|
logging.basicConfig(level=get_settings().log_level)
|
|
role = get_settings().worker_role
|
|
|
|
coordinator = SyncCoordinator() if role in ("sync", "all") else None
|
|
if coordinator is not None:
|
|
await coordinator.start()
|
|
arq_tasks = [asyncio.create_task(w.async_run()) for w in build_arq_workers(role)]
|
|
logger.info("worker starting: role=%s (%d arq worker(s))", role, len(arq_tasks))
|
|
|
|
stop = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
loop.add_signal_handler(sig, stop.set)
|
|
await stop.wait()
|
|
|
|
logger.info("worker stopping: releasing leases + draining")
|
|
if coordinator is not None:
|
|
try:
|
|
await asyncio.wait_for(coordinator.stop(), timeout=_SHUTDOWN_DRAIN_S)
|
|
except TimeoutError:
|
|
logger.warning("coordinator stop exceeded %.0fs drain budget", _SHUTDOWN_DRAIN_S)
|
|
for task in arq_tasks:
|
|
task.cancel()
|
|
await asyncio.gather(*arq_tasks, return_exceptions=True)
|
|
|
|
from app.db import dispose_engine
|
|
from app.queue import close_arq_pool
|
|
from app.redis_client import close_redis
|
|
|
|
await dispose_engine()
|
|
await close_redis()
|
|
await close_arq_pool()
|
|
logger.info("worker stopped")
|
|
|
|
|
|
def main() -> None:
|
|
asyncio.run(_run())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|