"""Worker entrypoint (refactor §1, §7). One image, two roles selected by ``WORKER_ROLE``: - ``jobs`` — arq consumer for the ``interactive`` and ``bulk`` queues (both in one process, per-queue concurrency/timeout). - ``sync`` — runs the lease-owned account-sync coordinator (§2) directly. - ``all`` — both, for single-process development (the default). Graceful shutdown (SIGINT/SIGTERM): stop the coordinator first — it stops the claimer, cancels the runners, and releases every lease so a peer adopts the accounts immediately — then drain the arq consumers. Target < 10 s. Run with: ``python -m workers.run``. """ from __future__ import annotations import asyncio import logging import signal from typing import Any from arq.worker import Worker from app.config import get_settings from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, redis_settings from app.sync.coordinator import SyncCoordinator from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS logger = logging.getLogger(__name__) _SHUTDOWN_DRAIN_S = 10.0 # §7: release leases + drain within this budget def _worker(functions: list[Any], queue_name: str, max_jobs: int, job_timeout: int) -> Worker: return Worker( functions=functions, queue_name=queue_name, redis_settings=redis_settings(), max_jobs=max_jobs, job_timeout=job_timeout, handle_signals=False, # the runner owns signal handling (graceful shutdown) ) def build_arq_workers(role: str) -> list[Worker]: """Construct the arq Worker instances for the given role (jobs only).""" s = get_settings() if role not in ("jobs", "all"): return [] return [ _worker(INTERACTIVE_FUNCTIONS, QUEUE_INTERACTIVE, s.interactive_concurrency, 120), _worker(BULK_FUNCTIONS, QUEUE_BULK, s.bulk_concurrency, 3600), ] async def _run() -> None: logging.basicConfig(level=get_settings().log_level) role = get_settings().worker_role coordinator = SyncCoordinator() if role in ("sync", "all") else None if coordinator is not None: await coordinator.start() arq_tasks = [asyncio.create_task(w.async_run()) for w in build_arq_workers(role)] logger.info("worker starting: role=%s (%d arq worker(s))", role, len(arq_tasks)) stop = asyncio.Event() loop = asyncio.get_running_loop() for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, stop.set) await stop.wait() logger.info("worker stopping: releasing leases + draining") if coordinator is not None: try: await asyncio.wait_for(coordinator.stop(), timeout=_SHUTDOWN_DRAIN_S) except TimeoutError: logger.warning("coordinator stop exceeded %.0fs drain budget", _SHUTDOWN_DRAIN_S) for task in arq_tasks: task.cancel() await asyncio.gather(*arq_tasks, return_exceptions=True) from app.db import dispose_engine from app.queue import close_arq_pool from app.redis_client import close_redis await dispose_engine() await close_redis() await close_arq_pool() logger.info("worker stopped") def main() -> None: asyncio.run(_run()) if __name__ == "__main__": main()