sync-v2 step 1: worker roles + interactive/bulk queue split
- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY. - queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap goes to bulk, all other jobs to interactive. - workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue timeout/concurrency), role=sync runs the supervisor, all=both (dev default). - docker-compose: worker-sync + worker-jobs services (shared image, nofile ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets. - tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+27
-1
@@ -1,14 +1,33 @@
|
||||
"""arq queue helpers (enqueue side, used by the API)."""
|
||||
"""arq queue helpers (enqueue side, used by the API).
|
||||
|
||||
Two named queues (refactor §6): ``interactive`` (fast user actions) and ``bulk``
|
||||
(backfills). Enqueue via :func:`enqueue_job`, which routes each job to its queue —
|
||||
the API never needs to know queue names.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from arq import create_pool
|
||||
from arq.connections import ArqRedis, RedisSettings
|
||||
from arq.jobs import Job
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
_pool: ArqRedis | None = None
|
||||
|
||||
QUEUE_INTERACTIVE = "interactive"
|
||||
QUEUE_BULK = "bulk"
|
||||
|
||||
# Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
|
||||
_BULK_JOBS = frozenset({"sync_bootstrap_account"})
|
||||
|
||||
|
||||
def queue_for(job_name: str) -> str:
|
||||
"""Return the queue a job should run on."""
|
||||
return QUEUE_BULK if job_name in _BULK_JOBS else QUEUE_INTERACTIVE
|
||||
|
||||
|
||||
def redis_settings() -> RedisSettings:
|
||||
"""Return arq RedisSettings derived from ``REDIS_URL``."""
|
||||
@@ -23,6 +42,13 @@ async def get_arq_pool() -> ArqRedis:
|
||||
return _pool
|
||||
|
||||
|
||||
async def enqueue_job(name: str, *args: Any, **kwargs: Any) -> Job | None:
|
||||
"""Enqueue ``name`` onto its routed queue (interactive/bulk)."""
|
||||
pool = await get_arq_pool()
|
||||
kwargs.setdefault("_queue_name", queue_for(name))
|
||||
return await pool.enqueue_job(name, *args, **kwargs)
|
||||
|
||||
|
||||
async def close_arq_pool() -> None:
|
||||
"""Close the arq pool."""
|
||||
global _pool
|
||||
|
||||
Reference in New Issue
Block a user