sync-v2 step 1: worker roles + interactive/bulk queue split
- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY. - queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap goes to bulk, all other jobs to interactive. - workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue timeout/concurrency), role=sync runs the supervisor, all=both (dev default). - docker-compose: worker-sync + worker-jobs services (shared image, nofile ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets. - tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+26
-34
@@ -1,6 +1,8 @@
|
||||
"""arq worker entrypoint (SPEC §2 ``workers/worker.py``).
|
||||
"""Worker job functions and the sync-worker lifecycle (refactor §1, §6).
|
||||
|
||||
Run with: ``arq workers.worker.WorkerSettings`` (from ``backend/``).
|
||||
The process is launched via ``workers.run`` (``WORKER_ROLE`` decides which arq
|
||||
workers run). This module only defines the job callables and the two queue
|
||||
function-sets; the runner wires them onto the ``interactive`` / ``bulk`` queues.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -9,10 +11,8 @@ import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from app.config import get_settings
|
||||
from app.outbox.draft_sync import push_draft_to_imap_job, remove_imap_draft_job
|
||||
from app.outbox.jobs import send_job
|
||||
from app.queue import redis_settings as arq_redis_settings
|
||||
from app.sync.actions import (
|
||||
delete_thread_job,
|
||||
move_job,
|
||||
@@ -32,41 +32,33 @@ async def sync_bootstrap_account(ctx: dict[str, Any], account_id: str) -> int:
|
||||
return count
|
||||
|
||||
|
||||
async def on_startup(ctx: dict[str, Any]) -> None:
|
||||
logging.basicConfig(level=get_settings().log_level)
|
||||
# Fast, interactive user actions (§6): short timeout, high concurrency.
|
||||
INTERACTIVE_FUNCTIONS = [
|
||||
push_flags_job,
|
||||
move_job,
|
||||
move_thread_to_folder_job,
|
||||
delete_thread_job,
|
||||
send_job,
|
||||
push_draft_to_imap_job,
|
||||
remove_imap_draft_job,
|
||||
refresh_account_job,
|
||||
]
|
||||
|
||||
# Bulk work (§6): long timeout, low concurrency, isolated from interactive jobs.
|
||||
BULK_FUNCTIONS = [sync_bootstrap_account]
|
||||
|
||||
|
||||
async def sync_on_startup(ctx: dict[str, Any]) -> None:
|
||||
"""Sync-worker startup: launch the account-sync supervisor."""
|
||||
from app.sync.supervisor import start_supervisor
|
||||
|
||||
await start_supervisor(ctx)
|
||||
logger.info("webmail worker started")
|
||||
logger.info("sync worker: supervisor started")
|
||||
|
||||
|
||||
async def on_shutdown(ctx: dict[str, Any]) -> None:
|
||||
from app.db import dispose_engine
|
||||
from app.redis_client import close_redis
|
||||
async def sync_on_shutdown(ctx: dict[str, Any]) -> None:
|
||||
"""Sync-worker shutdown: stop the supervisor (process cleanup is in the runner)."""
|
||||
from app.sync.supervisor import stop_supervisor
|
||||
|
||||
await stop_supervisor(ctx)
|
||||
await dispose_engine()
|
||||
await close_redis()
|
||||
logger.info("webmail worker stopped")
|
||||
|
||||
|
||||
class WorkerSettings:
|
||||
"""arq worker configuration."""
|
||||
|
||||
functions = [
|
||||
sync_bootstrap_account,
|
||||
push_flags_job,
|
||||
move_job,
|
||||
move_thread_to_folder_job,
|
||||
delete_thread_job,
|
||||
send_job,
|
||||
push_draft_to_imap_job,
|
||||
remove_imap_draft_job,
|
||||
refresh_account_job,
|
||||
]
|
||||
on_startup = on_startup
|
||||
on_shutdown = on_shutdown
|
||||
max_jobs = 10 # global worker cap (SPEC §6.3)
|
||||
job_timeout = 3600 # backfill of a large account can be long
|
||||
redis_settings = arq_redis_settings()
|
||||
logger.info("sync worker: supervisor stopped")
|
||||
|
||||
Reference in New Issue
Block a user