sync-v2 step 1: worker roles + interactive/bulk queue split

- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY.
- queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap
  goes to bulk, all other jobs to interactive.
- workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue
  timeout/concurrency), role=sync runs the supervisor, all=both (dev default).
- docker-compose: worker-sync + worker-jobs services (shared image, nofile
  ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets.
- tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-18 19:53:31 +02:00
co-authored by Claude Opus 4.8
parent 9cb946f75d
commit e0a45c5a46
18 changed files with 262 additions and 102 deletions
+26 -34
View File
@@ -1,6 +1,8 @@
"""arq worker entrypoint (SPEC §2 ``workers/worker.py``).
"""Worker job functions and the sync-worker lifecycle (refactor §1, §6).
Run with: ``arq workers.worker.WorkerSettings`` (from ``backend/``).
The process is launched via ``workers.run`` (``WORKER_ROLE`` decides which arq
workers run). This module only defines the job callables and the two queue
function-sets; the runner wires them onto the ``interactive`` / ``bulk`` queues.
"""
from __future__ import annotations
@@ -9,10 +11,8 @@ import logging
import uuid
from typing import Any
from app.config import get_settings
from app.outbox.draft_sync import push_draft_to_imap_job, remove_imap_draft_job
from app.outbox.jobs import send_job
from app.queue import redis_settings as arq_redis_settings
from app.sync.actions import (
delete_thread_job,
move_job,
@@ -32,41 +32,33 @@ async def sync_bootstrap_account(ctx: dict[str, Any], account_id: str) -> int:
return count
async def on_startup(ctx: dict[str, Any]) -> None:
logging.basicConfig(level=get_settings().log_level)
# Fast, interactive user actions (§6): short timeout, high concurrency.
INTERACTIVE_FUNCTIONS = [
push_flags_job,
move_job,
move_thread_to_folder_job,
delete_thread_job,
send_job,
push_draft_to_imap_job,
remove_imap_draft_job,
refresh_account_job,
]
# Bulk work (§6): long timeout, low concurrency, isolated from interactive jobs.
BULK_FUNCTIONS = [sync_bootstrap_account]
async def sync_on_startup(ctx: dict[str, Any]) -> None:
"""Sync-worker startup: launch the account-sync supervisor."""
from app.sync.supervisor import start_supervisor
await start_supervisor(ctx)
logger.info("webmail worker started")
logger.info("sync worker: supervisor started")
async def on_shutdown(ctx: dict[str, Any]) -> None:
from app.db import dispose_engine
from app.redis_client import close_redis
async def sync_on_shutdown(ctx: dict[str, Any]) -> None:
"""Sync-worker shutdown: stop the supervisor (process cleanup is in the runner)."""
from app.sync.supervisor import stop_supervisor
await stop_supervisor(ctx)
await dispose_engine()
await close_redis()
logger.info("webmail worker stopped")
class WorkerSettings:
"""arq worker configuration."""
functions = [
sync_bootstrap_account,
push_flags_job,
move_job,
move_thread_to_folder_job,
delete_thread_job,
send_job,
push_draft_to_imap_job,
remove_imap_draft_job,
refresh_account_job,
]
on_startup = on_startup
on_shutdown = on_shutdown
max_jobs = 10 # global worker cap (SPEC §6.3)
job_timeout = 3600 # backfill of a large account can be long
redis_settings = arq_redis_settings()
logger.info("sync worker: supervisor stopped")