sync-v2 step 1: worker roles + interactive/bulk queue split

- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY.
- queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap
  goes to bulk, all other jobs to interactive.
- workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue
  timeout/concurrency), role=sync runs the supervisor, all=both (dev default).
- docker-compose: worker-sync + worker-jobs services (shared image, nofile
  ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets.
- tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-18 19:53:31 +02:00
co-authored by Claude Opus 4.8
parent 9cb946f75d
commit e0a45c5a46
18 changed files with 262 additions and 102 deletions
+31
View File
@@ -36,6 +36,36 @@ services:
timeout: 3s
retries: 10
# Sync-engine v2 workers (refactor §1): two roles sharing one image. Scale the
# sync role with `--scale worker-sync=N` (lease protocol prevents double-owning).
worker-sync:
build: { context: ../backend }
command: ["python", "-m", "workers.run"]
environment: &worker-env
WORKER_ROLE: sync
APP_DOMAIN: dev.webmail.local
DATABASE_URL: postgresql+asyncpg://webmail:webmail_dev@postgres:5432/webmail_dev
REDIS_URL: redis://redis:6379/0
MASTER_KEY: CHANGE_ME_base64_32_bytes
BLOB_ROOT: /var/blobs
volumes: [blobs:/var/blobs]
ulimits: { nofile: 65536 }
depends_on:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
worker-jobs:
build: { context: ../backend }
command: ["python", "-m", "workers.run"]
environment:
<<: *worker-env
WORKER_ROLE: jobs
volumes: [blobs:/var/blobs]
ulimits: { nofile: 65536 }
depends_on:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
# Test mail servers (integration tests + manual dev against a throwaway mailbox)
greenmail:
image: greenmail/standalone:2.0.1
@@ -51,3 +81,4 @@ services:
volumes:
pgdata:
meilidata:
blobs: