sync-v2 step 1: worker roles + interactive/bulk queue split

- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY.
- queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap
  goes to bulk, all other jobs to interactive.
- workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue
  timeout/concurrency), role=sync runs the supervisor, all=both (dev default).
- docker-compose: worker-sync + worker-jobs services (shared image, nofile
  ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets.
- tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-18 19:53:31 +02:00
co-authored by Claude Opus 4.8
parent 9cb946f75d
commit e0a45c5a46
18 changed files with 262 additions and 102 deletions
+5 -9
View File
@@ -15,7 +15,7 @@ from app.db import get_session
from app.events import get_event_bus
from app.identity.deps import CurrentUser, require_csrf
from app.models import Mailbox, Message, Thread
from app.queue import get_arq_pool
from app.queue import enqueue_job
from app.sync.actions import apply_flags_to_messages, sanitize_flags
router = APIRouter(tags=["mail"], dependencies=[Depends(require_csrf)])
@@ -26,10 +26,9 @@ DbSession = Annotated[AsyncSession, Depends(get_session)]
async def _push_and_publish(
user_id: uuid.UUID, changed: list[Message], add: frozenset[str], remove: frozenset[str]
) -> None:
pool = await get_arq_pool()
bus = get_event_bus()
for message in changed:
await pool.enqueue_job("push_flags_job", str(message.id), sorted(add), sorted(remove))
await enqueue_job("push_flags_job", str(message.id), sorted(add), sorted(remove))
await bus.publish(
"flags.changed",
{
@@ -95,8 +94,7 @@ async def _move_thread(
)
if thread is None:
raise_api_error(404, "not_found", "thread not found")
pool = await get_arq_pool()
await pool.enqueue_job("move_thread_to_folder_job", str(thread_id), special)
await enqueue_job("move_thread_to_folder_job", str(thread_id), special)
@router.post("/threads/{thread_id}/archive")
@@ -126,8 +124,7 @@ async def delete_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession)
)
if thread is None:
raise_api_error(404, "not_found", "thread not found")
pool = await get_arq_pool()
await pool.enqueue_job("delete_thread_job", str(thread_id))
await enqueue_job("delete_thread_job", str(thread_id))
return {"status": "queued"}
@@ -154,6 +151,5 @@ async def move_message(
message.mailbox_id = dst.id # optimistic
await db.commit()
pool = await get_arq_pool()
await pool.enqueue_job("move_job", str(message.id), str(dst.id), str(src_mailbox_id))
await enqueue_job("move_job", str(message.id), str(dst.id), str(src_mailbox_id))
return {"id": str(message.id), "mailbox_id": str(dst.id)}