sync-v2 step 1: worker roles + interactive/bulk queue split

- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY.
- queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap
  goes to bulk, all other jobs to interactive.
- workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue
  timeout/concurrency), role=sync runs the supervisor, all=both (dev default).
- docker-compose: worker-sync + worker-jobs services (shared image, nofile
  ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets.
- tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/api/accounts.py

@@ -30,7 +30,7 @@ from app.api.schemas import AccountOut
 from app.db import get_session
 from app.identity.deps import CurrentUser, require_csrf
 from app.models import Account
-from app.queue import get_arq_pool
+from app.queue import enqueue_job
 from app.redis_client import get_redis
 from app.security.crypto import encrypt_secret
 
@@ -91,9 +91,8 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
 async def sync_all_accounts(user: CurrentUser, db: DbSession) -> dict[str, str]:
     """Poll every folder of each of the user's accounts (manual Refresh)."""
     account_ids = list(await db.scalars(select(Account.id).where(Account.user_id == user.id)))
-    pool = await get_arq_pool()
     for account_id in account_ids:
-        await pool.enqueue_job("refresh_account_job", str(account_id))
+        await enqueue_job("refresh_account_job", str(account_id))
     return {"status": "queued"}
 
 
@@ -145,8 +144,7 @@ async def create_new_account(
         raise_api_error(409, "account_exists", "an account for this email already exists")
     await db.commit()
 
-    pool = await get_arq_pool()
-    await pool.enqueue_job("sync_bootstrap_account", str(account.id))
+    await enqueue_job("sync_bootstrap_account", str(account.id))
 
     return _detail(account)
 
@@ -227,8 +225,7 @@ async def update_account(
     await db.commit()
 
     if conn_change:
-        pool = await get_arq_pool()
-        await pool.enqueue_job("sync_bootstrap_account", str(account.id))
+        await enqueue_job("sync_bootstrap_account", str(account.id))
 
     return _detail(account)
 

backend/app/api/actions.py

@@ -15,7 +15,7 @@ from app.db import get_session
 from app.events import get_event_bus
 from app.identity.deps import CurrentUser, require_csrf
 from app.models import Mailbox, Message, Thread
-from app.queue import get_arq_pool
+from app.queue import enqueue_job
 from app.sync.actions import apply_flags_to_messages, sanitize_flags
 
 router = APIRouter(tags=["mail"], dependencies=[Depends(require_csrf)])
@@ -26,10 +26,9 @@ DbSession = Annotated[AsyncSession, Depends(get_session)]
 async def _push_and_publish(
     user_id: uuid.UUID, changed: list[Message], add: frozenset[str], remove: frozenset[str]
 ) -> None:
-    pool = await get_arq_pool()
     bus = get_event_bus()
     for message in changed:
-        await pool.enqueue_job("push_flags_job", str(message.id), sorted(add), sorted(remove))
+        await enqueue_job("push_flags_job", str(message.id), sorted(add), sorted(remove))
         await bus.publish(
             "flags.changed",
             {
@@ -95,8 +94,7 @@ async def _move_thread(
     )
     if thread is None:
         raise_api_error(404, "not_found", "thread not found")
-    pool = await get_arq_pool()
-    await pool.enqueue_job("move_thread_to_folder_job", str(thread_id), special)
+    await enqueue_job("move_thread_to_folder_job", str(thread_id), special)
 
 
 @router.post("/threads/{thread_id}/archive")
@@ -126,8 +124,7 @@ async def delete_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession)
     )
     if thread is None:
         raise_api_error(404, "not_found", "thread not found")
-    pool = await get_arq_pool()
-    await pool.enqueue_job("delete_thread_job", str(thread_id))
+    await enqueue_job("delete_thread_job", str(thread_id))
     return {"status": "queued"}
 
 
@@ -154,6 +151,5 @@ async def move_message(
     message.mailbox_id = dst.id  # optimistic
     await db.commit()
 
-    pool = await get_arq_pool()
-    await pool.enqueue_job("move_job", str(message.id), str(dst.id), str(src_mailbox_id))
+    await enqueue_job("move_job", str(message.id), str(dst.id), str(src_mailbox_id))
     return {"id": str(message.id), "mailbox_id": str(dst.id)}

backend/app/api/compose.py

@@ -25,7 +25,7 @@ from app.outbox.schemas import (
     SendResponse,
 )
 from app.outbox.service import DraftError, cancel_outbox, create_outbox, upsert_draft
-from app.queue import get_arq_pool
+from app.queue import enqueue_job
 
 router = APIRouter(tags=["compose"], dependencies=[Depends(require_csrf)])
 
@@ -155,9 +155,7 @@ async def list_remote_drafts(user: CurrentUser, db: DbSession) -> list[RemoteDra
 
 
 @router.post("/drafts/adopt/{message_id}", response_model=DraftOut)
-async def adopt_remote_draft(
-    message_id: uuid.UUID, user: CurrentUser, db: DbSession
-) -> DraftOut:
+async def adopt_remote_draft(message_id: uuid.UUID, user: CurrentUser, db: DbSession) -> DraftOut:
     """Turn a Drafts-folder message into an editable local draft (linked by UID)."""
     msg = await db.scalar(
         select(Message)
@@ -235,8 +233,7 @@ async def publish_draft(draft_id: uuid.UUID, user: CurrentUser, db: DbSession) -
     draft = await db.scalar(select(Draft).where(Draft.id == draft_id, Draft.user_id == user.id))
     if draft is None:
         raise_api_error(404, "not_found", "draft not found")
-    pool = await get_arq_pool()
-    await pool.enqueue_job("push_draft_to_imap_job", str(draft_id))
+    await enqueue_job("push_draft_to_imap_job", str(draft_id))
     return {"status": "queued"}
 
 
@@ -250,8 +247,7 @@ async def delete_draft(draft_id: uuid.UUID, user: CurrentUser, db: DbSession) ->
     await db.commit()
     # Remove the mirrored copy from the server Drafts folder too.
     if imap_uid:
-        pool = await get_arq_pool()
-        await pool.enqueue_job("remove_imap_draft_job", str(account_id), imap_uid)
+        await enqueue_job("remove_imap_draft_job", str(account_id), imap_uid)
     return {"status": "deleted"}
 
 
@@ -265,8 +261,7 @@ async def send(body: SendRequest, user: CurrentUser, db: DbSession) -> SendRespo
     outbox = await create_outbox(db, user_id=user.id, draft=draft)
     await db.commit()
 
-    pool = await get_arq_pool()
-    await pool.enqueue_job(
+    await enqueue_job(
         "send_job",
         str(outbox.id),
         _defer_by=timedelta(seconds=get_settings().undo_seconds),

backend/app/config.py

@@ -34,6 +34,13 @@ class Settings(BaseSettings):
     max_attachment_mb: int = Field(default=25, alias="MAX_ATTACHMENT_MB")
     log_level: str = Field(default="INFO", alias="LOG_LEVEL")
 
+    # --- Sync engine v2 (workers layer) --------------------------------------
+    # Worker role: "sync" (lease-owned account runners) | "jobs" (arq consumer)
+    # | "all" (both, single-process dev default).
+    worker_role: str = Field(default="all", alias="WORKER_ROLE")
+    interactive_concurrency: int = Field(default=20, alias="INTERACTIVE_CONCURRENCY")
+    bulk_concurrency: int = Field(default=2, alias="BULK_CONCURRENCY")
+
     oauth_google_client_id: str = Field(default="", alias="OAUTH_GOOGLE_CLIENT_ID")
     oauth_google_client_secret: str = Field(default="", alias="OAUTH_GOOGLE_CLIENT_SECRET")
     oauth_ms_client_id: str = Field(default="", alias="OAUTH_MS_CLIENT_ID")

backend/app/outbox/draft_sync.py

@@ -34,9 +34,7 @@ async def _ensure_drafts_folder(provider: Any, account: Account, user_id: uuid.U
     uidvalidity = next((r.uidvalidity for r in remote if r.path == path), 0)
     async with session_scope() as session:
         existing = await session.scalar(
-            select(Mailbox).where(
-                Mailbox.account_id == account.id, Mailbox.special_use == "drafts"
-            )
+            select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )
         if existing is not None:
             return existing.path
@@ -72,9 +70,7 @@ async def push_draft_to_imap_job(ctx: dict[str, Any], draft_id: str) -> None:
         attachments = await load_attachments(session, user_id, draft)
         _, sent_raw, _, _ = build_mime(account, draft, reply_to, attachments)
         drafts_mb = await session.scalar(
-            select(Mailbox).where(
-                Mailbox.account_id == account.id, Mailbox.special_use == "drafts"
-            )
+            select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "drafts")
         )
         drafts_path = drafts_mb.path if drafts_mb else None
         account_obj = account

backend/app/outbox/jobs.py

@@ -52,9 +52,7 @@ async def _harvest_recipients(session: AsyncSession, user_id: uuid.UUID, emails:
         await session.execute(stmt)
 
 
-async def _persist_sent_copy(
-    user_id: uuid.UUID, account: Account, raw: bytes, uid: int
-) -> None:
+async def _persist_sent_copy(user_id: uuid.UUID, account: Account, raw: bytes, uid: int) -> None:
     """Store the just-sent message in the DB (Sent mailbox) and notify the UI.
 
     No-op without a UIDPLUS append UID (uid <= 0); in that case a future Sent sync
@@ -64,9 +62,7 @@ async def _persist_sent_copy(
         return
     async with session_scope() as session:
         sent_mb = await session.scalar(
-            select(Mailbox).where(
-                Mailbox.account_id == account.id, Mailbox.special_use == "sent"
-            )
+            select(Mailbox).where(Mailbox.account_id == account.id, Mailbox.special_use == "sent")
         )
         if sent_mb is None:
             return
@@ -99,9 +95,7 @@ async def _persist_sent_copy(
         },
     )
     if thread_id is not None:
-        await bus.publish(
-            "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}
-        )
+        await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
 
 
 async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:

backend/app/queue.py

@@ -1,14 +1,33 @@
-"""arq queue helpers (enqueue side, used by the API)."""
+"""arq queue helpers (enqueue side, used by the API).
+
+Two named queues (refactor §6): ``interactive`` (fast user actions) and ``bulk``
+(backfills). Enqueue via :func:`enqueue_job`, which routes each job to its queue —
+the API never needs to know queue names.
+"""
 
 from __future__ import annotations
 
+from typing import Any
+
 from arq import create_pool
 from arq.connections import ArqRedis, RedisSettings
+from arq.jobs import Job
 
 from app.config import get_settings
 
 _pool: ArqRedis | None = None
 
+QUEUE_INTERACTIVE = "interactive"
+QUEUE_BULK = "bulk"
+
+# Jobs default to the interactive queue; only bulk backfills go to the bulk queue.
+_BULK_JOBS = frozenset({"sync_bootstrap_account"})
+
+
+def queue_for(job_name: str) -> str:
+    """Return the queue a job should run on."""
+    return QUEUE_BULK if job_name in _BULK_JOBS else QUEUE_INTERACTIVE
+
 
 def redis_settings() -> RedisSettings:
     """Return arq RedisSettings derived from ``REDIS_URL``."""
@@ -23,6 +42,13 @@ async def get_arq_pool() -> ArqRedis:
     return _pool
 
 
+async def enqueue_job(name: str, *args: Any, **kwargs: Any) -> Job | None:
+    """Enqueue ``name`` onto its routed queue (interactive/bulk)."""
+    pool = await get_arq_pool()
+    kwargs.setdefault("_queue_name", queue_for(name))
+    return await pool.enqueue_job(name, *args, **kwargs)
+
+
 async def close_arq_pool() -> None:
     """Close the arq pool."""
     global _pool

backend/app/sync/actions.py

@@ -273,9 +273,7 @@ async def delete_thread_job(ctx: dict[str, Any], thread_id: str) -> None:
                 .where(Draft.reply_to_message_id.in_(msg_ids))
                 .values(reply_to_message_id=None)
             )
-        await session.execute(
-            update(Draft).where(Draft.thread_id == tid).values(thread_id=None)
-        )
+        await session.execute(update(Draft).where(Draft.thread_id == tid).values(thread_id=None))
         for m in msgs:
             await session.delete(m)
         thread = await session.get(Thread, tid)