sync-v2 step 1: worker roles + interactive/bulk queue split
- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY. - queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap goes to bulk, all other jobs to interactive. - workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue timeout/concurrency), role=sync runs the supervisor, all=both (dev default). - docker-compose: worker-sync + worker-jobs services (shared image, nofile ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets. - tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -30,7 +30,7 @@ from app.api.schemas import AccountOut
|
||||
from app.db import get_session
|
||||
from app.identity.deps import CurrentUser, require_csrf
|
||||
from app.models import Account
|
||||
from app.queue import get_arq_pool
|
||||
from app.queue import enqueue_job
|
||||
from app.redis_client import get_redis
|
||||
from app.security.crypto import encrypt_secret
|
||||
|
||||
@@ -91,9 +91,8 @@ async def list_accounts(user: CurrentUser, db: DbSession) -> list[AccountOut]:
|
||||
async def sync_all_accounts(user: CurrentUser, db: DbSession) -> dict[str, str]:
|
||||
"""Poll every folder of each of the user's accounts (manual Refresh)."""
|
||||
account_ids = list(await db.scalars(select(Account.id).where(Account.user_id == user.id)))
|
||||
pool = await get_arq_pool()
|
||||
for account_id in account_ids:
|
||||
await pool.enqueue_job("refresh_account_job", str(account_id))
|
||||
await enqueue_job("refresh_account_job", str(account_id))
|
||||
return {"status": "queued"}
|
||||
|
||||
|
||||
@@ -145,8 +144,7 @@ async def create_new_account(
|
||||
raise_api_error(409, "account_exists", "an account for this email already exists")
|
||||
await db.commit()
|
||||
|
||||
pool = await get_arq_pool()
|
||||
await pool.enqueue_job("sync_bootstrap_account", str(account.id))
|
||||
await enqueue_job("sync_bootstrap_account", str(account.id))
|
||||
|
||||
return _detail(account)
|
||||
|
||||
@@ -227,8 +225,7 @@ async def update_account(
|
||||
await db.commit()
|
||||
|
||||
if conn_change:
|
||||
pool = await get_arq_pool()
|
||||
await pool.enqueue_job("sync_bootstrap_account", str(account.id))
|
||||
await enqueue_job("sync_bootstrap_account", str(account.id))
|
||||
|
||||
return _detail(account)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user