sync-v2 step 1: worker roles + interactive/bulk queue split
- config: WORKER_ROLE (sync|jobs|all), INTERACTIVE_CONCURRENCY, BULK_CONCURRENCY. - queue: named queues (interactive/bulk) + enqueue_job() router; sync_bootstrap goes to bulk, all other jobs to interactive. - workers.run: single entrypoint; role=jobs runs both arq consumers (per-queue timeout/concurrency), role=sync runs the supervisor, all=both (dev default). - docker-compose: worker-sync + worker-jobs services (shared image, nofile ulimit); backend/Dockerfile; Makefile worker/worker-sync/worker-jobs targets. - tests repointed to app.queue.get_arq_pool; supervisor behavior unchanged. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,7 @@ from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app import queue as app_queue
|
||||
from app.api import accounts as accounts_api
|
||||
from app.config import get_settings
|
||||
from app.identity.passwords import hash_password
|
||||
@@ -114,14 +115,14 @@ async def test_create_account_enqueues_sync(ctx: Ctx, monkeypatch) -> None: # t
|
||||
enqueued: list[tuple[str, str]] = []
|
||||
|
||||
class FakePool:
|
||||
async def enqueue_job(self, name: str, arg: str) -> None:
|
||||
async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
|
||||
enqueued.append((name, arg))
|
||||
|
||||
async def fake_pool() -> FakePool:
|
||||
return FakePool()
|
||||
|
||||
monkeypatch.setattr(accounts_api, "test_account", fake_test)
|
||||
monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
|
||||
monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
|
||||
|
||||
resp = await ctx.client.post(
|
||||
"/api/accounts",
|
||||
@@ -151,14 +152,14 @@ async def _make_account(ctx: Ctx, monkeypatch) -> str: # type: ignore[no-untype
|
||||
return None
|
||||
|
||||
class FakePool:
|
||||
async def enqueue_job(self, name: str, arg: str) -> None:
|
||||
async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
async def fake_pool() -> FakePool:
|
||||
return FakePool()
|
||||
|
||||
monkeypatch.setattr(accounts_api, "test_account", fake_test)
|
||||
monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
|
||||
monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
|
||||
resp = await ctx.client.post(
|
||||
"/api/accounts",
|
||||
json={**GOOD_CFG, "email": "me@example.com", "display_name": "Me", "password": "secret"},
|
||||
@@ -174,13 +175,13 @@ async def test_sync_all_enqueues_refresh_per_account(ctx: Ctx, monkeypatch) -> N
|
||||
enqueued: list[tuple[str, str]] = []
|
||||
|
||||
class FakePool:
|
||||
async def enqueue_job(self, name: str, arg: str) -> None:
|
||||
async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
|
||||
enqueued.append((name, arg))
|
||||
|
||||
async def fake_pool() -> FakePool:
|
||||
return FakePool()
|
||||
|
||||
monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
|
||||
monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
|
||||
resp = await ctx.client.post("/api/accounts/sync", headers=_hdr())
|
||||
assert resp.status_code == 200
|
||||
assert enqueued == [("refresh_account_job", aid)]
|
||||
@@ -221,7 +222,7 @@ async def test_update_password_reencrypts_and_resyncs(ctx: Ctx, monkeypatch) ->
|
||||
enqueued: list[tuple[str, str]] = []
|
||||
|
||||
class FakePool:
|
||||
async def enqueue_job(self, name: str, arg: str) -> None:
|
||||
async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
|
||||
enqueued.append((name, arg))
|
||||
|
||||
async def fake_pool() -> FakePool:
|
||||
@@ -231,7 +232,7 @@ async def test_update_password_reencrypts_and_resyncs(ctx: Ctx, monkeypatch) ->
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(accounts_api, "test_account", fake_test)
|
||||
monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
|
||||
monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
|
||||
|
||||
resp = await ctx.client.patch(
|
||||
f"/api/accounts/{aid}", json={"password": "newsecret"}, headers=_hdr()
|
||||
@@ -259,14 +260,14 @@ async def test_delete_account(ctx: Ctx, monkeypatch) -> None: # type: ignore[no
|
||||
return None
|
||||
|
||||
class FakePool:
|
||||
async def enqueue_job(self, name: str, arg: str) -> None:
|
||||
async def enqueue_job(self, name: str, arg: str, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
async def fake_pool() -> FakePool:
|
||||
return FakePool()
|
||||
|
||||
monkeypatch.setattr(accounts_api, "test_account", fake_test)
|
||||
monkeypatch.setattr(accounts_api, "get_arq_pool", fake_pool)
|
||||
monkeypatch.setattr(app_queue, "get_arq_pool", fake_pool)
|
||||
|
||||
created = await ctx.client.post(
|
||||
"/api/accounts",
|
||||
|
||||
Reference in New Issue
Block a user