sync-v2 step 5: activity tiers (hot/warm/cold)

Each owned account is tiered by the coordinator; the runner shapes its work to match — correctness identical, only latency/socket cost differ. - hot: persistent INBOX IDLE + secondary poll every POLL_WARM. - warm: no persistent socket; transient INBOX+special poll every POLL_WARM. - cold: same transient poll every POLL_COLD. - engine: poll_all_loop (transient full poll); secondary_poll_loop interval is now a parameter; POLL_INTERVAL removed in favor of POLL_WARM/POLL_COLD. - coordinator: tier eval every 120s from session heat (sess:active:{user}) + recent mail (max internal_date within TIER_HOT_ACTIVITY_H), with the HOT_BUDGET_PER_WORKER guard shedding least-recently-active session-less hot accounts; user.active event promotes a user's accounts to hot at once (≤5s). - API: login + WS connect/ping publish user.active and refresh sess:active (TTL TIER_SESSION_GRACE_MIN). New sync:tier:{id} key, default warm on claim. - config: POLL_WARM/POLL_COLD/TIER_*/HOT_BUDGET_PER_WORKER. - tests: 5 tier tests (session/recent-mail promotion, cold, user.active, budget demotion) alongside the lease/lifecycle acceptance tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-19 07:03:54 +02:00
parent 245672d3f2
commit 19335116da
9 changed files with 365 additions and 47 deletions
@@ -19,16 +19,18 @@ import asyncio
 import uuid
 from collections import defaultdict
 from collections.abc import AsyncIterator, Callable
+from datetime import UTC, datetime, timedelta

 import pytest_asyncio
-from sqlalchemy import delete
+from sqlalchemy import delete, select
 from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
 from sqlalchemy.pool import NullPool

 from app.config import get_settings
+from app.db import session_scope
 from app.events import get_event_bus
 from app.identity.passwords import hash_password
-from app.models import Account, User
+from app.models import Account, Mailbox, Message, User
 from app.redis_client import get_redis
 from app.sync import coordinator as coordinator_mod
 from app.sync import leases, runner
@@ -39,22 +41,31 @@ from app.sync.coordinator import SyncCoordinator

@pytest_asyncio.fixture
 async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]:  # type: ignore[no-untyped-def]
-    """Patch the runner's loops; yield a live-connection count per account."""
+    """Patch the runner's loops; yield a live (hot-tier IDLE) connection count."""
    conns: dict[uuid.UUID, int] = defaultdict(int)

    async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None:
-        conns[account_id] += 1  # one IDLE connection while the loop is alive
+        conns[account_id] += 1  # one persistent IDLE connection while alive (hot)
        try:
            await stop.wait()
        finally:
            conns[account_id] -= 1

-    async def fake_secondary(account_id: uuid.UUID, stop: asyncio.Event | None = None) -> None:
+    async def fake_secondary(
+        account_id: uuid.UUID, stop: asyncio.Event | None = None, interval: float | None = None
+    ) -> None:
        assert stop is not None
        await stop.wait()

+    async def fake_poll_all(
+        account_id: uuid.UUID, interval: float, stop: asyncio.Event | None = None
+    ) -> None:
+        assert stop is not None  # warm/cold: no persistent connection, just wait
+        await stop.wait()
+
    monkeypatch.setattr(runner, "live_loop", fake_live_loop)
    monkeypatch.setattr(runner, "secondary_poll_loop", fake_secondary)
+    monkeypatch.setattr(runner, "poll_all_loop", fake_poll_all)
    yield conns


@@ -119,13 +130,44 @@ def _fast(c: SyncCoordinator) -> SyncCoordinator:
    c.reconcile_interval = 1000.0
    c.heartbeat_refresh = 1000.0
    c.rebalance_interval = 1000.0
+    c.tier_eval_interval = 1000.0  # lease tests pin the tier; no eval churn
    c._reconcile_once = _no_reconcile  # type: ignore[method-assign]
    return c


-async def _seed(ids: set[uuid.UUID]) -> None:
+async def _seed(ids: set[uuid.UUID], tier: str = "hot") -> None:
+    """Register accounts as live and pin their tier (hot => persistent IDLE)."""
+    redis = get_redis()
    for account_id in ids:
-        await leases.mark_live(get_redis(), account_id)
+        await leases.mark_live(redis, account_id)
+        await redis.set(leases.tier_key(account_id), tier)
+
+
+async def _user_of(account_id: uuid.UUID) -> uuid.UUID:
+    async with session_scope() as session:
+        user_id = await session.scalar(select(Account.user_id).where(Account.id == account_id))
+    assert user_id is not None
+    return user_id
+
+
+async def _add_recent_mail(account_id: uuid.UUID, when: datetime) -> None:
+    """Insert one INBOX message dated ``when`` so the account reads as active."""
+    user_id = await _user_of(account_id)
+    async with session_scope() as session:
+        mailbox = Mailbox(account_id=account_id, user_id=user_id, path="INBOX", special_use="inbox")
+        session.add(mailbox)
+        await session.flush()
+        session.add(
+            Message(
+                user_id=user_id,
+                account_id=account_id,
+                mailbox_id=mailbox.id,
+                uid=1,
+                internal_date=when,
+                blob_sha256="0" * 64,
+            )
+        )
+        await session.commit()


 async def _until(pred: Callable[[], bool], timeout_s: float = 6.0) -> None:
@@ -210,6 +252,7 @@ async def test_acceptance_3_create_delete(connections, make_accounts) -> None:
        # A newly-live account (as the bootstrap job's _set_state('live') would do):
        # register it in sync:live; the running coordinator must claim it, no restart.
        (aid,) = await make_accounts(1)
+        await get_redis().set(leases.tier_key(aid), "hot")  # persistent IDLE
        await leases.mark_live(get_redis(), aid)
        await _until(lambda: aid in c.owned)
        assert connections[aid] == 1
@@ -254,5 +297,90 @@ async def test_event_driven_teardown(connections, make_accounts) -> None:  # typ
        await c.stop()


+# --- §5 activity tiers -------------------------------------------------------
+
+
+async def test_tier_session_promotes_to_hot(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    (aid,) = await make_accounts(1)
+    user_id = await _user_of(aid)
+    await leases.mark_live(get_redis(), aid)  # claimed as warm by default
+    await leases.mark_user_active(get_redis(), user_id, 300)  # active session -> hot
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        await _until(lambda: connections[aid] == 1)  # hot => persistent IDLE
+        assert c._runners[aid].tier == "hot"
+    finally:
+        await c.stop()
+
+
+async def test_tier_no_session_no_mail_goes_cold(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    (aid,) = await make_accounts(1)
+    await leases.mark_live(get_redis(), aid)  # no session, no mail
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        await _until(lambda: c._runners[aid].tier == "cold")
+        assert connections[aid] == 0  # cold holds no persistent connection
+    finally:
+        await c.stop()
+
+
+async def test_tier_recent_mail_promotes_to_hot(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    (aid,) = await make_accounts(1)
+    await _add_recent_mail(aid, datetime.now(UTC))  # activity within TIER_HOT_ACTIVITY_H
+    await leases.mark_live(get_redis(), aid)
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    await c.start()
+    try:
+        await _until(lambda: aid in c.owned)
+        await _until(lambda: connections[aid] == 1)  # recent mail => hot
+        assert c._runners[aid].tier == "hot"
+    finally:
+        await c.stop()
+
+
+async def test_user_active_event_promotes_immediately(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    aid0, aid1 = await make_accounts(2)  # both under one user
+    user_id = await _user_of(aid0)
+    await leases.mark_live(get_redis(), aid0)
+    await leases.mark_live(get_redis(), aid1)
+    c = _fast(SyncCoordinator("w1"))  # tier eval disabled: only the event can promote
+    await c.start()
+    try:
+        await _until(lambda: {aid0, aid1} <= c.owned)
+        assert connections[aid0] == 0 and connections[aid1] == 0  # warm to start
+        await get_event_bus().publish("user.active", {"user_id": str(user_id)})
+        await _until(lambda: connections[aid0] == 1 and connections[aid1] == 1, timeout_s=3.0)
+    finally:
+        await c.stop()
+
+
+async def test_hot_budget_demotes_least_active(connections, make_accounts) -> None:  # type: ignore[no-untyped-def]
+    aid_old, aid_new = await make_accounts(2)  # activity-hot, same user, NO session
+    now = datetime.now(UTC)
+    await _add_recent_mail(aid_old, now - timedelta(hours=1))  # least recently active
+    await _add_recent_mail(aid_new, now)
+    await leases.mark_live(get_redis(), aid_old)
+    await leases.mark_live(get_redis(), aid_new)
+    c = _fast(SyncCoordinator("w1"))
+    c.tier_eval_interval = 0.1
+    c.hot_budget = 1  # only one may be hot
+    await c.start()
+    try:
+        await _until(lambda: {aid_old, aid_new} <= c.owned)
+        # The newer account stays hot; the older is shed to warm by the budget.
+        await _until(lambda: c._runners[aid_new].tier == "hot")
+        await _until(lambda: c._runners[aid_old].tier == "warm")
+        assert connections[aid_new] == 1 and connections[aid_old] == 0
+    finally:
+        await c.stop()
+
+
 def test_worker_id_is_unique() -> None:
    assert coordinator_mod._new_worker_id() != coordinator_mod._new_worker_id()