lukas/webmail-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync-v2 step 5: activity tiers (hot/warm/cold)

Browse Source 
Each owned account is tiered by the coordinator; the runner shapes its work to
match — correctness identical, only latency/socket cost differ.

- hot: persistent INBOX IDLE + secondary poll every POLL_WARM.
- warm: no persistent socket; transient INBOX+special poll every POLL_WARM.
- cold: same transient poll every POLL_COLD.
- engine: poll_all_loop (transient full poll); secondary_poll_loop interval is
  now a parameter; POLL_INTERVAL removed in favor of POLL_WARM/POLL_COLD.
- coordinator: tier eval every 120s from session heat (sess:active:{user}) +
  recent mail (max internal_date within TIER_HOT_ACTIVITY_H), with the
  HOT_BUDGET_PER_WORKER guard shedding least-recently-active session-less hot
  accounts; user.active event promotes a user's accounts to hot at once (≤5s).
- API: login + WS connect/ping publish user.active and refresh sess:active
  (TTL TIER_SESSION_GRACE_MIN). New sync:tier:{id} key, default warm on claim.
- config: POLL_WARM/POLL_COLD/TIER_*/HOT_BUDGET_PER_WORKER.
- tests: 5 tier tests (session/recent-mail promotion, cold, user.active,
  budget demotion) alongside the lease/lifecycle acceptance tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
lukas
2026-07-19 07:03:54 +02:00
co-authored by Claude Opus 4.8
parent 245672d3f2
commit 19335116da
9 changed files with 365 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sync/coordinator.py
+103 -7
View File
@@ -25,13 +25,14 @@ import secrets
import socket
import time
import uuid
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy import func, select
from app.config import get_settings
from app.db import session_scope
from app.events import channel_for
from app.models import Account
from app.models import Account, Message
from app.redis_client import aw, get_redis
from app.sync import leases
from app.sync.runner import AccountRunner
@@ -65,6 +66,11 @@ class SyncCoordinator:
self.heartbeat_ttl = 30
self.heartbeat_refresh = 10.0
self.rebalance_interval = 60.0
self.tier_eval_interval = 120.0
self.session_grace_s = s.tier_session_grace_min * 60
self.hot_activity = timedelta(hours=s.tier_hot_activity_h)
self.cold_after = timedelta(days=s.tier_cold_after_d)
self.hot_budget = s.hot_budget_per_worker
# --- lifecycle -----------------------------------------------------------
@@ -80,6 +86,7 @@ class SyncCoordinator:
asyncio.create_task(self._loop(self._reconcile_once, self.reconcile_interval)),
asyncio.create_task(self._loop(self._heartbeat_once, self.heartbeat_refresh)),
asyncio.create_task(self._loop(self._rebalance_once, self.rebalance_interval)),
asyncio.create_task(self._loop(self._evaluate_tiers_once, self.tier_eval_interval)),
asyncio.create_task(self._event_subscriber()),
]
logger.info("sync coordinator %s started", self.worker_id)
@@ -131,7 +138,7 @@ class SyncCoordinator:
if account_id in self._runners:
continue
if await leases.claim(self._redis, account_id, self.worker_id, self.lease_ttl):
self._start_runner(account_id)
await self._start_runner(account_id)
async def _renew_once(self) -> None:
for account_id in list(self._runners):
@@ -160,8 +167,11 @@ class SyncCoordinator:
if runner is not None:
await runner.stop()
def _start_runner(self, account_id: uuid.UUID) -> None:
runner = AccountRunner(account_id)
async def _start_runner(self, account_id: uuid.UUID) -> None:
# Inherit any tier a previous owner recorded; default warm on first claim.
tier = await aw(self._redis.get(leases.tier_key(account_id))) or "warm"
await aw(self._redis.set(leases.tier_key(account_id), tier))
runner = AccountRunner(account_id, tier)
runner.start()
self._runners[account_id] = runner
@@ -225,7 +235,9 @@ class SyncCoordinator:
the lease to lapse.
"""
pubsub = self._redis.pubsub()
await pubsub.subscribe(channel_for("account.state_changed"))
state_ch = channel_for("account.state_changed")
active_ch = channel_for("user.active")
await pubsub.subscribe(state_ch, active_ch)
try:
async for message in pubsub.listen():
if self._stop.is_set():
@@ -236,7 +248,10 @@ class SyncCoordinator:
payload = json.loads(message["data"]).get("payload", {})
except (ValueError, TypeError):
continue
await self._on_state_changed(payload)
if message.get("channel") == active_ch:
await self._on_user_active(payload)
else:
await self._on_state_changed(payload)
finally:
with contextlib.suppress(Exception):
await pubsub.aclose() # type: ignore[no-untyped-call]
@@ -250,3 +265,84 @@ class SyncCoordinator:
await leases.mark_live(self._redis, account_id) # claimable next cycle
elif account_id in self._runners:
await self._release(account_id) # disabled/deleted/error -> stop now
async def _on_user_active(self, payload: dict[str, object]) -> None:
"""Promote all this user's owned accounts to hot at once (≤5s, §5)."""
raw = payload.get("user_id")
if not isinstance(raw, str):
return
user_id = uuid.UUID(raw)
owned = list(self._runners)
if not owned:
return
async with session_scope() as session:
aids = list(
await session.scalars(
select(Account.id).where(Account.id.in_(owned), Account.user_id == user_id)
)
)
for account_id in aids:
await self._set_tier(account_id, "hot")
# --- tier evaluation (§5) ------------------------------------------------
async def _set_tier(self, account_id: uuid.UUID, tier: str) -> None:
runner = self._runners.get(account_id)
if runner is None:
return
await aw(self._redis.set(leases.tier_key(account_id), tier))
await runner.apply_tier(tier)
async def _account_activity(
self, ids: list[uuid.UUID]
) -> dict[uuid.UUID, tuple[uuid.UUID, datetime | None]]:
"""Return {account_id: (user_id, last_mail_at)} for the owned accounts."""
async with session_scope() as session:
rows = (
await session.execute(
select(Account.id, Account.user_id, func.max(Message.internal_date))
.select_from(Account)
.outerjoin(Message, Message.account_id == Account.id)
.where(Account.id.in_(ids))
.group_by(Account.id, Account.user_id)
)
).all()
return {row[0]: (row[1], row[2]) for row in rows}
async def _evaluate_tiers_once(self) -> None:
"""Recompute each owned account's tier from session heat + recent mail."""
owned = list(self._runners)
if not owned:
return
activity = await self._account_activity(owned)
now = datetime.now(UTC)
sessions: dict[uuid.UUID, bool] = {}
desired: dict[uuid.UUID, str] = {}
demotable: list[tuple[datetime, uuid.UUID]] = [] # activity-hot, no session
hot_count = 0
for account_id in owned:
user_id, last_mail = activity.get(account_id, (None, None))
session_active = False
if user_id is not None:
if user_id not in sessions:
sessions[user_id] = bool(
await aw(self._redis.exists(leases.session_key(user_id)))
)
session_active = sessions[user_id]
recent = last_mail is not None and (now - last_mail) < self.hot_activity
if session_active or recent:
desired[account_id] = "hot"
hot_count += 1
if not session_active and last_mail is not None:
demotable.append((last_mail, account_id))
elif last_mail is not None and (now - last_mail) < self.cold_after:
desired[account_id] = "warm"
else:
desired[account_id] = "cold"
# Global hot budget: shed the least-recently-active session-less hot accounts.
if hot_count > self.hot_budget:
demotable.sort(key=lambda item: item[0]) # oldest activity first
for _, account_id in demotable[: hot_count - self.hot_budget]:
desired[account_id] = "warm"
for account_id, tier in desired.items():
await self._set_tier(account_id, tier)