sync-v2 §10.7: mock-provider soak/load harness + poll jitter
- loadtest/soak_sync.py + mock_provider.py: fully isolated soak for acceptance criterion 7. Creates a throwaway webmail_soak DB + uses Redis DB 14 (both torn down), simulates N accounts (default 1000, 20% active sessions) across 3 in-process coordinators with a mock MailProvider (no real IMAP). Asserts: all accounts owned, persistent sockets <= hot*1.05, oldest lag < POLL_COLD*1.15, RSS stable. Steady state = after every account has completed its first sync. Configurable via SOAK_* / POLL_* env for fast smoke runs. - engine: poll_all_loop and secondary_poll_loop now jitter their waits (first at a random point in the interval, later +/-10%) so a fleet claimed together doesn't poll in lockstep — anti-thundering-herd, a real improvement surfaced by the soak. Validated: 60-account smoke passes all four criteria; full gate green (238 tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import random
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
@@ -353,10 +354,15 @@ async def secondary_poll_loop(
|
||||
"""
|
||||
stop = stop or asyncio.Event()
|
||||
period = interval if interval is not None else get_settings().poll_warm
|
||||
first = True
|
||||
while not stop.is_set():
|
||||
# Wait first: the initial state was already covered by the backfill.
|
||||
# Wait first (initial state was covered by backfill). Spread the fleet: the
|
||||
# first wait is a random point in the period, later waits jitter ±10%, so
|
||||
# accounts claimed together don't poll in lockstep (thundering herd).
|
||||
wait = random.uniform(0, period) if first else period * random.uniform(0.9, 1.1)
|
||||
first = False
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(stop.wait(), timeout=period)
|
||||
await asyncio.wait_for(stop.wait(), timeout=wait)
|
||||
if stop.is_set():
|
||||
break
|
||||
await _poll_secondary_once(account_id)
|
||||
@@ -371,9 +377,14 @@ async def poll_all_loop(
|
||||
is held between passes. Correctness matches the hot tier — only latency differs.
|
||||
"""
|
||||
stop = stop or asyncio.Event()
|
||||
first = True
|
||||
while not stop.is_set():
|
||||
# Spread the fleet: first poll at a random point in the interval, then
|
||||
# jitter ±10% so a batch claimed together doesn't poll in lockstep.
|
||||
wait = random.uniform(0, interval) if first else interval * random.uniform(0.9, 1.1)
|
||||
first = False
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(stop.wait(), timeout=interval)
|
||||
await asyncio.wait_for(stop.wait(), timeout=wait)
|
||||
if stop.is_set():
|
||||
break
|
||||
await poll_folders_once(account_id, include_inbox=True)
|
||||
|
||||
Reference in New Issue
Block a user