diff --git a/backend/loadtest/soak_sync.py b/backend/loadtest/soak_sync.py index 2af0bad..80f73da 100644 --- a/backend/loadtest/soak_sync.py +++ b/backend/loadtest/soak_sync.py @@ -193,10 +193,11 @@ class Sample: max_lag: float rss_mb: float synced: int # accounts that have completed >=1 poll (steady-state gate) + dead: int # owned runners whose work loops have died (zombies) -async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, int]: - owned = hot = warm = cold = 0 +async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, int, int]: + owned = hot = warm = cold = dead = 0 for c in coords: for runner in list(c._runners.values()): owned += 1 @@ -206,7 +207,9 @@ async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, in warm += 1 else: cold += 1 - return owned, hot, warm, cold + if not runner.running(): # owned but its work loops have died = zombie + dead += 1 + return owned, hot, warm, cold, dead def _max_lag(now: float, seeded: int, start: float) -> float: @@ -250,8 +253,15 @@ async def _soak(cfg: Config) -> int: await refresh_sessions() + # Size the per-worker cap so total capacity is just above the fleet: claims + # fill each worker roughly evenly and the rebalancer barely has to shed + # (mirrors how you'd tune MAX_ACCOUNTS_PER_WORKER in a real deployment). + import math + + cap = max(50, math.ceil(cfg.accounts / cfg.workers * 1.2)) coords = [SyncCoordinator(f"soak-w{i}") for i in range(cfg.workers)] for c in coords: + c.max_accounts = cap c.claim_interval = cfg.claim_s c.claim_jitter = 1.0 c.tier_eval_interval = cfg.tier_eval_s @@ -272,7 +282,7 @@ async def _soak(cfg: Config) -> int: ) print( f"{'t':>6} {'owned':>6} {'hot':>5} {'warm':>5} {'cold':>5} " - f"{'persist':>8} {'open':>5} {'lag_s':>7} {'rss_mb':>8}", + f"{'persist':>8} {'open':>5} {'lag_s':>7} {'rss_mb':>8} {'dead':>5}", flush=True, ) try: @@ -282,7 +292,7 @@ async def _soak(cfg: Config) -> int: if now >= next_session_refresh: await refresh_sessions() next_session_refresh = now + cfg.session_grace_s / 2 - owned, hot, warm, cold = await _tier_counts(coords) + owned, hot, warm, cold, dead = await _tier_counts(coords) s = Sample( t=now - start, owned=owned, @@ -294,11 +304,13 @@ async def _soak(cfg: Config) -> int: max_lag=_max_lag(now, cfg.accounts, start), rss_mb=_rss_mb(), synced=len(MockStats.last_sync), + dead=dead, ) samples.append(s) print( f"{s.t:6.0f} {s.owned:6d} {s.hot:5d} {s.warm:5d} {s.cold:5d} " - f"{s.persistent:8d} {s.total_open:5d} {s.max_lag:7.0f} {s.rss_mb:8.1f}", + f"{s.persistent:8d} {s.total_open:5d} {s.max_lag:7.0f} {s.rss_mb:8.1f} " + f"{s.dead:5d}", flush=True, ) finally: