From ee4dd99ad19b3be22c24ddff156aff94fedf5f2a Mon Sep 17 00:00:00 2001
From: Lukas Remis <lukas.remis@gmail.com>
Date: Sun, 19 Jul 2026 08:25:03 +0200
Subject: [PATCH] =?UTF-8?q?sync-v2=20=C2=A710.7:=20soak=20stability=20?=
 =?UTF-8?q?=E2=80=94=20even=20per-worker=20cap=20+=20zombie=20diagnostic?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Size MAX_ACCOUNTS_PER_WORKER to ~1.2x the fair share so the fleet fills each
  worker evenly at startup and the rebalancer barely sheds; without this, three
  coordinators racing for the whole fleet shed/reclaim in a thrash that reset
  reclaimed accounts' first-poll clocks and dragged the sync-lag ramp.
- Sample a "dead" count (owned runners whose work loops have stopped) to
  distinguish a real zombie-runner bug from a slow ramp. Validated: dead=0
  throughout; 400-account run holds lag ~48s under a 52s budget across 13 steady
  samples, persistent sockets == hot, RSS flat.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 backend/loadtest/soak_sync.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/backend/loadtest/soak_sync.py b/backend/loadtest/soak_sync.py
index 2af0bad..80f73da 100644
--- a/backend/loadtest/soak_sync.py
+++ b/backend/loadtest/soak_sync.py
@@ -193,10 +193,11 @@ class Sample:
     max_lag: float
     rss_mb: float
     synced: int  # accounts that have completed >=1 poll (steady-state gate)
+    dead: int  # owned runners whose work loops have died (zombies)
 
 
-async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, int]:
-    owned = hot = warm = cold = 0
+async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, int, int]:
+    owned = hot = warm = cold = dead = 0
     for c in coords:
         for runner in list(c._runners.values()):
             owned += 1
@@ -206,7 +207,9 @@ async def _tier_counts(coords: list[SyncCoordinator]) -> tuple[int, int, int, in
                 warm += 1
             else:
                 cold += 1
-    return owned, hot, warm, cold
+            if not runner.running():  # owned but its work loops have died = zombie
+                dead += 1
+    return owned, hot, warm, cold, dead
 
 
 def _max_lag(now: float, seeded: int, start: float) -> float:
@@ -250,8 +253,15 @@ async def _soak(cfg: Config) -> int:
 
     await refresh_sessions()
 
+    # Size the per-worker cap so total capacity is just above the fleet: claims
+    # fill each worker roughly evenly and the rebalancer barely has to shed
+    # (mirrors how you'd tune MAX_ACCOUNTS_PER_WORKER in a real deployment).
+    import math
+
+    cap = max(50, math.ceil(cfg.accounts / cfg.workers * 1.2))
     coords = [SyncCoordinator(f"soak-w{i}") for i in range(cfg.workers)]
     for c in coords:
+        c.max_accounts = cap
         c.claim_interval = cfg.claim_s
         c.claim_jitter = 1.0
         c.tier_eval_interval = cfg.tier_eval_s
@@ -272,7 +282,7 @@ async def _soak(cfg: Config) -> int:
     )
     print(
         f"{'t':>6} {'owned':>6} {'hot':>5} {'warm':>5} {'cold':>5} "
-        f"{'persist':>8} {'open':>5} {'lag_s':>7} {'rss_mb':>8}",
+        f"{'persist':>8} {'open':>5} {'lag_s':>7} {'rss_mb':>8} {'dead':>5}",
         flush=True,
     )
     try:
@@ -282,7 +292,7 @@ async def _soak(cfg: Config) -> int:
             if now >= next_session_refresh:
                 await refresh_sessions()
                 next_session_refresh = now + cfg.session_grace_s / 2
-            owned, hot, warm, cold = await _tier_counts(coords)
+            owned, hot, warm, cold, dead = await _tier_counts(coords)
             s = Sample(
                 t=now - start,
                 owned=owned,
@@ -294,11 +304,13 @@ async def _soak(cfg: Config) -> int:
                 max_lag=_max_lag(now, cfg.accounts, start),
                 rss_mb=_rss_mb(),
                 synced=len(MockStats.last_sync),
+                dead=dead,
             )
             samples.append(s)
             print(
                 f"{s.t:6.0f} {s.owned:6d} {s.hot:5d} {s.warm:5d} {s.cold:5d} "
-                f"{s.persistent:8d} {s.total_open:5d} {s.max_lag:7.0f} {s.rss_mb:8.1f}",
+                f"{s.persistent:8d} {s.total_open:5d} {s.max_lag:7.0f} {s.rss_mb:8.1f} "
+                f"{s.dead:5d}",
                 flush=True,
             )
     finally: