sync-v2 §10.7: mail-injecting mode for the soak harness

Optional SOAK_MAIL=1 makes the mock deliver synthetic messages (unique UIDs,
well-formed RFC822) on INBOX polls, so the full parse -> blob -> DB -> index path
actually runs and CPU/RSS move under real throughput. Tunables: SOAK_MAIL_PER_POLL,
SOAK_MSG_KB, SOAK_BIG_MSG_PCT (percent above STREAM_PARSE_MB -> streaming/§8 path).
Blob store + search backend are isolated (temp BLOB_ROOT, SEARCH_BACKEND=pg) and
torn down. Output gains msgs + msg/s columns and a delivered/peak-RSS summary.

RSS check is now a trend (early-third vs late-third mean, not min-to-max) so it
detects a real leak without flaking on transient large-message parse spikes.

Validated: mail run delivers ~4.2 msg/s (incl. 5% 10MB messages); RSS oscillates
123-267MB but its mean is flat (+3.7% over the run) -> no leak, parse memory
bound holds (§8); tier re-promotion on new mail exercised; all §10.7 checks pass.
Full gate green (238 tests).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-19 09:28:48 +02:00
co-authored by Claude Opus 4.8
parent ee4dd99ad1
commit 3d962a183c
2 changed files with 115 additions and 10 deletions
+50 -1
View File
@@ -16,9 +16,13 @@ so the harness can compute the oldest sync lag across all accounts.
from __future__ import annotations
import asyncio
import random
import time
import uuid
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from email.message import EmailMessage
from email.utils import format_datetime
from app.providers.base import MailboxDelta, RemoteMailbox, RemoteMessage
@@ -28,11 +32,40 @@ class MockStats:
persistent_open = 0
last_sync: dict[uuid.UUID, float] = {}
# --- optional mail injection (exercises parse/blob/DB/index under load) ----
mail_enabled = False
mail_per_poll = 0.0 # expected new INBOX messages per poll (fractional => prob)
msg_bytes = 10_000
big_pct = 0 # percent of messages that exceed STREAM_PARSE_MB (streaming path)
big_bytes = 10_000_000
delivered = 0 # total synthetic messages handed to the sync engine
_uid: dict[uuid.UUID, int] = {}
@classmethod
def reset(cls) -> None:
cls.open = 0
cls.persistent_open = 0
cls.last_sync = {}
cls.delivered = 0
cls._uid = {}
def _make_raw(size_bytes: int) -> bytes:
"""A unique, well-formed RFC822 message padded to roughly ``size_bytes``."""
msg = EmailMessage()
msg["Subject"] = f"soak {uuid.uuid4().hex[:12]}"
msg["From"] = "sender@load.local"
msg["To"] = "rcpt@load.local"
msg["Message-ID"] = f"<{uuid.uuid4().hex}@load.local>"
msg["Date"] = format_datetime(datetime.now(UTC))
msg.set_content("x" * max(0, size_bytes - 256))
return msg.as_bytes()
def _draw_count() -> int:
n = int(MockStats.mail_per_poll)
frac = MockStats.mail_per_poll - n
return n + (1 if random.random() < frac else 0)
class MockProvider:
@@ -61,8 +94,24 @@ class MockProvider:
) -> MailboxDelta:
self._open_socket()
MockStats.last_sync[self.account_id] = time.time()
new: list[RemoteMessage] = []
if MockStats.mail_enabled and mailbox == "INBOX":
for _ in range(_draw_count()):
uid = MockStats._uid.get(self.account_id, 0) + 1
MockStats._uid[self.account_id] = uid
big = random.random() < MockStats.big_pct / 100.0
size = MockStats.big_bytes if big else MockStats.msg_bytes
new.append(
RemoteMessage(
uid=uid,
raw=_make_raw(size),
flags=frozenset(),
internal_date=datetime.now(UTC),
)
)
MockStats.delivered += len(new)
return MailboxDelta(
new=[],
new=new,
flag_changes=[],
expunged_uids=[],
new_uidvalidity=None,