sync-v2 §10.7: mail-injecting mode for the soak harness
Optional SOAK_MAIL=1 makes the mock deliver synthetic messages (unique UIDs, well-formed RFC822) on INBOX polls, so the full parse -> blob -> DB -> index path actually runs and CPU/RSS move under real throughput. Tunables: SOAK_MAIL_PER_POLL, SOAK_MSG_KB, SOAK_BIG_MSG_PCT (percent above STREAM_PARSE_MB -> streaming/§8 path). Blob store + search backend are isolated (temp BLOB_ROOT, SEARCH_BACKEND=pg) and torn down. Output gains msgs + msg/s columns and a delivered/peak-RSS summary. RSS check is now a trend (early-third vs late-third mean, not min-to-max) so it detects a real leak without flaking on transient large-message parse spikes. Validated: mail run delivers ~4.2 msg/s (incl. 5% 10MB messages); RSS oscillates 123-267MB but its mean is flat (+3.7% over the run) -> no leak, parse memory bound holds (§8); tier re-promotion on new mail exercised; all §10.7 checks pass. Full gate green (238 tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -16,9 +16,13 @@ so the harness can compute the oldest sync lag across all accounts.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from email.message import EmailMessage
|
||||
from email.utils import format_datetime
|
||||
|
||||
from app.providers.base import MailboxDelta, RemoteMailbox, RemoteMessage
|
||||
|
||||
@@ -28,11 +32,40 @@ class MockStats:
|
||||
persistent_open = 0
|
||||
last_sync: dict[uuid.UUID, float] = {}
|
||||
|
||||
# --- optional mail injection (exercises parse/blob/DB/index under load) ----
|
||||
mail_enabled = False
|
||||
mail_per_poll = 0.0 # expected new INBOX messages per poll (fractional => prob)
|
||||
msg_bytes = 10_000
|
||||
big_pct = 0 # percent of messages that exceed STREAM_PARSE_MB (streaming path)
|
||||
big_bytes = 10_000_000
|
||||
delivered = 0 # total synthetic messages handed to the sync engine
|
||||
_uid: dict[uuid.UUID, int] = {}
|
||||
|
||||
@classmethod
|
||||
def reset(cls) -> None:
|
||||
cls.open = 0
|
||||
cls.persistent_open = 0
|
||||
cls.last_sync = {}
|
||||
cls.delivered = 0
|
||||
cls._uid = {}
|
||||
|
||||
|
||||
def _make_raw(size_bytes: int) -> bytes:
|
||||
"""A unique, well-formed RFC822 message padded to roughly ``size_bytes``."""
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = f"soak {uuid.uuid4().hex[:12]}"
|
||||
msg["From"] = "sender@load.local"
|
||||
msg["To"] = "rcpt@load.local"
|
||||
msg["Message-ID"] = f"<{uuid.uuid4().hex}@load.local>"
|
||||
msg["Date"] = format_datetime(datetime.now(UTC))
|
||||
msg.set_content("x" * max(0, size_bytes - 256))
|
||||
return msg.as_bytes()
|
||||
|
||||
|
||||
def _draw_count() -> int:
|
||||
n = int(MockStats.mail_per_poll)
|
||||
frac = MockStats.mail_per_poll - n
|
||||
return n + (1 if random.random() < frac else 0)
|
||||
|
||||
|
||||
class MockProvider:
|
||||
@@ -61,8 +94,24 @@ class MockProvider:
|
||||
) -> MailboxDelta:
|
||||
self._open_socket()
|
||||
MockStats.last_sync[self.account_id] = time.time()
|
||||
new: list[RemoteMessage] = []
|
||||
if MockStats.mail_enabled and mailbox == "INBOX":
|
||||
for _ in range(_draw_count()):
|
||||
uid = MockStats._uid.get(self.account_id, 0) + 1
|
||||
MockStats._uid[self.account_id] = uid
|
||||
big = random.random() < MockStats.big_pct / 100.0
|
||||
size = MockStats.big_bytes if big else MockStats.msg_bytes
|
||||
new.append(
|
||||
RemoteMessage(
|
||||
uid=uid,
|
||||
raw=_make_raw(size),
|
||||
flags=frozenset(),
|
||||
internal_date=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
MockStats.delivered += len(new)
|
||||
return MailboxDelta(
|
||||
new=[],
|
||||
new=new,
|
||||
flag_changes=[],
|
||||
expunged_uids=[],
|
||||
new_uidvalidity=None,
|
||||
|
||||
Reference in New Issue
Block a user