sync-v2 step 6: memory + connection bounds (§7, §4)
- concurrency.loop_semaphore: per-event-loop semaphores (safe across pytest loops and process restarts via a WeakKeyDictionary keyed by the running loop). - provider: holds a per-account (≤4) and per-host (PER_HOST_CONN_CAP) slot for the whole lifetime of an IMAP socket, released on close; SMTP send takes a per-host slot for its duration. A worker can no longer open hundreds of sockets to one host/account. - indexer: parsing goes through a PARSE_CONCURRENCY semaphore, and a message over STREAM_PARSE_MB is parsed from its blob file (parse_message_file) instead of an in-memory tree, so peak RAM ~= PARSE_CONCURRENCY x STREAM_PARSE_MB regardless of mailbox sizes. Backfill parallelism stays bounded by the bulk queue only. - config: PER_HOST_CONN_CAP, PARSE_CONCURRENCY, STREAM_PARSE_MB. - tests: per-account cap blocks the 5th connection / distinct accounts independent; file-vs-bytes parse equivalence + large-message streaming routing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
"""Bounded / streaming MIME parse (refactor §7).
|
||||
|
||||
Large messages are parsed from their blob file rather than an in-memory tree, and
|
||||
all parsing passes through the PARSE_CONCURRENCY semaphore.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from email.message import EmailMessage
|
||||
|
||||
from app.indexer import indexer
|
||||
from app.indexer.mime import parse_message, parse_message_file
|
||||
|
||||
|
||||
def _raw(body: str = "hello") -> bytes:
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Streamed"
|
||||
msg["From"] = "a@example.com"
|
||||
msg["To"] = "b@example.com"
|
||||
msg["Message-ID"] = "<s1@example.com>"
|
||||
msg.set_content(body)
|
||||
msg.add_attachment(
|
||||
b"x" * 1024, maintype="application", subtype="octet-stream", filename="a.bin"
|
||||
)
|
||||
return msg.as_bytes()
|
||||
|
||||
|
||||
def test_parse_file_matches_bytes(tmp_path) -> None: # type: ignore[no-untyped-def]
|
||||
raw = _raw()
|
||||
path = tmp_path / "msg.eml"
|
||||
path.write_bytes(raw)
|
||||
# Parsing from the file handle yields exactly the same ParsedEmail as from bytes.
|
||||
assert parse_message_file(path) == parse_message(raw)
|
||||
|
||||
|
||||
async def test_parse_bounded_streams_large_messages(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
from app.config import get_settings
|
||||
|
||||
threshold = get_settings().stream_parse_mb * 1024 * 1024
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_file(path: object) -> str:
|
||||
calls.append("file")
|
||||
return "file"
|
||||
|
||||
def fake_bytes(raw: bytes) -> str:
|
||||
calls.append("bytes")
|
||||
return "bytes"
|
||||
|
||||
monkeypatch.setattr(indexer, "parse_message_file", fake_file)
|
||||
monkeypatch.setattr(indexer, "parse_message", fake_bytes)
|
||||
|
||||
user_id = uuid.uuid4()
|
||||
small = await indexer._parse_bounded(user_id, "sha", b"x" * 16)
|
||||
big = await indexer._parse_bounded(user_id, "sha", b"x" * (threshold + 1))
|
||||
|
||||
assert small == "bytes" # small stays in memory
|
||||
assert big == "file" # over threshold => parsed from the blob file
|
||||
assert calls == ["bytes", "file"]
|
||||
Reference in New Issue
Block a user