61 lines
1.9 KiB
Python
61 lines
1.9 KiB
Python
 
|
"""Bounded / streaming MIME parse (refactor §7).
|
||
|
|
|
||
|
|
Large messages are parsed from their blob file rather than an in-memory tree, and
|
||
|
|
all parsing passes through the PARSE_CONCURRENCY semaphore.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import uuid
|
||
|
|
from email.message import EmailMessage
|
||
|
|
|
||
|
|
from app.indexer import indexer
|
||
|
|
from app.indexer.mime import parse_message, parse_message_file
|
||
|
|
|
||
|
|
|
||
|
|
def _raw(body: str = "hello") -> bytes:
|
||
|
|
msg = EmailMessage()
|
||
|
|
msg["Subject"] = "Streamed"
|
||
|
|
msg["From"] = "a@example.com"
|
||
|
|
msg["To"] = "b@example.com"
|
||
|
|
msg["Message-ID"] = "<s1@example.com>"
|
||
|
|
msg.set_content(body)
|
||
|
|
msg.add_attachment(
|
||
|
|
b"x" * 1024, maintype="application", subtype="octet-stream", filename="a.bin"
|
||
|
|
)
|
||
|
|
return msg.as_bytes()
|
||
|
|
|
||
|
|
|
||
|
|
def test_parse_file_matches_bytes(tmp_path) -> None: # type: ignore[no-untyped-def]
|
||
|
|
raw = _raw()
|
||
|
|
path = tmp_path / "msg.eml"
|
||
|
|
path.write_bytes(raw)
|
||
|
|
# Parsing from the file handle yields exactly the same ParsedEmail as from bytes.
|
||
|
|
assert parse_message_file(path) == parse_message(raw)
|
||
|
|
|
||
|
|
|
||
|
|
async def test_parse_bounded_streams_large_messages(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||
|
|
from app.config import get_settings
|
||
|
|
|
||
|
|
threshold = get_settings().stream_parse_mb * 1024 * 1024
|
||
|
|
calls: list[str] = []
|
||
|
|
|
||
|
|
def fake_file(path: object) -> str:
|
||
|
|
calls.append("file")
|
||
|
|
return "file"
|
||
|
|
|
||
|
|
def fake_bytes(raw: bytes) -> str:
|
||
|
|
calls.append("bytes")
|
||
|
|
return "bytes"
|
||
|
|
|
||
|
|
monkeypatch.setattr(indexer, "parse_message_file", fake_file)
|
||
|
|
monkeypatch.setattr(indexer, "parse_message", fake_bytes)
|
||
|
|
|
||
|
|
user_id = uuid.uuid4()
|
||
|
|
small = await indexer._parse_bounded(user_id, "sha", b"x" * 16)
|
||
|
|
big = await indexer._parse_bounded(user_id, "sha", b"x" * (threshold + 1))
|
||
|
|
|
||
|
|
assert small == "bytes" # small stays in memory
|
||
|
|
assert big == "file" # over threshold => parsed from the blob file
|
||
|
|
assert calls == ["bytes", "file"]
|