lukasandClaude Opus 4.8
9dbc5ed998
- concurrency.loop_semaphore: per-event-loop semaphores (safe across pytest loops and process restarts via a WeakKeyDictionary keyed by the running loop). - provider: holds a per-account (≤4) and per-host (PER_HOST_CONN_CAP) slot for the whole lifetime of an IMAP socket, released on close; SMTP send takes a per-host slot for its duration. A worker can no longer open hundreds of sockets to one host/account. - indexer: parsing goes through a PARSE_CONCURRENCY semaphore, and a message over STREAM_PARSE_MB is parsed from its blob file (parse_message_file) instead of an in-memory tree, so peak RAM ~= PARSE_CONCURRENCY x STREAM_PARSE_MB regardless of mailbox sizes. Backfill parallelism stays bounded by the bulk queue only. - config: PER_HOST_CONN_CAP, PARSE_CONCURRENCY, STREAM_PARSE_MB. - tests: per-account cap blocks the 5th connection / distinct accounts independent; file-vs-bytes parse equivalence + large-message streaming routing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
59 lines
1.9 KiB
Python
59 lines
1.9 KiB
Python
"""Per-account / per-host connection caps (refactor §4).
|
|
|
|
The provider holds a per-account (≤4) and per-host semaphore slot for the whole
|
|
lifetime of an open socket, so a worker can never open hundreds of simultaneous
|
|
connections to one account or host.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
|
|
from app.providers.imap_smtp.provider import _PER_ACCOUNT_CONN_CAP, ImapSmtpProvider, ProviderConfig
|
|
|
|
|
|
def _cfg(username: str, host: str) -> ProviderConfig:
|
|
return ProviderConfig(
|
|
imap_host=host,
|
|
imap_port=993,
|
|
imap_tls="ssl",
|
|
smtp_host=host,
|
|
smtp_port=465,
|
|
smtp_tls="ssl",
|
|
username=username,
|
|
password="pw",
|
|
)
|
|
|
|
|
|
async def test_per_account_connection_cap() -> None:
|
|
cfg = _cfg("user@example.com", "imap.example.com")
|
|
providers = [ImapSmtpProvider(cfg) for _ in range(_PER_ACCOUNT_CONN_CAP + 1)]
|
|
|
|
# Fill the account's slots (host cap defaults to 50, so account is the limiter).
|
|
for provider in providers[:_PER_ACCOUNT_CONN_CAP]:
|
|
await provider._acquire_conn_slots()
|
|
|
|
# The next acquire blocks until a slot frees.
|
|
with pytest.raises(TimeoutError):
|
|
await asyncio.wait_for(providers[-1]._acquire_conn_slots(), timeout=0.2)
|
|
|
|
providers[0]._release_conn_slots()
|
|
await asyncio.wait_for(providers[-1]._acquire_conn_slots(), timeout=0.5) # now succeeds
|
|
|
|
for provider in providers[1:]:
|
|
provider._release_conn_slots()
|
|
|
|
|
|
async def test_distinct_accounts_do_not_share_slots() -> None:
|
|
# Saturating account A must not block account B (independent semaphores).
|
|
a = [ImapSmtpProvider(_cfg("a@example.com", "imap.example.com")) for _ in range(4)]
|
|
for provider in a:
|
|
await provider._acquire_conn_slots()
|
|
b = ImapSmtpProvider(_cfg("b@example.com", "imap.example.com"))
|
|
await asyncio.wait_for(b._acquire_conn_slots(), timeout=0.5)
|
|
b._release_conn_slots()
|
|
for provider in a:
|
|
provider._release_conn_slots()
|