lukas/webmail-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync-v2 step 6: memory + connection bounds (§7, §4)

Browse Source 
- concurrency.loop_semaphore: per-event-loop semaphores (safe across pytest loops
  and process restarts via a WeakKeyDictionary keyed by the running loop).
- provider: holds a per-account (≤4) and per-host (PER_HOST_CONN_CAP) slot for the
  whole lifetime of an IMAP socket, released on close; SMTP send takes a per-host
  slot for its duration. A worker can no longer open hundreds of sockets to one
  host/account.
- indexer: parsing goes through a PARSE_CONCURRENCY semaphore, and a message over
  STREAM_PARSE_MB is parsed from its blob file (parse_message_file) instead of an
  in-memory tree, so peak RAM ~= PARSE_CONCURRENCY x STREAM_PARSE_MB regardless of
  mailbox sizes. Backfill parallelism stays bounded by the bulk queue only.
- config: PER_HOST_CONN_CAP, PARSE_CONCURRENCY, STREAM_PARSE_MB.
- tests: per-account cap blocks the 5th connection / distinct accounts independent;
  file-vs-bytes parse equivalence + large-message streaming routing.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
lukas
2026-07-19 07:13:03 +02:00
co-authored by Claude Opus 4.8
parent 19335116da
commit 9dbc5ed998
7 changed files with 254 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/providers/imap_smtp/provider.py
+68 -31
View File
@@ -17,6 +17,8 @@ from dataclasses import dataclass
import aiosmtplib
from aioimaplib import aioimaplib
from app.concurrency import loop_semaphore
from app.config import get_settings
from app.providers.base import (
FlagChange,
MailboxDelta,
@@ -37,6 +39,7 @@ from app.providers.imap_smtp.parsing import (
_FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
_IDLE_TIMEOUT = 25 * 60 # SPEC §5.1: re-issue IDLE at least every 25 min
_CONNECT_TIMEOUT = 20
_PER_ACCOUNT_CONN_CAP = 4 # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
_APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
_COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")
@@ -59,34 +62,65 @@ class ImapSmtpProvider(MailProvider):
self._imap: aioimaplib.IMAP4 | None = None
self._selected: str | None = None
self._uidvalidity_seen: dict[str, int] = {}
self._conn_slots: list[asyncio.Semaphore] = []
# --- connection lifecycle ------------------------------------------------
async def _acquire_conn_slots(self) -> None:
"""Bound simultaneous open sockets: ≤4 per account and ≤cap per host (§4).
Held for the connection's whole lifetime and released in :meth:`close`.
"""
cfg = self._cfg
account = loop_semaphore(
"imap_account", f"{cfg.username}@{cfg.imap_host}", _PER_ACCOUNT_CONN_CAP
)
host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
await account.acquire() # consistent order (account, then host) avoids deadlock
try:
await host.acquire()
except BaseException:
account.release()
raise
self._conn_slots = [host, account]
def _release_conn_slots(self) -> None:
for sem in self._conn_slots:
sem.release()
self._conn_slots = []
async def _connect(self) -> aioimaplib.IMAP4:
if self._imap is not None:
return self._imap
cfg = self._cfg
await self._acquire_conn_slots()
try:
if cfg.imap_tls == "ssl":
imap = aioimaplib.IMAP4_SSL(
host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
)
else:
imap = aioimaplib.IMAP4(
host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
)
await imap.wait_hello_from_server()
if cfg.imap_tls == "starttls":
await imap.starttls()
except (OSError, TimeoutError) as exc:
raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc
try:
if cfg.imap_tls == "ssl":
imap = aioimaplib.IMAP4_SSL(
host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
)
else:
imap = aioimaplib.IMAP4(
host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
)
await imap.wait_hello_from_server()
if cfg.imap_tls == "starttls":
await imap.starttls()
except (OSError, TimeoutError) as exc:
raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc
resp = await imap.login(cfg.username, cfg.password)
if resp.result != "OK":
detail = (
b" ".join(resp.lines).decode("utf-8", "ignore") if resp.lines else "login failed"
)
raise ProviderAuthError(detail)
resp = await imap.login(cfg.username, cfg.password)
if resp.result != "OK":
detail = (
b" ".join(resp.lines).decode("utf-8", "ignore")
if resp.lines
else "login failed"
)
raise ProviderAuthError(detail)
except BaseException:
self._release_conn_slots() # never hold a slot without an open socket
raise
self._imap = imap
return imap
@@ -104,6 +138,7 @@ class ImapSmtpProvider(MailProvider):
await self._imap.logout()
self._imap = None
self._selected = None
self._release_conn_slots() # free the per-account / per-host slots
# --- MailProvider protocol ----------------------------------------------
@@ -293,19 +328,21 @@ class ImapSmtpProvider(MailProvider):
cfg = self._cfg
use_tls = cfg.smtp_tls == "ssl"
start_tls = cfg.smtp_tls == "starttls"
host_slot = loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
try:
await aiosmtplib.send(
raw,
sender=envelope_from,
recipients=envelope_to,
hostname=cfg.smtp_host,
port=cfg.smtp_port,
username=cfg.username,
password=cfg.password,
use_tls=use_tls,
start_tls=start_tls,
timeout=_CONNECT_TIMEOUT,
)
async with host_slot: # bound simultaneous sockets to the SMTP host (§4)
await aiosmtplib.send(
raw,
sender=envelope_from,
recipients=envelope_to,
hostname=cfg.smtp_host,
port=cfg.smtp_port,
username=cfg.username,
password=cfg.password,
use_tls=use_tls,
start_tls=start_tls,
timeout=_CONNECT_TIMEOUT,
)
except aiosmtplib.SMTPAuthenticationError as exc:
raise ProviderAuthError(f"SMTP auth failed: {exc}") from exc
except aiosmtplib.SMTPException as exc: