webmail-client/backend/app/providers/imap_smtp/provider.py

"""``ImapSmtpProvider`` — IMAP (aioimaplib) + SMTP (aiosmtplib) MailProvider.

One instance == one IMAP connection (SPEC §6.3 caps connections per account);
the sync engine creates separate instances for the IDLE loop vs. backfill/jobs.
Credentials arrive already decrypted (decryption lives only in workers /
accounts.test per SPEC §10.2).
"""

from __future__ import annotations

import asyncio
import contextlib
import re
from collections.abc import AsyncIterator
from dataclasses import dataclass

import aiosmtplib
from aioimaplib import aioimaplib

from app.concurrency import loop_semaphore
from app.config import get_settings
from app.providers.base import (
    FlagChange,
    MailboxDelta,
    MailProvider,
    ProviderAuthError,
    ProviderConnectError,
    ProviderNotSupported,
    RemoteMailbox,
    RemoteMessage,
)
from app.providers.imap_smtp.parsing import (
    parse_fetch_lines,
    parse_list_line,
    parse_search_uids,
    parse_select_untagged,
)

_FETCH_ITEMS = "(UID FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[])"
_IDLE_TIMEOUT = 25 * 60  # SPEC §5.1: re-issue IDLE at least every 25 min
_CONNECT_TIMEOUT = 20
_PER_ACCOUNT_CONN_CAP = 4  # SPEC §6.3 / §4: IDLE + transient + short-lived jobs
_APPENDUID_RE = re.compile(rb"APPENDUID\s+\d+\s+(\d+)")
_COPYUID_RE = re.compile(rb"COPYUID\s+\d+\s+\S+\s+(\d+)")


@dataclass(frozen=True)
class ProviderConfig:
    imap_host: str
    imap_port: int
    imap_tls: str  # "ssl" | "starttls" | "plain"
    smtp_host: str
    smtp_port: int
    smtp_tls: str  # "ssl" | "starttls" | "plain"
    username: str
    password: str  # plaintext; caller decrypts per §10.2


class ImapSmtpProvider(MailProvider):
    def __init__(self, config: ProviderConfig) -> None:
        self._cfg = config
        self._imap: aioimaplib.IMAP4 | None = None
        self._selected: str | None = None
        self._uidvalidity_seen: dict[str, int] = {}
        self._conn_slots: list[asyncio.Semaphore] = []

    # --- connection lifecycle ------------------------------------------------

    async def _acquire_conn_slots(self) -> None:
        """Bound simultaneous open sockets: ≤4 per account and ≤cap per host (§4).

        Held for the connection's whole lifetime and released in :meth:`close`.
        """
        cfg = self._cfg
        account = loop_semaphore(
            "imap_account", f"{cfg.username}@{cfg.imap_host}", _PER_ACCOUNT_CONN_CAP
        )
        host = loop_semaphore("imap_host", cfg.imap_host, get_settings().per_host_conn_cap)
        await account.acquire()  # consistent order (account, then host) avoids deadlock
        try:
            await host.acquire()
        except BaseException:
            account.release()
            raise
        self._conn_slots = [host, account]

    def _release_conn_slots(self) -> None:
        for sem in self._conn_slots:
            sem.release()
        self._conn_slots = []

    async def _connect(self) -> aioimaplib.IMAP4:
        if self._imap is not None:
            return self._imap
        cfg = self._cfg
        await self._acquire_conn_slots()
        try:
            try:
                if cfg.imap_tls == "ssl":
                    imap = aioimaplib.IMAP4_SSL(
                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
                    )
                else:
                    imap = aioimaplib.IMAP4(
                        host=cfg.imap_host, port=cfg.imap_port, timeout=_CONNECT_TIMEOUT
                    )
                await imap.wait_hello_from_server()
                if cfg.imap_tls == "starttls":
                    await imap.starttls()
            except (OSError, TimeoutError) as exc:
                raise ProviderConnectError(f"cannot connect to {cfg.imap_host}: {exc}") from exc

            resp = await imap.login(cfg.username, cfg.password)
            if resp.result != "OK":
                detail = (
                    b" ".join(resp.lines).decode("utf-8", "ignore")
                    if resp.lines
                    else "login failed"
                )
                raise ProviderAuthError(detail)
        except BaseException:
            self._release_conn_slots()  # never hold a slot without an open socket
            raise
        self._imap = imap
        return imap

    async def _select(self, mailbox: str) -> dict[str, int]:
        imap = await self._connect()
        resp = await imap.select(mailbox)
        if resp.result != "OK":
            raise ProviderConnectError(f"cannot select {mailbox!r}")
        self._selected = mailbox
        return parse_select_untagged(resp.lines)

    async def close(self) -> None:
        if self._imap is not None:
            with contextlib.suppress(Exception):  # best-effort teardown
                await self._imap.logout()
            self._imap = None
            self._selected = None
            self._release_conn_slots()  # free the per-account / per-host slots

    # --- MailProvider protocol ----------------------------------------------

    async def verify(self) -> None:
        imap = await self._connect()
        resp = await imap.list('""', "*")
        if resp.result != "OK":
            raise ProviderConnectError("LIST failed after login")

    async def list_mailboxes(self) -> list[RemoteMailbox]:
        imap = await self._connect()
        resp = await imap.list('""', "*")
        if resp.result != "OK":
            raise ProviderConnectError("LIST failed")
        out: list[RemoteMailbox] = []
        for line in resp.lines:
            if not isinstance(line, (bytes, bytearray)):
                continue
            parsed = parse_list_line(bytes(line))
            if parsed is None:
                continue
            uidvalidity = await self._mailbox_uidvalidity(parsed.path)
            out.append(
                RemoteMailbox(
                    path=parsed.path, special_use=parsed.special_use, uidvalidity=uidvalidity
                )
            )
        return out

    async def _mailbox_uidvalidity(self, path: str) -> int:
        imap = await self._connect()
        try:
            resp = await imap.status(path, "(UIDVALIDITY)")
        except Exception:
            return 0
        if resp.result != "OK":
            return 0
        for line in resp.lines:
            if isinstance(line, (bytes, bytearray)):
                m = re.search(rb"UIDVALIDITY\s+(\d+)", bytes(line))
                if m:
                    return int(m.group(1))
        return 0

    async def backfill(
        self, mailbox: str, *, newest_first: bool = True, batch_size: int = 200
    ) -> AsyncIterator[list[RemoteMessage]]:
        info = await self._select(mailbox)
        self._uidvalidity_seen[mailbox] = info.get("uidvalidity", 0)
        imap = await self._connect()
        search = await imap.uid_search("ALL")
        uids = parse_search_uids(search.lines)
        uids.sort(reverse=newest_first)
        for start in range(0, len(uids), batch_size):
            batch = uids[start : start + batch_size]
            messages = await self._fetch_messages(imap, batch)
            if messages:
                yield messages

    async def list_uids(self, mailbox: str) -> set[int]:
        """Full set of UIDs currently present in the mailbox (expunge reconciliation).

        Cheap — a single ``UID SEARCH ALL``. The CONDSTORE poll only reports new and
        changed messages, not vanished ones (that needs QRESYNC/VANISHED); callers use
        this to detect server-side expunges for small folders like Drafts.
        """
        await self._select(mailbox)
        imap = await self._connect()
        resp = await imap.uid_search("ALL")
        return set(parse_search_uids(resp.lines))

    async def _fetch_messages(self, imap: aioimaplib.IMAP4, uids: list[int]) -> list[RemoteMessage]:
        if not uids:
            return []
        uid_set = ",".join(str(u) for u in uids)
        resp = await imap.uid("fetch", uid_set, _FETCH_ITEMS)
        if resp.result != "OK":
            raise ProviderConnectError("FETCH failed")
        out: list[RemoteMessage] = []
        for pf in parse_fetch_lines(resp.lines):
            if pf.raw is None or pf.internal_date is None:
                continue
            out.append(
                RemoteMessage(
                    uid=pf.uid, raw=pf.raw, flags=pf.flags, internal_date=pf.internal_date
                )
            )
        return out

    async def poll(
        self, mailbox: str, *, last_seen_uid: int, highestmodseq: int | None
    ) -> MailboxDelta:
        info = await self._select(mailbox)
        imap = await self._connect()
        current_uidvalidity = info.get("uidvalidity", 0)
        new_uidvalidity: int | None = None
        prev = self._uidvalidity_seen.get(mailbox)
        if prev is not None and current_uidvalidity != prev:
            new_uidvalidity = current_uidvalidity
        self._uidvalidity_seen[mailbox] = current_uidvalidity
        if new_uidvalidity is not None:
            return MailboxDelta([], [], [], new_uidvalidity, info.get("highestmodseq"))

        # New messages: UID SEARCH beyond last_seen_uid (filter the M:* boundary echo).
        search = await imap.uid_search(f"UID {last_seen_uid + 1}:*")
        candidate_uids = [u for u in parse_search_uids(search.lines) if u > last_seen_uid]
        new_messages = await self._fetch_messages(imap, sorted(candidate_uids))

        # Flag changes via CONDSTORE when we have a modseq baseline.
        flag_changes: list[FlagChange] = []
        if highestmodseq is not None:
            fresp = await imap.uid("fetch", "1:*", f"(UID FLAGS) (CHANGEDSINCE {highestmodseq})")
            if fresp.result == "OK":
                new_uid_set = {m.uid for m in new_messages}
                for pf in parse_fetch_lines(fresp.lines):
                    if pf.uid not in new_uid_set:
                        flag_changes.append(FlagChange(uid=pf.uid, flags=pf.flags))

        # TODO(spec): expunge detection needs QRESYNC/VANISHED (M2 hardening).
        return MailboxDelta(
            new=new_messages,
            flag_changes=flag_changes,
            expunged_uids=[],
            new_uidvalidity=None,
            new_highestmodseq=info.get("highestmodseq"),
        )

    async def idle(self, mailbox: str) -> None:
        imap = await self._connect()
        if "IDLE" not in (imap.protocol.capabilities or set()):
            raise ProviderNotSupported("server does not support IDLE")
        await self._select(mailbox)
        idle_task = await imap.idle_start(timeout=_IDLE_TIMEOUT)
        try:
            await imap.wait_server_push()
        finally:
            imap.idle_done()
            with contextlib.suppress(Exception):
                await asyncio.wait_for(idle_task, timeout=10)

    async def push_flags(
        self, mailbox: str, uid: int, add: frozenset[str], remove: frozenset[str]
    ) -> None:
        await self._select(mailbox)
        imap = await self._connect()
        if add:
            await imap.uid("store", str(uid), "+FLAGS.SILENT", f"({' '.join(sorted(add))})")
        if remove:
            await imap.uid("store", str(uid), "-FLAGS.SILENT", f"({' '.join(sorted(remove))})")

    async def create_mailbox(self, path: str) -> None:
        """Create a mailbox on the server (idempotent — ignores 'already exists')."""
        imap = await self._connect()
        with contextlib.suppress(Exception):
            await imap.create(path)

    async def move(self, uid: int, src: str, dst: str) -> int:
        await self._select(src)
        imap = await self._connect()
        resp = await imap.uid("move", str(uid), dst)
        if resp.result != "OK":
            raise ProviderConnectError(f"MOVE {uid} -> {dst} failed")
        for line in resp.lines:
            if isinstance(line, (bytes, bytearray)):
                m = _COPYUID_RE.search(bytes(line))
                if m:
                    return int(m.group(1))
        return uid

    async def delete_messages(self, mailbox: str, uids: list[int]) -> None:
        """Flag the given UIDs \\Deleted and EXPUNGE them from ``mailbox``."""
        if not uids:
            return
        await self._select(mailbox)
        imap = await self._connect()
        uid_set = ",".join(str(u) for u in uids)
        await imap.uid("store", uid_set, "+FLAGS.SILENT", "(\\Deleted)")
        try:
            # UID EXPUNGE (UIDPLUS) removes only our UIDs; fall back to a plain
            # EXPUNGE (all \Deleted in the mailbox) when unsupported.
            await imap.uid("expunge", uid_set)
        except Exception:
            await imap.expunge()

    async def append(self, mailbox: str, raw: bytes, flags: frozenset[str]) -> int:
        imap = await self._connect()
        flag_str = f"({' '.join(sorted(flags))})" if flags else None
        resp = await imap.append(raw, mailbox=mailbox, flags=flag_str)
        if resp.result != "OK":
            raise ProviderConnectError(f"APPEND to {mailbox} failed")
        for line in resp.lines:
            if isinstance(line, (bytes, bytearray)):
                m = _APPENDUID_RE.search(bytes(line))
                if m:
                    return int(m.group(1))
        return 0

    async def send(self, raw: bytes, envelope_from: str, envelope_to: list[str]) -> None:
        cfg = self._cfg
        use_tls = cfg.smtp_tls == "ssl"
        start_tls = cfg.smtp_tls == "starttls"
        host_slot = loop_semaphore("smtp_host", cfg.smtp_host, get_settings().per_host_conn_cap)
        try:
            async with host_slot:  # bound simultaneous sockets to the SMTP host (§4)
                await aiosmtplib.send(
                    raw,
                    sender=envelope_from,
                    recipients=envelope_to,
                    hostname=cfg.smtp_host,
                    port=cfg.smtp_port,
                    username=cfg.username,
                    password=cfg.password,
                    use_tls=use_tls,
                    start_tls=start_tls,
                    timeout=_CONNECT_TIMEOUT,
                )
        except aiosmtplib.SMTPAuthenticationError as exc:
            raise ProviderAuthError(f"SMTP auth failed: {exc}") from exc
        except aiosmtplib.SMTPException as exc:
            raise ProviderConnectError(f"SMTP send failed: {exc}") from exc