"""MIME parsing via the stdlib ``email`` package with ``policy.default`` (SPEC §6.4). Primary parsing is stdlib per SPEC §3; ``mail-parser`` is available for helpers only. Header RFC2047 decoding is automatic under ``policy.default``. """ from __future__ import annotations import email import os import re from dataclasses import dataclass, field from datetime import datetime from email.headerregistry import AddressHeader from email.message import EmailMessage from email.policy import default as default_policy import nh3 from app.threading_.normalize import normalize_message_id, normalize_reference_list _SNIPPET_LEN = 160 # Undecodable header/body bytes come out of the stdlib parser as lone UTF-16 # surrogates (U+D800–U+DFFF). Those are invalid UTF-8, so Postgres rejects them in # text/JSONB columns — scrub them to U+FFFD before anything is stored. _SURROGATE_RE = re.compile("[\ud800-\udfff]") def _scrub(value: str) -> str: return _SURROGATE_RE.sub("�", value) if value else value def _scrub_addrs(addrs: list[dict[str, str]]) -> list[dict[str, str]]: return [{k: _scrub(v) for k, v in addr.items()} for addr in addrs] @dataclass(frozen=True) class ParsedAttachment: filename: str mime_type: str content_id: str | None payload: bytes @dataclass class ParsedEmail: message_id: str | None in_reply_to: str | None references: list[str] # References + In-Reply-To, normalized, in order (§6.4) subject: str from_addr: list[dict[str, str]] to_addr: list[dict[str, str]] cc_addr: list[dict[str, str]] bcc_addr: list[dict[str, str]] date: datetime | None text_body: str html_body: str | None attachments: list[ParsedAttachment] = field(default_factory=list) @property def has_attachments(self) -> bool: # Inline resources (cid: images) carry a content_id and render in-body, not # as attachment chips — only real attachments flip the paperclip. return any(a.content_id is None for a in self.attachments) @property def snippet(self) -> str: return " ".join(self.text_body.split())[:_SNIPPET_LEN] def _addresses(msg: EmailMessage, header: str) -> list[dict[str, str]]: value = msg[header] if value is None: return [] out: list[dict[str, str]] = [] if isinstance(value, AddressHeader): for addr in value.addresses: if addr.addr_spec: out.append({"name": str(addr.display_name or ""), "email": addr.addr_spec}) return out def _html_to_text(html: str) -> str: """Derive plain text from HTML by stripping all tags (SPEC §6.4 nh3 strip).""" return nh3.clean( html, tags=set(), attributes={}, strip_comments=True, # The document is not body text; unwrapped it would lead the snippet. clean_content_tags={"script", "style", "title"}, ) def _get_body_content(msg: EmailMessage, subtype: str) -> str | None: part = msg.get_body(preferencelist=(subtype,)) if part is None: return None try: content = part.get_content() except (LookupError, ValueError, KeyError): payload = part.get_payload(decode=True) if not isinstance(payload, bytes): return None content = payload.decode("utf-8", "replace") return content if isinstance(content, str) else None def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]: # Walk the whole tree, not iter_attachments(): the latter is shallow and treats # multipart/related as a body, so an inline image nested in mixed>related>alt is # never yielded (its cid: reference then renders broken). A part is an attachment # or inline resource if it has a filename, a Content-ID, or attachment disposition; # the body text/plain and text/html parts have none of these and are skipped. out: list[ParsedAttachment] = [] for part in msg.walk(): if part.is_multipart() or not isinstance(part, EmailMessage): continue cid = part.get("Content-ID") filename = part.get_filename() disposition = part.get_content_disposition() if not (filename or cid or disposition == "attachment"): continue payload = part.get_payload(decode=True) if not isinstance(payload, bytes): continue # A part is an INLINE resource (cid: image rendered in-body, hidden from the # attachment list) only when it carries a Content-ID AND is not explicitly an # attachment. A part marked `Content-Disposition: attachment` is a real # attachment even when it also has a Content-ID (e.g. Fastmail's invite.ics), # so we drop the cid for it and it shows as a downloadable chip. inline = cid is not None and disposition != "attachment" out.append( ParsedAttachment( filename=_scrub(filename or "attachment"), mime_type=part.get_content_type(), content_id=cid.strip("<>") if (inline and cid is not None) else None, payload=payload, ) ) # An invitation sent only as the text/calendar alternative (Outlook, often) has # no file to show: keep it as invite.ics, so the event can be seen, answered # and downloaded like a Gmail one. Not when the message attaches its own .ics. if not any(a.filename.lower().endswith(".ics") for a in out): for part in msg.walk(): if part.get_content_type() != "text/calendar" or part.get_filename(): continue payload = part.get_payload(decode=True) if isinstance(payload, bytes) and payload.strip(): out.append( ParsedAttachment( filename="invite.ics", mime_type="text/calendar", content_id=None, payload=payload, ) ) break return out def parse_message(raw: bytes) -> ParsedEmail: """Parse a raw RFC822 message (in memory) into a :class:`ParsedEmail`.""" msg = email.message_from_bytes(raw, policy=default_policy) assert isinstance(msg, EmailMessage) return _parse_email(msg) def parse_message_file(path: str | os.PathLike[str]) -> ParsedEmail: """Parse a message straight from a blob file handle (large-message path, §7).""" with open(path, "rb") as handle: msg = email.message_from_binary_file(handle, policy=default_policy) assert isinstance(msg, EmailMessage) return _parse_email(msg) def _parse_email(msg: EmailMessage) -> ParsedEmail: message_id = normalize_message_id(msg.get("Message-ID")) in_reply_to_ids = normalize_reference_list(msg.get("In-Reply-To")) in_reply_to = in_reply_to_ids[0] if in_reply_to_ids else None # references_hdr = References then In-Reply-To message-ids, in order, deduped (§6.4). references: list[str] = [] for ref in normalize_reference_list(msg.get("References")) + in_reply_to_ids: if ref not in references: references.append(ref) subject = str(msg.get("Subject", "") or "") date: datetime | None = None date_header = msg["Date"] if date_header is not None: try: date = date_header.datetime except (AttributeError, ValueError): date = None html_body = _get_body_content(msg, "html") text_body = _get_body_content(msg, "plain") if text_body is None and html_body is not None: text_body = _html_to_text(html_body) return ParsedEmail( message_id=message_id, in_reply_to=in_reply_to, references=references, subject=_scrub(subject), from_addr=_scrub_addrs(_addresses(msg, "From")), to_addr=_scrub_addrs(_addresses(msg, "To")), cc_addr=_scrub_addrs(_addresses(msg, "Cc")), bcc_addr=_scrub_addrs(_addresses(msg, "Bcc")), date=date, text_body=_scrub(text_body or ""), html_body=_scrub(html_body) if html_body is not None else None, attachments=_extract_attachments(msg), )