Initial commit: webmail client (M0/M1)

Three-pane webmail client. - Backend: FastAPI + async SQLAlchemy + arq worker (Postgres/Redis), IMAP/SMTP sync, threading, search, drafts/outbox with undo, attachments, 2FA. - Frontend: React 18 + Vite + TypeScript + TanStack Query + Tailwind; three-pane UI with rich-text composer, light/dark themes, resizable panels. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-17 23:20:34 +02:00
commit a5ec0d21cb
174 changed files with 22239 additions and 0 deletions
@@ -0,0 +1,43 @@
+"""Content-addressed blob store (SPEC §4, §6.4)."""
+
+from __future__ import annotations
+
+import uuid
+
+import pytest
+
+from app.indexer import blobstore
+
+
+@pytest.fixture(autouse=True)
+def _blob_root(tmp_path, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    from app.config import get_settings
+
+    settings = get_settings()
+    monkeypatch.setattr(settings, "blob_root", str(tmp_path))
+
+
+def test_store_and_load_roundtrip() -> None:
+    uid = uuid.uuid4()
+    data = b"hello world"
+    sha = blobstore.store(uid, data)
+    assert blobstore.load(uid, sha) == data
+    assert blobstore.exists(uid, sha)
+
+
+def test_store_is_content_addressed_and_dedupes() -> None:
+    uid = uuid.uuid4()
+    sha1 = blobstore.store(uid, b"same payload")
+    sha2 = blobstore.store(uid, b"same payload")
+    assert sha1 == sha2
+    # path layout: {user}/{sha[:2]}/{sha}
+    path = blobstore.blob_path(uid, sha1)
+    assert path.parent.name == sha1[:2]
+    assert path.parent.parent.name == str(uid)
+
+
+def test_different_users_isolated() -> None:
+    u1, u2 = uuid.uuid4(), uuid.uuid4()
+    sha = blobstore.store(u1, b"payload")
+    assert blobstore.exists(u1, sha)
+    assert not blobstore.exists(u2, sha)
@@ -0,0 +1,124 @@
+"""Duplicate Message-ID handling in the indexer (SPEC §7.1).
+
+Real-world trigger: home.pl's welcome mailer reuses one static Message-ID for
+every recipient, so the same id lands in different accounts as different messages.
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime, timedelta
+from email.message import EmailMessage
+
+import pytest_asyncio
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+from app.indexer.indexer import persist_message
+from app.models import Account, Mailbox, Message, User
+from app.providers.base import RemoteMessage
+
+
+class Dupenv:
+    def __init__(self, session: AsyncSession, user_id: uuid.UUID) -> None:
+        self.session = session
+        self.user_id = user_id
+        self.accounts: dict[str, uuid.UUID] = {}
+        self.mailboxes: dict[str, uuid.UUID] = {}
+        self._uid = 0
+
+    async def add_account(self, label: str) -> None:
+        acc = Account(
+            user_id=self.user_id,
+            email=f"{label}@example.com",
+            imap_config={},
+            smtp_config={},
+            secret_ct=b"",
+            discovery_src="manual",
+        )
+        self.session.add(acc)
+        await self.session.flush()
+        self.accounts[label] = acc.id
+
+    async def add_mailbox(self, account_label: str, path: str) -> None:
+        mb = Mailbox(
+            account_id=self.accounts[account_label],
+            user_id=self.user_id,
+            path=path,
+            special_use="inbox",
+        )
+        self.session.add(mb)
+        await self.session.flush()
+        self.mailboxes[f"{account_label}/{path}"] = mb.id
+
+    async def ingest(
+        self, account_label: str, path: str, *, mid: str, minutes: int = 0
+    ) -> Message | None:
+        self._uid += 1
+        m = EmailMessage()
+        m["Subject"] = "Pierwsze kroki z pocztą w home.pl"
+        m["From"] = "info@home.pl"
+        m["Message-ID"] = mid
+        m["Date"] = "Sun, 23 Nov 2025 12:47:30 +0000"
+        m.set_content("welcome template body")
+        remote = RemoteMessage(
+            uid=self._uid,
+            raw=m.as_bytes(),
+            flags=frozenset(),
+            internal_date=datetime(2026, 1, 1, tzinfo=UTC) + timedelta(minutes=minutes),
+        )
+        result = await persist_message(
+            self.session,
+            user_id=self.user_id,
+            account_id=self.accounts[account_label],
+            mailbox_id=self.mailboxes[f"{account_label}/{path}"],
+            mailbox_special_use="inbox",
+            remote=remote,
+        )
+        return result.message if result else None
+
+
+@pytest_asyncio.fixture
+async def dup() -> AsyncIterator[Dupenv]:
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    session = sm()
+    user = User(handle=f"dup{uuid.uuid4().hex[:8]}@test.local", password_hash="x")
+    session.add(user)
+    await session.flush()
+    env = Dupenv(session, user.id)
+    await env.add_account("a")
+    await env.add_account("b")
+    await env.add_mailbox("a", "INBOX")
+    await env.add_mailbox("a", "Archive")
+    await env.add_mailbox("b", "INBOX")
+    try:
+        yield env
+    finally:
+        await session.execute(delete(User).where(User.id == user.id))
+        await session.commit()
+        await session.close()
+        await engine.dispose()
+
+
+async def test_cross_account_duplicate_isolates(dup: Dupenv) -> None:
+    # Same static Message-ID delivered to two different accounts -> two threads.
+    a = await dup.ingest("a", "INBOX", mid="<static@home.pl>", minutes=0)
+    b = await dup.ingest("b", "INBOX", mid="<static@home.pl>", minutes=60)
+    assert a is not None and b is not None
+    assert a.thread_id != b.thread_id
+    # The second one was stored with a synthesized, non-authoritative id.
+    assert a.message_id_hdr == "static@home.pl"
+    assert b.message_id_hdr.endswith("@synthetic.local")
+
+
+async def test_same_account_duplicate_merges(dup: Dupenv) -> None:
+    # Same message in INBOX + Archive of ONE account (Gmail-style) -> one thread.
+    a = await dup.ingest("a", "INBOX", mid="<msg@x.com>", minutes=0)
+    b = await dup.ingest("a", "Archive", mid="<msg@x.com>", minutes=1)
+    assert a is not None and b is not None
+    assert a.thread_id == b.thread_id
+    assert b.message_id_hdr == "msg@x.com"
@@ -0,0 +1,103 @@
+"""MIME parsing (SPEC §6.4)."""
+
+from __future__ import annotations
+
+from email.message import EmailMessage
+
+from app.indexer.mime import parse_message
+
+
+def _build(
+    *,
+    subject: str = "Hello",
+    from_addr: str = "Alice <alice@example.com>",
+    to_addr: str = "bob@example.com",
+    message_id: str = "<m1@example.com>",
+    references: str | None = None,
+    in_reply_to: str | None = None,
+    text: str | None = "plain body",
+    html: str | None = None,
+) -> bytes:
+    msg = EmailMessage()
+    msg["Subject"] = subject
+    msg["From"] = from_addr
+    msg["To"] = to_addr
+    msg["Message-ID"] = message_id
+    msg["Date"] = "Mon, 09 Mar 2026 08:36:37 +0100"
+    if references:
+        msg["References"] = references
+    if in_reply_to:
+        msg["In-Reply-To"] = in_reply_to
+    if text is not None:
+        msg.set_content(text)
+    if html is not None:
+        if text is not None:
+            msg.add_alternative(html, subtype="html")
+        else:
+            msg.set_content(html, subtype="html")
+    return msg.as_bytes()
+
+
+def test_parse_basic_headers_and_body() -> None:
+    parsed = parse_message(_build())
+    assert parsed.subject == "Hello"
+    assert parsed.message_id == "m1@example.com"
+    assert parsed.from_addr == [{"name": "Alice", "email": "alice@example.com"}]
+    assert parsed.to_addr == [{"name": "", "email": "bob@example.com"}]
+    assert "plain body" in parsed.text_body
+    assert parsed.date is not None
+    assert parsed.snippet == "plain body"
+
+
+def test_references_include_in_reply_to_in_order() -> None:
+    parsed = parse_message(_build(references="<a@x.com> <b@x.com>", in_reply_to="<b@x.com>"))
+    # References first, then In-Reply-To, deduped
+    assert parsed.references == ["a@x.com", "b@x.com"]
+    assert parsed.in_reply_to == "b@x.com"
+
+
+def test_html_only_derives_text() -> None:
+    parsed = parse_message(_build(text=None, html="<p>Hello <b>world</b></p>"))
+    assert parsed.html_body is not None
+    assert "Hello" in parsed.text_body
+    assert "world" in parsed.text_body
+    assert "<b>" not in parsed.text_body  # tags stripped for indexing text
+
+
+def test_multipart_alternative_prefers_html_for_display() -> None:
+    parsed = parse_message(_build(text="text version", html="<p>html version</p>"))
+    assert parsed.html_body is not None
+    assert "html version" in parsed.html_body
+    assert "text version" in parsed.text_body
+
+
+def test_rfc2047_encoded_subject_is_decoded() -> None:
+    # "Próba" base64-encoded UTF-8
+    parsed = parse_message(_build(subject="=?UTF-8?B?UHLDs2Jh?="))
+    assert parsed.subject == "Próba"
+
+
+def test_attachment_extracted() -> None:
+    msg = EmailMessage()
+    msg["Subject"] = "with attachment"
+    msg["From"] = "a@b.com"
+    msg["To"] = "c@d.com"
+    msg["Message-ID"] = "<att@example.com>"
+    msg.set_content("see attached")
+    msg.add_attachment(
+        b"\x00\x01\x02data", maintype="application", subtype="octet-stream", filename="file.bin"
+    )
+    parsed = parse_message(msg.as_bytes())
+    assert parsed.has_attachments
+    assert parsed.attachments[0].filename == "file.bin"
+    assert parsed.attachments[0].payload == b"\x00\x01\x02data"
+
+
+def test_missing_message_id_returns_none() -> None:
+    msg = EmailMessage()
+    msg["Subject"] = "no id"
+    msg["From"] = "a@b.com"
+    msg["To"] = "c@d.com"
+    msg.set_content("body")
+    parsed = parse_message(msg.as_bytes())
+    assert parsed.message_id is None
@@ -0,0 +1,186 @@
+"""End-to-end indexer pipeline + Postgres search (SPEC §6.4, §8).
+
+Uses a committed session (search reads through its own connection) and deletes
+the user on teardown; the user FK cascade removes all created rows.
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime
+from email.message import EmailMessage
+
+import pytest_asyncio
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.config import get_settings
+from app.indexer.indexer import persist_message
+from app.models import Account, BodyPart, KnownRecipient, Mailbox, Message, User
+from app.providers.base import RemoteMessage
+from app.search.pg import PgSearchBackend
+from app.search.queryparser import to_search_query
+
+
+class Fixture:
+    def __init__(
+        self,
+        session: AsyncSession,
+        user_id: uuid.UUID,
+        account_id: uuid.UUID,
+        mailbox_id: uuid.UUID,
+    ) -> None:
+        self.session = session
+        self.user_id = user_id
+        self.account_id = account_id
+        self.mailbox_id = mailbox_id
+        self.backend = PgSearchBackend()
+        self._uid = 0
+
+    async def ingest(self, raw: bytes, *, flags: frozenset[str] = frozenset()) -> Message | None:
+        self._uid += 1
+        remote = RemoteMessage(uid=self._uid, raw=raw, flags=flags, internal_date=datetime.now(UTC))
+        result = await persist_message(
+            self.session,
+            user_id=self.user_id,
+            account_id=self.account_id,
+            mailbox_id=self.mailbox_id,
+            mailbox_special_use="inbox",
+            remote=remote,
+        )
+        await self.session.commit()
+        if result is not None:
+            await self.backend.index(result.search_doc)
+            return result.message
+        return None
+
+
+@pytest_asyncio.fixture
+async def fx() -> AsyncIterator[Fixture]:
+    engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    session = sm()
+    user = User(handle=f"pipe{uuid.uuid4().hex[:8]}@test.local", password_hash="x")
+    session.add(user)
+    await session.flush()
+    account = Account(
+        user_id=user.id,
+        email="me@example.com",
+        imap_config={},
+        smtp_config={},
+        secret_ct=b"",
+        discovery_src="manual",
+    )
+    session.add(account)
+    await session.flush()
+    mailbox = Mailbox(account_id=account.id, user_id=user.id, path="INBOX", special_use="inbox")
+    session.add(mailbox)
+    await session.commit()
+    try:
+        yield Fixture(session, user.id, account.id, mailbox.id)
+    finally:
+        await session.execute(delete(User).where(User.id == user.id))
+        await session.commit()
+        await session.close()
+        await engine.dispose()
+
+
+def _msg(
+    subject: str,
+    body: str,
+    *,
+    message_id: str,
+    frm: str = "Alice <alice@example.com>",
+    html: str | None = None,
+) -> bytes:
+    m = EmailMessage()
+    m["Subject"] = subject
+    m["From"] = frm
+    m["To"] = "me@example.com"
+    m["Message-ID"] = message_id
+    m["Date"] = "Mon, 09 Mar 2026 08:36:37 +0100"
+    m.set_content(body)
+    if html:
+        m.add_alternative(html, subtype="html")
+    return m.as_bytes()
+
+
+async def test_ingest_creates_rows_and_thread(fx: Fixture) -> None:
+    msg = await fx.ingest(
+        _msg("Invoice 2026", "Please find the invoice attached.", message_id="<a@example.com>")
+    )
+    assert msg is not None
+    assert msg.thread_id is not None
+    body = (await fx.session.scalars(select(BodyPart).where(BodyPart.message_id == msg.id))).all()
+    kinds = {b.kind for b in body}
+    assert "text" in kinds
+    # known_recipients harvested the sender
+    kr = await fx.session.scalar(
+        select(KnownRecipient).where(KnownRecipient.email == "alice@example.com")
+    )
+    assert kr is not None and kr.times_used == 1
+
+
+async def test_idempotent_reingest(fx: Fixture) -> None:
+    raw = _msg("Dup", "body", message_id="<dup@example.com>")
+    first = await fx.ingest(raw)
+    assert first is not None
+    # Same (mailbox_id, uid) — but ingest increments uid; force same uid path:
+    remote = RemoteMessage(uid=1, raw=raw, flags=frozenset(), internal_date=datetime.now(UTC))
+    dup = await persist_message(
+        fx.session,
+        user_id=fx.user_id,
+        account_id=fx.account_id,
+        mailbox_id=fx.mailbox_id,
+        mailbox_special_use="inbox",
+        remote=remote,
+    )
+    assert dup is None  # (mailbox_id, uid=1) already exists
+
+
+async def test_search_finds_indexed_message(fx: Fixture) -> None:
+    await fx.ingest(
+        _msg("Quarterly report", "revenue numbers and projections", message_id="<r@example.com>")
+    )
+    await fx.ingest(_msg("Lunch plans", "let us grab lunch tomorrow", message_id="<l@example.com>"))
+
+    q = to_search_query("revenue", fx.user_id)
+    hits, cursor = await fx.backend.query(q, limit=50, cursor=None)
+    assert len(hits) == 1
+    assert hits[0].highlights  # revenue highlighted
+
+    q2 = to_search_query("lunch", fx.user_id)
+    hits2, _ = await fx.backend.query(q2, limit=50, cursor=None)
+    assert len(hits2) == 1
+
+
+async def test_search_operator_from_filter(fx: Fixture) -> None:
+    await fx.ingest(
+        _msg("Hi", "hello there", message_id="<h1@example.com>", frm="Bob <bob@example.com>")
+    )
+    await fx.ingest(
+        _msg("Yo", "hello again", message_id="<h2@example.com>", frm="Carol <carol@example.com>")
+    )
+    q = to_search_query("hello from:bob", fx.user_id)
+    hits, _ = await fx.backend.query(q, limit=50, cursor=None)
+    assert len(hits) == 1
+
+
+async def test_html_sanitized_stored(fx: Fixture) -> None:
+    msg = await fx.ingest(
+        _msg(
+            "HTML",
+            "text fallback",
+            message_id="<html@example.com>",
+            html="<p>Hello <script>evil()</script><b>world</b></p>",
+        )
+    )
+    assert msg is not None
+    html_part = await fx.session.scalar(
+        select(BodyPart).where(BodyPart.message_id == msg.id, BodyPart.kind == "html_sanitized")
+    )
+    assert html_part is not None
+    assert "<script" not in html_part.content.lower()
+    assert "world" in html_part.content
@@ -0,0 +1,56 @@
+"""HTML sanitization (SPEC §6.4, §10.4, I-8)."""
+
+from __future__ import annotations
+
+import uuid
+
+from app.indexer.sanitize import sanitize_html
+
+MID = uuid.uuid4()
+
+
+def test_scripts_and_handlers_are_removed() -> None:
+    html = '<div onclick="evil()">hi<script>alert(1)</script></div>'
+    result = sanitize_html(html, MID)
+    assert "<script" not in result.html.lower()
+    assert "onclick" not in result.html.lower()
+    assert "hi" in result.html
+
+
+def test_remote_image_is_blocked_and_flagged() -> None:
+    html = '<p>see <img src="http://tracker.example/pixel.gif"> this</p>'
+    result = sanitize_html(html, MID)
+    assert result.has_remote_images
+    assert "tracker.example" not in result.html
+
+
+def test_cid_image_is_rewritten_to_endpoint() -> None:
+    html = '<img src="cid:logo123">'
+    result = sanitize_html(html, MID)
+    assert f"/api/messages/{MID}/cid/logo123" in result.html
+    assert not result.has_remote_images
+
+
+def test_data_image_is_preserved() -> None:
+    html = '<img src="data:image/png;base64,iVBORw0KGgo=">'
+    result = sanitize_html(html, MID)
+    assert "data:image/png" in result.html
+    assert not result.has_remote_images
+
+
+def test_safe_links_survive_with_rel() -> None:
+    html = '<a href="https://example.com">link</a>'
+    result = sanitize_html(html, MID)
+    assert "https://example.com" in result.html
+    assert "javascript:" not in result.html.lower()
+
+
+def test_javascript_uri_is_stripped() -> None:
+    html = '<a href="javascript:alert(1)">x</a>'
+    result = sanitize_html(html, MID)
+    assert "javascript:" not in result.html.lower()
+
+
+def test_sanitizer_version_reported() -> None:
+    result = sanitize_html("<p>hi</p>", MID)
+    assert result.sanitizer_v >= 1