M6 hosted mailboxes (off by default); dark message mode; fixes
Hosted mailboxes (SPEC App. D), behind HOSTED_MAILBOXES (off: unchanged):
- Stalwart mail server, provisioned by `python -m app.hosted.provision`;
directory read from a Postgres view, app passwords only for mail apps
- native account per user, the worker signs in as master for the user
- raw messages of hosted mailboxes only cached, fetched back on demand
- migrations 0025 (accounts.kind, directory view), 0026 (IMAP ids bigint)
Mail:
- dark message mode switch in the reading pane (Thunderbird's algorithm)
- moves and folder creation quote the folder name ("Deleted Items"):
swipe-to-delete / Trash no longer bounce back
- thread unread counters: lock before recount, sorted lock order,
UIDVALIDITY re-import recounts, nightly repair_threads
- sidebar accounts sorted alphabetically, more space above headings
Settings: Security headings renamed; antispam accounts are explicit
(none ticked = none filtered), input widths, Activity hidden.
Plugin bundles served with Cache-Control: no-cache.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,269 @@
|
||||
"""Hosted mailboxes (SPEC App. D) against the dev stack's Stalwart container.
|
||||
|
||||
Sign-in through the directory (password, 2FA, block), app passwords made by the
|
||||
mail server, the permanent native account, and raw messages kept only by the
|
||||
mail server (fetched back on demand). Skipped when the container is not up.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import imaplib
|
||||
import smtplib
|
||||
import ssl
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from email.message import EmailMessage
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import session_scope
|
||||
from app.hosted import admin, native
|
||||
from app.identity import app_passwords
|
||||
from app.identity.passwords import hash_password
|
||||
from app.identity.sessions import create_session
|
||||
from app.identity.user_admin import delete_user, set_blocked
|
||||
from app.indexer import blobstore, rawstore
|
||||
from app.main import create_app
|
||||
from app.models import Account, Attachment, Mailbox, Message, User
|
||||
|
||||
PASSWORD = "Correct-Horse-7"
|
||||
|
||||
|
||||
async def _server_domain() -> str | None:
|
||||
try:
|
||||
settings = await admin.get_one("SystemSettings")
|
||||
domain = await admin.get_one("Domain", str(settings["defaultDomainId"]))
|
||||
except admin.MailServerError:
|
||||
return None
|
||||
return str(domain["name"])
|
||||
|
||||
|
||||
@dataclass
|
||||
class Ctx:
|
||||
user: User
|
||||
web: httpx.AsyncClient
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def ctx(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Ctx]:
|
||||
domain = await _server_domain()
|
||||
if domain is None:
|
||||
pytest.skip("the dev mail server (docker compose service stalwart) is not running")
|
||||
settings = get_settings()
|
||||
monkeypatch.setattr(settings, "hosted_mailboxes", True)
|
||||
monkeypatch.setattr(settings, "app_domain", domain)
|
||||
handle = f"hosted{uuid.uuid4().hex[:8]}@{domain}"
|
||||
async with session_scope() as db:
|
||||
user = User(handle=handle, password_hash=hash_password(PASSWORD))
|
||||
db.add(user)
|
||||
token = await create_session(user.id, "pytest")
|
||||
web = httpx.AsyncClient(
|
||||
transport=ASGITransport(app=create_app()),
|
||||
base_url="http://testserver",
|
||||
cookies={"session": token, "csrf": "c"},
|
||||
headers={"X-CSRF": "c"},
|
||||
)
|
||||
try:
|
||||
yield Ctx(user, web)
|
||||
finally:
|
||||
await web.aclose()
|
||||
await admin.delete_account(handle)
|
||||
async with session_scope() as db:
|
||||
await db.execute(delete(User).where(User.id == user.id))
|
||||
|
||||
|
||||
def _imap_login(user: str, password: str) -> bool:
|
||||
tls = ssl.create_default_context()
|
||||
tls.check_hostname = False
|
||||
tls.verify_mode = ssl.CERT_NONE
|
||||
settings = get_settings()
|
||||
try:
|
||||
# A blocked account's sign-in is not answered: the server drops the
|
||||
# connection after a minute. Not answering within seconds is a refusal.
|
||||
conn = imaplib.IMAP4_SSL(
|
||||
settings.stalwart_imap_host, settings.stalwart_imap_port, ssl_context=tls, timeout=5
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
conn.login(user, password)
|
||||
conn.logout()
|
||||
return True
|
||||
except (imaplib.IMAP4.error, OSError):
|
||||
return False
|
||||
|
||||
|
||||
async def _can_sign_in(user: str, password: str) -> bool:
|
||||
return await asyncio.to_thread(_imap_login, user, password)
|
||||
|
||||
|
||||
def _deliver(to: str, subject: str, attachment: bytes) -> None:
|
||||
msg = EmailMessage()
|
||||
msg["From"] = "Anna Test <anna@example.org>"
|
||||
msg["To"] = to
|
||||
msg["Subject"] = subject
|
||||
msg.set_content("Hello,\n\nthe report is attached.\n")
|
||||
msg.add_attachment(attachment, maintype="application", subtype="octet-stream", filename="r.bin")
|
||||
with smtplib.SMTP(get_settings().stalwart_imap_host, 1025, timeout=20) as smtp:
|
||||
smtp.send_message(msg)
|
||||
|
||||
|
||||
async def test_the_webmail_password_never_opens_the_mailbox(ctx: Ctx) -> None:
|
||||
"""Mail apps sign in with app passwords only — 2FA or not."""
|
||||
assert not await _can_sign_in(ctx.user.handle, PASSWORD)
|
||||
assert not await _can_sign_in(ctx.user.handle, "wrong")
|
||||
|
||||
|
||||
async def test_app_passwords_work_for_mail_apps_and_dav_and_are_revoked_everywhere(
|
||||
ctx: Ctx,
|
||||
) -> None:
|
||||
handle = ctx.user.handle
|
||||
async with session_scope() as db:
|
||||
row, password = await app_passwords.create(db, ctx.user.id, "Thunderbird")
|
||||
assert password.startswith("app_") and len(password) == app_passwords.SERVER_LENGTH
|
||||
assert row.server_id
|
||||
assert await _can_sign_in(handle, password)
|
||||
async with session_scope() as db:
|
||||
assert await app_passwords.verify(db, ctx.user.id, password) is not None # DAV
|
||||
|
||||
# 2FA turned on changes nothing for it.
|
||||
async with session_scope() as db:
|
||||
user = await db.get(User, ctx.user.id)
|
||||
assert user is not None
|
||||
user.totp_secret = b"x"
|
||||
assert await _can_sign_in(handle, password)
|
||||
|
||||
# Blocked: nothing opens the mailbox; unblocked: it does again.
|
||||
async with session_scope() as db:
|
||||
user = await db.get(User, ctx.user.id)
|
||||
assert user is not None
|
||||
await set_blocked(db, user, True)
|
||||
assert not await _can_sign_in(handle, password)
|
||||
async with session_scope() as db:
|
||||
user = await db.get(User, ctx.user.id)
|
||||
assert user is not None
|
||||
await set_blocked(db, user, False)
|
||||
assert await _can_sign_in(handle, password)
|
||||
|
||||
async with session_scope() as db:
|
||||
assert await app_passwords.revoke(db, ctx.user.id, row.id)
|
||||
assert not await _can_sign_in(handle, password)
|
||||
|
||||
|
||||
async def test_the_native_account_keeps_mail_once_and_cannot_be_removed(ctx: Ctx) -> None:
|
||||
from app.sync.engine import bootstrap_and_backfill
|
||||
|
||||
async with session_scope() as db:
|
||||
user = await db.get(User, ctx.user.id)
|
||||
assert user is not None
|
||||
account = await native.ensure(db, user)
|
||||
assert account is not None
|
||||
account_id = account.id
|
||||
payload = uuid.uuid4().bytes * 5000
|
||||
await asyncio.to_thread(_deliver, ctx.user.handle, "Report", payload)
|
||||
|
||||
# Synced like any IMAP account (delivery is asynchronous: retry briefly).
|
||||
message = None
|
||||
for _ in range(20):
|
||||
await bootstrap_and_backfill(account_id)
|
||||
async with session_scope() as db:
|
||||
message = await db.scalar(
|
||||
select(Message).where(Message.account_id == account_id, Message.subject == "Report")
|
||||
)
|
||||
if message is not None:
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
assert message is not None
|
||||
|
||||
# A new mailbox has the plain folder names (not "Deleted Items", …).
|
||||
async with session_scope() as db:
|
||||
folders = dict(
|
||||
(
|
||||
await db.execute(
|
||||
select(Mailbox.special_use, Mailbox.path).where(
|
||||
Mailbox.account_id == account_id
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
assert folders["trash"] == "Trash" and folders["sent"] == "Sent"
|
||||
assert folders["junk"] == "Junk" and folders["archive"] == "Archive"
|
||||
|
||||
# The raw message is not kept for good — only cached.
|
||||
sha = message.blob_sha256
|
||||
assert not blobstore.exists(ctx.user.id, sha)
|
||||
assert rawstore.cache_path(ctx.user.id, sha).exists()
|
||||
|
||||
# Gone from the cache, the attachment is fetched back from the mail server.
|
||||
rawstore.drop(ctx.user.id, sha)
|
||||
async with session_scope() as db:
|
||||
att = await db.scalar(select(Attachment).where(Attachment.message_id == message.id))
|
||||
assert att is not None
|
||||
got = await ctx.web.get(f"/api/messages/{message.id}/attachments/{att.id}?download=1")
|
||||
assert got.status_code == 200 and got.content == payload
|
||||
assert rawstore.cache_path(ctx.user.id, sha).exists()
|
||||
|
||||
# Permanent: not removable, its server settings not changeable.
|
||||
listed = (await ctx.web.get("/api/accounts")).json()
|
||||
assert [a["kind"] for a in listed] == ["native"]
|
||||
assert (await ctx.web.delete(f"/api/accounts/{account_id}")).status_code == 409
|
||||
patched = await ctx.web.patch(f"/api/accounts/{account_id}", json={"password": "x"})
|
||||
assert patched.status_code == 400 and patched.json()["error"]["code"] == "native_account"
|
||||
|
||||
|
||||
async def test_deleting_the_user_deletes_the_mailbox(ctx: Ctx) -> None:
|
||||
await admin.ensure_account(ctx.user.handle) # brings it into the server
|
||||
assert await admin.account_id(ctx.user.handle) is not None
|
||||
async with session_scope() as db:
|
||||
user = await db.get(User, ctx.user.id)
|
||||
assert user is not None
|
||||
await delete_user(db, user)
|
||||
assert await admin.account_id(ctx.user.handle) is None
|
||||
async with session_scope() as db:
|
||||
assert await db.scalar(select(Account.id).where(Account.user_id == ctx.user.id)) is None
|
||||
|
||||
|
||||
async def test_the_cache_keeps_the_most_recently_used(ctx: Ctx) -> None:
|
||||
import os
|
||||
import time
|
||||
|
||||
uid = ctx.user.id
|
||||
old = rawstore.put_cached(uid, b"a" * 1000)
|
||||
new = rawstore.put_cached(uid, b"b" * 1000)
|
||||
past = time.time() - 60
|
||||
os.utime(rawstore.cache_path(uid, old), (past, past))
|
||||
rawstore.prune(uid, max_bytes=1500)
|
||||
assert not rawstore.cache_path(uid, old).exists()
|
||||
assert rawstore.cache_path(uid, new).exists()
|
||||
stale = time.time() - rawstore.KEEP_S - 10
|
||||
os.utime(rawstore.cache_path(uid, new), (stale, stale))
|
||||
rawstore.prune(uid)
|
||||
assert not rawstore.cache_path(uid, new).exists()
|
||||
|
||||
|
||||
async def test_moves_reach_folders_with_spaces_in_their_names(ctx: Ctx) -> None:
|
||||
"""Older mailboxes (and many providers) say "Deleted Items": the name goes
|
||||
over the wire quoted, or the move is refused and Trash never fills."""
|
||||
from app.providers.imap_smtp import ImapSmtpProvider
|
||||
|
||||
await admin.ensure_account(ctx.user.handle)
|
||||
msg = EmailMessage()
|
||||
msg["From"] = "a@example.com"
|
||||
msg["To"] = ctx.user.handle
|
||||
msg["Subject"] = "Swipe me"
|
||||
msg.set_content("x")
|
||||
provider = ImapSmtpProvider(native.native_provider_config(ctx.user.handle))
|
||||
try:
|
||||
await provider.create_mailbox("Deleted Items")
|
||||
uid = await provider.append("INBOX", msg.as_bytes(), frozenset())
|
||||
moved = await provider.move(uid, "INBOX", "Deleted Items")
|
||||
assert moved > 0
|
||||
finally:
|
||||
await provider.close()
|
||||
Reference in New Issue
Block a user