270 lines
10 KiB
Python
270 lines
10 KiB
Python
 
|
"""Hosted mailboxes (SPEC App. D) against the dev stack's Stalwart container.
|
||
|
|
|
||
|
|
Sign-in through the directory (password, 2FA, block), app passwords made by the
|
||
|
|
mail server, the permanent native account, and raw messages kept only by the
|
||
|
|
mail server (fetched back on demand). Skipped when the container is not up.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import imaplib
|
||
|
|
import smtplib
|
||
|
|
import ssl
|
||
|
|
import uuid
|
||
|
|
from collections.abc import AsyncIterator
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from email.message import EmailMessage
|
||
|
|
|
||
|
|
import httpx
|
||
|
|
import pytest
|
||
|
|
import pytest_asyncio
|
||
|
|
from httpx import ASGITransport
|
||
|
|
from sqlalchemy import delete, select
|
||
|
|
|
||
|
|
from app.config import get_settings
|
||
|
|
from app.db import session_scope
|
||
|
|
from app.hosted import admin, native
|
||
|
|
from app.identity import app_passwords
|
||
|
|
from app.identity.passwords import hash_password
|
||
|
|
from app.identity.sessions import create_session
|
||
|
|
from app.identity.user_admin import delete_user, set_blocked
|
||
|
|
from app.indexer import blobstore, rawstore
|
||
|
|
from app.main import create_app
|
||
|
|
from app.models import Account, Attachment, Mailbox, Message, User
|
||
|
|
|
||
|
|
PASSWORD = "Correct-Horse-7"
|
||
|
|
|
||
|
|
|
||
|
|
async def _server_domain() -> str | None:
|
||
|
|
try:
|
||
|
|
settings = await admin.get_one("SystemSettings")
|
||
|
|
domain = await admin.get_one("Domain", str(settings["defaultDomainId"]))
|
||
|
|
except admin.MailServerError:
|
||
|
|
return None
|
||
|
|
return str(domain["name"])
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Ctx:
|
||
|
|
user: User
|
||
|
|
web: httpx.AsyncClient
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def ctx(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Ctx]:
|
||
|
|
domain = await _server_domain()
|
||
|
|
if domain is None:
|
||
|
|
pytest.skip("the dev mail server (docker compose service stalwart) is not running")
|
||
|
|
settings = get_settings()
|
||
|
|
monkeypatch.setattr(settings, "hosted_mailboxes", True)
|
||
|
|
monkeypatch.setattr(settings, "app_domain", domain)
|
||
|
|
handle = f"hosted{uuid.uuid4().hex[:8]}@{domain}"
|
||
|
|
async with session_scope() as db:
|
||
|
|
user = User(handle=handle, password_hash=hash_password(PASSWORD))
|
||
|
|
db.add(user)
|
||
|
|
token = await create_session(user.id, "pytest")
|
||
|
|
web = httpx.AsyncClient(
|
||
|
|
transport=ASGITransport(app=create_app()),
|
||
|
|
base_url="http://testserver",
|
||
|
|
cookies={"session": token, "csrf": "c"},
|
||
|
|
headers={"X-CSRF": "c"},
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
yield Ctx(user, web)
|
||
|
|
finally:
|
||
|
|
await web.aclose()
|
||
|
|
await admin.delete_account(handle)
|
||
|
|
async with session_scope() as db:
|
||
|
|
await db.execute(delete(User).where(User.id == user.id))
|
||
|
|
|
||
|
|
|
||
|
|
def _imap_login(user: str, password: str) -> bool:
|
||
|
|
tls = ssl.create_default_context()
|
||
|
|
tls.check_hostname = False
|
||
|
|
tls.verify_mode = ssl.CERT_NONE
|
||
|
|
settings = get_settings()
|
||
|
|
try:
|
||
|
|
# A blocked account's sign-in is not answered: the server drops the
|
||
|
|
# connection after a minute. Not answering within seconds is a refusal.
|
||
|
|
conn = imaplib.IMAP4_SSL(
|
||
|
|
settings.stalwart_imap_host, settings.stalwart_imap_port, ssl_context=tls, timeout=5
|
||
|
|
)
|
||
|
|
except OSError:
|
||
|
|
return False
|
||
|
|
try:
|
||
|
|
conn.login(user, password)
|
||
|
|
conn.logout()
|
||
|
|
return True
|
||
|
|
except (imaplib.IMAP4.error, OSError):
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
async def _can_sign_in(user: str, password: str) -> bool:
|
||
|
|
return await asyncio.to_thread(_imap_login, user, password)
|
||
|
|
|
||
|
|
|
||
|
|
def _deliver(to: str, subject: str, attachment: bytes) -> None:
|
||
|
|
msg = EmailMessage()
|
||
|
|
msg["From"] = "Anna Test <anna@example.org>"
|
||
|
|
msg["To"] = to
|
||
|
|
msg["Subject"] = subject
|
||
|
|
msg.set_content("Hello,\n\nthe report is attached.\n")
|
||
|
|
msg.add_attachment(attachment, maintype="application", subtype="octet-stream", filename="r.bin")
|
||
|
|
with smtplib.SMTP(get_settings().stalwart_imap_host, 1025, timeout=20) as smtp:
|
||
|
|
smtp.send_message(msg)
|
||
|
|
|
||
|
|
|
||
|
|
async def test_the_webmail_password_never_opens_the_mailbox(ctx: Ctx) -> None:
|
||
|
|
"""Mail apps sign in with app passwords only — 2FA or not."""
|
||
|
|
assert not await _can_sign_in(ctx.user.handle, PASSWORD)
|
||
|
|
assert not await _can_sign_in(ctx.user.handle, "wrong")
|
||
|
|
|
||
|
|
|
||
|
|
async def test_app_passwords_work_for_mail_apps_and_dav_and_are_revoked_everywhere(
|
||
|
|
ctx: Ctx,
|
||
|
|
) -> None:
|
||
|
|
handle = ctx.user.handle
|
||
|
|
async with session_scope() as db:
|
||
|
|
row, password = await app_passwords.create(db, ctx.user.id, "Thunderbird")
|
||
|
|
assert password.startswith("app_") and len(password) == app_passwords.SERVER_LENGTH
|
||
|
|
assert row.server_id
|
||
|
|
assert await _can_sign_in(handle, password)
|
||
|
|
async with session_scope() as db:
|
||
|
|
assert await app_passwords.verify(db, ctx.user.id, password) is not None # DAV
|
||
|
|
|
||
|
|
# 2FA turned on changes nothing for it.
|
||
|
|
async with session_scope() as db:
|
||
|
|
user = await db.get(User, ctx.user.id)
|
||
|
|
assert user is not None
|
||
|
|
user.totp_secret = b"x"
|
||
|
|
assert await _can_sign_in(handle, password)
|
||
|
|
|
||
|
|
# Blocked: nothing opens the mailbox; unblocked: it does again.
|
||
|
|
async with session_scope() as db:
|
||
|
|
user = await db.get(User, ctx.user.id)
|
||
|
|
assert user is not None
|
||
|
|
await set_blocked(db, user, True)
|
||
|
|
assert not await _can_sign_in(handle, password)
|
||
|
|
async with session_scope() as db:
|
||
|
|
user = await db.get(User, ctx.user.id)
|
||
|
|
assert user is not None
|
||
|
|
await set_blocked(db, user, False)
|
||
|
|
assert await _can_sign_in(handle, password)
|
||
|
|
|
||
|
|
async with session_scope() as db:
|
||
|
|
assert await app_passwords.revoke(db, ctx.user.id, row.id)
|
||
|
|
assert not await _can_sign_in(handle, password)
|
||
|
|
|
||
|
|
|
||
|
|
async def test_the_native_account_keeps_mail_once_and_cannot_be_removed(ctx: Ctx) -> None:
|
||
|
|
from app.sync.engine import bootstrap_and_backfill
|
||
|
|
|
||
|
|
async with session_scope() as db:
|
||
|
|
user = await db.get(User, ctx.user.id)
|
||
|
|
assert user is not None
|
||
|
|
account = await native.ensure(db, user)
|
||
|
|
assert account is not None
|
||
|
|
account_id = account.id
|
||
|
|
payload = uuid.uuid4().bytes * 5000
|
||
|
|
await asyncio.to_thread(_deliver, ctx.user.handle, "Report", payload)
|
||
|
|
|
||
|
|
# Synced like any IMAP account (delivery is asynchronous: retry briefly).
|
||
|
|
message = None
|
||
|
|
for _ in range(20):
|
||
|
|
await bootstrap_and_backfill(account_id)
|
||
|
|
async with session_scope() as db:
|
||
|
|
message = await db.scalar(
|
||
|
|
select(Message).where(Message.account_id == account_id, Message.subject == "Report")
|
||
|
|
)
|
||
|
|
if message is not None:
|
||
|
|
break
|
||
|
|
await asyncio.sleep(0.5)
|
||
|
|
assert message is not None
|
||
|
|
|
||
|
|
# A new mailbox has the plain folder names (not "Deleted Items", …).
|
||
|
|
async with session_scope() as db:
|
||
|
|
folders = dict(
|
||
|
|
(
|
||
|
|
await db.execute(
|
||
|
|
select(Mailbox.special_use, Mailbox.path).where(
|
||
|
|
Mailbox.account_id == account_id
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
)
|
||
|
|
assert folders["trash"] == "Trash" and folders["sent"] == "Sent"
|
||
|
|
assert folders["junk"] == "Junk" and folders["archive"] == "Archive"
|
||
|
|
|
||
|
|
# The raw message is not kept for good — only cached.
|
||
|
|
sha = message.blob_sha256
|
||
|
|
assert not blobstore.exists(ctx.user.id, sha)
|
||
|
|
assert rawstore.cache_path(ctx.user.id, sha).exists()
|
||
|
|
|
||
|
|
# Gone from the cache, the attachment is fetched back from the mail server.
|
||
|
|
rawstore.drop(ctx.user.id, sha)
|
||
|
|
async with session_scope() as db:
|
||
|
|
att = await db.scalar(select(Attachment).where(Attachment.message_id == message.id))
|
||
|
|
assert att is not None
|
||
|
|
got = await ctx.web.get(f"/api/messages/{message.id}/attachments/{att.id}?download=1")
|
||
|
|
assert got.status_code == 200 and got.content == payload
|
||
|
|
assert rawstore.cache_path(ctx.user.id, sha).exists()
|
||
|
|
|
||
|
|
# Permanent: not removable, its server settings not changeable.
|
||
|
|
listed = (await ctx.web.get("/api/accounts")).json()
|
||
|
|
assert [a["kind"] for a in listed] == ["native"]
|
||
|
|
assert (await ctx.web.delete(f"/api/accounts/{account_id}")).status_code == 409
|
||
|
|
patched = await ctx.web.patch(f"/api/accounts/{account_id}", json={"password": "x"})
|
||
|
|
assert patched.status_code == 400 and patched.json()["error"]["code"] == "native_account"
|
||
|
|
|
||
|
|
|
||
|
|
async def test_deleting_the_user_deletes_the_mailbox(ctx: Ctx) -> None:
|
||
|
|
await admin.ensure_account(ctx.user.handle) # brings it into the server
|
||
|
|
assert await admin.account_id(ctx.user.handle) is not None
|
||
|
|
async with session_scope() as db:
|
||
|
|
user = await db.get(User, ctx.user.id)
|
||
|
|
assert user is not None
|
||
|
|
await delete_user(db, user)
|
||
|
|
assert await admin.account_id(ctx.user.handle) is None
|
||
|
|
async with session_scope() as db:
|
||
|
|
assert await db.scalar(select(Account.id).where(Account.user_id == ctx.user.id)) is None
|
||
|
|
|
||
|
|
|
||
|
|
async def test_the_cache_keeps_the_most_recently_used(ctx: Ctx) -> None:
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
|
||
|
|
uid = ctx.user.id
|
||
|
|
old = rawstore.put_cached(uid, b"a" * 1000)
|
||
|
|
new = rawstore.put_cached(uid, b"b" * 1000)
|
||
|
|
past = time.time() - 60
|
||
|
|
os.utime(rawstore.cache_path(uid, old), (past, past))
|
||
|
|
rawstore.prune(uid, max_bytes=1500)
|
||
|
|
assert not rawstore.cache_path(uid, old).exists()
|
||
|
|
assert rawstore.cache_path(uid, new).exists()
|
||
|
|
stale = time.time() - rawstore.KEEP_S - 10
|
||
|
|
os.utime(rawstore.cache_path(uid, new), (stale, stale))
|
||
|
|
rawstore.prune(uid)
|
||
|
|
assert not rawstore.cache_path(uid, new).exists()
|
||
|
|
|
||
|
|
|
||
|
|
async def test_moves_reach_folders_with_spaces_in_their_names(ctx: Ctx) -> None:
|
||
|
|
"""Older mailboxes (and many providers) say "Deleted Items": the name goes
|
||
|
|
over the wire quoted, or the move is refused and Trash never fills."""
|
||
|
|
from app.providers.imap_smtp import ImapSmtpProvider
|
||
|
|
|
||
|
|
await admin.ensure_account(ctx.user.handle)
|
||
|
|
msg = EmailMessage()
|
||
|
|
msg["From"] = "a@example.com"
|
||
|
|
msg["To"] = ctx.user.handle
|
||
|
|
msg["Subject"] = "Swipe me"
|
||
|
|
msg.set_content("x")
|
||
|
|
provider = ImapSmtpProvider(native.native_provider_config(ctx.user.handle))
|
||
|
|
try:
|
||
|
|
await provider.create_mailbox("Deleted Items")
|
||
|
|
uid = await provider.append("INBOX", msg.as_bytes(), frozenset())
|
||
|
|
moved = await provider.move(uid, "INBOX", "Deleted Items")
|
||
|
|
assert moved > 0
|
||
|
|
finally:
|
||
|
|
await provider.close()
|