diff --git a/backend/app/accounts/testing.py b/backend/app/accounts/testing.py index c26b7d9..d070b28 100644 --- a/backend/app/accounts/testing.py +++ b/backend/app/accounts/testing.py @@ -14,6 +14,7 @@ import aiosmtplib from app.accounts.ssrf import SsrfError, assert_host_is_public from app.accounts.types import ServerConfig +from app.config import smtp_helo_name from app.providers.base import ProviderAuthError, ProviderConnectError from app.providers.imap_smtp import ImapSmtpProvider, ProviderConfig @@ -53,6 +54,7 @@ async def _test_smtp(config: ServerConfig, username: str, password: str) -> None use_tls=config.tls == "ssl", start_tls=config.tls == "starttls", timeout=_SMTP_TIMEOUT, + local_hostname=smtp_helo_name(), ) try: await smtp.connect() diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index d36002c..51bc7b5 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -16,6 +16,7 @@ def register_routers(app: FastAPI) -> None: from app.api.admin import router as admin_router from app.api.avatar import router as avatar_router from app.api.compose import router as compose_router + from app.api.invitations import router as invitations_router from app.api.mailboxes import router as mailboxes_router from app.api.messages import router as messages_router from app.api.plugins import router as plugins_router @@ -46,6 +47,7 @@ def register_routers(app: FastAPI) -> None: admin_router, spellcheck_router, push_router, + invitations_router, ): app.include_router(router, prefix="/api") app.include_router(ws_router, prefix="/api") diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 2535eb9..07e1e12 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -9,7 +9,9 @@ their own account, so the instance can never lock out its last admin that way. from __future__ import annotations +import base64 import datetime as dt +import re import uuid from typing import Annotated @@ -18,11 +20,24 @@ from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from app.api.errors import raise_api_error +from app.config import get_settings from app.db import get_session from app.hosted.admin import MailServerError +from app.identity import invitations from app.identity.deps import CurrentAdmin, require_csrf -from app.identity.mailer import LOGO_HTML, MailerError, mailer_config, send_mail +from app.identity.mailer import ( + LOGO_CID, + LOGO_DATA_URL, + LOGO_HTML, + MailerError, + mailer_config, + send_mail, +) from app.identity.schemas import ( + AdminInvitePreviewIn, + AdminInvitePreviewOut, + AdminInviteTemplateIn, + AdminInviteTemplateOut, AdminMailIn, AdminMailOut, AdminMailTestIn, @@ -46,6 +61,7 @@ def _out(row: AppSettings) -> AdminSettingsOut: return AdminSettingsOut( registration_open=row.registration_open, default_quota_bytes=row.default_quota_bytes, + invitations_enabled=row.invitations_enabled, ) @@ -68,6 +84,8 @@ async def write_settings( row.registration_open = body.registration_open if body.default_quota_bytes is not None: row.default_quota_bytes = body.default_quota_bytes + if body.invitations_enabled is not None: + row.invitations_enabled = body.invitations_enabled row.updated_at = dt.datetime.now(dt.UTC) await db.commit() return _out(row) @@ -141,6 +159,109 @@ async def test_mail(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) - return MessageOut() +# --- Invitation email ------------------------------------------------------------ + +_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+") +_SAMPLE = {"invitee_name": "Anna Nowak", "inviter_name": "Jan Kowalski"} + + +def _template_out(row: AppSettings) -> AdminInviteTemplateOut: + return AdminInviteTemplateOut( + subject=row.invite_subject, + html=row.invite_template, + from_address=row.invite_from, + default_subject=invitations.DEFAULT_SUBJECT, + default_html=invitations.DEFAULT_TEMPLATE, + default_from=row.smtp_from, + placeholders=list(invitations.PLACEHOLDERS), + ) + + +@router.get("/invitations/template", response_model=AdminInviteTemplateOut) +async def read_invite_template(admin: CurrentAdmin, db: DbSession) -> AdminInviteTemplateOut: + return _template_out(await app_settings(db)) + + +@router.put( + "/invitations/template", + response_model=AdminInviteTemplateOut, + dependencies=[Depends(require_csrf)], +) +async def write_invite_template( + body: AdminInviteTemplateIn, admin: CurrentAdmin, db: DbSession +) -> AdminInviteTemplateOut: + """Upload the invitation email's HTML template, subject and sender.""" + row = await app_settings(db, for_update=True) + if body.html is not None: + html = body.html.strip() + if html and (problem := invitations.template_problem(html)): + raise_api_error(400, "invalid_template", problem) + row.invite_template = html + if body.subject is not None: + row.invite_subject = " ".join(body.subject.split()) + if body.from_address is not None: + sender = body.from_address.strip() + if sender and not _EMAIL_RE.fullmatch(sender): + raise_api_error(400, "invalid_email", "the sender must be an email address") + row.invite_from = sender + row.updated_at = dt.datetime.now(dt.UTC) + await db.commit() + return _template_out(row) + + +def _sample(row: AppSettings, subject: str, html: str) -> invitations.RenderedMail: + """The invitation as a friend would get it, with sample names.""" + if html.strip() and (problem := invitations.template_problem(html)): + raise_api_error(400, "invalid_template", problem) + draft = AppSettings(invite_subject=subject, invite_template=html.strip()) + base = row.public_url.rstrip("/") or f"https://{get_settings().app_domain}" + return invitations.render(draft, link=f"{base}/register#invite=sample", **_SAMPLE) + + +@router.post( + "/invitations/preview", + response_model=AdminInvitePreviewOut, + dependencies=[Depends(require_csrf)], +) +async def preview_invite( + body: AdminInvitePreviewIn, admin: CurrentAdmin, db: DbSession +) -> AdminInvitePreviewOut: + """Render a template (saved or not) with sample names, for the panel to show.""" + mail = _sample(await app_settings(db), body.subject, body.html) + html = mail.html + # The preview frame has no message parts: show the images it would carry. + for part in mail.inline: + data = base64.b64encode(part.data).decode() + html = html.replace(f"cid:{part.cid}", f"data:{part.mime_type};base64,{data}") + html = html.replace(f"cid:{LOGO_CID}", LOGO_DATA_URL) + return AdminInvitePreviewOut(subject=mail.subject, html=html) + + +@router.post("/invitations/test", response_model=MessageOut, dependencies=[Depends(require_csrf)]) +async def test_invite(body: AdminMailTestIn, admin: CurrentAdmin, db: DbSession) -> MessageOut: + """Send the saved invitation email, with sample names and a dummy link.""" + row = await app_settings(db) + cfg = mailer_config(row) + if cfg is None: + raise_api_error( + 409, "mail_not_configured", "set the SMTP server, From address and public address first" + ) + mail = _sample(row, row.invite_subject, row.invite_template) + try: + await send_mail( + cfg, + body.to.strip(), + mail.subject, + mail.text, + mail.html, + sender=invitations.sender_address(row), + inline=mail.inline, + ) + except MailerError as exc: + raise_api_error(502, "mail_failed", f"sending failed: {exc}") + return MessageOut() + + # --- Users -------------------------------------------------------------------- diff --git a/backend/app/api/invitations.py b/backend/app/api/invitations.py new file mode 100644 index 0000000..f57af5f --- /dev/null +++ b/backend/app/api/invitations.py @@ -0,0 +1,123 @@ +"""Invite a friend (app/identity/invitations.py): what the sidebar entry and its +dialog need — how many invitations are left, and sending one.""" + +from __future__ import annotations + +import datetime as dt +import logging +import re +from typing import Annotated + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import raise_api_error +from app.db import get_session +from app.identity import invitations +from app.identity.deps import CurrentUser, require_csrf +from app.identity.mailer import MailerError, mailer_config, send_mail +from app.identity.service import app_settings +from app.models import Invitation + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/invitations", tags=["invitations"]) + +DbSession = Annotated[AsyncSession, Depends(get_session)] + +_EMAIL_RE = re.compile(r"[^@\s<>\"]+@[^@\s<>\"]+\.[^@\s<>\"]+") + + +class SentInvitationOut(BaseModel): + invitee_name: str + invitee_email: str + created_at: str + used: bool + expired: bool + + +class InvitationsOut(BaseModel): + # Invitations are switched on and system mail can send them. + available: bool + remaining: int + per_user: int + sent: list[SentInvitationOut] + + +class InvitationIn(BaseModel): + name: str = Field(min_length=1, max_length=100) + email: str = Field(min_length=3, max_length=320) + + +@router.get("", response_model=InvitationsOut) +async def read(user: CurrentUser, db: DbSession) -> InvitationsOut: + row = await app_settings(db) + now = dt.datetime.now(dt.UTC) + return InvitationsOut( + available=row.invitations_enabled and mailer_config(row) is not None, + remaining=await invitations.remaining(db, user.id), + per_user=invitations.PER_USER, + sent=[ + SentInvitationOut( + invitee_name=i.invitee_name, + invitee_email=i.invitee_email, + created_at=i.created_at.isoformat(), + used=i.used_at is not None, + expired=i.used_at is None and i.expires_at <= now, + ) + for i in await invitations.sent(db, user.id) + ], + ) + + +@router.post("", response_model=InvitationsOut, dependencies=[Depends(require_csrf)]) +async def send(body: InvitationIn, user: CurrentUser, db: DbSession) -> InvitationsOut: + """Email a friend a single-use sign-up link. Sent before answering, so the + user hears at once if it could not go out (and keeps the invitation).""" + name = " ".join(body.name.split()) + email = body.email.strip() + if not name: + raise_api_error(400, "invalid_name", "enter your friend's name") + if not _EMAIL_RE.fullmatch(email): + raise_api_error(400, "invalid_email", "enter a valid email address") + row = await app_settings(db) + if not row.invitations_enabled: + raise_api_error(403, "invitations_disabled", "invitations are switched off") + cfg = mailer_config(row) + if cfg is None: + raise_api_error(503, "mail_not_configured", "the server cannot send email yet") + try: + invitation, token = await invitations.create(db, user, name=name, email=email) + except invitations.InvitationError as exc: + raise_api_error(409, exc.code, exc.message) + await db.commit() + + # The token rides in the fragment: never sent to a server, so it stays out + # of access logs and Referer headers (as with password-reset links). + link = f"{cfg.public_url}/register#invite={token}" + mail = invitations.render( + row, + invitee_name=name, + inviter_name=await invitations.inviter_name(db, user), + link=link, + ) + try: + await send_mail( + cfg, + email, + mail.subject, + mail.text, + mail.html, + sender=invitations.sender_address(row), + reply_to=user.handle, # an answer reaches the friend who invited + inline=mail.inline, + ) + except MailerError as exc: + logger.warning("invitation from %s could not be sent: %s", user.handle, exc) + # Not sent: the user keeps the invitation. + await db.execute(delete(Invitation).where(Invitation.id == invitation.id)) + await db.commit() + raise_api_error(502, "mail_failed", "the invitation could not be sent — try again later") + return await read(user, db) diff --git a/backend/app/config.py b/backend/app/config.py index 00fd888..ce2c775 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -74,6 +74,9 @@ class Settings(BaseSettings): # New passwords are checked against known data breaches (Have I Been Pwned's # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server). password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK") + # The name the webmail gives itself when it connects to a mail server to + # send (SMTP EHLO). Empty: APP_DOMAIN — not the container's random name. + smtp_helo_name: str = Field(default="", alias="SMTP_HELO_NAME") # Auto-purge Trash/Spam messages older than this many days (0 disables). trash_retention_days: int = Field(default=30, alias="TRASH_RETENTION_DAYS") @@ -109,3 +112,9 @@ class Settings(BaseSettings): def get_settings() -> Settings: """Return the process-wide settings singleton.""" return Settings() # values are sourced from the environment + + +def smtp_helo_name() -> str: + """The name to introduce ourselves with to SMTP servers (EHLO).""" + settings = get_settings() + return settings.smtp_helo_name.strip() or settings.app_domain diff --git a/backend/app/identity/invitations.py b/backend/app/identity/invitations.py new file mode 100644 index 0000000..10f4a87 --- /dev/null +++ b/backend/app/identity/invitations.py @@ -0,0 +1,240 @@ +"""Invitations: a user sends a friend a single-use sign-up link. + +While the administrator has invitations switched on, every user may send +``PER_USER`` of them. A link lets exactly one account be created — even with +public registration closed — and works for ``VALID_FOR``; one that ran out +unused is handed back to its sender. Only a hash of the link's token is kept. + +The email is built from the administrator's template (or the built-in one) +with these placeholders, written ``{{name}}``: + + invitee_name the friend's name, as the sender typed it + inviter_name the sender's name (their mail account's display name, + else their address) + invite_link the sign-up address itself + invite_button a ready-made button linking to it + logo the Ultimo Email logo + +Names are HTML-escaped where they go into the HTML: they come from users and +end up in mail to strangers. +""" + +from __future__ import annotations + +import datetime as dt +import hashlib +import html as html_lib +import re +import secrets +import uuid +from dataclasses import dataclass + +import nh3 +from sqlalchemy import func, or_, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import get_settings +from app.identity.mailer import LOGO_HTML +from app.models import Account, AppSettings, Invitation, User +from app.outbox.inline_images import InlinePart, data_image + +PER_USER = 2 +VALID_FOR = dt.timedelta(days=14) +MAX_TEMPLATE_BYTES = 500 * 1024 + +PLACEHOLDERS = ("invitee_name", "inviter_name", "invite_link", "invite_button", "logo") +_PLACEHOLDER = re.compile(r"\{\{\s*([a-z_]+)\s*\}\}") +_DATA_SRC = re.compile(r"""(\bsrc\s*=\s*)(["'])(data:image/[^"']+)\2""", re.IGNORECASE) + +DEFAULT_SUBJECT = "{{inviter_name}} invites you to Ultimo Email" +_BODY = "font-family:Arial,Helvetica,sans-serif;font-size:15px;color:#1f2937;line-height:1.5" +_MUTED = "font-size:13px;color:#6b7280" +_BUTTON = ( + "display:inline-block;background:#2563eb;color:#ffffff;text-decoration:none;" + "padding:10px 18px;border-radius:8px;font-weight:bold" +) +DEFAULT_TEMPLATE = f""" + +{{{{logo}}}} +

Hi {{{{invitee_name}}}},

+

{{{{inviter_name}}}} invites you to Ultimo Email — private email, +calendar and contacts in one place.

+

{{{{invite_button}}}}

+

Or paste this address into your browser:
{{{{invite_link}}}}

+

The invitation works once and expires in 14 days. If you +weren't expecting it, you can ignore this email.

+ +""" + + +class InvitationError(Exception): + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +@dataclass(frozen=True) +class RenderedMail: + subject: str + text: str + html: str + inline: list[InlinePart] + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +def sender_address(row: AppSettings) -> str: + """The invitation's From. Empty: the system-mail From address, the one the + SMTP account is known to be allowed to send as.""" + return row.invite_from.strip() or row.smtp_from + + +def template_problem(html: str) -> str | None: + """Why ``html`` cannot be used as the invitation template, or None.""" + if len(html.encode()) > MAX_TEMPLATE_BYTES: + return f"the template is larger than {MAX_TEMPLATE_BYTES // 1024} KB" + used = set(_PLACEHOLDER.findall(html)) + unknown = sorted(used - set(PLACEHOLDERS)) + if unknown: + return "unknown placeholder: " + ", ".join("{{" + u + "}}" for u in unknown) + if not used & {"invite_link", "invite_button"}: + return "the template must contain {{invite_link}} or {{invite_button}}" + return None + + +def render(row: AppSettings, *, invitee_name: str, inviter_name: str, link: str) -> RenderedMail: + """The invitation email for one friend, from the configured template.""" + template = row.invite_template or DEFAULT_TEMPLATE + subject_template = row.invite_subject or DEFAULT_SUBJECT + esc_link = html_lib.escape(link, quote=True) + html_values = { + "invitee_name": html_lib.escape(invitee_name), + "inviter_name": html_lib.escape(inviter_name), + "invite_link": f'{esc_link}', + "invite_button": f'Accept the invitation', + "logo": LOGO_HTML, + } + text_values = { + "invitee_name": invitee_name, + "inviter_name": inviter_name, + "invite_link": link, + "invite_button": link, + "logo": "", + } + + def fill(source: str, values: dict[str, str]) -> str: + return _PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), source) + + # An image written into the template itself (data:) goes as an inline part: + # Gmail does not show data: images. + inline: list[InlinePart] = [] + + def to_cid(match: re.Match[str]) -> str: + part = data_image(match.group(3), get_settings().app_domain) + if part is None: + return match.group(0) + inline.append(part) + return f"{match.group(1)}{match.group(2)}cid:{part.cid}{match.group(2)}" + + html = _DATA_SRC.sub(to_cid, fill(template, html_values)) + text = fill(template, text_values) + # Keep the paragraphs and line breaks the HTML shows once its tags are gone. + text = re.sub(r"", "\n", text, flags=re.IGNORECASE) + text = re.sub(r"", "\n\n", text, flags=re.IGNORECASE) + text = nh3.clean(text, tags=set(), attributes={}, strip_comments=True) + text = html_lib.unescape(re.sub(r"\n\s*\n\s*", "\n\n", text)).strip() + "\n" + subject = " ".join(fill(subject_template, text_values).split()) + return RenderedMail(subject=subject, text=text, html=html, inline=inline) + + +async def inviter_name(db: AsyncSession, user: User) -> str: + """How the friend is told who invited them.""" + names = ( + await db.execute( + select(Account.email, Account.display_name).where(Account.user_id == user.id) + ) + ).all() + own = next((n for e, n in names if e.lower() == user.handle.lower() and n), None) + return own or next((n for _, n in names if n), None) or user.handle + + +def _counts(now: dt.datetime): # type: ignore[no-untyped-def] + # Used, or still valid: an invitation that expired unused is given back. + return or_(Invitation.used_at.isnot(None), Invitation.expires_at > now) + + +async def remaining(db: AsyncSession, user_id: uuid.UUID) -> int: + now = dt.datetime.now(dt.UTC) + taken = await db.scalar( + select(func.count()) + .select_from(Invitation) + .where(Invitation.inviter_id == user_id, _counts(now)) + ) + return max(0, PER_USER - int(taken or 0)) + + +async def sent(db: AsyncSession, user_id: uuid.UUID) -> list[Invitation]: + return list( + await db.scalars( + select(Invitation) + .where(Invitation.inviter_id == user_id) + .order_by(Invitation.created_at.desc()) + ) + ) + + +async def create(db: AsyncSession, user: User, *, name: str, email: str) -> tuple[Invitation, str]: + """A new invitation from ``user`` (not committed): the row and the token. + Raises :class:`InvitationError` when the user has none left.""" + # One at a time per user, so two quick clicks cannot both take the last one. + await db.execute(select(User.id).where(User.id == user.id).with_for_update()) + if await remaining(db, user.id) <= 0: + raise InvitationError("no_invitations_left", "you have no invitations left") + token = secrets.token_urlsafe(32) + invitation = Invitation( + inviter_id=user.id, + token_hash=hash_token(token), + invitee_name=name, + invitee_email=email, + expires_at=dt.datetime.now(dt.UTC) + VALID_FOR, + ) + db.add(invitation) + await db.flush() + return invitation, token + + +async def find_valid(db: AsyncSession, token: str) -> Invitation | None: + """The unused, unexpired invitation behind ``token``, without using it.""" + invitation: Invitation | None = await db.scalar( + select(Invitation).where( + Invitation.token_hash == hash_token(token), + Invitation.used_at.is_(None), + Invitation.expires_at > dt.datetime.now(dt.UTC), + ) + ) + return invitation + + +async def claim(db: AsyncSession, token: str) -> uuid.UUID | None: + """Use the invitation up (in the caller's transaction): its id, or None if + it is unknown, used or expired. Atomic — two sign-ups with one link cannot + both get it.""" + return await db.scalar( + update(Invitation) + .where( + Invitation.token_hash == hash_token(token), + Invitation.used_at.is_(None), + Invitation.expires_at > dt.datetime.now(dt.UTC), + ) + .values(used_at=func.now()) + .returning(Invitation.id) + ) + + +async def record_user(db: AsyncSession, invitation_id: uuid.UUID, user_id: uuid.UUID) -> None: + await db.execute( + update(Invitation).where(Invitation.id == invitation_id).values(used_by=user_id) + ) diff --git a/backend/app/identity/mailer.py b/backend/app/identity/mailer.py index e1be7f0..ed467c5 100644 --- a/backend/app/identity/mailer.py +++ b/backend/app/identity/mailer.py @@ -1,6 +1,7 @@ """Outgoing system mail (SPEC §6.1): mail the server sends on its own behalf. -Today that is only the password-reset link. It goes out through the SMTP +Password-reset links, and invitations users send their friends +(app/identity/invitations.py). It goes out through the SMTP account the administrator configured in the admin panel (``app_settings``), never through a user's own mail account — the whole point is that the user cannot sign in. The administrator's server is trusted, so unlike user mail @@ -10,19 +11,26 @@ on the LAN). from __future__ import annotations +import base64 import html as html_lib import logging +from collections.abc import Sequence from dataclasses import dataclass from email.message import EmailMessage -from email.utils import formataddr, make_msgid +from email.utils import formataddr, formatdate, make_msgid from functools import cache from pathlib import Path +from typing import TYPE_CHECKING import aiosmtplib +from app.config import smtp_helo_name from app.models import AppSettings from app.security.crypto import decrypt_secret +if TYPE_CHECKING: + from app.outbox.inline_images import InlinePart + logger = logging.getLogger(__name__) SENDER_NAME = "Ultimo Email" @@ -33,9 +41,9 @@ _TIMEOUT = 20.0 # because mail clients (Gmail first) do not display SVG. Rendered at twice the # size it is shown at, for sharp high-DPI screens. _LOGO_FILE = Path(__file__).parent / "assets" / "ultimo-logo.png" -_LOGO_CID = "ultimo-logo@ultimo.email" +LOGO_CID = "ultimo-logo@ultimo.email" LOGO_HTML = ( - f'

Ultimo Email

' ) @@ -45,6 +53,10 @@ def _logo() -> bytes: return _LOGO_FILE.read_bytes() +# For previews in the admin panel, which show the HTML without message parts. +LOGO_DATA_URL = "data:image/png;base64," + base64.b64encode(_LOGO_FILE.read_bytes()).decode() + + class MailerError(Exception): """System mail is not configured, or the SMTP server refused the message.""" @@ -76,19 +88,49 @@ def mailer_config(row: AppSettings) -> MailerConfig | None: ) -async def send_mail(cfg: MailerConfig, to: str, subject: str, text: str, html: str) -> None: +async def send_mail( + cfg: MailerConfig, + to: str, + subject: str, + text: str, + html: str, + *, + sender: str | None = None, + reply_to: str | None = None, + inline: Sequence[InlinePart] = (), +) -> None: + """Send one system message. ``sender`` replaces the configured From address + (the SMTP account must be allowed to send as it); ``inline`` are images the + HTML names by ``cid:``.""" msg = EmailMessage() - msg["From"] = formataddr((SENDER_NAME, cfg.sender)) + msg["From"] = formataddr((SENDER_NAME, sender or cfg.sender)) msg["To"] = to + if reply_to: + msg["Reply-To"] = reply_to msg["Subject"] = subject + # RFC 5322 requires it, and mail without one scores as spam (Gmail). + msg["Date"] = formatdate(localtime=True) msg["Message-ID"] = make_msgid(domain=cfg.sender.rsplit("@", 1)[-1]) msg["Auto-Submitted"] = "auto-generated" # RFC 3834: do not auto-reply to this msg.set_content(text) msg.add_alternative(html, subtype="html") - if f"cid:{_LOGO_CID}" in html: + if f"cid:{LOGO_CID}" in html: html_part = msg.get_body(("html",)) if isinstance(html_part, EmailMessage): - html_part.add_related(_logo(), maintype="image", subtype="png", cid=f"<{_LOGO_CID}>") + html_part.add_related(_logo(), maintype="image", subtype="png", cid=f"<{LOGO_CID}>") + if inline: + html_part = msg.get_body(("html",)) + assert isinstance(html_part, EmailMessage) + for part in inline: + maintype, _, subtype = part.mime_type.partition("/") + html_part.add_related( + part.data, + maintype=maintype, + subtype=subtype, + cid=f"<{part.cid}>", + filename=part.filename, + disposition="inline", + ) try: await aiosmtplib.send( msg, @@ -99,6 +141,7 @@ async def send_mail(cfg: MailerConfig, to: str, subject: str, text: str, html: s use_tls=cfg.security == "ssl", start_tls=cfg.security == "starttls", timeout=_TIMEOUT, + local_hostname=smtp_helo_name(), ) except (aiosmtplib.SMTPException, OSError) as exc: raise MailerError(str(exc) or type(exc).__name__) from exc diff --git a/backend/app/identity/routes.py b/backend/app/identity/routes.py index def3c71..07cde71 100644 --- a/backend/app/identity/routes.py +++ b/backend/app/identity/routes.py @@ -18,7 +18,7 @@ from app.config import get_settings from app.db import get_session from app.events import get_event_bus from app.hosted.admin import MailServerError -from app.identity import app_passwords, trusted_devices +from app.identity import app_passwords, invitations, trusted_devices from app.identity import totp as totp_mod from app.identity.cookies import ( clear_auth_cookies, @@ -40,6 +40,8 @@ from app.identity.schemas import ( AppPasswordCreatedOut, AppPasswordIn, AppPasswordOut, + InvitationCheckIn, + InvitationCheckOut, LoginRequest, MessageOut, PasswordResetCheckIn, @@ -116,12 +118,31 @@ def _user_out(user: User) -> UserOut: async def registration(db: DbSession) -> RegistrationStatusOut: """Whether signup is open. Unauthenticated: the signup page needs it to render.""" status = await registration_status(db) - return RegistrationStatusOut(open=status.open, bootstrap=status.bootstrap) + return RegistrationStatusOut( + open=status.open, bootstrap=status.bootstrap, domain=get_settings().app_domain + ) _REGISTRATION_ERROR_STATUS = {"handle_taken": 409, "registration_closed": 403} +@router.post("/invitation/check", response_model=InvitationCheckOut) +async def invitation_check(body: InvitationCheckIn, db: DbSession) -> InvitationCheckOut: + """Whether an invitation link still works, and who it is from — for the + sign-up page. Does not use it up.""" + if not (await app_settings(db)).invitations_enabled: + return InvitationCheckOut(valid=False) + invitation = await invitations.find_valid(db, body.token) + if invitation is None: + return InvitationCheckOut(valid=False) + inviter = await db.get(User, invitation.inviter_id) + return InvitationCheckOut( + valid=True, + invitee_name=invitation.invitee_name, + inviter_name=await invitations.inviter_name(db, inviter) if inviter else "", + ) + + @router.post("/register", response_model=UserOut) async def register( body: RegisterRequest, request: Request, response: Response, db: DbSession @@ -130,15 +151,28 @@ async def register( await check_new_password(body.password, resolve_handle(body.localpart)) except PasswordRejected as exc: raise_api_error(400, exc.code, exc.message) + invitation_id: uuid.UUID | None = None + if body.invite: + # Used up in this same transaction: if the sign-up fails (the address + # is taken, …) the rollback hands the invitation back. + if (await app_settings(db)).invitations_enabled: + invitation_id = await invitations.claim(db, body.invite) + if invitation_id is None: + raise_api_error( + 400, "invalid_invitation", "this invitation has already been used or has expired" + ) try: user = await register_user( db, localpart=body.localpart, password=body.password, recovery_email=body.recovery_email, + invited=invitation_id is not None, ) except RegistrationError as exc: raise_api_error(_REGISTRATION_ERROR_STATUS.get(exc.code, 400), exc.code, exc.message) + if invitation_id is not None: + await invitations.record_user(db, invitation_id, user.id) await db.commit() # The user's own mailbox (App. D §D.4): it exists on the mail server as soon # as the user does (the directory reads the users table); this adds it to diff --git a/backend/app/identity/schemas.py b/backend/app/identity/schemas.py index 290e79a..3371169 100644 --- a/backend/app/identity/schemas.py +++ b/backend/app/identity/schemas.py @@ -12,6 +12,9 @@ class RegisterRequest(BaseModel): localpart: str = Field(min_length=3, max_length=30) password: str = Field(min_length=8, max_length=128) recovery_email: str | None = None + # A friend's invitation (app/identity/invitations.py): signs up even while + # public registration is closed, and is used up by it. + invite: str | None = Field(default=None, max_length=200) class LoginRequest(BaseModel): @@ -98,23 +101,39 @@ class UserOut(BaseModel): mail_host: str | None = None +class InvitationCheckIn(BaseModel): + token: str = Field(min_length=1, max_length=200) + + +class InvitationCheckOut(BaseModel): + valid: bool + invitee_name: str = "" + inviter_name: str = "" + + class RegistrationStatusOut(BaseModel): """Public, unauthenticated: what the signup page should show.""" open: bool # True when the instance has no users yet, so signup creates the administrator. bootstrap: bool + # The domain every handle lives under (localpart@domain), shown beside the + # handle field. + domain: str class AdminSettingsOut(BaseModel): registration_open: bool default_quota_bytes: int + # Users may send friends sign-up invitations (app/identity/invitations.py). + invitations_enabled: bool class AdminSettingsIn(BaseModel): """Merge semantics: omitted fields are left as they are.""" registration_open: bool | None = None + invitations_enabled: bool | None = None # Applies to users created after the change; existing mailboxes keep theirs. default_quota_bytes: int | None = Field(default=None, gt=0) @@ -145,6 +164,36 @@ class AdminMailIn(BaseModel): public_url: str | None = Field(default=None, max_length=500) +class AdminInviteTemplateOut(BaseModel): + # Empty strings: the built-in subject / template / sender is used. + subject: str + html: str + from_address: str + default_subject: str + default_html: str + default_from: str + placeholders: list[str] + + +class AdminInviteTemplateIn(BaseModel): + """Merge semantics: omitted fields are left as they are; an empty string + goes back to the built-in one.""" + + subject: str | None = Field(default=None, max_length=300) + html: str | None = None + from_address: str | None = Field(default=None, max_length=320) + + +class AdminInvitePreviewIn(BaseModel): + subject: str = Field(default="", max_length=300) + html: str = "" + + +class AdminInvitePreviewOut(BaseModel): + subject: str + html: str + + class AdminMailTestIn(BaseModel): to: str = Field(min_length=3, max_length=320) diff --git a/backend/app/identity/service.py b/backend/app/identity/service.py index 65a9a21..94fbc5d 100644 --- a/backend/app/identity/service.py +++ b/backend/app/identity/service.py @@ -97,7 +97,12 @@ async def registration_status(db: AsyncSession) -> RegistrationStatus: async def register_user( - db: AsyncSession, *, localpart: str, password: str, recovery_email: str | None + db: AsyncSession, + *, + localpart: str, + password: str, + recovery_email: str | None, + invited: bool = False, ) -> User: """Create a new user. Raises :class:`RegistrationError` on validation/uniqueness. @@ -114,7 +119,7 @@ async def register_user( settings = await app_settings(db, for_update=True) bootstrap = not await _has_users(db) - if not bootstrap and not settings.registration_open: + if not bootstrap and not settings.registration_open and not invited: raise RegistrationError("registration_closed", "registration is closed on this server") user = User( diff --git a/backend/app/models.py b/backend/app/models.py index 18cbebf..8120d0b 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -87,6 +87,13 @@ class AppSettings(Base): # take it (base64url, uncompressed P-256 point). vapid_public: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) vapid_private_ct: Mapped[bytes | None] = mapped_column(BYTEA) + # Users may invite friends (app/identity/invitations.py); the invitation + # email's From address (empty: postmaster@APP_DOMAIN), subject and HTML + # template (empty: the built-in ones). + invitations_enabled: Mapped[bool] = mapped_column(nullable=False, server_default=text("false")) + invite_from: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) + invite_subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) + invite_template: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) updated_at: Mapped[dt.datetime] = _now() @@ -376,6 +383,27 @@ class UserAppPassword(Base): server_id: Mapped[str | None] = mapped_column(Text) +class Invitation(Base): + """A single-use link that lets one person sign up, sent by a user.""" + + __tablename__ = "invitations" + + id: Mapped[uuid.UUID] = _uuid_pk() + inviter_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + # sha256 of the token in the link; the token itself is only in the email. + token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + invitee_name: Mapped[str] = mapped_column(Text, nullable=False) + invitee_email: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[dt.datetime] = _now() + expires_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False) + used_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True)) + used_by: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL") + ) + + class PushSubscription(Base): """A browser or installed app to tell about new mail (Web Push).""" diff --git a/backend/app/outbox/inline_images.py b/backend/app/outbox/inline_images.py index b14c2c6..bad7470 100644 --- a/backend/app/outbox/inline_images.py +++ b/backend/app/outbox/inline_images.py @@ -7,11 +7,19 @@ Sent as is, that URL means nothing to the recipient (and to anyone else, including ourselves once the original is gone). Before sending, each such image is put back into the message as an inline part with a new Content-ID — as any mail program does — and one that can no longer be had is left out. + +An image written into the HTML itself (``data:image/…;base64,…`` — a signature +logo pasted into the editor) goes the same way: Gmail and other mail programs +do not show ``data:`` images in mail they receive, so each becomes an inline +part too (one per distinct image). """ from __future__ import annotations import asyncio +import base64 +import binascii +import hashlib import re import uuid from dataclasses import dataclass @@ -26,6 +34,9 @@ from app.indexer import rawstore from app.indexer.sanitize import INTERNAL_CID from app.models import Attachment, Message +_DATA_IMG = re.compile(r"^data:(image/[a-z0-9.+-]+);base64,(.*)$", re.IGNORECASE | re.DOTALL) +_EXT = {"jpeg": "jpg", "svg+xml": "svg"} + _IMG = re.compile(r"]*?\bsrc\s*=\s*([\"'])(.*?)\1[^>]*>", re.IGNORECASE | re.DOTALL) @@ -37,6 +48,23 @@ class InlinePart: data: bytes +def data_image(src: str, domain: str) -> InlinePart | None: + """The inline part for a ``data:image/…;base64`` URL, or None if it is not one.""" + m = _DATA_IMG.match(src.strip()) + if m is None: + return None + try: + data = base64.b64decode(re.sub(r"\s+", "", m.group(2)), validate=True) + except (binascii.Error, ValueError): + return None + if not data: + return None + mime_type = m.group(1).lower() + subtype = mime_type.split("/", 1)[1] + name = f"image.{_EXT.get(subtype, subtype)}" + return InlinePart(make_msgid(domain=domain)[1:-1], mime_type, name, data) + + def internal_cid(src: str) -> tuple[uuid.UUID, str] | None: """The (message id, content id) an image URL of this server points at.""" value = src.strip() @@ -57,9 +85,20 @@ async def embed( """``html`` with this server's image URLs turned into ``cid:`` references, and the parts to send them in.""" parts: dict[tuple[uuid.UUID, str], InlinePart | None] = {} + # Images written into the HTML, by content: the same logo twice is one part. + data_parts: dict[str, InlinePart | None] = {} + + def data_key(src: str) -> str: + return hashlib.sha256(src.strip().encode()).hexdigest() for match in _IMG.finditer(html): - target = internal_cid(match.group(2)) + src = match.group(2) + if src.strip().lower().startswith("data:"): + key = data_key(src) + if key not in data_parts: + data_parts[key] = data_image(src, domain) + continue + target = internal_cid(src) if target is None or target in parts: continue message_id, content_id = target @@ -90,16 +129,25 @@ async def embed( parts[target] = part def replace(match: re.Match[str]) -> str: + quote = match.group(1) + src = match.group(2) + if src.strip().lower().startswith("data:"): + data_part = data_parts.get(data_key(src)) + if data_part is None: + return match.group(0) # not an image we can carry: left as it was + return match.group(0).replace( + f"src={quote}{src}{quote}", f"src={quote}cid:{data_part.cid}{quote}", 1 + ) target = internal_cid(match.group(2)) if target is None: return match.group(0) part = parts.get(target) if part is None: return "" # gone: no broken image for the recipient - quote = match.group(1) return match.group(0).replace( f"src={quote}{match.group(2)}{quote}", f"src={quote}cid:{part.cid}{quote}", 1 ) - out = _IMG.sub(replace, html) if parts else html - return out, [p for p in parts.values() if p is not None] + out = _IMG.sub(replace, html) if parts or data_parts else html + quoted = [p for p in parts.values() if p is not None] + return out, quoted + [p for p in data_parts.values() if p is not None] diff --git a/backend/app/providers/imap_smtp/provider.py b/backend/app/providers/imap_smtp/provider.py index f83d42d..09337f3 100644 --- a/backend/app/providers/imap_smtp/provider.py +++ b/backend/app/providers/imap_smtp/provider.py @@ -20,7 +20,7 @@ import aiosmtplib from aioimaplib import aioimaplib from app.concurrency import loop_semaphore -from app.config import get_settings +from app.config import get_settings, smtp_helo_name from app.providers.base import ( FlagChange, MailboxDelta, @@ -579,6 +579,8 @@ class ImapSmtpProvider(MailProvider): start_tls=start_tls, tls_context=cfg.tls_context, timeout=_CONNECT_TIMEOUT, + # Our own name, not the container's random host name. + local_hostname=smtp_helo_name(), ) except aiosmtplib.SMTPException as exc: raise _smtp_error(exc) from exc diff --git a/backend/migrations/versions/0027_invitations.py b/backend/migrations/versions/0027_invitations.py new file mode 100644 index 0000000..59668d5 --- /dev/null +++ b/backend/migrations/versions/0027_invitations.py @@ -0,0 +1,63 @@ +"""invitations: users invite friends, even while sign-up is closed + +An administrator switch lets every user send a few single-use invitation links +by email; a link opens sign-up for the person it was sent to even when public +registration is off. The email is built from a template the administrator +uploads (or the built-in one) and sent from the instance's own address. + +Revision ID: 0027 +Revises: 0026 +Create Date: 2026-10-14 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0027" +down_revision: str | None = "0026" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE app_settings + ADD COLUMN invitations_enabled boolean NOT NULL DEFAULT false, + ADD COLUMN invite_from text NOT NULL DEFAULT '', + ADD COLUMN invite_subject text NOT NULL DEFAULT '', + ADD COLUMN invite_template text NOT NULL DEFAULT '' + """ + ) + op.execute( + """ + CREATE TABLE invitations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + inviter_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash text NOT NULL UNIQUE, + invitee_name text NOT NULL, + invitee_email text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + used_at timestamptz, + used_by uuid REFERENCES users(id) ON DELETE SET NULL + ) + """ + ) + op.execute("CREATE INDEX ix_invitations_inviter ON invitations (inviter_id)") + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS invitations") + op.execute( + """ + ALTER TABLE app_settings + DROP COLUMN IF EXISTS invitations_enabled, + DROP COLUMN IF EXISTS invite_from, + DROP COLUMN IF EXISTS invite_subject, + DROP COLUMN IF EXISTS invite_template + """ + ) diff --git a/backend/tests/identity/test_invitations.py b/backend/tests/identity/test_invitations.py new file mode 100644 index 0000000..31c7d8c --- /dev/null +++ b/backend/tests/identity/test_invitations.py @@ -0,0 +1,349 @@ +"""Invite a friend: each user sends up to two single-use sign-up links, which +work even while registration is closed (app/identity/invitations.py). + +Runs against the dev database; the settings row is put back as it was and every +user made here is removed on the way out. Mail is captured, never sent. +""" + +from __future__ import annotations + +import base64 +import datetime as dt +import re +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest +import pytest_asyncio +from httpx import ASGITransport +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool + +from app.api import invitations as invitations_api +from app.config import get_settings +from app.identity import invitations +from app.identity.mailer import MailerError +from app.identity.passwords import hash_password +from app.identity.sessions import create_session +from app.main import create_app +from app.models import Account, AppSettings, Invitation, User + +CSRF = "csrf-token" +_SETTINGS = ( + "registration_open", + "invitations_enabled", + "smtp_host", + "smtp_from", + "public_url", + "invite_from", + "invite_subject", + "invite_template", +) + + +class World: + def __init__(self, session: AsyncSession) -> None: + self.session = session + self.created: list[uuid.UUID] = [] + self.handles: list[str] = [] + self.sent: list[dict[str, Any]] = [] + + async def user(self, *, admin: bool = False, name: str = "") -> User: + handle = f"inv{uuid.uuid4().hex[:8]}@test.local" + user = User(handle=handle, password_hash=hash_password("pw"), is_admin=admin) + self.session.add(user) + await self.session.flush() + if name: + self.session.add( + Account( + user_id=user.id, + email=handle, + display_name=name, + imap_config={}, + smtp_config={}, + secret_ct=b"x", + discovery_src="manual", + ) + ) + await self.session.commit() + self.created.append(user.id) + return user + + async def client(self, user: User | None = None) -> httpx.AsyncClient: + cookies = {"csrf": CSRF} + if user is not None: + cookies["session"] = await create_session(user.id, "pytest") + return httpx.AsyncClient( + transport=ASGITransport(app=create_app()), + base_url="http://testserver", + cookies=cookies, + headers={"X-CSRF": CSRF}, + ) + + async def settings(self, **values: Any) -> None: + await self.session.execute(update(AppSettings).where(AppSettings.id == 1).values(**values)) + await self.session.commit() + + +@pytest_asyncio.fixture +async def world(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[World]: + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + session = async_sessionmaker(engine, expire_on_commit=False)() + row = await session.scalar(select(AppSettings).where(AppSettings.id == 1)) + assert row is not None + saved = {name: getattr(row, name) for name in _SETTINGS} + w = World(session) + + async def capture(cfg, to, subject, text, html, **kw): # type: ignore[no-untyped-def] + w.sent.append({"to": to, "subject": subject, "text": text, "html": html, **kw}) + + monkeypatch.setattr(invitations_api, "send_mail", capture) + await w.settings( + registration_open=False, + invitations_enabled=True, + smtp_host="smtp.example.com", + smtp_from="system@example.com", + public_url="https://mail.example.com", + invite_from="", + invite_subject="", + invite_template="", + ) + try: + yield w + finally: + await session.rollback() + await w.settings(**saved) + await session.execute(delete(User).where(User.id.in_(w.created))) + if w.handles: + await session.execute(delete(User).where(User.handle.in_(w.handles))) + await session.commit() + await session.close() + await engine.dispose() + + +def _token(mail: dict[str, Any]) -> str: + m = re.search(r"#invite=([A-Za-z0-9_-]+)", mail["text"]) + assert m is not None, mail["text"] + return m.group(1) + + +async def _register(w: World, token: str | None) -> httpx.Response: + localpart = f"new{uuid.uuid4().hex[:8]}" + w.handles.append(f"{localpart}@{get_settings().app_domain}") + body: dict[str, Any] = {"localpart": localpart, "password": "correct horse battery"} + if token is not None: + body["invite"] = token + async with await w.client() as anon: + return await anon.post("/api/auth/register", json=body) + + +async def test_two_invitations_then_none(world: World) -> None: + inviter = await world.user(name="Jan Kowalski") + async with await world.client(inviter) as c: + state = (await c.get("/api/invitations")).json() + assert state["available"] is True and state["remaining"] == 2 + + for i, name in enumerate(["Anna", "Piotr"]): + resp = await c.post( + "/api/invitations", json={"name": name, "email": f"f{i}@example.org"} + ) + assert resp.status_code == 200, resp.text + assert resp.json()["remaining"] == 0 + assert [s["invitee_name"] for s in resp.json()["sent"]] == ["Piotr", "Anna"] + + third = await c.post("/api/invitations", json={"name": "Ola", "email": "o@example.org"}) + assert third.status_code == 409 + assert third.json()["error"]["code"] == "no_invitations_left" + + mail = world.sent[0] + assert mail["to"] == "f0@example.org" + assert mail["sender"] == "system@example.com" + assert mail["reply_to"] == inviter.handle + assert mail["subject"] == "Jan Kowalski invites you to Ultimo Email" + assert "Hi Anna" in mail["html"] and "https://mail.example.com/register#invite=" in mail["html"] + + +async def test_an_invitation_signs_up_once_while_registration_is_closed(world: World) -> None: + inviter = await world.user() + async with await world.client(inviter) as c: + await c.post("/api/invitations", json={"name": "Anna", "email": "anna@example.org"}) + token = _token(world.sent[0]) + + closed = await _register(world, None) + assert closed.status_code == 403 # registration itself stays closed + + async with await world.client() as anon: + check = (await anon.post("/api/auth/invitation/check", json={"token": token})).json() + assert check == {"valid": True, "invitee_name": "Anna", "inviter_name": inviter.handle} + + first = await _register(world, token) + assert first.status_code == 200, first.text + again = await _register(world, token) + assert again.status_code == 400 + assert again.json()["error"]["code"] == "invalid_invitation" + + row = await world.session.scalar(select(Invitation).where(Invitation.inviter_id == inviter.id)) + assert row is not None and row.used_at is not None and row.used_by is not None + + +async def test_a_failed_sign_up_keeps_the_invitation(world: World) -> None: + inviter = await world.user() + async with await world.client(inviter) as c: + await c.post("/api/invitations", json={"name": "Anna", "email": "anna@example.org"}) + token = _token(world.sent[0]) + weak = await world.client() + async with weak: + resp = await weak.post( + "/api/auth/register", + json={"localpart": "taken" + uuid.uuid4().hex[:6], "password": "password123"}, + ) + assert resp.status_code == 400 # refused password: nothing used + taken = await _register(world, token) + assert taken.status_code == 200 + # Already used now; but a sign-up that failed on a taken address must not use it: + second_inviter = await world.user() + async with await world.client(second_inviter) as c: + await c.post("/api/invitations", json={"name": "B", "email": "b@example.org"}) + token2 = _token(world.sent[-1]) + async with await world.client() as anon: + dup = await anon.post( + "/api/auth/register", + json={ + "localpart": world.handles[-1].split("@")[0], + "password": "correct horse battery", + "invite": token2, + }, + ) + assert dup.status_code == 409 + assert (await invitations.find_valid(world.session, token2)) is not None + + +async def test_switched_off_means_no_sending_and_no_signing_up(world: World) -> None: + inviter = await world.user() + async with await world.client(inviter) as c: + await c.post("/api/invitations", json={"name": "Anna", "email": "anna@example.org"}) + token = _token(world.sent[0]) + await world.settings(invitations_enabled=False) + assert (await c.get("/api/invitations")).json()["available"] is False + refused = await c.post("/api/invitations", json={"name": "B", "email": "b@example.org"}) + assert refused.status_code == 403 + assert (await _register(world, token)).status_code == 400 + + +async def test_a_mail_that_could_not_go_out_is_given_back( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + async def failing(*args, **kwargs): # type: ignore[no-untyped-def] + raise MailerError("relay refused") + + monkeypatch.setattr(invitations_api, "send_mail", failing) + inviter = await world.user() + async with await world.client(inviter) as c: + resp = await c.post("/api/invitations", json={"name": "Anna", "email": "a@example.org"}) + assert resp.status_code == 502 + assert (await c.get("/api/invitations")).json()["remaining"] == 2 + + +async def test_an_expired_unused_invitation_is_given_back(world: World) -> None: + inviter = await world.user() + async with await world.client(inviter) as c: + await c.post("/api/invitations", json={"name": "Anna", "email": "a@example.org"}) + await world.session.execute( + update(Invitation) + .where(Invitation.inviter_id == inviter.id) + .values(expires_at=dt.datetime.now(dt.UTC) - dt.timedelta(minutes=1)) + ) + await world.session.commit() + state = (await c.get("/api/invitations")).json() + assert state["remaining"] == 2 and state["sent"][0]["expired"] is True + assert (await _register(world, _token(world.sent[0]))).status_code == 400 + + +async def test_bad_input_is_refused(world: World) -> None: + inviter = await world.user() + async with await world.client(inviter) as c: + bad = await c.post("/api/invitations", json={"name": "A", "email": "not-an-address"}) + assert bad.status_code == 400 + blank = await c.post("/api/invitations", json={"name": " ", "email": "a@example.org"}) + assert blank.status_code == 400 + + +# --- the email ------------------------------------------------------------------------ + + +def test_names_are_escaped_and_placeholders_filled() -> None: + row = AppSettings( + invite_subject="{{inviter_name}} → {{invitee_name}}", + invite_template="

Hi {{ invitee_name }}

{{inviter_name}}

{{invite_button}}", + ) + mail = invitations.render( + row, invitee_name="Eve", inviter_name='Bob "x"', link="https://m.example/r#invite=t" + ) + assert "<b>Eve</b>" in mail.html and "Eve" not in mail.html + assert 'href="https://m.example/r#invite=t"' in mail.html + assert mail.subject == 'Bob "x" → Eve' # a header, not HTML + assert "https://m.example/r#invite=t" in mail.text and "

" not in mail.text + + +def test_template_images_travel_inside_the_mail() -> None: + png = base64.b64encode(b"\x89PNG\r\n\x1a\nfake").decode() + row = AppSettings(invite_template=f'{{{{invite_link}}}}') + mail = invitations.render(row, invitee_name="A", inviter_name="B", link="https://x/y") + assert len(mail.inline) == 1 and "data:image" not in mail.html + assert f'src="cid:{mail.inline[0].cid}"' in mail.html + + +@pytest.mark.parametrize( + "html,ok", + [ + ("

{{invite_link}}

", True), + ("

{{invite_button}}

", True), + ("

Hello {{invitee_name}}

", False), # no way to accept + ("

{{invite_link}} {{password}}

", False), # unknown placeholder + ], +) +def test_template_checks(html: str, ok: bool) -> None: + assert (invitations.template_problem(html) is None) is ok + + +async def test_admin_uploads_previews_and_resets_the_template(world: World) -> None: + admin = await world.user(admin=True) + async with await world.client(admin) as c: + bad = await c.put("/api/admin/invitations/template", json={"html": "

hi

"}) + assert bad.status_code == 400 + saved = await c.put( + "/api/admin/invitations/template", + json={ + "html": "

Dear {{invitee_name}}, {{invite_button}}

", + "subject": "Join {{inviter_name}}", + "from_address": "postmaster@example.org", + }, + ) + assert saved.status_code == 200 + assert saved.json()["from_address"] == "postmaster@example.org" + preview = await c.post( + "/api/admin/invitations/preview", + json={"subject": saved.json()["subject"], "html": saved.json()["html"]}, + ) + assert preview.json()["subject"] == "Join Jan Kowalski" + assert "Dear Anna Nowak" in preview.json()["html"] + reset = await c.put( + "/api/admin/invitations/template", + json={"html": "", "subject": "", "from_address": ""}, + ) + assert reset.json()["html"] == "" and reset.json()["default_html"] + user = await world.user() + async with await world.client(user) as c: + assert (await c.get("/api/admin/invitations/template")).status_code == 403 + + +async def test_admin_switch(world: World) -> None: + admin = await world.user(admin=True) + async with await world.client(admin) as c: + off = await c.put("/api/admin/settings", json={"invitations_enabled": False}) + assert off.json()["invitations_enabled"] is False + on = await c.put("/api/admin/settings", json={"invitations_enabled": True}) + assert on.json()["invitations_enabled"] is True diff --git a/backend/tests/identity/test_registration_policy.py b/backend/tests/identity/test_registration_policy.py index 734dead..ebf7a01 100644 --- a/backend/tests/identity/test_registration_policy.py +++ b/backend/tests/identity/test_registration_policy.py @@ -137,7 +137,11 @@ async def test_register_is_refused_while_closed( status = await client.get("/api/auth/registration") assert status.status_code == 200 - assert status.json() == {"open": False, "bootstrap": False} + assert status.json() == { + "open": False, + "bootstrap": False, + "domain": get_settings().app_domain, + } refused = await client.post( "/api/auth/register", json={"localpart": _localpart(), "password": PASSWORD} diff --git a/backend/tests/outbox/test_inline_images.py b/backend/tests/outbox/test_inline_images.py index 1c58a9c..c994be5 100644 --- a/backend/tests/outbox/test_inline_images.py +++ b/backend/tests/outbox/test_inline_images.py @@ -131,3 +131,41 @@ def test_links_already_sent_load_from_here() -> None: result = sanitize_html(f'', uuid.uuid4(), allow_remote_images=True) assert f'src="/api/messages/{mid}/cid/logo@x"' in result.html, src assert not result.has_remote_images + + +async def test_images_written_into_the_html_go_as_inline_parts() -> None: + """A signature logo pasted into the editor is a data: URL, which Gmail does + not show in mail it receives: it is sent as an inline part instead, once + however often it appears.""" + import base64 + + data_url = "data:image/png;base64," + base64.b64encode(LOGO).decode() + html = ( + f'

Hi

--

' + '' # not an image: left alone + ) + out, parts = await inline_images.embed(None, uuid.uuid4(), html, "example.com") # type: ignore[arg-type] + assert len(parts) == 1 + assert parts[0].data == LOGO and parts[0].mime_type == "image/png" + assert parts[0].filename == "image.png" + assert out.count(f'src="cid:{parts[0].cid}"') == 2 + assert "data:image/png" not in out + assert 'width="40"' in out + + account = SimpleNamespace(email="me@example.com", display_name="Me") + draft = SimpleNamespace(payload={"to": ["a@example.org"], "subject": "Sig", "html": html}) + raw, _, _, _ = build_mime(account, draft, None, [], html=out, inline=parts) # type: ignore[arg-type] + msg = email.message_from_bytes(raw, policy=policy.default) + images = [p for p in msg.walk() if p.get_content_type() == "image/png"] + assert len(images) == 1 and images[0].get_content() == LOGO + assert images[0]["Content-ID"] == f"<{parts[0].cid}>" + + +def test_smtp_servers_are_greeted_with_our_domain(monkeypatch) -> None: # type: ignore[no-untyped-def] + from app.config import smtp_helo_name + + settings = get_settings() + monkeypatch.setattr(settings, "smtp_helo_name", "") + assert smtp_helo_name() == settings.app_domain + monkeypatch.setattr(settings, "smtp_helo_name", "mail.example.org") + assert smtp_helo_name() == "mail.example.org" diff --git a/docker-compose.yml b/docker-compose.yml index 6236589..33d6cbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,6 +79,8 @@ x-backend-env: &backend-env SENDER_AVATARS_ENABLED: ${SENDER_AVATARS_ENABLED:-true} # New passwords checked against Have I Been Pwned (k-anonymity; docs §5.2). PASSWORD_BREACH_CHECK: ${PASSWORD_BREACH_CHECK:-true} + # Name given to SMTP servers when sending (EHLO). Empty: APP_DOMAIN. + SMTP_HELO_NAME: ${SMTP_HELO_NAME:-} GOOGLE_OAUTH_ENABLED: ${GOOGLE_OAUTH_ENABLED:-false} OAUTH_GOOGLE_CLIENT_ID: ${OAUTH_GOOGLE_CLIENT_ID:-} OAUTH_GOOGLE_CLIENT_SECRET: ${OAUTH_GOOGLE_CLIENT_SECRET:-} diff --git a/docs/dns-recommendations.md b/docs/dns-recommendations.md new file mode 100644 index 0000000..31e8c17 --- /dev/null +++ b/docs/dns-recommendations.md @@ -0,0 +1,107 @@ +# DNS recommendations — ultimo.email and imaginary-pixels.com + +Checked 2026-10-14, after the security assessment (DMARC/SPF finding) and the +first DMARC test mails. In priority order. See also `docs/mail-server-network.md` +for the hosted-mail (Stalwart) plan. + +## State when checked + +| | ultimo.email (DNS at home.pl) | imaginary-pixels.com (DNS at Cloudflare) | +|---|---|---| +| MX | `serwer2295308.home.pl` (46.242.246.139) | `_dc-mx…` → 46.242.246.139 (home.pl) | +| SPF | `v=spf1 a mx ~all` | `v=spf1 mx a ptr ~all` | +| DKIM | selector `dkim` present | selector `dkim` present, signing verified (PASS at Gmail) | +| DMARC | `p=none; rua=mailto:postmaster@ultimo.email; fo=1` | `p=none; rua=mailto:admin@imaginary-pixels.com` (added 2026-10-14) | +| CAA | none | none | +| DNSSEC | no | no | +| Other | wildcard `*.ultimo.email` → `ultimo.email`; SRV `_imaps`/`_submission` → `ultimo.email:993/587` (ports closed) | — | + +Mail from the webmail goes out through home.pl's SMTP (46.242.246.139), which +both domains' SPF already allow through `mx`. The home IP 178.219.143.96 has +reverse DNS `pub-143.96.multiplay.pl` (cannot be changed by the ISP), so the +QNAP must not send directly to the internet. + +## 1. Remove the mail-app SRV records for ultimo.email — now + +``` +_imaps._tcp.ultimo.email SRV 10 5 993 ultimo.email. +_submission._tcp.ultimo.email SRV 10 5 587 ultimo.email. +``` + +They tell Thunderbird, iPhones and other mail apps that ultimo.email's mail +server is `ultimo.email` on 993/587 — the home IP, where those ports are closed +(hosted mail is not live; mail is at home.pl). Apps that set themselves up from +them fail to connect. Delete both in the home.pl DNS panel; add them back when +Stalwart goes live. + +## 2. Remove the wildcard `*.ultimo.email` — now + +Every made-up name (e.g. `random-xyz123.ultimo.email`) resolves to the home IP +and lands on the Synology; lookups such as `_mta-sts.ultimo.email` and +`_smtp._tls.ultimo.email` come back with the SPF record. Delete the wildcard and +keep explicit records only for names in use: `ultimo.email` itself, plus `www` +if wanted (point 3). `mail.` and `autoconfig.` come back with the Stalwart +launch. Check first whether the wildcard was a home.pl default or deliberate. + +## 3. `www.ultimo.email` — decide + +It resolves but has no certificate, so `https://www.ultimo.email` shows a +browser error. Either drop it (it disappears with the wildcard), or keep it as +`CNAME ultimo.email`, add it to the DSM certificate and add a reverse-proxy +rule for it (HTTP rule with `X-Forwarded-Proto: http`, HTTPS rule like the +main one). + +## 4. CAA for ultimo.email — recommended + +Only the named certificate authority may issue certificates for the domain. +The Synology uses Let's Encrypt: + +``` +ultimo.email. CAA 0 issue "letsencrypt.org" +``` + +Before adding it, confirm with home.pl that they don't issue a certificate for +ultimo.email themselves; if they do, add their CA as another `issue` line or +skip CAA. Not needed for imaginary-pixels.com (Cloudflare manages its +certificates there). + +## 5. SPF cleanups — after reviewing the DMARC reports + +- **ultimo.email:** `v=spf1 mx -all`. The QNAP sends through home.pl (covered + by `mx`), so `a` — the home IP — is not needed. Change it together with the + move to `p=quarantine`. +- **imaginary-pixels.com:** `v=spf1 mx ~all`. `a` points at Cloudflare's web + servers (never send mail); `ptr` is deprecated. First check the reports for + any other legitimate sender (e.g. a website contact form). + +## 6. DMARC staging — both domains + +1. Collect aggregate reports for about 2 weeks (`p=none`). Make sure + `postmaster@ultimo.email` exists at home.pl, or its reports go nowhere. +2. When every legitimate sender passes: `p=quarantine`. +3. After another 2–4 weeks without legitimate failures: `p=reject`. + +DMARC TXT records have a 5-minute TTL, but a missing record is cached for up +to 30 minutes (Cloudflare SOA minimum) — retest a while after creating one. + +## 7. DNSSEC — optional, worth doing + +Protects MX and other records against forged DNS answers. + +- **imaginary-pixels.com:** Cloudflare → DNS → Settings → Enable DNSSEC, then + add the DS record it shows at the registrar. +- **ultimo.email:** check the home.pl panel for a DNSSEC option. + +## Later — when hosted mail (Stalwart) goes live + +- MX to `mail.ultimo.email`; SPF to match the real sending path (with the + home.pl relay, `mx`/home.pl keeps covering it). +- SRV (`_imaps`, `_submission`, `_submissions`) and `autoconfig` back, pointing + at `mail.ultimo.email`. +- Optional MTA-STS + TLS-RPT (needs a real `mta-sts.ultimo.email` with a + certificate — another reason to remove the wildcard now). +- `SMTP_HELO_NAME=mail.ultimo.email` in `docker/.env.prod` if Stalwart's + hostname should be the one announced. + +Not fixable in DNS: the home IP's reverse DNS — avoided by always sending +through home.pl. diff --git a/frontend/src/admin/AdminApp.tsx b/frontend/src/admin/AdminApp.tsx index 25b774c..3ccec19 100644 --- a/frontend/src/admin/AdminApp.tsx +++ b/frontend/src/admin/AdminApp.tsx @@ -1,5 +1,6 @@ import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect, useState } from "react"; +import { createPortal } from "react-dom"; import { useT } from "../i18n"; import { api, ApiError } from "../lib/api"; @@ -163,6 +164,8 @@ function Panel({ handle }: { handle: string }) { // The quota field is a draft until Save — unlike the toggle, you do not want a // write per keystroke. `null` means "not edited yet, show the server's value". const [quotaDraft, setQuotaDraft] = useState(null); + // The invitation email settings open in a dialog of their own. + const [configuring, setConfiguring] = useState(false); const save = useMutation({ mutationFn: api.saveAdminSettings, @@ -216,6 +219,39 @@ function Panel({ handle }: { handle: string }) {

{t("admin.registrationHelp")}

+
+
+ {t("admin.invitations")} +
+ {([true, false] as const).map((value) => ( + + ))} +
+
+

{t("admin.invitationsHelp")}

+ {settings.data.invitations_enabled && ( + + )} +
+ {configuring && setConfiguring(false)} />} +
{t("admin.defaultQuota")} @@ -445,6 +481,225 @@ function SystemMail() { ); } +/** "Configure" under Allow invitations: the invitation email settings in a dialog. */ +function InvitationEmailDialog({ onClose }: { onClose: () => void }) { + const t = useT(); + useEffect(() => { + const onKey = (e: KeyboardEvent) => e.key === "Escape" && onClose(); + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, [onClose]); + return createPortal( +
+
e.stopPropagation()} + > + + +
+
, + document.body, + ); +} + +/** The invitation email: sender, subject and an uploaded HTML template with + * placeholders; previewed with sample names, and sent as a test. */ +function InvitationEmail() { + const t = useT(); + const queryClient = useQueryClient(); + const tpl = useQuery({ queryKey: ["adminInviteTemplate"], queryFn: api.adminInviteTemplate }); + // Sender and subject are drafts until Save; null shows the server's value. + const [from, setFrom] = useState(null); + const [subject, setSubject] = useState(null); + const [testTo, setTestTo] = useState(""); + + const save = useMutation({ + mutationFn: api.saveAdminInviteTemplate, + onSuccess: (data) => { + queryClient.setQueryData(["adminInviteTemplate"], data); + setFrom(null); + setSubject(null); + }, + }); + const test = useMutation({ mutationFn: () => api.testInvite(testTo.trim()) }); + const data = tpl.data; + const preview = useQuery({ + queryKey: ["adminInvitePreview", data?.subject, data?.html], + queryFn: () => api.previewInvite(data?.subject ?? "", data?.html ?? ""), + enabled: !!data, + }); + + if (!data) { + return tpl.error instanceof ApiError ?

{tpl.error.message}

: null; + } + const custom = data.html !== ""; + const dirty = (from !== null && from !== data.from_address) || (subject !== null && subject !== data.subject); + const field = + "w-full rounded border border-app-border bg-app-surface px-2 py-1.5 text-sm focus:border-app-accent focus:outline-none"; + const label = "block space-y-1 text-sm"; + + const upload = async (file: File | undefined) => { + if (!file) return; + save.mutate({ html: await file.text() }); + }; + const download = () => { + const blob = new Blob([data.html || data.default_html], { type: "text/html" }); + const a = document.createElement("a"); + a.href = URL.createObjectURL(blob); + a.download = custom ? "invitation.html" : "invitation-default.html"; + a.click(); + URL.revokeObjectURL(a.href); + }; + + return ( +
+

{t("admin.inviteMail")}

+

{t("admin.inviteMailHelp")}

+
{ + e.preventDefault(); + save.mutate({ + ...(from !== null ? { from_address: from } : {}), + ...(subject !== null ? { subject } : {}), + }); + }} + > + + +
+ +
+
+ +
+

+ {custom ? t("admin.inviteTemplateCustom") : t("admin.inviteTemplateDefault")} +

+
+ + + {custom && ( + + )} +
+

+ {t("admin.invitePlaceholders")}{" "} + {data.placeholders.map((p, i) => ( + + {i > 0 && ", "} + {`{{${p}}}`} + + ))} +

+

{t("admin.invitePlaceholdersHelp")}

+ {save.error instanceof ApiError &&

{save.error.message}

} +
+ + {preview.data && ( +
+

+ {t("admin.invitePreview")} {preview.data.subject} +

+ {/* Sandboxed, script-less: the admin's own HTML, shown as mail. */} +