Productivity features, UI translations, local email translation
- Snooze (with a Snoozed folder), keyboard shortcuts with a Settings section, mail templates (Compose > More > Templates), a Subscriptions folder with bulk unsubscribe/delete, and tracker blocking with a Privacy toggle (on by default) - Own dialogs instead of browser popups; emoji picker in compose; compose windows survive a visit to Settings - Sentence completion learned from the user's sent mail (no AI) - Interface in English and Polish (one file per language, English as the fallback), picked from the browser on first run, then kept - Email translation on the server via LibreTranslate: Translate button and "Translate to...", messages in several languages translated part by part, names/addresses/emoji/file names/phone numbers kept, an animated line while translating - Antispam "Mark as spam" loads with the message, not when the menu opens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+176
-36
@@ -6,6 +6,7 @@ import asyncio
|
||||
import contextlib
|
||||
import html as html_lib
|
||||
import io
|
||||
import logging
|
||||
import uuid
|
||||
import zipfile
|
||||
from typing import Annotated
|
||||
@@ -14,11 +15,18 @@ from urllib.parse import quote
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from fastapi.responses import FileResponse, HTMLResponse, Response
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import translation
|
||||
from app.api.errors import raise_api_error
|
||||
from app.api.schemas import AttachmentOut
|
||||
from app.api.settings import add_allowed_image_sender, allowed_image_senders
|
||||
from app.api.settings import (
|
||||
add_allowed_image_sender,
|
||||
allowed_image_senders,
|
||||
blocks_trackers,
|
||||
)
|
||||
from app.completion import html_to_text
|
||||
from app.db import get_session
|
||||
from app.identity.deps import CurrentUser, require_csrf
|
||||
from app.indexer import attachments as att_store
|
||||
@@ -26,7 +34,18 @@ from app.indexer import rawstore, thumbs
|
||||
from app.indexer.mime import parse_message
|
||||
from app.indexer.sanitize import SANITIZER_VERSION, sanitize_html
|
||||
from app.indexer.thumbs import thumb_type
|
||||
from app.models import Attachment, BodyPart, ComposeAttachment, Message
|
||||
from app.models import (
|
||||
Account,
|
||||
Attachment,
|
||||
BodyPart,
|
||||
ComposeAttachment,
|
||||
Message,
|
||||
MessageTranslation,
|
||||
User,
|
||||
)
|
||||
from app.translation_prep import find_names_in_signature
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/messages", tags=["mail"])
|
||||
|
||||
@@ -101,8 +120,16 @@ _MESSAGE_CSP = (
|
||||
)
|
||||
|
||||
|
||||
def _html(document: str) -> HTMLResponse:
|
||||
return HTMLResponse(content=document, headers={"Content-Security-Policy": _MESSAGE_CSP})
|
||||
def _html(document: str, trackers: int = 0, extra: dict[str, str] | None = None) -> HTMLResponse:
|
||||
# The page reads what it needs from headers: the body is the message.
|
||||
return HTMLResponse(
|
||||
content=document,
|
||||
headers={
|
||||
"Content-Security-Policy": _MESSAGE_CSP,
|
||||
"X-Trackers-Blocked": str(trackers),
|
||||
**(extra or {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _owned_message(db: AsyncSession, user_id: uuid.UUID, message_id: uuid.UUID) -> Message:
|
||||
@@ -118,54 +145,167 @@ def _sender_email(message: Message) -> str:
|
||||
return message.from_addr[0].get("email", "").lower() if message.from_addr else ""
|
||||
|
||||
|
||||
async def _render_fragment(
|
||||
db: AsyncSession, message: Message, allow_remote: bool, block: bool
|
||||
) -> tuple[str, int]:
|
||||
"""The sanitized body the user sees, and how many trackers came out of it.
|
||||
|
||||
Re-sanitizes from the raw blob when remote images are allowed, the stored
|
||||
copy predates the current sanitizer (SPEC §10.4), or trackers are to stay
|
||||
in; otherwise the stored copy is served as is.
|
||||
"""
|
||||
parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message.id)))
|
||||
html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
|
||||
|
||||
stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
|
||||
# The stored copy has its trackers taken out; with blocking switched off, a
|
||||
# message that had some is rendered afresh with them left in.
|
||||
trackers_kept = not block and html_part is not None and html_part.trackers_blocked > 0
|
||||
if html_part is not None and not allow_remote and not stale and not trackers_kept:
|
||||
return html_part.content, html_part.trackers_blocked if block else 0
|
||||
|
||||
raw = None
|
||||
if html_part is not None or allow_remote: # a plain-text message has nothing to redo
|
||||
try:
|
||||
raw = await rawstore.ensure_bytes(message)
|
||||
except FileNotFoundError:
|
||||
raw = None
|
||||
parsed = parse_message(raw) if raw is not None else None
|
||||
if parsed is not None and parsed.html_body is not None:
|
||||
if stale and html_part is not None:
|
||||
# The stored copy is always the plain one: no remote images, trackers out.
|
||||
upgraded = sanitize_html(parsed.html_body, message.id)
|
||||
html_part.content = upgraded.html
|
||||
html_part.sanitizer_v = upgraded.sanitizer_v
|
||||
html_part.trackers_blocked = upgraded.trackers_blocked
|
||||
await db.commit()
|
||||
if block and not allow_remote:
|
||||
return upgraded.html, upgraded.trackers_blocked
|
||||
result = sanitize_html(
|
||||
parsed.html_body, message.id, allow_remote_images=allow_remote, block_trackers=block
|
||||
)
|
||||
return result.html, result.trackers_blocked
|
||||
|
||||
if html_part is not None:
|
||||
return html_part.content, html_part.trackers_blocked
|
||||
text_part = next((p for p in parts if p.kind == "text"), None)
|
||||
body = html_lib.escape(text_part.content) if text_part else ""
|
||||
return f'<div class="wm-plaintext">{body}</div>', 0
|
||||
|
||||
|
||||
async def _content_language(db: AsyncSession, message: Message) -> str:
|
||||
"""The message's language, found (and kept) the first time it is shown."""
|
||||
if message.content_language is not None or not translation.enabled():
|
||||
return message.content_language or ""
|
||||
text = await db.scalar(
|
||||
select(BodyPart.content).where(BodyPart.message_id == message.id, BodyPart.kind == "text")
|
||||
)
|
||||
try:
|
||||
found = await translation.detect(text or message.snippet or "")
|
||||
except translation.TranslationError as exc:
|
||||
logger.info("language detection unavailable: %s", exc)
|
||||
return "" # not kept: tried again once the service is back
|
||||
message.content_language = found
|
||||
await db.commit()
|
||||
return found
|
||||
|
||||
|
||||
@router.get("/{message_id}/html", response_class=HTMLResponse)
|
||||
async def message_html(
|
||||
message_id: uuid.UUID,
|
||||
user: CurrentUser,
|
||||
db: DbSession,
|
||||
images: bool = Query(False),
|
||||
translate: str | None = Query(None, pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$"),
|
||||
) -> HTMLResponse:
|
||||
"""Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8).
|
||||
|
||||
Re-sanitizes from the raw blob when the sender is image-approved, or when the
|
||||
stored copy predates the current sanitizer version (SPEC §10.4). ``images=1``
|
||||
is a one-time "Load images" — render remote images for this request only,
|
||||
without persisting the sender as trusted.
|
||||
``images=1`` is a one-time "Load images" — render remote images for this
|
||||
request only, without persisting the sender as trusted. ``translate=pl``
|
||||
returns it translated (app/translation.py). Headers tell the page what it
|
||||
got: X-Trackers-Blocked, X-Content-Language (the message's own language,
|
||||
when known) and, translated, X-Translated-From / X-Translated-To.
|
||||
"""
|
||||
message = await _owned_message(db, user.id, message_id)
|
||||
allow_remote = images or _sender_email(message) in allowed_image_senders(user)
|
||||
block = blocks_trackers(user)
|
||||
fragment, trackers = await _render_fragment(db, message, allow_remote, block)
|
||||
|
||||
parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id)))
|
||||
html_part = next((p for p in parts if p.kind == "html_sanitized"), None)
|
||||
headers: dict[str, str] = {}
|
||||
language = await _content_language(db, message)
|
||||
if language:
|
||||
headers["X-Content-Language"] = language
|
||||
if translate:
|
||||
fragment, source = await _translated(
|
||||
db, user, message, fragment, language, translate, allow_remote, block
|
||||
)
|
||||
headers["X-Translated-From"] = source
|
||||
headers["X-Translated-To"] = translate
|
||||
return _html(_wrap(fragment), trackers, headers)
|
||||
|
||||
stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION
|
||||
if html_part is not None and not allow_remote and not stale:
|
||||
return _html(_wrap(html_part.content))
|
||||
|
||||
# Re-sanitize the original HTML (image-approved sender, or a stale stored copy).
|
||||
# If the raw blob is unavailable, fall back to whatever is already stored.
|
||||
if allow_remote or stale:
|
||||
try:
|
||||
raw = await rawstore.ensure_bytes(message)
|
||||
except FileNotFoundError:
|
||||
raw = None
|
||||
if raw is not None:
|
||||
parsed = parse_message(raw)
|
||||
if parsed.html_body is not None:
|
||||
result = sanitize_html(
|
||||
parsed.html_body, message.id, allow_remote_images=allow_remote
|
||||
)
|
||||
if html_part is not None and not allow_remote:
|
||||
html_part.content = result.html # persist the upgraded sanitization
|
||||
html_part.sanitizer_v = result.sanitizer_v
|
||||
await db.commit()
|
||||
return _html(_wrap(result.html))
|
||||
async def _names(db: AsyncSession, user: User, message: Message) -> list[str]:
|
||||
"""Names a translation leaves as they are if the model would change them:
|
||||
the people on the message (and the sender's single names, as signed:
|
||||
"Milan"), the user's own names (accounts, signatures)."""
|
||||
names: list[str] = []
|
||||
for people in (message.from_addr, message.to_addr, message.cc_addr):
|
||||
names += [p.get("name", "") for p in people if p.get("name")]
|
||||
for p in message.from_addr:
|
||||
names += [w for w in str(p.get("name", "")).split() if len(w) >= 3 and w[0].isupper()]
|
||||
rows = await db.execute(
|
||||
select(Account.display_name, Account.signature).where(Account.user_id == user.id)
|
||||
)
|
||||
for display_name, signature in rows:
|
||||
if display_name:
|
||||
names.append(display_name)
|
||||
if signature:
|
||||
names += find_names_in_signature(html_to_text(signature))
|
||||
return names
|
||||
|
||||
if html_part is not None:
|
||||
return _html(_wrap(html_part.content))
|
||||
text_part = next((p for p in parts if p.kind == "text"), None)
|
||||
body = html_lib.escape(text_part.content) if text_part else ""
|
||||
return _html(_wrap(f'<div class="wm-plaintext">{body}</div>'))
|
||||
|
||||
async def _translated(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
message: Message,
|
||||
fragment: str,
|
||||
language: str,
|
||||
target: str,
|
||||
allow_remote: bool,
|
||||
block: bool,
|
||||
) -> tuple[str, str]:
|
||||
"""The fragment in ``target`` — from the cache, or translated and kept."""
|
||||
version = SANITIZER_VERSION * 1000 + translation.PIPELINE_VERSION
|
||||
cached = await db.get(MessageTranslation, (message.id, target, allow_remote))
|
||||
if cached is not None and cached.sanitizer_v == version:
|
||||
return cached.html, cached.source
|
||||
try:
|
||||
out = await translation.translate_html(
|
||||
fragment, language or "auto", target, names=await _names(db, user, message)
|
||||
)
|
||||
except translation.TranslationError as exc:
|
||||
raise_api_error(503, "translation_failed", str(exc))
|
||||
# The translation comes back as HTML: cleaned like any HTML we show.
|
||||
clean = sanitize_html(
|
||||
out, message.id, allow_remote_images=allow_remote, block_trackers=block
|
||||
).html
|
||||
source = language or "auto"
|
||||
stmt = insert(MessageTranslation).values(
|
||||
message_id=message.id,
|
||||
target=target,
|
||||
images=allow_remote,
|
||||
source=source,
|
||||
html=clean,
|
||||
sanitizer_v=version,
|
||||
)
|
||||
await db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["message_id", "target", "images"],
|
||||
set_={"html": clean, "source": source, "sanitizer_v": version},
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
return clean, source
|
||||
|
||||
|
||||
@router.post("/{message_id}/allow-remote-images", dependencies=[Depends(require_csrf)])
|
||||
|
||||
Reference in New Issue
Block a user