From 8a1b9979e9d989db3889d322639c92c0b600e363 Mon Sep 17 00:00:00 2001 From: Lukas Remis Date: Sun, 18 Oct 2026 20:00:31 +0200 Subject: [PATCH] Productivity features, UI translations, local email translation - Snooze (with a Snoozed folder), keyboard shortcuts with a Settings section, mail templates (Compose > More > Templates), a Subscriptions folder with bulk unsubscribe/delete, and tracker blocking with a Privacy toggle (on by default) - Own dialogs instead of browser popups; emoji picker in compose; compose windows survive a visit to Settings - Sentence completion learned from the user's sent mail (no AI) - Interface in English and Polish (one file per language, English as the fallback), picked from the browser on first run, then kept - Email translation on the server via LibreTranslate: Translate button and "Translate to...", messages in several languages translated part by part, names/addresses/emoji/file names/phone numbers kept, an animated line while translating - Antispam "Mark as spam" loads with the message, not when the menu opens Co-Authored-By: Claude Opus 5.5 --- Makefile | 2 +- backend/app/api/__init__.py | 10 + backend/app/api/completion.py | 44 + backend/app/api/mailboxes.py | 18 +- backend/app/api/messages.py | 212 ++++- backend/app/api/schemas.py | 10 + backend/app/api/settings.py | 32 + backend/app/api/snooze.py | 76 ++ backend/app/api/subscriptions.py | 372 ++++++++ backend/app/api/templates.py | 129 +++ backend/app/api/threads.py | 80 +- backend/app/api/translation.py | 30 + backend/app/completion.py | 391 ++++++++ backend/app/config.py | 3 + backend/app/indexer/indexer.py | 6 + backend/app/indexer/sanitize.py | 105 ++- backend/app/models.py | 91 ++ backend/app/queue.py | 9 +- backend/app/snooze.py | 92 ++ backend/app/subscriptions.py | 151 +++ backend/app/translation.py | 303 ++++++ backend/app/translation_prep.py | 332 +++++++ .../0028_snooze_templates_subscriptions.py | 108 +++ .../migrations/versions/0029_completion.py | 49 + .../migrations/versions/0030_translation.py | 45 + backend/tests/api/conftest.py | 149 +++ backend/tests/api/test_api.py | 2 +- backend/tests/api/test_completion_api.py | 41 + backend/tests/api/test_snooze.py | 80 ++ backend/tests/api/test_subscriptions.py | 239 +++++ backend/tests/api/test_templates.py | 54 ++ backend/tests/api/test_trackers.py | 45 + backend/tests/api/test_translation.py | 86 ++ backend/tests/indexer/test_sanitize.py | 71 ++ backend/tests/test_completion_phrases.py | 89 ++ backend/tests/test_translation_prep.py | 128 +++ backend/tests/test_translation_text.py | 123 +++ backend/workers/run.py | 3 + backend/workers/worker.py | 9 + docker-compose.yml | 31 +- docker/docker-compose.yml | 19 + docs/feature-ideas.md | 298 ++++++ frontend/package-lock.json | 14 + frontend/package.json | 2 + frontend/src/App.tsx | 36 +- frontend/src/admin/AdminApp.tsx | 12 +- frontend/src/components/AppDialog.tsx | 218 +++++ frontend/src/components/ShortcutList.tsx | 61 ++ .../src/components/compose/ComposeDock.tsx | 3 +- frontend/src/components/compose/Composer.tsx | 187 +++- .../src/components/compose/EmojiButton.tsx | 154 +++ .../src/components/compose/RichTextEditor.tsx | 113 ++- .../components/compose/SaveTemplateDialog.tsx | 69 ++ frontend/src/components/compose/completion.ts | 110 +++ frontend/src/components/compose/restore.ts | 77 ++ frontend/src/components/compose/spellcheck.ts | 2 +- frontend/src/components/list/MessageList.tsx | 16 +- .../src/components/list/SubscriptionsList.tsx | 527 +++++++++++ frontend/src/components/mail/LabelChips.tsx | 4 +- frontend/src/components/mail/RightPanel.tsx | 3 +- frontend/src/components/mail/SnoozeMenu.tsx | 143 +++ .../src/components/mail/SubscriptionPane.tsx | 151 +++ .../src/components/thread/MessageBody.tsx | 98 +- frontend/src/components/thread/ThreadView.tsx | 331 +++++-- frontend/src/i18n/en.ts | 172 +++- frontend/src/i18n/index.tsx | 150 ++- frontend/src/i18n/pl.ts | 890 ++++++++++++++++++ frontend/src/index.css | 47 + frontend/src/lib/api.ts | 55 ++ frontend/src/lib/print.ts | 16 +- frontend/src/lib/shortcuts.ts | 48 + frontend/src/lib/timeFormat.ts | 12 +- frontend/src/lib/types.ts | 57 ++ frontend/src/lib/useKeyboardShortcuts.ts | 92 +- frontend/src/routes/MailLayout.tsx | 355 ++++++- frontend/src/routes/settings/Settings.tsx | 191 +++- frontend/src/test/antispamMenu.test.ts | 24 + frontend/src/test/completionGhost.test.ts | 64 ++ frontend/src/test/composeRestore.test.ts | 43 + frontend/src/test/i18nCatalogs.test.tsx | 87 ++ frontend/src/test/snoozePresets.test.ts | 36 + frontend/src/test/subscriptionBulk.test.tsx | 115 +++ .../webmail_plugin_antispam/ui/menu.js | 18 +- 83 files changed, 8677 insertions(+), 293 deletions(-) create mode 100644 backend/app/api/completion.py create mode 100644 backend/app/api/snooze.py create mode 100644 backend/app/api/subscriptions.py create mode 100644 backend/app/api/templates.py create mode 100644 backend/app/api/translation.py create mode 100644 backend/app/completion.py create mode 100644 backend/app/snooze.py create mode 100644 backend/app/subscriptions.py create mode 100644 backend/app/translation.py create mode 100644 backend/app/translation_prep.py create mode 100644 backend/migrations/versions/0028_snooze_templates_subscriptions.py create mode 100644 backend/migrations/versions/0029_completion.py create mode 100644 backend/migrations/versions/0030_translation.py create mode 100644 backend/tests/api/conftest.py create mode 100644 backend/tests/api/test_completion_api.py create mode 100644 backend/tests/api/test_snooze.py create mode 100644 backend/tests/api/test_subscriptions.py create mode 100644 backend/tests/api/test_templates.py create mode 100644 backend/tests/api/test_trackers.py create mode 100644 backend/tests/api/test_translation.py create mode 100644 backend/tests/test_completion_phrases.py create mode 100644 backend/tests/test_translation_prep.py create mode 100644 backend/tests/test_translation_text.py create mode 100644 docs/feature-ideas.md create mode 100644 frontend/src/components/AppDialog.tsx create mode 100644 frontend/src/components/ShortcutList.tsx create mode 100644 frontend/src/components/compose/EmojiButton.tsx create mode 100644 frontend/src/components/compose/SaveTemplateDialog.tsx create mode 100644 frontend/src/components/compose/completion.ts create mode 100644 frontend/src/components/compose/restore.ts create mode 100644 frontend/src/components/list/SubscriptionsList.tsx create mode 100644 frontend/src/components/mail/SnoozeMenu.tsx create mode 100644 frontend/src/components/mail/SubscriptionPane.tsx create mode 100644 frontend/src/i18n/pl.ts create mode 100644 frontend/src/lib/shortcuts.ts create mode 100644 frontend/src/test/completionGhost.test.ts create mode 100644 frontend/src/test/composeRestore.test.ts create mode 100644 frontend/src/test/i18nCatalogs.test.tsx create mode 100644 frontend/src/test/snoozePresets.test.ts create mode 100644 frontend/src/test/subscriptionBulk.test.tsx diff --git a/Makefile b/Makefile index c76b76c..cdca34c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ up: ## start services - docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool + docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool libretranslate down: ## stop services (data kept) docker compose -f docker/docker-compose.yml down reset-db: ## DESTRUCTIVE: wipe db + search + blobs, re-migrate diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 51bc7b5..718454d 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -15,6 +15,7 @@ def register_routers(app: FastAPI) -> None: from app.api.actions import router as actions_router from app.api.admin import router as admin_router from app.api.avatar import router as avatar_router + from app.api.completion import router as completion_router from app.api.compose import router as compose_router from app.api.invitations import router as invitations_router from app.api.mailboxes import router as mailboxes_router @@ -25,8 +26,12 @@ def register_routers(app: FastAPI) -> None: from app.api.saved_searches import router as saved_searches_router from app.api.search import router as search_router from app.api.settings import router as settings_router + from app.api.snooze import router as snooze_router from app.api.spellcheck import router as spellcheck_router + from app.api.subscriptions import router as subscriptions_router + from app.api.templates import router as templates_router from app.api.threads import router as threads_router + from app.api.translation import router as translation_router from app.identity.routes import router as auth_router from app.ws import router as ws_router @@ -48,6 +53,11 @@ def register_routers(app: FastAPI) -> None: spellcheck_router, push_router, invitations_router, + templates_router, + snooze_router, + subscriptions_router, + completion_router, + translation_router, ): app.include_router(router, prefix="/api") app.include_router(ws_router, prefix="/api") diff --git a/backend/app/api/completion.py b/backend/app/api/completion.py new file mode 100644 index 0000000..d4f1a04 --- /dev/null +++ b/backend/app/api/completion.py @@ -0,0 +1,44 @@ +"""Sentence completion for the compose window (app/completion.py).""" + +from __future__ import annotations + +import datetime as dt +from typing import Annotated + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app import completion +from app.db import get_session +from app.identity.deps import CurrentUser, require_csrf +from app.models import CompletionIndexState +from app.queue import enqueue_job + +router = APIRouter(prefix="/completion", tags=["compose"], dependencies=[Depends(require_csrf)]) + +DbSession = Annotated[AsyncSession, Depends(get_session)] + + +class CompletionIn(BaseModel): + # The text before the cursor (its last line is all that matters). A POST, + # not a query string: what the user is typing stays out of access logs. + before: str = Field(max_length=2000) + + +class CompletionOut(BaseModel): + suggestion: str | None + + +@router.post("", response_model=CompletionOut) +async def suggest(body: CompletionIn, user: CurrentUser, db: DbSession) -> CompletionOut: + before = body.before[-300:] + built = await db.scalar( + select(CompletionIndexState.built_at).where(CompletionIndexState.user_id == user.id) + ) + if built is None or built < dt.datetime.now(dt.UTC) - completion.REBUILD_AFTER: + # In the background; until it is built, the starter phrases still answer. + await enqueue_job("build_index_job", str(user.id), _job_id=f"complete:{user.id}") + personal = await completion.personal_phrases(db, user.id, completion.keys_for(before)) + return CompletionOut(suggestion=completion.complete(personal, before)) diff --git a/backend/app/api/mailboxes.py b/backend/app/api/mailboxes.py index 0a4aa81..f215958 100644 --- a/backend/app/api/mailboxes.py +++ b/backend/app/api/mailboxes.py @@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import get_session from app.identity.deps import CurrentUser -from app.models import Draft, Mailbox, Message +from app.models import Draft, Mailbox, Message, Thread from app.outbox.service import not_being_sent router = APIRouter(prefix="/mailboxes", tags=["mail"]) @@ -30,6 +30,7 @@ class MailboxOut(BaseModel): class FolderCounts(BaseModel): inbox_unread: int # unread messages sitting in Inbox mailboxes drafts: int # drafts shown in the Drafts folder (local + un-adopted remote) + snoozed: int = 0 # conversations snoozed out of the Inbox @router.get("", response_model=list[MailboxOut]) @@ -54,8 +55,17 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts: Message.user_id == user.id, Mailbox.special_use == "inbox", ~Message.flags.contains(["\\Seen"]), + # A snoozed conversation is out of the Inbox until it wakes. + ~Message.thread_id.in_( + select(Thread.id).where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None)) + ), ) ) + snoozed = await db.scalar( + select(func.count()) + .select_from(Thread) + .where(Thread.user_id == user.id, Thread.snoozed_until.isnot(None)) + ) # Drafts folder = local drafts + IMAP-Drafts messages we haven't adopted (mirror # UIDs excluded), matching what list_drafts + list_remote_drafts return. @@ -77,4 +87,8 @@ async def folder_counts(user: CurrentUser, db: DbSession) -> FolderCounts: ) remote = sum(1 for r in remote_rows if (r.account_id, r.uid) not in owned) - return FolderCounts(inbox_unread=int(inbox_unread or 0), drafts=int(local or 0) + remote) + return FolderCounts( + inbox_unread=int(inbox_unread or 0), + drafts=int(local or 0) + remote, + snoozed=int(snoozed or 0), + ) diff --git a/backend/app/api/messages.py b/backend/app/api/messages.py index fd469f3..42a9a3f 100644 --- a/backend/app/api/messages.py +++ b/backend/app/api/messages.py @@ -6,6 +6,7 @@ import asyncio import contextlib import html as html_lib import io +import logging import uuid import zipfile from typing import Annotated @@ -14,11 +15,18 @@ from urllib.parse import quote from fastapi import APIRouter, Depends, Query from fastapi.responses import FileResponse, HTMLResponse, Response from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession +from app import translation from app.api.errors import raise_api_error from app.api.schemas import AttachmentOut -from app.api.settings import add_allowed_image_sender, allowed_image_senders +from app.api.settings import ( + add_allowed_image_sender, + allowed_image_senders, + blocks_trackers, +) +from app.completion import html_to_text from app.db import get_session from app.identity.deps import CurrentUser, require_csrf from app.indexer import attachments as att_store @@ -26,7 +34,18 @@ from app.indexer import rawstore, thumbs from app.indexer.mime import parse_message from app.indexer.sanitize import SANITIZER_VERSION, sanitize_html from app.indexer.thumbs import thumb_type -from app.models import Attachment, BodyPart, ComposeAttachment, Message +from app.models import ( + Account, + Attachment, + BodyPart, + ComposeAttachment, + Message, + MessageTranslation, + User, +) +from app.translation_prep import find_names_in_signature + +logger = logging.getLogger(__name__) router = APIRouter(prefix="/messages", tags=["mail"]) @@ -101,8 +120,16 @@ _MESSAGE_CSP = ( ) -def _html(document: str) -> HTMLResponse: - return HTMLResponse(content=document, headers={"Content-Security-Policy": _MESSAGE_CSP}) +def _html(document: str, trackers: int = 0, extra: dict[str, str] | None = None) -> HTMLResponse: + # The page reads what it needs from headers: the body is the message. + return HTMLResponse( + content=document, + headers={ + "Content-Security-Policy": _MESSAGE_CSP, + "X-Trackers-Blocked": str(trackers), + **(extra or {}), + }, + ) async def _owned_message(db: AsyncSession, user_id: uuid.UUID, message_id: uuid.UUID) -> Message: @@ -118,54 +145,167 @@ def _sender_email(message: Message) -> str: return message.from_addr[0].get("email", "").lower() if message.from_addr else "" +async def _render_fragment( + db: AsyncSession, message: Message, allow_remote: bool, block: bool +) -> tuple[str, int]: + """The sanitized body the user sees, and how many trackers came out of it. + + Re-sanitizes from the raw blob when remote images are allowed, the stored + copy predates the current sanitizer (SPEC §10.4), or trackers are to stay + in; otherwise the stored copy is served as is. + """ + parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message.id))) + html_part = next((p for p in parts if p.kind == "html_sanitized"), None) + + stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION + # The stored copy has its trackers taken out; with blocking switched off, a + # message that had some is rendered afresh with them left in. + trackers_kept = not block and html_part is not None and html_part.trackers_blocked > 0 + if html_part is not None and not allow_remote and not stale and not trackers_kept: + return html_part.content, html_part.trackers_blocked if block else 0 + + raw = None + if html_part is not None or allow_remote: # a plain-text message has nothing to redo + try: + raw = await rawstore.ensure_bytes(message) + except FileNotFoundError: + raw = None + parsed = parse_message(raw) if raw is not None else None + if parsed is not None and parsed.html_body is not None: + if stale and html_part is not None: + # The stored copy is always the plain one: no remote images, trackers out. + upgraded = sanitize_html(parsed.html_body, message.id) + html_part.content = upgraded.html + html_part.sanitizer_v = upgraded.sanitizer_v + html_part.trackers_blocked = upgraded.trackers_blocked + await db.commit() + if block and not allow_remote: + return upgraded.html, upgraded.trackers_blocked + result = sanitize_html( + parsed.html_body, message.id, allow_remote_images=allow_remote, block_trackers=block + ) + return result.html, result.trackers_blocked + + if html_part is not None: + return html_part.content, html_part.trackers_blocked + text_part = next((p for p in parts if p.kind == "text"), None) + body = html_lib.escape(text_part.content) if text_part else "" + return f'
{body}
', 0 + + +async def _content_language(db: AsyncSession, message: Message) -> str: + """The message's language, found (and kept) the first time it is shown.""" + if message.content_language is not None or not translation.enabled(): + return message.content_language or "" + text = await db.scalar( + select(BodyPart.content).where(BodyPart.message_id == message.id, BodyPart.kind == "text") + ) + try: + found = await translation.detect(text or message.snippet or "") + except translation.TranslationError as exc: + logger.info("language detection unavailable: %s", exc) + return "" # not kept: tried again once the service is back + message.content_language = found + await db.commit() + return found + + @router.get("/{message_id}/html", response_class=HTMLResponse) async def message_html( message_id: uuid.UUID, user: CurrentUser, db: DbSession, images: bool = Query(False), + translate: str | None = Query(None, pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$"), ) -> HTMLResponse: """Return sanitized HTML wrapped in a document for the sandboxed iframe (SPEC §9.4, I-8). - Re-sanitizes from the raw blob when the sender is image-approved, or when the - stored copy predates the current sanitizer version (SPEC §10.4). ``images=1`` - is a one-time "Load images" — render remote images for this request only, - without persisting the sender as trusted. + ``images=1`` is a one-time "Load images" — render remote images for this + request only, without persisting the sender as trusted. ``translate=pl`` + returns it translated (app/translation.py). Headers tell the page what it + got: X-Trackers-Blocked, X-Content-Language (the message's own language, + when known) and, translated, X-Translated-From / X-Translated-To. """ message = await _owned_message(db, user.id, message_id) allow_remote = images or _sender_email(message) in allowed_image_senders(user) + block = blocks_trackers(user) + fragment, trackers = await _render_fragment(db, message, allow_remote, block) - parts = list(await db.scalars(select(BodyPart).where(BodyPart.message_id == message_id))) - html_part = next((p for p in parts if p.kind == "html_sanitized"), None) + headers: dict[str, str] = {} + language = await _content_language(db, message) + if language: + headers["X-Content-Language"] = language + if translate: + fragment, source = await _translated( + db, user, message, fragment, language, translate, allow_remote, block + ) + headers["X-Translated-From"] = source + headers["X-Translated-To"] = translate + return _html(_wrap(fragment), trackers, headers) - stale = html_part is not None and html_part.sanitizer_v < SANITIZER_VERSION - if html_part is not None and not allow_remote and not stale: - return _html(_wrap(html_part.content)) - # Re-sanitize the original HTML (image-approved sender, or a stale stored copy). - # If the raw blob is unavailable, fall back to whatever is already stored. - if allow_remote or stale: - try: - raw = await rawstore.ensure_bytes(message) - except FileNotFoundError: - raw = None - if raw is not None: - parsed = parse_message(raw) - if parsed.html_body is not None: - result = sanitize_html( - parsed.html_body, message.id, allow_remote_images=allow_remote - ) - if html_part is not None and not allow_remote: - html_part.content = result.html # persist the upgraded sanitization - html_part.sanitizer_v = result.sanitizer_v - await db.commit() - return _html(_wrap(result.html)) +async def _names(db: AsyncSession, user: User, message: Message) -> list[str]: + """Names a translation leaves as they are if the model would change them: + the people on the message (and the sender's single names, as signed: + "Milan"), the user's own names (accounts, signatures).""" + names: list[str] = [] + for people in (message.from_addr, message.to_addr, message.cc_addr): + names += [p.get("name", "") for p in people if p.get("name")] + for p in message.from_addr: + names += [w for w in str(p.get("name", "")).split() if len(w) >= 3 and w[0].isupper()] + rows = await db.execute( + select(Account.display_name, Account.signature).where(Account.user_id == user.id) + ) + for display_name, signature in rows: + if display_name: + names.append(display_name) + if signature: + names += find_names_in_signature(html_to_text(signature)) + return names - if html_part is not None: - return _html(_wrap(html_part.content)) - text_part = next((p for p in parts if p.kind == "text"), None) - body = html_lib.escape(text_part.content) if text_part else "" - return _html(_wrap(f'
{body}
')) + +async def _translated( + db: AsyncSession, + user: User, + message: Message, + fragment: str, + language: str, + target: str, + allow_remote: bool, + block: bool, +) -> tuple[str, str]: + """The fragment in ``target`` — from the cache, or translated and kept.""" + version = SANITIZER_VERSION * 1000 + translation.PIPELINE_VERSION + cached = await db.get(MessageTranslation, (message.id, target, allow_remote)) + if cached is not None and cached.sanitizer_v == version: + return cached.html, cached.source + try: + out = await translation.translate_html( + fragment, language or "auto", target, names=await _names(db, user, message) + ) + except translation.TranslationError as exc: + raise_api_error(503, "translation_failed", str(exc)) + # The translation comes back as HTML: cleaned like any HTML we show. + clean = sanitize_html( + out, message.id, allow_remote_images=allow_remote, block_trackers=block + ).html + source = language or "auto" + stmt = insert(MessageTranslation).values( + message_id=message.id, + target=target, + images=allow_remote, + source=source, + html=clean, + sanitizer_v=version, + ) + await db.execute( + stmt.on_conflict_do_update( + index_elements=["message_id", "target", "images"], + set_={"html": clean, "source": source, "sanitizer_v": version}, + ) + ) + await db.commit() + return clean, source @router.post("/{message_id}/allow-remote-images", dependencies=[Depends(require_csrf)]) diff --git a/backend/app/api/schemas.py b/backend/app/api/schemas.py index 68a0f38..f90d6d0 100644 --- a/backend/app/api/schemas.py +++ b/backend/app/api/schemas.py @@ -27,6 +27,8 @@ class ThreadSummary(BaseModel): has_attachments: bool pinned: bool last_message_at: dt.datetime + # Set while the thread is snoozed: when it comes back to the Inbox. + snoozed_until: dt.datetime | None = None class ThreadListOut(BaseModel): @@ -64,6 +66,13 @@ class MessageEnvelope(BaseModel): snippet: str # Other accounts this same delivery also landed in (collapsed duplicates). also_in_account_ids: list[uuid.UUID] = [] + # Bulk mail: the Subscriptions group it belongs to, how it can be left + # (one_click | mailto | web), and whether the user already did. + subscription_key: str | None = None + unsubscribe_via: str | None = None + unsubscribed: bool = False + # Its language, once found (the first time it is shown): "" unknown. + content_language: str = "" class ThreadDetail(BaseModel): @@ -72,6 +81,7 @@ class ThreadDetail(BaseModel): order: str # "asc" | "desc" pinned: bool messages: list[MessageEnvelope] # always ASC (SPEC §9.4) + snoozed_until: dt.datetime | None = None class FlagChangeRequest(BaseModel): diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 3af7fc8..4ac365b 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -27,6 +27,7 @@ _ALLOWED_INVITE_FROM = {"contacts", "anyone"} # "auto", or a LanguageTool language code such as "pl-PL", "en-GB", "de". _LANG_CODE = re.compile(r"auto|[a-z]{2,3}(-[A-Za-z0-9]{2,8})*") +_UI_LANGUAGE = re.compile(r"[a-z]{2,3}(-[A-Za-z]{2,4})?") class ReminderPref(BaseModel): @@ -52,6 +53,16 @@ class SettingsOut(BaseModel): # ("auto" detects it per message). spellcheck_enabled: bool = True spellcheck_language: str = "auto" + # Gmail-style keyboard shortcuts in the mail view (Settings → Keyboard shortcuts). + shortcuts_enabled: bool = True + # Remove tracking pixels from mail (Settings → Privacy). + block_trackers: bool = True + # Grey sentence endings in the compose window, taken with Tab. + completion_enabled: bool = True + # Interface language ("en", "pl", …); "" until chosen — the web app then + # picks one from the browser on first sign-in and saves it here. + ui_language: str = "" + # Names and words email translation leaves as they are. class SettingsPatch(BaseModel): @@ -68,6 +79,10 @@ class SettingsPatch(BaseModel): invitation_file: bool | None = None spellcheck_enabled: bool | None = None spellcheck_language: str | None = None + shortcuts_enabled: bool | None = None + block_trackers: bool | None = None + completion_enabled: bool | None = None + ui_language: str | None = None class QuotaOut(BaseModel): @@ -82,6 +97,11 @@ def allowed_image_senders(user: User) -> set[str]: return set() +def blocks_trackers(user: User) -> bool: + """Whether tracking pixels are taken out of this user's mail (on unless switched off).""" + return bool(user.settings.get("block_trackers", True)) + + def add_allowed_image_sender(user: User, email: str) -> None: current = sorted(allowed_image_senders(user)) if email.lower() not in current: @@ -127,6 +147,10 @@ def _settings_out(user: User) -> SettingsOut: allowed_image_senders=sorted(allowed_image_senders(user)), spellcheck_enabled=bool(s.get("spellcheck_enabled", True)), spellcheck_language=str(s.get("spellcheck_language", "auto")), + shortcuts_enabled=bool(s.get("shortcuts_enabled", True)), + block_trackers=blocks_trackers(user), + completion_enabled=bool(s.get("completion_enabled", True)), + ui_language=str(s.get("ui_language", "")), ) @@ -175,6 +199,14 @@ async def patch_settings(body: SettingsPatch, user: CurrentUser, db: DbSession) updates["spellcheck_enabled"] = body.spellcheck_enabled if body.spellcheck_language is not None and _LANG_CODE.fullmatch(body.spellcheck_language): updates["spellcheck_language"] = body.spellcheck_language + if body.shortcuts_enabled is not None: + updates["shortcuts_enabled"] = body.shortcuts_enabled + if body.block_trackers is not None: + updates["block_trackers"] = body.block_trackers + if body.completion_enabled is not None: + updates["completion_enabled"] = body.completion_enabled + if body.ui_language is not None and _UI_LANGUAGE.fullmatch(body.ui_language): + updates["ui_language"] = body.ui_language user.settings = updates await db.commit() return _settings_out(user) diff --git a/backend/app/api/snooze.py b/backend/app/api/snooze.py new file mode 100644 index 0000000..c295d4c --- /dev/null +++ b/backend/app/api/snooze.py @@ -0,0 +1,76 @@ +"""Snooze endpoints: hide a conversation from the Inbox until a set time.""" + +from __future__ import annotations + +import datetime as dt +import uuid +from typing import Annotated + +from fastapi import APIRouter, Depends +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app import snooze +from app.api.errors import raise_api_error +from app.db import get_session +from app.events import get_event_bus +from app.identity.deps import CurrentUser, require_csrf +from app.models import Thread + +router = APIRouter(prefix="/threads", tags=["mail"], dependencies=[Depends(require_csrf)]) + +DbSession = Annotated[AsyncSession, Depends(get_session)] + + +class SnoozeIn(BaseModel): + until: dt.datetime + + +class SnoozeOut(BaseModel): + thread_id: uuid.UUID + snoozed_until: dt.datetime | None + + +async def _thread(db: AsyncSession, user_id: uuid.UUID, thread_id: uuid.UUID) -> Thread: + thread = await db.scalar( + select(Thread).where(Thread.id == thread_id, Thread.user_id == user_id) + ) + if thread is None: + raise_api_error(404, "not_found", "thread not found") + return thread + + +async def _announce(user_id: uuid.UUID, thread_id: uuid.UUID) -> None: + await get_event_bus().publish( + "thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)} + ) + + +@router.post("/{thread_id}/snooze", response_model=SnoozeOut) +async def snooze_thread( + thread_id: uuid.UUID, body: SnoozeIn, user: CurrentUser, db: DbSession +) -> SnoozeOut: + until = body.until + if until.tzinfo is None: + raise_api_error(400, "invalid_time", "the time needs a time zone") + now = dt.datetime.now(dt.UTC) + if until <= now: + raise_api_error(400, "invalid_time", "pick a time in the future") + if until > now + snooze.MAX_SNOOZE: + raise_api_error(400, "invalid_time", "snooze for at most a year") + thread = await _thread(db, user.id, thread_id) + snooze.snooze(thread, until, now) + await db.commit() + await _announce(user.id, thread_id) + return SnoozeOut(thread_id=thread_id, snoozed_until=thread.snoozed_until) + + +@router.post("/{thread_id}/unsnooze", response_model=SnoozeOut) +async def unsnooze_thread(thread_id: uuid.UUID, user: CurrentUser, db: DbSession) -> SnoozeOut: + thread = await _thread(db, user.id, thread_id) + if thread.snoozed_until is not None: + snooze.wake(thread, dt.datetime.now(dt.UTC)) + await db.commit() + await _announce(user.id, thread_id) + return SnoozeOut(thread_id=thread_id, snoozed_until=None) diff --git a/backend/app/api/subscriptions.py b/backend/app/api/subscriptions.py new file mode 100644 index 0000000..eaa0280 --- /dev/null +++ b/backend/app/api/subscriptions.py @@ -0,0 +1,372 @@ +"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py).""" + +from __future__ import annotations + +import datetime as dt +import logging +import uuid +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy import case, func, literal, or_, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql.elements import ColumnElement + +from app import subscriptions as subs +from app.accounts.ssrf import guarded_request +from app.api.errors import raise_api_error +from app.config import get_settings +from app.db import get_session +from app.identity.deps import CurrentUser, require_csrf +from app.models import Mailbox, Message, SubscriptionState +from app.outbox.schemas import DraftIn, DraftPayload +from app.outbox.service import DraftError, create_outbox, upsert_draft +from app.queue import enqueue_job + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)]) + +DbSession = Annotated[AsyncSession, Depends(get_session)] + +# RFC 8058 senders should stop within two days of a one-click request. +GRACE = dt.timedelta(days=2) +# Not counted: mail the user sent, and mail already thrown away (Trash, Spam) — +# a sender whose mail was all deleted drops off the list. +_LEFT_OUT = ("sent", "drafts", "trash", "junk") + + +class SubscriptionOut(BaseModel): + key: str + name: str + email: str + list_id: str | None + total: int + unread: int + last_30_days: int + per_month: float + first_at: dt.datetime + last_at: dt.datetime + last_subject: str + one_click: bool + has_mailto: bool + has_web: bool + unsubscribed_at: dt.datetime | None + method: str | None + still_sending: bool + + +class SubscriptionListOut(BaseModel): + subscriptions: list[SubscriptionOut] # one page + total: int # senders in all + pending: int # messages whose headers are still being read + + +class KeyIn(BaseModel): + key: str + + +class UnsubscribeIn(KeyIn): + # From the reading pane: this message's own header, wherever it is filed + # (Spam included, which the page's groups leave out). + message_id: uuid.UUID | None = None + + +class UnsubscribeOut(BaseModel): + status: str # done | open + method: str # one_click | mailto | web + url: str | None = None + + +def _sender() -> ColumnElement[Any]: + return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, "")) + + +def _key() -> ColumnElement[Any]: + return case( + (Message.list_id.isnot(None), literal("list:") + Message.list_id), + else_=literal("from:") + _sender(), + ) + + +def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]: + return [ + Message.user_id == user_id, + Message.list_unsubscribe.isnot(None), + Message.list_unsubscribe != "", + or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)), + ] + + +async def _start_backfill(user_id: uuid.UUID) -> None: + # One job per user at a time: arq refuses a second while the first exists. + await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}") + + +@router.get("", response_model=SubscriptionListOut) +async def list_subscriptions( + user: CurrentUser, + db: DbSession, + offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=200), + order: str = Query("desc"), # desc: heard from most recently first | asc: longest ago +) -> SubscriptionListOut: + """One page of senders, by when their latest mail came.""" + pending = await subs.pending_count(db, user.id) + if pending: + await _start_backfill(user.id) + + now = dt.datetime.now(dt.UTC) + key = _key().label("key") + seen = Message.flags.contains(["\\Seen"]) + groups = ( + select( + key, + func.count().label("total"), + func.count().filter(~seen).label("unread"), + func.count() + .filter(Message.internal_date > now - dt.timedelta(days=30)) + .label("recent"), + func.min(Message.internal_date).label("first_at"), + func.max(Message.internal_date).label("last_at"), + ) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where(*_bulk(user.id)) + .group_by(key) + ) + total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0 + agg_rows = ( + await db.execute( + groups.order_by( + func.max(Message.internal_date).asc() + if order == "asc" + else func.max(Message.internal_date).desc(), + key, + ) + .offset(offset) + .limit(limit) + ) + ).all() + page_keys = [r.key for r in agg_rows] + if not page_keys: + return SubscriptionListOut(subscriptions=[], total=total, pending=pending) + + in_page = _key().in_(page_keys) + latest_rows = ( + await db.execute( + select( + key, + Message.from_addr, + Message.subject, + Message.list_id, + Message.list_unsubscribe, + Message.list_unsubscribe_post, + ) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where(*_bulk(user.id), in_page) + .distinct(key) + .order_by(key, Message.internal_date.desc()) + ) + ).all() + latest = {r.key: r for r in latest_rows} + states = { + s.key: s + for s in await db.scalars( + select(SubscriptionState).where( + SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys) + ) + ) + } + # Senders still mailing after the grace period, for the whole page at once. + still_sending = set( + await db.scalars( + select(key) + .select_from(Message) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .join( + SubscriptionState, + (SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()), + ) + .where( + *_bulk(user.id), + in_page, + Message.internal_date > SubscriptionState.unsubscribed_at + GRACE, + ) + .distinct() + ) + ) + + out: list[SubscriptionOut] = [] + for row in agg_rows: + last = latest.get(row.key) + if last is None: + continue + sender = last.from_addr[0] if last.from_addr else {} + targets = subs.unsubscribe_targets(last.list_unsubscribe or "") + state = states.get(row.key) + months = max(1.0, (row.last_at - row.first_at).days / 30.44) + out.append( + SubscriptionOut( + key=row.key, + name=sender.get("name") or sender.get("email", ""), + email=sender.get("email", ""), + list_id=last.list_id, + total=row.total, + unread=row.unread, + last_30_days=row.recent, + per_month=round(row.total / months, 1), + first_at=row.first_at, + last_at=row.last_at, + last_subject=last.subject, + one_click=bool(last.list_unsubscribe_post and targets.https), + has_mailto=targets.mailto is not None, + has_web=targets.https is not None, + unsubscribed_at=state.unsubscribed_at if state else None, + method=state.method if state else None, + still_sending=row.key in still_sending, + ) + ) + return SubscriptionListOut(subscriptions=out, total=total, pending=pending) + + +@router.get("/threads", response_model=list[uuid.UUID]) +async def subscription_threads( + user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200) +) -> list[uuid.UUID]: + """The group's conversations, newest first (for the reading pane).""" + rows = await db.execute( + select(Message.thread_id, func.max(Message.internal_date).label("at")) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None)) + .group_by(Message.thread_id) + .order_by(func.max(Message.internal_date).desc()) + .limit(limit) + ) + return [r.thread_id for r in rows if r.thread_id is not None] + + +async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message: + message = await db.scalar( + select(Message) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where(*_bulk(user_id), _key() == key) + .order_by(Message.internal_date.desc()) + .limit(1) + ) + if message is None: + raise_api_error(404, "not_found", "no mail from this sender") + return message + + +async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None: + stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method) + await db.execute( + stmt.on_conflict_do_update( + index_elements=["user_id", "key"], + set_={"method": method, "unsubscribed_at": func.now()}, + ) + ) + await db.commit() + + +async def _one_click(url: str) -> bool: + try: + resp = await guarded_request( + "POST", + url, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + content="List-Unsubscribe=One-Click", + max_bytes=256 * 1024, + ) + except Exception as exc: # SSRF refusal, network, TLS: any failure tries the next way + logger.info("one-click unsubscribe failed: %s", exc) + return False + return 200 <= resp.status < 400 + + +async def _send_mailto( + db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets +) -> bool: + assert targets.mailto is not None + body = DraftIn( + account_id=message.account_id, + kind="new", + payload=DraftPayload( + to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body + ), + ) + try: + draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None) + except DraftError: + return False + outbox = await create_outbox(db, user_id=user_id, draft=draft) + await db.commit() + await enqueue_job( + "send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds) + ) + return True + + +@router.post("/unsubscribe", response_model=UnsubscribeOut) +async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut: + """Unsubscribe from a sender: one-click if offered, else by mail, else the web page.""" + if body.message_id is not None: + found = await db.scalar( + select(Message).where( + Message.id == body.message_id, + Message.user_id == user.id, + Message.list_unsubscribe.isnot(None), + Message.list_unsubscribe != "", + ) + ) + if found is None: + raise_api_error(404, "not_found", "this message offers no way to unsubscribe") + message = found + else: + message = await _latest_with_header(db, user.id, body.key) + targets = subs.unsubscribe_targets(message.list_unsubscribe or "") + if message.list_unsubscribe_post and targets.https and await _one_click(targets.https): + await _record(db, user.id, body.key, "one_click") + return UnsubscribeOut(status="done", method="one_click") + if targets.mailto and await _send_mailto(db, user.id, message, targets): + await _record(db, user.id, body.key, "mailto") + return UnsubscribeOut(status="done", method="mailto") + if targets.https: + # The sender wants it done on their page: the user opens it. + await _record(db, user.id, body.key, "web") + return UnsubscribeOut(status="open", method="web", url=targets.https) + raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe") + + +@router.post("/trash") +async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]: + """Move every conversation from this sender to Trash. + + Only conversations made up entirely of this sender's mail: one where the + user replied, or someone else wrote, stays where it is. + """ + rows = ( + await db.execute( + select(Message.thread_id, _key().label("key"), Mailbox.special_use) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where( + Message.user_id == user.id, + Message.thread_id.in_( + select(Message.thread_id) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where(*_bulk(user.id), _key() == body.key) + ), + ) + ) + ).all() + own: dict[uuid.UUID, bool] = {} + for thread_id, key, special in rows: + if thread_id is None or special == "trash": + continue + own[thread_id] = own.get(thread_id, True) and key == body.key + moved = [tid for tid, only_theirs in own.items() if only_theirs] + for tid in moved: + await enqueue_job("move_thread_to_folder_job", str(tid), "trash") + return {"moved": len(moved)} diff --git a/backend/app/api/templates.py b/backend/app/api/templates.py new file mode 100644 index 0000000..16942e8 --- /dev/null +++ b/backend/app/api/templates.py @@ -0,0 +1,129 @@ +"""Templates: saved replies, inserted from the compose window's More menu.""" + +from __future__ import annotations + +import datetime as dt +import uuid +from typing import Annotated + +import nh3 +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import raise_api_error +from app.db import get_session +from app.identity.deps import CurrentUser, require_csrf +from app.models import MailTemplate + +router = APIRouter(prefix="/templates", tags=["compose"], dependencies=[Depends(require_csrf)]) + +DbSession = Annotated[AsyncSession, Depends(get_session)] + +MAX_TEMPLATES = 200 +MAX_BODY_BYTES = 256 * 1024 + +# What the compose editor writes. A template goes back into the editor as HTML, +# so it is cleaned like any HTML the app renders: no scripts, no handlers. +_TAGS = { + "a", "b", "blockquote", "br", "code", "div", "em", "font", "h1", "h2", "h3", + "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strike", "strong", + "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "u", "ul", +} # fmt: skip +_GENERIC = {"style", "class", "dir", "align"} +_SPECIFIC = { + "a": {"href", "title"}, + "img": {"src", "alt", "width", "height"}, + "font": {"color", "face", "size"}, + "td": {"colspan", "rowspan"}, + "th": {"colspan", "rowspan"}, +} +# nh3 takes no wildcard: the generic attributes go onto every tag. +_ATTRS = {tag: _GENERIC | _SPECIFIC.get(tag, set()) for tag in _TAGS} + + +class TemplateOut(BaseModel): + id: uuid.UUID + name: str + subject: str + body_html: str + updated_at: dt.datetime + + +class TemplateIn(BaseModel): + name: str = Field(min_length=1, max_length=80) + subject: str = Field(default="", max_length=500) + body_html: str = "" + + +def _out(t: MailTemplate) -> TemplateOut: + return TemplateOut( + id=t.id, name=t.name, subject=t.subject, body_html=t.body_html, updated_at=t.updated_at + ) + + +def clean_body(html: str) -> str: + return nh3.clean( + html, + tags=_TAGS, + attributes=_ATTRS, + url_schemes={"http", "https", "mailto", "data"}, + link_rel="noopener noreferrer", + ) + + +@router.get("", response_model=list[TemplateOut]) +async def list_templates(user: CurrentUser, db: DbSession) -> list[TemplateOut]: + rows = await db.scalars( + select(MailTemplate) + .where(MailTemplate.user_id == user.id) + .order_by(func.lower(MailTemplate.name)) + ) + return [_out(t) for t in rows] + + +@router.post("", response_model=TemplateOut) +async def save_template(body: TemplateIn, user: CurrentUser, db: DbSession) -> TemplateOut: + """Save a template. Saving under a name that exists replaces that template.""" + name = " ".join(body.name.split()) + if not name: + raise_api_error(400, "invalid", "a template needs a name") + if len(body.body_html.encode()) > MAX_BODY_BYTES: + raise_api_error(400, "too_large", "the template is too large") + html = clean_body(body.body_html) + subject = " ".join(body.subject.split()) + existing = await db.scalar( + select(MailTemplate).where( + MailTemplate.user_id == user.id, func.lower(MailTemplate.name) == name.lower() + ) + ) + if existing is not None: + existing.name, existing.subject, existing.body_html = name, subject, html + existing.updated_at = dt.datetime.now(dt.UTC) + saved = existing + else: + count = await db.scalar( + select(func.count()).select_from(MailTemplate).where(MailTemplate.user_id == user.id) + ) + if (count or 0) >= MAX_TEMPLATES: + raise_api_error(409, "too_many", f"at most {MAX_TEMPLATES} templates") + saved = MailTemplate(user_id=user.id, name=name, subject=subject, body_html=html) + db.add(saved) + await db.commit() + await db.refresh(saved) + return _out(saved) + + +@router.delete("/{template_id}") +async def delete_template( + template_id: uuid.UUID, user: CurrentUser, db: DbSession +) -> dict[str, str]: + row = await db.scalar( + select(MailTemplate).where(MailTemplate.id == template_id, MailTemplate.user_id == user.id) + ) + if row is None: + raise_api_error(404, "not_found", "template not found") + await db.delete(row) + await db.commit() + return {"status": "deleted"} diff --git a/backend/app/api/threads.py b/backend/app/api/threads.py index 2de3e07..c4a02fc 100644 --- a/backend/app/api/threads.py +++ b/backend/app/api/threads.py @@ -4,13 +4,14 @@ from __future__ import annotations import uuid from collections.abc import Collection -from typing import Annotated +from typing import Annotated, Any from fastapi import APIRouter, Depends, Query from sqlalchemy import Text, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement +from app import subscriptions as subs from app.api.errors import raise_api_error from app.api.schemas import ( Address, @@ -19,10 +20,10 @@ from app.api.schemas import ( ThreadListOut, ThreadSummary, ) -from app.api.settings import allowed_image_senders +from app.api.settings import allowed_image_senders, blocks_trackers from app.db import get_session from app.identity.deps import CurrentUser -from app.models import Mailbox, Message, Thread, User +from app.models import BodyPart, Mailbox, Message, SubscriptionState, Thread, User router = APIRouter(prefix="/threads", tags=["mail"]) @@ -50,6 +51,7 @@ def _summary( has_attachments=thread.has_attachments, pinned=thread.pinned, last_message_at=thread.last_message_at, + snoozed_until=thread.snoozed_until, ) @@ -74,6 +76,7 @@ def _envelope(msg: Message, allowed_senders: Collection[str] = ()) -> MessageEnv images_allowed=bool(sender) and sender in allowed_senders, size_bytes=msg.size_bytes, snippet=msg.snippet, + content_language=msg.content_language or "", ) @@ -130,6 +133,10 @@ async def list_threads( .where(Message.user_id == user.id, Mailbox.special_use == special) ) conditions.append(Thread.id.in_(in_folder)) + if special == "inbox": + conditions.append(Thread.snoozed_until.is_(None)) # back when it wakes + elif view == "snoozed": + conditions.append(Thread.snoozed_until.isnot(None)) elif view != "unified": raise_api_error(400, "bad_view", "unsupported view") @@ -144,11 +151,20 @@ async def list_threads( ) ) or 0 - order_by = ( - (Thread.last_message_at.asc(), Thread.id.asc()) - if order == "asc" - else (Thread.last_message_at.desc(), Thread.id.desc()) + # A thread woken from snooze sorts as if it had just arrived. + sort_at = func.greatest( + Thread.last_message_at, func.coalesce(Thread.unsnoozed_at, Thread.last_message_at) ) + if view == "snoozed": + # Soonest back first, whatever the list order. + order_by: tuple[ColumnElement[Any], ...] = ( + Thread.snoozed_until.asc(), + Thread.id.asc(), + ) + elif order == "asc": + order_by = (sort_at.asc(), Thread.id.asc()) + else: + order_by = (sort_at.desc(), Thread.id.desc()) stmt = select(Thread).where(*conditions).order_by(*order_by).offset(offset).limit(limit) threads = list(await db.scalars(stmt)) @@ -177,6 +193,41 @@ async def list_threads( ) +async def _mark_subscriptions( + db: AsyncSession, + user_id: uuid.UUID, + messages: list[Message], + envelopes: list[MessageEnvelope], +) -> None: + """Tell the reading pane which messages can be unsubscribed from, and how.""" + keys: dict[uuid.UUID, str] = {} + for m, env in zip(messages, envelopes, strict=True): + if not m.list_unsubscribe: + continue + targets = subs.unsubscribe_targets(m.list_unsubscribe) + if m.list_unsubscribe_post and targets.https: + env.unsubscribe_via = "one_click" + elif targets.mailto: + env.unsubscribe_via = "mailto" + elif targets.https: + env.unsubscribe_via = "web" + else: + continue + sender = m.from_addr[0].get("email", "") if m.from_addr else "" + env.subscription_key = keys[m.id] = subs.group_key(m.list_id, sender) + if keys: + done = set( + await db.scalars( + select(SubscriptionState.key).where( + SubscriptionState.user_id == user_id, + SubscriptionState.key.in_(set(keys.values())), + ) + ) + ) + for env in envelopes: + env.unsubscribed = env.subscription_key in done + + def _visible_in(current_special: str | None) -> ColumnElement[bool]: """Which of a thread's messages a view shows (see ``_REMOVAL_FOLDERS``): Trash/Spam show only their own messages; every other view hides them.""" @@ -281,6 +332,20 @@ async def get_thread( env = _envelope(m, allowed_senders) env.also_in_account_ids = also[m.delivery_id or f"id:{m.id}"] envelopes.append(env) + await _mark_subscriptions(db, user.id, ordered, envelopes) + if not blocks_trackers(user) and ordered: + # Trackers left in are remote images: offer "Load images" for them too. + with_trackers = set( + await db.scalars( + select(BodyPart.message_id).where( + BodyPart.message_id.in_([m.id for m in ordered]), + BodyPart.kind == "html_sanitized", + BodyPart.trackers_blocked > 0, + ) + ) + ) + for env in envelopes: + env.has_remote_images = env.has_remote_images or env.id in with_trackers order = "desc" if sort == "desc" else "asc" return ThreadDetail( @@ -289,4 +354,5 @@ async def get_thread( order=order, pinned=thread.pinned, messages=envelopes, + snoozed_until=thread.snoozed_until, ) diff --git a/backend/app/api/translation.py b/backend/app/api/translation.py new file mode 100644 index 0000000..e6807c3 --- /dev/null +++ b/backend/app/api/translation.py @@ -0,0 +1,30 @@ +"""What email translation offers (app/translation.py), for the reading pane.""" + +from __future__ import annotations + +from fastapi import APIRouter +from pydantic import BaseModel + +from app import translation +from app.identity.deps import CurrentUser + +router = APIRouter(prefix="/translation", tags=["mail"]) + + +class LanguageOut(BaseModel): + code: str + name: str + + +class TranslationInfoOut(BaseModel): + available: bool + languages: list[LanguageOut] + + +@router.get("/languages", response_model=TranslationInfoOut) +async def translation_languages(user: CurrentUser) -> TranslationInfoOut: + found = await translation.languages() + return TranslationInfoOut( + available=bool(found), + languages=[LanguageOut(code=x.code, name=x.name) for x in found], + ) diff --git a/backend/app/completion.py b/backend/app/completion.py new file mode 100644 index 0000000..1151533 --- /dev/null +++ b/backend/app/completion.py @@ -0,0 +1,391 @@ +"""Sentence completion in the compose window: phrases the user writes, finished. + +No AI: a per-user index of the word sequences the user has written at least +twice, built from their own sent mail (quoted replies and signatures cut off), +plus a few common email phrases per language to start from. Typing "I look +forward" finds what usually follows those words — "to hearing from you." — and +the compose window shows it in grey until Tab takes it. + +It works in any language the user writes in: matching is on the words just +typed, so a Polish sentence only ever meets Polish phrases. + +The index lives in ``completion_phrases`` (context → continuation, with how +often it was seen) and is rebuilt in the background at most once a day, when +the compose window asks for suggestions (:func:`build_index_job`). +""" + +from __future__ import annotations + +import datetime as dt +import html as html_lib +import logging +import re +import uuid +from collections import Counter +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import delete, select +from sqlalchemy.dialects.postgresql import insert + +from app.db import session_scope +from app.models import ( + Account, + BodyPart, + CompletionIndexState, + CompletionPhrase, + Mailbox, + Message, +) + +logger = logging.getLogger(__name__) + +START = "^" # a sentence's start, as a word: "^ pozdr" completes the first word +MAX_CONTEXT = 3 # words before the cursor matched on +MAX_CONTINUATION = 6 # words suggested at most +MIN_COUNT = 2 # seen this often to be offered; one-offs are noise (and private) +EXTEND_SHARE = 0.6 # a longer suggestion only while it is what usually follows +CORPUS_MESSAGES = 500 # most recent sent messages read +CORPUS_CHARS = 3000 # of each +REBUILD_AFTER = dt.timedelta(days=1) + +# Where the user's own words end and quoted mail or a signature begins: a reply +# header ("On … wrote:", Polish Gmail's "… napisał(a):", …), a forwarded or +# original-message marker, or an Outlook-style header block. +_QUOTE_START = re.compile( + r"^\s*(?:" + r"-{2,}\s*(?:original message|forwarded message|wiadomość oryginalna" + r"|wiadomość przekazana dalej|ursprüngliche nachricht|weitergeleitete nachricht)" + r"|(?:from|od|von|de):\s.+" + r"|sent from my\s" + r")", + re.IGNORECASE, +) +_QUOTE_HEADER_END = re.compile( + # German names the writer after the verb: "Am … schrieb Anna:". + r"(?:wrote|napisał\(a\)|napisała?|pisze|schrieb(?:\s+.{0,100})?|a écrit|escribió)\s*:\s*$", + re.IGNORECASE, +) +_SIGNATURE = re.compile(r"^--\s*$") +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+|\n+") +_WORD = re.compile(r"\S+") +_EDGE_PUNCT = re.compile(r"^[\"'“”„«»(\[–—-]+|[\"'“”„«»)\],.;:!?…–—-]+$") +# Words a phrase is never learned across: numbers (phones, dates, prices), +# addresses and links — useless to complete, and often private. +_NOT_A_WORD = re.compile(r"\d|@|www\.|https?:", re.IGNORECASE) + + +def html_to_text(html: str) -> str: + """An HTML signature as text lines (block tags and
break lines).""" + text = re.sub(r"(?i)|", "\n", html) + text = re.sub(r"<[^>]+>", "", text) + return html_lib.unescape(text) + + +def own_text(text: str, signatures: Iterable[str] = ()) -> str: + """The part of a sent message the user wrote: no quoted mail, no signature + (the ``--`` kind, or one of ``signatures`` — the accounts' own, as text).""" + sig_lines = {norm_line(x) for sig in signatures for x in sig.splitlines() if norm_line(x)} + kept: list[str] = [] + for line in text.splitlines(): + if _SIGNATURE.match(line) or _QUOTE_START.match(line) or _QUOTE_HEADER_END.search(line): + break + if norm_line(line) in sig_lines: + continue + if line.lstrip().startswith(">"): + continue # an inline-quoted line + kept.append(line) + return "\n".join(kept) + + +def norm_line(line: str) -> str: + return " ".join(line.lower().split()) + + +def norm(word: str) -> str: + """A word as matched: lower case, outer punctuation off ("Hello," -> "hello").""" + return _EDGE_PUNCT.sub("", word.lower()) + + +def sentences(text: str) -> Iterable[list[str]]: + for chunk in _SENTENCE_END.split(text): + words = _WORD.findall(chunk) + if words: + yield words + + +@dataclass(frozen=True) +class Phrase: + context: str # normalized words before the cursor, space-joined ("^ i look") + continuation: str # the words that follow, as written ("forward to hearing") + count: int + + +def build_phrases(texts: Iterable[str], weight: int = 1) -> dict[tuple[str, str], int]: + """(context, continuation) -> count, for every sequence seen MIN_COUNT times. + + Counts each sentence's word sequences (with the start marker), keeps the + frequent ones, and splits each into "what was typed" and "what follows". + """ + counts: Counter[tuple[str, ...]] = Counter() + surface: dict[tuple[str, ...], tuple[str, ...]] = {} + longest = MAX_CONTEXT + MAX_CONTINUATION + for text in texts: + for words in sentences(text): + seq = [START, *words] + keys = [START, *("" if _NOT_A_WORD.search(w) else norm(w) for w in words)] + for i in range(len(seq)): + for n in range(2, min(longest, len(seq) - i) + 1): + gram = tuple(keys[i : i + n]) + if "" in gram: + break # a lone punctuation mark: no phrase across it + counts[gram] += weight + surface.setdefault(gram, tuple(seq[i : i + n])) + out: dict[tuple[str, str], int] = {} + for gram, count in counts.items(): + if count < MIN_COUNT: + continue + written = surface[gram] + for k in range(1, min(MAX_CONTEXT, len(gram) - 1) + 1): + rest = len(gram) - k + if rest > MAX_CONTINUATION: + continue + key = (" ".join(gram[:k]), " ".join(written[k:])) + out[key] = max(out.get(key, 0), count) + return out + + +# A few common phrases per language, for users with little sent mail yet. +STARTER_PHRASES: dict[str, list[str]] = { + "en": [ + "I look forward to hearing from you.", + "I'm looking forward to hearing from you.", + "Looking forward to hearing from you.", + "Looking forward to working with you.", + "Thank you for your email.", + "Thank you for your help.", + "Thank you for your quick reply.", + "Thanks in advance.", + "Please let me know if you have any questions.", + "Let me know if you have any questions.", + "Let me know what you think.", + "Please find attached the", + "I hope this email finds you well.", + "I hope you're doing well.", + "Sorry for the late reply.", + "I would be happy to help.", + "Feel free to contact me if you need anything else.", + "Could you please send me the", + "Please confirm at your earliest convenience.", + "Have a great day!", + "Have a nice weekend!", + "Best regards,", + "Kind regards,", + ], + "pl": [ + "Z poważaniem,", + "Pozdrawiam serdecznie,", + "Dziękuję za wiadomość.", + "Dziękuję za szybką odpowiedź.", + "Dziękuję za pomoc.", + "Z góry dziękuję.", + "W załączniku przesyłam", + "Proszę o informację zwrotną.", + "W razie pytań proszę o kontakt.", + "Czekam na odpowiedź.", + "Przepraszam za opóźnienie w odpowiedzi.", + "Będę wdzięczny za odpowiedź.", + "Będę wdzięczna za odpowiedź.", + "Zgodnie z naszą rozmową", + "Proszę o potwierdzenie.", + "Daj znać, co myślisz.", + "Miłego dnia!", + "Miłego weekendu!", + ], + "de": [ + "Mit freundlichen Grüßen", + "Viele Grüße", + "Beste Grüße", + "Vielen Dank für Ihre E-Mail.", + "Vielen Dank für Ihre schnelle Antwort.", + "Vielen Dank im Voraus.", + "Ich freue mich auf Ihre Antwort.", + "Bei Fragen stehe ich Ihnen gerne zur Verfügung.", + "Im Anhang finden Sie", + "Entschuldigen Sie die späte Antwort.", + "Ich hoffe, es geht Ihnen gut.", + "Schönen Tag noch!", + "Schönes Wochenende!", + ], +} + +# Built once: every starter phrase counts as seen MIN_COUNT times. +_STARTERS: dict[str, Counter[str]] = {} +for (_ctx, _cont), _n in build_phrases( + (p for phrases in STARTER_PHRASES.values() for p in phrases), weight=MIN_COUNT +).items(): + _STARTERS.setdefault(_ctx, Counter())[_cont] = _n + +_LEADING_PUNCT = re.compile(r"^[\"'“”„«»(\[]+") + + +def _candidates(phrases: dict[str, Counter[str]], context: str, partial: str) -> Counter[str]: + """Continuations of ``context`` whose first word starts with ``partial``.""" + out: Counter[str] = Counter() + for continuation, count in phrases.get(context, Counter()).items(): + first = continuation.split(" ", 1)[0] + if partial and not first.lower().startswith(partial.lower()): + continue + if partial and len(first) <= len(partial): + continue # that word is typed in full already + out[continuation] = count + return out + + +def choose(candidates: Counter[str]) -> str | None: + """The suggestion: the most frequent next word, extended word by word only + while the longer phrase is what usually follows (EXTEND_SHARE).""" + if not candidates: + return None + by_first: Counter[str] = Counter() + for continuation, count in candidates.items(): + first = continuation.split(" ", 1)[0] + by_first[first] = max(by_first[first], count) + chosen, chosen_count = by_first.most_common(1)[0] + while True: + depth = chosen.count(" ") + 1 + longer = [ + (c, n) + for c, n in candidates.items() + if c.startswith(chosen + " ") and c.count(" ") == depth + ] + if not longer: + return chosen + best, best_count = max(longer, key=lambda x: x[1]) + if best_count < max(MIN_COUNT, EXTEND_SHARE * chosen_count): + return chosen + chosen, chosen_count = best, best_count + + +def contexts(before: str) -> tuple[list[str], str]: + """For the text before the cursor: the contexts to try, longest first, and + the partly typed word ('' after a space). No contexts: nothing to suggest + (an empty sentence, or the cursor right after a punctuation mark).""" + if not before or not before.strip(): + return [], "" + last = before[-1] + if not last.isspace() and not last.isalnum() and last not in "'’-": + return [], "" # "you," or "end." — not mid-word, and no space typed yet + tail = _SENTENCE_END.split(before)[-1] + words = _WORD.findall(tail) + partial = "" + if words and not last.isspace(): + partial = _LEADING_PUNCT.sub("", words.pop()) + if not words and not partial: + return [], "" # a new sentence with nothing in it yet + keys = [START, *(norm(w) for w in words)] + if "" in keys: # a lone punctuation mark: the phrase starts after it + keys = keys[len(keys) - keys[::-1].index("") :] + if not keys: + return [], "" + return [" ".join(keys[-k:]) for k in range(min(MAX_CONTEXT, len(keys)), 0, -1)], partial + + +def suggest(personal: dict[str, Counter[str]], before: str) -> str | None: + """What to show after ``before`` (the text up to the cursor), or None. + + The returned text starts where the cursor is: the rest of a partly typed + word, or (after a space) the next words. The user's own phrases and the + starter phrases count together. + """ + tried, partial = contexts(before) + for context in tried: + merged = _candidates(personal, context, partial) + for cont, count in _candidates(_STARTERS, context, partial).items(): + merged[cont] = merged.get(cont, 0) + count + if " " not in context and not partial: + # A single word before a space is a weak hint: only a strong habit. + merged = Counter({c: n for c, n in merged.items() if n >= 2 * MIN_COUNT}) + best = choose(merged) + if best: + return best[len(partial) :] if partial else best + return None + + +def complete(personal: dict[str, Counter[str]], before: str) -> str | None: + """:func:`suggest`, and for a word typed in full but no space yet ("I look + forward"), what follows it — with the space ("␣to hearing from you.").""" + found = suggest(personal, before) + if found is None and before[-1:].isalnum(): + after_space = suggest(personal, before + " ") + if after_space: + return " " + after_space + return found + + +def keys_for(before: str) -> list[str]: + """The contexts :func:`complete` may look up, to fetch them from the index.""" + keys = contexts(before)[0] + if before[-1:].isalnum(): + keys += contexts(before + " ")[0] + return keys + + +async def personal_phrases( + session: Any, user_id: uuid.UUID, keys: list[str] +) -> dict[str, Counter[str]]: + rows = await session.execute( + select( + CompletionPhrase.context, CompletionPhrase.continuation, CompletionPhrase.count + ).where(CompletionPhrase.user_id == user_id, CompletionPhrase.context.in_(keys)) + ) + out: dict[str, Counter[str]] = {} + for context, continuation, count in rows: + out.setdefault(context, Counter())[continuation] = count + return out + + +async def build_index_job(ctx: dict[str, Any], user_id: str) -> int: + """Rebuild a user's phrase index from their recent sent mail. Returns its size.""" + uid = uuid.UUID(user_id) + async with session_scope() as session: + texts = list( + await session.scalars( + select(BodyPart.content) + .join(Message, Message.id == BodyPart.message_id) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where( + Message.user_id == uid, + Mailbox.special_use == "sent", + BodyPart.kind == "text", + ) + .order_by(Message.internal_date.desc()) + .limit(CORPUS_MESSAGES) + ) + ) + signatures = [ + html_to_text(sig) + for sig in await session.scalars( + select(Account.signature).where(Account.user_id == uid, Account.signature != "") + ) + ] + phrases = build_phrases(own_text(t, signatures)[:CORPUS_CHARS] for t in texts) + await session.execute(delete(CompletionPhrase).where(CompletionPhrase.user_id == uid)) + rows = [ + {"user_id": uid, "context": c, "continuation": cont, "count": n} + for (c, cont), n in phrases.items() + ] + for start in range(0, len(rows), 1000): + await session.execute(insert(CompletionPhrase), rows[start : start + 1000]) + stmt = insert(CompletionIndexState).values(user_id=uid, built_at=dt.datetime.now(dt.UTC)) + await session.execute( + stmt.on_conflict_do_update( + index_elements=["user_id"], set_={"built_at": stmt.excluded.built_at} + ) + ) + await session.commit() + logger.info( + "completion index for %s: %d phrases from %d messages", user_id, len(rows), len(texts) + ) + return len(rows) diff --git a/backend/app/config.py b/backend/app/config.py index ce2c775..5ed920e 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -71,6 +71,9 @@ class Settings(BaseSettings): # Spelling and grammar checking in the compose window (LanguageTool server, # e.g. http://languagetool:8010). Empty: the check is off for everyone. languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL") + # Email translation: a LibreTranslate server (app/translation.py), e.g. + # http://libretranslate:5000. Empty: translation is off. + libretranslate_url: str = Field(default="", alias="LIBRETRANSLATE_URL") # New passwords are checked against known data breaches (Have I Been Pwned's # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server). password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK") diff --git a/backend/app/indexer/indexer.py b/backend/app/indexer/indexer.py index 3783959..39246bb 100644 --- a/backend/app/indexer/indexer.py +++ b/backend/app/indexer/indexer.py @@ -25,6 +25,7 @@ from app.indexer.sanitize import sanitize_html from app.models import Account, Attachment, BodyPart, KnownRecipient, Message from app.providers.base import RemoteMessage from app.search.base import SearchDoc +from app.subscriptions import read_list_headers from app.threading_.normalize import synthesize_message_id from app.threading_.threader import thread_message @@ -167,6 +168,7 @@ async def persist_message( else: sha = rawstore.put_cached(user_id, remote.raw) parsed = await _parse_bounded(user_id, sha, remote.raw) + list_headers = read_list_headers(remote.raw) # for the Subscriptions page message_id_hdr = await _resolve_message_id( session, user_id=user_id, account_id=account_id, parsed_id=parsed.message_id, raw=remote.raw @@ -207,6 +209,9 @@ async def persist_message( has_attachments=parsed.has_attachments, snippet=parsed.snippet, blob_sha256=sha, + list_id=list_headers.list_id, + list_unsubscribe=list_headers.unsubscribe, + list_unsubscribe_post=list_headers.one_click, ) session.add(message) await session.flush() @@ -224,6 +229,7 @@ async def persist_message( kind="html_sanitized", content=result.html, sanitizer_v=result.sanitizer_v, + trackers_blocked=result.trackers_blocked, ) ) diff --git a/backend/app/indexer/sanitize.py b/backend/app/indexer/sanitize.py index bde58c0..015c38f 100644 --- a/backend/app/indexer/sanitize.py +++ b/backend/app/indexer/sanitize.py @@ -19,7 +19,10 @@ import nh3 # render re-sanitizes stale content from the raw blob (SPEC §10.4). # v5: this server's own inline-image URLs quoted in a message load from here. # v6: links keep only http(s) and mailto (no data: pages). -SANITIZER_VERSION = 6 +# v7: tracking pixels are removed (and counted), even when images are allowed. +# v8: CSS sizes in other units (in, em, %, pt) no longer read as pixels — v7 +# took 0.31in signature logos for trackers. +SANITIZER_VERSION = 8 # This server's inline-image URL, however it ended up written: "/api/…", # "https://host/api/…", or mangled to "http:///api/…" by an earlier copy. @@ -154,6 +157,94 @@ class SanitizeResult: html: str has_remote_images: bool sanitizer_v: int + trackers_blocked: int = 0 + + +# --- Tracking pixels --------------------------------------------------------------- +# A remote image that is invisible (1×1, or hidden by CSS) has no use but to tell +# the sender the mail was opened, and by whom. A few well-known open-tracking URL +# shapes are caught whatever their size. Matching runs on the raw HTML before nh3: +# a miss only leaves an image in, and nh3 still cleans whatever remains. +_IMG_TAG = re.compile(r"]*>", re.IGNORECASE) +_ATTR = re.compile(r"""([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("[^"]*"|'[^']*'|[^\s"'>]+)""") +_CSS_HIDDEN = re.compile( + r"display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?![.\d]*[1-9])", + re.IGNORECASE, +) +_TRACKER_URL = re.compile( + r"//[^/]*mailtrack\.io/" + r"|/track/open\b" + r"|/wf/open\b" + r"|/open\.(?:php|aspx|gif)\b" + r"|[?&]tracking[_-]?pixel=", + re.IGNORECASE, +) + + +_PX = re.compile(r"\s*(\d+(?:\.\d*)?|\.\d+)\s*(px)?\s*$", re.IGNORECASE) + + +def _px(value: str | None) -> float | None: + """A size in pixels ("1", "1px"), or None for anything else: other units + (0.31in is a 30-pixel signature logo), auto, %, or malformed (".", "1.2.3").""" + if value is None: + return None + match = _PX.match(value) + return float(match.group(1)) if match else None + + +def _tiny(value: str | None) -> bool: + px = _px(value) + return px is not None and px <= 2 + + +def _css(style: str) -> dict[str, str]: + """An inline style's declarations, lower-cased, without !important.""" + out: dict[str, str] = {} + for decl in style.split(";"): + name, _, value = decl.partition(":") + value = value.lower().replace("!important", "").strip() + if name.strip() and value: + out[name.strip().lower()] = value + return out + + +def is_tracking_pixel(tag: str) -> bool: + """Whether an ```` tag is a remote open-tracking pixel.""" + attrs = {name.lower(): value.strip("\"'") for name, value in _ATTR.findall(tag)} + src = attrs.get("src", "").strip() + if not src.lower().startswith(("http:", "https:", "//")): + return False # inline (cid:/data:) images can't report anything + if _TRACKER_URL.search(src): + return True + style = attrs.get("style", "") + if _CSS_HIDDEN.search(style): + return True + # The size the browser draws: a style width/height overrides the attribute + # (and "auto", % or another unit is no tiny size); a max-width/max-height + # of a pixel or two caps it; height "auto" follows the width. + css = _css(style) + wide = _tiny(css["width"]) if "width" in css else _tiny(attrs.get("width")) + tall = _tiny(css["height"]) if "height" in css else _tiny(attrs.get("height")) + wide = wide or _tiny(css.get("max-width")) + tall = tall or _tiny(css.get("max-height")) + if css.get("height") == "auto": + tall = wide + return wide and tall + + +def strip_tracking_pixels(html: str) -> tuple[str, int]: + """Remove tracking pixels from raw HTML; returns the HTML and how many.""" + count = 0 + + def repl(match: re.Match[str]) -> str: + nonlocal count + if is_tracking_pixel(match.group(0)): + count += 1 + return "" + return match.group(0) + + return _IMG_TAG.sub(repl, html), count def _sanitize_style(value: str, allow_remote: bool) -> str | None: @@ -188,15 +279,24 @@ def _sanitize_style_elements(html: str, allow_remote: bool, state: dict[str, boo def sanitize_html( - html: str, message_id: uuid.UUID, *, allow_remote_images: bool = False + html: str, + message_id: uuid.UUID, + *, + allow_remote_images: bool = False, + block_trackers: bool = True, ) -> SanitizeResult: """Sanitize ``html`` for the message ``message_id``. Returns the safe HTML, whether any remote images were blocked, and the sanitizer version. Pass ``allow_remote_images=True`` after the user approves a sender to render remote images and CSS backgrounds (SPEC §9.4). + ``block_trackers=False`` (Settings → Privacy) leaves tracking pixels in; they + are remote images like any other then. """ state = {"has_remote": False} + trackers = 0 + if block_trackers: + html, trackers = strip_tracking_pixels(html) def attribute_filter(tag: str, attr: str, value: str) -> str | None: if attr == "style": @@ -247,4 +347,5 @@ def sanitize_html( html=cleaned, has_remote_images=state["has_remote"], sanitizer_v=SANITIZER_VERSION, + trackers_blocked=trackers, ) diff --git a/backend/app/models.py b/backend/app/models.py index 8120d0b..55715b8 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -199,6 +199,12 @@ class Thread(Base): has_attachments: Mapped[bool] = mapped_column(nullable=False, server_default=text("false")) pinned: Mapped[bool] = mapped_column(nullable=False, server_default=text("false")) last_message_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False) + # Snooze: out of the Inbox until this time (NULL when not snoozed). A message + # that arrives after ``snoozed_at`` wakes it early; ``unsnoozed_at`` (when it + # last woke) puts it back at the top of the Inbox. + snoozed_until: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True)) + snoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True)) + unsnoozed_at: Mapped[dt.datetime | None] = mapped_column(TIMESTAMP(timezone=True)) class Message(Base): @@ -263,6 +269,17 @@ class Message(Base): snippet: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) blob_sha256: Mapped[str] = mapped_column(Text, nullable=False) body_indexed: Mapped[str | None] = mapped_column(TSVECTOR) + # Mailing-list headers, for the Subscriptions page. ``list_unsubscribe`` is + # NULL until read (a backfill job fills older mail), '' when there is none. + list_id: Mapped[str | None] = mapped_column(Text) + list_unsubscribe: Mapped[str | None] = mapped_column(Text) + # RFC 8058: the sender takes a one-click unsubscribe POST. + list_unsubscribe_post: Mapped[bool] = mapped_column( + nullable=False, server_default=text("false") + ) + # The language of its text (app/translation.py), found the first time it is + # shown: NULL not looked at yet, '' could not tell. + content_language: Mapped[str | None] = mapped_column(Text) class BodyPart(Base): @@ -276,6 +293,8 @@ class BodyPart(Base): content: Mapped[str] = mapped_column(Text, nullable=False) # SPEC §10.4: re-sanitize on render if the sanitizer version changed. sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + # Tracking pixels taken out of this HTML (shown as "N trackers blocked"). + trackers_blocked: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) class Attachment(Base): @@ -404,6 +423,78 @@ class Invitation(Base): ) +class SubscriptionState(Base): + """What a user did about a bulk sender on the Subscriptions page.""" + + __tablename__ = "subscription_states" + + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + # The sender group: "list:" or "from:
". + key: Mapped[str] = mapped_column(Text, primary_key=True) + unsubscribed_at: Mapped[dt.datetime] = _now() + method: Mapped[str] = mapped_column(Text, nullable=False) # one_click | mailto | web + + +class MailTemplate(Base): + """A saved reply, inserted from the compose window.""" + + __tablename__ = "mail_templates" + + id: Mapped[uuid.UUID] = _uuid_pk() + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + name: Mapped[str] = mapped_column(Text, nullable=False) + subject: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) + body_html: Mapped[str] = mapped_column(Text, nullable=False, server_default=text("''")) + created_at: Mapped[dt.datetime] = _now() + updated_at: Mapped[dt.datetime] = _now() + + +class MessageTranslation(Base): + """A message's HTML translated into a language (app/translation.py), kept so + it is translated once. ``images``: with remote images shown or not.""" + + __tablename__ = "message_translations" + + message_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True + ) + target: Mapped[str] = mapped_column(Text, primary_key=True) + images: Mapped[bool] = mapped_column(primary_key=True) + source: Mapped[str] = mapped_column(Text, nullable=False) + html: Mapped[str] = mapped_column(Text, nullable=False) + sanitizer_v: Mapped[int] = mapped_column(Integer, nullable=False) + created_at: Mapped[dt.datetime] = _now() + + +class CompletionPhrase(Base): + """Sentence completion (app/completion.py): after these words, the user + usually writes this — seen ``count`` times in their sent mail.""" + + __tablename__ = "completion_phrases" + + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + context: Mapped[str] = mapped_column(Text, primary_key=True) # normalized words + continuation: Mapped[str] = mapped_column(Text, primary_key=True) # as written + count: Mapped[int] = mapped_column(Integer, nullable=False) + + +class CompletionIndexState(Base): + """When a user's completion index was last built.""" + + __tablename__ = "completion_index_state" + + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + built_at: Mapped[dt.datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False) + + class PushSubscription(Base): """A browser or installed app to tell about new mail (Web Push).""" diff --git a/backend/app/queue.py b/backend/app/queue.py index b3c2d79..4b0ebb6 100644 --- a/backend/app/queue.py +++ b/backend/app/queue.py @@ -21,7 +21,14 @@ QUEUE_INTERACTIVE = "interactive" QUEUE_BULK = "bulk" # Jobs default to the interactive queue; only bulk backfills go to the bulk queue. -_BULK_JOBS = frozenset({"sync_bootstrap_account", "dedupe_existing_job"}) +_BULK_JOBS = frozenset( + { + "sync_bootstrap_account", + "dedupe_existing_job", + "backfill_list_headers_job", + "build_index_job", + } +) def queue_for(job_name: str) -> str: diff --git a/backend/app/snooze.py b/backend/app/snooze.py new file mode 100644 index 0000000..9919e65 --- /dev/null +++ b/backend/app/snooze.py @@ -0,0 +1,92 @@ +"""Snooze: a conversation leaves the Inbox until a chosen time, then comes back. + +Kept in this database only (the mail server knows nothing of it): a snoozed +thread is hidden from the Inbox view and listed under Snoozed. The wake job, +run every minute, brings back threads whose time has come — or that got a new +message meanwhile, as Gmail does — at the top of the Inbox, the newest message +marked unread. +""" + +from __future__ import annotations + +import datetime as dt +import logging +import uuid +from typing import Any + +from sqlalchemy import or_, select + +from app.db import session_scope +from app.events import get_event_bus +from app.models import Mailbox, Message, Thread +from app.queue import enqueue_job +from app.sync.actions import apply_flags_to_messages + +logger = logging.getLogger(__name__) + +MAX_SNOOZE = dt.timedelta(days=366) +_SEEN = frozenset({"\\Seen"}) + + +def snooze(thread: Thread, until: dt.datetime, now: dt.datetime) -> None: + thread.snoozed_until = until + thread.snoozed_at = now + + +def wake(thread: Thread, now: dt.datetime) -> None: + thread.snoozed_until = None + thread.snoozed_at = None + thread.unsnoozed_at = now + + +async def wake_due_job(ctx: dict[str, Any]) -> int: + """Cron (every minute): wake snoozed threads that are due. Returns how many.""" + now = dt.datetime.now(dt.UTC) + woken: list[tuple[uuid.UUID, uuid.UUID]] = [] + unread: list[tuple[uuid.UUID, Message]] = [] + async with session_scope() as session: + due = list( + await session.scalars( + select(Thread) + .where( + Thread.snoozed_until.isnot(None), + or_(Thread.snoozed_until <= now, Thread.last_message_at > Thread.snoozed_at), + ) + .with_for_update(skip_locked=True) + ) + ) + for thread in due: + on_time = thread.snoozed_until is not None and thread.snoozed_until <= now + wake(thread, now) + woken.append((thread.user_id, thread.id)) + if not on_time: + continue # a new message woke it: that message is unread already + # Back as new: the newest Inbox message becomes unread again. + newest = await session.scalar( + select(Message) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where( + Message.thread_id == thread.id, + Message.user_id == thread.user_id, + Mailbox.special_use == "inbox", + ) + .order_by(Message.internal_date.desc()) + .limit(1) + ) + if newest is not None and "\\Seen" in newest.flags: + await apply_flags_to_messages(session, [newest], frozenset(), _SEEN) + unread.append((thread.user_id, newest)) + await session.commit() + + bus = get_event_bus() + for user_id, message in unread: + await enqueue_job("push_flags_job", str(message.id), [], ["\\Seen"]) + await bus.publish( + "flags.changed", + {"user_id": str(user_id), "message_id": str(message.id), "flags": list(message.flags)}, + ) + for user_id, thread_id in woken: + await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)}) + if woken: + logger.info("woke %d snoozed thread(s)", len(woken)) + return len(woken) diff --git a/backend/app/subscriptions.py b/backend/app/subscriptions.py new file mode 100644 index 0000000..6aed045 --- /dev/null +++ b/backend/app/subscriptions.py @@ -0,0 +1,151 @@ +"""Subscriptions: the bulk senders mailing a user, and unsubscribing from them. + +No service records what a user signed up for, so the list is worked out from +received mail, as Gmail does: a message is a subscription's when it carries a +``List-Unsubscribe`` header (required of bulk senders by Gmail and Yahoo since +2024). Messages are grouped by ``List-Id`` when there is one (a company can run +several lists), otherwise by sender address. + +The headers are read when a message is stored (indexer.persist_message); older +mail is read once by :func:`backfill_list_headers_job`. Unsubscribing uses the +sender's one-click POST (RFC 8058) when offered, else mails the ``mailto:`` +address, else hands the web page back for the user to open. +""" + +from __future__ import annotations + +import logging +import re +import uuid +from dataclasses import dataclass +from email.parser import BytesHeaderParser +from email.policy import default as default_policy +from typing import Any +from urllib.parse import parse_qs, unquote, urlsplit + +from sqlalchemy import select, update + +from app.db import session_scope +from app.indexer import rawstore +from app.models import Message + +logger = logging.getLogger(__name__) + +_MAX_HEADER = 2000 +_ANGLE = re.compile(r"<([^<>]+)>") +_BATCH = 200 + + +@dataclass(frozen=True) +class ListHeaders: + list_id: str | None + unsubscribe: str # '' when the message has none + one_click: bool + + +def read_list_headers(raw: bytes) -> ListHeaders: + """The mailing-list headers of a raw message (headers only are parsed).""" + end = raw.find(b"\r\n\r\n") + if end < 0: + end = raw.find(b"\n\n") + head = raw if end < 0 else raw[: end + 2] + try: + msg = BytesHeaderParser(policy=default_policy).parsebytes(head) + list_id_raw = str(msg.get("List-Id") or "") + unsubscribe = str(msg.get("List-Unsubscribe") or "") + post = str(msg.get("List-Unsubscribe-Post") or "") + except Exception: # a malformed header must not stop the message being stored + return ListHeaders(None, "", False) + list_id = None + if list_id_raw: + match = _ANGLE.search(list_id_raw) + list_id = (match.group(1) if match else list_id_raw).strip().lower()[:300] or None + unsubscribe = " ".join(unsubscribe.split())[:_MAX_HEADER] + one_click = "list-unsubscribe=one-click" in post.replace(" ", "").lower() + return ListHeaders(list_id, unsubscribe, one_click and bool(unsubscribe)) + + +@dataclass(frozen=True) +class UnsubscribeTargets: + https: str | None + mailto: str | None # the address + mailto_subject: str + mailto_body: str + + +def unsubscribe_targets(header: str) -> UnsubscribeTargets: + """The https and mailto targets listed in a ``List-Unsubscribe`` header.""" + https = mailto = None + subject, body = "unsubscribe", "unsubscribe" + for target in _ANGLE.findall(header): + target = target.strip() + lowered = target.lower() + if https is None and lowered.startswith("https://"): + https = target + elif mailto is None and lowered.startswith("mailto:"): + parts = urlsplit(target) + address = unquote(parts.path).strip() + if "@" in address and "\n" not in address and "\r" not in address: + mailto = address + query = parse_qs(parts.query) + subject = (query.get("subject") or [subject])[0][:200] + body = (query.get("body") or [body])[0][:2000] + return UnsubscribeTargets(https, mailto, subject, body) + + +def group_key(list_id: str | None, sender: str) -> str: + return f"list:{list_id}" if list_id else f"from:{sender.lower()}" + + +async def pending_count(session: Any, user_id: uuid.UUID) -> int: + from sqlalchemy import func + + return int( + await session.scalar( + select(func.count()) + .select_from(Message) + .where(Message.user_id == user_id, Message.list_unsubscribe.is_(None)) + ) + or 0 + ) + + +async def backfill_list_headers_job(ctx: dict[str, Any], user_id: str) -> int: + """Read the list headers of a user's older mail (stored before this feature). + + Works in batches; a message whose raw copy can't be had is marked as having + none, so it is not tried again. Returns how many messages were read. + """ + uid = uuid.UUID(user_id) + done = 0 + while True: + async with session_scope() as session: + batch = list( + await session.scalars( + select(Message) + .where(Message.user_id == uid, Message.list_unsubscribe.is_(None)) + .limit(_BATCH) + ) + ) + if not batch: + break + for message in batch: + try: + raw = await rawstore.ensure_bytes(message) + headers = read_list_headers(raw) + except Exception: # missing blob, server unreachable: give up on it + headers = ListHeaders(None, "", False) + await session.execute( + update(Message) + .where(Message.id == message.id) + .values( + list_id=headers.list_id, + list_unsubscribe=headers.unsubscribe, + list_unsubscribe_post=headers.one_click, + ) + ) + await session.commit() + done += len(batch) + if done: + logger.info("read list headers of %d message(s) for user %s", done, user_id) + return done diff --git a/backend/app/translation.py b/backend/app/translation.py new file mode 100644 index 0000000..14384aa --- /dev/null +++ b/backend/app/translation.py @@ -0,0 +1,303 @@ +"""Email translation, on this server: LibreTranslate (Argos models) in its own +container, reached only by the API. A message's text never leaves the server. + +- A message's language is detected the first time it is shown (from its own + text: quoted replies and signatures left out) and kept on the message, so the + reading pane can offer "Translate" when it differs from the interface language. +- A translation is of the message HTML the user sees (already sanitized; the + result is sanitized again), cached per message, target language and whether + remote images were shown. + +Off when ``LIBRETRANSLATE_URL`` is empty. +""" + +from __future__ import annotations + +import asyncio +import html as html_lib +import logging +import re +import time +from collections.abc import Iterable +from dataclasses import dataclass + +import httpx + +from app import translation_prep as prep +from app.completion import own_text +from app.config import get_settings + +logger = logging.getLogger(__name__) + +DETECT_CHARS = 1500 # of the message's own text: plenty to tell the language +MIN_CONFIDENCE = 50.0 # below this, LibreTranslate is guessing +MAX_HTML_CHARS = 150_000 # a translation request bigger than this is refused +_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # long messages take a while on a CPU +_LANG_CACHE_S = 600 + + +class TranslationError(Exception): + """Translation is off, the service is unreachable, or it refused the text.""" + + +@dataclass(frozen=True) +class Language: + code: str + name: str + + +def enabled() -> bool: + return bool(get_settings().libretranslate_url.strip()) + + +def _url(path: str) -> str: + return get_settings().libretranslate_url.rstrip("/") + path + + +async def _post(path: str, payload: dict[str, object]) -> object: + if not enabled(): + raise TranslationError("translation is not set up on this server") + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + resp = await client.post(_url(path), json=payload) + except httpx.HTTPError as exc: + raise TranslationError(f"the translation service is unreachable: {exc}") from exc + if resp.status_code != 200: + detail = resp.text[:200] + raise TranslationError(f"the translation service refused: {resp.status_code} {detail}") + return resp.json() + + +_languages: tuple[float, list[Language]] = (0.0, []) + + +async def languages() -> list[Language]: + """What the service can translate between (cached for a few minutes).""" + global _languages + stamp, cached = _languages + if cached and time.monotonic() - stamp < _LANG_CACHE_S: + return cached + if not enabled(): + return [] + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client: + resp = await client.get(_url("/languages")) + resp.raise_for_status() + rows = resp.json() + except (httpx.HTTPError, ValueError) as exc: + logger.info("translation languages unavailable: %s", exc) + return [] + found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r] + _languages = (time.monotonic(), found) + return found + + +_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]") +MIN_LETTERS = 25 # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto) +MAX_DETECT = 60 # pieces of one message asked about +MIN_SHARE = 0.15 # of a message's letters, for its language to count + + +async def _detect_one(text: str) -> str: + result = await _post("/detect", {"q": text[:DETECT_CHARS]}) + if not isinstance(result, list) or not result: + return "" + best = result[0] + if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE: + return "" + return str(best.get("language", "")) + + +def _weight(text: str) -> int: + """How much text this is: letters, a Chinese/Japanese/Korean sign as a word.""" + return sum(3 if _CJK.match(c) else 1 for c in text if c.isalpha()) + + +async def languages_of(parts: list[str]) -> list[str]: + """The language of each of ``parts`` (paragraphs, in order). Parts too + short to tell take the language of the next part that could be told (a + greeting goes with what follows it), the last ones that of the one before; + parts without letters get ''.""" + # Addresses say nothing about a language ("E-mail: lianshaojuan.lsj@…" → Estonian). + parts = [_ADDRESS.sub(" ", part) for part in parts] + told: list[str] = [""] * len(parts) + asked = [ + i + for i, part in enumerate(parts) + if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS + ][:MAX_DETECT] + gate = asyncio.Semaphore(4) + + async def one(i: int) -> None: + async with gate: + told[i] = await _detect_one(parts[i]) + + await asyncio.gather(*(one(i) for i in asked)) + out = list(told) + following = "" + for i in reversed(range(len(parts))): + if told[i]: + following = told[i] + elif any(c.isalpha() for c in parts[i]): + out[i] = following + previous = "" + for i, part in enumerate(parts): + if told[i]: + previous = told[i] + elif not out[i] and any(c.isalpha() for c in part): + out[i] = previous + return out + + +async def detect(text: str) -> str: + """The languages of ``text`` (a message's plain text), most of it first, + comma-separated ("zh-Hans,pl,en"); '' if unsure. A message can be in + several: each language that makes up a good part of it counts.""" + own = own_text(text) + if len(" ".join(own.split())) < 20: + return "" # a line or two says too little + lines: list[str] = [] + total = 0 + for line in (x.strip() for x in own.splitlines()): + if line and total < DETECT_CHARS * 4: + lines.append(line) + total += len(line) + found = await languages_of(lines) + weight: dict[str, int] = {} + for line, language in zip(lines, found, strict=True): + if language: + weight[language] = weight.get(language, 0) + _weight(_ADDRESS.sub(" ", line)) + if not weight: + return await _detect_one(" ".join(own.split())) + all_of_it = sum(weight.values()) + ranked = sorted(weight, key=lambda k: weight[k], reverse=True) + return ",".join(k for k in ranked if weight[k] / all_of_it >= MIN_SHARE) + + +def _base(code: str) -> str: + return code.lower().split("-")[0] + + +async def _translate_paragraphs(html: str, target: str) -> str: + """``html`` in more than one language, in ``target``: each paragraph from + its own language; paragraphs already in ``target`` are left as they are.""" + pieces, paragraphs = prep.split_pieces(html) + found = await languages_of([p.text for p in paragraphs]) + # Chinese and Japanese go without spaces: two pieces of one sentence + # ("…奎琳，我从…") need one between them once translated. + followed = {i for p in paragraphs for i in p.pieces[:-1]} + by_language: dict[str, list[int]] = {} + for paragraph, language in zip(paragraphs, found, strict=True): + if language and _base(language) != _base(target): + by_language.setdefault(language, []).extend(paragraph.pieces) + for language, where in by_language.items(): + todo: list[tuple[int, str, str, str]] = [] + for i in where: + m = _EDGE_SPACE.match(html_lib.unescape(pieces[i])) + lead, core, trail = m.groups() if m else ("", "", "") + if not trail and i in followed and _base(language) in _NO_SPACES: + trail = " " + if any(c.isalnum() for c in core): + todo.append((i, lead, core, trail)) + for start in range(0, len(todo), 100): + batch = todo[start : start + 100] + result = await _post( + "/translate", + {"q": [core for _, _, core, _ in batch], "source": language, "target": target}, + ) + out = result.get("translatedText") if isinstance(result, dict) else None + if not isinstance(out, list) or len(out) != len(batch): + raise TranslationError("the translation service sent no translation") + for (i, lead, _, trail), text in zip(batch, out, strict=True): + pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False) + return "".join(pieces) + + +_NO_SPACES = {"zh", "ja", "th"} +_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL) + + +# Web and mail addresses in the text: the models break them up ("https: / / +# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps. +_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE) +_TAG = re.compile(r"(<[^>]*>)") + + +def protect_addresses(html: str) -> str: + """Mark the addresses in ``html``'s text as not to be translated (style + blocks and tag attributes are left alone).""" + out: list[str] = [] + in_style = False + for piece in _TAG.split(html): + if piece.startswith("<"): + lowered = piece.lower() + if lowered.startswith("{m.group(0)}', piece)) + return "".join(out) + + +_PUNCT = re.compile(r"[\W_]+") +MAX_NAMES = 200 # names checked for one message + + +async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]: + """Of ``names``, those the model changes when it translates them alone + ("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one + request for them all. If the check fails, all of them.""" + wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES] + if not wanted: + return set() + try: + result = await _post( + "/translate", {"q": wanted, "source": source or "auto", "target": target} + ) + except TranslationError: + return set(wanted) + out = result.get("translatedText") if isinstance(result, dict) else None + if not isinstance(out, list) or len(out) != len(wanted): + return set(wanted) + + def same(a: str, b: str) -> bool: + return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold() + + return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))} + + +# Bumped when what goes to the service, or what is done with the result, +# changes: translations kept from before are made again. +PIPELINE_VERSION = 5 + + +async def translate_html( + html: str, source: str, target: str, keep: Iterable[str] = (), names: Iterable[str] = () +) -> str: + """``html`` (a sanitized message body) in ``target``; source "auto", a code, + or several ("zh-Hans,pl,en": then each paragraph from its own language). + ``keep``: words always left as they are; ``names``: left as they are if + the model would change them (translation_prep).""" + if len(html) > MAX_HTML_CHARS: + raise TranslationError("this message is too long to translate") + sources = [x for x in (source or "").split(",") if x and x != "auto"] + prepared = prep.join_wrapped_lines(html) + for language in sources: + prepared = prep.translate_phrases(prepared, language, target) + single = sources[0] if len(sources) == 1 else "auto" + candidates = {*names, *prep.find_names(prepared, single if sources else "")} + terms = {*keep, *await names_to_keep(candidates, single, target)} + prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms))) + if len(sources) > 1: + return prep.tidy(await _translate_paragraphs(prepared, target)) + result = await _post( + "/translate", + {"q": prepared, "source": source or "auto", "target": target, "format": "html"}, + ) + if not isinstance(result, dict) or "translatedText" not in result: + raise TranslationError("the translation service sent no translation") + return prep.tidy(str(result["translatedText"])) diff --git a/backend/app/translation_prep.py b/backend/app/translation_prep.py new file mode 100644 index 0000000..aff7cf1 --- /dev/null +++ b/backend/app/translation_prep.py @@ -0,0 +1,332 @@ +"""Getting a message ready for translation, and tidying what comes back. + +LibreTranslate translates each piece of text between tags on its own, and its +small models stumble on things a reader would not: a sentence wrapped onto two +lines becomes two sentences ("ask you about" / "Plan for next month"), +"Best," becomes "Najlepsze," and a company name gets translated word by word. +So before a message goes out: + +- lines that carry on a sentence (no end punctuation, next starts lowercase) + are joined, in HTML (
, one
per line) and in wrapped plain text; +- greetings and sign-offs that make a whole line are put in the target + language from a small phrase table (English, Polish, German); +- names are marked translate="no", which LibreTranslate keeps — the people + on the message, the user's own names and signature names, and capitalised + multi-word names mid-sentence — but only the names the model would change + ("Milan" → "Mediolan"): a kept name splits its sentence, and LibreTranslate + translates each piece alone ("Dear Lukas Remis," → + "Kochanie."), so a name the model leaves alone stays in the sentence; +- so are emoji, file names ("A_0023C017_p1CAC.mov") and text without a letter; +- a message in more than one language is translated paragraph by paragraph, + each from its own language (split_pieces); those in the target stay; +- and afterwards, the space LibreTranslate drops after a kept word comes back. +""" + +from __future__ import annotations + +import html as html_lib +import re +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field + +KEEP_OPEN = '' +_TAG = re.compile(r"(<[^>]*>)") +_LOWER_START = "a-ząćęłńóśźżäöüßàâçéèêëîïôûùüÿñ" +_UPPER = "A-ZĄĆĘŁŃÓŚŹŻÄÖÜÀÂÇÉÈÊËÎÏÔÛÙÜŸÑ" + +# --- joining wrapped lines ------------------------------------------------------- + +# A line break between a word (or comma) and a lowercase word carries a sentence on. +_HTML_SOFT_BREAK = re.compile( + rf"([\w,;–-])[ \t]*(?:|
\s*]*>|

\s*]*>)\s*(?=[{_LOWER_START}])" +) +_TEXT_SOFT_BREAK = re.compile(rf"([^\s.!?:;])[ \t]*\n[ \t]*(?=[{_LOWER_START}])") + + +def join_wrapped_lines(html: str) -> str: + html = _HTML_SOFT_BREAK.sub(r"\1 ", html) + # Plain text (wm-plaintext): newlines inside text, outside tags. + return "".join( + p if p.startswith("<") else _TEXT_SOFT_BREAK.sub(r"\1 ", p) for p in _TAG.split(html) + ) + + +# --- greetings and sign-offs ----------------------------------------------------- + +# source language → the line as written (lower case) → target language → line. +_PHRASES: dict[str, dict[str, dict[str, str]]] = { + "en": { + "best": {"pl": "Pozdrawiam", "de": "Viele Grüße"}, + "best regards": {"pl": "Pozdrawiam serdecznie", "de": "Beste Grüße"}, + "best wishes": {"pl": "Najlepsze życzenia", "de": "Beste Grüße"}, + "kind regards": {"pl": "Z pozdrowieniami", "de": "Freundliche Grüße"}, + "warm regards": {"pl": "Serdecznie pozdrawiam", "de": "Herzliche Grüße"}, + "regards": {"pl": "Pozdrawiam", "de": "Grüße"}, + "cheers": {"pl": "Pozdrawiam", "de": "Viele Grüße"}, + "sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"}, + "yours sincerely": {"pl": "Z poważaniem", "de": "Mit freundlichen Grüßen"}, + "thanks": {"pl": "Dzięki", "de": "Danke"}, + "many thanks": {"pl": "Wielkie dzięki", "de": "Vielen Dank"}, + "thank you": {"pl": "Dziękuję", "de": "Vielen Dank"}, + "thanks in advance": {"pl": "Z góry dziękuję", "de": "Vielen Dank im Voraus"}, + "all the best": {"pl": "Wszystkiego dobrego", "de": "Alles Gute"}, + "talk soon": {"pl": "Do usłyszenia", "de": "Bis bald"}, + "have a great day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"}, + "have a nice day": {"pl": "Miłego dnia", "de": "Einen schönen Tag noch"}, + }, + "pl": { + "pozdrawiam": {"en": "Best regards", "de": "Viele Grüße"}, + "pozdrawiam serdecznie": {"en": "Warm regards", "de": "Herzliche Grüße"}, + "serdecznie pozdrawiam": {"en": "Warm regards", "de": "Herzliche Grüße"}, + "z poważaniem": {"en": "Kind regards", "de": "Mit freundlichen Grüßen"}, + "z pozdrowieniami": {"en": "Kind regards", "de": "Freundliche Grüße"}, + "dziękuję": {"en": "Thank you", "de": "Vielen Dank"}, + "dzięki": {"en": "Thanks", "de": "Danke"}, + "z góry dziękuję": {"en": "Thanks in advance", "de": "Vielen Dank im Voraus"}, + "miłego dnia": {"en": "Have a nice day", "de": "Einen schönen Tag noch"}, + "do usłyszenia": {"en": "Talk soon", "de": "Bis bald"}, + }, + "de": { + "mit freundlichen grüßen": {"en": "Kind regards", "pl": "Z poważaniem"}, + "freundliche grüße": {"en": "Kind regards", "pl": "Z pozdrowieniami"}, + "viele grüße": {"en": "Best regards", "pl": "Pozdrawiam"}, + "beste grüße": {"en": "Best regards", "pl": "Pozdrawiam"}, + "liebe grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"}, + "herzliche grüße": {"en": "Warm regards", "pl": "Serdecznie pozdrawiam"}, + "danke": {"en": "Thanks", "pl": "Dzięki"}, + "vielen dank": {"en": "Many thanks", "pl": "Bardzo dziękuję"}, + "bis bald": {"en": "Talk soon", "pl": "Do usłyszenia"}, + }, +} +# A whole line: after a tag or line start, before a tag or line end. +_LINE = re.compile(r"(^|>|\n)([ \t]*)([^<>\n]{2,40}?)([ \t]*[,.!]?)([ \t]*)(?=<|\n|$)") + + +def _base(code: str) -> str: + return code.lower().split("-")[0] + + +def translate_phrases(html: str, source: str, target: str) -> str: + table = _PHRASES.get(_base(source), {}) + if not table: + return html + tgt = _base(target) + + def repl(m: re.Match[str]) -> str: + found = table.get(" ".join(m.group(3).lower().split()), {}).get(tgt) + if found is None: + return m.group(0) + return f"{m.group(1)}{m.group(2)}{KEEP_OPEN}{found}{m.group(4).strip()}{m.group(5)}" + + # The pattern never crosses a tag: it stops at "<" and ">". + return _LINE.sub(repl, html) + + +# --- names ------------------------------------------------------------------------ + +# A capitalised word; dots only inside it ("U.S"), never at its end — a name +# does not run on over a sentence's end ("NukeX. My work…"). +_WORD = rf"[{_UPPER}][\w'’&-]*(?:\.[\w'’&-]+)*" +# Two or more capitalised words in a row ("Imaginary Pixels", "Trusted Partner +# Network"), after a lowercase word: a name, not the start of a sentence. +_MULTI_NAME = re.compile( + rf"(?<=[{_LOWER_START},] )({_WORD}(?: (?:&|of|the|de|von|van|{_WORD}))* {_WORD})" +) +# Languages that capitalise every noun: capitals say nothing about names there. +_CAPITALISE_NOUNS = {"de", "lb"} + + +def _title_case(text: str) -> bool: + """A Headline Like This: nearly every longer word capitalised (short ones + — of, and, in — often are not). A sentence full of names is not one.""" + words = [w for w in re.findall(r"[^\W\d_][\w'’-]*", text) if len(w) >= 4] + return len(words) >= 3 and sum(w[0].isupper() for w in words) / len(words) > 0.8 + + +def find_names(html: str, source: str) -> set[str]: + """Capitalised multi-word names in the middle of sentences (not in headings + or Title Case lines, and not for languages that capitalise all nouns).""" + if _base(source) in _CAPITALISE_NOUNS: + return set() + found: set[str] = set() + for piece in _TAG.split(html): + if piece.startswith("<") or _title_case(piece): + continue + found.update(m.group(1).rstrip(".") for m in _MULTI_NAME.finditer(piece)) + return found + + +_SIG_NAME = re.compile(rf"{_WORD}(?: (?:&|of|the|{_WORD}))* {_WORD}") + + +def find_names_in_signature(text: str) -> list[str]: + """Capitalised multi-word names in a signature ("Imaginary Pixels"): the + user's company or title, which a translation should leave alone.""" + return [m.group(0).rstrip(".") for m in _SIG_NAME.finditer(text)] + + +def keep_terms(html: str, terms: Iterable[str]) -> str: + """Mark each of ``terms`` in ``html``'s text translate="no" (longest first; + whole words; style blocks and tags left alone).""" + words = sorted({t.strip() for t in terms if len(t.strip()) >= 2}, key=len, reverse=True) + if not words: + return html + pattern = re.compile(r"(?])(" + "|".join(re.escape(w) for w in words) + r")(?![\w<])") + return _mark(html, pattern) + + +# Emoji and other pictographs (with their skin tones, joiners and variation +# selectors): on its own, an emoji comes back as a made-up word ("😊" → +# "Uzasadnienie"). +_SYMBOLS = re.compile( + "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff" + "\U0001f000-\U0001faff]" + "[\u2190-\u21ff\u2300-\u23ff\u2460-\u27bf\u2900-\u297f\u2b00-\u2bff" + "\U0001f000-\U0001faff\ufe0f\u200d\u20e3]*" +) +# File names and codes with underscores ("A_0023C017_260521_p1CAC.mov", +# "final_v2"): the models split them at every "_" and dot. +_FILE_NAME = re.compile( + r"(? str: + """Mark emoji, file names, handles ("Jacqueline_"), phone numbers and text + without a letter in ``html`` translate="no".""" + html = _mark(_mark(html, _SYMBOLS), _PHONE) + + def is_file(m: re.Match[str]) -> bool: + word = m.group(0) + if "_" in word and word.strip("_"): + return True + return bool(_FILE_EXT.search(word)) and any(c.isdigit() or c in "_-" for c in word) + + return _map_text(_mark(html, _FILE_NAME, keep=is_file), _keep_letterless) + + +_ENTITY = re.compile(r"&#?\w+;") +_EDGES = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL) + + +def _keep_letterless(text: str) -> str: + """A piece of text with no letter or digit — an invisible byte-order mark, + a lone symbol — kept as it is: the models make up a word for it ("\ufeff" + → "Uzasadnienie").""" + if any(c.isalnum() for c in _ENTITY.sub("", text)): + return text + lead, core, trail = _EDGES.match(text).groups() # type: ignore[union-attr] + if not _ENTITY.sub("", core).strip(): + return text # only spaces and   + return f"{lead}{KEEP_OPEN}{core}{trail}" + + +def _mark( + html: str, pattern: re.Pattern[str], keep: Callable[[re.Match[str]], bool] | None = None +) -> str: + """Wrap ``pattern``'s matches in ``html``'s text in translate="no" spans + (where ``keep(match)`` says so).""" + + def repl(m: re.Match[str]) -> str: + if keep is not None and not keep(m): + return m.group(0) + return f"{KEEP_OPEN}{m.group(0)}" + + return _map_text(html, lambda text: pattern.sub(repl, text)) + + +def _map_text(html: str, fn: Callable[[str], str]) -> str: + """``fn`` applied to each piece of ``html``'s text; style blocks, tags and + text already kept are left alone.""" + out: list[str] = [] + in_style = in_keep = False + for piece in _TAG.split(html): + if piece.startswith("<"): + low = piece.lower() + if low.startswith(" tuple[list[str], list[Paragraph]]: + """``html`` as a list of tags and text, and its paragraphs. Text in style + blocks is no paragraph's; text already kept counts for the language but + is not translated.""" + pieces = _TAG.split(html) + paragraphs: list[Paragraph] = [Paragraph()] + in_style = in_keep = False + for i, piece in enumerate(pieces): + if piece.startswith("<"): + low = piece.lower() + if low.startswith("i twoje"). +# Only kept spans: they still carry translate="no" here (the sanitizer that +# runs next takes it off); other spans are the sender's and stay as they are. +_KEPT = re.compile(r'([^<]*)') + + +def tidy(html: str) -> str: + def repl(m: re.Match[str]) -> str: + before = html[m.start() - 1] if m.start() > 0 else "" + after = html[m.end()] if m.end() < len(html) else "" + lead = " " if re.match(r"[\w,;:)]", before) else "" + trail = " " if re.match(r"[\w(„\"“]", after) else "" + return f"{lead}{m.group(1)}{trail}" + + return _KEPT.sub(repl, html) diff --git a/backend/migrations/versions/0028_snooze_templates_subscriptions.py b/backend/migrations/versions/0028_snooze_templates_subscriptions.py new file mode 100644 index 0000000..5a01ee8 --- /dev/null +++ b/backend/migrations/versions/0028_snooze_templates_subscriptions.py @@ -0,0 +1,108 @@ +"""snooze, templates, subscriptions, tracker counts + +- Snooze: a thread can leave the Inbox until a set time (``snoozed_until``); + ``snoozed_at`` lets a reply that arrives meanwhile wake it early, and + ``unsnoozed_at`` brings a woken thread to the top of the Inbox. +- Templates: saved replies, per user, inserted from the compose window. +- Subscriptions: the mailing-list headers of each message (``List-Id``, + ``List-Unsubscribe``, one-click support), read during sync. NULL + ``list_unsubscribe`` means "not read yet" (filled in by a backfill job), '' + means the message has none. ``subscription_states`` records what the user did + about a sender (unsubscribed when). +- Tracker blocking: how many tracking pixels were removed from a message's HTML. + +Revision ID: 0028 +Revises: 0027 +Create Date: 2026-10-18 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0028" +down_revision: str | None = "0027" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE threads + ADD COLUMN snoozed_until timestamptz, + ADD COLUMN snoozed_at timestamptz, + ADD COLUMN unsnoozed_at timestamptz + """ + ) + op.execute( + "CREATE INDEX ix_threads_snoozed ON threads (snoozed_until) WHERE snoozed_until IS NOT NULL" + ) + op.execute( + """ + ALTER TABLE messages + ADD COLUMN list_id text, + ADD COLUMN list_unsubscribe text, + ADD COLUMN list_unsubscribe_post boolean NOT NULL DEFAULT false + """ + ) + op.execute( + "CREATE INDEX ix_messages_user_list ON messages (user_id) " + "WHERE list_unsubscribe IS NOT NULL AND list_unsubscribe <> ''" + ) + op.execute( + "CREATE INDEX ix_messages_list_unread ON messages (user_id) WHERE list_unsubscribe IS NULL" + ) + op.execute( + """ + CREATE TABLE subscription_states ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + key text NOT NULL, + unsubscribed_at timestamptz NOT NULL DEFAULT now(), + method text NOT NULL, + PRIMARY KEY (user_id, key) + ) + """ + ) + op.execute( + """ + CREATE TABLE mail_templates ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + name text NOT NULL, + subject text NOT NULL DEFAULT '', + body_html text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + op.execute("CREATE INDEX ix_mail_templates_user ON mail_templates (user_id)") + op.execute("ALTER TABLE body_parts ADD COLUMN trackers_blocked integer NOT NULL DEFAULT 0") + + +def downgrade() -> None: + op.execute("ALTER TABLE body_parts DROP COLUMN IF EXISTS trackers_blocked") + op.execute("DROP TABLE IF EXISTS mail_templates") + op.execute("DROP TABLE IF EXISTS subscription_states") + op.execute("DROP INDEX IF EXISTS ix_messages_list_unread") + op.execute("DROP INDEX IF EXISTS ix_messages_user_list") + op.execute( + """ + ALTER TABLE messages + DROP COLUMN IF EXISTS list_id, + DROP COLUMN IF EXISTS list_unsubscribe, + DROP COLUMN IF EXISTS list_unsubscribe_post + """ + ) + op.execute("DROP INDEX IF EXISTS ix_threads_snoozed") + op.execute( + """ + ALTER TABLE threads + DROP COLUMN IF EXISTS snoozed_until, + DROP COLUMN IF EXISTS snoozed_at, + DROP COLUMN IF EXISTS unsnoozed_at + """ + ) diff --git a/backend/migrations/versions/0029_completion.py b/backend/migrations/versions/0029_completion.py new file mode 100644 index 0000000..1a96994 --- /dev/null +++ b/backend/migrations/versions/0029_completion.py @@ -0,0 +1,49 @@ +"""sentence completion: a per-user index of phrases from their sent mail + +``completion_phrases`` maps the words before the cursor (normalized) to what +the user usually writes next, with how often; rebuilt from the sent mail at +most daily (app/completion.py). The primary key leads with (user_id, context), +which is the lookup. + +Revision ID: 0029 +Revises: 0028 +Create Date: 2026-10-18 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0029" +down_revision: str | None = "0028" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE completion_phrases ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + context text NOT NULL, + continuation text NOT NULL, + count integer NOT NULL, + PRIMARY KEY (user_id, context, continuation) + ) + """ + ) + op.execute( + """ + CREATE TABLE completion_index_state ( + user_id uuid PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + built_at timestamptz NOT NULL + ) + """ + ) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS completion_index_state") + op.execute("DROP TABLE IF EXISTS completion_phrases") diff --git a/backend/migrations/versions/0030_translation.py b/backend/migrations/versions/0030_translation.py new file mode 100644 index 0000000..aab3f57 --- /dev/null +++ b/backend/migrations/versions/0030_translation.py @@ -0,0 +1,45 @@ +"""email translation: a message's detected language, and translations kept + +``messages.content_language``: NULL until the message is first shown, '' when +the language could not be told. ``message_translations``: one translated HTML +per message, target language and remote-images choice (app/translation.py); +dropped with the message. + +Revision ID: 0030 +Revises: 0029 +Create Date: 2026-10-18 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0030" +down_revision: str | None = "0029" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("ALTER TABLE messages ADD COLUMN content_language text") + op.execute( + """ + CREATE TABLE message_translations ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + target text NOT NULL, + images boolean NOT NULL, + source text NOT NULL, + html text NOT NULL, + sanitizer_v integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (message_id, target, images) + ) + """ + ) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS message_translations") + op.execute("ALTER TABLE messages DROP COLUMN IF EXISTS content_language") diff --git a/backend/tests/api/conftest.py b/backend/tests/api/conftest.py new file mode 100644 index 0000000..86fc639 --- /dev/null +++ b/backend/tests/api/conftest.py @@ -0,0 +1,149 @@ +"""A signed-in user with an API client, for endpoint tests in this package.""" + +from __future__ import annotations + +import datetime as dt +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from email.message import EmailMessage + +import httpx +import pytest_asyncio +from httpx import ASGITransport +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool + +from app.config import get_settings +from app.identity.passwords import hash_password +from app.identity.sessions import create_session +from app.indexer.indexer import persist_message +from app.main import create_app +from app.models import Account, Mailbox, Message, User +from app.providers.base import RemoteMessage + +CSRF = "csrf" +HDR = {"X-CSRF": CSRF} + + +@dataclass +class UserCtx: + client: httpx.AsyncClient + user_id: uuid.UUID + session: AsyncSession # for arranging rows directly + + +@pytest_asyncio.fixture +async def user_ctx() -> AsyncIterator[UserCtx]: + engine = create_async_engine(get_settings().database_url, poolclass=NullPool) + sm = async_sessionmaker(engine, expire_on_commit=False) + session = sm() + user = User(handle=f"t{uuid.uuid4().hex[:10]}@test.local", password_hash=hash_password("pw")) + session.add(user) + await session.commit() + user_id = user.id # kept: a test may expire the session's objects + token = await create_session(user_id, "pytest") + client = httpx.AsyncClient( + transport=ASGITransport(app=create_app()), + base_url="http://testserver", + cookies={"session": token, "csrf": CSRF}, + ) + try: + yield UserCtx(client=client, user_id=user_id, session=session) + finally: + await client.aclose() + await session.rollback() + await session.execute(delete(User).where(User.id == user_id)) + await session.commit() + await session.close() + await engine.dispose() + + +@dataclass +class MailCtx(UserCtx): + account_id: uuid.UUID + inbox_id: uuid.UUID + sent_id: uuid.UUID + + async def deliver( + self, + raw: bytes, + uid: int, + *, + seen: bool = False, + at: dt.datetime | None = None, + folder: str = "inbox", + ) -> Message: + """Store a message the way sync does (indexer.persist_message), in the + Inbox or (``folder="sent"``) the Sent folder.""" + remote = RemoteMessage( + uid=uid, + raw=raw, + flags=frozenset({"\\Seen"}) if seen else frozenset(), + internal_date=at or dt.datetime.now(dt.UTC), + ) + result = await persist_message( + self.session, + user_id=self.user_id, + account_id=self.account_id, + mailbox_id=self.sent_id if folder == "sent" else self.inbox_id, + mailbox_special_use=folder, + remote=remote, + ) + await self.session.commit() + assert result is not None + return result.message + + +def raw_mail( + subject: str, + *, + sender: str = "News ", + headers: dict[str, str] | None = None, + mid: str | None = None, + html: str | None = None, +) -> bytes: + m = EmailMessage() + m["From"] = sender + m["To"] = "me@example.com" + m["Subject"] = subject + m["Message-ID"] = mid or f"<{uuid.uuid4().hex}@shop.example>" + for name, value in (headers or {}).items(): + m[name] = value + m.set_content(f"{subject} body") + if html is not None: + m.add_alternative(html, subtype="html") + return m.as_bytes() + + +@pytest_asyncio.fixture +async def mail_ctx(user_ctx: UserCtx) -> MailCtx: + s = user_ctx.session + account = Account( + user_id=user_ctx.user_id, + email="me@example.com", + imap_config={}, + smtp_config={}, + secret_ct=b"", + discovery_src="manual", + ) + s.add(account) + await s.flush() + inbox = Mailbox( + account_id=account.id, user_id=user_ctx.user_id, path="INBOX", special_use="inbox" + ) + trash = Mailbox( + account_id=account.id, user_id=user_ctx.user_id, path="Trash", special_use="trash" + ) + sent = Mailbox(account_id=account.id, user_id=user_ctx.user_id, path="Sent", special_use="sent") + s.add_all([inbox, trash, sent]) + await s.commit() + return MailCtx( + client=user_ctx.client, + user_id=user_ctx.user_id, + session=s, + account_id=account.id, + inbox_id=inbox.id, + sent_id=sent.id, + ) diff --git a/backend/tests/api/test_api.py b/backend/tests/api/test_api.py index c3ed44c..ae04424 100644 --- a/backend/tests/api/test_api.py +++ b/backend/tests/api/test_api.py @@ -391,7 +391,7 @@ async def test_folder_counts(api: ApiCtx) -> None: # All three seeded inbox messages are unread; no drafts yet. resp = await api.client.get("/api/mailboxes/counts") assert resp.status_code == 200 - assert resp.json() == {"inbox_unread": 3, "drafts": 0} + assert resp.json() == {"inbox_unread": 3, "drafts": 0, "snoozed": 0} # Marking one message \Seen lowers the unread count. msg = await api.session.get(Message, api.message_ids[0]) diff --git a/backend/tests/api/test_completion_api.py b/backend/tests/api/test_completion_api.py new file mode 100644 index 0000000..babad3d --- /dev/null +++ b/backend/tests/api/test_completion_api.py @@ -0,0 +1,41 @@ +"""Sentence completion over the API, from the user's sent mail.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from app.api import completion as api_completion +from app.completion import build_index_job +from tests.api.conftest import HDR, MailCtx, raw_mail + + +async def _suggest(ctx: MailCtx, before: str) -> str | None: + resp = await ctx.client.post("/api/completion", json={"before": before}, headers=HDR) + assert resp.status_code == 200, resp.text + return resp.json()["suggestion"] + + +async def test_suggests_from_sent_mail(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None: + queued: list[tuple[Any, ...]] = [] + + async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None: + queued.append((name, *args)) + + monkeypatch.setattr(api_completion, "enqueue_job", fake_enqueue) + body = "Hi Anna,\nThe renders will be ready by Thursday evening.\n\nOn Mon Anna wrote:\n> ok" + for uid in (1, 2): + await mail_ctx.deliver( + raw_mail("Renders", sender="me@example.com").replace(b"Renders body", body.encode()), + uid, + folder="sent", + ) + + # No index yet: a build is asked for, and the starter phrases still answer. + assert await _suggest(mail_ctx, "I look forward ") == "to hearing from you." + assert queued and queued[0][0] == "build_index_job" + + assert await build_index_job({}, str(mail_ctx.user_id)) > 0 + assert await _suggest(mail_ctx, "The renders will be re") == "ady by Thursday evening." + assert await _suggest(mail_ctx, "> ok") is None # quoted text was not learned diff --git a/backend/tests/api/test_snooze.py b/backend/tests/api/test_snooze.py new file mode 100644 index 0000000..d940abb --- /dev/null +++ b/backend/tests/api/test_snooze.py @@ -0,0 +1,80 @@ +"""Snooze: a conversation leaves the Inbox until a set time, then comes back.""" + +from __future__ import annotations + +import datetime as dt + +from sqlalchemy import select + +from app.models import Message, Thread +from app.snooze import wake_due_job +from tests.api.conftest import HDR, MailCtx, raw_mail + + +def _in(days: float = 1) -> str: + return (dt.datetime.now(dt.UTC) + dt.timedelta(days=days)).isoformat() + + +async def _inbox_ids(ctx: MailCtx) -> list[str]: + resp = await ctx.client.get("/api/threads", params={"view": "folder:inbox"}) + return [t["id"] for t in resp.json()["threads"]] + + +async def test_snoozed_thread_leaves_the_inbox(mail_ctx: MailCtx) -> None: + old = await mail_ctx.deliver( + raw_mail("Old"), 1, at=dt.datetime.now(dt.UTC) - dt.timedelta(days=2) + ) + new = await mail_ctx.deliver(raw_mail("New"), 2) + c = mail_ctx.client + resp = await c.post(f"/api/threads/{new.thread_id}/snooze", json={"until": _in()}, headers=HDR) + assert resp.status_code == 200, resp.text + + assert await _inbox_ids(mail_ctx) == [str(old.thread_id)] + snoozed = (await c.get("/api/threads", params={"view": "snoozed"})).json()["threads"] + assert [t["id"] for t in snoozed] == [str(new.thread_id)] + assert snoozed[0]["snoozed_until"] is not None + counts = (await c.get("/api/mailboxes/counts")).json() + assert counts["snoozed"] == 1 and counts["inbox_unread"] == 1 + + # Unsnooze: straight back, at the top. + await c.post(f"/api/threads/{new.thread_id}/unsnooze", headers=HDR) + assert await _inbox_ids(mail_ctx) == [str(new.thread_id), str(old.thread_id)] + + +async def test_snooze_needs_a_future_time(mail_ctx: MailCtx) -> None: + msg = await mail_ctx.deliver(raw_mail("x"), 1) + for until in (_in(-1), _in(400), "2030-01-01T09:00:00"): + resp = await mail_ctx.client.post( + f"/api/threads/{msg.thread_id}/snooze", json={"until": until}, headers=HDR + ) + assert resp.status_code == 400, until + + +async def test_due_thread_wakes_on_top_and_unread(mail_ctx: MailCtx) -> None: + week_ago = dt.datetime.now(dt.UTC) - dt.timedelta(days=7) + snoozed = await mail_ctx.deliver(raw_mail("Snoozed"), 1, seen=True, at=week_ago) + newer = await mail_ctx.deliver(raw_mail("Newer"), 2, seen=True) + thread = await mail_ctx.session.get(Thread, snoozed.thread_id) + assert thread is not None + thread.snoozed_until = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=1) + thread.snoozed_at = week_ago + await mail_ctx.session.commit() + + assert await wake_due_job({}) >= 1 + assert await _inbox_ids(mail_ctx) == [str(snoozed.thread_id), str(newer.thread_id)] + snoozed_id = snoozed.id + mail_ctx.session.expire_all() + flags = await mail_ctx.session.scalar(select(Message.flags).where(Message.id == snoozed_id)) + assert flags is not None and "\\Seen" not in flags + + +async def test_a_reply_wakes_a_snoozed_thread(mail_ctx: MailCtx) -> None: + first = await mail_ctx.deliver(raw_mail("Question", mid=""), 1) + await mail_ctx.client.post( + f"/api/threads/{first.thread_id}/snooze", json={"until": _in(3)}, headers=HDR + ) + m = raw_mail("Re: Question", headers={"In-Reply-To": "", "References": ""}) + await mail_ctx.deliver(m, 2, at=dt.datetime.now(dt.UTC) + dt.timedelta(seconds=5)) + + await wake_due_job({}) + assert await _inbox_ids(mail_ctx) == [str(first.thread_id)] diff --git a/backend/tests/api/test_subscriptions.py b/backend/tests/api/test_subscriptions.py new file mode 100644 index 0000000..3a5e693 --- /dev/null +++ b/backend/tests/api/test_subscriptions.py @@ -0,0 +1,239 @@ +"""Subscriptions: bulk senders worked out from List-* headers, and unsubscribing.""" + +from __future__ import annotations + +import datetime as dt +from typing import Any + +import pytest +from sqlalchemy import select, update + +from app.api import subscriptions as api_subs +from app.models import Draft, Mailbox, Message +from app.subscriptions import backfill_list_headers_job, read_list_headers, unsubscribe_targets +from tests.api.conftest import HDR, MailCtx, raw_mail + +NEWS = { + "List-Id": '"Weekly news" ', + "List-Unsubscribe": ", ", + "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", +} + + +def test_read_list_headers() -> None: + h = read_list_headers(raw_mail("x", headers=NEWS)) + assert h.list_id == "weekly.shop.example" + assert h.unsubscribe.startswith("") + assert h.one_click is True + plain = read_list_headers(raw_mail("hi")) + assert (plain.list_id, plain.unsubscribe, plain.one_click) == (None, "", False) + + +def test_unsubscribe_targets() -> None: + t = unsubscribe_targets(NEWS["List-Unsubscribe"]) + assert t.https == "https://shop.example/u/1" + assert t.mailto == "leave@shop.example" and t.mailto_subject == "stop" + assert unsubscribe_targets("").https is None + assert unsubscribe_targets("").mailto is None + + +async def _list(ctx: MailCtx) -> list[dict[str, Any]]: + resp = await ctx.client.get("/api/subscriptions") + assert resp.status_code == 200, resp.text + return list(resp.json()["subscriptions"]) + + +async def test_groups_by_list_then_sender(mail_ctx: MailCtx) -> None: + await mail_ctx.deliver(raw_mail("Issue 1", headers=NEWS), 1, seen=True) + await mail_ctx.deliver(raw_mail("Issue 2", headers=NEWS), 2) + await mail_ctx.deliver( + raw_mail( + "Sale", + sender="Deals ", + headers={"List-Unsubscribe": ""}, + ), + 3, + ) + await mail_ctx.deliver(raw_mail("Hi from a friend", sender="ann@friend.example"), 4) + + subs = {s["key"]: s for s in await _list(mail_ctx)} + assert set(subs) == {"list:weekly.shop.example", "from:deals@other.example"} + news = subs["list:weekly.shop.example"] + assert (news["total"], news["unread"], news["name"]) == (2, 1, "News") + assert news["one_click"] and news["has_mailto"] and news["has_web"] + assert news["last_subject"] == "Issue 2" + threads = await mail_ctx.client.get( + "/api/subscriptions/threads", params={"key": "list:weekly.shop.example"} + ) + assert len(threads.json()) == 2 + + +async def test_one_click_unsubscribe(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None: + await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1) + posted: list[str] = [] + + async def fake_one_click(url: str) -> bool: + posted.append(url) + return True + + monkeypatch.setattr(api_subs, "_one_click", fake_one_click) + resp = await mail_ctx.client.post( + "/api/subscriptions/unsubscribe", json={"key": "list:weekly.shop.example"}, headers=HDR + ) + assert resp.json() == {"status": "done", "method": "one_click", "url": None} + assert posted == ["https://shop.example/u/1"] + (news,) = await _list(mail_ctx) + assert news["unsubscribed_at"] is not None and news["still_sending"] is False + + +async def test_mailto_unsubscribe_and_still_sending( + mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch +) -> None: + enqueued: list[tuple[Any, ...]] = [] + + async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None: + enqueued.append((name, *args)) + + monkeypatch.setattr(api_subs, "enqueue_job", fake_enqueue) + headers = {"List-Unsubscribe": ""} + await mail_ctx.deliver(raw_mail("Issue", headers=headers), 1) + resp = await mail_ctx.client.post( + "/api/subscriptions/unsubscribe", json={"key": "from:news@shop.example"}, headers=HDR + ) + assert resp.json()["method"] == "mailto" + assert [e[0] for e in enqueued] == ["send_job"] + draft = await mail_ctx.session.scalar(select(Draft).where(Draft.user_id == mail_ctx.user_id)) + assert draft is not None + assert draft.payload["to"] == ["leave@shop.example"] and draft.payload["subject"] == "stop" + + # Mail still arriving after the grace period is flagged. + later = dt.datetime.now(dt.UTC) + dt.timedelta(days=3) + await mail_ctx.deliver(raw_mail("Issue again", headers=headers), 2, at=later) + (sub,) = await _list(mail_ctx) + assert sub["still_sending"] is True + + +async def test_web_only_unsubscribe_hands_back_the_page(mail_ctx: MailCtx) -> None: + headers = {"List-Unsubscribe": ""} + await mail_ctx.deliver(raw_mail("Issue", headers=headers), 1) + resp = await mail_ctx.client.post( + "/api/subscriptions/unsubscribe", json={"key": "from:news@shop.example"}, headers=HDR + ) + assert resp.json() == {"status": "open", "method": "web", "url": "https://shop.example/prefs"} + + +async def test_backfill_reads_older_mail(mail_ctx: MailCtx) -> None: + msg = await mail_ctx.deliver(raw_mail("Old issue", headers=NEWS), 1) + await mail_ctx.session.execute( + update(Message).where(Message.id == msg.id).values(list_unsubscribe=None, list_id=None) + ) + await mail_ctx.session.commit() + resp = await mail_ctx.client.get("/api/subscriptions") + assert resp.json()["pending"] == 1 + + assert await backfill_list_headers_job({}, str(mail_ctx.user_id)) == 1 + resp = await mail_ctx.client.get("/api/subscriptions") + assert resp.json()["pending"] == 0 + assert [s["key"] for s in resp.json()["subscriptions"]] == ["list:weekly.shop.example"] + + +async def test_trash_all_leaves_mixed_conversations( + mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch +) -> None: + moved: list[str] = [] + + async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None: + moved.append(args[0]) + + monkeypatch.setattr(api_subs, "enqueue_job", fake_enqueue) + only = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS, mid=""), 1) + await mail_ctx.deliver(raw_mail("Issue 2", headers=NEWS, mid=""), 2) + reply = raw_mail( + "Re: Issue 2", + sender="me@example.com", + headers={"In-Reply-To": "", "References": ""}, + ) + await mail_ctx.deliver(reply, 3) + resp = await mail_ctx.client.post( + "/api/subscriptions/trash", json={"key": "list:weekly.shop.example"}, headers=HDR + ) + assert resp.json() == {"moved": 1} + assert moved == [str(only.thread_id)] + + +async def test_deleted_mail_is_left_out(mail_ctx: MailCtx) -> None: + msg = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1) + trash = await mail_ctx.session.scalar( + select(Mailbox.id).where( + Mailbox.account_id == mail_ctx.account_id, Mailbox.special_use == "trash" + ) + ) + await mail_ctx.session.execute( + update(Message).where(Message.id == msg.id).values(mailbox_id=trash) + ) + await mail_ctx.session.commit() + assert await _list(mail_ctx) == [] + + +async def test_reading_pane_offers_unsubscribe( + mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch +) -> None: + async def fake_one_click(url: str) -> bool: + return True + + monkeypatch.setattr(api_subs, "_one_click", fake_one_click) + news = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1) + plain = await mail_ctx.deliver(raw_mail("Hi", sender="ann@friend.example"), 2) + c = mail_ctx.client + + (env,) = (await c.get(f"/api/threads/{news.thread_id}")).json()["messages"] + assert env["subscription_key"] == "list:weekly.shop.example" + assert env["unsubscribe_via"] == "one_click" and env["unsubscribed"] is False + (other,) = (await c.get(f"/api/threads/{plain.thread_id}")).json()["messages"] + assert other["subscription_key"] is None and other["unsubscribe_via"] is None + + resp = await c.post( + "/api/subscriptions/unsubscribe", + json={"key": env["subscription_key"], "message_id": env["id"]}, + headers=HDR, + ) + assert resp.json()["status"] == "done" + (env,) = (await c.get(f"/api/threads/{news.thread_id}")).json()["messages"] + assert env["unsubscribed"] is True + + # A message without the header can't be used. + resp = await c.post( + "/api/subscriptions/unsubscribe", + json={"key": "from:ann@friend.example", "message_id": str(plain.id)}, + headers=HDR, + ) + assert resp.status_code == 404 + + +async def test_senders_come_in_pages_newest_first(mail_ctx: MailCtx) -> None: + now = dt.datetime.now(dt.UTC) + for i, name in enumerate(["old", "mid", "new"]): + await mail_ctx.deliver( + raw_mail( + f"From {name}", + sender=f"{name}@lists.example", + headers={"List-Unsubscribe": f""}, + ), + i + 1, + at=now - dt.timedelta(days=10 - i), + ) + c = mail_ctx.client + first = (await c.get("/api/subscriptions", params={"limit": 2})).json() + assert first["total"] == 3 + assert [s["email"] for s in first["subscriptions"]] == [ + "new@lists.example", + "mid@lists.example", + ] + rest = (await c.get("/api/subscriptions", params={"limit": 2, "offset": 2})).json() + assert [s["email"] for s in rest["subscriptions"]] == ["old@lists.example"] + assert (await c.get("/api/subscriptions", params={"offset": 9})).json()["subscriptions"] == [] + oldest = (await c.get("/api/subscriptions", params={"limit": 2, "order": "asc"})).json() + assert [s["email"] for s in oldest["subscriptions"]] == [ + "old@lists.example", + "mid@lists.example", + ] diff --git a/backend/tests/api/test_templates.py b/backend/tests/api/test_templates.py new file mode 100644 index 0000000..3aa2247 --- /dev/null +++ b/backend/tests/api/test_templates.py @@ -0,0 +1,54 @@ +"""Templates: saved replies from the compose window's More menu.""" + +from __future__ import annotations + +from tests.api.conftest import HDR, UserCtx + + +async def test_save_list_replace_delete(user_ctx: UserCtx) -> None: + c = user_ctx.client + assert (await c.get("/api/templates")).json() == [] + + resp = await c.post( + "/api/templates", + json={"name": " Thanks ", "subject": "Re: offer", "body_html": "

Thank you

"}, + headers=HDR, + ) + assert resp.status_code == 200, resp.text + first = resp.json() + assert first["name"] == "Thanks" and first["subject"] == "Re: offer" + + # Same name, any case: replaces rather than adding a second one. + resp = await c.post( + "/api/templates", json={"name": "thanks", "body_html": "

Cheers

"}, headers=HDR + ) + assert resp.json()["id"] == first["id"] + await c.post("/api/templates", json={"name": "Apology", "body_html": "x"}, headers=HDR) + listed = (await c.get("/api/templates")).json() + assert [t["name"] for t in listed] == ["Apology", "thanks"] + assert listed[1]["body_html"] == "

Cheers

" + + assert (await c.delete(f"/api/templates/{first['id']}", headers=HDR)).status_code == 200 + assert [t["name"] for t in (await c.get("/api/templates")).json()] == ["Apology"] + + +async def test_template_html_is_cleaned(user_ctx: UserCtx) -> None: + resp = await user_ctx.client.post( + "/api/templates", + json={ + "name": "x", + "body_html": '

Hi' + 'l

', + }, + headers=HDR, + ) + html = resp.json()["body_html"] + assert "script" not in html and "onclick" not in html and "onerror" not in html + assert "javascript:" not in html and "Hi" in html + + +async def test_template_needs_a_name(user_ctx: UserCtx) -> None: + resp = await user_ctx.client.post( + "/api/templates", json={"name": " ", "body_html": "x"}, headers=HDR + ) + assert resp.status_code == 400 diff --git a/backend/tests/api/test_trackers.py b/backend/tests/api/test_trackers.py new file mode 100644 index 0000000..1e05c59 --- /dev/null +++ b/backend/tests/api/test_trackers.py @@ -0,0 +1,45 @@ +"""Tracker blocking (Settings → Privacy): on by default, and it can be switched off.""" + +from __future__ import annotations + +from tests.api.conftest import HDR, MailCtx, raw_mail + +PIXEL = "https://t.example.com/open.gif?u=42" +HTML = f'

Hello

' + + +async def _html(ctx: MailCtx, message_id: object, images: bool = False) -> tuple[str, str]: + resp = await ctx.client.get( + f"/api/messages/{message_id}/html", params={"images": "1"} if images else {} + ) + assert resp.status_code == 200 + return resp.text, resp.headers["X-Trackers-Blocked"] + + +async def test_trackers_blocked_by_default(mail_ctx: MailCtx) -> None: + msg = await mail_ctx.deliver(raw_mail("News", html=HTML), 1) + assert (await mail_ctx.client.get("/api/settings")).json()["block_trackers"] is True + body, count = await _html(mail_ctx, msg.id, images=True) + assert count == "1" and PIXEL not in body and "Hello" in body + (env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"] + assert env["has_remote_images"] is False # nothing left worth loading + + +async def test_switched_off_trackers_are_ordinary_remote_images(mail_ctx: MailCtx) -> None: + msg = await mail_ctx.deliver(raw_mail("News", html=HTML), 1) + resp = await mail_ctx.client.patch("/api/settings", json={"block_trackers": False}, headers=HDR) + assert resp.json()["block_trackers"] is False + + # Still a remote image: not loaded until images are allowed, but offered. + body, count = await _html(mail_ctx, msg.id) + assert count == "0" and PIXEL not in body + (env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"] + assert env["has_remote_images"] is True + + body, count = await _html(mail_ctx, msg.id, images=True) + assert count == "0" and PIXEL in body.replace("&", "&") + + # Back on: gone again. + await mail_ctx.client.patch("/api/settings", json={"block_trackers": True}, headers=HDR) + body, count = await _html(mail_ctx, msg.id, images=True) + assert count == "1" and PIXEL not in body diff --git a/backend/tests/api/test_translation.py b/backend/tests/api/test_translation.py new file mode 100644 index 0000000..1795341 --- /dev/null +++ b/backend/tests/api/test_translation.py @@ -0,0 +1,86 @@ +"""Email translation over the API, with LibreTranslate replaced by a stand-in.""" + +from __future__ import annotations + +import pytest + +from app import translation +from tests.api.conftest import MailCtx, raw_mail + +TEXT = "Hello Lukas, I would like to ask about the project schedule for next month." + + +@pytest.fixture +def fake_service(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: + calls: dict[str, object] = {"detect": 0, "translate": 0} + + async def detect(text: str) -> str: + calls["detect"] = int(calls["detect"]) + 1 # type: ignore[call-overload] + return "en" + + async def translate_html( + html: str, source: str, target: str, keep: object = (), names: object = () + ) -> str: + calls["translate"] = int(calls["translate"]) + 1 # type: ignore[call-overload] + calls["keep"] = list(keep) # type: ignore[call-overload] + calls["names"] = list(names) # type: ignore[call-overload] + assert source == "en" and target == "pl" + return "

Cześć Łukasz

" + + monkeypatch.setattr(translation, "enabled", lambda: True) + monkeypatch.setattr(translation, "detect", detect) + monkeypatch.setattr(translation, "translate_html", translate_html) + return calls + + +async def test_language_is_found_once_and_kept( + mail_ctx: MailCtx, fake_service: dict[str, object] +) -> None: + msg = await mail_ctx.deliver(raw_mail("Project", html=f"

{TEXT}

"), 1) + for _ in range(2): + resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html") + assert resp.headers["X-Content-Language"] == "en" + assert fake_service["detect"] == 1 + (env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"] + assert env["content_language"] == "en" + + +async def test_translation_is_cleaned_and_kept( + mail_ctx: MailCtx, fake_service: dict[str, object] +) -> None: + msg = await mail_ctx.deliver(raw_mail("Project", html=f"

{TEXT}

"), 1) + for _ in range(2): + resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html", params={"translate": "pl"}) + assert resp.status_code == 200 + assert "Cześć Łukasz" in resp.text and "alert(1)" not in resp.text + assert resp.headers["X-Translated-From"] == "en" + assert resp.headers["X-Translated-To"] == "pl" + assert fake_service["translate"] == 1 # the second came from the cache + # The people on the message are names to keep if the model would change them. + assert "News" in fake_service["names"] # type: ignore[operator] + + +async def test_service_trouble_is_a_clear_error( + mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch, fake_service: dict[str, object] +) -> None: + async def broken( + html: str, source: str, target: str, keep: object = (), names: object = () + ) -> str: + raise translation.TranslationError("the translation service is unreachable") + + monkeypatch.setattr(translation, "translate_html", broken) + msg = await mail_ctx.deliver(raw_mail("Project", html=f"

{TEXT}

"), 1) + resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html", params={"translate": "pl"}) + assert resp.status_code == 503 + assert resp.json()["error"]["code"] == "translation_failed" + # The message itself still shows. + assert (await mail_ctx.client.get(f"/api/messages/{msg.id}/html")).status_code == 200 + + +async def test_off_without_a_service(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(translation, "enabled", lambda: False) + msg = await mail_ctx.deliver(raw_mail("Project", html=f"

{TEXT}

"), 1) + resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html") + assert "X-Content-Language" not in resp.headers + info = (await mail_ctx.client.get("/api/translation/languages")).json() + assert info == {"available": False, "languages": []} diff --git a/backend/tests/indexer/test_sanitize.py b/backend/tests/indexer/test_sanitize.py index 9b9960a..fb7978d 100644 --- a/backend/tests/indexer/test_sanitize.py +++ b/backend/tests/indexer/test_sanitize.py @@ -4,6 +4,8 @@ from __future__ import annotations import uuid +import pytest + from app.indexer.sanitize import sanitize_html MID = uuid.uuid4() @@ -99,3 +101,72 @@ def test_script_content_still_stripped_with_style_allowed() -> None: result = sanitize_html(html, MID) assert "alert(1)" not in result.html assert ".a{color:red}" in result.html + + +# --- Tracking pixels --------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tag", + [ + '', + '', + "", + '', + '', + '', + '', + # Squeezed to a pixel by max-width, its height following. + '', + '', + ], +) +def test_tracking_pixels_are_removed_and_counted(tag: str) -> None: + html = f"

Hello

{tag}

Bye

" + for allow in (False, True): + result = sanitize_html(html, MID, allow_remote_images=allow) + assert result.trackers_blocked == 1 + assert "', + '', + '', + '', + # Outlook signature logos: sizes in inches, not pixels. + '', + '', + '', + '', + # Attributes say 1x1, the style draws it full width. + '', + ], +) +def test_real_images_are_kept(tag: str) -> None: + result = sanitize_html(f"

x

{tag}", MID, allow_remote_images=True) + assert result.trackers_blocked == 0 + assert " None: + tag = f'' + result = sanitize_html(f"

x

{tag}", MID, allow_remote_images=True) + assert result.trackers_blocked == 0 + css = f'' + assert sanitize_html(css, MID, allow_remote_images=True).trackers_blocked == 0 + + +def test_tracker_blocking_can_be_switched_off() -> None: + tag = '' + kept = sanitize_html(f"

x

{tag}", MID, allow_remote_images=True, block_trackers=False) + assert kept.trackers_blocked == 0 and "t.example.com/o.gif" in kept.html diff --git a/backend/tests/test_completion_phrases.py b/backend/tests/test_completion_phrases.py new file mode 100644 index 0000000..1da0767 --- /dev/null +++ b/backend/tests/test_completion_phrases.py @@ -0,0 +1,89 @@ +"""Sentence completion: phrases from the user's own sent mail, plus starters.""" + +from __future__ import annotations + +from collections import Counter + +from app.completion import build_phrases, choose, complete, html_to_text, own_text + + +def personal(*texts: str) -> dict[str, Counter[str]]: + out: dict[str, Counter[str]] = {} + for (ctx, cont), n in build_phrases(texts).items(): + out.setdefault(ctx, Counter())[cont] = n + return out + + +def test_own_text_drops_quotes_and_signature() -> None: + for quote in ( + "On Mon, 5 Oct 2026 Anna wrote:", + "W dniu 5.10.2026 Anna pisze:", + "Am 05.10.2026 um 10:00 schrieb Anna:", + "-----Original Message-----", + "From: Anna ", + "--", + ): + text = f"Sure, I'll send it today.\n\n{quote}\nOld text here." + assert own_text(text).strip() == "Sure, I'll send it today.", quote + assert own_text("Yes.\n> quoted line\nAgreed.") == "Yes.\nAgreed." + + +def test_only_repeated_phrases_are_kept() -> None: + phrases = build_phrases( + ["I'll send the drawings by Friday.", "I'll send the drawings by Friday."] + ) + assert phrases[("the", "drawings by Friday.")] == 2 + once = build_phrases(["A private sentence written once."]) + assert once == {} + + +def test_personal_phrases_complete_in_any_language() -> None: + mine = personal( + "I'll send the drawings by Friday.", + "I'll send the drawings by Friday.", + "Prześlę rysunki do piątku.", + "Prześlę rysunki do piątku.", + ) + assert complete(mine, "I'll send the dr") == "awings by Friday." + assert complete(mine, "Prześlę rys") == "unki do piątku." + assert complete(mine, "Prześlę rysunki ") == "do piątku." + + +def test_starter_phrases_and_spacing() -> None: + assert complete({}, "I look forward ") == "to hearing from you." + assert complete({}, "I look forward") == " to hearing from you." # space added + assert complete({}, "Z pow") == "ażaniem," + assert complete({}, "Mit freundlichen ") == "Grüßen" + + +def test_nothing_after_punctuation_or_on_an_empty_line() -> None: + assert complete({}, "Thanks,") is None + assert complete({}, "Done. ") is None + assert complete({}, "") is None + assert complete({}, " ") is None + + +def test_suggestion_stops_where_habits_part() -> None: + # "see you" is followed by "soon" or "tomorrow" equally: suggest only "you". + assert choose(Counter({"you": 4, "you soon": 2, "you tomorrow": 2})) == "you" + assert choose(Counter({"you": 4, "you soon": 4, "you soon then": 3})) == "you soon then" + + +def test_polish_gmail_quotes_and_own_signature_are_left_out() -> None: + text = ( + "Dziękuję, prześlę jutro.\n" + "Imaginary Pixels – visual effects studio\n" + "pon., 5 paź 2026 o 10:00 Anna Nowak napisał(a):\n" + "> stary tekst" + ) + sig = html_to_text( + "
Imaginary Pixels – visual effects studio
+48 602 691 231
" + ) + assert own_text(text, [sig]).strip() == "Dziękuję, prześlę jutro." + + +def test_numbers_addresses_and_links_are_never_learned() -> None: + texts = ["Call me at +48 602 691 231 or write to me@x.example today."] * 3 + learned = " ".join(f"{c} {n}" for (c, n) in build_phrases(texts)) + assert "602" not in learned and "@" not in learned + assert ("call me", "at") in build_phrases(texts) diff --git a/backend/tests/test_translation_prep.py b/backend/tests/test_translation_prep.py new file mode 100644 index 0000000..04bb82f --- /dev/null +++ b/backend/tests/test_translation_prep.py @@ -0,0 +1,128 @@ +"""Getting a message ready for translation (app/translation_prep.py).""" + +from __future__ import annotations + +from app.translation_prep import ( + find_names, + find_names_in_signature, + join_wrapped_lines, + keep_symbols, + keep_terms, + split_pieces, + tidy, + translate_phrases, +) + +KEPT = '' + + +def test_wrapped_lines_are_joined_but_paragraphs_kept() -> None: + assert join_wrapped_lines("

ask you about
the schedule.

") == ( + "

ask you about the schedule.

" + ) + assert join_wrapped_lines("
ask you about
the schedule.
") == ( + "
ask you about the schedule.
" + ) + text = '
ask you about\nthe schedule.\n\nThanks,\nAnna
' + assert join_wrapped_lines(text) == ( + '
ask you about the schedule.\n\nThanks,\nAnna
' + ) + # A new sentence, or a capital after the break: left alone. + for html in ("

Done.
then more

", "

Hi Lukas,
Natalia

"): + assert join_wrapped_lines(html) == html + + +def test_sign_offs_come_from_the_phrase_table() -> None: + assert translate_phrases("

Best,
Natalia

", "en", "pl") == ( + f"

{KEPT}Pozdrawiam,
Natalia

" + ) + assert translate_phrases("
Mit freundlichen Grüßen
", "de", "en") == ( + f"
{KEPT}Kind regards
" + ) + assert "Pozdrawiam" not in translate_phrases("

Best results so far

", "en", "pl") + assert translate_phrases("

Best,

", "fr", "pl") == "

Best,

" # no table + + +def test_names_mid_sentence_but_not_headlines_or_german() -> None: + html = "

I came across Imaginary Pixels and Trusted Partner Network, in Spain.

" + assert find_names(html, "en") == {"Imaginary Pixels", "Trusted Partner Network"} + assert find_names("

Why Matchmoving Matters, Editing Digger And More

", "en") == set() + assert find_names("

Wir haben das Neue Angebot gesehen.

", "de") == set() + # Not across a sentence's end. + assert find_names("

working mostly in NukeX. My work includes paint.

", "en") == set() + assert find_names_in_signature("Imaginary Pixels – visual effects studio") == [ + "Imaginary Pixels" + ] + + +def test_kept_terms_and_spacing() -> None: + out = keep_terms( + "

Hi Natalia Raz, see Imaginary Pixels.

", ["Natalia Raz", "Imaginary Pixels"] + ) + assert out == f"

Hi {KEPT}Natalia Raz, see {KEPT}Imaginary Pixels.

" + assert keep_terms("", ["Natalia"]) == "" + assert tidy(f"nad {KEPT}Brutoni dalej") == f"nad {KEPT}Bruton i dalej" + assert tidy("ab") == "ab" # the sender's own spans + + +def test_emoji_kept() -> None: + html = ( + '

nice to e-meet you \n😊  

' + "

Great 👍🏽 work ❤️

" + ) + out = keep_symbols(html) + assert f"{KEPT}😊" in out + assert f"{KEPT}👍🏽" in out + assert f"{KEPT}❤️" in out + assert "nice to e-meet you" in out # the words still go to translation + + +def test_file_names_kept() -> None: + html = ( + "
  • A_0023C017_260521_130157_p1CAC.mov
    • " + "

    See final_v2 and plate-03.exr, e.g. the U.S cut.

    " + ) + out = keep_symbols(html) + assert f"{KEPT}A_0023C017_260521_130157_p1CAC.mov" in out + assert f"{KEPT}final_v2" in out + assert f"{KEPT}plate-03.exr" in out + assert "e.g. the U.S cut" in out # no file names there + + +def test_symbols_not_marked_twice_or_in_attributes() -> None: + html = ( + f'{KEPT}https://x.io/a_b.mov' + "" + ) + assert keep_symbols(html) == html + + +def test_text_without_letters_is_kept() -> None: + # A byte-order mark alone between tags came back as "Uzasadnienie". + out = keep_symbols( + '
    \ufeff
    Dear Lukas,
     

    –

    ' + ) + assert f"{KEPT}\ufeff" in out + assert "
     
    " in out # only spaces: nothing to keep + assert f"

    {KEPT}–

    " in out + assert "
    Dear Lukas,
    " in out + + +def test_handles_and_phone_numbers_kept() -> None: + out = keep_symbols( + "

    WeChat: Jacquelineeeee_ WhatsApp: +48 780 549 877, for 1300 - 2500 Euro

    " + ) + assert f"{KEPT}Jacquelineeeee_" in out + assert f"{KEPT}+48 780 549 877" in out + assert "1300 - 2500 Euro" in out # plain numbers stay in their sentence + + +def test_paragraphs_split_at_block_tags() -> None: + html = ( + f"
    Dear Anna,
    hello {KEPT}x.mov there
    " + "

     

  • Item
    • " + ) + pieces, paragraphs = split_pieces(html) + assert [p.text for p in paragraphs] == ["Dear Anna,", "hello x.mov there", "Item"] + # The kept file name counts for the language but is not translated. + assert [pieces[i] for i in paragraphs[1].pieces] == ["hello ", " there"] diff --git a/backend/tests/test_translation_text.py b/backend/tests/test_translation_text.py new file mode 100644 index 0000000..28d6b96 --- /dev/null +++ b/backend/tests/test_translation_text.py @@ -0,0 +1,123 @@ +"""What is sent to the translation service (app/translation.py).""" + +from __future__ import annotations + +import pytest + +from app import translation +from app.translation import detect, languages_of, names_to_keep, protect_addresses, translate_html + + +def test_addresses_in_text_are_kept_from_translation() -> None: + html = ( + '

    My reel: https://vimeo.com/1, ' + "write to anna@x.example or see www.example.com today.

    " + ) + out = protect_addresses(html) + assert 'https://vimeo.com/1' in out + assert 'anna@x.example' in out + assert 'www.example.com' in out + assert '' in out # attributes untouched + + +def test_style_blocks_are_left_alone() -> None: + html = "

    Hi

    " + assert protect_addresses(html) == html + + +async def test_only_names_the_model_changes_are_kept(monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[object] = [] + + async def post(path: str, payload: dict[str, object]) -> object: + sent.append(payload["q"]) + alone = { + "Lukas Remis": "Lukas Remis.", + "Milan": "Mediolan", + "Imaginary Pixels": "Imaginanal", + } + return {"translatedText": [alone[q] for q in payload["q"]]} # type: ignore[attr-defined] + + monkeypatch.setattr(translation, "_post", post) + kept = await names_to_keep( + ["Milan", "Lukas Remis", "Imaginary Pixels", "Milan", "x"], "en", "pl" + ) + assert kept == {"Milan", "Imaginary Pixels"} # "Lukas Remis" stays in its sentence + assert len(sent) == 1 # one request for all + + +async def test_all_names_kept_when_the_check_fails(monkeypatch: pytest.MonkeyPatch) -> None: + async def post(path: str, payload: dict[str, object]) -> object: + raise translation.TranslationError("down") + + monkeypatch.setattr(translation, "_post", post) + assert await names_to_keep(["Milan", "Lukas Remis"], "en", "pl") == {"Milan", "Lukas Remis"} + + +def _fake_detector(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """A detector that knows Chinese signs, Polish letters, else English.""" + asked: list[str] = [] + + async def detect_one(text: str) -> str: + asked.append(text) + if any("\u4e00" <= c <= "\u9fff" for c in text): + return "zh-Hans" + return "pl" if any(c in "ąćęłńóśźż" for c in text.lower()) else "en" + + monkeypatch.setattr(translation, "_detect_one", detect_one) + return asked + + +async def test_short_parts_take_the_next_language(monkeypatch: pytest.MonkeyPatch) -> None: + asked = _fake_detector(monkeypatch) + parts = [ + "您好！", + "Szanowny Kliencie,", # short: goes with the Polish that follows + "Widzę, że złożyłeś wniosek o otwarcie sklepu na platformie.", + "-----", + "Hello, my name is Jacqueline, the manager for Europe.", + "Jacqueline Lian", # short, last: goes with the one before + ] + assert await languages_of(parts) == ["zh-Hans", "pl", "pl", "", "en", "en"] + assert "Jacqueline Lian" not in asked # too short to ask about + + +async def test_a_message_in_three_languages(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_detector(monkeypatch) + text = ( + "尊敬的客户，我是速卖通的官方客户经理，我从后台看到您申请入驻速卖通平台开店。\n" + "Szanowny Kliencie,\n" + "Widzę, że złożyłeś wniosek o otwarcie sklepu na platformie AliExpress.\n" + "Dears,\nI noticed from the backend that you have applied to join the platform.\n" + ) + assert set((await detect(text)).split(",")) == {"zh-Hans", "pl", "en"} + # One language: one code. + assert await detect("I noticed from the backend that you applied, thanks a lot.") == "en" + + +async def test_each_paragraph_from_its_own_language(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_detector(monkeypatch) + calls: list[tuple[str, list[str]]] = [] + + async def post(path: str, payload: dict[str, object]) -> object: + q = payload["q"] + if path == "/translate" and isinstance(q, list) and payload.get("source") != "auto": + calls.append((str(payload["source"]), list(q))) + return {"translatedText": [f"PL({x})" for x in q]} + return {"translatedText": list(q) if isinstance(q, list) else q} # names: unchanged + + monkeypatch.setattr(translation, "_post", post) + html = ( + "
    我是速卖通的官方客户经理奎琳，我从后台看到您申请入驻速卖通平台开店。
    " + "
    Widzę, że złożyłeś wniosek o otwarcie sklepu & więcej.
    " + "
    I noticed that you applied to join the platform & more.
    " + "
    WhatsApp: +48 780 549 877
    " + ) + out = await translate_html(html, "zh-Hans,pl,en", "pl") + assert [c[0] for c in calls] == ["zh-Hans", "en"] # the Polish is left alone + assert "
    Widzę, że złożyłeś wniosek o otwarcie sklepu & więcej.
    " in out + # Two pieces of one Chinese sentence get a space between them. + assert "PL(我是速卖通的官方客户经理奎琳，) PL(" in out + assert "PL(I noticed that you applied to join the platform & more.)" in out.replace( + "&", "&" + ) + assert '+48 780 549 877' in out diff --git a/backend/workers/run.py b/backend/workers/run.py index 86fa8dd..e210422 100644 --- a/backend/workers/run.py +++ b/backend/workers/run.py @@ -27,6 +27,7 @@ from arq.worker import Worker from app.config import get_settings from app.indexer.gc import sweep_job from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, redis_settings +from app.snooze import wake_due_job from app.sync.actions import autopurge_trash_spam from app.sync.coordinator import SyncCoordinator from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS @@ -34,7 +35,9 @@ from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS # Core scheduled jobs. Daily Trash/Spam retention purge at 03:17 (off-peak, and a # minute that won't collide with typical top-of-hour crons); then the clean-up of # stored files nothing uses any more (the purge's among them) at 03:47. +# Snoozed conversations are checked every minute (cheap: a partial index). _CORE_CRON_JOBS = [ + cron(wake_due_job, second=5), cron(autopurge_trash_spam, hour=3, minute=17), cron(sweep_job, hour=3, minute=47, timeout=3600), ] diff --git a/backend/workers/worker.py b/backend/workers/worker.py index b5feab3..502c3ef 100644 --- a/backend/workers/worker.py +++ b/backend/workers/worker.py @@ -14,10 +14,12 @@ from typing import Any from arq import func +from app.completion import build_index_job from app.indexer.attachments import dedupe_existing_job from app.outbox.draft_sync import push_draft_to_imap_job, remove_imap_draft_job from app.outbox.jobs import file_sent_copy_job, send_job from app.push.service import push_new_mail_job +from app.subscriptions import backfill_list_headers_job from app.sync.actions import ( ACTION_MAX_TRIES, delete_thread_job, @@ -67,4 +69,11 @@ BULK_FUNCTIONS = [ sync_bootstrap_account, # Resumable (each batch is saved), so a long first run may span timeouts. func(dedupe_existing_job, timeout=6 * 3600), + # Batches are saved as they go, so a run cut short resumes where it stopped. + # No stored result: its job id (one per user) must only block a second run + # while one is queued or running — a kept result would refuse new scans for + # an hour after each one. + func(backfill_list_headers_job, timeout=6 * 3600, keep_result=0), + # Sentence completion: one per user at a time, and none kept (as above). + func(build_index_job, timeout=1800, keep_result=0), ] diff --git a/docker-compose.yml b/docker-compose.yml index 33d6cbf..2147137 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,8 @@ x-backend-env: &backend-env RSPAMD_CONTROLLER_PASSWORD: ${RSPAMD_CONTROLLER_PASSWORD} # Spelling and grammar check in the compose window. Empty turns it off. LANGUAGETOOL_URL: ${LANGUAGETOOL_URL-http://languagetool:8010} + # Email translation; empty turns it off. + LIBRETRANSLATE_URL: ${LIBRETRANSLATE_URL-http://libretranslate:5000} LOG_LEVEL: ${LOG_LEVEL:-INFO} UNDO_SECONDS: ${UNDO_SECONDS:-10} SESSION_TTL_DAYS: ${SESSION_TTL_DAYS:-30} @@ -109,12 +111,13 @@ services: - | set -e mkdir -p /mnt/postgres /mnt/blobs /mnt/backups /mnt/redis /mnt/meili /mnt/state \ - /mnt/languagetool-ngrams + /mnt/languagetool-ngrams /mnt/libretranslate # Only recurse when ownership is actually wrong: `chown -R` over a large # blob store on every start would be minutes of pointless disk churn. [ "$$(stat -c %u /mnt/blobs)" = "10001" ] || chown -R 10001:10001 /mnt/blobs [ "$$(stat -c %u /mnt/backups)" = "999" ] || chown -R 999:999 /mnt/backups [ "$$(stat -c %u /mnt/redis)" = "999" ] || chown -R 999:999 /mnt/redis + [ "$$(stat -c %u /mnt/libretranslate)" = "1032" ] || chown -R 1032:65534 /mnt/libretranslate chmod 700 /mnt/backups echo "[init] data directories ready" volumes: @@ -313,6 +316,32 @@ services: cpus: ${LANGUAGETOOL_CPUS:-1.0} logging: *logging + # Email translation (app/translation.py), all on this server. The first + # start downloads every language model (a few GB) into DATA_ROOT; later + # starts reuse them. A model is loaded into memory the first time its + # language is used. + libretranslate: + image: libretranslate/libretranslate:v1.9.6 + environment: + LT_DISABLE_WEB_UI: "true" + LT_UPDATE_MODELS: "false" + LT_THREADS: ${LIBRETRANSLATE_THREADS:-2} + TZ: ${TZ:-UTC} + volumes: + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/libretranslate:/home/libretranslate/.local + healthcheck: + test: ["CMD-SHELL", "python3 -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/languages', timeout=4)\""] + interval: 30s + timeout: 6s + retries: 5 + start_period: 30m # the first start downloads the models + depends_on: + init: { condition: service_completed_successfully } + restart: unless-stopped + mem_limit: ${LIBRETRANSLATE_MEM_LIMIT:-6g} + cpus: ${LIBRETRANSLATE_CPUS:-2.0} + logging: *logging + # Bayes tokens and fuzzy hashes. Separate from the app's redis on purpose # (§C.2): this data is derived and retrainable, so it trades durability for # room to grow — RDB snapshots every 5 min, and an eviction policy, where the diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0e6573c..1593809 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -158,6 +158,24 @@ services: retries: 12 restart: unless-stopped + # Email translation (app/translation.py). The first start downloads every + # language model (several GB) into the volume; later starts reuse them. + libretranslate: + image: libretranslate/libretranslate:v1.9.6 + environment: + LT_DISABLE_WEB_UI: "true" + LT_UPDATE_MODELS: "false" + ports: + - "127.0.0.1:5005:5000" + volumes: + - ltmodels:/home/libretranslate/.local + healthcheck: + test: ["CMD-SHELL", "python3 -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/languages', timeout=4)\""] + interval: 15s + timeout: 6s + retries: 240 + restart: unless-stopped + greenmail: image: greenmail/standalone:2.0.1 environment: @@ -170,6 +188,7 @@ services: - "127.0.0.1:3993:3993" # IMAPS volumes: + ltmodels: pgdata: meilidata: blobs: diff --git a/docs/feature-ideas.md b/docs/feature-ideas.md new file mode 100644 index 0000000..f388b14 --- /dev/null +++ b/docs/feature-ideas.md @@ -0,0 +1,298 @@ +# Feature ideas — making Ultimo competitive with Gmail + +Status (2026-10-18): **built** (uncommitted at the time of writing) — snooze, +keyboard shortcuts, subscriptions page + unsubscribe, templates, tracker +blocking. Left out of that first pass: follow-up reminders (§1.1), the image +proxy (§2.1), the footer-link fallback and "auto-archive in future" filter on the +Subscriptions page (§1.3), and bulk snooze from the list. Everything else here is +still a proposal. Each idea notes +what it is, why it matters, how it fits the existing architecture, and a rough +size (S = days, M = 1–2 weeks, L = several weeks). + +Already built, so not repeated here: undo send, scheduled send, labels, filters, +antispam, search operators (Meilisearch), VIP contacts, calendar (CalDAV, iMIP), +contacts (CardDAV), offline read-only mode, PWA + Web Push, 2FA, invitations, +hosted mailboxes (Stalwart). + +--- + +## 1. What people miss when leaving Gmail + +### 1.1 Snooze and follow-up reminders — M +- **Snooze:** "bring this back Monday 09:00". The thread leaves the inbox and + returns at the chosen time, marked unread and pinned to the top. +- **Follow-up:** when sending, "remind me if nobody replies in 3 days". If no + reply has arrived in the thread by then, it comes back to the inbox with a + "No reply yet" chip. +- **Fit:** a `snoozes` table (thread, user, wake_at, kind) plus an arq cron job + that wakes due threads. Snoozed threads move to a server-side "Snoozed" folder + so other IMAP clients see the same state. The follow-up check watches + `message.new` events for the thread. Could be a builtin plugin. + +### 1.2 Keyboard shortcuts — S/M +- Gmail-compatible: `j/k` next/previous, `o`/Enter open, `u` back to list, + `r`/`a`/`f` reply / reply-all / forward, `e` archive, `#` delete, `s` star, + `l` label, `v` move, `/` search, `c` compose, `x` select, `Shift+I/U` + read/unread, `g i` / `g s` go to inbox / starred, `?` shows them all. +- **Fit:** frontend only. One central key handler that respects focus in + inputs and the compose editor, plus a setting to turn it off. + +### 1.3 One-click unsubscribe and a subscriptions page — M +- An "Unsubscribe" button on mail that carries `List-Unsubscribe`, using the + one-click POST (RFC 8058) when offered, `mailto:` otherwise. +- A **Subscriptions** page: every sender who sends mailing-list mail, how often, + when last read, with Unsubscribe / "Delete all from this sender" / + "Auto-archive in future" (creates a filter). +- **How subscriptions are found:** no service records what a user signed up + for, so the page infers it from received mail (as Gmail does). A message is + bulk mail when it carries `List-Unsubscribe` (required of bulk senders by + Gmail/Yahoo since 2024), `List-Id`, `Precedence: bulk|list`, or mailing-service + markers (`Feedback-ID`, ESP headers). A footer "unsubscribe" link is a weaker + fallback, shown as "probably a newsletter". Mail is grouped by `List-Id`, else + by sender address. +- **Per group:** messages per month, last received, and **read rate** from the + `\Seen` flag ("opened 0 of 23"), sorted most frequent / least read first. +- **After unsubscribing:** mark the group unsubscribed with the date. If mail + still arrives after a grace period (RFC 8058 senders should stop within 2 + days), flag "still sending" and offer block / auto-archive via a filter. +- **Limits:** lists that haven't sent anything yet, or send no list headers, + can't be seen. The page shows "bulk senders currently mailing you". +- **Fit:** the plugin SDK can read raw headers + (`backend/app/plugins/actions.py`), but it parses the stored RFC822 per + message, and with hosted mailboxes the raw message is only cached. So extract + `List-Id`, `List-Unsubscribe` and `List-Unsubscribe-Post` during sync into a + small `message_list_headers` table, backfilled once for existing mail. The + spec (§12, M3) already describes the button as an example `message.actions` + plugin. The one-click POST goes from the server, through the SSRF guard. + +### 1.4 Templates (canned replies) — S +- Saved replies with placeholders (`{{first_name}}`, `{{my_name}}`), inserted + from a compose toolbar menu or by typing `/` in the editor. +- **Fit:** per-user table plus a compose-toolbar plugin slot. + +### 1.5 Out-of-office reply — M +- Start/end dates, subject, message; replies at most once per sender per N days; + never to mailing lists or `Auto-Submitted` mail (RFC 3834). +- **Fit:** for hosted mailboxes, write a Sieve `vacation` script through + Stalwart. For external accounts, the outbox sends the reply from an arq job + on `message.new`. + +--- + +## 2. Where Ultimo can beat Gmail + +### 2.1 Privacy as the selling point — M +- **Tracker blocking:** detect tracking pixels (1×1 images, known tracker + domains, per-recipient URL tokens) and strip them even when remote images are + allowed. Show "3 trackers blocked" on the message. +- **Image proxy:** when remote images are allowed, load them through the server + (SSRF-guarded, cached), so senders never see the reader's IP address or when + the mail was opened. +- **Aliases on the hosted domain:** unlimited addresses such as + `shop.amazon@ultimo.email` or `name+tag@`. Each one can be switched off, and + the app shows which alias leaked to spammers. +- **Fit:** the sanitizer (`backend/app/indexer/sanitize.py`) already rewrites + HTML and blocks remote images. Aliases are Stalwart principals or catch-all + rules managed from the M6 hosted-mailbox code. + +### 2.2 Encrypted mail (OpenPGP) — L +- Optional, done in the browser (OpenPGP.js), so the **private key never + reaches the server unencrypted**. +- Decrypt and verify received PGP/MIME mail. Sign and encrypt when sending. + Look up recipients' keys automatically (WKD, keys.openpgp.org) and publish + WKD for the hosted domain. +- **Fit:** compose and reading-panel plugin slots. The key backup is encrypted + with a passphrase before upload. Indexing can't search encrypted bodies (say + so in the UI). + +### 2.3 AI assistant (opt-in) — L +Detailed design in **§5** below. + +--- + +## 3. Productivity polish + +### 3.1 Split inbox / categories — M +Tabs such as *Important / Newsletters / Notifications / Receipts / Other*, +filled by rules (sender, `List-Id`, labels, filters) and optionally by the AI +classifier (§5.3). The user can move a sender between tabs and the app learns +from it. + +### 3.2 Everything you've received — S/M +A view listing all **attachments, links and photos**, filterable by sender, type +and date, with preview. Builds on the right panel's Attachments tab and the +deduplicated attachment store. + +### 3.3 Mute a thread; send and archive — S +- **Mute:** new replies in the thread skip the inbox (still searchable). Good for + noisy reply-all threads. +- **Send and archive:** one button that sends the reply and archives the thread. + +### 3.4 Easy move from Gmail — M +- Copy existing mail from Gmail over IMAP (Google OAuth is already planned), + with progress, resume, and labels → labels. +- Import Gmail filters (Gmail exports them as XML) into the filters plugin. +- Import contacts (vCard/CSV) and calendars (ICS). +- The hardest part of winning users is getting them to switch. This makes + switching painless. + +### 3.5 Calendar from mail — M +- "Create event from this mail": date, time and place pre-filled from the text + (rules first, AI optional). +- "Find a time": free/busy for the thread's participants (for those whose + calendar we can see), then send an invitation. +- "Turn into task" once a tasks list exists (VTODO over CalDAV). + +### 3.6 Shared mailboxes and delegation — L +For families and small teams: +- A shared `info@` mailbox visible to several users. +- Assign a thread to someone, with status (open / waiting / done). +- Internal notes on a thread that are never sent. +- Delegated access ("my assistant can read and send as me"). + +--- + +## 4. Suggested order + +| Milestone | Contents | Why | +|---|---|---| +| **Next** | Snooze + follow-up, keyboard shortcuts, unsubscribe/subscriptions, templates, tracker blocking + image proxy | Moderate size, fits the current architecture, users notice right away; starts the privacy story | +| Then | Out-of-office reply, mute / send and archive, everything-received view, Gmail import | Smaller wins + removes the switching barrier | +| Later | AI assistant (§5), split inbox, calendar from mail | AI unlocks smart categories and event extraction | +| Separate | OpenPGP, aliases, shared mailboxes | Larger security / multi-user projects | + +--- + +## 5. AI assistant — design + +### 5.1 Principles +1. **Off by default, opt-in twice.** The administrator turns the plugin on and + chooses a provider. Each user then opts in, per feature. Nothing about a + user's mail is sent anywhere until they switch a feature on. +2. **The server calls the AI, never the browser.** API keys stay on the server, + encrypted with the vault (`MASTER_KEY`), like mail-account passwords. +3. **The AI suggests, the user acts.** No AI output sends mail, deletes, moves + or unsubscribes on its own. Drafts land in the compose editor; categories + are labels the user can change. +4. **Mail is untrusted input** (see §5.6). An email can contain text written to + manipulate the model ("ignore your instructions and…"). The design makes + this harmless rather than trying to detect it. +5. **Provider-independent.** One small adapter interface; Anthropic (Claude) is + the default. Other providers or a local model can be plugged in. + +### 5.2 Features + +| Feature | Trigger | What is sent to the model | Output | +|---|---|---|---| +| **Thread summary** | User clicks "Summarize" (or auto for threads over N messages, if enabled) | Plain-text bodies of the thread (quoted history stripped), senders, dates | 3–5 bullets + "what's asked of you" + deadlines; streamed into the reading panel | +| **Reply drafts** | "Suggest reply" in the reading panel, or a short instruction ("decline politely, offer next week") | The thread + the user's instruction + their signature/name + optional tone examples from their own sent mail | A draft inserted into the compose editor; the user edits and sends | +| **Rewrite** | Select text in compose → "Shorter / More formal / Friendlier / Fix grammar / Translate to …" | Only the selected text | Replacement text with a preview and undo | +| **Categorization** | Background, on each new message (only for users who opted in) | Sender, subject, `List-*` headers, first ~2,000 characters of text | One category from a fixed list (structured output) + optional "needs reply" flag → label | +| **Ask your mailbox** | A search-bar mode: "When does my car insurance expire?" | Only the messages that a search finds relevant (see §5.4) | An answer with links to the source messages | +| **Event extraction** | "Create event" on a message | That message | Title, start/end, place, attendees as structured data → pre-filled event form | + +### 5.3 How it plugs into Ultimo +- **A new builtin plugin `webmail_plugin_ai`**, using the existing SDK: + reading-panel and compose slots for the buttons, its own tables for settings, + usage and cached results, and arq jobs for background work. +- **Provider adapter:** `complete(system, messages, schema=None, stream=False)`, + with implementations for Anthropic (official `anthropic` Python SDK), and + later Bedrock / Vertex / a local OpenAI-compatible server (Ollama, vLLM). +- **Streaming:** summaries and drafts stream token by token to the browser over + the existing WebSocket (`backend/app/ws.py`), so text appears right away. +- **Structured outputs:** categorization and event extraction use the API's + structured output mode (`output_config.format` / `messages.parse()` against + a JSON schema). The model can only answer with an allowed category, so a + malicious mail can't make it invent a "trusted" label. +- **Caching results:** a summary is stored keyed by thread + the newest message + id, so reopening a thread costs nothing until a new message arrives. +- **Prompt caching:** the fixed instructions (system prompt, category list, + style guide) come first and are marked cacheable, so repeated calls pay for + them at the much cheaper cache-read rate. Volatile parts (the email text, the + date) come after. +- **Bulk backlog:** classifying existing mail when a user first opts in goes + through the Message Batches API (asynchronous, half price). +- **Quotas and audit:** per-user monthly token budget set by the admin; a usage + table (user, feature, model, input/output tokens) shown in the admin panel. + Calls are logged without message content. +- **Refusals and errors:** check `stop_reason` (including `refusal`) before using + output; on rate limits, retry later (arq retry); the UI says "AI unavailable" + rather than failing the mail action. + +### 5.4 "Ask your mailbox" in detail +Searching all mail with the model would be slow and expensive, so this uses +retrieval: +1. The question goes to the model together with **one read-only tool**, + `search_mail(query, from?, after?, before?)`, backed by the existing + Meilisearch index and run **as that user** (row-level security; it can only + see their own mail). +2. The model makes one or more searches; the server returns short snippets + (sender, date, subject, ~500 characters). +3. If needed, the model asks for the full text of a few messages with a second + read-only tool, `read_message(id)`. +4. It answers and cites message ids; the UI turns them into links. + +There are no tools that write anything. Keyword search is enough to start. +Semantic search (embeddings in pgvector) could come later, but would need a +separate embeddings provider or local model, since that's a different service +from text generation. + +### 5.5 Who provides the AI +The administrator chooses one in the admin panel: + +| Option | Privacy | Quality | Cost | Notes | +|---|---|---|---|---| +| **Anthropic Claude API** (default) | Mail text leaves the server for Anthropic | Best | Pay per token | Simplest; official Python SDK; streaming, structured outputs, prompt caching, batches | +| **Claude through Google Vertex AI or Amazon Bedrock** | Same model, contract and data processing with Google/AWS; regional endpoints (e.g. EU) | Best | Partner pricing | Choose if EU data residency or an existing cloud contract matters | +| **Local model** (Ollama / vLLM on a GPU machine) | Nothing leaves your network | Noticeably weaker; fine for categorization and rewriting, weaker for summaries and Q&A | Hardware + electricity | The QNAP has no GPU — CPU-only is too slow for interactive use; needs a separate box | +| **Bring your own key** (per user) | As the chosen provider | — | User pays | Optional; lets power users use their own account | + +**Models (Anthropic, prices cached 2026-10-06, per million tokens in / out):** + +| Model | Price | Suggested use | +|---|---|---| +| `claude-opus-5-5` | $4 / $20 | Default for everything; best quality for drafts, summaries, Q&A | +| `claude-sonnet-5-5` | $2 / $10 | Option for drafts / summaries if cost matters | +| `claude-haiku-5-5` | $0.10 / $0.50 | Option for high-volume background work (categorization) | + +The model is configurable **per feature** in the admin panel. Default to Opus 5.5 +everywhere and switch individual features down only after checking that quality +holds. Categorization is the one where volume matters. Example: 500 new +messages/day × ~1,000 tokens ≈ 15M input tokens/month ≈ $60/month on Opus 5.5 +vs. ≈ $1.50 on Haiku 5.5 (before caching). Interactive features are cheap +either way: a summary of a long thread is a few thousand tokens. Prices change; +check the provider's current price list before deciding. + +**Data handling to verify before launch:** the provider's commercial terms on +training (Anthropic's API terms state that API inputs are not used for model +training by default — confirm the current wording), how long requests are +retained, and a data processing agreement (DPA). Hosting other people's mail in +the EU makes Ultimo the GDPR controller, so the opt-in screen must say which +provider receives the text. + +### 5.6 Security: mail as untrusted input +- **No side effects from model output.** Every action needs a user click. Drafts + are never sent automatically. +- **Read-only tools only**, always running as the requesting user (RLS), so an + email that says "search for passwords and include them in the reply" can at + most show the user something from their own mailbox, to the user. +- **Fixed output shapes** (structured outputs) for anything that changes state, + like labels. +- **No remote content:** links in model output are shown as text or go through + the normal link-safety warning; images are never loaded from model output. +- **Clear boundaries in the prompt:** email text is wrapped and labelled as + data, not instructions. This helps, but the guarantees above don't depend on + it. +- **Minimal data:** attachments are not sent by default; bodies are trimmed of + quoted history and signatures; hidden HTML text is dropped by the sanitizer + before the model sees it. + +### 5.7 Build steps +1. Plugin skeleton, provider adapter (Anthropic), admin settings (key, model per + feature, quotas), per-user opt-in screen, usage table. +2. Thread summary (streaming) + rewrite in compose. +3. Reply drafts with tone from the user's sent mail. +4. Categorization (structured output, batch backfill) → feeds split inbox (§3.1). +5. Ask your mailbox (read-only tool loop over Meilisearch). +6. Event extraction → calendar plugin. +7. Optional: Vertex/Bedrock and local-model adapters; bring-your-own-key. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f892cf9..02bb376 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,9 +8,11 @@ "name": "webmail-frontend", "version": "0.1.0", "dependencies": { + "@emoji-mart/data": "~1.2.1", "@tanstack/react-query": "~5.51.0", "@tanstack/react-virtual": "~3.8.0", "docx-preview": "^0.4.0", + "emoji-mart": "~5.6.0", "jszip": "^3.10.1", "pdfjs-dist": "^4.10.38", "qrcode.react": "~4.1.0", @@ -524,6 +526,12 @@ "node": ">=18" } }, + "node_modules/@emoji-mart/data": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emoji-mart/data/-/data-1.2.1.tgz", + "integrity": "sha512-no2pQMWiBy6gpBEiqGeU77/bFejDqUTRY7KX+0+iur13op3bqUsXdnwoZs6Xb1zbv0gAj5VvS1PWoUUckSr5Dw==", + "license": "MIT" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -3457,6 +3465,12 @@ "dev": true, "license": "ISC" }, + "node_modules/emoji-mart": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/emoji-mart/-/emoji-mart-5.6.0.tgz", + "integrity": "sha512-eJp3QRe79pjwa+duv+n7+5YsNhRcMl812EcFVwrnRvYKoNPoQb5qxU8DG6Bgwji0akHdp6D4Ln6tYLG58MFSow==", + "license": "MIT" + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", diff --git a/frontend/package.json b/frontend/package.json index f5b1bc0..069f096 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,9 +13,11 @@ "gen:api": "openapi-typescript http://127.0.0.1:8000/openapi.json -o src/api/schema.ts" }, "dependencies": { + "@emoji-mart/data": "~1.2.1", "@tanstack/react-query": "~5.51.0", "@tanstack/react-virtual": "~3.8.0", "docx-preview": "^0.4.0", + "emoji-mart": "~5.6.0", "jszip": "^3.10.1", "pdfjs-dist": "^4.10.38", "qrcode.react": "~4.1.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index de70d94..1cfd812 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,11 +1,11 @@ -import { useQueryClient } from "@tanstack/react-query"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect } from "react"; import { Navigate, Route, Routes, useLocation } from "react-router-dom"; import { OfflineController } from "./components/Offline"; import { PushController } from "./components/PushController"; -import { useT } from "./i18n"; -import { isOfflineError } from "./lib/api"; +import { detectLanguage, supportedLanguage, useLocale, useT } from "./i18n"; +import { api, isOfflineError } from "./lib/api"; import { useMe } from "./lib/auth"; import { connectWs } from "./lib/ws"; import { PluginPage } from "./plugins/PluginPage"; @@ -17,6 +17,33 @@ import { Register } from "./routes/Register"; import { ResetPassword } from "./routes/ResetPassword"; import { Settings } from "./routes/settings/Settings"; +/** The interface language follows the account (Settings → Appearance), on + * every device. An account that has never chosen one gets the browser's or + * system's language (English if we have no catalog for it), saved right away + * — so it stays put on a device set to another language later. */ +function LanguageSync() { + const { locale, setLocale } = useLocale(); + const queryClient = useQueryClient(); + const settings = useQuery({ queryKey: ["settings"], queryFn: api.settings }); + const saved = settings.data?.ui_language; + useEffect(() => { + if (saved === undefined) return; // not loaded yet + if (saved) { + const code = supportedLanguage(saved) ?? "en"; + if (code !== locale) setLocale(code); + return; + } + const chosen = detectLanguage(); + setLocale(chosen); + void api + .patchSettings({ ui_language: chosen }) + .then((data) => queryClient.setQueryData(["settings"], data)) + .catch(() => undefined); // tried again on the next visit + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [saved]); + return null; +} + function RequireAuth({ children }: { children: JSX.Element }) { const { data, isLoading, isError, error, refetch } = useMe(); const t = useT(); @@ -29,7 +56,7 @@ function RequireAuth({ children }: { children: JSX.Element }) { }, [data, queryClient]); if (isLoading) { - return
    Loading…
    ; + return
    {t("common.loading")}
    ; } if (isError && isOfflineError(error)) { // No connection and nothing saved on this device: not a reason to sign out. @@ -50,6 +77,7 @@ function RequireAuth({ children }: { children: JSX.Element }) { } return ( <> + {children} diff --git a/frontend/src/admin/AdminApp.tsx b/frontend/src/admin/AdminApp.tsx index 3ccec19..ac254e6 100644 --- a/frontend/src/admin/AdminApp.tsx +++ b/frontend/src/admin/AdminApp.tsx @@ -6,6 +6,7 @@ import { useT } from "../i18n"; import { api, ApiError } from "../lib/api"; import { useMe } from "../lib/auth"; import type { AdminMail, AdminUser } from "../lib/types"; +import { useConfirm } from "../components/AppDialog"; const GIB = 1024 ** 3; @@ -519,6 +520,7 @@ function InvitationEmailDialog({ onClose }: { onClose: () => void }) { * placeholders; previewed with sample names, and sent as a test. */ function InvitationEmail() { const t = useT(); + const [confirm, confirmDialog] = useConfirm(); const queryClient = useQueryClient(); const tpl = useQuery({ queryKey: ["adminInviteTemplate"], queryFn: api.adminInviteTemplate }); // Sender and subject are drafts until Save; null shows the server's value. @@ -566,6 +568,7 @@ function InvitationEmail() { return (
    + {confirmDialog}

    {t("admin.inviteMail")}

    {t("admin.inviteMailHelp")}

    confirm(t("admin.inviteResetConfirm")) && save.mutate({ html: "" })} + onClick={async () => { + const ok = await confirm({ + title: t("admin.inviteReset"), + message: t("admin.inviteResetConfirm"), + confirmLabel: t("admin.inviteResetAction"), + }); + if (ok) save.mutate({ html: "" }); + }} > {t("admin.inviteReset")} diff --git a/frontend/src/components/AppDialog.tsx b/frontend/src/components/AppDialog.tsx new file mode 100644 index 0000000..ec16ad4 --- /dev/null +++ b/frontend/src/components/AppDialog.tsx @@ -0,0 +1,218 @@ +import { useCallback, useEffect, useState, type ReactNode } from "react"; +import { createPortal } from "react-dom"; + +import { useT } from "../i18n"; + +// The app's own dialog, in place of the browser's alert/confirm/prompt boxes. +// Looks like Settings' confirmations. Drawn on , so a compose window +// (or any other clipped, transformed box) it is opened from can't cut it off. +// Escape cancels; Enter in a form submits it. +export function AppDialog({ + title, + children, + onClose, +}: { + title: string; + children: ReactNode; + onClose: () => void; +}) { + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if (e.key === "Escape") { + e.stopPropagation(); + onClose(); + } + }; + // Capture: before the app's own Escape handling (minimizing compose windows). + window.addEventListener("keydown", onKey, true); + return () => window.removeEventListener("keydown", onKey, true); + }, [onClose]); + + return createPortal( +
    +
    e.stopPropagation()} + > +

    {title}

    + {children} +
    +
    , + document.body, + ); +} + +export function DialogButtons({ + confirmLabel, + onCancel, + danger = false, + disabled = false, + submit = false, + onConfirm, +}: { + confirmLabel: string; + onCancel: () => void; + danger?: boolean; + disabled?: boolean; + /** The confirm button submits the surrounding form (Enter works too). */ + submit?: boolean; + onConfirm?: () => void; +}) { + const t = useT(); + return ( +
    + + +
    + ); +} + +export function ConfirmDialog({ + title, + message, + detail, + confirmLabel, + danger = false, + onConfirm, + onCancel, +}: { + title: string; + message?: string; + /** A second, smaller paragraph (what exactly will happen). */ + detail?: string; + confirmLabel: string; + danger?: boolean; + onConfirm: () => void; + onCancel: () => void; +}) { + return ( + + {message &&

    {message}

    } + {detail &&

    {detail}

    } + +     + ); +} + +export interface ConfirmOptions { + title: string; + message?: string; + detail?: string; + confirmLabel: string; + danger?: boolean; +} + +/** `await confirm({...})` in place of window.confirm: true when confirmed. + * Render the returned element somewhere in the component. */ +export function useConfirm(): [(options: ConfirmOptions) => Promise, ReactNode] { + const [request, setRequest] = useState<(ConfirmOptions & { resolve: (ok: boolean) => void }) | null>( + null, + ); + const confirm = useCallback( + (options: ConfirmOptions) => new Promise((resolve) => setRequest({ ...options, resolve })), + [], + ); + const finish = (ok: boolean) => { + request?.resolve(ok); + setRequest(null); + }; + const element = request ? ( + finish(true)} + onCancel={() => finish(false)} + /> + ) : null; + return [confirm, element]; +} + +export interface PromptOptions { + title: string; + message?: string; + label: string; + initial?: string; + placeholder?: string; + confirmLabel: string; + maxLength?: number; +} + +function PromptDialog({ + options, + onDone, +}: { + options: PromptOptions; + onDone: (value: string | null) => void; +}) { + const [value, setValue] = useState(options.initial ?? ""); + const cancel = useCallback(() => onDone(null), [onDone]); + return ( + + { + e.preventDefault(); + if (value.trim()) onDone(value.trim()); + }} + > + {options.message &&

    {options.message}

    } + + + +     + ); +} + +/** `await prompt({...})` in place of window.prompt: the text, or null if cancelled. */ +export function usePrompt(): [(options: PromptOptions) => Promise, ReactNode] { + const [request, setRequest] = useState< + (PromptOptions & { resolve: (value: string | null) => void }) | null + >(null); + const prompt = useCallback( + (options: PromptOptions) => + new Promise((resolve) => setRequest({ ...options, resolve })), + [], + ); + const onDone = useCallback( + (value: string | null) => { + request?.resolve(value); + setRequest(null); + }, + [request], + ); + const element = request ? : null; + return [prompt, element]; +} + +/** A short notice in the corner (components/Toast.tsx), in place of window.alert. */ +export function notify(text: string, ms?: number): void { + window.dispatchEvent(new CustomEvent("webmail:toast", { detail: { text, ms } })); +} diff --git a/frontend/src/components/ShortcutList.tsx b/frontend/src/components/ShortcutList.tsx new file mode 100644 index 0000000..e7ac34b --- /dev/null +++ b/frontend/src/components/ShortcutList.tsx @@ -0,0 +1,61 @@ +import { useT } from "../i18n"; +import { SHORTCUT_GROUPS } from "../lib/shortcuts"; + +/** Every shortcut in groups, keys drawn as keycaps. Settings and the "?" overlay. */ +export function ShortcutList() { + const t = useT(); + return ( +
    + {SHORTCUT_GROUPS.map((group) => ( +
    +

    + {t(group.title)} +

    +
      + {group.items.map((item) => ( +
    • + {t(item.label)} + + {item.keys.map((k, i) => ( + + {i > 0 && ( + {item.keys[0] === "Shift" ? "+" : t("shortcuts.then")} + )} + + {k} + + + ))} + +
      • + ))} +
    +
    + ))} +
    + ); +} + +/** The "?" overlay. */ +export function ShortcutsHelp({ onClose }: { onClose: () => void }) { + const t = useT(); + return ( +
    +
    e.stopPropagation()} + > +
    +

    {t("shortcuts.title")}

    + +
    + +
    +
    + ); +} diff --git a/frontend/src/components/compose/ComposeDock.tsx b/frontend/src/components/compose/ComposeDock.tsx index 0676eed..e49a508 100644 --- a/frontend/src/components/compose/ComposeDock.tsx +++ b/frontend/src/components/compose/ComposeDock.tsx @@ -82,7 +82,7 @@ export function ComposeDock({ : "hidden" } > - onClose(win.id)} fullWindow /> + onClose(win.id)} fullWindow /> ))} @@ -162,6 +162,7 @@ export function ComposeDock({ > noteTitle(win.id, s)} minimized={!open} expanded={open && win.expanded} diff --git a/frontend/src/components/compose/Composer.tsx b/frontend/src/components/compose/Composer.tsx index e4ec02f..14b7a1e 100644 --- a/frontend/src/components/compose/Composer.tsx +++ b/frontend/src/components/compose/Composer.tsx @@ -1,4 +1,4 @@ -import { useQuery } from "@tanstack/react-query"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect, useRef, useState } from "react"; import { useT } from "../../i18n"; @@ -16,11 +16,14 @@ import { putLocal, uploadFiles, } from "../../lib/scheduled"; -import type { AccountOut, AttachmentOut, DraftInput, DraftOut } from "../../lib/types"; +import type { AccountOut, AttachmentOut, DraftInput, DraftOut, MailTemplate } from "../../lib/types"; import { showOfflineNotice } from "../../lib/offlineNotice"; import { RecipientChips } from "./RecipientChips"; import { RichTextEditor, type RichTextEditorHandle } from "./RichTextEditor"; import { hasUndoHost, offerUndo } from "./undoSend"; +import { ConfirmDialog } from "../AppDialog"; +import { SaveTemplateDialog } from "./SaveTemplateDialog"; +import { noteSnapshotDraft, saveSnapshot } from "./restore"; const MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024; // 25 MB per file @@ -48,6 +51,9 @@ export interface ComposerInit { localState?: LocalState; // "scheduled" when reopened from Scheduled attachments?: AttachmentOut[]; // known already: no need to ask the server localFiles?: LocalFile[]; + // Brought back after leaving the mail view (./restore.ts) with changes that + // may not have been saved yet: save again. + restoredDirty?: boolean; } // Build the composer seed from a saved draft (used to reopen it in place or in a @@ -161,6 +167,7 @@ const caret = ( export function Composer({ init, + snapshotKey, onClose, expanded = false, onToggleExpand, @@ -171,6 +178,9 @@ export function Composer({ fullWindow = false, }: { init: ComposerInit; + /** Keep this window's content when it goes away for any reason but + * Send/Discard (restore.ts), under this key. */ + snapshotKey?: string; onClose: () => void; expanded?: boolean; onToggleExpand?: () => void; @@ -211,9 +221,21 @@ export function Composer({ const [spellLang, setSpellLang] = useState(null); const [spellMenuOpen, setSpellMenuOpen] = useState(false); const settingsQuery = useQuery({ queryKey: ["settings"], queryFn: api.settings }); - // The language list folds away whenever the More menu closes. + // More → Templates: saved replies (api/templates.py). + const [templatesOpen, setTemplatesOpen] = useState(false); + const queryClient = useQueryClient(); + const templatesQuery = useQuery({ + queryKey: ["templates"], + queryFn: api.templates, + enabled: moreOpen, + staleTime: 60_000, + }); + // The language and template lists fold away whenever the More menu closes. useEffect(() => { - if (!moreOpen) setSpellMenuOpen(false); + if (!moreOpen) { + setSpellMenuOpen(false); + setTemplatesOpen(false); + } }, [moreOpen]); // What the engine can check — and whether this server has one at all. const spellLanguages = useQuery({ @@ -250,6 +272,9 @@ export function Composer({ onTitleRef.current?.(subject); }, [subject]); const draftId = useRef(init.draftId ?? null); + // Where this window keeps its content when it goes away (restore.ts). + const snapshotKeyRef = useRef(snapshotKey); + snapshotKeyRef.current = snapshotKey; // This message's copy on this device, while there is one (lib/scheduled.ts). const localId = useRef(init.localId ?? null); const discardedRef = useRef(false); @@ -258,7 +283,7 @@ export function Composer({ const sentRef = useRef(false); // send initiated -> don't mirror to Drafts on close // Whether the user actually edited anything. Merely opening a draft to view it // must not re-save/re-mirror it (that spawned duplicate IMAP copies). - const dirtyRef = useRef(false); + const dirtyRef = useRef(Boolean(init.restoredDirty)); const markDirty = () => { dirtyRef.current = true; }; @@ -278,6 +303,41 @@ export function Composer({ return () => document.removeEventListener("mousedown", onDown); }, [moreOpen, fromOpen, minimized]); + // A template holds what was written: not the signature, not the quoted mail. + const templateContent = (): string => { + const box = document.createElement("div"); + box.innerHTML = bodyHtml; + box.querySelectorAll(".wmc-signature, .wmc-quoted, .wmc-quote-toggle").forEach((n) => n.remove()); + // Blank lines at either end (the editor keeps one above the signature). + const blank = (n: ChildNode | null) => + n !== null && + (n.nodeType === Node.TEXT_NODE + ? !n.textContent?.trim() + : n instanceof HTMLElement && !n.textContent?.trim() && !n.querySelector("img")); + while (blank(box.firstChild)) box.firstChild?.remove(); + while (blank(box.lastChild)) box.lastChild?.remove(); + return box.textContent?.trim() || box.querySelector("img") ? box.innerHTML.trim() : ""; + }; + // Our own dialogs (components/AppDialog.tsx), not the browser's prompt/confirm. + const [savingTemplate, setSavingTemplate] = useState(null); // the content + const [deletingTemplate, setDeletingTemplate] = useState(null); + const saveTemplate = async (name: string) => { + if (savingTemplate === null) return; + await api.saveTemplate(name, subject, savingTemplate); + void queryClient.invalidateQueries({ queryKey: ["templates"] }); + }; + const insertTemplate = (tpl: MailTemplate) => { + setMoreOpen(false); + markDirty(); + if (!subject.trim() && tpl.subject) setSubject(tpl.subject); + editorRef.current?.insertHtml(tpl.body_html); + }; + const deleteTemplate = async (tpl: MailTemplate) => { + setDeletingTemplate(null); + await api.deleteTemplate(tpl.id).catch(() => undefined); + void queryClient.invalidateQueries({ queryKey: ["templates"] }); + }; + const account = accounts.find((a) => a.id === accountId); const fromName = account?.display_name || ""; const fromEmail = account?.email ?? ""; @@ -533,7 +593,11 @@ export function Composer({ setLocalFiles([]); } if (draftId.current) await api.updateDraft(draftId.current, payload(atts)); - else draftId.current = (await api.createDraft(payload(atts))).id; + else { + draftId.current = (await api.createDraft(payload(atts))).id; + // This window may have gone away while the draft was being made. + if (snapshotKeyRef.current) noteSnapshotDraft(snapshotKeyRef.current, draftId.current); + } if (localId.current) { await deleteLocal(localId.current); localId.current = null; @@ -586,8 +650,44 @@ export function Composer({ const saveLocalRef = useRef(saveLocal); saveLocalRef.current = saveLocal; + // What this window holds right now, to bring it back as it is (restore.ts). + // A ref: the unmount below sees only the first render's values otherwise. + const snapshotRef = useRef<() => ComposerInit>(() => init); + snapshotRef.current = () => ({ + ...init, + accountId, + to, + cc, + bcc, + subject, + html: bodyHtml || initialHtml, + text: undefined, + quote: undefined, // inside html now + forwardFrom: undefined, // its attachments are in `attachments` now + draftId: draftId.current ?? undefined, + localId: localId.current ?? undefined, + localState: localState.current, + attachments, + localFiles, + restoredDirty: dirtyRef.current, + }); + + const keepSnapshot = () => { + const key = snapshotKeyRef.current; + if (key && !sentRef.current && !discardedRef.current) saveSnapshot(key, snapshotRef.current()); + }; + const keepSnapshotRef = useRef(keepSnapshot); + keepSnapshotRef.current = keepSnapshot; + // A reload or a closed tab unmounts nothing: keep the content then too. + useEffect(() => { + const onPageHide = () => keepSnapshotRef.current(); + window.addEventListener("pagehide", onPageHide); + return () => window.removeEventListener("pagehide", onPageHide); + }, []); + useEffect( () => () => { + keepSnapshotRef.current(); // leaving the mail view (Settings…): back as it was if (undoTimer.current) window.clearTimeout(undoTimer.current); // On close, mirror the draft to the IMAP Drafts folder so it can be // continued on any mail client — but only if the user actually edited it @@ -773,6 +873,24 @@ export function Composer({ {dragKind === "body" ? t("compose.dropBody") : t("compose.dropAttach")} )} + {savingTemplate !== null && ( + setSavingTemplate(null)} + /> + )} + {deletingTemplate && ( + void deleteTemplate(deletingTemplate)} + onCancel={() => setDeletingTemplate(null)} + /> + )} {showClosePrompt && (
    {t("compose.insertSignature")} + + {templatesOpen && ( +
    + {(() => { + const content = templateContent(); + return ( + + ); + })()} +
    + {templatesQuery.data?.length === 0 && ( +
    {t("compose.noTemplates")}
    + )} + {templatesQuery.data?.map((tpl) => ( +
    + + +
    + ))} +
    + )} {spellLanguages.data && (() => { // The language in force: this message's choice, else Settings. const fallback = settingsQuery.data?.spellcheck_enabled diff --git a/frontend/src/components/compose/EmojiButton.tsx b/frontend/src/components/compose/EmojiButton.tsx new file mode 100644 index 0000000..67cafb2 --- /dev/null +++ b/frontend/src/components/compose/EmojiButton.tsx @@ -0,0 +1,154 @@ +import { useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; + +import { useT } from "../../i18n"; +import { useTheme } from "../../lib/theme"; + +// emoji-mart, loaded on first use: its emoji data is a few hundred KB that the +// rest of the app does without. The data is bundled — nothing is fetched from +// outside this server. +const loadData = async () => (await import("@emoji-mart/data")).default; + +const PICKER_W = 352; +const PICKER_H = 435; + +/** emoji-mart's picker (a web component), made and removed here: its React + * wrapper builds it twice under StrictMode, doubling sections. */ +function PickerHost({ + theme, + onSelect, + loading, +}: { + theme: "light" | "dark"; + onSelect: (native: string) => void; + loading: string; +}) { + const hostRef = useRef(null); + const onSelectRef = useRef(onSelect); + onSelectRef.current = onSelect; + const [ready, setReady] = useState(false); + useEffect(() => { + let picker: HTMLElement | null = null; + let alive = true; + void import("emoji-mart").then(({ Picker }) => { + if (!alive || !hostRef.current) return; + picker = new Picker({ + data: loadData, + theme, + previewPosition: "none", + skinTonePosition: "search", + maxFrequentRows: 2, + onEmojiSelect: (emoji: { native?: string }) => { + if (emoji.native) onSelectRef.current(emoji.native); + }, + }) as unknown as HTMLElement; + hostRef.current.appendChild(picker); + setReady(true); + }); + return () => { + alive = false; + picker?.remove(); + }; + }, [theme]); + return ( +
    + {!ready && ( +
    + {loading} +
    + )} +
    + ); +} + +/** The formatting bar's emoji button and its picker. ``onOpen`` runs before + * the picker takes focus (the editor keeps its caret); ``onPick`` inserts. */ +export function EmojiButton({ + onOpen, + onPick, + className, +}: { + onOpen: () => void; + onPick: (emoji: string) => void; + className: string; +}) { + const t = useT(); + const { resolved } = useTheme(); + const btnRef = useRef(null); + const popRef = useRef(null); + const [pos, setPos] = useState<{ left: number; top: number } | null>(null); + + const open = () => { + const r = btnRef.current?.getBoundingClientRect(); + if (!r) return; + onOpen(); + // Below the button, or above it when the window ends first; inside the viewport. + const below = r.bottom + 6 + PICKER_H <= window.innerHeight; + setPos({ + left: Math.max(8, Math.min(r.left, window.innerWidth - PICKER_W - 8)), + top: below ? r.bottom + 6 : Math.max(8, r.top - 6 - PICKER_H), + }); + }; + + useEffect(() => { + if (!pos) return; + const onDown = (e: MouseEvent) => { + const target = e.target as Node; + if (!popRef.current?.contains(target) && !btnRef.current?.contains(target)) setPos(null); + }; + const onKey = (e: KeyboardEvent) => { + if (e.key === "Escape") { + e.stopPropagation(); // the picker, not the compose window + setPos(null); + } + }; + document.addEventListener("mousedown", onDown); + window.addEventListener("keydown", onKey, true); + return () => { + document.removeEventListener("mousedown", onDown); + window.removeEventListener("keydown", onKey, true); + }; + }, [pos]); + + return ( + <> + + {pos && + createPortal( +
    + { + onPick(native); + setPos(null); + }} + /> +
    , + document.body, + )} + + ); +} diff --git a/frontend/src/components/compose/RichTextEditor.tsx b/frontend/src/components/compose/RichTextEditor.tsx index e8809d7..bbad126 100644 --- a/frontend/src/components/compose/RichTextEditor.tsx +++ b/frontend/src/components/compose/RichTextEditor.tsx @@ -11,6 +11,17 @@ import { import { useT } from "../../i18n"; import { api, ApiError } from "../../lib/api"; +import { notify } from "../AppDialog"; +import { EmojiButton } from "./EmojiButton"; +import { + GHOST, + acceptGhost, + caretContext, + ghostEl, + removeGhost, + showGhost, + typedIntoGhost, +} from "./completion"; import { canHighlight, clearPaint, @@ -94,9 +105,10 @@ function addQuoteToggle(el: HTMLElement, title: string) { } function contentHtml(el: HTMLElement): string { - if (!el.querySelector(`.${QUOTE_TOGGLE}`)) return el.innerHTML; + // The quote pill and a suggestion's grey text are the editor's, not the message's. + if (!el.querySelector(`.${QUOTE_TOGGLE}, .${GHOST}`)) return el.innerHTML; const copy = el.cloneNode(true) as HTMLElement; - copy.querySelectorAll(`.${QUOTE_TOGGLE}`).forEach((n) => n.remove()); + copy.querySelectorAll(`.${QUOTE_TOGGLE}, .${GHOST}`).forEach((n) => n.remove()); return copy.innerHTML; } @@ -247,6 +259,70 @@ export const RichTextEditor = forwardRef< const [spellPos, setSpellPos] = useState<{ left: number; top: number } | null>(null); const spellPopupRef = useRef(null); + // Sentence completion (./completion.ts): grey text after the caret, Tab takes it. + const completionOn = settings.data?.completion_enabled ?? true; + const completionOnRef = useRef(completionOn); + completionOnRef.current = completionOn; + const completionSeq = useRef(0); + const completionTimer = useRef(null); + const composing = useRef(false); // an input method is building a character + const scheduleCompletion = () => { + const el = ref.current; + if (completionTimer.current) window.clearTimeout(completionTimer.current); + const seq = ++completionSeq.current; + if (!el || !completionOnRef.current || ghostEl(el)) return; + completionTimer.current = window.setTimeout(async () => { + if (composing.current || document.activeElement !== el) return; + const at = caretContext(el); + if (!at) return; + let suggestion: string | null = null; + try { + suggestion = (await api.complete(at.before)).suggestion; + } catch { + return; // offline, or the server says no: simply no suggestion + } + // Still where it was asked for? (The user may have typed on, or moved.) + const now = caretContext(el); + if (!suggestion || seq !== completionSeq.current || !now || now.before !== at.before) return; + removeGhost(el); + showGhost(now.range, suggestion, "Tab"); + }, 300); + }; + const dropGhost = () => { + completionSeq.current++; + if (ref.current) removeGhost(ref.current); + }; + + // Native input events the ghost answers to: typing over it, leaving, IME. + useEffect(() => { + const el = ref.current; + if (!el) return; + const onBeforeInput = (e: InputEvent) => { + if (typedIntoGhost(el, e)) completionSeq.current++; // kept: no new request + }; + const onCompositionStart = () => { + composing.current = true; + dropGhost(); + }; + const onCompositionEnd = () => { + composing.current = false; + }; + el.addEventListener("beforeinput", onBeforeInput); + el.addEventListener("blur", dropGhost); + el.addEventListener("mousedown", dropGhost); + el.addEventListener("compositionstart", onCompositionStart); + el.addEventListener("compositionend", onCompositionEnd); + return () => { + el.removeEventListener("beforeinput", onBeforeInput); + el.removeEventListener("blur", dropGhost); + el.removeEventListener("mousedown", dropGhost); + el.removeEventListener("compositionstart", onCompositionStart); + el.removeEventListener("compositionend", onCompositionEnd); + if (completionTimer.current) window.clearTimeout(completionTimer.current); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + // A reply's quoted original starts folded away behind the "•••" pill. const [quoteOpen, setQuoteOpen] = useState(false); @@ -274,6 +350,7 @@ export const RichTextEditor = forwardRef< replaceSignature: (html: string) => { const el = ref.current; if (!el) return; + removeGhost(el); const sig = el.querySelector(".wmc-signature"); if (html) { if (sig) sig.innerHTML = html; @@ -285,6 +362,7 @@ export const RichTextEditor = forwardRef< insertSignature: (html: string) => { const el = ref.current; if (!el || !html) return; + removeGhost(el); el.querySelector(".wmc-signature")?.remove(); const spacerBefore = document.createElement("div"); spacerBefore.innerHTML = "
    "; @@ -304,6 +382,7 @@ export const RichTextEditor = forwardRef< insertHtml: (html: string) => { const el = ref.current; if (!el || !html) return; + removeGhost(el); el.focus(); // Insert at the current caret; if that fails (no selection in the editor), // append before the signature/quoted block. @@ -430,6 +509,7 @@ export const RichTextEditor = forwardRef< }; const exec = (command: string, value?: string) => { + dropGhost(); ref.current?.focus(); document.execCommand(command, false, value); emit(); @@ -481,7 +561,7 @@ export const RichTextEditor = forwardRef< const insertImage = (file: File) => { if (file.size > 5 * 1024 * 1024) { - window.alert(t("compose.imageTooLarge")); + notify(t("compose.imageTooLarge"), 5000); return; } const reader = new FileReader(); @@ -771,6 +851,23 @@ export const RichTextEditor = forwardRef< }; const onEditorKeyDown = (e: React.KeyboardEvent) => { + const el = ref.current; + if (el && ghostEl(el)) { + if (e.key === "Tab" && !e.shiftKey && !e.altKey && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + if (acceptGhost(el)) emit(); + return; + } + if (e.key === "Escape") { + e.preventDefault(); + e.stopPropagation(); // the suggestion, not the compose window + dropGhost(); + return; + } + // Moving, deleting, Enter…: the suggestion no longer fits. Typing is + // handled by beforeinput (it may shorten the suggestion instead). + if (e.key.length > 1 && e.key !== "Shift" && e.key !== "Dead") dropGhost(); + } // Typing (or Escape) in the message puts the link popup away. Escape stops // here: the mail view's own Escape would otherwise minimise the whole window. if (spellIssue) { @@ -926,6 +1023,15 @@ export const RichTextEditor = forwardRef< e.target.value = ""; }} /> + { + ref.current?.focus(); + restoreSelection(); + exec("insertText", emoji); + }} + /> exec("justifyLeft")} @@ -964,6 +1070,7 @@ export const RichTextEditor = forwardRef< onInput={(e) => { onInput(contentHtml(e.currentTarget)); scheduleSpellcheck(); + scheduleCompletion(); }} // Our own checker replaces the browser's, rather than doubling it up. spellCheck={!spellOn} diff --git a/frontend/src/components/compose/SaveTemplateDialog.tsx b/frontend/src/components/compose/SaveTemplateDialog.tsx new file mode 100644 index 0000000..783f388 --- /dev/null +++ b/frontend/src/components/compose/SaveTemplateDialog.tsx @@ -0,0 +1,69 @@ +import { useState } from "react"; + +import { useT } from "../../i18n"; +import { ApiError } from "../../lib/api"; +import type { MailTemplate } from "../../lib/types"; +import { AppDialog, DialogButtons } from "../AppDialog"; + +/** Compose → More → Templates → Save as template: name it, save it. A name + * already taken turns the button into "Replace" rather than asking again. */ +export function SaveTemplateDialog({ + initialName, + existing, + onSave, + onClose, +}: { + initialName: string; + existing: MailTemplate[]; + onSave: (name: string) => Promise; + onClose: () => void; +}) { + const t = useT(); + const [name, setName] = useState(initialName); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(""); + const trimmed = name.trim(); + const taken = existing.some((x) => x.name.toLowerCase() === trimmed.toLowerCase()); + + const submit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!trimmed || saving) return; + setSaving(true); + setError(""); + try { + await onSave(trimmed); + onClose(); + } catch (err) { + setError(err instanceof ApiError ? err.message : t("compose.templateSaveFailed")); + setSaving(false); + } + }; + + return ( + +
    +

    {t("compose.saveTemplateHint")}

    + + {taken &&

    {t("compose.templateReplaceNote")}

    } + {error &&

    {error}

    } + + +
    + ); +} diff --git a/frontend/src/components/compose/completion.ts b/frontend/src/components/compose/completion.ts new file mode 100644 index 0000000..9841a87 --- /dev/null +++ b/frontend/src/components/compose/completion.ts @@ -0,0 +1,110 @@ +// Sentence completion in the compose editor: the grey "ghost" text after the +// caret (suggested by /api/completion, app/completion.py) that Tab accepts. +// +// The ghost is a non-editable span right after the caret, so it wraps with the +// paragraph like real text. It is never part of the message: contentHtml() +// leaves it out, the spell checker skips it, and it goes on any other key. + +export const GHOST = "wmc-ghost"; + +// Where a suggestion would not belong. +const NO_GHOST = ".wmc-signature, .wmc-quoted, .wmc-quote-toggle, blockquote, a, pre, code"; +const BLOCKS = "div, p, li, h1, h2, h3"; + +export function ghostEl(root: HTMLElement): HTMLElement | null { + return root.querySelector(`.${GHOST}`); +} + +export function removeGhost(root: HTMLElement): void { + root.querySelectorAll(`.${GHOST}`).forEach((n) => n.remove()); +} + +/** The current line's text up to the caret, when a suggestion may be shown + * there: a collapsed caret at the end of its line, outside signature, quote + * and links. Null otherwise. */ +export function caretContext(root: HTMLElement): { before: string; range: Range } | null { + const sel = window.getSelection(); + if (!sel || sel.rangeCount === 0 || !sel.isCollapsed) return null; + const range = sel.getRangeAt(0); + if (!root.contains(range.startContainer)) return null; + const node = range.startContainer; + const el = node instanceof Element ? node : node.parentElement; + if (!el || el.closest(NO_GHOST)) return null; + const block = el.closest(BLOCKS); + const scope = block && block !== root && root.contains(block) ? block : root; + + // Only at the end of the line: nothing but whitespace after the caret. + const after = document.createRange(); + after.selectNodeContents(scope); + after.setStart(range.startContainer, range.startOffset); + if (after.toString().replace(/[\s\u200b]/g, "")) return null; + + // The text before the caret, a
    counting as a line break. + let text = ""; + const walker = document.createTreeWalker(scope, NodeFilter.SHOW_TEXT | NodeFilter.SHOW_ELEMENT); + for (let n: Node | null = walker.nextNode(); n; n = walker.nextNode()) { + if (n === range.startContainer && n.nodeType === Node.TEXT_NODE) { + text += (n as Text).data.slice(0, range.startOffset); + break; + } + if (n.nodeType === Node.ELEMENT_NODE && n === range.startContainer) { + // The caret sits between children: take those before it. + const kids = Array.from(n.childNodes).slice(0, range.startOffset); + text += kids.map((k) => k.textContent ?? "").join(""); + break; + } + if (range.comparePoint(n, 0) > 0) break; // past the caret + if (n.nodeType === Node.TEXT_NODE) text += (n as Text).data; + else if ((n as Element).tagName === "BR") text += "\n"; + } + const line = (text.split("\n").pop() ?? "").replace(/\u00a0/g, " "); + if (!line.trim()) return null; + return { before: line, range: range.cloneRange() }; +} + +/** Show ``text`` in grey right after the caret (``range``), the caret kept before it. */ +export function showGhost(range: Range, text: string, hint: string): void { + const span = document.createElement("span"); + span.className = GHOST; + span.setAttribute("contenteditable", "false"); + span.setAttribute("aria-hidden", "true"); + span.dataset.hint = hint; // "Tab", drawn after the text by CSS + span.textContent = text; + range.insertNode(span); + const caret = document.createRange(); + caret.setStartBefore(span); + caret.collapse(true); + const sel = window.getSelection(); + sel?.removeAllRanges(); + sel?.addRange(caret); +} + +/** Take the suggestion: real, undoable text where the ghost was. */ +export function acceptGhost(root: HTMLElement): boolean { + const ghost = ghostEl(root); + const text = ghost?.textContent ?? ""; + if (!ghost || !text) return false; + const caret = document.createRange(); + caret.setStartBefore(ghost); + caret.collapse(true); + ghost.remove(); + const sel = window.getSelection(); + sel?.removeAllRanges(); + sel?.addRange(caret); + document.execCommand("insertText", false, text); + return true; +} + +/** Typing what the ghost shows shortens it ("hea" → "ring from you") rather + * than throwing it away. True when the ghost stays. */ +export function typedIntoGhost(root: HTMLElement, e: InputEvent): boolean { + const ghost = ghostEl(root); + if (!ghost) return false; + const shown = ghost.textContent ?? ""; + if (e.inputType === "insertText" && e.data && shown.length > e.data.length && shown.startsWith(e.data)) { + ghost.textContent = shown.slice(e.data.length); + return true; + } + ghost.remove(); + return false; +} diff --git a/frontend/src/components/compose/restore.ts b/frontend/src/components/compose/restore.ts new file mode 100644 index 0000000..b1a7dd2 --- /dev/null +++ b/frontend/src/components/compose/restore.ts @@ -0,0 +1,77 @@ +// Open compose windows outlive leaving the mail view (Settings, a plugin page) +// and a reload: kept for this tab in sessionStorage, and put back when the mail +// view opens again. +// +// Two parts: the window list (which windows, minimized or expanded), written as +// it changes; and each window's content, written by its Composer as it goes +// away (with the latest text, which autosave may not have reached yet). Only a +// window with content comes back: one that was sent or discarded leaves none. + +import type { ComposerInit } from "./Composer"; +import type { ComposeWin } from "./windows"; + +const LIST = "compose.windows"; +const SNAP = "compose.snap."; + +type Listed = Pick; + +function read(key: string): T | null { + try { + const raw = sessionStorage.getItem(key); + return raw ? (JSON.parse(raw) as T) : null; + } catch { + return null; // storage blocked or garbled: nothing to restore + } +} + +function write(key: string, value: unknown): void { + try { + sessionStorage.setItem(key, JSON.stringify(value)); + } catch { + /* storage blocked or full: the windows just won't come back */ + } +} + +export function saveWindowList(wins: ComposeWin[]): void { + write( + LIST, + wins.map(({ id, minimized, expanded, activated }) => ({ id, minimized, expanded, activated })), + ); +} + +/** A window's content as it goes away (called by its Composer). */ +export function saveSnapshot(id: string, init: ComposerInit): void { + write(SNAP + id, init); +} + +/** A save that finished after its window went away: the kept content learns + * its draft id, so the window brought back updates that draft instead of + * starting a second one. */ +export function noteSnapshotDraft(id: string, draftId: string): void { + const kept = read(SNAP + id); + if (kept && !kept.draftId) write(SNAP + id, { ...kept, draftId }); +} + +/** The windows to reopen, each with its last content. Reads only: React may + * call a state initializer twice (StrictMode), and both must see the same. */ +export function restoredWindows(): ComposeWin[] { + const listed = read(LIST) ?? []; + const out: ComposeWin[] = []; + for (const w of listed) { + const init = read(SNAP + w.id); + if (init) out.push({ ...w, init }); + } + return out; +} + +/** Drop the kept contents once the windows are open again (they keep their own). */ +export function clearSnapshots(): void { + try { + for (let i = sessionStorage.length - 1; i >= 0; i--) { + const key = sessionStorage.key(i); + if (key?.startsWith(SNAP)) sessionStorage.removeItem(key); + } + } catch { + /* nothing to clear */ + } +} diff --git a/frontend/src/components/compose/spellcheck.ts b/frontend/src/components/compose/spellcheck.ts index 069ea8c..caf3549 100644 --- a/frontend/src/components/compose/spellcheck.ts +++ b/frontend/src/components/compose/spellcheck.ts @@ -22,7 +22,7 @@ export interface PlacedIssue extends SpellIssue { } // The signature and the quoted original are not the user's writing to check. -const SKIP = ".wmc-signature, .wmc-quoted, .wmc-quote-toggle"; +const SKIP = ".wmc-signature, .wmc-quoted, .wmc-quote-toggle, .wmc-ghost"; const BLOCK = /^(DIV|P|LI|UL|OL|BLOCKQUOTE|H[1-6]|TABLE|TR|PRE|SECTION)$/; /** diff --git a/frontend/src/components/list/MessageList.tsx b/frontend/src/components/list/MessageList.tsx index aa83d97..2b0337d 100644 --- a/frontend/src/components/list/MessageList.tsx +++ b/frontend/src/components/list/MessageList.tsx @@ -10,6 +10,7 @@ import { useSavedThreads } from "../../lib/offline"; import { formatClock, getDateLocale } from "../../lib/timeFormat"; import type { Address, ThreadSummary } from "../../lib/types"; import { Icon } from "../Icon"; +import { Clock } from "../Offline"; import { AvatarImage } from "../SenderAvatar"; import { LabelChips } from "../mail/LabelChips"; @@ -22,6 +23,17 @@ function timeLabel(iso: string): string { return d.toLocaleDateString(getDateLocale(), { year: "2-digit", month: "short", day: "numeric" }); } +// A row's date: when the conversation comes back, while it is snoozed. +function RowDate({ thread }: { thread: ThreadSummary }) { + if (!thread.snoozed_until) return <>{timeLabel(thread.last_message_at)}; + return ( + + + {timeLabel(thread.snoozed_until)} + + ); +} + // Stable per-sender hue so a sender's avatar keeps a consistent colour. function initials(p: Address | undefined): string { const name = (p?.name || p?.email || "?").trim(); @@ -354,7 +366,7 @@ export function MessageList({ unread ? "font-bold" : "" }`} > - {timeLabel(thread.last_message_at)} + {/* A narrow desktop list: the actions take the date's place on hover. */} @@ -505,7 +517,7 @@ export function MessageList({ unread ? "font-bold" : "" }`} > - {timeLabel(thread.last_message_at)} +
    diff --git a/frontend/src/components/list/SubscriptionsList.tsx b/frontend/src/components/list/SubscriptionsList.tsx new file mode 100644 index 0000000..365e96c --- /dev/null +++ b/frontend/src/components/list/SubscriptionsList.tsx @@ -0,0 +1,527 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useState } from "react"; + +import { useT } from "../../i18n"; +import { api } from "../../lib/api"; +import { avatarColor } from "../../lib/avatarColor"; +import { useListDividers } from "../../lib/listDividers"; +import { formatClock, getDateLocale } from "../../lib/timeFormat"; +import type { Subscription, SubscriptionList } from "../../lib/types"; +import { useConfirm } from "../AppDialog"; +import { EmptyIllustration } from "../EmptyIllustration"; +import { Icon } from "../Icon"; +import { AvatarImage } from "../SenderAvatar"; + +export function dateLabel(iso: string): string { + const d = new Date(iso); + const now = new Date(); + if (d.toDateString() === now.toDateString()) return formatClock(d); + if (d.getFullYear() === now.getFullYear()) + return d.toLocaleDateString(getDateLocale(), { month: "short", day: "numeric" }); + return d.toLocaleDateString(getDateLocale(), { year: "2-digit", month: "short", day: "numeric" }); +} + +export function initialsOf(name: string): string { + const words = name.trim().split(/\s+/).filter(Boolean); + if (words.length >= 2) return (words[0][0] + words[words.length - 1][0]).toUpperCase(); + return (name.trim() || "?").slice(0, 2).toUpperCase(); +} + +function toast(text: string) { + window.dispatchEvent(new CustomEvent("webmail:toast", { detail: { text, ms: 5000 } })); +} + +/** What to unsubscribe from: a sender group, or one message's own header. */ +export interface UnsubscribeTarget { + key: string; + name: string; + via: "one_click" | "mailto" | "web"; + messageId?: string; +} + +export function targetOf(sub: Subscription): UnsubscribeTarget { + return { + key: sub.key, + name: sub.name, + via: sub.one_click ? "one_click" : sub.has_mailto ? "mailto" : "web", + }; +} + +/** Unsubscribe and delete-all, with their confirmations, for the list, the + * sender pane and a message in the reading pane. */ +export function useSubscriptionActions() { + const t = useT(); + const queryClient = useQueryClient(); + const [confirm, dialog] = useConfirm(); + const refresh = () => { + void queryClient.invalidateQueries({ queryKey: ["subscriptions"] }); + }; + + const unsubscribeRun = useMutation({ + mutationFn: async (target: UnsubscribeTarget) => { + // Opened now, while the click on Unsubscribe still counts as the user's: + // a page opened after the request returns would be stopped as a pop-up. + const page = target.via === "web" ? window.open("", "_blank") : null; + const result = await api.unsubscribe(target.key, target.messageId); + if (result.status === "open" && result.url) { + if (page) page.location.href = result.url; + else window.open(result.url, "_blank", "noopener"); + } else page?.close(); + return result; + }, + onSuccess: (result, sub) => { + if (!result) return; + refresh(); + void queryClient.invalidateQueries({ queryKey: ["thread"] }); // the reading pane's link + toast( + result.method === "web" + ? t("subs.doneWeb") + : result.method === "mailto" + ? t("subs.doneMailto", { name: sub.name }) + : t("subs.doneOneClick", { name: sub.name }), + ); + }, + onError: () => toast(t("subs.failed")), + }); + + const trashRun = useMutation({ + mutationFn: (sub: Subscription) => api.trashSubscription(sub.key), + onSuccess: (result) => { + if (!result) return; + toast(t("subs.trashed", { count: result.moved })); + // The moves land one by one in the worker; look again once they have. + window.setTimeout(() => { + refresh(); + void queryClient.invalidateQueries({ queryKey: ["threads"] }); + }, 2500); + }, + }); + + // Asked first, in our own dialog; the request (and its busy state) only after. + const unsubscribe = { + ...unsubscribeRun, + mutate: async (target: UnsubscribeTarget) => { + const ok = await confirm({ + title: t("subs.confirm", { name: target.name }), + message: + target.via === "one_click" + ? t("subs.confirmOneClick") + : target.via === "mailto" + ? t("subs.confirmMailto") + : t("subs.confirmWeb"), + confirmLabel: t("subs.unsubscribe"), + }); + if (ok) unsubscribeRun.mutate(target); + }, + }; + const trash = { + ...trashRun, + mutate: async (sub: Subscription) => { + const ok = await confirm({ + title: t("subs.trashAll"), + message: t("subs.confirmTrash", { name: sub.name }), + confirmLabel: t("subs.trashAllAction"), + danger: true, + }); + if (ok) trashRun.mutate(sub); + }, + }; + + /** Render `dialog` wherever the hook is used: it is the confirmation. */ + return { unsubscribe, trash, dialog }; +} + +// Requests in flight at once during a bulk unsubscribe: a one-click request can +// take seconds, and twenty of them one after another would be slow. +const BULK_PARALLEL = 4; + +async function eachLimited(items: T[], run: (item: T) => Promise): Promise { + let next = 0; + const worker = async () => { + while (next < items.length) await run(items[next++]); + }; + await Promise.all(Array.from({ length: Math.min(BULK_PARALLEL, items.length) }, worker)); +} + +const isDone = (sub: Subscription) => sub.unsubscribed_at !== null && !sub.still_sending; + +/** Unsubscribe from / delete the mail of several senders at once (the + * Subscriptions folder's checkboxes). Senders that can only be left on their + * own web page are collected for the user to open one by one: a browser opens + * one new tab per click, not twenty. */ +export function useBulkSubscriptionActions(onDone: () => void) { + const t = useT(); + const queryClient = useQueryClient(); + const [confirm, dialog] = useConfirm(); + const [running, setRunning] = useState(false); + const [finishOnPage, setFinishOnPage] = useState<{ name: string; url: string }[]>([]); + + const unsubscribeMany = async (subs: Subscription[]) => { + const todo = subs.filter((s) => !isDone(s)); + if (todo.length === 0) { + toast(t("subs.bulkNothing")); + return; + } + const ok = await confirm({ + title: t("subs.bulkConfirm", { count: todo.length }), + message: t("subs.bulkHow"), + confirmLabel: t("subs.unsubscribe"), + }); + if (!ok) return; + setRunning(true); + let done = 0; + let failed = 0; + const pages: { name: string; url: string }[] = []; + await eachLimited(todo, async (sub) => { + try { + const r = await api.unsubscribe(sub.key); + if (r.status === "open" && r.url) pages.push({ name: sub.name, url: r.url }); + else done += 1; + } catch { + failed += 1; + } + }); + setRunning(false); + void queryClient.invalidateQueries({ queryKey: ["subscriptions"] }); + void queryClient.invalidateQueries({ queryKey: ["thread"] }); + toast( + [ + t("subs.bulkDone", { count: done }), + pages.length ? t("subs.bulkPages", { count: pages.length }) : "", + failed ? t("subs.bulkFailed", { count: failed }) : "", + ] + .filter(Boolean) + .join(" · "), + ); + setFinishOnPage(pages); + onDone(); + }; + + const trashMany = async (subs: Subscription[]) => { + if (subs.length === 0) return; + const ok = await confirm({ + title: t("subs.trashAll"), + message: t("subs.bulkTrashConfirm", { count: subs.length }), + confirmLabel: t("subs.trashAllAction"), + danger: true, + }); + if (!ok) return; + setRunning(true); + let moved = 0; + await eachLimited(subs, async (sub) => { + try { + moved += (await api.trashSubscription(sub.key)).moved; + } catch { + /* the others still go */ + } + }); + setRunning(false); + toast(t("subs.trashed", { count: moved })); + onDone(); + // The moves land one by one in the worker; look again once they have. + window.setTimeout(() => { + void queryClient.invalidateQueries({ queryKey: ["subscriptions"] }); + void queryClient.invalidateQueries({ queryKey: ["threads"] }); + }, 2500); + }; + + return { + unsubscribeMany, + trashMany, + running, + finishOnPage, + closeFinish: () => setFinishOnPage([]), + dialog, // render it: the confirmation + }; +} + +/** After a bulk unsubscribe: the senders to finish on their own page. */ +export function FinishOnPageDialog({ + items, + onClose, +}: { + items: { name: string; url: string }[]; + onClose: () => void; +}) { + const t = useT(); + const [opened, setOpened] = useState>(new Set()); + return ( +     + ); +} + +/** The two bulk buttons, for the reading pane's bar (or above the list on a phone). */ +export function SubscriptionBulkButtons({ + selected, + bulk, +}: { + selected: Subscription[]; + bulk: ReturnType; +}) { + const t = useT(); + const cls = + "wm-online-only flex items-center gap-1.5 rounded px-2.5 py-1 text-app-muted hover:bg-app-hover hover:text-app-fg disabled:opacity-50"; + return ( + <> + + + {bulk.running && ( + + )} + + ); +} + +function CheckBox({ checked, partial = false }: { checked: boolean; partial?: boolean }) { + return ( + + {checked ? ( + + + + ) : partial ? ( + + + + ) : null} + + ); +} + +/** One line of figures about a sender: how much, how often, how much is read. */ +export function SubscriptionStats({ sub }: { sub: Subscription }) { + const t = useT(); + return ( + + {t(sub.total === 1 ? "subs.email" : "subs.emails", { count: sub.total })} + {" · "} + {t("subs.perMonth", { count: sub.per_month })} + {sub.unread > 0 && ` · ${t("subs.unread", { count: sub.unread })}`} + + ); +} + +export function SubscriptionStatus({ sub }: { sub: Subscription }) { + const t = useT(); + if (sub.still_sending) + return ( + + {t("subs.stillSending")} + + ); + if (sub.unsubscribed_at) + return ( + + {t("subs.unsubscribed")} + + ); + return null; +} + +/** The Subscriptions folder in the list panel: one row per bulk sender. */ +export function SubscriptionsList({ + data, + loading, + selectedKey, + onSelect, + checkedKeys, + onToggleCheck, + busyKeys, + compact = false, +}: { + data: SubscriptionList | undefined; + loading: boolean; + selectedKey: string | null; + onSelect: (key: string) => void; + /** Checked rows (bulk actions); a row's avatar turns into its checkbox. */ + checkedKeys: Set; + onToggleCheck: (key: string) => void; + /** Rows a bulk action is working on (dimmed). */ + busyKeys?: Set; + compact?: boolean; +}) { + const t = useT(); + const { unsubscribe, trash, dialog } = useSubscriptionActions(); + const subs = data?.subscriptions ?? []; + const checkMode = checkedKeys.size > 0; + const dividers = useListDividers(); // Mailbox display → dividing lines, as in the mail list + + return ( +
    + {dialog} + {data && data.pending > 0 && ( +
    + + {t("subs.scanning", { count: data.pending })} +
    + )} + {!loading && subs.length === 0 && (data?.pending ?? 0) === 0 && ( +
    + +

    {t("subs.none")}

    +

    {t("subs.noneHint")}

    +
    + )} + {subs.map((sub, index) => { + const selected = sub.key === selectedKey; + const checked = checkedKeys.has(sub.key); + const busy = + busyKeys?.has(sub.key) || + (unsubscribe.isPending && unsubscribe.variables?.key === sub.key) || + (trash.isPending && trash.variables?.key === sub.key); + const done = sub.unsubscribed_at !== null && !sub.still_sending; + return ( +
    onSelect(sub.key)} + onKeyDown={(e) => (e.key === "Enter" || e.key === " ") && onSelect(sub.key)} + className={`group relative cursor-pointer px-2 ${compact ? "pt-3" : "pt-[7px]"} ${busy ? "opacity-50" : ""}`} + > + {dividers && index > 0 && ( + // Midway through the gap above the row, lined up with its content. +
    + )} +
    +
    + {sub.unread > 0 && ( + + )} + {/* The avatar gives way to a checkbox on hover, and everywhere once + anything is checked — as in the mail list. */} + + {initialsOf(sub.name)} + + + +
    +
    +
    + 0 ? "font-black" : ""}`}> + {sub.name} + + + + {dateLabel(sub.last_at)} + + {!compact && ( + + + + + )} +
    +
    {sub.last_subject || t("subs.noSubject")}
    +
    + +
    +
    +
    +
    + ); + })} +
    + ); +} diff --git a/frontend/src/components/mail/LabelChips.tsx b/frontend/src/components/mail/LabelChips.tsx index 1772d68..e212bce 100644 --- a/frontend/src/components/mail/LabelChips.tsx +++ b/frontend/src/components/mail/LabelChips.tsx @@ -1,3 +1,5 @@ +import { translate } from "../../i18n"; + // Small muted pills for a message/thread's labels, shown before the subject in the // list and after the subject in the reading pane. With `onRemove`, each pill gets a // clickable ✕ (used in the reading pane to drop a label from the thread). @@ -23,7 +25,7 @@ export function LabelChips({ {onRemove && ( + ))} +
    + {custom ? ( +
    + setValue(e.target.value)} + autoFocus + /> +
    + + +
    +
    + ) : ( + + )} +
    + ); +} diff --git a/frontend/src/components/mail/SubscriptionPane.tsx b/frontend/src/components/mail/SubscriptionPane.tsx new file mode 100644 index 0000000..492a2bb --- /dev/null +++ b/frontend/src/components/mail/SubscriptionPane.tsx @@ -0,0 +1,151 @@ +import { useQuery } from "@tanstack/react-query"; + +import { useT } from "../../i18n"; +import { api } from "../../lib/api"; +import { avatarColor } from "../../lib/avatarColor"; +import type { Subscription } from "../../lib/types"; +import { + dateLabel, + initialsOf, + SubscriptionStatus, + targetOf, + useSubscriptionActions, +} from "../list/SubscriptionsList"; +import { Icon } from "../Icon"; +import { AvatarImage } from "../SenderAvatar"; + +/** The reading pane for a sender picked in Subscriptions: figures, actions, its mail. */ +export function SubscriptionPane({ + sub, + onOpenThread, + barClass, + onBack, +}: { + sub: Subscription; + onOpenThread: (id: string) => void; + barClass: string; + onBack?: () => void; +}) { + const t = useT(); + const { unsubscribe, trash, dialog } = useSubscriptionActions(); + const threads = useQuery({ + queryKey: ["subscription-threads", sub.key, sub.total], + queryFn: async () => { + const ids = await api.subscriptionThreads(sub.key); + return ids.length ? (await api.threadsByIds(ids)).threads : []; + }, + }); + const done = sub.unsubscribed_at !== null && !sub.still_sending; + const stat = (label: string, value: string) => ( +
    +
    {label}
    +
    {value}
    +
    + ); + const readShare = sub.total ? Math.round(((sub.total - sub.unread) / sub.total) * 100) : 0; + + return ( +
    + {dialog} +
    + {onBack && ( + + )} + + +
    +
    +
    +
    + + {initialsOf(sub.name)} + + +
    +
    +

    {sub.name}

    + +
    +
    {sub.email}
    + {sub.list_id &&
    {t("subs.list", { id: sub.list_id })}
    } +
    +
    + + {sub.still_sending && ( +

    + {t("subs.stillSendingHint")} +

    + )} + {sub.unsubscribed_at && !sub.still_sending && ( +

    + {t("subs.unsubscribedOn", { date: new Date(sub.unsubscribed_at).toLocaleDateString() })} +

    + )} + +
    + {stat(t("subs.statTotal"), String(sub.total))} + {stat(t("subs.statPerMonth"), String(sub.per_month))} + {stat(t("subs.statRead"), `${readShare}%`)} + {stat(t("subs.statSince"), dateLabel(sub.first_at))} +
    + +
    +

    + {t("subs.recent")} +

    +
      + {(threads.data ?? []).map((th) => ( +
    • + +
      • + ))} + {threads.isLoading &&
    • {t("mail.loadingThread")}
      • } +
    +
    +
    +
    +
    + ); +} diff --git a/frontend/src/components/thread/MessageBody.tsx b/frontend/src/components/thread/MessageBody.tsx index b1dc3b4..c078ab3 100644 --- a/frontend/src/components/thread/MessageBody.tsx +++ b/frontend/src/components/thread/MessageBody.tsx @@ -1,7 +1,7 @@ -import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { keepPreviousData, useQuery, useQueryClient } from "@tanstack/react-query"; import { useCallback, useEffect, useRef, useState } from "react"; -import { useT } from "../../i18n"; +import { languageName, sameLanguage, splitLanguages, useLocale, useT } from "../../i18n"; import { api } from "../../lib/api"; import { adaptForDarkMode, useDarkMessages, withDarkPage } from "../../lib/darkMessage"; import { fixInvisibleText } from "../../lib/readability"; @@ -19,11 +19,20 @@ import { Attachments } from "./Attachments"; export function MessageBody({ message, onMailto, + translateTo = null, + onLanguage, + onShowOriginal, }: { message: MessageEnvelope; onMailto?: (email: string) => void; + /** Show it translated into this language (app/translation.py), or as sent. */ + translateTo?: string | null; + /** The message's own language, once the server knows it. */ + onLanguage?: (code: string) => void; + onShowOriginal?: () => void; }) { const t = useT(); + const { locale } = useLocale(); const queryClient = useQueryClient(); // Seed from the server's trusted-sender flag so an already-approved sender // loads images and shows no "blocked" prompt on reopen. @@ -42,18 +51,52 @@ export function MessageBody({ const mobileRef = useRef(false); mobileRef.current = useIsMobile(); - const { data, isLoading, isError, error } = useQuery({ - queryKey: ["message-html", message.id, allowed], + const { data, isLoading, isError, error, isPlaceholderData } = useQuery({ + queryKey: ["message-html", message.id, allowed, translateTo], queryFn: async () => { // ?images=1 asks the server to render remote images for this one request // (the "Load images" one-off); "Always from this sender" persists separately. - const url = `/api/messages/${message.id}/html${allowed ? "?images=1" : ""}`; + const params = new URLSearchParams(); + if (allowed) params.set("images", "1"); + if (translateTo) params.set("translate", translateTo); + const query = params.toString(); + const url = `/api/messages/${message.id}/html${query ? `?${query}` : ""}`; const resp = await fetch(url, { credentials: "include" }); if (resp.headers.has("X-WM-Offline") && !resp.ok) throw new Error("offline"); - if (!resp.ok) throw new Error("failed to load message"); - return resp.text(); + if (!resp.ok) { + if (translateTo) throw new Error("translate"); + throw new Error("failed to load message"); + } + // Tracking pixels the server took out of the HTML (sanitize.py). + const trackers = Number(resp.headers.get("X-Trackers-Blocked")) || 0; + return { + html: await resp.text(), + trackers, + language: resp.headers.get("X-Content-Language") ?? "", + translatedFrom: resp.headers.get("X-Translated-From"), + }; }, + staleTime: translateTo ? Infinity : 0, // a translation does not change + // While a translation (or the images) loads, the message stays as it was. + placeholderData: keepPreviousData, }); + const translating = translateTo !== null && isPlaceholderData; + // "Translated from Chinese, English": the languages it had besides the one shown. + const translatedFromLabel = (from: string) => { + const names = splitLanguages(from) + .filter((code) => !translateTo || !sameLanguage(code, translateTo)) + .map((code) => languageName(code, locale)); + if (names.length === 0) return t("mail.translated"); + if (names.length === 1) return t("mail.translatedFrom", { language: names[0] }); + return t("mail.translatedFromMany", { languages: names.join(", ") }); + }; + // Tell the header which language this is (for its Translate button). + const reportedLanguage = data?.language; + const onLanguageRef = useRef(onLanguage); + onLanguageRef.current = onLanguage; + useEffect(() => { + if (reportedLanguage) onLanguageRef.current?.(reportedLanguage); + }, [reportedLanguage]); const resize = useCallback(() => { const frame = frameRef.current; @@ -247,6 +290,14 @@ export function MessageBody({ return (
    + {translating && ( +
    + )} {message.has_remote_images && !allowed && (
    {t("mail.imagesBlocked")} @@ -258,8 +309,37 @@ export function MessageBody({
    )} + {data?.translatedFrom && ( +
    + {translatedFromLabel(data.translatedFrom)} + +
    + )} + {isError && error.message === "translate" && ( +
    + {t("mail.translateFailed")} + +
    + )} + {data !== undefined && data.trackers > 0 && ( + // A darker strip of its own, like the amber one about images above it. +
    + + + + + {data.trackers === 1 ? t("mail.trackerBlocked") : t("mail.trackersBlocked", { count: data.trackers })} +
    + )} {isLoading &&
    {t("mail.loadingThread")}
    } - {isError && ( + {isError && error.message !== "translate" && (
    {error.message === "offline" ? t("offline.notAvailable") : t("mail.loadFailed")}
    @@ -271,7 +351,7 @@ export function MessageBody({ ref={frameRef} title={`message-${message.id}`} sandbox="allow-same-origin" - srcDoc={dark ? withDarkPage(data) : data} + srcDoc={dark ? withDarkPage(data.html) : data.html} onLoad={onLoad} style={{ height }} className={`block w-full rounded-lg border-0 ${dark ? "bg-[#212427]" : "bg-white"}`} diff --git a/frontend/src/components/thread/ThreadView.tsx b/frontend/src/components/thread/ThreadView.tsx index ec6454b..387dd93 100644 --- a/frontend/src/components/thread/ThreadView.tsx +++ b/frontend/src/components/thread/ThreadView.tsx @@ -1,7 +1,8 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Fragment, useEffect, useMemo, useRef, useState } from "react"; -import { useT } from "../../i18n"; +import { languageName, needsTranslation, sameLanguage, translate, useLocale, useT } from "../../i18n"; +import type { MessageKey } from "../../i18n/en"; import { personHue } from "../../lib/avatarColor"; import { setDarkMessages, useDarkMessages } from "../../lib/darkMessage"; import { api } from "../../lib/api"; @@ -17,6 +18,7 @@ import { Icon } from "../Icon"; import { AvatarImage } from "../SenderAvatar"; import { clearPdfPasswordCache } from "./AttachmentPreview"; import { LabelChips } from "../mail/LabelChips"; +import { useSubscriptionActions } from "../list/SubscriptionsList"; import { MessageBody } from "./MessageBody"; // Direction-agnostic thread view (invariant I-5 / SPEC §9.4). @@ -54,13 +56,13 @@ function attributionFor(msg: MessageEnvelope): string { day: "numeric", year: "numeric", }); - const time = d.toLocaleTimeString("en-US", { + const time = d.toLocaleTimeString(getDateLocale(), { hour: "numeric", minute: "2-digit", hour12: getHour12(), }); const who = msg.from_addr[0]?.name || msg.from_addr[0]?.email || ""; - return `On ${date}, at ${time}, ${who} wrote:`; + return translate("compose.attribution", { date, time, who }); } /** The header block above a forwarded message, as mail programs write it: @@ -137,16 +139,16 @@ function hue(msg: MessageEnvelope): number { function fmtRelative(iso: string): string { const d = new Date(iso); - const base = d.toLocaleDateString("en-US", { month: "short", day: "numeric" }); + const base = d.toLocaleDateString(getDateLocale(), { month: "short", day: "numeric" }); const diff = Date.now() - d.getTime(); const mins = Math.round(diff / 60000); const hrs = Math.round(diff / 3600000); const days = Math.round(diff / 86400000); let rel = ""; - if (mins < 1) rel = "just now"; - else if (mins < 60) rel = `${mins} min ago`; - else if (hrs < 24) rel = `${hrs} hour${hrs === 1 ? "" : "s"} ago`; - else if (days < 30) rel = `${days} day${days === 1 ? "" : "s"} ago`; + if (mins < 1) rel = translate("time.justNow"); + else if (mins < 60) rel = translate("time.minAgo", { count: mins }); + else if (hrs < 24) rel = translate("time.hoursAgo", { count: hrs }); + else if (days < 30) rel = translate("time.daysAgo", { count: days }); return rel ? `${base} (${rel})` : `${base} ${d.getFullYear()}`; } @@ -162,23 +164,56 @@ function fmtFull(iso: string): string { }); } +// "Unsubscribe" beside the sender of bulk mail (its List-Unsubscribe header), +// as Gmail has it; "Unsubscribed" once done. +function UnsubscribeLink({ message, name }: { message: MessageEnvelope; name: string }) { + const t = useT(); + const { unsubscribe, dialog } = useSubscriptionActions(); + if (message.unsubscribed) { + return {t("subs.unsubscribed")}; + } + return ( + <> + {dialog} + + + ); +} + function fmtSize(bytes: number): string { if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`; return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; } -const LOCATION_LABEL: Record = { - inbox: "Inbox", - sent: "Sent", - archive: "Archive", - drafts: "Drafts", - trash: "Trash", - junk: "Spam", +// A message's folder, named as in the sidebar (in the interface language). +const LOCATION_LABEL: Record = { + inbox: "nav.inbox", + sent: "nav.sent", + archive: "nav.archive", + drafts: "nav.drafts", + trash: "nav.trash", + junk: "nav.spam", }; function ActionsMenu({ messageId, + mailboxId, onReply, onPrint, onDelete, @@ -187,8 +222,13 @@ function ActionsMenu({ messageLabels, onToggleLabel, onCreateLabel, + translateLanguages, + translateTo, + onTranslateTo, }: { messageId: string; + /** The message's folder: a move gives plugin items a fresh start. */ + mailboxId: string; onReply: (kind: string) => void; onPrint: () => void; onDelete: () => void; @@ -197,10 +237,16 @@ function ActionsMenu({ messageLabels: string[]; onToggleLabel: (label: string) => void; onCreateLabel: (name: string) => void; + /** Languages a message can be translated into; empty: translation is off. */ + translateLanguages: string[]; + translateTo: string | null; + onTranslateTo: (code: string | null) => void; }) { const t = useT(); + const { locale } = useLocale(); const [open, setOpen] = useState(false); const [labelsOpen, setLabelsOpen] = useState(false); + const [translateOpen, setTranslateOpen] = useState(false); const [creating, setCreating] = useState(false); const [newName, setNewName] = useState(""); const ref = useRef(null); @@ -220,7 +266,7 @@ function ActionsMenu({ }; }, [open]); useEffect(() => { - if (!open) { setLabelsOpen(false); setCreating(false); setNewName(""); } + if (!open) { setLabelsOpen(false); setTranslateOpen(false); setCreating(false); setNewName(""); } }, [open]); const item = (label: string, onClick: () => void, disabled = false) => ( @@ -249,79 +295,121 @@ function ActionsMenu({ {t("mail.actions")} ▾ - {open && ( -
    - {item(t("mail.reply"), () => onReply("reply"), replyDisabled)} - {item(t("mail.replyAll"), () => onReply("reply_all"), replyDisabled)} - {item(t("mail.forward"), () => onReply("forward"), replyDisabled)} - {item(t("mail.forwardAsAttachment"), () => onReply("forward"), replyDisabled)} - {item(t("mail.print"), onPrint)} -
    - {/* Labels submenu: multi-select checkboxes (menu stays open) + create new. */} - - {labelsOpen && ( -
    - {allLabels.map((l) => ( - - ))} - {allLabels.length === 0 && ( -
    {t("mail.noLabels")}
    - )} - {creating ? ( + {/* Always rendered, hidden while closed: plugin items (below) load with + the message, not when the menu opens — on a slow connection their + fetch would otherwise hold them back for seconds. */} +