Translation on Firefox Translations models; new logo

- Email translation moves from LibreTranslate to our own translator
  service (translator/): Mozilla's Firefox Translations models on the
  Bergamot engine, fastText language detection (176 languages). Polish,
  English, German, French, Spanish and Italian on first start (~350 MB
  instead of ~10 GB); Admin panel > Email translation installs and
  removes languages. Built and pushed as the `translator` image.
- Every message is translated paragraph by paragraph as plain text;
  names travel with their sentence and are kept only where the model
  changed them; a sentence start the model copied ("Enclosed") is tried
  again in lower case; list markers are not sent
- The Translate button shows only for installed languages
- New logo: favicon, app/maskable/iPhone icons, notification badge,
  email logo and calendar copy regenerated; BIMI logo rebuilt in SVG
  Tiny PS with its gradients; masks render in browsers
- Launch screen background is the app's dark grey

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-18 22:04:04 +02:00
co-authored by Claude Opus 5.5
parent 8a1b9979e9
commit 972e10876b
38 changed files with 1627 additions and 223 deletions
+61 -1
View File
@@ -15,10 +15,12 @@ import re
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Query
from fastapi import APIRouter, Depends, Path, Query
from pydantic import BaseModel
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app import translation
from app.api.errors import raise_api_error
from app.config import get_settings
from app.db import get_session
@@ -332,3 +334,61 @@ async def remove_user(user_id: uuid.UUID, admin: CurrentAdmin, db: DbSession) ->
except MailServerError as exc:
raise_api_error(502, "mail_server_unavailable", f"the mail server did not answer: {exc}")
return {"status": "deleted"}
# --- email translation languages (translator service) ------------------------------
class InstallProgressOut(BaseModel):
done: int
total: int
class AdminTranslationOut(BaseModel):
enabled: bool # TRANSLATOR_URL is set
reachable: bool
installed: list[str] = []
installing: dict[str, InstallProgressOut] = {}
available: list[str] = [] # what Mozilla has models for (empty: offline)
errors: dict[str, str] = {} # why a language's last install failed
_LANG = Path(pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$")
@router.get("/translation", response_model=AdminTranslationOut)
async def translation_status(admin: CurrentAdmin) -> AdminTranslationOut:
if not translation.enabled():
return AdminTranslationOut(enabled=False, reachable=False)
try:
data = await translation.status()
except translation.TranslationError:
return AdminTranslationOut(enabled=True, reachable=False)
return AdminTranslationOut(enabled=True, reachable=True, **data)
@router.post(
"/translation/languages/{code}",
response_model=AdminTranslationOut,
dependencies=[Depends(require_csrf)],
)
async def install_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
"""Download a language's models (to and from English) in the background."""
try:
await translation.install(code)
except translation.TranslationError as exc:
raise_api_error(502, "translator_failed", str(exc))
return await translation_status(admin)
@router.delete(
"/translation/languages/{code}",
response_model=AdminTranslationOut,
dependencies=[Depends(require_csrf)],
)
async def remove_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut:
try:
await translation.remove(code)
except translation.TranslationError as exc:
raise_api_error(502, "translator_failed", str(exc))
return await translation_status(admin)
+1 -1
View File
@@ -25,6 +25,6 @@ class TranslationInfoOut(BaseModel):
async def translation_languages(user: CurrentUser) -> TranslationInfoOut:
found = await translation.languages()
return TranslationInfoOut(
available=bool(found),
available=len(found) > 1, # English alone translates nothing
languages=[LanguageOut(code=x.code, name=x.name) for x in found],
)
+3 -3
View File
@@ -71,9 +71,9 @@ class Settings(BaseSettings):
# Spelling and grammar checking in the compose window (LanguageTool server,
# e.g. http://languagetool:8010). Empty: the check is off for everyone.
languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL")
# Email translation: a LibreTranslate server (app/translation.py), e.g.
# http://libretranslate:5000. Empty: translation is off.
libretranslate_url: str = Field(default="", alias="LIBRETRANSLATE_URL")
# Email translation: the translator service (translator/, app/translation.py),
# e.g. http://translator:5000. Empty: translation is off.
translator_url: str = Field(default="", alias="TRANSLATOR_URL")
# New passwords are checked against known data breaches (Have I Been Pwned's
# k-anonymity range API: only 5 hex digits of a SHA-1 leave the server).
password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK")
Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

+209 -89
View File
@@ -1,19 +1,22 @@
"""Email translation, on this server: LibreTranslate (Argos models) in its own
container, reached only by the API. A message's text never leaves the server.
"""Email translation, on this server: the translator service (translator/:
Mozilla's Firefox Translations models and fastText language identification)
in its own container, reached only by the API. A message's text never leaves
the server.
- A message's language is detected the first time it is shown (from its own
text: quoted replies and signatures left out) and kept on the message, so the
reading pane can offer "Translate" when it differs from the interface language.
- A translation is of the message HTML the user sees (already sanitized; the
result is sanitized again), cached per message, target language and whether
remote images were shown.
remote images were shown. It is made paragraph by paragraph, as plain text
(translation_prep.split_pieces), each paragraph from its own language.
- The languages on offer are those installed on the service (admin panel).
Off when ``LIBRETRANSLATE_URL`` is empty.
Off when ``TRANSLATOR_URL`` is empty.
"""
from __future__ import annotations
import asyncio
import html as html_lib
import logging
import re
@@ -30,10 +33,10 @@ from app.config import get_settings
logger = logging.getLogger(__name__)
DETECT_CHARS = 1500 # of the message's own text: plenty to tell the language
MIN_CONFIDENCE = 50.0 # below this, LibreTranslate is guessing
MIN_CONFIDENCE = 50.0 # below this, the detector is guessing
MAX_HTML_CHARS = 150_000 # a translation request bigger than this is refused
_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # long messages take a while on a CPU
_LANG_CACHE_S = 600
_LANG_CACHE_S = 60
class TranslationError(Exception):
@@ -47,65 +50,108 @@ class Language:
def enabled() -> bool:
return bool(get_settings().libretranslate_url.strip())
return bool(get_settings().translator_url.strip())
def _url(path: str) -> str:
return get_settings().libretranslate_url.rstrip("/") + path
return get_settings().translator_url.rstrip("/") + path
async def _post(path: str, payload: dict[str, object]) -> object:
async def _call(method: str, path: str, payload: dict[str, object] | None = None) -> object:
if not enabled():
raise TranslationError("translation is not set up on this server")
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
resp = await client.post(_url(path), json=payload)
resp = await client.request(method, _url(path), json=payload)
except httpx.HTTPError as exc:
raise TranslationError(f"the translation service is unreachable: {exc}") from exc
if resp.status_code != 200:
if resp.status_code not in (200, 202):
detail = resp.text[:200]
raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
return resp.json()
async def _post(path: str, payload: dict[str, object]) -> object:
return await _call("POST", path, payload)
_languages: tuple[float, list[Language]] = (0.0, [])
async def languages() -> list[Language]:
"""What the service can translate between (cached for a few minutes)."""
"""The installed languages, English among them (cached for a minute). The
browser names them in the interface language; ``name`` is the code."""
global _languages
if not enabled():
return []
stamp, cached = _languages
if cached and time.monotonic() - stamp < _LANG_CACHE_S:
return cached
if not enabled():
return []
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
resp = await client.get(_url("/languages"))
resp.raise_for_status()
rows = resp.json()
except (httpx.HTTPError, ValueError) as exc:
codes = resp.json()["installed"]
except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc:
logger.info("translation languages unavailable: %s", exc)
return []
found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r]
found = [Language(code=str(c), name=str(c)) for c in codes]
_languages = (time.monotonic(), found)
return found
def forget_languages() -> None:
"""After an install or removal: ask the service again."""
global _languages
_languages = (0.0, [])
async def status() -> dict[str, object]:
"""The service's own account (admin panel): installed, installing (bytes
done of total), available from Mozilla, and why an install failed."""
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=3.0)) as client:
try:
resp = await client.get(_url("/languages"))
resp.raise_for_status()
data = resp.json()
except (httpx.HTTPError, ValueError) as exc:
raise TranslationError(f"the translation service is unreachable: {exc}") from exc
return data if isinstance(data, dict) else {}
async def install(code: str) -> None:
await _call("POST", f"/languages/{code}")
forget_languages()
async def remove(code: str) -> None:
await _call("DELETE", f"/languages/{code}")
forget_languages()
_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
MIN_LETTERS = 25 # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
MAX_DETECT = 60 # pieces of one message asked about
MIN_SHARE = 0.15 # of a message's letters, for its language to count
async def _detect_many(texts: list[str]) -> list[str]:
"""The language of each of ``texts`` ('' where unsure), in one request."""
if not texts:
return []
result = await _post("/detect", {"q": [t[:DETECT_CHARS] for t in texts]})
if not isinstance(result, list) or len(result) != len(texts):
return [""] * len(texts)
return [
str(r.get("language", ""))
if isinstance(r, dict) and float(r.get("confidence", 0)) >= MIN_CONFIDENCE
else ""
for r in result
]
async def _detect_one(text: str) -> str:
result = await _post("/detect", {"q": text[:DETECT_CHARS]})
if not isinstance(result, list) or not result:
return ""
best = result[0]
if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE:
return ""
return str(best.get("language", ""))
return (await _detect_many([text]))[0]
def _weight(text: str) -> int:
@@ -126,13 +172,8 @@ async def languages_of(parts: list[str]) -> list[str]:
for i, part in enumerate(parts)
if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
][:MAX_DETECT]
gate = asyncio.Semaphore(4)
async def one(i: int) -> None:
async with gate:
told[i] = await _detect_one(parts[i])
await asyncio.gather(*(one(i) for i in asked))
for i, language in zip(asked, await _detect_many([parts[i] for i in asked]), strict=True):
told[i] = language
out = list(told)
following = ""
for i in reversed(range(len(parts))):
@@ -178,47 +219,163 @@ def _base(code: str) -> str:
return code.lower().split("-")[0]
async def _translate_paragraphs(html: str, target: str) -> str:
"""``html`` in more than one language, in ``target``: each paragraph from
its own language; paragraphs already in ``target`` are left as they are."""
async def _texts(texts: list[str], source: str, target: str) -> list[str]:
"""``texts`` translated, in batches."""
out: list[str] = []
for start in range(0, len(texts), 100):
batch = texts[start : start + 100]
result = await _post("/translate", {"q": batch, "source": source, "target": target})
got = result.get("translatedText") if isinstance(result, dict) else None
if not isinstance(got, list) or len(got) != len(batch):
raise TranslationError("the translation service sent no translation")
out += [str(x) for x in got]
return out
def _kept(name: str, translated: str) -> bool:
"""Whether ``name`` came through a translation: each of its words still
there, give or take an ending ("Lukas" → "Lukasie": yes; "Milan" →
"Mediolan": no)."""
return all(
word[: max(3, len(word) - 2)] in translated for word in name.split() if len(word) >= 2
)
# A capitalised word that starts a sentence ("Enclosed is a summary…").
_SENTENCE_START = re.compile(r"(?:^|(?<=[.!?…]\s))(?P<word>[^\W\d_][^\W\d_'’-]{2,})\b")
def _copied_starts(core: str, text: str, names: list[str]) -> list[str]:
"""Words that start a sentence of ``core`` and came through ``text``
untranslated, and are no known name: the models take a capitalised
sentence start for a name ("Enclosed is…" → "Enclosed jest…")."""
found: list[str] = []
for m in _SENTENCE_START.finditer(core):
word = m.group("word")
if not word[0].isupper() or word.isupper() or any(word in n for n in names):
continue
if re.search(rf"(?<!\w){re.escape(word)}(?!\w)", text):
found.append(word)
return found
def _lowered(core: str, words: list[str]) -> str:
"""``core`` with ``words`` lower-cased where they start a sentence."""
return _SENTENCE_START.sub(
lambda m: m.group(0).lower() if m.group("word") in words else m.group(0), core
)
def _capitalised(text: str) -> str:
"""Sentence starts capitalised again (after ``_lowered``): the first
letter, and one after a sentence's end — a full stop after a word of four
letters or more, so not after "r.", "np.", "tzw." (abbreviations)."""
text = re.sub(r"^(\W*)([^\W\d_])", lambda m: m.group(1) + m.group(2).upper(), text)
return re.sub(
r"((?:[^\W\d_]{4,}[.…]|[!?])\s+)([^\W\d_])",
lambda m: m.group(1) + m.group(2).upper(),
text,
)
def _as_before(text: str, first: str) -> str:
"""A word after a comma capitalised in ``text`` but not in ``first`` (the
first try) is lower-cased: lower-cased input makes the models capitalise
elsewhere ("…2026, od 13 lipca" → "…2026 r., Od 13 lipca")."""
def repl(m: re.Match[str]) -> str:
word = m.group(2)
lower = word[0].lower() + word[1:]
return m.group(1) + (lower if f", {lower}" in first else word)
return re.sub(r"(,\s+)([^\W\d_][\w'’-]*)", repl, text)
async def _with_names_kept(core: str, names: list[str], source: str, target: str) -> str:
"""``core`` translated around ``names``, which are left as they are."""
pattern = re.compile("(" + "|".join(re.escape(n) for n in names) + ")")
parts = pattern.split(core)
todo = [
(i, m.groups())
for i, part in enumerate(parts)
if i % 2 == 0 and (m := _EDGE_SPACE.match(part)) and any(c.isalnum() for c in part)
]
done = await _texts([groups[1] for _, groups in todo], source, target)
for (i, (lead, _, trail)), text in zip(todo, done, strict=True):
parts[i] = f"{lead}{text}{trail}"
return "".join(parts)
async def _translate_paragraphs(
html: str, target: str, sources: list[str], names: Iterable[str] = ()
) -> str:
"""``html`` in ``target``, as plain text paragraph by paragraph: from the
one language in ``sources``, or (several, or none known) each paragraph
from its own. Paragraphs already in ``target``, or in a language that is
not installed, are left as they are.
``names`` go along with their sentence, which then translates as a whole
(kept apart, "I came across <Imaginary Pixels>" lost its verb: "Przyszedłem
Imaginary Pixels"). Only a piece where one came out changed ("Milan" →
"Mediolan") is translated again with it left as it is."""
watch = sorted({n.strip() for n in names if len(n.strip()) >= 2}, key=len, reverse=True)
pieces, paragraphs = prep.split_pieces(html)
found = await languages_of([p.text for p in paragraphs])
if len(sources) == 1:
found = [sources[0]] * len(paragraphs)
else:
found = await languages_of([p.text for p in paragraphs])
installed = {x.code for x in await languages()}
# Chinese and Japanese go without spaces: two pieces of one sentence
# ("…奎琳,</span>我从…") need one between them once translated.
followed = {i for p in paragraphs for i in p.pieces[:-1]}
by_language: dict[str, list[int]] = {}
for paragraph, language in zip(paragraphs, found, strict=True):
if language and _base(language) != _base(target):
if language in installed and _base(language) != _base(target):
by_language.setdefault(language, []).extend(paragraph.pieces)
for language, where in by_language.items():
todo: list[tuple[int, str, str, str]] = []
for i in where:
m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
lead, core, trail = m.groups() if m else ("", "", "")
# A list marker stays out: the models repeat it ("1. Thursday" →
# "1. 1. Czwartek").
marker = _LIST_MARKER.match(core)
if marker:
lead, core = lead + marker.group(0), core[marker.end() :]
if not trail and i in followed and _base(language) in _NO_SPACES:
trail = " "
if any(c.isalnum() for c in core):
todo.append((i, lead, core, trail))
for start in range(0, len(todo), 100):
batch = todo[start : start + 100]
result = await _post(
"/translate",
{"q": [core for _, _, core, _ in batch], "source": language, "target": target},
done = await _texts([core for _, _, core, _ in todo], language, target)
# Sentence starts that came through untranslated: once more, lower-case.
again = [
(k, words)
for k, ((_, _, core, _), text) in enumerate(zip(todo, done, strict=True))
if (words := _copied_starts(core, text, watch))
]
if again:
retried = await _texts(
[_lowered(todo[k][2], words) for k, words in again], language, target
)
out = result.get("translatedText") if isinstance(result, dict) else None
if not isinstance(out, list) or len(out) != len(batch):
raise TranslationError("the translation service sent no translation")
for (i, lead, _, trail), text in zip(batch, out, strict=True):
pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
for (k, words), text in zip(again, retried, strict=True):
# Still there: a name after all, and the first try stands.
if len(_copied_starts(todo[k][2], text, watch)) < len(words):
done[k] = _as_before(_capitalised(text), done[k])
for (i, lead, core, trail), text in zip(todo, done, strict=True):
lost = [n for n in watch if n in core and not _kept(n, text)]
if lost:
text = await _with_names_kept(core, lost, language, target)
pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
return "".join(pieces)
_NO_SPACES = {"zh", "ja", "th"}
_LIST_MARKER = re.compile(r"(?:\d{1,3}[.)]|[a-zA-Z][.)]|[-•*–])\s+(?=\S)")
_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
# Web and mail addresses in the text: the models break them up ("https: / /
# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps.
# vimeo.com"), so they are marked translate="no" and not sent.
_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
_TAG = re.compile(r"(<[^>]*>)")
@@ -243,36 +400,9 @@ def protect_addresses(html: str) -> str:
return "".join(out)
_PUNCT = re.compile(r"[\W_]+")
MAX_NAMES = 200 # names checked for one message
async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]:
"""Of ``names``, those the model changes when it translates them alone
("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one
request for them all. If the check fails, all of them."""
wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES]
if not wanted:
return set()
try:
result = await _post(
"/translate", {"q": wanted, "source": source or "auto", "target": target}
)
except TranslationError:
return set(wanted)
out = result.get("translatedText") if isinstance(result, dict) else None
if not isinstance(out, list) or len(out) != len(wanted):
return set(wanted)
def same(a: str, b: str) -> bool:
return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold()
return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))}
# Bumped when what goes to the service, or what is done with the result,
# changes: translations kept from before are made again.
PIPELINE_VERSION = 5
PIPELINE_VERSION = 8
async def translate_html(
@@ -288,16 +418,6 @@ async def translate_html(
prepared = prep.join_wrapped_lines(html)
for language in sources:
prepared = prep.translate_phrases(prepared, language, target)
single = sources[0] if len(sources) == 1 else "auto"
candidates = {*names, *prep.find_names(prepared, single if sources else "")}
terms = {*keep, *await names_to_keep(candidates, single, target)}
prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms)))
if len(sources) > 1:
return prep.tidy(await _translate_paragraphs(prepared, target))
result = await _post(
"/translate",
{"q": prepared, "source": source or "auto", "target": target, "format": "html"},
)
if not isinstance(result, dict) or "translatedText" not in result:
raise TranslationError("the translation service sent no translation")
return prep.tidy(str(result["translatedText"]))
candidates = {*names, *prep.find_names(prepared, sources[0] if sources else "")}
prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, keep)))
return prep.tidy(await _translate_paragraphs(prepared, target, sources, candidates))
+15 -15
View File
@@ -1,25 +1,25 @@
"""Getting a message ready for translation, and tidying what comes back.
LibreTranslate translates each piece of text between tags on its own, and its
small models stumble on things a reader would not: a sentence wrapped onto two
lines becomes two sentences ("ask you about" / "Plan for next month"),
"Best," becomes "Najlepsze," and a company name gets translated word by word.
So before a message goes out:
A message is translated as plain text, piece by piece (the text between
tags), and the models stumble on things a reader would not: a sentence
wrapped onto two lines becomes two sentences ("ask you about" / "Plan for next
month"), "Best," becomes "Najlepsze," and a company name gets translated word
by word. So before a message goes out:
- lines that carry on a sentence (no end punctuation, next starts lowercase)
are joined, in HTML (<br>, one <div> per line) and in wrapped plain text;
- greetings and sign-offs that make a whole line are put in the target
language from a small phrase table (English, Polish, German);
- names are marked translate="no", which LibreTranslate keeps — the people
on the message, the user's own names and signature names, and capitalised
multi-word names mid-sentence — but only the names the model would change
("Milan" → "Mediolan"): a kept name splits its sentence, and LibreTranslate
translates each piece alone ("Dear <kept>Lukas Remis</kept>," →
"Kochanie."), so a name the model leaves alone stays in the sentence;
- names are marked translate="no" and not sent — the people on the message,
the user's own names and signature names, and capitalised multi-word names
mid-sentence — but only the names the model would change ("Milan" →
"Mediolan"): a kept name splits its sentence into pieces translated alone
("Dear <kept>Lukas Remis</kept>," → "Kochanie."), so a name the model
leaves alone stays in the sentence;
- so are emoji, file names ("A_0023C017_p1CAC.mov") and text without a letter;
- a message in more than one language is translated paragraph by paragraph,
each from its own language (split_pieces); those in the target stay;
- and afterwards, the space LibreTranslate drops after a kept word comes back.
- the message is translated paragraph by paragraph (split_pieces), each from
its own language when there are several; those in the target stay;
- and afterwards, a kept word gets back the spaces around it.
"""
from __future__ import annotations
@@ -315,7 +315,7 @@ def split_pieces(html: str) -> tuple[list[str], list[Paragraph]]:
# --- after translation -------------------------------------------------------------
# LibreTranslate drops the space after a kept span ("Bruton</span>i twoje").
# A kept span can lose the space beside it ("Bruton</span>i twoje").
# Only kept spans: they still carry translate="no" here (the sanitizer that
# runs next takes it off); other spans are the sender's and stay as they are.
_KEPT = re.compile(r'(<span translate="no">[^<]*</span>)')