lukas/webmail-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Translation on Firefox Translations models; new logo

Browse Source 
- Email translation moves from LibreTranslate to our own translator
  service (translator/): Mozilla's Firefox Translations models on the
  Bergamot engine, fastText language detection (176 languages). Polish,
  English, German, French, Spanish and Italian on first start (~350 MB
  instead of ~10 GB); Admin panel > Email translation installs and
  removes languages. Built and pushed as the `translator` image.
- Every message is translated paragraph by paragraph as plain text;
  names travel with their sentence and are kept only where the model
  changed them; a sentence start the model copied ("Enclosed") is tried
  again in lower case; list markers are not sent
- The Translate button shows only for installed languages
- New logo: favicon, app/maskable/iPhone icons, notification badge,
  email logo and calendar copy regenerated; BIMI logo rebuilt in SVG
  Tiny PS with its gradients; masks render in browsers
- Launch screen background is the app's dark grey

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lukas
2026-10-18 22:04:04 +02:00
co-authored by Claude Opus 5.5
parent 8a1b9979e9
commit 972e10876b
38 changed files with 1627 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/translation.py
+209 -89
View File
@@ -1,19 +1,22 @@
"""Email translation, on this server: LibreTranslate (Argos models) in its own
container, reached only by the API. A message's text never leaves the server.
"""Email translation, on this server: the translator service (translator/:
Mozilla's Firefox Translations models and fastText language identification)
in its own container, reached only by the API. A message's text never leaves
the server.
- A message's language is detected the first time it is shown (from its own
text: quoted replies and signatures left out) and kept on the message, so the
reading pane can offer "Translate" when it differs from the interface language.
- A translation is of the message HTML the user sees (already sanitized; the
result is sanitized again), cached per message, target language and whether
remote images were shown.
remote images were shown. It is made paragraph by paragraph, as plain text
(translation_prep.split_pieces), each paragraph from its own language.
- The languages on offer are those installed on the service (admin panel).
Off when ``LIBRETRANSLATE_URL`` is empty.
Off when ``TRANSLATOR_URL`` is empty.
"""
from __future__ import annotations
import asyncio
import html as html_lib
import logging
import re
@@ -30,10 +33,10 @@ from app.config import get_settings
logger = logging.getLogger(__name__)
DETECT_CHARS = 1500 # of the message's own text: plenty to tell the language
MIN_CONFIDENCE = 50.0 # below this, LibreTranslate is guessing
MIN_CONFIDENCE = 50.0 # below this, the detector is guessing
MAX_HTML_CHARS = 150_000 # a translation request bigger than this is refused
_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # long messages take a while on a CPU
_LANG_CACHE_S = 600
_LANG_CACHE_S = 60
class TranslationError(Exception):
@@ -47,65 +50,108 @@ class Language:
def enabled() -> bool:
return bool(get_settings().libretranslate_url.strip())
return bool(get_settings().translator_url.strip())
def _url(path: str) -> str:
return get_settings().libretranslate_url.rstrip("/") + path
return get_settings().translator_url.rstrip("/") + path
async def _post(path: str, payload: dict[str, object]) -> object:
async def _call(method: str, path: str, payload: dict[str, object] | None = None) -> object:
if not enabled():
raise TranslationError("translation is not set up on this server")
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
resp = await client.post(_url(path), json=payload)
resp = await client.request(method, _url(path), json=payload)
except httpx.HTTPError as exc:
raise TranslationError(f"the translation service is unreachable: {exc}") from exc
if resp.status_code != 200:
if resp.status_code not in (200, 202):
detail = resp.text[:200]
raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
return resp.json()
async def _post(path: str, payload: dict[str, object]) -> object:
return await _call("POST", path, payload)
_languages: tuple[float, list[Language]] = (0.0, [])
async def languages() -> list[Language]:
"""What the service can translate between (cached for a few minutes)."""
"""The installed languages, English among them (cached for a minute). The
browser names them in the interface language; ``name`` is the code."""
global _languages
if not enabled():
return []
stamp, cached = _languages
if cached and time.monotonic() - stamp < _LANG_CACHE_S:
return cached
if not enabled():
return []
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
resp = await client.get(_url("/languages"))
resp.raise_for_status()
rows = resp.json()
except (httpx.HTTPError, ValueError) as exc:
codes = resp.json()["installed"]
except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc:
logger.info("translation languages unavailable: %s", exc)
return []
found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r]
found = [Language(code=str(c), name=str(c)) for c in codes]
_languages = (time.monotonic(), found)
return found
def forget_languages() -> None:
"""After an install or removal: ask the service again."""
global _languages
_languages = (0.0, [])
async def status() -> dict[str, object]:
"""The service's own account (admin panel): installed, installing (bytes
done of total), available from Mozilla, and why an install failed."""
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=3.0)) as client:
try:
resp = await client.get(_url("/languages"))
resp.raise_for_status()
data = resp.json()
except (httpx.HTTPError, ValueError) as exc:
raise TranslationError(f"the translation service is unreachable: {exc}") from exc
return data if isinstance(data, dict) else {}
async def install(code: str) -> None:
await _call("POST", f"/languages/{code}")
forget_languages()
async def remove(code: str) -> None:
await _call("DELETE", f"/languages/{code}")
forget_languages()
_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
MIN_LETTERS = 25 # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
MAX_DETECT = 60 # pieces of one message asked about
MIN_SHARE = 0.15 # of a message's letters, for its language to count
async def _detect_many(texts: list[str]) -> list[str]:
"""The language of each of ``texts`` ('' where unsure), in one request."""
if not texts:
return []
result = await _post("/detect", {"q": [t[:DETECT_CHARS] for t in texts]})
if not isinstance(result, list) or len(result) != len(texts):
return [""] * len(texts)
return [
str(r.get("language", ""))
if isinstance(r, dict) and float(r.get("confidence", 0)) >= MIN_CONFIDENCE
else ""
for r in result
]
async def _detect_one(text: str) -> str:
result = await _post("/detect", {"q": text[:DETECT_CHARS]})
if not isinstance(result, list) or not result:
return ""
best = result[0]
if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE:
return ""
return str(best.get("language", ""))
return (await _detect_many([text]))[0]
def _weight(text: str) -> int:
@@ -126,13 +172,8 @@ async def languages_of(parts: list[str]) -> list[str]:
for i, part in enumerate(parts)
if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
][:MAX_DETECT]
gate = asyncio.Semaphore(4)
async def one(i: int) -> None:
async with gate:
told[i] = await _detect_one(parts[i])
await asyncio.gather(*(one(i) for i in asked))
for i, language in zip(asked, await _detect_many([parts[i] for i in asked]), strict=True):
told[i] = language
out = list(told)
following = ""
for i in reversed(range(len(parts))):
@@ -178,47 +219,163 @@ def _base(code: str) -> str:
return code.lower().split("-")[0]
async def _translate_paragraphs(html: str, target: str) -> str:
"""``html`` in more than one language, in ``target``: each paragraph from
its own language; paragraphs already in ``target`` are left as they are."""
async def _texts(texts: list[str], source: str, target: str) -> list[str]:
"""``texts`` translated, in batches."""
out: list[str] = []
for start in range(0, len(texts), 100):
batch = texts[start : start + 100]
result = await _post("/translate", {"q": batch, "source": source, "target": target})
got = result.get("translatedText") if isinstance(result, dict) else None
if not isinstance(got, list) or len(got) != len(batch):
raise TranslationError("the translation service sent no translation")
out += [str(x) for x in got]
return out
def _kept(name: str, translated: str) -> bool:
"""Whether ``name`` came through a translation: each of its words still
there, give or take an ending ("Lukas" → "Lukasie": yes; "Milan" →
"Mediolan": no)."""
return all(
word[: max(3, len(word) - 2)] in translated for word in name.split() if len(word) >= 2
)
# A capitalised word that starts a sentence ("Enclosed is a summary…").
_SENTENCE_START = re.compile(r"(?:^|(?<=[.!?…]\s))(?P<word>[^\W\d_][^\W\d_'’-]{2,})\b")
def _copied_starts(core: str, text: str, names: list[str]) -> list[str]:
"""Words that start a sentence of ``core`` and came through ``text``
untranslated, and are no known name: the models take a capitalised
sentence start for a name ("Enclosed is…" → "Enclosed jest…")."""
found: list[str] = []
for m in _SENTENCE_START.finditer(core):
word = m.group("word")
if not word[0].isupper() or word.isupper() or any(word in n for n in names):
continue
if re.search(rf"(?<!\w){re.escape(word)}(?!\w)", text):
found.append(word)
return found
def _lowered(core: str, words: list[str]) -> str:
"""``core`` with ``words`` lower-cased where they start a sentence."""
return _SENTENCE_START.sub(
lambda m: m.group(0).lower() if m.group("word") in words else m.group(0), core
)
def _capitalised(text: str) -> str:
"""Sentence starts capitalised again (after ``_lowered``): the first
letter, and one after a sentence's end — a full stop after a word of four
letters or more, so not after "r.", "np.", "tzw." (abbreviations)."""
text = re.sub(r"^(\W*)([^\W\d_])", lambda m: m.group(1) + m.group(2).upper(), text)
return re.sub(
r"((?:[^\W\d_]{4,}[.…]|[!?])\s+)([^\W\d_])",
lambda m: m.group(1) + m.group(2).upper(),
text,
)
def _as_before(text: str, first: str) -> str:
"""A word after a comma capitalised in ``text`` but not in ``first`` (the
first try) is lower-cased: lower-cased input makes the models capitalise
elsewhere ("…2026, od 13 lipca" → "…2026 r., Od 13 lipca")."""
def repl(m: re.Match[str]) -> str:
word = m.group(2)
lower = word[0].lower() + word[1:]
return m.group(1) + (lower if f", {lower}" in first else word)
return re.sub(r"(,\s+)([^\W\d_][\w'’-]*)", repl, text)
async def _with_names_kept(core: str, names: list[str], source: str, target: str) -> str:
"""``core`` translated around ``names``, which are left as they are."""
pattern = re.compile("(" + "|".join(re.escape(n) for n in names) + ")")
parts = pattern.split(core)
todo = [
(i, m.groups())
for i, part in enumerate(parts)
if i % 2 == 0 and (m := _EDGE_SPACE.match(part)) and any(c.isalnum() for c in part)
]
done = await _texts([groups[1] for _, groups in todo], source, target)
for (i, (lead, _, trail)), text in zip(todo, done, strict=True):
parts[i] = f"{lead}{text}{trail}"
return "".join(parts)
async def _translate_paragraphs(
html: str, target: str, sources: list[str], names: Iterable[str] = ()
) -> str:
"""``html`` in ``target``, as plain text paragraph by paragraph: from the
one language in ``sources``, or (several, or none known) each paragraph
from its own. Paragraphs already in ``target``, or in a language that is
not installed, are left as they are.
``names`` go along with their sentence, which then translates as a whole
(kept apart, "I came across <Imaginary Pixels>" lost its verb: "Przyszedłem
Imaginary Pixels"). Only a piece where one came out changed ("Milan" →
"Mediolan") is translated again with it left as it is."""
watch = sorted({n.strip() for n in names if len(n.strip()) >= 2}, key=len, reverse=True)
pieces, paragraphs = prep.split_pieces(html)
found = await languages_of([p.text for p in paragraphs])
if len(sources) == 1:
found = [sources[0]] * len(paragraphs)
else:
found = await languages_of([p.text for p in paragraphs])
installed = {x.code for x in await languages()}
# Chinese and Japanese go without spaces: two pieces of one sentence
# ("…奎琳，</span>我从…") need one between them once translated.
followed = {i for p in paragraphs for i in p.pieces[:-1]}
by_language: dict[str, list[int]] = {}
for paragraph, language in zip(paragraphs, found, strict=True):
if language and _base(language) != _base(target):
if language in installed and _base(language) != _base(target):
by_language.setdefault(language, []).extend(paragraph.pieces)
for language, where in by_language.items():
todo: list[tuple[int, str, str, str]] = []
for i in where:
m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
lead, core, trail = m.groups() if m else ("", "", "")
# A list marker stays out: the models repeat it ("1. Thursday" →
# "1. 1. Czwartek").
marker = _LIST_MARKER.match(core)
if marker:
lead, core = lead + marker.group(0), core[marker.end() :]
if not trail and i in followed and _base(language) in _NO_SPACES:
trail = " "
if any(c.isalnum() for c in core):
todo.append((i, lead, core, trail))
for start in range(0, len(todo), 100):
batch = todo[start : start + 100]
result = await _post(
"/translate",
{"q": [core for _, _, core, _ in batch], "source": language, "target": target},
done = await _texts([core for _, _, core, _ in todo], language, target)
# Sentence starts that came through untranslated: once more, lower-case.
again = [
(k, words)
for k, ((_, _, core, _), text) in enumerate(zip(todo, done, strict=True))
if (words := _copied_starts(core, text, watch))
]
if again:
retried = await _texts(
[_lowered(todo[k][2], words) for k, words in again], language, target
)
out = result.get("translatedText") if isinstance(result, dict) else None
if not isinstance(out, list) or len(out) != len(batch):
raise TranslationError("the translation service sent no translation")
for (i, lead, _, trail), text in zip(batch, out, strict=True):
pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
for (k, words), text in zip(again, retried, strict=True):
# Still there: a name after all, and the first try stands.
if len(_copied_starts(todo[k][2], text, watch)) < len(words):
done[k] = _as_before(_capitalised(text), done[k])
for (i, lead, core, trail), text in zip(todo, done, strict=True):
lost = [n for n in watch if n in core and not _kept(n, text)]
if lost:
text = await _with_names_kept(core, lost, language, target)
pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
return "".join(pieces)
_NO_SPACES = {"zh", "ja", "th"}
_LIST_MARKER = re.compile(r"(?:\d{1,3}[.)]|[a-zA-Z][.)]|[-•*–])\s+(?=\S)")
_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
# Web and mail addresses in the text: the models break them up ("https: / /
# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps.
# vimeo.com"), so they are marked translate="no" and not sent.
_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
_TAG = re.compile(r"(<[^>]*>)")
@@ -243,36 +400,9 @@ def protect_addresses(html: str) -> str:
return "".join(out)
_PUNCT = re.compile(r"[\W_]+")
MAX_NAMES = 200 # names checked for one message
async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]:
"""Of ``names``, those the model changes when it translates them alone
("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one
request for them all. If the check fails, all of them."""
wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES]
if not wanted:
return set()
try:
result = await _post(
"/translate", {"q": wanted, "source": source or "auto", "target": target}
)
except TranslationError:
return set(wanted)
out = result.get("translatedText") if isinstance(result, dict) else None
if not isinstance(out, list) or len(out) != len(wanted):
return set(wanted)
def same(a: str, b: str) -> bool:
return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold()
return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))}
# Bumped when what goes to the service, or what is done with the result,
# changes: translations kept from before are made again.
PIPELINE_VERSION = 5
PIPELINE_VERSION = 8
async def translate_html(
@@ -288,16 +418,6 @@ async def translate_html(
prepared = prep.join_wrapped_lines(html)
for language in sources:
prepared = prep.translate_phrases(prepared, language, target)
single = sources[0] if len(sources) == 1 else "auto"
candidates = {*names, *prep.find_names(prepared, single if sources else "")}
terms = {*keep, *await names_to_keep(candidates, single, target)}
prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms)))
if len(sources) > 1:
return prep.tidy(await _translate_paragraphs(prepared, target))
result = await _post(
"/translate",
{"q": prepared, "source": source or "auto", "target": target, "format": "html"},
)
if not isinstance(result, dict) or "translatedText" not in result:
raise TranslationError("the translation service sent no translation")
return prep.tidy(str(result["translatedText"]))
candidates = {*names, *prep.find_names(prepared, sources[0] if sources else "")}
prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, keep)))
return prep.tidy(await _translate_paragraphs(prepared, target, sources, candidates))