diff --git a/.env.example b/.env.example index 86bb653..1640352 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,5 @@ RSPAMD_CONTROLLER_PASSWORD=devpass # Spelling/grammar check in the compose window (LanguageTool). Empty = off. LANGUAGETOOL_URL=http://127.0.0.1:8010 +# Email translation (the translator service, translator/). Empty = off. +TRANSLATOR_URL=http://127.0.0.1:5005 diff --git a/Makefile b/Makefile index cdca34c..2c4cc3a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ up: ## start services - docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool libretranslate + docker compose -f docker/docker-compose.yml up -d postgres redis meilisearch languagetool translator down: ## stop services (data kept) docker compose -f docker/docker-compose.yml down reset-db: ## DESTRUCTIVE: wipe db + search + blobs, re-migrate diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 07e1e12..4606a82 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -15,10 +15,12 @@ import re import uuid from typing import Annotated -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Path, Query +from pydantic import BaseModel from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession +from app import translation from app.api.errors import raise_api_error from app.config import get_settings from app.db import get_session @@ -332,3 +334,61 @@ async def remove_user(user_id: uuid.UUID, admin: CurrentAdmin, db: DbSession) -> except MailServerError as exc: raise_api_error(502, "mail_server_unavailable", f"the mail server did not answer: {exc}") return {"status": "deleted"} + + +# --- email translation languages (translator service) ------------------------------ + + +class InstallProgressOut(BaseModel): + done: int + total: int + + +class AdminTranslationOut(BaseModel): + enabled: bool # TRANSLATOR_URL is set + reachable: bool + installed: list[str] = [] + installing: dict[str, InstallProgressOut] = {} + available: list[str] = [] # what Mozilla has models for (empty: offline) + errors: dict[str, str] = {} # why a language's last install failed + + +_LANG = Path(pattern=r"^[a-z]{2,3}(-[A-Za-z]{2,4})?$") + + +@router.get("/translation", response_model=AdminTranslationOut) +async def translation_status(admin: CurrentAdmin) -> AdminTranslationOut: + if not translation.enabled(): + return AdminTranslationOut(enabled=False, reachable=False) + try: + data = await translation.status() + except translation.TranslationError: + return AdminTranslationOut(enabled=True, reachable=False) + return AdminTranslationOut(enabled=True, reachable=True, **data) + + +@router.post( + "/translation/languages/{code}", + response_model=AdminTranslationOut, + dependencies=[Depends(require_csrf)], +) +async def install_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut: + """Download a language's models (to and from English) in the background.""" + try: + await translation.install(code) + except translation.TranslationError as exc: + raise_api_error(502, "translator_failed", str(exc)) + return await translation_status(admin) + + +@router.delete( + "/translation/languages/{code}", + response_model=AdminTranslationOut, + dependencies=[Depends(require_csrf)], +) +async def remove_language(admin: CurrentAdmin, code: str = _LANG) -> AdminTranslationOut: + try: + await translation.remove(code) + except translation.TranslationError as exc: + raise_api_error(502, "translator_failed", str(exc)) + return await translation_status(admin) diff --git a/backend/app/api/translation.py b/backend/app/api/translation.py index e6807c3..829eef6 100644 --- a/backend/app/api/translation.py +++ b/backend/app/api/translation.py @@ -25,6 +25,6 @@ class TranslationInfoOut(BaseModel): async def translation_languages(user: CurrentUser) -> TranslationInfoOut: found = await translation.languages() return TranslationInfoOut( - available=bool(found), + available=len(found) > 1, # English alone translates nothing languages=[LanguageOut(code=x.code, name=x.name) for x in found], ) diff --git a/backend/app/config.py b/backend/app/config.py index 5ed920e..c48723e 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -71,9 +71,9 @@ class Settings(BaseSettings): # Spelling and grammar checking in the compose window (LanguageTool server, # e.g. http://languagetool:8010). Empty: the check is off for everyone. languagetool_url: str = Field(default="", alias="LANGUAGETOOL_URL") - # Email translation: a LibreTranslate server (app/translation.py), e.g. - # http://libretranslate:5000. Empty: translation is off. - libretranslate_url: str = Field(default="", alias="LIBRETRANSLATE_URL") + # Email translation: the translator service (translator/, app/translation.py), + # e.g. http://translator:5000. Empty: translation is off. + translator_url: str = Field(default="", alias="TRANSLATOR_URL") # New passwords are checked against known data breaches (Have I Been Pwned's # k-anonymity range API: only 5 hex digits of a SHA-1 leave the server). password_breach_check: bool = Field(default=True, alias="PASSWORD_BREACH_CHECK") diff --git a/backend/app/identity/assets/ultimo-logo.png b/backend/app/identity/assets/ultimo-logo.png index 0965a0e..2a91459 100644 Binary files a/backend/app/identity/assets/ultimo-logo.png and b/backend/app/identity/assets/ultimo-logo.png differ diff --git a/backend/app/translation.py b/backend/app/translation.py index 14384aa..6897e14 100644 --- a/backend/app/translation.py +++ b/backend/app/translation.py @@ -1,19 +1,22 @@ -"""Email translation, on this server: LibreTranslate (Argos models) in its own -container, reached only by the API. A message's text never leaves the server. +"""Email translation, on this server: the translator service (translator/: +Mozilla's Firefox Translations models and fastText language identification) +in its own container, reached only by the API. A message's text never leaves +the server. - A message's language is detected the first time it is shown (from its own text: quoted replies and signatures left out) and kept on the message, so the reading pane can offer "Translate" when it differs from the interface language. - A translation is of the message HTML the user sees (already sanitized; the result is sanitized again), cached per message, target language and whether - remote images were shown. + remote images were shown. It is made paragraph by paragraph, as plain text + (translation_prep.split_pieces), each paragraph from its own language. +- The languages on offer are those installed on the service (admin panel). -Off when ``LIBRETRANSLATE_URL`` is empty. +Off when ``TRANSLATOR_URL`` is empty. """ from __future__ import annotations -import asyncio import html as html_lib import logging import re @@ -30,10 +33,10 @@ from app.config import get_settings logger = logging.getLogger(__name__) DETECT_CHARS = 1500 # of the message's own text: plenty to tell the language -MIN_CONFIDENCE = 50.0 # below this, LibreTranslate is guessing +MIN_CONFIDENCE = 50.0 # below this, the detector is guessing MAX_HTML_CHARS = 150_000 # a translation request bigger than this is refused _TIMEOUT = httpx.Timeout(120.0, connect=5.0) # long messages take a while on a CPU -_LANG_CACHE_S = 600 +_LANG_CACHE_S = 60 class TranslationError(Exception): @@ -47,65 +50,108 @@ class Language: def enabled() -> bool: - return bool(get_settings().libretranslate_url.strip()) + return bool(get_settings().translator_url.strip()) def _url(path: str) -> str: - return get_settings().libretranslate_url.rstrip("/") + path + return get_settings().translator_url.rstrip("/") + path -async def _post(path: str, payload: dict[str, object]) -> object: +async def _call(method: str, path: str, payload: dict[str, object] | None = None) -> object: if not enabled(): raise TranslationError("translation is not set up on this server") try: async with httpx.AsyncClient(timeout=_TIMEOUT) as client: - resp = await client.post(_url(path), json=payload) + resp = await client.request(method, _url(path), json=payload) except httpx.HTTPError as exc: raise TranslationError(f"the translation service is unreachable: {exc}") from exc - if resp.status_code != 200: + if resp.status_code not in (200, 202): detail = resp.text[:200] raise TranslationError(f"the translation service refused: {resp.status_code} {detail}") return resp.json() +async def _post(path: str, payload: dict[str, object]) -> object: + return await _call("POST", path, payload) + + _languages: tuple[float, list[Language]] = (0.0, []) async def languages() -> list[Language]: - """What the service can translate between (cached for a few minutes).""" + """The installed languages, English among them (cached for a minute). The + browser names them in the interface language; ``name`` is the code.""" global _languages + if not enabled(): + return [] stamp, cached = _languages if cached and time.monotonic() - stamp < _LANG_CACHE_S: return cached - if not enabled(): - return [] try: async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client: resp = await client.get(_url("/languages")) resp.raise_for_status() - rows = resp.json() - except (httpx.HTTPError, ValueError) as exc: + codes = resp.json()["installed"] + except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc: logger.info("translation languages unavailable: %s", exc) return [] - found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r] + found = [Language(code=str(c), name=str(c)) for c in codes] _languages = (time.monotonic(), found) return found +def forget_languages() -> None: + """After an install or removal: ask the service again.""" + global _languages + _languages = (0.0, []) + + +async def status() -> dict[str, object]: + """The service's own account (admin panel): installed, installing (bytes + done of total), available from Mozilla, and why an install failed.""" + async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=3.0)) as client: + try: + resp = await client.get(_url("/languages")) + resp.raise_for_status() + data = resp.json() + except (httpx.HTTPError, ValueError) as exc: + raise TranslationError(f"the translation service is unreachable: {exc}") from exc + return data if isinstance(data, dict) else {} + + +async def install(code: str) -> None: + await _call("POST", f"/languages/{code}") + forget_languages() + + +async def remove(code: str) -> None: + await _call("DELETE", f"/languages/{code}") + forget_languages() + + _CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]") MIN_LETTERS = 25 # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto) MAX_DETECT = 60 # pieces of one message asked about MIN_SHARE = 0.15 # of a message's letters, for its language to count +async def _detect_many(texts: list[str]) -> list[str]: + """The language of each of ``texts`` ('' where unsure), in one request.""" + if not texts: + return [] + result = await _post("/detect", {"q": [t[:DETECT_CHARS] for t in texts]}) + if not isinstance(result, list) or len(result) != len(texts): + return [""] * len(texts) + return [ + str(r.get("language", "")) + if isinstance(r, dict) and float(r.get("confidence", 0)) >= MIN_CONFIDENCE + else "" + for r in result + ] + + async def _detect_one(text: str) -> str: - result = await _post("/detect", {"q": text[:DETECT_CHARS]}) - if not isinstance(result, list) or not result: - return "" - best = result[0] - if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE: - return "" - return str(best.get("language", "")) + return (await _detect_many([text]))[0] def _weight(text: str) -> int: @@ -126,13 +172,8 @@ async def languages_of(parts: list[str]) -> list[str]: for i, part in enumerate(parts) if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS ][:MAX_DETECT] - gate = asyncio.Semaphore(4) - - async def one(i: int) -> None: - async with gate: - told[i] = await _detect_one(parts[i]) - - await asyncio.gather(*(one(i) for i in asked)) + for i, language in zip(asked, await _detect_many([parts[i] for i in asked]), strict=True): + told[i] = language out = list(told) following = "" for i in reversed(range(len(parts))): @@ -178,47 +219,163 @@ def _base(code: str) -> str: return code.lower().split("-")[0] -async def _translate_paragraphs(html: str, target: str) -> str: - """``html`` in more than one language, in ``target``: each paragraph from - its own language; paragraphs already in ``target`` are left as they are.""" +async def _texts(texts: list[str], source: str, target: str) -> list[str]: + """``texts`` translated, in batches.""" + out: list[str] = [] + for start in range(0, len(texts), 100): + batch = texts[start : start + 100] + result = await _post("/translate", {"q": batch, "source": source, "target": target}) + got = result.get("translatedText") if isinstance(result, dict) else None + if not isinstance(got, list) or len(got) != len(batch): + raise TranslationError("the translation service sent no translation") + out += [str(x) for x in got] + return out + + +def _kept(name: str, translated: str) -> bool: + """Whether ``name`` came through a translation: each of its words still + there, give or take an ending ("Lukas" → "Lukasie": yes; "Milan" → + "Mediolan": no).""" + return all( + word[: max(3, len(word) - 2)] in translated for word in name.split() if len(word) >= 2 + ) + + +# A capitalised word that starts a sentence ("Enclosed is a summary…"). +_SENTENCE_START = re.compile(r"(?:^|(?<=[.!?…]\s))(?P[^\W\d_][^\W\d_'’-]{2,})\b") + + +def _copied_starts(core: str, text: str, names: list[str]) -> list[str]: + """Words that start a sentence of ``core`` and came through ``text`` + untranslated, and are no known name: the models take a capitalised + sentence start for a name ("Enclosed is…" → "Enclosed jest…").""" + found: list[str] = [] + for m in _SENTENCE_START.finditer(core): + word = m.group("word") + if not word[0].isupper() or word.isupper() or any(word in n for n in names): + continue + if re.search(rf"(? str: + """``core`` with ``words`` lower-cased where they start a sentence.""" + return _SENTENCE_START.sub( + lambda m: m.group(0).lower() if m.group("word") in words else m.group(0), core + ) + + +def _capitalised(text: str) -> str: + """Sentence starts capitalised again (after ``_lowered``): the first + letter, and one after a sentence's end — a full stop after a word of four + letters or more, so not after "r.", "np.", "tzw." (abbreviations).""" + text = re.sub(r"^(\W*)([^\W\d_])", lambda m: m.group(1) + m.group(2).upper(), text) + return re.sub( + r"((?:[^\W\d_]{4,}[.…]|[!?])\s+)([^\W\d_])", + lambda m: m.group(1) + m.group(2).upper(), + text, + ) + + +def _as_before(text: str, first: str) -> str: + """A word after a comma capitalised in ``text`` but not in ``first`` (the + first try) is lower-cased: lower-cased input makes the models capitalise + elsewhere ("…2026, od 13 lipca" → "…2026 r., Od 13 lipca").""" + + def repl(m: re.Match[str]) -> str: + word = m.group(2) + lower = word[0].lower() + word[1:] + return m.group(1) + (lower if f", {lower}" in first else word) + + return re.sub(r"(,\s+)([^\W\d_][\w'’-]*)", repl, text) + + +async def _with_names_kept(core: str, names: list[str], source: str, target: str) -> str: + """``core`` translated around ``names``, which are left as they are.""" + pattern = re.compile("(" + "|".join(re.escape(n) for n in names) + ")") + parts = pattern.split(core) + todo = [ + (i, m.groups()) + for i, part in enumerate(parts) + if i % 2 == 0 and (m := _EDGE_SPACE.match(part)) and any(c.isalnum() for c in part) + ] + done = await _texts([groups[1] for _, groups in todo], source, target) + for (i, (lead, _, trail)), text in zip(todo, done, strict=True): + parts[i] = f"{lead}{text}{trail}" + return "".join(parts) + + +async def _translate_paragraphs( + html: str, target: str, sources: list[str], names: Iterable[str] = () +) -> str: + """``html`` in ``target``, as plain text paragraph by paragraph: from the + one language in ``sources``, or (several, or none known) each paragraph + from its own. Paragraphs already in ``target``, or in a language that is + not installed, are left as they are. + + ``names`` go along with their sentence, which then translates as a whole + (kept apart, "I came across " lost its verb: "Przyszedłem + Imaginary Pixels"). Only a piece where one came out changed ("Milan" → + "Mediolan") is translated again with it left as it is.""" + watch = sorted({n.strip() for n in names if len(n.strip()) >= 2}, key=len, reverse=True) pieces, paragraphs = prep.split_pieces(html) - found = await languages_of([p.text for p in paragraphs]) + if len(sources) == 1: + found = [sources[0]] * len(paragraphs) + else: + found = await languages_of([p.text for p in paragraphs]) + installed = {x.code for x in await languages()} # Chinese and Japanese go without spaces: two pieces of one sentence # ("…奎琳，我从…") need one between them once translated. followed = {i for p in paragraphs for i in p.pieces[:-1]} by_language: dict[str, list[int]] = {} for paragraph, language in zip(paragraphs, found, strict=True): - if language and _base(language) != _base(target): + if language in installed and _base(language) != _base(target): by_language.setdefault(language, []).extend(paragraph.pieces) for language, where in by_language.items(): todo: list[tuple[int, str, str, str]] = [] for i in where: m = _EDGE_SPACE.match(html_lib.unescape(pieces[i])) lead, core, trail = m.groups() if m else ("", "", "") + # A list marker stays out: the models repeat it ("1. Thursday" → + # "1. 1. Czwartek"). + marker = _LIST_MARKER.match(core) + if marker: + lead, core = lead + marker.group(0), core[marker.end() :] if not trail and i in followed and _base(language) in _NO_SPACES: trail = " " if any(c.isalnum() for c in core): todo.append((i, lead, core, trail)) - for start in range(0, len(todo), 100): - batch = todo[start : start + 100] - result = await _post( - "/translate", - {"q": [core for _, _, core, _ in batch], "source": language, "target": target}, + done = await _texts([core for _, _, core, _ in todo], language, target) + # Sentence starts that came through untranslated: once more, lower-case. + again = [ + (k, words) + for k, ((_, _, core, _), text) in enumerate(zip(todo, done, strict=True)) + if (words := _copied_starts(core, text, watch)) + ] + if again: + retried = await _texts( + [_lowered(todo[k][2], words) for k, words in again], language, target ) - out = result.get("translatedText") if isinstance(result, dict) else None - if not isinstance(out, list) or len(out) != len(batch): - raise TranslationError("the translation service sent no translation") - for (i, lead, _, trail), text in zip(batch, out, strict=True): - pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False) + for (k, words), text in zip(again, retried, strict=True): + # Still there: a name after all, and the first try stands. + if len(_copied_starts(todo[k][2], text, watch)) < len(words): + done[k] = _as_before(_capitalised(text), done[k]) + for (i, lead, core, trail), text in zip(todo, done, strict=True): + lost = [n for n in watch if n in core and not _kept(n, text)] + if lost: + text = await _with_names_kept(core, lost, language, target) + pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False) return "".join(pieces) _NO_SPACES = {"zh", "ja", "th"} +_LIST_MARKER = re.compile(r"(?:\d{1,3}[.)]|[a-zA-Z][.)]|[-•*–])\s+(?=\S)") _EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL) # Web and mail addresses in the text: the models break them up ("https: / / -# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps. +# vimeo.com"), so they are marked translate="no" and not sent. _ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE) _TAG = re.compile(r"(<[^>]*>)") @@ -243,36 +400,9 @@ def protect_addresses(html: str) -> str: return "".join(out) -_PUNCT = re.compile(r"[\W_]+") -MAX_NAMES = 200 # names checked for one message - - -async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]: - """Of ``names``, those the model changes when it translates them alone - ("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one - request for them all. If the check fails, all of them.""" - wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES] - if not wanted: - return set() - try: - result = await _post( - "/translate", {"q": wanted, "source": source or "auto", "target": target} - ) - except TranslationError: - return set(wanted) - out = result.get("translatedText") if isinstance(result, dict) else None - if not isinstance(out, list) or len(out) != len(wanted): - return set(wanted) - - def same(a: str, b: str) -> bool: - return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold() - - return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))} - - # Bumped when what goes to the service, or what is done with the result, # changes: translations kept from before are made again. -PIPELINE_VERSION = 5 +PIPELINE_VERSION = 8 async def translate_html( @@ -288,16 +418,6 @@ async def translate_html( prepared = prep.join_wrapped_lines(html) for language in sources: prepared = prep.translate_phrases(prepared, language, target) - single = sources[0] if len(sources) == 1 else "auto" - candidates = {*names, *prep.find_names(prepared, single if sources else "")} - terms = {*keep, *await names_to_keep(candidates, single, target)} - prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms))) - if len(sources) > 1: - return prep.tidy(await _translate_paragraphs(prepared, target)) - result = await _post( - "/translate", - {"q": prepared, "source": source or "auto", "target": target, "format": "html"}, - ) - if not isinstance(result, dict) or "translatedText" not in result: - raise TranslationError("the translation service sent no translation") - return prep.tidy(str(result["translatedText"])) + candidates = {*names, *prep.find_names(prepared, sources[0] if sources else "")} + prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, keep))) + return prep.tidy(await _translate_paragraphs(prepared, target, sources, candidates)) diff --git a/backend/app/translation_prep.py b/backend/app/translation_prep.py index aff7cf1..256e7d3 100644 --- a/backend/app/translation_prep.py +++ b/backend/app/translation_prep.py @@ -1,25 +1,25 @@ """Getting a message ready for translation, and tidying what comes back. -LibreTranslate translates each piece of text between tags on its own, and its -small models stumble on things a reader would not: a sentence wrapped onto two -lines becomes two sentences ("ask you about" / "Plan for next month"), -"Best," becomes "Najlepsze," and a company name gets translated word by word. -So before a message goes out: +A message is translated as plain text, piece by piece (the text between +tags), and the models stumble on things a reader would not: a sentence +wrapped onto two lines becomes two sentences ("ask you about" / "Plan for next +month"), "Best," becomes "Najlepsze," and a company name gets translated word +by word. So before a message goes out: - lines that carry on a sentence (no end punctuation, next starts lowercase) are joined, in HTML (
, one
per line) and in wrapped plain text; - greetings and sign-offs that make a whole line are put in the target language from a small phrase table (English, Polish, German); -- names are marked translate="no", which LibreTranslate keeps — the people - on the message, the user's own names and signature names, and capitalised - multi-word names mid-sentence — but only the names the model would change - ("Milan" → "Mediolan"): a kept name splits its sentence, and LibreTranslate - translates each piece alone ("Dear Lukas Remis," → - "Kochanie."), so a name the model leaves alone stays in the sentence; +- names are marked translate="no" and not sent — the people on the message, + the user's own names and signature names, and capitalised multi-word names + mid-sentence — but only the names the model would change ("Milan" → + "Mediolan"): a kept name splits its sentence into pieces translated alone + ("Dear Lukas Remis," → "Kochanie."), so a name the model + leaves alone stays in the sentence; - so are emoji, file names ("A_0023C017_p1CAC.mov") and text without a letter; -- a message in more than one language is translated paragraph by paragraph, - each from its own language (split_pieces); those in the target stay; -- and afterwards, the space LibreTranslate drops after a kept word comes back. +- the message is translated paragraph by paragraph (split_pieces), each from + its own language when there are several; those in the target stay; +- and afterwards, a kept word gets back the spaces around it. """ from __future__ import annotations @@ -315,7 +315,7 @@ def split_pieces(html: str) -> tuple[list[str], list[Paragraph]]: # --- after translation ------------------------------------------------------------- -# LibreTranslate drops the space after a kept span ("Brutoni twoje"). +# A kept span can lose the space beside it ("Brutoni twoje"). # Only kept spans: they still carry translate="no" here (the sanitizer that # runs next takes it off); other spans are the sender's and stay as they are. _KEPT = re.compile(r'([^<]*)') diff --git a/backend/tests/api/test_admin_translation.py b/backend/tests/api/test_admin_translation.py new file mode 100644 index 0000000..9dd1b0c --- /dev/null +++ b/backend/tests/api/test_admin_translation.py @@ -0,0 +1,81 @@ +"""Email translation's languages in the admin panel (app/api/admin.py), with +the translator service replaced by a stand-in.""" + +from __future__ import annotations + +import pytest + +from app import translation +from tests.api.test_admin_users import World, world # noqa: F401 — the fixture + + +@pytest.fixture +def service(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[str]]: + state: dict[str, list[str]] = {"installed": ["en", "pl"], "calls": []} + + async def status() -> dict[str, object]: + return { + "installed": list(state["installed"]), + "installing": {"de": {"done": 10, "total": 40}} if "de" in state["calls"] else {}, + "available": ["en", "pl", "de", "hu"], + "errors": {}, + } + + async def install(code: str) -> None: + state["calls"].append(code) + + async def remove(code: str) -> None: + state["installed"].remove(code) + + monkeypatch.setattr(translation, "enabled", lambda: True) + monkeypatch.setattr(translation, "status", status) + monkeypatch.setattr(translation, "install", install) + monkeypatch.setattr(translation, "remove", remove) + return state + + +async def test_admin_installs_and_removes_languages( + world: World, # noqa: F811 + service: dict[str, list[str]], +) -> None: + async with await world.client(await world.user(admin=True)) as c: + got = (await c.get("/api/admin/translation")).json() + assert got["reachable"] and got["installed"] == ["en", "pl"] + assert got["available"] == ["en", "pl", "de", "hu"] + + got = (await c.post("/api/admin/translation/languages/de")).json() + assert service["calls"] == ["de"] + assert got["installing"] == {"de": {"done": 10, "total": 40}} + + got = (await c.delete("/api/admin/translation/languages/pl")).json() + assert got["installed"] == ["en"] + + assert (await c.post("/api/admin/translation/languages/x;rm")).status_code == 422 + + +async def test_only_admins(world: World, service: dict[str, list[str]]) -> None: # noqa: F811 + async with await world.client(await world.user()) as c: + assert (await c.get("/api/admin/translation")).status_code == 403 + assert (await c.post("/api/admin/translation/languages/de")).status_code == 403 + assert service["calls"] == [] + + +async def test_unreachable_service_is_reported( + world: World, # noqa: F811 + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def down() -> dict[str, object]: + raise translation.TranslationError("down") + + monkeypatch.setattr(translation, "enabled", lambda: True) + monkeypatch.setattr(translation, "status", down) + async with await world.client(await world.user(admin=True)) as c: + got = (await c.get("/api/admin/translation")).json() + assert got == { + "enabled": True, + "reachable": False, + "installed": [], + "installing": {}, + "available": [], + "errors": {}, + } diff --git a/backend/tests/test_translation_text.py b/backend/tests/test_translation_text.py index 28d6b96..e500d7d 100644 --- a/backend/tests/test_translation_text.py +++ b/backend/tests/test_translation_text.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from app import translation -from app.translation import detect, languages_of, names_to_keep, protect_addresses, translate_html +from app.translation import detect, languages_of, protect_addresses, translate_html def test_addresses_in_text_are_kept_from_translation() -> None: @@ -25,45 +25,55 @@ def test_style_blocks_are_left_alone() -> None: assert protect_addresses(html) == html -async def test_only_names_the_model_changes_are_kept(monkeypatch: pytest.MonkeyPatch) -> None: - sent: list[object] = [] +async def test_names_go_with_their_sentence_unless_the_model_changes_them( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sent: list[list[str]] = [] + model = { + "I came across Imaginary Pixels today.": "Natknąłem się dziś na Imaginary Pixels.", + "Dear Lukas,": "Drogi Lukasie,", # an ending: still the name + "Regards, Milan": "Pozdrawiam, Mediolan", # changed: done again around it + "Regards,": "Pozdrawiam,", + } async def post(path: str, payload: dict[str, object]) -> object: - sent.append(payload["q"]) - alone = { - "Lukas Remis": "Lukas Remis.", - "Milan": "Mediolan", - "Imaginary Pixels": "Imaginanal", - } - return {"translatedText": [alone[q] for q in payload["q"]]} # type: ignore[attr-defined] + q = payload["q"] + assert isinstance(q, list) + sent.append(list(q)) + return {"translatedText": [model.get(x, x) for x in q]} + + async def installed() -> list[translation.Language]: + return [translation.Language(c, c) for c in ("en", "pl")] monkeypatch.setattr(translation, "_post", post) - kept = await names_to_keep( - ["Milan", "Lukas Remis", "Imaginary Pixels", "Milan", "x"], "en", "pl" + monkeypatch.setattr(translation, "languages", installed) + html = "

I came across Imaginary Pixels today.

Dear Lukas,

Regards, Milan

" + out = await translate_html(html, "en", "pl", names=["Imaginary Pixels", "Lukas", "Milan"]) + assert out == ( + "

Natknąłem się dziś na Imaginary Pixels.

Drogi Lukasie,

" + "

Pozdrawiam, Milan

" ) - assert kept == {"Milan", "Imaginary Pixels"} # "Lukas Remis" stays in its sentence - assert len(sent) == 1 # one request for all - - -async def test_all_names_kept_when_the_check_fails(monkeypatch: pytest.MonkeyPatch) -> None: - async def post(path: str, payload: dict[str, object]) -> object: - raise translation.TranslationError("down") - - monkeypatch.setattr(translation, "_post", post) - assert await names_to_keep(["Milan", "Lukas Remis"], "en", "pl") == {"Milan", "Lukas Remis"} + assert sent[1:] == [["Regards,"]] # only the piece that lost a name went again def _fake_detector(monkeypatch: pytest.MonkeyPatch) -> list[str]: """A detector that knows Chinese signs, Polish letters, else English.""" asked: list[str] = [] - async def detect_one(text: str) -> str: + def one(text: str) -> str: asked.append(text) if any("\u4e00" <= c <= "\u9fff" for c in text): return "zh-Hans" return "pl" if any(c in "ąćęłńóśźż" for c in text.lower()) else "en" - monkeypatch.setattr(translation, "_detect_one", detect_one) + async def detect_many(texts: list[str]) -> list[str]: + return [one(t) for t in texts] + + async def installed() -> list[translation.Language]: + return [translation.Language(c, c) for c in ("en", "pl", "zh-Hans")] + + monkeypatch.setattr(translation, "_detect_many", detect_many) + monkeypatch.setattr(translation, "languages", installed) return asked @@ -102,7 +112,7 @@ async def test_each_paragraph_from_its_own_language(monkeypatch: pytest.MonkeyPa q = payload["q"] if path == "/translate" and isinstance(q, list) and payload.get("source") != "auto": calls.append((str(payload["source"]), list(q))) - return {"translatedText": [f"PL({x})" for x in q]} + return {"translatedText": [f"PL({x.upper()})" for x in q]} return {"translatedText": list(q) if isinstance(q, list) else q} # names: unchanged monkeypatch.setattr(translation, "_post", post) @@ -117,7 +127,84 @@ async def test_each_paragraph_from_its_own_language(monkeypatch: pytest.MonkeyPa assert "
Widzę, że złożyłeś wniosek o otwarcie sklepu & więcej.
" in out # Two pieces of one Chinese sentence get a space between them. assert "PL(我是速卖通的官方客户经理奎琳，) PL(" in out - assert "PL(I noticed that you applied to join the platform & more.)" in out.replace( + assert "PL(I NOTICED THAT YOU APPLIED TO JOIN THE PLATFORM & MORE.)" in out.replace( "&", "&" ) assert '+48 780 549 877' in out + + +async def test_one_language_needs_no_detector_and_skips_what_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + asked = _fake_detector(monkeypatch) + calls: list[tuple[str, list[str]]] = [] + + async def post(path: str, payload: dict[str, object]) -> object: + q = payload["q"] + if path == "/translate" and isinstance(q, list) and payload.get("source") == "en": + calls.append(("en", list(q))) + return {"translatedText": [f"PL({x.upper()})" for x in q]} + return {"translatedText": list(q) if isinstance(q, list) else q} + + monkeypatch.setattr(translation, "_post", post) + html = "

Hello there, how are you?

Short one.

1. Thursday 2nd July

" + out = await translate_html(html, "en", "pl") + assert out == ( + "

PL(HELLO THERE, HOW ARE YOU?)

PL(SHORT ONE.)

" + "

1. PL(THURSDAY 2ND JULY)

" # the list marker is not sent + ) + assert asked == [] # the message's one language is known: no detection + # Hungarian is not installed: left as it is, nothing sent. + calls.clear() + assert await translate_html("

Jó napot kívánok

", "hu", "pl") == "

Jó napot kívánok

" + assert calls == [] + + +async def test_a_sentence_start_the_model_copied_is_tried_lower_case( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model = { + "Enclosed is a summary. Natalia wrote it.": ( + "Enclosed jest podsumowaniem. Natalia to napisała." + ), + "enclosed is a summary. natalia wrote it.": ( + "załączone jest podsumowanie. Natalia to napisała." + ), + "Natalia wrote it.": "Natalia to napisała.", + "natalia wrote it.": "Natalia to napisała.", + } + sent: list[str] = [] + + async def post(path: str, payload: dict[str, object]) -> object: + q = payload["q"] + assert isinstance(q, list) + sent.extend(q) + return {"translatedText": [model[x] for x in q]} + + async def installed() -> list[translation.Language]: + return [translation.Language(c, c) for c in ("en", "pl")] + + monkeypatch.setattr(translation, "_post", post) + monkeypatch.setattr(translation, "languages", installed) + out = await translate_html("

Enclosed is a summary. Natalia wrote it.

", "en", "pl") + assert out == "

Załączone jest podsumowanie. Natalia to napisała.

" + # A name comes back the same lower-case too: the first try stands. + sent.clear() + assert await translate_html("

Natalia wrote it.

", "en", "pl") == ( + "

Natalia to napisała.

" + ) + assert sent == ["Natalia wrote it.", "natalia wrote it."] + + +def test_capitalised_again_but_not_after_abbreviations() -> None: + assert translation._capitalised("załączony na lipiec 2026 r., od 13 lipca. dalej np. tak") == ( + "Załączony na lipiec 2026 r., od 13 lipca. Dalej np. tak" + ) + + +def test_a_comma_word_keeps_the_first_try_case() -> None: + first = "Enclosed jest podsumowaniem na lipiec 2026, od 13 lipca." + again = "Załączony jest podsumowanie na lipiec 2026 r., Od 13 lipca, Anna wie." + assert translation._as_before(again, first) == ( + "Załączony jest podsumowanie na lipiec 2026 r., od 13 lipca, Anna wie." + ) diff --git a/deploy/build-push.sh b/deploy/build-push.sh index 6f64278..568bafe 100755 --- a/deploy/build-push.sh +++ b/deploy/build-push.sh @@ -43,15 +43,17 @@ build() { "$ROOT_DIR" } -build backend Dockerfile.backend -build web Dockerfile.frontend +build backend Dockerfile.backend +build web Dockerfile.frontend +build translator Dockerfile.translator if [ "$MODE" = "--save" ]; then out="${ROOT_DIR}/webmail-${TAG}.tar" - echo "== saving both images to ${out}" + echo "== saving the images to ${out}" docker save -o "$out" \ "${REGISTRY_PREFIX}/backend:${TAG}" \ - "${REGISTRY_PREFIX}/web:${TAG}" + "${REGISTRY_PREFIX}/web:${TAG}" \ + "${REGISTRY_PREFIX}/translator:${TAG}" echo "done. Copy it to the NAS and run: docker load -i $(basename "$out")" exit 0 fi @@ -72,6 +74,8 @@ fi echo "== pushing" docker push "${REGISTRY_PREFIX}/backend:${TAG}" docker push "${REGISTRY_PREFIX}/web:${TAG}" +docker push "${REGISTRY_PREFIX}/translator:${TAG}" docker push "${REGISTRY_PREFIX}/backend:latest" docker push "${REGISTRY_PREFIX}/web:latest" +docker push "${REGISTRY_PREFIX}/translator:latest" echo "pushed ${TAG}. On the NAS: ./deploy/webmail.sh update ${TAG}" diff --git a/deploy/webmail.sh b/deploy/webmail.sh index 0aefc16..3cb407b 100755 --- a/deploy/webmail.sh +++ b/deploy/webmail.sh @@ -436,6 +436,24 @@ print('OFF' if not url else urllib.request.urlopen(url + '/v2/languages', timeou printf ' %s\n' "$lt" ;; esac + # Email translation: is the translator answering, with which languages? + tr="$(compose exec -T api python -c " +import json, os, urllib.request +url = os.environ.get('TRANSLATOR_URL', '') +if not url: + print('OFF') +else: + d = json.load(urllib.request.urlopen(url + '/languages', timeout=5)) + busy = ', '.join(d.get('installing') or {}) + print('OK ' + ' '.join(d['installed']) + (' (installing: ' + busy + ')' if busy else '')) +" 2>&1 | tail -1)" || true + case "$tr" in + OK*) grn " translation answering; languages: ${tr#OK }" ;; + OFF) ylw " translation is off (TRANSLATOR_URL is empty)" ;; + *) ylw " translation not answering — the Translate button stays hidden" + printf ' %s\n' "$tr" ;; + esac + if command -v ss >/dev/null 2>&1; then local p for p in "WEB_PORT:${WEB_PORT:-8088}" "ADMIN_PORT:${ADMIN_PORT:-8089}"; do @@ -503,7 +521,7 @@ cmd_pull() { Set REGISTRY_PREFIX and IMAGE_TAG in ${ENV_FILE}, or use: webmail.sh up --build" detect_compose IMAGE_TAG="$tag" compose pull || die "pull failed — is the registry reachable and logged in?" - grn "pulled ${REGISTRY_PREFIX}/{backend,web}:${tag}. Next: ./deploy/webmail.sh up" + grn "pulled ${REGISTRY_PREFIX}/{backend,web,translator}:${tag}. Next: ./deploy/webmail.sh up" } cmd_down() { require_env; compose --profile meili down; grn "stopped (data kept in ${DATA_ROOT})"; } @@ -587,6 +605,10 @@ cmd_update() { # then the API, then the web front end. IMAGE_TAG="$target" compose up -d worker-sync worker-jobs IMAGE_TAG="$target" compose up -d api web + # The translator is ours too, so it follows the tag. Named here (not left + # to --no-recreate below) so it is replaced on every update, and so init, + # which it depends on, runs again when a release adds a data directory. + IMAGE_TAG="$target" compose up -d translator # Converge anything the ordered roll does not name — a release that adds a # service (the antispam engine did) would otherwise never start it, and the # stack would look updated while missing a container. --no-recreate so this diff --git a/docker-compose.yml b/docker-compose.yml index 2147137..1e0f706 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ x-backend-env: &backend-env # Spelling and grammar check in the compose window. Empty turns it off. LANGUAGETOOL_URL: ${LANGUAGETOOL_URL-http://languagetool:8010} # Email translation; empty turns it off. - LIBRETRANSLATE_URL: ${LIBRETRANSLATE_URL-http://libretranslate:5000} + TRANSLATOR_URL: ${TRANSLATOR_URL-http://translator:5000} LOG_LEVEL: ${LOG_LEVEL:-INFO} UNDO_SECONDS: ${UNDO_SECONDS:-10} SESSION_TTL_DAYS: ${SESSION_TTL_DAYS:-30} @@ -111,13 +111,13 @@ services: - | set -e mkdir -p /mnt/postgres /mnt/blobs /mnt/backups /mnt/redis /mnt/meili /mnt/state \ - /mnt/languagetool-ngrams /mnt/libretranslate + /mnt/languagetool-ngrams /mnt/translator # Only recurse when ownership is actually wrong: `chown -R` over a large # blob store on every start would be minutes of pointless disk churn. [ "$$(stat -c %u /mnt/blobs)" = "10001" ] || chown -R 10001:10001 /mnt/blobs [ "$$(stat -c %u /mnt/backups)" = "999" ] || chown -R 999:999 /mnt/backups [ "$$(stat -c %u /mnt/redis)" = "999" ] || chown -R 999:999 /mnt/redis - [ "$$(stat -c %u /mnt/libretranslate)" = "1032" ] || chown -R 1032:65534 /mnt/libretranslate + [ "$$(stat -c %u /mnt/translator)" = "10001" ] || chown -R 10001:10001 /mnt/translator chmod 700 /mnt/backups echo "[init] data directories ready" volumes: @@ -316,30 +316,26 @@ services: cpus: ${LANGUAGETOOL_CPUS:-1.0} logging: *logging - # Email translation (app/translation.py), all on this server. The first - # start downloads every language model (a few GB) into DATA_ROOT; later - # starts reuse them. A model is loaded into memory the first time its - # language is used. - libretranslate: - image: libretranslate/libretranslate:v1.9.6 + # Email translation (translator/, app/translation.py), all on this server: + # Mozilla's Firefox Translations models and fastText language detection. + # The first start downloads the default languages (about 350 MB) into + # DATA_ROOT; more are installed from the admin panel. + translator: + image: ${REGISTRY_PREFIX:-webmail}/translator:${IMAGE_TAG:-local} + build: + context: . + dockerfile: docker/Dockerfile.translator environment: - LT_DISABLE_WEB_UI: "true" - LT_UPDATE_MODELS: "false" - LT_THREADS: ${LIBRETRANSLATE_THREADS:-2} + TRANSLATOR_LANGUAGES: ${TRANSLATOR_LANGUAGES:-pl,de,fr,es,it} + TRANSLATOR_THREADS: ${TRANSLATOR_THREADS:-2} TZ: ${TZ:-UTC} volumes: - - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/libretranslate:/home/libretranslate/.local - healthcheck: - test: ["CMD-SHELL", "python3 -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/languages', timeout=4)\""] - interval: 30s - timeout: 6s - retries: 5 - start_period: 30m # the first start downloads the models + - ${DATA_ROOT:-/NO-.env-FOUND-RUN-deploy_webmail.sh-init-FIRST}/translator:/data depends_on: init: { condition: service_completed_successfully } restart: unless-stopped - mem_limit: ${LIBRETRANSLATE_MEM_LIMIT:-6g} - cpus: ${LIBRETRANSLATE_CPUS:-2.0} + mem_limit: ${TRANSLATOR_MEM_LIMIT:-2g} + cpus: ${TRANSLATOR_CPUS:-2.0} logging: *logging # Bayes tokens and fuzzy hashes. Separate from the app's redis on purpose diff --git a/docker/Dockerfile.translator b/docker/Dockerfile.translator new file mode 100644 index 0000000..5fccf76 --- /dev/null +++ b/docker/Dockerfile.translator @@ -0,0 +1,38 @@ +# Translation service (translator/translator.py): Mozilla's Firefox +# Translations models on the Bergamot engine, and fastText language +# identification. Python 3.10: the newest the bergamot package is built for. +# +# docker build -f docker/Dockerfile.translator -t webmail/translator . +# +# The models are not in the image: they are downloaded into /data (a bind +# mount) on the first start and from the admin panel. + +FROM python:3.10-slim + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + DATA_DIR=/data \ + HOME=/tmp + +COPY translator/requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt + +# fastText's language identifier (lid.176, compressed; CC BY-SA 3.0). +ADD --checksum=sha256:8f3472cfe8738a7b6099e8e999c3cbfae0dcd15696aac7d7738a8039db603e83 \ + https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz /opt/lid.176.ftz + +COPY translator/translator.py /app/translator.py + +# The same uid as the backend: DATA_ROOT/translator is owned by it (init). +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin translator \ + && mkdir -p /data && chown translator /data /opt/lid.176.ftz +USER translator +WORKDIR /app +EXPOSE 5000 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=5 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/health', timeout=4)" + +CMD ["uvicorn", "translator:app", "--host", "0.0.0.0", "--port", "5000", "--no-access-log"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1593809..67ae976 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -158,22 +158,18 @@ services: retries: 12 restart: unless-stopped - # Email translation (app/translation.py). The first start downloads every - # language model (several GB) into the volume; later starts reuse them. - libretranslate: - image: libretranslate/libretranslate:v1.9.6 - environment: - LT_DISABLE_WEB_UI: "true" - LT_UPDATE_MODELS: "false" + # Email translation (translator/, app/translation.py): Firefox Translations + # models and fastText. The first start downloads the default languages + # (about 350 MB) into the volume; the admin panel adds more. + translator: + build: + context: .. + dockerfile: docker/Dockerfile.translator + image: webmail/translator:dev ports: - "127.0.0.1:5005:5000" volumes: - - ltmodels:/home/libretranslate/.local - healthcheck: - test: ["CMD-SHELL", "python3 -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/languages', timeout=4)\""] - interval: 15s - timeout: 6s - retries: 240 + - translatordata:/data restart: unless-stopped greenmail: @@ -188,7 +184,7 @@ services: - "127.0.0.1:3993:3993" # IMAPS volumes: - ltmodels: + translatordata: pgdata: meilidata: blobs: diff --git a/docs/nas-install.md b/docs/nas-install.md index 944a1c9..ee6f37a 100644 --- a/docs/nas-install.md +++ b/docs/nas-install.md @@ -701,6 +701,23 @@ priority, so even the first checks after a restart are served from RAM. On mechanical disks that warm-up takes several minutes; an NVMe cache in front of the volume makes it — and any later cache misses — fast. +### Email translation (translator) + +`translator` translates email in the reading pane, entirely on this server: +Mozilla's Firefox Translations models (the ones Firefox uses for page +translation) and fastText for telling which language a message is in. + +- The first start downloads the languages in `TRANSLATOR_LANGUAGES` (default + `pl,de,fr,es,it`; English is always there) into `$DATA_ROOT/translator`, + about 35 MB per language and direction (~350 MB for the default set). +- More languages are installed, and unused ones removed, from the **admin + panel → Translation**. That list is kept in `$DATA_ROOT/translator`, so a + restart does not undo it. +- Models translate to and from English; other pairs go through English. +- Capped at 2 GB of RAM and 2 CPUs (`TRANSLATOR_MEM_LIMIT`, `TRANSLATOR_CPUS`). +- `./deploy/webmail.sh status` shows whether it answers and which languages + are installed. `TRANSLATOR_URL=` (empty) in `.env` turns translation off. + Two NAS-specific realities worth knowing: - **Disk hibernation won't happen.** Active accounts hold IMAP IDLE connections diff --git a/frontend/icons/ultimo-logo-signet.svg b/frontend/icons/ultimo-logo-signet.svg index f5ff865..ae125ed 100644 --- a/frontend/icons/ultimo-logo-signet.svg +++ b/frontend/icons/ultimo-logo-signet.svg @@ -8,9 +8,31 @@ version="1.1" id="svg1" xml:space="preserve" + sodipodi:docname="ultimo-logo-signet.svg" + inkscape:version="1.4.4 (dcaf3e7d9e, 2026-05-05)" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" - xmlns:svg="http://www.w3.org/2000/svg"> + gradientTransform="matrix(0.75684298,0,0,0.80151105,3.0076311,94.135157)" /> diff --git a/frontend/icons/ultimo-logo.svg b/frontend/icons/ultimo-logo.svg index 4d24abe..a62679f 100644 --- a/frontend/icons/ultimo-logo.svg +++ b/frontend/icons/ultimo-logo.svg @@ -8,10 +8,48 @@ version="1.1" id="svg1" xml:space="preserve" + sodipodi:docname="ultimo-logo.svg" + inkscape:version="1.4.4 (dcaf3e7d9e, 2026-05-05)" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" - xmlns:svg="http://www.w3.org/2000/svg"> + gradientTransform="matrix(0.75684298,0,0,0.80151105,3.0076311,94.135157)" />ltimo diff --git a/frontend/public/apple-touch-icon.png b/frontend/public/apple-touch-icon.png index 648f6a8..fb785d5 100644 Binary files a/frontend/public/apple-touch-icon.png and b/frontend/public/apple-touch-icon.png differ diff --git a/frontend/public/badge-96.png b/frontend/public/badge-96.png index 638eb24..29a5ec4 100644 Binary files a/frontend/public/badge-96.png and b/frontend/public/badge-96.png differ diff --git a/frontend/public/bimi/ultimo-logo.svg b/frontend/public/bimi/ultimo-logo.svg index 808c5cd..9077174 100644 --- a/frontend/public/bimi/ultimo-logo.svg +++ b/frontend/public/bimi/ultimo-logo.svg @@ -1,14 +1,7 @@ Ultimo Email Ultimo Email logo, for BIMI (SVG Tiny Portable/Secure) - - - - - - + - - - + diff --git a/frontend/public/favicon.ico b/frontend/public/favicon.ico index 2293c69..30c35a4 100644 Binary files a/frontend/public/favicon.ico and b/frontend/public/favicon.ico differ diff --git a/frontend/public/icon-192.png b/frontend/public/icon-192.png index 971a11b..cf4284b 100644 Binary files a/frontend/public/icon-192.png and b/frontend/public/icon-192.png differ diff --git a/frontend/public/icon-512.png b/frontend/public/icon-512.png index 5695f87..76f19c4 100644 Binary files a/frontend/public/icon-512.png and b/frontend/public/icon-512.png differ diff --git a/frontend/public/icon-maskable-192.png b/frontend/public/icon-maskable-192.png index facbea5..395f73c 100644 Binary files a/frontend/public/icon-maskable-192.png and b/frontend/public/icon-maskable-192.png differ diff --git a/frontend/public/icon-maskable-512.png b/frontend/public/icon-maskable-512.png index 71a0250..43336b8 100644 Binary files a/frontend/public/icon-maskable-512.png and b/frontend/public/icon-maskable-512.png differ diff --git a/frontend/public/manifest.webmanifest b/frontend/public/manifest.webmanifest index 0434241..6f0755e 100644 --- a/frontend/public/manifest.webmanifest +++ b/frontend/public/manifest.webmanifest @@ -7,7 +7,7 @@ "scope": "/", "display": "standalone", "orientation": "any", - "background_color": "#ffffff", + "background_color": "#1b1e20", "theme_color": "#f3f4f6", "categories": ["productivity", "business"], "icons": [ diff --git a/frontend/public/ultimo-logo-signet.svg b/frontend/public/ultimo-logo-signet.svg index f5ff865..567fffc 100644 --- a/frontend/public/ultimo-logo-signet.svg +++ b/frontend/public/ultimo-logo-signet.svg @@ -26,9 +26,87 @@ x2="54.340946" y2="162.66415" gradientUnits="userSpaceOnUse" - gradientTransform="matrix(0.75684298,0,0,0.80151105,3.0076311,94.135157)" /> + gradientTransform="matrix(0.75684298,0,0,0.80151105,3.0076311,94.135157)" /> diff --git a/frontend/src/admin/AdminApp.tsx b/frontend/src/admin/AdminApp.tsx index ac254e6..7386034 100644 --- a/frontend/src/admin/AdminApp.tsx +++ b/frontend/src/admin/AdminApp.tsx @@ -2,10 +2,10 @@ import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tansta import { useEffect, useState } from "react"; import { createPortal } from "react-dom"; -import { useT } from "../i18n"; +import { languageName, useLocale, useT } from "../i18n"; import { api, ApiError } from "../lib/api"; import { useMe } from "../lib/auth"; -import type { AdminMail, AdminUser } from "../lib/types"; +import type { AdminMail, AdminTranslation, AdminUser } from "../lib/types"; import { useConfirm } from "../components/AppDialog"; const GIB = 1024 ** 3; @@ -44,8 +44,9 @@ export function AdminApp() {
{/* The divider runs the full height of the taller column. */} -
+
+
)} @@ -287,6 +288,136 @@ function Panel({ handle }: { handle: string }) { ); } +/** Email translation's languages: installed ones (removable, but English), + * downloads in progress, and any language Mozilla has models for to add. */ +function Translation() { + const t = useT(); + const { locale } = useLocale(); + const queryClient = useQueryClient(); + const [confirm, confirmDialog] = useConfirm(); + const [adding, setAdding] = useState(""); + const info = useQuery({ + queryKey: ["adminTranslation"], + queryFn: api.adminTranslation, + // While something downloads, follow its progress. + refetchInterval: (q) => + Object.keys((q.state.data as AdminTranslation | undefined)?.installing ?? {}).length ? 1500 : false, + }); + const done = (data: AdminTranslation) => { + queryClient.setQueryData(["adminTranslation"], data); + void queryClient.invalidateQueries({ queryKey: ["translation-languages"] }); + }; + const install = useMutation({ + mutationFn: api.installTranslationLanguage, + onSuccess: (data) => { + done(data); + setAdding(""); + }, + }); + const remove = useMutation({ mutationFn: api.removeTranslationLanguage, onSuccess: done }); + + const name = (code: string) => languageName(code, locale); + const data = info.data; + const busy = install.isPending || remove.isPending; + const installing = Object.entries(data?.installing ?? {}); + const addable = (data?.available ?? []) + .filter((c) => !data?.installed.includes(c) && !(c in (data?.installing ?? {}))) + .sort((a, b) => name(a).localeCompare(name(b), locale)); + const error = (install.error ?? remove.error) as unknown; + + return ( +
+

{t("admin.translation")}

+

{t("admin.translationHelp")}

+ {info.error instanceof ApiError &&

{info.error.message}

} + {data && !data.enabled &&

{t("admin.translationOff")}

} + {data?.enabled && !data.reachable && ( +

{t("admin.translationUnreachable")}

+ )} + {data?.reachable && ( + <> +
    + {[...data.installed] + .sort((a, b) => name(a).localeCompare(name(b), locale)) + .map((code) => ( +
  • + + {name(code)} {code} + + {code === "en" ? ( + {t("admin.translationAlways")} + ) : ( + + )} +
    • + ))} + {installing.map(([code, p]) => ( +
  • + {name(code)} + + {t("admin.translationInstalling", { + percent: p.total ? Math.floor((p.done / p.total) * 100) : 0, + })} + +
    • + ))} +
+ {Object.entries(data.errors).map(([code, why]) => ( +

+ {t("admin.translationFailed", { language: name(code) })} {why} +

+ ))} + {addable.length > 0 ? ( +
+ + +
+ ) : ( + data.available.length === 0 && ( +

{t("admin.translationCatalogueOffline")}

+ ) + )} + {error instanceof ApiError &&

{error.message}

} + + )} + {confirmDialog} +
+ ); +} + type MailDraft = Omit; /** The SMTP account the server sends password-reset links from. */ diff --git a/frontend/src/components/thread/ThreadView.tsx b/frontend/src/components/thread/ThreadView.tsx index 387dd93..6ef64a7 100644 --- a/frontend/src/components/thread/ThreadView.tsx +++ b/frontend/src/components/thread/ThreadView.tsx @@ -478,7 +478,7 @@ function MessageItem({ : []; const offerTranslate = translateCodes.length > 0 && - (translateTo !== null || needsTranslation(language, locale)); + (translateTo !== null || needsTranslation(language, locale, translateCodes)); const target = translateCodes.find((c) => sameLanguage(c, locale)) ?? null; const replyBtn = (kind: string, label: string, icon: string) => (