Translation on Firefox Translations models; new logo

- Email translation moves from LibreTranslate to our own translator service (translator/): Mozilla's Firefox Translations models on the Bergamot engine, fastText language detection (176 languages). Polish, English, German, French, Spanish and Italian on first start (~350 MB instead of ~10 GB); Admin panel > Email translation installs and removes languages. Built and pushed as the `translator` image. - Every message is translated paragraph by paragraph as plain text; names travel with their sentence and are kept only where the model changed them; a sentence start the model copied ("Enclosed") is tried again in lower case; list markers are not sent - The Translate button shows only for installed languages - New logo: favicon, app/maskable/iPhone icons, notification badge, email logo and calendar copy regenerated; BIMI logo rebuilt in SVG Tiny PS with its gradients; masks render in browsers - Launch screen background is the app's dark grey Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 22:04:04 +02:00
parent 8a1b9979e9
commit 972e10876b
38 changed files with 1627 additions and 223 deletions
@@ -5,7 +5,7 @@ from __future__ import annotations
 import pytest

 from app import translation
-from app.translation import detect, languages_of, names_to_keep, protect_addresses, translate_html
+from app.translation import detect, languages_of, protect_addresses, translate_html


 def test_addresses_in_text_are_kept_from_translation() -> None:
@@ -25,45 +25,55 @@ def test_style_blocks_are_left_alone() -> None:
    assert protect_addresses(html) == html


-async def test_only_names_the_model_changes_are_kept(monkeypatch: pytest.MonkeyPatch) -> None:
-    sent: list[object] = []
+async def test_names_go_with_their_sentence_unless_the_model_changes_them(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    sent: list[list[str]] = []
+    model = {
+        "I came across Imaginary Pixels today.": "Natknąłem się dziś na Imaginary Pixels.",
+        "Dear Lukas,": "Drogi Lukasie,",  # an ending: still the name
+        "Regards, Milan": "Pozdrawiam, Mediolan",  # changed: done again around it
+        "Regards,": "Pozdrawiam,",
+    }

    async def post(path: str, payload: dict[str, object]) -> object:
-        sent.append(payload["q"])
-        alone = {
-            "Lukas Remis": "Lukas Remis.",
-            "Milan": "Mediolan",
-            "Imaginary Pixels": "Imaginanal",
-        }
-        return {"translatedText": [alone[q] for q in payload["q"]]}  # type: ignore[attr-defined]
+        q = payload["q"]
+        assert isinstance(q, list)
+        sent.append(list(q))
+        return {"translatedText": [model.get(x, x) for x in q]}
+
+    async def installed() -> list[translation.Language]:
+        return [translation.Language(c, c) for c in ("en", "pl")]

    monkeypatch.setattr(translation, "_post", post)
-    kept = await names_to_keep(
-        ["Milan", "Lukas Remis", "Imaginary Pixels", "Milan", "x"], "en", "pl"
+    monkeypatch.setattr(translation, "languages", installed)
+    html = "<p>I came across Imaginary Pixels today.</p><p>Dear Lukas,</p><p>Regards, Milan</p>"
+    out = await translate_html(html, "en", "pl", names=["Imaginary Pixels", "Lukas", "Milan"])
+    assert out == (
+        "<p>Natknąłem się dziś na Imaginary Pixels.</p><p>Drogi Lukasie,</p>"
+        "<p>Pozdrawiam, Milan</p>"
    )
-    assert kept == {"Milan", "Imaginary Pixels"}  # "Lukas Remis" stays in its sentence
-    assert len(sent) == 1  # one request for all
-
-
-async def test_all_names_kept_when_the_check_fails(monkeypatch: pytest.MonkeyPatch) -> None:
-    async def post(path: str, payload: dict[str, object]) -> object:
-        raise translation.TranslationError("down")
-
-    monkeypatch.setattr(translation, "_post", post)
-    assert await names_to_keep(["Milan", "Lukas Remis"], "en", "pl") == {"Milan", "Lukas Remis"}
+    assert sent[1:] == [["Regards,"]]  # only the piece that lost a name went again


 def _fake_detector(monkeypatch: pytest.MonkeyPatch) -> list[str]:
    """A detector that knows Chinese signs, Polish letters, else English."""
    asked: list[str] = []

-    async def detect_one(text: str) -> str:
+    def one(text: str) -> str:
        asked.append(text)
        if any("\u4e00" <= c <= "\u9fff" for c in text):
            return "zh-Hans"
        return "pl" if any(c in "ąćęłńóśźż" for c in text.lower()) else "en"

-    monkeypatch.setattr(translation, "_detect_one", detect_one)
+    async def detect_many(texts: list[str]) -> list[str]:
+        return [one(t) for t in texts]
+
+    async def installed() -> list[translation.Language]:
+        return [translation.Language(c, c) for c in ("en", "pl", "zh-Hans")]
+
+    monkeypatch.setattr(translation, "_detect_many", detect_many)
+    monkeypatch.setattr(translation, "languages", installed)
    return asked


@@ -102,7 +112,7 @@ async def test_each_paragraph_from_its_own_language(monkeypatch: pytest.MonkeyPa
        q = payload["q"]
        if path == "/translate" and isinstance(q, list) and payload.get("source") != "auto":
            calls.append((str(payload["source"]), list(q)))
-            return {"translatedText": [f"PL({x})" for x in q]}
+            return {"translatedText": [f"PL({x.upper()})" for x in q]}
        return {"translatedText": list(q) if isinstance(q, list) else q}  # names: unchanged

    monkeypatch.setattr(translation, "_post", post)
@@ -117,7 +127,84 @@ async def test_each_paragraph_from_its_own_language(monkeypatch: pytest.MonkeyPa
    assert "<div>Widzę, że złożyłeś wniosek o otwarcie sklepu &amp; więcej.</div>" in out
    # Two pieces of one Chinese sentence get a space between them.
    assert "PL(我是速卖通的官方客户经理奎琳，) </span>PL(" in out
-    assert "PL(I noticed that you applied to join the platform & more.)" in out.replace(
+    assert "PL(I NOTICED THAT YOU APPLIED TO JOIN THE PLATFORM & MORE.)" in out.replace(
        "&amp;", "&"
    )
    assert '<span translate="no">+48 780 549 877</span>' in out
+
+
+async def test_one_language_needs_no_detector_and_skips_what_is_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    asked = _fake_detector(monkeypatch)
+    calls: list[tuple[str, list[str]]] = []
+
+    async def post(path: str, payload: dict[str, object]) -> object:
+        q = payload["q"]
+        if path == "/translate" and isinstance(q, list) and payload.get("source") == "en":
+            calls.append(("en", list(q)))
+            return {"translatedText": [f"PL({x.upper()})" for x in q]}
+        return {"translatedText": list(q) if isinstance(q, list) else q}
+
+    monkeypatch.setattr(translation, "_post", post)
+    html = "<p>Hello there, how are you?</p><p>Short one.</p><p><b>1. Thursday 2nd July</b></p>"
+    out = await translate_html(html, "en", "pl")
+    assert out == (
+        "<p>PL(HELLO THERE, HOW ARE YOU?)</p><p>PL(SHORT ONE.)</p>"
+        "<p><b>1. PL(THURSDAY 2ND JULY)</b></p>"  # the list marker is not sent
+    )
+    assert asked == []  # the message's one language is known: no detection
+    # Hungarian is not installed: left as it is, nothing sent.
+    calls.clear()
+    assert await translate_html("<p>Jó napot kívánok</p>", "hu", "pl") == "<p>Jó napot kívánok</p>"
+    assert calls == []
+
+
+async def test_a_sentence_start_the_model_copied_is_tried_lower_case(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    model = {
+        "Enclosed is a summary. Natalia wrote it.": (
+            "Enclosed jest podsumowaniem. Natalia to napisała."
+        ),
+        "enclosed is a summary. natalia wrote it.": (
+            "załączone jest podsumowanie. Natalia to napisała."
+        ),
+        "Natalia wrote it.": "Natalia to napisała.",
+        "natalia wrote it.": "Natalia to napisała.",
+    }
+    sent: list[str] = []
+
+    async def post(path: str, payload: dict[str, object]) -> object:
+        q = payload["q"]
+        assert isinstance(q, list)
+        sent.extend(q)
+        return {"translatedText": [model[x] for x in q]}
+
+    async def installed() -> list[translation.Language]:
+        return [translation.Language(c, c) for c in ("en", "pl")]
+
+    monkeypatch.setattr(translation, "_post", post)
+    monkeypatch.setattr(translation, "languages", installed)
+    out = await translate_html("<p>Enclosed is a summary. Natalia wrote it.</p>", "en", "pl")
+    assert out == "<p>Załączone jest podsumowanie. Natalia to napisała.</p>"
+    # A name comes back the same lower-case too: the first try stands.
+    sent.clear()
+    assert await translate_html("<p>Natalia wrote it.</p>", "en", "pl") == (
+        "<p>Natalia to napisała.</p>"
+    )
+    assert sent == ["Natalia wrote it.", "natalia wrote it."]
+
+
+def test_capitalised_again_but_not_after_abbreviations() -> None:
+    assert translation._capitalised("załączony na lipiec 2026 r., od 13 lipca. dalej np. tak") == (
+        "Załączony na lipiec 2026 r., od 13 lipca. Dalej np. tak"
+    )
+
+
+def test_a_comma_word_keeps_the_first_try_case() -> None:
+    first = "Enclosed jest podsumowaniem na lipiec 2026, od 13 lipca."
+    again = "Załączony jest podsumowanie na lipiec 2026 r., Od 13 lipca, Anna wie."
+    assert translation._as_before(again, first) == (
+        "Załączony jest podsumowanie na lipiec 2026 r., od 13 lipca, Anna wie."
+    )