webmail-client/backend/app/translation.py

"""Email translation, on this server: LibreTranslate (Argos models) in its own
container, reached only by the API. A message's text never leaves the server.

- A message's language is detected the first time it is shown (from its own
  text: quoted replies and signatures left out) and kept on the message, so the
  reading pane can offer "Translate" when it differs from the interface language.
- A translation is of the message HTML the user sees (already sanitized; the
  result is sanitized again), cached per message, target language and whether
  remote images were shown.

Off when ``LIBRETRANSLATE_URL`` is empty.
"""

from __future__ import annotations

import asyncio
import html as html_lib
import logging
import re
import time
from collections.abc import Iterable
from dataclasses import dataclass

import httpx

from app import translation_prep as prep
from app.completion import own_text
from app.config import get_settings

logger = logging.getLogger(__name__)

DETECT_CHARS = 1500  # of the message's own text: plenty to tell the language
MIN_CONFIDENCE = 50.0  # below this, LibreTranslate is guessing
MAX_HTML_CHARS = 150_000  # a translation request bigger than this is refused
_TIMEOUT = httpx.Timeout(120.0, connect=5.0)  # long messages take a while on a CPU
_LANG_CACHE_S = 600


class TranslationError(Exception):
    """Translation is off, the service is unreachable, or it refused the text."""


@dataclass(frozen=True)
class Language:
    code: str
    name: str


def enabled() -> bool:
    return bool(get_settings().libretranslate_url.strip())


def _url(path: str) -> str:
    return get_settings().libretranslate_url.rstrip("/") + path


async def _post(path: str, payload: dict[str, object]) -> object:
    if not enabled():
        raise TranslationError("translation is not set up on this server")
    try:
        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
            resp = await client.post(_url(path), json=payload)
    except httpx.HTTPError as exc:
        raise TranslationError(f"the translation service is unreachable: {exc}") from exc
    if resp.status_code != 200:
        detail = resp.text[:200]
        raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
    return resp.json()


_languages: tuple[float, list[Language]] = (0.0, [])


async def languages() -> list[Language]:
    """What the service can translate between (cached for a few minutes)."""
    global _languages
    stamp, cached = _languages
    if cached and time.monotonic() - stamp < _LANG_CACHE_S:
        return cached
    if not enabled():
        return []
    try:
        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
            resp = await client.get(_url("/languages"))
        resp.raise_for_status()
        rows = resp.json()
    except (httpx.HTTPError, ValueError) as exc:
        logger.info("translation languages unavailable: %s", exc)
        return []
    found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r]
    _languages = (time.monotonic(), found)
    return found


_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
MIN_LETTERS = 25  # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
MAX_DETECT = 60  # pieces of one message asked about
MIN_SHARE = 0.15  # of a message's letters, for its language to count


async def _detect_one(text: str) -> str:
    result = await _post("/detect", {"q": text[:DETECT_CHARS]})
    if not isinstance(result, list) or not result:
        return ""
    best = result[0]
    if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE:
        return ""
    return str(best.get("language", ""))


def _weight(text: str) -> int:
    """How much text this is: letters, a Chinese/Japanese/Korean sign as a word."""
    return sum(3 if _CJK.match(c) else 1 for c in text if c.isalpha())


async def languages_of(parts: list[str]) -> list[str]:
    """The language of each of ``parts`` (paragraphs, in order). Parts too
    short to tell take the language of the next part that could be told (a
    greeting goes with what follows it), the last ones that of the one before;
    parts without letters get ''."""
    # Addresses say nothing about a language ("E-mail: lianshaojuan.lsj@…" → Estonian).
    parts = [_ADDRESS.sub(" ", part) for part in parts]
    told: list[str] = [""] * len(parts)
    asked = [
        i
        for i, part in enumerate(parts)
        if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
    ][:MAX_DETECT]
    gate = asyncio.Semaphore(4)

    async def one(i: int) -> None:
        async with gate:
            told[i] = await _detect_one(parts[i])

    await asyncio.gather(*(one(i) for i in asked))
    out = list(told)
    following = ""
    for i in reversed(range(len(parts))):
        if told[i]:
            following = told[i]
        elif any(c.isalpha() for c in parts[i]):
            out[i] = following
    previous = ""
    for i, part in enumerate(parts):
        if told[i]:
            previous = told[i]
        elif not out[i] and any(c.isalpha() for c in part):
            out[i] = previous
    return out


async def detect(text: str) -> str:
    """The languages of ``text`` (a message's plain text), most of it first,
    comma-separated ("zh-Hans,pl,en"); '' if unsure. A message can be in
    several: each language that makes up a good part of it counts."""
    own = own_text(text)
    if len(" ".join(own.split())) < 20:
        return ""  # a line or two says too little
    lines: list[str] = []
    total = 0
    for line in (x.strip() for x in own.splitlines()):
        if line and total < DETECT_CHARS * 4:
            lines.append(line)
            total += len(line)
    found = await languages_of(lines)
    weight: dict[str, int] = {}
    for line, language in zip(lines, found, strict=True):
        if language:
            weight[language] = weight.get(language, 0) + _weight(_ADDRESS.sub(" ", line))
    if not weight:
        return await _detect_one(" ".join(own.split()))
    all_of_it = sum(weight.values())
    ranked = sorted(weight, key=lambda k: weight[k], reverse=True)
    return ",".join(k for k in ranked if weight[k] / all_of_it >= MIN_SHARE)


def _base(code: str) -> str:
    return code.lower().split("-")[0]


async def _translate_paragraphs(html: str, target: str) -> str:
    """``html`` in more than one language, in ``target``: each paragraph from
    its own language; paragraphs already in ``target`` are left as they are."""
    pieces, paragraphs = prep.split_pieces(html)
    found = await languages_of([p.text for p in paragraphs])
    # Chinese and Japanese go without spaces: two pieces of one sentence
    # ("…奎琳，</span>我从…") need one between them once translated.
    followed = {i for p in paragraphs for i in p.pieces[:-1]}
    by_language: dict[str, list[int]] = {}
    for paragraph, language in zip(paragraphs, found, strict=True):
        if language and _base(language) != _base(target):
            by_language.setdefault(language, []).extend(paragraph.pieces)
    for language, where in by_language.items():
        todo: list[tuple[int, str, str, str]] = []
        for i in where:
            m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
            lead, core, trail = m.groups() if m else ("", "", "")
            if not trail and i in followed and _base(language) in _NO_SPACES:
                trail = " "
            if any(c.isalnum() for c in core):
                todo.append((i, lead, core, trail))
        for start in range(0, len(todo), 100):
            batch = todo[start : start + 100]
            result = await _post(
                "/translate",
                {"q": [core for _, _, core, _ in batch], "source": language, "target": target},
            )
            out = result.get("translatedText") if isinstance(result, dict) else None
            if not isinstance(out, list) or len(out) != len(batch):
                raise TranslationError("the translation service sent no translation")
            for (i, lead, _, trail), text in zip(batch, out, strict=True):
                pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
    return "".join(pieces)


_NO_SPACES = {"zh", "ja", "th"}
_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)


# Web and mail addresses in the text: the models break them up ("https: / /
# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps.
_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
_TAG = re.compile(r"(<[^>]*>)")


def protect_addresses(html: str) -> str:
    """Mark the addresses in ``html``'s text as not to be translated (style
    blocks and tag attributes are left alone)."""
    out: list[str] = []
    in_style = False
    for piece in _TAG.split(html):
        if piece.startswith("<"):
            lowered = piece.lower()
            if lowered.startswith("<style"):
                in_style = True
            elif lowered.startswith("</style"):
                in_style = False
            out.append(piece)
        elif in_style or not piece:
            out.append(piece)
        else:
            out.append(_ADDRESS.sub(lambda m: f'<span translate="no">{m.group(0)}</span>', piece))
    return "".join(out)


_PUNCT = re.compile(r"[\W_]+")
MAX_NAMES = 200  # names checked for one message


async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]:
    """Of ``names``, those the model changes when it translates them alone
    ("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one
    request for them all. If the check fails, all of them."""
    wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES]
    if not wanted:
        return set()
    try:
        result = await _post(
            "/translate", {"q": wanted, "source": source or "auto", "target": target}
        )
    except TranslationError:
        return set(wanted)
    out = result.get("translatedText") if isinstance(result, dict) else None
    if not isinstance(out, list) or len(out) != len(wanted):
        return set(wanted)

    def same(a: str, b: str) -> bool:
        return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold()

    return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))}


# Bumped when what goes to the service, or what is done with the result,
# changes: translations kept from before are made again.
PIPELINE_VERSION = 5


async def translate_html(
    html: str, source: str, target: str, keep: Iterable[str] = (), names: Iterable[str] = ()
) -> str:
    """``html`` (a sanitized message body) in ``target``; source "auto", a code,
    or several ("zh-Hans,pl,en": then each paragraph from its own language).
    ``keep``: words always left as they are; ``names``: left as they are if
    the model would change them (translation_prep)."""
    if len(html) > MAX_HTML_CHARS:
        raise TranslationError("this message is too long to translate")
    sources = [x for x in (source or "").split(",") if x and x != "auto"]
    prepared = prep.join_wrapped_lines(html)
    for language in sources:
        prepared = prep.translate_phrases(prepared, language, target)
    single = sources[0] if len(sources) == 1 else "auto"
    candidates = {*names, *prep.find_names(prepared, single if sources else "")}
    terms = {*keep, *await names_to_keep(candidates, single, target)}
    prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms)))
    if len(sources) > 1:
        return prep.tidy(await _translate_paragraphs(prepared, target))
    result = await _post(
        "/translate",
        {"q": prepared, "source": source or "auto", "target": target, "format": "html"},
    )
    if not isinstance(result, dict) or "translatedText" not in result:
        raise TranslationError("the translation service sent no translation")
    return prep.tidy(str(result["translatedText"]))