Productivity features, UI translations, local email translation
- Snooze (with a Snoozed folder), keyboard shortcuts with a Settings section, mail templates (Compose > More > Templates), a Subscriptions folder with bulk unsubscribe/delete, and tracker blocking with a Privacy toggle (on by default) - Own dialogs instead of browser popups; emoji picker in compose; compose windows survive a visit to Settings - Sentence completion learned from the user's sent mail (no AI) - Interface in English and Polish (one file per language, English as the fallback), picked from the browser on first run, then kept - Email translation on the server via LibreTranslate: Translate button and "Translate to...", messages in several languages translated part by part, names/addresses/emoji/file names/phone numbers kept, an animated line while translating - Antispam "Mark as spam" loads with the message, not when the menu opens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,303 @@
|
||||
"""Email translation, on this server: LibreTranslate (Argos models) in its own
|
||||
container, reached only by the API. A message's text never leaves the server.
|
||||
|
||||
- A message's language is detected the first time it is shown (from its own
|
||||
text: quoted replies and signatures left out) and kept on the message, so the
|
||||
reading pane can offer "Translate" when it differs from the interface language.
|
||||
- A translation is of the message HTML the user sees (already sanitized; the
|
||||
result is sanitized again), cached per message, target language and whether
|
||||
remote images were shown.
|
||||
|
||||
Off when ``LIBRETRANSLATE_URL`` is empty.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import html as html_lib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from app import translation_prep as prep
|
||||
from app.completion import own_text
|
||||
from app.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECT_CHARS = 1500 # of the message's own text: plenty to tell the language
|
||||
MIN_CONFIDENCE = 50.0 # below this, LibreTranslate is guessing
|
||||
MAX_HTML_CHARS = 150_000 # a translation request bigger than this is refused
|
||||
_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # long messages take a while on a CPU
|
||||
_LANG_CACHE_S = 600
|
||||
|
||||
|
||||
class TranslationError(Exception):
|
||||
"""Translation is off, the service is unreachable, or it refused the text."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Language:
|
||||
code: str
|
||||
name: str
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return bool(get_settings().libretranslate_url.strip())
|
||||
|
||||
|
||||
def _url(path: str) -> str:
|
||||
return get_settings().libretranslate_url.rstrip("/") + path
|
||||
|
||||
|
||||
async def _post(path: str, payload: dict[str, object]) -> object:
|
||||
if not enabled():
|
||||
raise TranslationError("translation is not set up on this server")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.post(_url(path), json=payload)
|
||||
except httpx.HTTPError as exc:
|
||||
raise TranslationError(f"the translation service is unreachable: {exc}") from exc
|
||||
if resp.status_code != 200:
|
||||
detail = resp.text[:200]
|
||||
raise TranslationError(f"the translation service refused: {resp.status_code} {detail}")
|
||||
return resp.json()
|
||||
|
||||
|
||||
_languages: tuple[float, list[Language]] = (0.0, [])
|
||||
|
||||
|
||||
async def languages() -> list[Language]:
|
||||
"""What the service can translate between (cached for a few minutes)."""
|
||||
global _languages
|
||||
stamp, cached = _languages
|
||||
if cached and time.monotonic() - stamp < _LANG_CACHE_S:
|
||||
return cached
|
||||
if not enabled():
|
||||
return []
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
|
||||
resp = await client.get(_url("/languages"))
|
||||
resp.raise_for_status()
|
||||
rows = resp.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.info("translation languages unavailable: %s", exc)
|
||||
return []
|
||||
found = [Language(code=str(r["code"]), name=str(r["name"])) for r in rows if "code" in r]
|
||||
_languages = (time.monotonic(), found)
|
||||
return found
|
||||
|
||||
|
||||
_CJK = re.compile(r"[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]")
|
||||
MIN_LETTERS = 25 # fewer, and the detector guesses ("Jacqueline Lian" → Esperanto)
|
||||
MAX_DETECT = 60 # pieces of one message asked about
|
||||
MIN_SHARE = 0.15 # of a message's letters, for its language to count
|
||||
|
||||
|
||||
async def _detect_one(text: str) -> str:
|
||||
result = await _post("/detect", {"q": text[:DETECT_CHARS]})
|
||||
if not isinstance(result, list) or not result:
|
||||
return ""
|
||||
best = result[0]
|
||||
if not isinstance(best, dict) or float(best.get("confidence", 0)) < MIN_CONFIDENCE:
|
||||
return ""
|
||||
return str(best.get("language", ""))
|
||||
|
||||
|
||||
def _weight(text: str) -> int:
|
||||
"""How much text this is: letters, a Chinese/Japanese/Korean sign as a word."""
|
||||
return sum(3 if _CJK.match(c) else 1 for c in text if c.isalpha())
|
||||
|
||||
|
||||
async def languages_of(parts: list[str]) -> list[str]:
|
||||
"""The language of each of ``parts`` (paragraphs, in order). Parts too
|
||||
short to tell take the language of the next part that could be told (a
|
||||
greeting goes with what follows it), the last ones that of the one before;
|
||||
parts without letters get ''."""
|
||||
# Addresses say nothing about a language ("E-mail: lianshaojuan.lsj@…" → Estonian).
|
||||
parts = [_ADDRESS.sub(" ", part) for part in parts]
|
||||
told: list[str] = [""] * len(parts)
|
||||
asked = [
|
||||
i
|
||||
for i, part in enumerate(parts)
|
||||
if _CJK.search(part) or sum(c.isalpha() for c in part) >= MIN_LETTERS
|
||||
][:MAX_DETECT]
|
||||
gate = asyncio.Semaphore(4)
|
||||
|
||||
async def one(i: int) -> None:
|
||||
async with gate:
|
||||
told[i] = await _detect_one(parts[i])
|
||||
|
||||
await asyncio.gather(*(one(i) for i in asked))
|
||||
out = list(told)
|
||||
following = ""
|
||||
for i in reversed(range(len(parts))):
|
||||
if told[i]:
|
||||
following = told[i]
|
||||
elif any(c.isalpha() for c in parts[i]):
|
||||
out[i] = following
|
||||
previous = ""
|
||||
for i, part in enumerate(parts):
|
||||
if told[i]:
|
||||
previous = told[i]
|
||||
elif not out[i] and any(c.isalpha() for c in part):
|
||||
out[i] = previous
|
||||
return out
|
||||
|
||||
|
||||
async def detect(text: str) -> str:
|
||||
"""The languages of ``text`` (a message's plain text), most of it first,
|
||||
comma-separated ("zh-Hans,pl,en"); '' if unsure. A message can be in
|
||||
several: each language that makes up a good part of it counts."""
|
||||
own = own_text(text)
|
||||
if len(" ".join(own.split())) < 20:
|
||||
return "" # a line or two says too little
|
||||
lines: list[str] = []
|
||||
total = 0
|
||||
for line in (x.strip() for x in own.splitlines()):
|
||||
if line and total < DETECT_CHARS * 4:
|
||||
lines.append(line)
|
||||
total += len(line)
|
||||
found = await languages_of(lines)
|
||||
weight: dict[str, int] = {}
|
||||
for line, language in zip(lines, found, strict=True):
|
||||
if language:
|
||||
weight[language] = weight.get(language, 0) + _weight(_ADDRESS.sub(" ", line))
|
||||
if not weight:
|
||||
return await _detect_one(" ".join(own.split()))
|
||||
all_of_it = sum(weight.values())
|
||||
ranked = sorted(weight, key=lambda k: weight[k], reverse=True)
|
||||
return ",".join(k for k in ranked if weight[k] / all_of_it >= MIN_SHARE)
|
||||
|
||||
|
||||
def _base(code: str) -> str:
|
||||
return code.lower().split("-")[0]
|
||||
|
||||
|
||||
async def _translate_paragraphs(html: str, target: str) -> str:
|
||||
"""``html`` in more than one language, in ``target``: each paragraph from
|
||||
its own language; paragraphs already in ``target`` are left as they are."""
|
||||
pieces, paragraphs = prep.split_pieces(html)
|
||||
found = await languages_of([p.text for p in paragraphs])
|
||||
# Chinese and Japanese go without spaces: two pieces of one sentence
|
||||
# ("…奎琳，</span>我从…") need one between them once translated.
|
||||
followed = {i for p in paragraphs for i in p.pieces[:-1]}
|
||||
by_language: dict[str, list[int]] = {}
|
||||
for paragraph, language in zip(paragraphs, found, strict=True):
|
||||
if language and _base(language) != _base(target):
|
||||
by_language.setdefault(language, []).extend(paragraph.pieces)
|
||||
for language, where in by_language.items():
|
||||
todo: list[tuple[int, str, str, str]] = []
|
||||
for i in where:
|
||||
m = _EDGE_SPACE.match(html_lib.unescape(pieces[i]))
|
||||
lead, core, trail = m.groups() if m else ("", "", "")
|
||||
if not trail and i in followed and _base(language) in _NO_SPACES:
|
||||
trail = " "
|
||||
if any(c.isalnum() for c in core):
|
||||
todo.append((i, lead, core, trail))
|
||||
for start in range(0, len(todo), 100):
|
||||
batch = todo[start : start + 100]
|
||||
result = await _post(
|
||||
"/translate",
|
||||
{"q": [core for _, _, core, _ in batch], "source": language, "target": target},
|
||||
)
|
||||
out = result.get("translatedText") if isinstance(result, dict) else None
|
||||
if not isinstance(out, list) or len(out) != len(batch):
|
||||
raise TranslationError("the translation service sent no translation")
|
||||
for (i, lead, _, trail), text in zip(batch, out, strict=True):
|
||||
pieces[i] = html_lib.escape(f"{lead}{text}{trail}", quote=False)
|
||||
return "".join(pieces)
|
||||
|
||||
|
||||
_NO_SPACES = {"zh", "ja", "th"}
|
||||
_EDGE_SPACE = re.compile(r"^(\s*)(.*?)(\s*)$", re.DOTALL)
|
||||
|
||||
|
||||
# Web and mail addresses in the text: the models break them up ("https: / /
|
||||
# vimeo.com"), so they are marked translate="no", which LibreTranslate keeps.
|
||||
_ADDRESS = re.compile(r"(?:https?://|www\.)[^\s<>\"']+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+", re.IGNORECASE)
|
||||
_TAG = re.compile(r"(<[^>]*>)")
|
||||
|
||||
|
||||
def protect_addresses(html: str) -> str:
|
||||
"""Mark the addresses in ``html``'s text as not to be translated (style
|
||||
blocks and tag attributes are left alone)."""
|
||||
out: list[str] = []
|
||||
in_style = False
|
||||
for piece in _TAG.split(html):
|
||||
if piece.startswith("<"):
|
||||
lowered = piece.lower()
|
||||
if lowered.startswith("<style"):
|
||||
in_style = True
|
||||
elif lowered.startswith("</style"):
|
||||
in_style = False
|
||||
out.append(piece)
|
||||
elif in_style or not piece:
|
||||
out.append(piece)
|
||||
else:
|
||||
out.append(_ADDRESS.sub(lambda m: f'<span translate="no">{m.group(0)}</span>', piece))
|
||||
return "".join(out)
|
||||
|
||||
|
||||
_PUNCT = re.compile(r"[\W_]+")
|
||||
MAX_NAMES = 200 # names checked for one message
|
||||
|
||||
|
||||
async def names_to_keep(names: Iterable[str], source: str, target: str) -> set[str]:
|
||||
"""Of ``names``, those the model changes when it translates them alone
|
||||
("Milan" → "Mediolan", "Imaginary Pixels" → "Imaginanal Pixels"); one
|
||||
request for them all. If the check fails, all of them."""
|
||||
wanted = sorted({n.strip() for n in names if len(n.strip()) >= 2})[:MAX_NAMES]
|
||||
if not wanted:
|
||||
return set()
|
||||
try:
|
||||
result = await _post(
|
||||
"/translate", {"q": wanted, "source": source or "auto", "target": target}
|
||||
)
|
||||
except TranslationError:
|
||||
return set(wanted)
|
||||
out = result.get("translatedText") if isinstance(result, dict) else None
|
||||
if not isinstance(out, list) or len(out) != len(wanted):
|
||||
return set(wanted)
|
||||
|
||||
def same(a: str, b: str) -> bool:
|
||||
return _PUNCT.sub("", a).casefold() == _PUNCT.sub("", b).casefold()
|
||||
|
||||
return {name for name, got in zip(wanted, out, strict=True) if not same(name, str(got))}
|
||||
|
||||
|
||||
# Bumped when what goes to the service, or what is done with the result,
|
||||
# changes: translations kept from before are made again.
|
||||
PIPELINE_VERSION = 5
|
||||
|
||||
|
||||
async def translate_html(
|
||||
html: str, source: str, target: str, keep: Iterable[str] = (), names: Iterable[str] = ()
|
||||
) -> str:
|
||||
"""``html`` (a sanitized message body) in ``target``; source "auto", a code,
|
||||
or several ("zh-Hans,pl,en": then each paragraph from its own language).
|
||||
``keep``: words always left as they are; ``names``: left as they are if
|
||||
the model would change them (translation_prep)."""
|
||||
if len(html) > MAX_HTML_CHARS:
|
||||
raise TranslationError("this message is too long to translate")
|
||||
sources = [x for x in (source or "").split(",") if x and x != "auto"]
|
||||
prepared = prep.join_wrapped_lines(html)
|
||||
for language in sources:
|
||||
prepared = prep.translate_phrases(prepared, language, target)
|
||||
single = sources[0] if len(sources) == 1 else "auto"
|
||||
candidates = {*names, *prep.find_names(prepared, single if sources else "")}
|
||||
terms = {*keep, *await names_to_keep(candidates, single, target)}
|
||||
prepared = prep.keep_symbols(protect_addresses(prep.keep_terms(prepared, terms)))
|
||||
if len(sources) > 1:
|
||||
return prep.tidy(await _translate_paragraphs(prepared, target))
|
||||
result = await _post(
|
||||
"/translate",
|
||||
{"q": prepared, "source": source or "auto", "target": target, "format": "html"},
|
||||
)
|
||||
if not isinstance(result, dict) or "translatedText" not in result:
|
||||
raise TranslationError("the translation service sent no translation")
|
||||
return prep.tidy(str(result["translatedText"]))
|
||||
Reference in New Issue
Block a user