Translator: languages that fail to download are tried again

The first start tried the default languages once; a DNS failure at that
moment left them missing for good. Languages asked for are now kept as
pending in state.json and retried in the background (30 s, doubling to
15 min) until they are in, across restarts; an install from the admin
panel joins the same queue. A state from before this, with nothing
installed, counts as an unfinished first start. The admin panel shows
what is waiting.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

translator/translator.py

@@ -6,9 +6,12 @@ internal network. A message's text never leaves the server.
 - Firefox's models translate to and from English; any other pair goes
   through English (Polish → English → German). So a language is installed as
   its two models, to and from English, and English is always there.
-- What is installed is kept in ``$DATA_DIR/state.json``. The first start
-  installs ``TRANSLATOR_LANGUAGES``; after that the admin panel adds and
-  removes languages (``POST``/``DELETE /languages/{code}``).
+- What is installed is kept in ``$DATA_DIR/state.json``, with the languages
+  still to install. The first start asks for ``TRANSLATOR_LANGUAGES``; after
+  that the admin panel adds and removes languages (``POST``/``DELETE
+  /languages/{code}``). A language that could not be downloaded (no network
+  yet) is tried again in the background, at growing intervals, until it is
+  in — across restarts too.
 - Models come from Mozilla's Remote Settings (where Firefox gets them), each
   file checked against its SHA-256; only release models (no test builds).
 - Language identification: fastText's lid.176 (176 languages), baked into
@@ -113,13 +116,30 @@ def _installed() -> list[str]:
     return [PIVOT] + [c for c in state.get("languages", []) if c != PIVOT]
 
 
-def _write_installed(codes: list[str]) -> None:
+def _pending() -> list[str]:
+    """Languages asked for and not installed yet (tried again until they are)."""
+    return list((_read_state() or {}).get("pending", []))
+
+
+def _write_state(installed: list[str], pending: list[str]) -> None:
     DATA.mkdir(parents=True, exist_ok=True)
     tmp = STATE.with_suffix(".tmp")
-    tmp.write_text(json.dumps({"languages": sorted(set(codes) - {PIVOT})}))
+    done = set(installed) - {PIVOT}
+    tmp.write_text(
+        json.dumps(
+            {
+                "languages": sorted(done),
+                "pending": [c for c in dict.fromkeys(pending) if c not in done and c != PIVOT],
+            }
+        )
+    )
     tmp.replace(STATE)
 
 
+def _write_installed(codes: list[str]) -> None:
+    _write_state(codes, _pending())
+
+
 # --- Mozilla's model catalogue -----------------------------------------------------------
 
 
@@ -211,12 +231,12 @@ def _install(code: str) -> None:
         for source, target in directions:
             _install_direction(code, source, target)
         with _lock:
-            _write_installed(_installed() + [code])
+            _write_state(_installed() + [code], [c for c in _pending() if c != code])
             for pair in (f"{code}-{PIVOT}", f"{PIVOT}-{code}"):
                 _loaded.pop(pair, None)  # a reinstall: load the new files
             _errors.pop(code, None)
         log.info("installed %s", code)
-    except Exception as exc:  # noqa: BLE001 — reported in /languages, tried again by hand
+    except Exception as exc:  # noqa: BLE001 — reported in /languages, tried again later
         log.warning("installing %s failed: %s", code, exc)
         with _lock:
             _errors[code] = str(exc)[:300]
@@ -225,34 +245,53 @@ def _install(code: str) -> None:
             _installing.pop(code, None)
 
 
-def _start_install(code: str) -> bool:
+_wake = threading.Event()  # a new language to install: try now, not after the pause
+_RETRY_FIRST, _RETRY_MAX = 30.0, 900.0
+
+
+def _installer() -> None:
+    """Install the pending languages, one after another; while some fail
+    (no network, Mozilla out of reach), try again after a pause that grows
+    from half a minute to a quarter of an hour."""
+    pause = _RETRY_FIRST
+    while True:
+        for code in _pending():
+            with _lock:
+                if code in _installing or code in _installed():
+                    continue
+                _installing[code] = {"done": 0, "total": 0}
+            _install(code)
+        if not _pending():
+            pause = _RETRY_FIRST
+            _wake.wait()  # nothing to do until a language is asked for
+        else:
+            _wake.wait(pause)
+            pause = min(pause * 2, _RETRY_MAX)
+        _wake.clear()
+
+
+def _ask_for(code: str) -> None:
     with _lock:
-        if code in _installing:
-            return False
-        _installing[code] = {"done": 0, "total": 0}
-    threading.Thread(target=_install, args=(code,), daemon=True).start()
-    return True
+        _write_state(_installed(), _pending() + [code])
+        _errors.pop(code, None)
+    _wake.set()
 
 
-def _first_start() -> None:
-    """The first start installs the default languages, one after another."""
-    if _read_state() is not None:
-        return
-    _write_installed([])
-
-    def run() -> None:
-        for code in DEFAULT_LANGUAGES:
-            if code and code != PIVOT:
-                with _lock:
-                    _installing[code] = {"done": 0, "total": 0}
-                _install(code)
-
-    threading.Thread(target=run, daemon=True).start()
+def _start() -> None:
+    """The first start asks for the default languages; every start resumes
+    what is still pending."""
+    state = _read_state()
+    # No state yet — or one from before pending languages were kept, with
+    # nothing installed (a first start that could not download): ask for the
+    # defaults.
+    if state is None or ("pending" not in state and not state.get("languages")):
+        _write_state([], [c for c in DEFAULT_LANGUAGES if c])
+    threading.Thread(target=_installer, daemon=True).start()
 
 
 @asynccontextmanager
 async def _lifespan(_: FastAPI) -> AsyncIterator[None]:
-    _first_start()
+    _start()
     yield
 
 
@@ -377,6 +416,7 @@ def languages() -> dict[str, Any]:
         return {
             "installed": _installed(),
             "installing": {k: dict(v) for k, v in _installing.items()},
+            "pending": [c for c in _pending() if c not in _installing],
             "available": available,
             "errors": dict(_errors),
         }
@@ -391,7 +431,10 @@ def install(code: str) -> dict[str, Any]:
         _release_files(PIVOT, code)
     except LookupError as exc:
         raise HTTPException(404, str(exc)) from exc
-    return {"started": _start_install(code)}
+    except Exception as exc:  # noqa: BLE001 — Mozilla out of reach: queue it anyway
+        log.info("model catalogue unavailable, %s queued: %s", code, exc)
+    _ask_for(code)
+    return {"started": True}
 
 
 @app.delete("/languages/{code}")
@@ -399,7 +442,8 @@ def remove(code: str) -> dict[str, bool]:
     if code == PIVOT:
         raise HTTPException(400, "English is always installed")
     with _lock:
-        _write_installed([c for c in _installed() if c != code])
+        _write_state([c for c in _installed() if c != code], [c for c in _pending() if c != code])
+        _errors.pop(code, None)
         for pair in (f"{code}-{PIVOT}", f"{PIVOT}-{code}"):
             _loaded.pop(pair, None)
             shutil.rmtree(MODELS / pair, ignore_errors=True)