lukas/lektor-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
lektor-ai/normalizer/normalizer.py
T

646 lines
28 KiB
Python
Raw Permalink Normal View History

Add subtitle dictionaries, Polish number normalizer, ComfyUI lifecycle control
2026-08-12 19:40:44 +02:00
#!/usr/bin/env python3
"""Polish number normalizer — digits to spoken Polish words, for TTS.
Higgs Audio v3 mispronounces raw digits, especially long ones, and sometimes
emits garbled non-words. This component rewrites them as fully spelled-out
Polish, in the right grammatical case — which is why it is a neural model and
not a lookup table: Polish numerals decline by case, gender and the noun they
govern.
>>> normalize("Wygrał 1247 złotych.")
'Wygrał tysiąc dwieście czterdzieści siedem złotych.'
>>> normalize("Mam 3 koty.") # nothing worth sending: left alone
'Mam 3 koty.'
Model: Folx/qwen3-0.6b-pl-text-normalization, loaded from a local directory and
run offline. If it is unavailable the component degrades to a num2words
fallback and says so in the log — it never raises and never crashes the caller.
Run `python normalizer.py "some text"` for a quick check, or `python worker.py`
for the JSON-lines service that lektor_ai_comfy.py talks to.
"""
import json
import logging
import os
import re
import sys
from pathlib import Path
LOG = logging.getLogger("pl_normalizer")
HERE = Path(__file__).resolve().parent
MODEL_DIR = Path(os.environ.get("PL_NORMALIZER_MODEL", HERE / "model"))
# "auto" uses the GPU when one is free, "cpu" forces RAM. Forcing CPU is a real
# use case, not a fallback: ComfyUI keeps ~8.4 GB of VRAM resident between runs,
# so a normalizer that insists on the GPU either fights it or has to wait.
DEVICE = os.environ.get("PL_NORMALIZER_DEVICE", "auto").strip().lower()
# The second model. The 0.6B fine-tune reads odd formats well — clock times,
# race positions, call signs — but loses the value of a large number while
# declining it ("z 8250 osobami" came back as 8205). A general Polish LLM has
# the opposite profile. Neither is asked to do the other's job; see route().
LLM_DIR = Path(os.environ.get("PL_NORMALIZER_LLM",
HERE / "models" / "PLLuM-4B-instruct-2512"))
# Fixed by the fine-tune — the model was trained with this exact system prompt.
SYSTEM_PROMPT = "Zamieniasz tekst na formę mówioną."
MAX_NEW_TOKENS = 400
LONG_INPUT_CHARS = 300 # quality degrades past this; warn, never truncate
# Where a number begins and ends. Numeric punctuation only continues a number
# when a digit follows it, which is what separates "3,5" and "14:30" (one number
# each, for the model to handle) from "Przód 85, tył 75." (two, each ending at
# ordinary sentence punctuation).
#
# The build brief writes these as (?<![\d.,:/\-]) and (?![\d.,:/\-]), which also
# rejects every number followed by a comma or a full stop — on real subtitles
# that silently skipped most of them.
_NUM_START = r'(?<!\d)(?<!\d[.,:/\-])'
_NUM_END = r'(?!\d)(?![.,:/\-]\d)'
# A standalone one-digit integer is pronounced correctly by the TTS, so it stays
# a digit; anything longer or more complex goes to the model.
#
# The two space guards are not in the brief either, and are there because
# without them "1 350 000" has its leading 1 protected as if it were standalone:
# the model then sees "KEEPNUMA 350 000" and returns "1 trzysta pięćdziesiąt
# tysięcy" — the million silently lost. A digit that sits next to a group of
# exactly three is part of a grouped number, not a number of its own.
KEEP_RE = re.compile(_NUM_START + r'(?<!\d[\u00a0\u202f ])'
r'\d'
+ _NUM_END + r'(?![\u00a0\u202f ]\d{3}(?!\d))')
DIGIT_RE = re.compile(r'\d')
# Fallback path only: integers of 2+ digits, or any number written in groups of
# three. Decimals, times and dates are deliberately left alone rather than
# mangled without the model's grammar.
_INT_RE = re.compile(_NUM_START +
r'(?:\d{1,3}(?:[\u00a0\u202f .]\d{3})+|\d{2,})'
+ _NUM_END)
# Banned alongside digits so the model cannot emit a symbol the TTS would read
# as a word of the wrong language, or not read at all.
_BANNED_SYMBOLS = ["$", "%", "€", "£", "¥", "°", "§", "@", "#", "&", "×", "÷", "±",
"©", "®", "™", "→", "←", "↑", "↓", "√", "∞", "≈", "≠", "≤", "≥"]
# The instruction the general model gets. It never sees a digit — num2words has
# already produced correct nominative words — so its only job is to inflect
# them. That is why it cannot get a value wrong: it is not reading numbers.
SYSTEM_INFLECT = (
"Otrzymujesz polskie zdanie, w którym liczebniki są już zapisane słowami, "
"ale w mianowniku. Popraw wyłącznie formę gramatyczną tych liczebników tak, "
"aby pasowały do zdania (przypadek, rodzaj, liczba). Nie zmieniaj, nie "
"dodawaj ani nie usuwaj żadnych innych słów. Zwróć samo zdanie."
)
# Pins down the three things it gets wrong unprompted: the case must follow the
# governing word, everything else must survive verbatim, and the answer is the
# sentence alone. The last example is a no-op on purpose — without it the model
# invents changes when none are needed.
FEWSHOT_INFLECT = [
("Mówili o dwieście pięćdziesiąt tysięcy uchodźcach.",
"Mówili o dwustu pięćdziesięciu tysiącach uchodźców."),
("Koncert dla osiem tysięcy dwieście pięćdziesiąt osób.",
"Koncert dla ośmiu tysięcy dwustu pięćdziesięciu osób."),
("Przyjechał z dwadzieścia pięć psami.",
"Przyjechał z dwudziestoma pięcioma psami."),
("Kosztuje tysiąc dwieście czterdzieści siedem złotych.",
"Kosztuje tysiąc dwieście czterdzieści siedem złotych."),
]
_state = {"loaded": False, "tok": None, "model": None, "banned": None, "error": None}
_llm = {"loaded": False, "tok": None, "model": None, "banned": None, "error": None}
# Counted per process and reported by the worker, so a run can say which leg did
# the work and how often a guard had to step in.
STATS = {"plain": 0, "complex": 0, "guarded": 0, "reasons": {}}
# ---------------------------------------------------------------------------
# Model
# ---------------------------------------------------------------------------
def _banned_ids(tok, cache_dir: Path = None) -> list:
"""Token ids that may never appear in the output.
Scanning the whole vocabulary means one decode call per token — a few
seconds for Qwen3's 150k entries — so the result is cached beside the model.
"""
cache = (cache_dir or MODEL_DIR) / "banned_tokens.json"
size = len(tok)
if cache.is_file():
try:
data = json.loads(cache.read_text(encoding="utf-8"))
if data.get("vocab_size") == size:
return data["ids"]
except (OSError, ValueError, KeyError):
pass # stale or corrupt — just recompute
banned = set()
for tid in range(size):
piece = tok.decode([tid]).strip()
if piece and piece.isdigit():
banned.add(tid)
for sym in _BANNED_SYMBOLS:
ids = tok.encode(sym, add_special_tokens=False)
if len(ids) == 1:
banned.add(ids[0])
ids = sorted(banned)
try:
cache.write_text(json.dumps({"vocab_size": size, "ids": ids}), encoding="utf-8")
except OSError:
pass # cache is an optimisation, not a requirement
return ids
def load_model() -> bool:
"""Load model, tokenizer and banned-token list once, and keep them.
Returns True when the neural path is available. A failure here is not an
error: it is logged and the component runs on the fallback for the rest of
the process. Loading is lazy rather than at import so that tests, --help and
fallback-only runs do not pay for a 1.1 GB model they never use.
"""
if _state["loaded"]:
return _state["model"] is not None
_state["loaded"] = True
if not (MODEL_DIR / "config.json").is_file():
_state["error"] = (f"no model in {MODEL_DIR} — see README.md for the "
f"one-time download")
LOG.warning("Neural normalizer unavailable (%s). Falling back to num2words: "
"numbers will be spelled out in the nominative case only.",
_state["error"])
return False
try:
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
tok = AutoTokenizer.from_pretrained(str(MODEL_DIR))
placement = {"": "cpu"} if DEVICE == "cpu" else "auto"
try:
model = AutoModelForCausalLM.from_pretrained(
str(MODEL_DIR), dtype=torch.bfloat16, device_map=placement)
except TypeError:
# transformers < 5 spells the same argument torch_dtype.
model = AutoModelForCausalLM.from_pretrained(
str(MODEL_DIR), torch_dtype=torch.bfloat16, device_map=placement)
model.eval()
_state["tok"], _state["model"] = tok, model
_state["banned"] = _banned_ids(tok)
LOG.info("Model loaded from %s on %s (%d banned tokens)",
MODEL_DIR, next(model.parameters()).device, len(_state["banned"]))
return True
except Exception as exc: # missing torch, no VRAM, corrupt download…
_state["tok"] = _state["model"] = None
_state["error"] = f"{type(exc).__name__}: {exc}"
LOG.warning("Could not load the neural normalizer (%s). Falling back to "
"num2words: numbers will be spelled out in the nominative "
"case only.", _state["error"])
return False
def model_available() -> bool:
return load_model()
def status() -> dict:
"""What the caller needs to report: which path is live, and why."""
ok = load_model()
device = ""
if ok:
try:
device = str(next(_state["model"].parameters()).device)
except Exception:
device = "unknown"
return {"mode": "model" if ok else "fallback",
"model_dir": str(MODEL_DIR),
"device": device,
"error": _state["error"]}
def _model_normalize(text: str) -> str:
import torch
tok, model = _state["tok"], _state["model"]
messages = [{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": text}]
prompt = tok.apply_chat_template(messages, tokenize=False,
add_generation_prompt=True)
inputs = tok(prompt, return_tensors="pt").to(model.device)
n = inputs["input_ids"].shape[1]
with torch.no_grad():
out = model.generate(**inputs,
max_new_tokens=MAX_NEW_TOKENS,
do_sample=False,
pad_token_id=tok.pad_token_id or tok.eos_token_id,
suppress_tokens=_state["banned"])
return tok.decode(out[0][n:], skip_special_tokens=True).strip()
def load_llm() -> bool:
"""Load the general Polish LLM used for the inflection leg.
Optional: without it every block goes to the fine-tune, which is what the
pipeline did before. Reported, never fatal.
"""
if _llm["loaded"]:
return _llm["model"] is not None
_llm["loaded"] = True
if not (LLM_DIR / "config.json").is_file():
_llm["error"] = f"no model in {LLM_DIR}"
LOG.warning("Inflection model unavailable (%s) — large numbers will be "
"normalized by the fine-tune alone.", _llm["error"])
return False
try:
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
tok = AutoTokenizer.from_pretrained(str(LLM_DIR))
placement = {"": "cpu"} if DEVICE == "cpu" else "auto"
model = AutoModelForCausalLM.from_pretrained(
str(LLM_DIR), dtype=torch.bfloat16, device_map=placement).eval()
_llm["tok"], _llm["model"] = tok, model
_llm["banned"] = _banned_ids(tok, cache_dir=LLM_DIR)
LOG.info("Inflection model loaded from %s on %s",
LLM_DIR, next(model.parameters()).device)
return True
except Exception as exc:
_llm["tok"] = _llm["model"] = None
_llm["error"] = f"{type(exc).__name__}: {exc}"
LOG.warning("Could not load the inflection model (%s) — large numbers "
"will be normalized by the fine-tune alone.", _llm["error"])
return False
def _llm_inflect(text: str) -> str:
import torch
tok, model = _llm["tok"], _llm["model"]
messages = [{"role": "system", "content": SYSTEM_INFLECT}]
for user, assistant in FEWSHOT_INFLECT:
messages.append({"role": "user", "content": user})
messages.append({"role": "assistant", "content": assistant})
messages.append({"role": "user", "content": text})
prompt = tok.apply_chat_template(messages, tokenize=False,
add_generation_prompt=True)
inputs = tok(prompt, return_tensors="pt").to(model.device)
n = inputs["input_ids"].shape[1]
with torch.no_grad():
out = model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS,
do_sample=False,
pad_token_id=tok.pad_token_id or tok.eos_token_id,
suppress_tokens=_llm["banned"])
answer = tok.decode(out[0][n:], skip_special_tokens=True).strip()
# Instruct models like to introduce their answer; the sentence is the last
# non-empty line in every case observed.
lines = [l.strip() for l in answer.splitlines() if l.strip()]
return lines[-1] if lines else answer
# ---------------------------------------------------------------------------
# Fallback
# ---------------------------------------------------------------------------
def _fallback(text: str) -> str:
"""Model unavailable. Nominative case only, bare integers only — wrong case
is still far better than the TTS garbling a long digit string."""
try:
from num2words import num2words
except ImportError:
LOG.warning("num2words is not installed — leaving %r unchanged", text)
return text
# A minus that stays a dash is a minus the TTS will not read, and -12
# spoken as 12 is the wrong temperature.
text = re.sub(r'(?<![\w])-(?=\d)', 'minus ', text)
def repl(m):
digits = re.sub(r'[\u00a0\u202f .]', '', m.group(0))
try:
return num2words(int(digits), lang="pl")
except Exception:
# num2words raises KeyError, not ValueError, past its largest scale
# word (10^66). Leaving the digits is bad; crashing a run is worse.
LOG.warning("num2words could not spell out %r — leaving it", m.group(0))
return m.group(0)
return _INT_RE.sub(repl, text)
# ---------------------------------------------------------------------------
# Routing
# ---------------------------------------------------------------------------
#
# The two models fail in opposite directions, measured on 40 real subtitle lines
# and a 32-case battery:
#
# fine-tune good: 6:05, P10, LO 3842, 95,7, 21.00, dates
# bad: loses the value of a large number while declining it
# ("z 8250 osobami" -> 8205), drops words
# num2words good: the value, always, whatever the digit grouping
# + LLM bad: destroys formats — a flight number becomes a quantity,
# "P10" becomes "Pdziesięć", times fall apart
#
# So a block goes to the second leg only when every number in it is a plain
# integer. On the sample, that is 77% of numbered blocks.
_MONTHS = ("stycznia|lutego|marca|kwietnia|maja|czerwca|lipca|sierpnia|"
"września|października|listopada|grudnia")
_COMPLEX_RE = re.compile(
r'\d[.,:/]\d' # 3,5 14:30 12/45 1.350
r'|\d\s*[-–—]\s*\d' # 10-15
r'|[^\W\d_]\d|\d[^\W\d_]' # P10, 1998cm3 — an identifier, not a count
r'|\b[A-ZĄĆĘŁŃÓŚŹŻ]{2,3}\s\d{2,}\b' # LO 3842 — call sign, read digit by digit
r'|\d+\s+(?i:' + _MONTHS + r')\b' # 13 maja — ordinal
r'|\d{4}\s+(?i:r(?:oku|\.))', # 2026 roku — ordinal
re.UNICODE)
# Case matters here, which is why IGNORECASE is scoped to the two alternatives
# that need it. Applied globally it turned the call-sign rule into "any short
# word before a number", routing "za 150 milionów" and "Przód 85" to the wrong
# leg — a third of the sample.
def route(text: str) -> str:
""""plain" when every number is a plain integer, else "complex"."""
return "complex" if _COMPLEX_RE.search(text) else "plain"
# ---------------------------------------------------------------------------
# Guards
# ---------------------------------------------------------------------------
#
# Both models can produce output that is worse than doing nothing: a repetition
# loop that runs to the token ceiling, a dropped minus sign that inverts a
# temperature, a vanished noun. These are cheap structural checks against the
# input — no second model pass — and a block that fails one falls back to
# num2words, which is dull but never wrong about the value.
_WORD_RE = re.compile(r'[^\W\d_]+', re.UNICODE)
_VOWELS = str.maketrans("", "", "aeiouyąęó")
_FOLD = str.maketrans("ąćęłńóśźż", "acelnoszz")
def _skeleton(word: str) -> str:
"""Consonants of a word, diacritics folded — a poor man's lemma, used to ask
whether a word survived generation at all. Three consonants is deliberate:
ludziach/ludzi and uchodźcach/uchodźców are the same word inflected, and a
longer key would report them as lost."""
return word.lower().translate(_FOLD).translate(_VOWELS)[:3]
# Numerals need identity, not similarity: "dwieście" and "dwustu" are the same
# number and must match, while "trzy" and "trzysta" differ by a factor of a
# hundred and must not. No consonant rule does both, so the closed class is
# written out. Patterns are tried in order, most specific first, against the
# lowercased and diacritic-folded word.
_NUMERAL_CANON = [
(r'zero', "0"),
(r'dwadziesc|dwudziest', "20"), (r'dwanasc|dwunast', "12"),
(r'dwiesc|dwust', "200"), (r'dw', "2"),
(r'trzynasc|trzynast', "13"), (r'trzydzies', "30"),
(r'trzyst', "300"), (r'trz', "3"),
(r'czternasc|czternast', "14"), (r'czterdzies', "40"),
(r'czteryst|czterest', "400"), (r'czter', "4"),
(r'pietnasc|pietnast', "15"), (r'piecdziesi', "50"),
(r'piecset', "500"), (r'pie[cs]', "5"),
(r'szesnasc|szesnast', "16"), (r'szescdziesi', "60"),
(r'szescset', "600"), (r'szes', "6"),
(r'siedemnasc|siedemnast', "17"), (r'siedemdziesi', "70"),
(r'siedemset', "700"), (r'siedem|siedmi', "7"),
(r'osiemnasc|osiemnast', "18"), (r'osiemdziesi', "80"),
(r'osiemset', "800"), (r'osiem|osm', "8"),
(r'dziewietnasc|dziewietnast', "19"), (r'dziewiecdziesi', "90"),
(r'dziewiecset', "900"), (r'dziewi', "9"),
(r'dziesi', "10"), (r'jedenasc|jedenast', "11"), (r'jed', "1"),
# Ordinals denote the same number and are often the better reading —
# "na pięćdziesiątym piątym okrążeniu". Without them the guard rejects
# correct output.
(r'pierwsz', "1"), (r'drug', "2"), (r'czwart', "4"), (r'piat', "5"),
(r'szost', "6"), (r'siodm', "7"), (r'setn', "100"),
(r'st[ou]$', "100"), (r'tysi', "1000"),
(r'miliard', "1e9"), (r'milion', "1e6"),
]
_NUMERAL_CANON = [(re.compile("^(?:" + pat + ")"), value)
for pat, value in _NUMERAL_CANON]
def _numeral(word: str) -> str:
"""The number a word denotes, whatever case it is in, or "" if it is not a
numeral."""
folded = word.lower().translate(_FOLD)
for pattern, value in _NUMERAL_CANON:
if pattern.match(folded):
return value
return ""
# Same as _INT_RE but from one digit up. Only ever applied to blocks the router
# called "plain", where every number is a bare integer — and there the model
# expands single digits too, so the expectation has to include them.
_ANY_INT_RE = re.compile(_NUM_START +
r'(?:\d{1,3}(?:[\u00a0\u202f .]\d{3})+|\d+)'
+ _NUM_END)
def _expected_numerals(text: str) -> list:
"""The numerals num2words would produce for `text`, canonicalised."""
try:
from num2words import num2words
except ImportError:
return []
def repl(m):
digits = re.sub(r'[\u00a0\u202f .]', '', m.group(0))
try:
return num2words(int(digits), lang="pl")
except Exception:
return m.group(0)
spelled = _ANY_INT_RE.sub(repl, text)
return sorted(n for n in map(_numeral, _WORD_RE.findall(spelled)) if n)
def _repeats(text: str) -> bool:
"""A word five times over, or a phrase three times — a generation loop.
PLLuM produced 'łącznie' eighty times on one input; the fine-tune produced
a wall of replacement characters on another."""
words = text.lower().split()
run = 1
for a, b in zip(words, words[1:]):
run = run + 1 if a == b else 1
if run >= 4:
return True
for size in (2, 3):
seen = {}
for i in range(len(words) - size + 1):
gram = tuple(words[i:i + size])
seen[gram] = seen.get(gram, 0) + 1
if seen[gram] >= 3:
return True
return False
def _guard(original: str, candidate: str, check_value: bool) -> str:
"""Return the reason `candidate` is unusable, or "" if it passes."""
if not candidate.strip():
return "empty"
if len(candidate) > 3 * len(original) + 60:
return "runaway length"
if _repeats(candidate):
return "repetition loop"
if "�" in candidate:
return "invalid characters"
if DIGIT_RE.search(candidate):
return "digits left in output"
if re.search(r'(?<![\w])-\s?\d', original) and "minus" not in candidate.lower():
return "minus sign lost"
if check_value:
# The plain leg's model is given text with no digits in it and asked
# only to inflect, so the numerals it returns must be exactly the ones
# num2words put there — same values, same count. A one-directional
# check is not enough: "dwadzieścia" -> "dwadzieścia pięć" keeps every
# expected word and still turns 20 into 25.
#
# The expectation comes from the full num2words rendering rather than
# from the digits alone, so numerals already written as words in the
# subtitle ("350 tysięcy") are counted too.
want = _expected_numerals(original)
got = sorted(n for n in map(_numeral, _WORD_RE.findall(candidate)) if n)
if want != got:
missing = [n for n in want if n not in got]
extra = [n for n in got if n not in want]
detail = (f"missing {','.join(missing)}" if missing else "") + \
(" " if missing and extra else "") + \
(f"added {','.join(extra)}" if extra else "reordered")
return f"numbers changed ({detail})"
# A word that vanished entirely. Short words are skipped because the
# dictionary legitimately expands them (ul. -> ulicy, zł -> złote).
have = [_skeleton(w) for w in _WORD_RE.findall(candidate)]
for word in _WORD_RE.findall(original):
if len(word) >= 5 and _skeleton(word) not in have:
return f"word lost ({word})"
return ""
# ---------------------------------------------------------------------------
# Decide / normalize
# ---------------------------------------------------------------------------
def needs_normalization(text: str) -> bool:
"""True when `text` holds a number the TTS would get wrong — anything with
a digit, once standalone single digits are discounted.
A block whose only numbers are standalone single digits is not sent at all:
the TTS says "pięć minut" for "5 minut" by itself, so there is nothing to
gain and a whole sentence of rewriting to risk. Inside a block that *is*
sent, single digits are normalized along with everything else.
lektor_ai_comfy.py applies the same rule before sending a block, so blocks
that need nothing never cross the process boundary. Keep the two in step.
"""
return bool(DIGIT_RE.search(KEEP_RE.sub("", text)))
def normalize(text: str) -> str:
"""Return `text` with its numbers spelled out in Polish. Never raises."""
return normalize_detail(text)[0]
def normalize_detail(text: str) -> tuple:
"""(result, leg, guard_reason) — the caller reports which leg did the work.
Plain integers go to num2words for the value and the LLM for the grammar,
which is the only arrangement measured to get both right. Everything else
goes to the fine-tune, which understands the formats. Either way the result
is checked before it is returned.
"""
if not text or not DIGIT_RE.search(text):
return text, "skipped", ""
if not needs_normalization(text):
return text, "skipped", "" # only standalone single digits
if len(text) > LONG_INPUT_CHARS:
LOG.warning("Input is %d characters; quality degrades past %d. "
"Processing it anyway.", len(text), LONG_INPUT_CHARS)
leg = route(text)
if leg == "plain" and load_llm():
STATS["plain"] += 1
nominative = _fallback(text) # deterministic: the value cannot be wrong
try:
candidate = _llm_inflect(nominative)
except Exception as exc:
LOG.warning("Inflection failed (%s: %s)", type(exc).__name__, exc)
candidate = nominative
safe, check_value = nominative, True
else:
leg = "complex" # no inflection model: the fine-tune takes it
STATS["complex"] += 1
if load_model():
try:
candidate = _model_normalize(text)
except Exception as exc:
LOG.warning("Generation failed (%s: %s)", type(exc).__name__, exc)
candidate = _fallback(text)
else:
candidate = _fallback(text)
# The value check needs a num2words expectation, which only exists for
# plain integers — "LO 3842" is correctly read digit by digit, and
# demanding "trzy tysiące osiemset" of it would reject a good answer.
safe, check_value = _fallback(text), False
reason = _guard(text, candidate, check_value)
if not reason:
return candidate, leg, ""
STATS["guarded"] += 1
STATS["reasons"][reason.split(" (")[0]] = \
STATS["reasons"].get(reason.split(" (")[0], 0) + 1
LOG.warning("Rejected %r -> %r (%s); using %r", text, candidate, reason, safe)
return (safe if safe.strip() else text), leg, reason
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main(argv=None) -> int:
import argparse
parser = argparse.ArgumentParser(
description="Spell out Polish numbers for TTS.",
epilog="With no TEXT, reads lines from stdin and writes them normalized.")
parser.add_argument("text", nargs="*", metavar="TEXT")
parser.add_argument("--status", action="store_true",
help="Report which path is live and exit")
parser.add_argument("--fallback-only", action="store_true",
help="Skip the model, use num2words (for testing)")
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING,
format="%(levelname)s: %(message)s", stream=sys.stderr)
if args.fallback_only:
_state.update(loaded=True, model=None, tok=None,
error="--fallback-only requested")
if args.status:
print(json.dumps(status(), ensure_ascii=False, indent=2))
return 0
if args.text:
print(normalize(" ".join(args.text)))
return 0
for line in sys.stdin:
print(normalize(line.rstrip("\n")), flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue Copy Permalink