#!/usr/bin/env python3
"""
lektor_ai_comfy — Convert SRT subtitles to AI voiceover via ComfyUI + Higgs Audio V3.
Usage:
python3 lektor_ai_comfy.py input/film.srt input/film.mkv --start 00:00 --end 01:40
Requires ComfyUI with TTS-Audio-Suite running at http://127.0.0.1:8189
"""
import argparse
import configparser
import json
import random
import re
import shutil
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
from urllib.request import urlopen, Request
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
COMFY_OUTPUT_DIR = Path("/home/lukas/comfy-tts-app/ComfyUI/output")
COMFY_INPUT_DIR = Path("/home/lukas/comfy-tts-app/ComfyUI/input")
VOICE_PROMPTS_DIR = Path(__file__).resolve().parent / "voices"
SETTINGS_FILE = Path(__file__).resolve().parent / "settings.ini"
# Fallback values, used when settings.ini is absent or omits a key. Keeping them
# here means the script still runs correctly with no settings file at all.
# Each entry is (ini_section, ini_key, type, value).
_DEFAULTS = {
"voice": ("tts", "voice", str, "gudowski_short"),
"speed": ("tts", "speed", float, 1.2),
"temperature": ("tts", "temperature", float, 0.9),
"max_tokens": ("tts", "max_tokens", int, 4096),
"strip_leading_ms": ("tts", "strip_leading_ms", float, 0.0),
"trailing_fade_ms": ("tts", "trailing_fade_ms", float, 80.0),
"orig_volume": ("mix", "orig_volume", float, 0.60),
"tts_volume": ("mix", "tts_volume", float, 1.8),
"output_dir": ("paths", "output_dir", str, "output"),
"comfy_url": ("comfy", "comfy_url", str, "http://127.0.0.1:8189"),
}
COMFY_URL = _DEFAULTS["comfy_url"][3] # rebound from settings/CLI in main()
def load_settings(path: Path) -> dict:
"""Read settings.ini, falling back to _DEFAULTS for anything missing.
Values here become argparse's defaults, so precedence ends up as
CLI flag > settings.ini > _DEFAULTS without any extra bookkeeping.
A malformed or unreadable file is reported and then ignored rather than
being fatal — a typo in a config file should not cost you a long run.
"""
values = {key: spec[3] for key, spec in _DEFAULTS.items()}
if not path.is_file():
return values
parser = configparser.ConfigParser(inline_comment_prefixes=("#", ";"))
try:
parser.read(path, encoding="utf-8")
except (configparser.Error, OSError) as exc:
print(f" WARNING: could not read {path.name} ({exc}) — using built-in defaults",
file=sys.stderr)
return values
for key, (section, option, cast, fallback) in _DEFAULTS.items():
if not parser.has_option(section, option):
continue
raw = parser.get(section, option).strip()
if not raw:
continue
try:
values[key] = cast(raw)
except ValueError:
print(f" WARNING: {path.name} [{section}] {option}={raw!r} is not a valid "
f"{cast.__name__} — using {fallback!r}", file=sys.stderr)
return values
# Subtitle codecs that carry real text and can therefore be extracted and fed
# to the TTS engine. Used only by the --embedded-subs picker; image-based
# formats (hdmv_pgs_subtitle, dvd_subtitle) hold bitmaps and would need OCR.
# The final mux copies every subtitle track regardless of codec.
_TEXT_SUB_CODECS = {"subrip", "srt", "ass", "ssa", "webvtt", "mov_text"}
# ---------------------------------------------------------------------------
# SRT helpers
# ---------------------------------------------------------------------------
_SRT_TIME_RE = re.compile(r'(\d+):(\d+):(\d+)[,.](\d+)')
def _parse_srt_time(s: str) -> float:
m = _SRT_TIME_RE.match(s.strip())
if not m:
raise ValueError(f"Bad SRT timestamp: {s!r}")
h, mn, sec, ms = int(m[1]), int(m[2]), int(m[3]), int(m[4])
return h * 3600 + mn * 60 + sec + ms / 1000
def _fmt_srt_time(seconds: float) -> str:
h = int(seconds // 3600)
m = int((seconds % 3600) // 60)
s = int(seconds % 60)
ms = int(round((seconds % 1) * 1000))
if ms == 1000:
ms = 999
return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
def _parse_mmss(value: str) -> float:
parts = value.strip().split(":")
if len(parts) == 2:
return int(parts[0]) * 60 + float(parts[1])
if len(parts) == 3:
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
raise ValueError(f"Cannot parse time: {value!r}")
def _ensure_terminal_punct(text: str, single_dot: bool = False) -> str:
if not text or text[-1] in ('.', ',', '!', '?', ';', ':'):
return text
return text + ('.' if single_dot else '...')
def _strip_sub_tags(line: str) -> str:
"""Remove SRT/ASS formatting codes from a subtitle line."""
line = re.sub(r'\{[^}]*\}', '', line) # ASS overrides: {\an8}, {\pos(…)}, {\c&H…&}
line = re.sub(r'<[^>]+>', '', line) # HTML tags: , , ,
return line
def _clean_text(lines: list) -> str:
# Strip formatting tags first so they don't interfere with allcaps detection
# or the leading-dash check.
lines = [_strip_sub_tags(l) for l in lines]
# Only split into multi-part (for dialogue splitting later) when lines
# have explicit speaker dashes. Wrapped single-speaker lines join with space.
has_dialogue = any(re.match(r"^-\s*\S", l.strip()) for l in lines if l.strip())
out = []
for line in lines:
line = line.strip()
line = re.sub(r"^-\s*", "", line)
if line:
out.append(line)
result = "\n".join(out) if has_dialogue else " ".join(out)
# If the whole block is ALL-CAPS, convert to sentence case and end with a single dot.
letters = [c for c in result if c.isalpha()]
was_allcaps = bool(letters) and all(c == c.upper() for c in letters)
if was_allcaps:
result = result.lower()
# Sentence fragments that continue from a previous subtitle start with a
# lowercase letter and confuse TTS models — capitalise the first character.
if result and result[0].islower():
result = result[0].upper() + result[1:]
# Ensure each sentence ends with punctuation so TTS produces a natural tail.
if has_dialogue:
result = "\n".join(_ensure_terminal_punct(p, was_allcaps) for p in result.split("\n"))
else:
result = _ensure_terminal_punct(result, was_allcaps)
return result
def load_tts_input_srt(path: Path) -> list:
"""Read a pre-processed tts_input.srt into (t_start, t_end, text) tuples.
Unlike parse_subtitle_blocks this does NOT re-run _clean_text or dialogue
splitting — the file is already in final form.
"""
text = path.read_text(encoding="utf-8")
blocks = re.split(r"\n{2,}", text.strip())
result = []
for block in blocks:
lines = block.strip().splitlines()
ti = next((i for i, l in enumerate(lines) if "-->" in l), None)
if ti is None:
continue
parts = lines[ti].split("-->")
if len(parts) != 2:
continue
t_start = _parse_srt_time(parts[0])
t_end = _parse_srt_time(parts[1])
payload = " ".join(l.strip() for l in lines[ti + 1:] if l.strip())
if payload:
result.append((t_start, t_end, payload))
return result
def parse_subtitle_blocks(srt_path: str, start_sec: float, end_sec: float) -> list:
"""Return list of (start_relative, end_relative, plain_text) blocks."""
text = Path(srt_path).read_text(encoding="utf-8-sig")
blocks = re.split(r"\n{2,}", text.strip())
result = []
for block in blocks:
lines = block.strip().splitlines()
ti = next((i for i, l in enumerate(lines) if "-->" in l), None)
if ti is None:
continue
parts = lines[ti].split("-->")
if len(parts) != 2:
continue
t_start = _parse_srt_time(parts[0])
t_end = _parse_srt_time(parts[1])
if t_start < start_sec or t_start > end_sec:
continue
plain = _clean_text(lines[ti + 1:])
if plain:
result.append((t_start - start_sec, t_end - start_sec, plain))
return result
def split_dialogue_blocks(blocks: list) -> list:
"""Split multi-line blocks into one entry per line with proportional timing."""
result = []
for t_start, t_end, text in blocks:
lines = [l for l in text.split("\n") if l.strip()]
if len(lines) <= 1:
result.append((t_start, t_end, text))
continue
duration = t_end - t_start
total_chars = sum(len(l) for l in lines)
current = t_start
for i, line in enumerate(lines):
share = len(line) / total_chars if total_chars else 1 / len(lines)
end = t_end if i == len(lines) - 1 else current + duration * share
result.append((current, end, line))
current = end
return result
def blocks_to_srt(blocks: list) -> str:
"""Convert (start, end, text) tuples back to SRT string for inspection."""
lines = []
for i, (t_start, t_end, text) in enumerate(blocks, 1):
lines.append(str(i))
lines.append(f"{_fmt_srt_time(t_start)} --> {_fmt_srt_time(t_end)}")
lines.append(text)
lines.append("")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# FFmpeg helpers
# ---------------------------------------------------------------------------
def _ffmpeg_ts(seconds: float) -> str:
h = int(seconds // 3600)
m = int((seconds % 3600) // 60)
s = seconds % 60
return f"{h:02d}:{m:02d}:{s:06.3f}"
def _run(cmd: list, label: str):
print(f" {label}...")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(f"ffmpeg error ({label}):\n{result.stderr[-2000:]}", file=sys.stderr)
sys.exit(1)
def _run_progress(cmd: list, label: str, total_seconds: float):
"""Run ffmpeg with a live progress bar that rewrites the current line."""
# Inject -progress pipe:1 before the output path (last arg)
prog_cmd = cmd[:-1] + ["-progress", "pipe:1", "-nostats", cmd[-1]]
proc = subprocess.Popen(
prog_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, bufsize=1,
)
stderr_chunks: list = []
def _drain():
for line in proc.stderr:
stderr_chunks.append(line)
threading.Thread(target=_drain, daemon=True).start()
BAR = 28
print(f" {label}...", end="", flush=True)
for line in proc.stdout:
if not line.startswith("out_time_us="):
continue
try:
us = int(line.split("=", 1)[1])
if us < 0:
continue
sec = us / 1_000_000
except (ValueError, IndexError):
continue
pct = min(sec / total_seconds, 1.0) if total_seconds > 0 else 0.0
filled = int(BAR * pct)
bar = "█" * filled + "░" * (BAR - filled)
t_cur = f"{int(sec // 60):02d}:{int(sec % 60):02d}"
t_tot = f"{int(total_seconds // 60):02d}:{int(total_seconds % 60):02d}"
print(f"\r {label}... [{bar}] {t_cur}/{t_tot}", end="", flush=True)
proc.wait()
if proc.returncode != 0:
err = "".join(stderr_chunks)
print(f"\nffmpeg error ({label}):\n{err[-2000:]}", file=sys.stderr)
sys.exit(1)
t_tot = f"{int(total_seconds // 60):02d}:{int(total_seconds % 60):02d}"
print(f"\r {label}... [{'█' * BAR}] {t_tot}/{t_tot}")
def get_audio_duration(path: Path) -> float:
result = subprocess.run(
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", str(path)],
capture_output=True, text=True,
)
info = json.loads(result.stdout)
for stream in info.get("streams", []):
if "duration" in stream:
return float(stream["duration"])
raise RuntimeError(f"Cannot determine duration of {path}")
def speed_up_audio(input_path: Path, output_path: Path, speed: float,
fade_out_ms: float = 0):
"""Apply atempo (and optional fade-out) to a single clip."""
filters = []
if speed != 1.0:
remaining = speed
while remaining > 2.0:
filters.append("atempo=2.0")
remaining /= 2.0
filters.append(f"atempo={remaining:.4f}")
if fade_out_ms > 0:
fade_sec = fade_out_ms / 1000
# Compute fade start from the post-speed duration so we can use a plain
# afade=t=out (areverse caused MP3 tail truncation on VBR files).
raw_dur = get_audio_duration(input_path)
post_dur = raw_dur / speed if speed != 1.0 else raw_dur
fade_start = max(0.0, post_dur - fade_sec)
filters.append(f"afade=t=out:st={fade_start:.3f}:d={fade_sec:.3f}")
if not filters:
shutil.copy2(input_path, output_path)
return
label = f"Speed {speed:.2f}×"
if fade_out_ms:
label += f" + fade-out {fade_out_ms:.0f}ms"
_run([
"ffmpeg", "-y",
"-i", str(input_path),
"-filter:a", ",".join(filters),
str(output_path),
], label)
def trim_audio(input_path: Path, output_path: Path, max_duration: float):
"""Hard-trim a clip to max_duration seconds."""
_run([
"ffmpeg", "-y",
"-i", str(input_path),
"-t", f"{max_duration:.3f}",
"-c", "copy",
str(output_path),
], f"Trim to {max_duration:.2f}s")
def strip_leading_audio(input_path: Path, output_path: Path, strip_ms: float):
"""Remove the first strip_ms milliseconds from a clip."""
_run([
"ffmpeg", "-y",
"-i", str(input_path),
"-ss", f"{strip_ms / 1000:.3f}",
"-c", "copy",
str(output_path),
], f"Strip {strip_ms:.0f}ms leading audio")
# A clip's trailing silence must not count towards its duration: placement uses
# a push-forward cursor, so padding at the end of one clip delays every clip
# after it. Usually that padding is a fraction of a second, but when the model
# fails to emit a stop token it generates until --max-tokens runs out — a short
# line then yields a clip of speech followed by two minutes of digital silence,
# which shunts the rest of the film out of sync.
SILENCE_DB = -45.0 # anything quieter counts as silence
SILENCE_MIN = 0.35 # ignore shorter gaps; commas and beats are not padding
SILENCE_KEEP_MS = 120 # leave this much tail so speech does not end abruptly
def trailing_silence_start(path: Path) -> float:
"""Time at which the clip's final run of silence begins, or None."""
result = subprocess.run(
["ffmpeg", "-hide_banner", "-i", str(path),
"-af", f"silencedetect=noise={SILENCE_DB}dB:d={SILENCE_MIN}",
"-f", "null", "-"],
capture_output=True, text=True,
)
try:
duration = get_audio_duration(path)
except RuntimeError:
return None
open_at = None
for kind, value in re.findall(r"silence_(start|end):\s*([0-9.]+)", result.stderr):
t = float(value)
if kind == "start":
open_at = t
elif t < duration - 0.05:
open_at = None # that silence ended mid-clip, so it is a pause
return open_at
# Some runaway generations pad with a steady tone instead of digital silence.
# It sits above any sane silence threshold, so silencedetect cannot see it —
# but real speech swings tens of dB between windows while a tone does not.
# Measured here: tone tails spread 3-5 dB, speech tails 43-61 dB.
FLAT_TAIL_DB = 15.0 # spread below this is not speech
FLAT_TAIL_MIN = 5.0 # the flat run must last at least this long
RUNAWAY_SECONDS = 20.0 # only look for tone padding on clips this long
def flat_tail_start(path: Path, window: float = 0.5) -> float:
"""Start of a long, unnaturally constant tail, or None."""
result = subprocess.run(
["ffmpeg", "-hide_banner", "-i", str(path),
"-af", f"astats=metadata=1:reset={window},"
"ametadata=print:key=lavfi.astats.Overall.RMS_level",
"-f", "null", "-"],
capture_output=True, text=True,
)
blob = result.stderr + result.stdout
values = [(-120.0 if "inf" in m else float(m))
for m in re.findall(r"RMS_level=(-?[0-9.]+|-?inf)", blob)]
need = int(FLAT_TAIL_MIN / window)
if len(values) < need + 1:
return None
for i in range(len(values) - need):
tail = values[i:]
if max(tail) - min(tail) <= FLAT_TAIL_DB:
return i * window
return None
def trim_trailing_silence(input_path: Path, output_path: Path) -> float:
"""Cut the final silence off a clip. Returns the seconds removed."""
duration = get_audio_duration(input_path)
start = trailing_silence_start(input_path)
if start is None and duration > RUNAWAY_SECONDS:
start = flat_tail_start(input_path)
keep = None if start is None else start + SILENCE_KEEP_MS / 1000
if keep is None or keep >= duration - 0.02 or keep <= 0.05:
if input_path != output_path:
shutil.copy2(input_path, output_path)
return 0.0
_run([
"ffmpeg", "-y",
"-i", str(input_path),
"-t", f"{keep:.3f}",
str(output_path),
], f"Trim {duration - keep:.1f}s trailing silence")
return duration - keep
def _assemble_batch(clips: list, output_path: str, label: str,
total_seconds: float = 0.0, start_offset: float = 0.0):
"""Mix a batch of (start_sec, path) clips with adelay into output_path.
start_offset is subtracted from every clip's start time so sub-batch files
contain no leading silence. The caller re-applies the offset at merge time.
"""
run = (lambda cmd, lbl: _run_progress(cmd, lbl, total_seconds)) if total_seconds > 0 else _run
if len(clips) == 1:
delay_ms = max(0, int((clips[0][0] - start_offset) * 1000))
run([
"ffmpeg", "-y",
"-i", str(clips[0][1]),
"-af", f"adelay={delay_ms}:all=1",
output_path,
], label)
return
inputs, filter_parts = [], []
for i, (start_sec, path) in enumerate(clips):
inputs += ["-i", str(path)]
delay_ms = max(0, int((start_sec - start_offset) * 1000))
filter_parts.append(f"[{i}:a]adelay={delay_ms}:all=1[a{i}]")
mix_inputs = "".join(f"[a{i}]" for i in range(len(clips)))
filter_parts.append(
f"{mix_inputs}amix=inputs={len(clips)}:duration=longest:normalize=0[out]"
)
run(
["ffmpeg", "-y"] + inputs +
["-filter_complex", ";".join(filter_parts), "-map", "[out]", output_path],
label,
)
def assemble_timed_audio(clips: list, output_path: str, batch_size: int = 500):
"""Place each clip at its start time via adelay then mix.
clips: list of (start_sec, Path).
Processes in batches to stay under OS file-descriptor limits.
"""
n_batches = (len(clips) - 1) // batch_size + 1
if len(clips) <= batch_size:
total_sec = clips[-1][0] if clips else 0.0
_assemble_batch(clips, output_path,
f"Assembling {len(clips)} timed clips",
total_seconds=total_sec)
return
tmp_dir = Path(output_path).parent / "_batch_tmp"
tmp_dir.mkdir(exist_ok=True)
batch_files = []
batch_offsets = []
try:
for b, start in enumerate(range(0, len(clips), batch_size)):
batch = clips[start:start + batch_size]
batch_offset = batch[0][0] # first clip's global start time
total_sec = batch[-1][0] - batch_offset # content duration only
batch_path = str(tmp_dir / f"batch_{b:04d}.flac")
_assemble_batch(
batch, batch_path,
f"Assembling batch {b + 1}/{n_batches} ({len(batch)} clips)",
total_seconds=total_sec,
start_offset=batch_offset,
)
batch_files.append(batch_path)
batch_offsets.append(batch_offset)
# Merge: re-apply each batch's offset so clips land at their global times
batch_clips = [(offset, Path(p)) for offset, p in zip(batch_offsets, batch_files)]
_assemble_batch(batch_clips, output_path, f"Merging {len(batch_files)} batches")
finally:
for p in batch_files:
Path(p).unlink(missing_ok=True)
tmp_dir.rmdir() if tmp_dir.exists() and not any(tmp_dir.iterdir()) else None
def extract_video_fragment(video_path: str, start_sec: float, end_sec: float,
output_path: str):
_run([
"ffmpeg", "-y",
"-ss", _ffmpeg_ts(start_sec),
"-i", video_path,
"-t", str(end_sec - start_sec),
"-c", "copy",
output_path,
], f"Extracting video {_ffmpeg_ts(start_sec)}–{_ffmpeg_ts(end_sec)}")
def list_subtitle_streams(video_path: str) -> list:
"""Return subtitle stream info dicts for all subtitle tracks in a video file."""
result = subprocess.run(
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", video_path],
capture_output=True, text=True,
)
subs = []
for s in json.loads(result.stdout).get("streams", []):
if s.get("codec_type") != "subtitle":
continue
tags = s.get("tags", {})
subs.append({
"index": s["index"],
"codec_name": s.get("codec_name", "unknown"),
"language": tags.get("language", ""),
"title": tags.get("title", ""),
})
return subs
def extract_subtitle_track(video_path: str, stream_index: int, output_path: str):
"""Extract one subtitle track from a video to an SRT file."""
_run([
"ffmpeg", "-y",
"-i", video_path,
"-map", f"0:{stream_index}",
"-c:s", "subrip",
output_path,
], f"Extracting subtitle stream #{stream_index}")
def get_media_duration(path: str) -> float:
"""Container duration in seconds (0.0 if unknown)."""
result = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "csv=p=0", path],
capture_output=True, text=True,
)
try:
return float(result.stdout.strip())
except ValueError:
return 0.0
def merge_voiceover(video_path: str, tts_path: str, output_path: str,
orig_volume: float = 0.60, tts_volume: float = 1.8):
"""Produce a video with two audio tracks:
track 0 — Original (first audio stream from source, stream-copied)
track 1 — Polish (lektor mix: TTS at tts_volume + original at orig_volume)
plus every subtitle track from the source, stream-copied.
Done in two passes:
1. Render the lektor mix on its own to a temporary audio file.
2. Stream-copy everything (video + both audio tracks + all subtitles)
into the final MKV.
Combining both in a single ffmpeg call fails on sources with many subtitle
tracks: the sparse subtitle streams stall the muxer while the audio filter
graph keeps producing, and ffmpeg eventually aborts with
"Failed to inject frame into filter network: Invalid argument".
Splitting the work keeps the filter graph free of subtitle streams, and
makes the final mux a pure copy (roughly 10x faster too).
MKV rips often have audio PTS starting at a non-zero offset (e.g. 4.043s).
Without correction amix can't output from PTS 0 until BOTH inputs have
frames, which delays the TTS. asetpts resets the PTS to 0 and adelay
re-inserts the offset as explicit silence, so the background audio stays at
its correct position while amix gets frames from both inputs from PTS 0.
"""
probe = subprocess.run(
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", video_path],
capture_output=True, text=True,
)
streams = json.loads(probe.stdout).get("streams", [])
audio_start = next(
(float(s.get("start_time", 0)) for s in streams if s.get("codec_type") == "audio"),
0.0,
)
if audio_start > 0.01:
print(f" Audio PTS offset: {audio_start:.3f}s — normalising orig_mix for lektor track")
delay_ms = int(audio_start * 1000)
norm = f"asetpts=PTS-STARTPTS,adelay={delay_ms}:all=1,aresample=async=1"
else:
norm = "aresample=async=1"
# The lektor track is stereo on purpose. ffmpeg's AAC encoder does not
# signal a 5.1 channel layout — it comes back as "unknown" in every
# container — so players cannot route the channels and the track plays
# silent. Stereo is declared correctly and plays everywhere; 192k is also
# a good rate for two channels where it is a poor one spread over six.
# The 5.1 source is downmixed with ffmpeg's standard coefficients, and the
# mono TTS becomes equal-level L/R, i.e. centred.
stereo = "aformat=channel_layouts=stereo"
filter_complex = (
f"[0:a:0]{norm},{stereo},volume={orig_volume}[orig_mix];"
f"[1:a]{stereo},volume={tts_volume}[tts];"
"[orig_mix][tts]amix=inputs=2:duration=first:normalize=0[lektor]"
)
out = Path(output_path)
mix_path = out.parent / f"{out.stem}_mix.m4a"
try:
# ── Pass 1: render the lektor mix on its own ─────────────────────────
_run_progress([
"ffmpeg", "-y",
"-i", video_path, "-i", tts_path,
"-filter_complex", filter_complex,
"-map", "[lektor]",
"-c:a", "aac", "-b:a", "192k",
str(mix_path),
], "Mixing lektor audio track", get_media_duration(video_path))
# ── Pass 2: stream-copy mux — video + both audio tracks + subtitles ──
sub_streams = [s for s in streams if s.get("codec_type") == "subtitle"]
if sub_streams:
print(f" Including {len(sub_streams)} subtitle track(s) in output")
cmd = [
"ffmpeg", "-y",
"-i", video_path, "-i", str(mix_path),
"-map", "0:v", "-map", "0:a:0", "-map", "1:a:0",
]
for s in sub_streams:
cmd += ["-map", f"0:{s['index']}"]
cmd += [
"-c", "copy",
# Must be an OUTPUT option (before the output path) — placed before
# -i it silently configures the demuxer instead and does nothing.
# Sparse subtitle tracks make the muxer give up on interleaving
# after its default 10s window, which strands the lektor audio
# hundreds of MB away from the video it belongs with. Players read
# MKV sequentially, so they never reach it and the track is silent.
# 0 means "buffer until every stream has a packet" — correct
# interleaving, and only a couple of GB of transient buffering.
"-max_interleave_delta", "0",
"-metadata:s:a:0", "title=Original",
"-metadata:s:a:1", "title=Polish",
"-metadata:s:a:1", "language=pol",
output_path,
]
_run(cmd, "Muxing final video (2 audio tracks + subtitles)")
finally:
mix_path.unlink(missing_ok=True)
# ---------------------------------------------------------------------------
# ComfyUI API helpers
# ---------------------------------------------------------------------------
def _comfy_post(path: str, data: dict) -> dict:
payload = json.dumps(data).encode()
req = Request(
f"{COMFY_URL}{path}",
data=payload,
headers={"Content-Type": "application/json"},
)
with urlopen(req, timeout=30) as r:
return json.loads(r.read())
def _comfy_get(path: str) -> dict:
with urlopen(f"{COMFY_URL}{path}", timeout=30) as r:
return json.loads(r.read())
def upload_voice_wav(wav_path: Path) -> tuple:
"""Copy voice WAV into ComfyUI's input dir; return (filename, reference_text)."""
dst = COMFY_INPUT_DIR / wav_path.name
if not dst.exists() or dst.stat().st_mtime < wav_path.stat().st_mtime:
shutil.copy2(wav_path, dst)
print(f" Copied {wav_path.name} → {dst}")
txt_path = wav_path.with_suffix(".txt")
ref_text = txt_path.read_text(encoding="utf-8").strip() if txt_path.exists() else ""
if ref_text:
print(f" Reference text loaded ({len(ref_text)} chars)")
else:
print(" WARNING: no reference .txt found — voice cloning quality will be lower")
return wav_path.name, ref_text
def build_text_workflow(text: str, voice_filename: str, voice_ref_text: str,
seed: int, temperature: float = 0.9,
max_new_tokens: int = 4096) -> dict:
"""ComfyUI workflow for a single text utterance with proper voice reference."""
return {
"1": {
"class_type": "HiggsAudioV3EngineNode",
"inputs": {
"model": "higgs-audio-v3-tts-4b",
"device": "auto",
"dtype": "auto",
"attention": "auto",
"temperature": temperature,
"top_p": 0.95,
"top_k": 50,
"max_new_tokens": max_new_tokens,
},
},
"2": {
"class_type": "LoadAudio",
"inputs": {"audio": voice_filename},
},
"3": {
# Pairs audio with its transcript so the model can clone the voice correctly
"class_type": "CharacterVoicesNode",
"inputs": {
"voice_name": "none",
"reference_text": voice_ref_text,
"opt_audio_input": ["2", 0],
},
},
"4": {
"class_type": "UnifiedTTSTextNode",
"inputs": {
"TTS_engine": ["1", 0],
"text": text,
"narrator_voice": "none",
"seed": seed,
"opt_narrator": ["3", 0],
"enable_audio_cache": False,
},
},
"5": {
"class_type": "SaveAudio",
"inputs": {
"audio": ["4", 0],
"filename_prefix": "audio/lektor_clip",
},
},
}
def submit_workflow(workflow: dict) -> str:
client_id = str(uuid.uuid4())
resp = _comfy_post("/prompt", {"prompt": workflow, "client_id": client_id})
return resp["prompt_id"]
def wait_for_completion(prompt_id: str, poll_interval: float = 0.25,
max_interval: float = 2.0, backoff_after: float = 10.0) -> dict:
"""Block until ComfyUI reports the job finished, then return its history entry.
The poll interval, not the GPU, governs perceived generation speed: a clip
that finishes 100 ms after a poll still waits for the next one. This used to
be a flat 3 s, which quantised every clip to a 3 s boundary — measured across
2075 real clips, 91% of the gaps landed exactly on a multiple of 3 s, and
roughly a third of each run was spent sleeping on already-completed work.
Polling starts tight so the common case (most clips finish in 3-6 s) is
picked up almost immediately, then backs off once a job looks slow so a
stuck or very long generation does not sit at 4 requests/second forever.
"""
print(f" Waiting for {prompt_id[:8]}…", end="", flush=True)
start = time.monotonic()
interval = poll_interval
next_dot = 1.0
while True:
try:
history = _comfy_get(f"/history/{prompt_id}")
except Exception as e:
print(f"\n Poll error: {e}", file=sys.stderr)
time.sleep(min(interval, 1.0))
continue
if prompt_id in history:
entry = history[prompt_id]
status = entry.get("status", {})
for msg_type, msg_data in status.get("messages", []):
if msg_type == "execution_error":
print(f"\nComfyUI error: {msg_data}", file=sys.stderr)
sys.exit(1)
if status.get("completed"):
print(f" done ({time.monotonic() - start:.1f}s).")
return entry
elapsed = time.monotonic() - start
# One dot per second regardless of poll rate, or the line fills with dots.
if elapsed >= next_dot:
print(".", end="", flush=True)
next_dot += 1.0
time.sleep(interval)
if elapsed > backoff_after:
interval = min(interval * 1.5, max_interval)
def extract_output_audio(history_entry: dict) -> Path:
outputs = history_entry.get("outputs", {})
for node_output in outputs.values():
for audio_info in node_output.get("audio", []):
subfolder = audio_info.get("subfolder", "")
path = COMFY_OUTPUT_DIR / subfolder / audio_info["filename"]
if path.exists():
return path
raise RuntimeError(f"Output audio not found in history: {outputs}")
def _generate_one(text: str, voice_filename: str, voice_ref_text: str,
seed: int, speed: float, out_dir: Path, label: str,
strip_leading_ms: float, temperature: float = 0.9,
trailing_fade_ms: float = 0, max_new_tokens: int = 4096) -> tuple:
"""Submit one TTS job and return (path, duration_after_speedup)."""
workflow = build_text_workflow(text, voice_filename, voice_ref_text, seed,
temperature, max_new_tokens)
prompt_id = submit_workflow(workflow)
history_entry = wait_for_completion(prompt_id)
raw_clip = extract_output_audio(history_entry)
if strip_leading_ms > 0:
stripped = out_dir / f"{label}_s.flac"
strip_leading_audio(raw_clip, stripped, strip_leading_ms)
raw_clip = stripped
raw_duration = get_audio_duration(raw_clip)
trimmed = out_dir / f"{label}_t.flac"
removed = trim_trailing_silence(raw_clip, trimmed)
if removed > 0:
raw_clip = trimmed
# Losing most of the clip to silence means the model never stopped and
# ran to the token ceiling. The speech is usually fine, but say so —
# it can also mean the line was cut short, which needs a --regen.
if removed > 20 and removed > raw_duration * 0.5:
print(f" ! {removed:.0f}s of trailing silence removed "
f"({raw_duration:.0f}s generated for {raw_duration - removed:.1f}s of "
f"speech) — the model did not stop on its own; check this clip")
fast_clip = out_dir / f"{label}.flac"
speed_up_audio(raw_clip, fast_clip, speed, fade_out_ms=trailing_fade_ms)
trimmed.unlink(missing_ok=True)
return fast_clip, get_audio_duration(fast_clip)
def generate_clips(blocks: list, voice_filename: str, voice_ref_text: str,
base_seed: int, speed: float, clips_dir: Path,
strip_leading_ms: float = 0.0, resume_from: int = 0,
temperature: float = 0.9, trailing_fade_ms: float = 0,
max_new_tokens: int = 4096) -> list:
"""
Generate one audio clip per block via ComfyUI, apply per-clip speed-up,
then calculate start positions with "push forward if overlap" logic.
If resume_from > 0, blocks 1..(resume_from-1) are loaded from existing
clip files instead of being regenerated (crash-recovery mode).
Returns list of (actual_start_sec, clip_path).
"""
clips = []
cursor = 0.0
total = len(blocks)
for idx, (t_start, t_end, text) in enumerate(blocks):
clip_n = idx + 1
label = f"clip_{clip_n:04d}"
canonical = clips_dir / f"{label}.flac"
if resume_from > 0 and clip_n < resume_from:
if not canonical.exists():
sys.exit(f"Resume from block {resume_from}: {canonical} not found — "
f"lower --resume-from or regenerate from an earlier block.")
duration = get_audio_duration(canonical)
print(f" [{clip_n}/{total}] skip (resuming) — {canonical.name} {duration:.2f}s")
actual_start = max(t_start, cursor)
clips.append((actual_start, canonical))
cursor = actual_start + duration
continue
display = text.replace("\n", " ")
print(f" [{clip_n}/{total}] {_fmt_srt_time(t_start)} {display[:60]!r}")
seed = base_seed + idx
fast_clip, duration = _generate_one(
text, voice_filename, voice_ref_text,
seed, speed, clips_dir, label, strip_leading_ms, temperature,
trailing_fade_ms=trailing_fade_ms, max_new_tokens=max_new_tokens,
)
# Place at SRT time or after previous clip, whichever is later.
# No trimming — clips play to their natural end; re-align at the next
# natural SRT gap that is wider than the accumulated overrun.
actual_start = max(t_start, cursor)
clips.append((actual_start, fast_clip))
cursor = actual_start + duration
delay = actual_start - t_start
delay_str = f" [+{delay:.2f}s behind SRT]" if delay > 0.05 else ""
print(f" → placed at {_fmt_srt_time(actual_start)}, duration {duration:.2f}s{delay_str}")
return clips
# ---------------------------------------------------------------------------
# Rebuild / Regen helpers
# ---------------------------------------------------------------------------
def job_dir(out_dir: Path, video_path: str) -> Path:
"""Per-job working directory: /clips/