From 7e2cee02e53a2cde8f6407bc0b7b867f8a37bfb5 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 10 Aug 2026 14:12:33 +0200 Subject: [PATCH] Survive a ComfyUI crash, retry clips the model cuts short MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crash recovery ComfyUI's queue and history do not persist across a restart, so a prompt submitted before a crash exists nowhere afterwards. The old poll loop waited on that dead prompt for ever, and its error branch skipped the backoff, so it hammered the closed port four times a second printing "Poll error" on every attempt. run_workflow() replaces wait_for_completion(). It reports once when the backend goes away and once when it returns, polls every 3s in between, and on reconnect checks both /history and /queue: if the prompt is in neither, the queue did not survive and the clip is submitted again. When it cannot tell, it assumes the job is still alive — resubmitting a running job would waste a generation. It also starts ComfyUI itself, via the script named by [comfy] start_command, in a new terminal window. Guarded by a 90s cooldown and 3 attempts per run so a backend that dies during startup cannot spawn an endless stream of windows; the window pauses on exit so a failed start leaves its error on screen. Falls back to a background launch with a log file when there is no display. Set [comfy] autostart = no, or pass --no-comfy-autostart, to turn it off. Clips cut short mid-word About one clip in twelve comes back truncated: the model stops before the line is finished. Measured on raw ComfyUI output, 6 of 80 recent clips, and 15 of 472 in one episode. It is invisible in the text but obvious in the audio — a finished clip decays to about -65 dB, a cut one stops at about -16 dB. Truncated clips are now regenerated with a distant seed, up to [tts] truncation_retries extra attempts, and a run that still has bad clips ends by printing them as a ready-made --regen list. --check-clips reports the same for a job already on disk. Worth recording what this is NOT: the trailing-silence trimming added earlier was suspected, and cleared. It only cuts silence that runs to end of file, so a pause between words never triggers it — verified against six synthetic cases including a 1.2s mid-clip pause, and against real clips containing pauses. The truncation is present in ComfyUI's own output before this script touches it. settings.ini New: [tts] truncation_retries, [comfy] autostart and start_command, all documented in place. The editor renders yes/no keys as a dropdown. Co-Authored-By: Claude Opus 5 --- lektor_ai_comfy.py | 307 ++++++++++++++++++++++++++++++++++++++++----- settings.ini | 15 +++ ui.py | 9 +- 3 files changed, 298 insertions(+), 33 deletions(-) diff --git a/lektor_ai_comfy.py b/lektor_ai_comfy.py index 1651aae..31f219c 100644 --- a/lektor_ai_comfy.py +++ b/lektor_ai_comfy.py @@ -11,8 +11,10 @@ Requires ComfyUI with TTS-Audio-Suite running at http://127.0.0.1:8189 import argparse import configparser import json +import os import random import re +import shlex import shutil import subprocess import sys @@ -32,6 +34,17 @@ VOICE_PROMPTS_DIR = Path(__file__).resolve().parent / "voices" SETTINGS_FILE = Path(__file__).resolve().parent / "settings.ini" +def _as_bool(raw) -> bool: + """yes/no setting. configparser's own getboolean is bypassed by the + per-key cast in load_settings, so accept the same spellings here.""" + value = str(raw).strip().lower() + if value in ("1", "true", "yes", "on"): + return True + if value in ("0", "false", "no", "off"): + return False + raise ValueError(f"{raw!r} is not a yes/no value") + + # Fallback values, used when settings.ini is absent or omits a key. Keeping them # here means the script still runs correctly with no settings file at all. # Each entry is (ini_section, ini_key, type, value). @@ -40,16 +53,23 @@ _DEFAULTS = { "speed": ("tts", "speed", float, 1.2), "temperature": ("tts", "temperature", float, 0.9), "max_tokens": ("tts", "max_tokens", int, 4096), + "truncation_retries": ("tts", "truncation_retries", int, 2), "strip_leading_ms": ("tts", "strip_leading_ms", float, 0.0), "trailing_fade_ms": ("tts", "trailing_fade_ms", float, 80.0), "orig_volume": ("mix", "orig_volume", float, 0.60), "tts_volume": ("mix", "tts_volume", float, 1.8), "output_dir": ("paths", "output_dir", str, "output"), "comfy_url": ("comfy", "comfy_url", str, "http://127.0.0.1:8189"), + "comfy_autostart": ("comfy", "autostart", _as_bool, True), + "comfy_start_command": ("comfy", "start_command", str, "start_comfyui.sh"), } COMFY_URL = _DEFAULTS["comfy_url"][3] # rebound from settings/CLI in main() +# Rebound in main() too; used when the backend stops answering. +COMFY_AUTOSTART = True +COMFY_START_SCRIPT = Path(__file__).resolve().parent / "start_comfyui.sh" + def load_settings(path: Path) -> dict: """Read settings.ini, falling back to _DEFAULTS for anything missing. @@ -416,6 +436,31 @@ FLAT_TAIL_MIN = 5.0 # the flat run must last at least this long RUNAWAY_SECONDS = 20.0 # only look for tone padding on clips this long +# A clip the model finished decays to near silence; one it cut off mid-word +# stops at full level. Measured across 80 raw outputs the two groups sit about +# 50 dB apart (-65 dB vs -16 dB), so the test is unambiguous. +TRUNCATION_PEAK_DB = -25.0 +TRUNCATION_WINDOW = 0.08 + + +def ends_abruptly(path: Path, window: float = TRUNCATION_WINDOW) -> bool: + """True when a clip stops at full level, i.e. the model cut it mid-word.""" + try: + duration = get_audio_duration(path) + except RuntimeError: + return False + if duration <= window: + return False + result = subprocess.run( + ["ffmpeg", "-hide_banner", "-ss", f"{duration - window:.3f}", + "-t", f"{window}", "-i", str(path), + "-af", "volumedetect", "-f", "null", "-"], + capture_output=True, text=True, + ) + peak = re.search(r"max_volume:\s*(-?[0-9.]+)", result.stderr) + return bool(peak) and float(peak.group(1)) > TRUNCATION_PEAK_DB + + def flat_tail_start(path: Path, window: float = 0.5) -> float: """Start of a long, unnaturally constant tail, or None.""" result = subprocess.run( @@ -781,33 +826,148 @@ def submit_workflow(workflow: dict) -> str: return resp["prompt_id"] -def wait_for_completion(prompt_id: str, poll_interval: float = 0.25, - max_interval: float = 2.0, backoff_after: float = 10.0) -> dict: - """Block until ComfyUI reports the job finished, then return its history entry. +BACKEND_RETRY_SECONDS = 3.0 # how often to look for ComfyUI while it is down - The poll interval, not the GPU, governs perceived generation speed: a clip - that finishes 100 ms after a poll still waits for the next one. This used to - be a flat 3 s, which quantised every clip to a 3 s boundary — measured across - 2075 real clips, 91% of the gaps landed exactly on a multiple of 3 s, and - roughly a third of each run was spent sleeping on already-completed work. - Polling starts tight so the common case (most clips finish in 3-6 s) is - picked up almost immediately, then backs off once a job looks slow so a - stuck or very long generation does not sit at 4 requests/second forever. +# Launching the backend is guarded: a ComfyUI that dies on startup would +# otherwise spawn a new terminal window every few seconds. +COMFY_AUTOSTART_MAX = 3 # attempts per run +COMFY_AUTOSTART_COOLDOWN = 90.0 # seconds between attempts +_autostart = {"attempts": 0, "last": 0.0} + +_TERMINALS = ( + ("gnome-terminal", lambda c: ["gnome-terminal", "--title", "ComfyUI", "--", + "bash", "-lc", c]), + ("konsole", lambda c: ["konsole", "-p", "tabtitle=ComfyUI", "-e", + "bash", "-lc", c]), + ("xfce4-terminal", lambda c: ["xfce4-terminal", "--title", "ComfyUI", + "-e", f"bash -lc {shlex.quote(c)}"]), + ("xterm", lambda c: ["xterm", "-T", "ComfyUI", "-e", "bash", "-lc", c]), +) + + +def start_comfy_backend(script: Path) -> bool: + """Launch ComfyUI in its own terminal window. True if one was started.""" + now = time.monotonic() + if _autostart["attempts"] >= COMFY_AUTOSTART_MAX: + return False + if _autostart["last"] and now - _autostart["last"] < COMFY_AUTOSTART_COOLDOWN: + return False + if not script.is_file(): + print(f" Cannot start ComfyUI automatically: {script} not found.", + file=sys.stderr) + _autostart["attempts"] = COMFY_AUTOSTART_MAX # do not keep complaining + return False + + _autostart["attempts"] += 1 + _autostart["last"] = now + # Pause on exit so a backend that fails to start leaves its error on screen + # instead of the window vanishing. + inner = (f"{shlex.quote(str(script))}; echo; " + f"echo '[ComfyUI exited — press any key to close]'; read -n 1") + + if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): + for name, build in _TERMINALS: + if not shutil.which(name): + continue + try: + subprocess.Popen(build(inner), start_new_session=True) + except OSError: + continue + print(f" Starting ComfyUI in a new {name} window " + f"(attempt {_autostart['attempts']} of {COMFY_AUTOSTART_MAX})…", + file=sys.stderr) + return True + + log = script.parent / "comfyui_autostart.log" + try: + with open(log, "ab") as handle: + subprocess.Popen([str(script)], stdout=handle, stderr=handle, + start_new_session=True) + except OSError as exc: + print(f" Could not start ComfyUI: {exc}", file=sys.stderr) + return False + print(f" No terminal available — started ComfyUI in the background, " + f"logging to {log}", file=sys.stderr) + return True + + +def _prompt_known(prompt_id: str) -> bool: + """True while ComfyUI still has this prompt queued, running or in history. + + A restart wipes both, so this is how a job lost to a crash is spotted. + If the answer cannot be determined, say yes: resubmitting a job that is + actually still running would waste a generation. """ - print(f" Waiting for {prompt_id[:8]}…", end="", flush=True) - start = time.monotonic() - interval = poll_interval - next_dot = 1.0 + try: + if prompt_id in _comfy_get(f"/history/{prompt_id}"): + return True + queue = _comfy_get("/queue") + except Exception: # noqa: BLE001 + return True + for key in ("queue_running", "queue_pending"): + for item in queue.get(key, []): + if isinstance(item, (list, tuple)) and len(item) > 1 and item[1] == prompt_id: + return True + return False + + +def run_workflow(workflow: dict, poll_interval: float = 0.25, + max_interval: float = 2.0, backoff_after: float = 10.0) -> dict: + """Submit a workflow, wait for it, and return its history entry. + + Survives ComfyUI dying mid-job. Its queue and history do not persist across + a restart, so a prompt submitted beforehand exists nowhere afterwards and + polling it would wait for ever — the job has to be sent again. While the + backend is down this reports once rather than once per poll, and slows to + BACKEND_RETRY_SECONDS instead of hammering a closed port. + + The poll interval, not the GPU, governs perceived speed: a clip that + finishes 100 ms after a poll still waits for the next one. It was a flat + 3 s, which quantised every clip to a 3 s boundary — across 2075 real clips + 91% of the gaps landed exactly on a multiple of 3 s. So polling starts + tight and only backs off once a job looks slow. + """ + prompt_id, offline = None, False + start, interval, next_dot = time.monotonic(), poll_interval, 1.0 while True: + if prompt_id is None: # first submission, or a resubmit + try: + prompt_id = submit_workflow(workflow) + except Exception as exc: # noqa: BLE001 + if not offline: + print(f"\n ComfyUI unreachable ({exc}).", file=sys.stderr) + offline = True + if not (COMFY_AUTOSTART and start_comfy_backend(COMFY_START_SCRIPT)): + print(" Waiting for it to come back — start it and this " + "will carry on.", file=sys.stderr) + time.sleep(BACKEND_RETRY_SECONDS) + continue + start, interval, next_dot = time.monotonic(), poll_interval, 1.0 + print(f" Waiting for {prompt_id[:8]}…", end="", flush=True) + try: history = _comfy_get(f"/history/{prompt_id}") - except Exception as e: - print(f"\n Poll error: {e}", file=sys.stderr) - time.sleep(min(interval, 1.0)) + except Exception as exc: # noqa: BLE001 + if not offline: + print(f"\n ComfyUI went away ({exc}).", file=sys.stderr) + offline = True + if not (COMFY_AUTOSTART and start_comfy_backend(COMFY_START_SCRIPT)): + print(" Waiting for it to come back — restart it and this " + "will carry on.", file=sys.stderr) + time.sleep(BACKEND_RETRY_SECONDS) continue + if offline: # it answered again + offline = False + print(" ComfyUI is back.", file=sys.stderr) + if not _prompt_known(prompt_id): + print(" The queue did not survive the restart — resubmitting " + "this clip.", file=sys.stderr) + prompt_id = None + continue + if prompt_id in history: entry = history[prompt_id] status = entry.get("status", {}) @@ -843,13 +1003,33 @@ def extract_output_audio(history_entry: dict) -> Path: def _generate_one(text: str, voice_filename: str, voice_ref_text: str, seed: int, speed: float, out_dir: Path, label: str, strip_leading_ms: float, temperature: float = 0.9, - trailing_fade_ms: float = 0, max_new_tokens: int = 4096) -> tuple: - """Submit one TTS job and return (path, duration_after_speedup).""" - workflow = build_text_workflow(text, voice_filename, voice_ref_text, seed, - temperature, max_new_tokens) - prompt_id = submit_workflow(workflow) - history_entry = wait_for_completion(prompt_id) - raw_clip = extract_output_audio(history_entry) + trailing_fade_ms: float = 0, max_new_tokens: int = 4096, + retries: int = 2) -> tuple: + """Submit one TTS job and return (path, duration_after_speedup, truncated). + + About one clip in twelve comes back cut off mid-word — the model stops + generating before the line is finished. It is not predictable from the + text, but it is obvious in the audio, so a clip that ends at full level is + simply asked for again with a different seed. + """ + attempt, truncated = 0, False + while True: + workflow = build_text_workflow(text, voice_filename, voice_ref_text, seed, + temperature, max_new_tokens) + raw_clip = extract_output_audio(run_workflow(workflow)) + + truncated = ends_abruptly(raw_clip) + if not truncated or attempt >= retries: + break + attempt += 1 + seed += 7919 # a distant seed, not the neighbouring sample + print(f" ! ends mid-word — retrying with seed {seed} " + f"(attempt {attempt + 1} of {retries + 1})") + + if truncated: + print(" ! still ends mid-word after " + f"{attempt + 1} attempt{'s' if attempt else ''} — " + "regenerate this one with --regen") if strip_leading_ms > 0: stripped = out_dir / f"{label}_s.flac" @@ -873,14 +1053,14 @@ def _generate_one(text: str, voice_filename: str, voice_ref_text: str, speed_up_audio(raw_clip, fast_clip, speed, fade_out_ms=trailing_fade_ms) trimmed.unlink(missing_ok=True) - return fast_clip, get_audio_duration(fast_clip) + return fast_clip, get_audio_duration(fast_clip), truncated def generate_clips(blocks: list, voice_filename: str, voice_ref_text: str, base_seed: int, speed: float, clips_dir: Path, strip_leading_ms: float = 0.0, resume_from: int = 0, temperature: float = 0.9, trailing_fade_ms: float = 0, - max_new_tokens: int = 4096) -> list: + max_new_tokens: int = 4096, retries: int = 2) -> list: """ Generate one audio clip per block via ComfyUI, apply per-clip speed-up, then calculate start positions with "push forward if overlap" logic. @@ -893,6 +1073,7 @@ def generate_clips(blocks: list, voice_filename: str, voice_ref_text: str, clips = [] cursor = 0.0 total = len(blocks) + still_truncated = [] for idx, (t_start, t_end, text) in enumerate(blocks): clip_n = idx + 1 @@ -914,11 +1095,14 @@ def generate_clips(blocks: list, voice_filename: str, voice_ref_text: str, print(f" [{clip_n}/{total}] {_fmt_srt_time(t_start)} {display[:60]!r}") seed = base_seed + idx - fast_clip, duration = _generate_one( + fast_clip, duration, truncated = _generate_one( text, voice_filename, voice_ref_text, seed, speed, clips_dir, label, strip_leading_ms, temperature, trailing_fade_ms=trailing_fade_ms, max_new_tokens=max_new_tokens, + retries=retries, ) + if truncated: + still_truncated.append(clip_n) # Place at SRT time or after previous clip, whichever is later. # No trimming — clips play to their natural end; re-align at the next @@ -931,6 +1115,11 @@ def generate_clips(blocks: list, voice_filename: str, voice_ref_text: str, delay_str = f" [+{delay:.2f}s behind SRT]" if delay > 0.05 else "" print(f" → placed at {_fmt_srt_time(actual_start)}, duration {duration:.2f}s{delay_str}") + if still_truncated: + print(f"\n {len(still_truncated)} clip(s) still end mid-word after retries. " + f"To try again:") + print(f" --regen {' '.join(str(n) for n in still_truncated)}") + return clips @@ -1023,6 +1212,37 @@ def _rebuild_video(args, out_dir: Path): print(f"\nDone! Output: {final_output}") +def _check_job_clips(args, out_dir: Path): + """List clips in this video's job folder that end mid-word. + + Prints them as a --regen argument list, so fixing them is a copy and paste. + """ + job = job_dir(out_dir, args.video) + clips = sorted(job.glob("clip_*.flac")) + if not clips: + sys.exit(f"No clips found in {job}." + _legacy_hint(out_dir, job)) + + print(f"\n Checking {len(clips)} clips in {job}…") + bad = [] + for clip in clips: + if ends_abruptly(clip): + try: + number = int(clip.stem.split("_")[1]) + except (IndexError, ValueError): + continue + bad.append(number) + print(f" {clip.name} ends at full level " + f"({get_audio_duration(clip):.2f}s)") + + if not bad: + print(f"\n No truncated clips. All {len(clips)} end cleanly.") + return + print(f"\n {len(bad)} of {len(clips)} clips ({100 * len(bad) / len(clips):.0f}%) " + f"end mid-word. To regenerate them:") + print(f" python3 {Path(__file__).name} \"{args.video}\" --regen " + f"{' '.join(str(n) for n in bad)}") + + def _trim_job_clips(args, out_dir: Path): """Trim trailing silence from every clip already on disk for this video. @@ -1100,12 +1320,13 @@ def _regen_clips(args, out_dir: Path): label = f"clip_{clip_n:04d}" print(f"\n REGEN [{clip_n}/{total}] {text[:70]!r}") seed = base_seed + idx - clip_path, duration = _generate_one( + clip_path, duration, truncated = _generate_one( text, voice_filename, voice_ref_text, seed, args.speed, clips_dir, label, args.strip_leading_ms, args.temperature, trailing_fade_ms=args.trailing_fade_ms, max_new_tokens=args.max_tokens, + retries=args.truncation_retries, ) print(f" → saved as {clip_path.name} ({duration:.2f}s)") @@ -1117,7 +1338,7 @@ def _regen_clips(args, out_dir: Path): # --------------------------------------------------------------------------- def main(): - global COMFY_URL + global COMFY_URL, COMFY_AUTOSTART, COMFY_START_SCRIPT # --settings is resolved before the main parser is built, because the file's # contents become that parser's defaults. @@ -1188,6 +1409,18 @@ def main(): "(same number as in clips/clip_NNNN.flac and tts_input.srt). " "Only replaces the selected FLAC files; run --rebuild " "afterwards to reassemble audio and rebuild the video.") + parser.add_argument("--no-comfy-autostart", action="store_true", + help="Do not launch ComfyUI automatically when it is not " + "answering. By default it is started via the script named " + "by [comfy] start_command in settings.ini.") + parser.add_argument("--truncation-retries", type=int, + default=cfg["truncation_retries"], metavar="N", + help=f"How many extra attempts to make when the model returns a " + f"clip cut off mid-word (default: {cfg['truncation_retries']}; " + f"0 disables retrying).") + parser.add_argument("--check-clips", action="store_true", + help="Report which clips in this video's job folder end mid-word, " + "as a ready-made --regen list, then stop.") parser.add_argument("--trim-silence", action="store_true", help="Trim trailing silence from the clips already generated for " "this video, then stop. Repairs runs where the model failed " @@ -1204,6 +1437,10 @@ def main(): args = parser.parse_args() COMFY_URL = args.comfy_url + COMFY_AUTOSTART = cfg["comfy_autostart"] and not args.no_comfy_autostart + start_cmd = Path(cfg["comfy_start_command"]).expanduser() + COMFY_START_SCRIPT = start_cmd if start_cmd.is_absolute() \ + else Path(__file__).resolve().parent / start_cmd if settings_path.is_file(): print(f" Settings: {settings_path}") @@ -1215,7 +1452,7 @@ def main(): # --regen without an SRT), argparse puts the lone positional into args.srt. # Detect this and move it to args.video. needs_srt = not (args.embedded_subs or args.rebuild or args.trim_silence - or bool(args.regen)) + or args.check_clips or bool(args.regen)) if args.video is None and args.srt is not None and not needs_srt: args.video = args.srt args.srt = None @@ -1228,6 +1465,11 @@ def main(): out_dir = Path(args.output_dir) out_dir.mkdir(parents=True, exist_ok=True) + # ── Maintenance: report clips the model cut short, then stop ───────────── + if args.check_clips: + _check_job_clips(args, out_dir) + return + # ── Maintenance: trim silence from existing clips, then stop ───────────── if args.trim_silence: _trim_job_clips(args, out_dir) @@ -1333,7 +1575,8 @@ def main(): resume_from=args.resume_from, temperature=args.temperature, trailing_fade_ms=args.trailing_fade_ms, - max_new_tokens=args.max_tokens) + max_new_tokens=args.max_tokens, + retries=args.truncation_retries) assembled_tts = clips_dir / "tts_assembled.flac" assemble_timed_audio(clips, str(assembled_tts)) diff --git a/settings.ini b/settings.ini index ee7fc61..588cb72 100644 --- a/settings.ini +++ b/settings.ini @@ -29,6 +29,11 @@ temperature = 0.6 # mid-word. max_tokens = 4096 +# Roughly one clip in twelve comes back cut off mid-word. Those are detected and +# asked for again with a different seed, up to this many extra attempts. 0 to +# switch retrying off. +truncation_retries = 2 + # Trim this many ms from the start of every clip, to remove an inhale artifact. # Try 80-150 if you hear one. Too high clips the first consonant. strip_leading_ms = 0 @@ -54,3 +59,13 @@ output_dir = output [comfy] # Address of the ComfyUI backend running Higgs Audio V3. comfy_url = http://127.0.0.1:8189 + +# Start ComfyUI automatically when it stops answering. A crash loses its queue, +# so the clip in flight is resubmitted once the backend is back. At most 3 +# attempts per run, 90s apart, so a backend that dies on startup cannot spawn an +# endless stream of terminal windows. yes/no. +autostart = yes + +# The script used to start it. Relative paths are resolved next to +# lektor_ai_comfy.py. +start_command = start_comfyui.sh diff --git a/ui.py b/ui.py index eabb6dd..a2a90ca 100644 --- a/ui.py +++ b/ui.py @@ -222,12 +222,15 @@ SETTING_LABELS = { "speed": "speed", "temperature": "speech temperature", "max_tokens": "max tokens per clip", + "truncation_retries": "retries if cut mid-word", "strip_leading_ms": "strip leading (ms)", "trailing_fade_ms": "trailing fade (ms)", "orig_volume": "sound volume", "tts_volume": "voiceover volume", "output_dir": "output folder", "comfy_url": "ComfyUI URL", + "comfy_autostart": "start ComfyUI if down", + "comfy_start_command": "start script", } SECTION_TITLES = {"tts": "Speech", "mix": "Mix", "paths": "Paths", "comfy": "Backend"} @@ -353,7 +356,11 @@ class SettingsDialog(tk.Toplevel): font=(F, 10)).grid(row=row, column=0, sticky="w", padx=(6, 14)) var = tk.StringVar(value=current[key]) self.vars[key] = var - if key == "voice" and voices: + if core._DEFAULTS[key][2] is core._as_bool: + w = ttk.Combobox(body, textvariable=var, values=["yes", "no"], + width=26, state="readonly", + style="Dark.TCombobox", font=(F, 10)) + elif key == "voice" and voices: w = ttk.Combobox(body, textvariable=var, values=voices, width=26, style="Dark.TCombobox", font=(F, 10)) else: