From 39452a4281bb1c13c02b124d8f314a2836324b3a Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 31 Aug 2026 21:15:27 +0200 Subject: [PATCH] Mux the spoken subtitles into the output, and fix MP4 subtitle copying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --include-subs adds the subtitles the voiceover was made from to the finished video as one more track, alongside whatever the source already carried. Useful where the source has no Polish subtitles of its own, and for checking the voiceover against the text it came from. Every run now writes clips//voiceover_subs.srt: the cue text verbatim, timestamps shifted by --start and cues outside --start..--end dropped. Verbatim because this is what a viewer reads, not what the model was handed — dictionary rewrites and spelled-out numbers must not leak into it. It is written whether or not --include-subs was passed, because --rebuild --include-subs is given only the video and would otherwise have nothing to mux; runs made before this existed fall back to tts_input.srt with a note that the wording is the model's. Copying subtitle tracks from an MP4 source was broken, and silently: MP4 stores them as mov_text (tx3g), which the Matroska muxer rejects — "Subtitle codec 94213 is not supported" — and then no output file is written at all. Those tracks are now decoded to SubRip in a temporary file and muxed from there, while image formats (PGS, VOBSUB) are still stream-copied since they cannot become text. Two details that follow from the detour: ffmpeg's tx3g decoder emits the format's style records as markup, which players either honour (subtitles rendered at whatever pixel size the MP4 specified, usually unreadably small) or print literally — so the tags are stripped; and a standalone .srt carries no language, so the source track's language and title tags are re-applied by hand or the output reads "undefined". An empty placeholder track, which rips often carry, is dropped rather than muxed as an empty one. UI: an "include the subtitles" checkbox next to the embedded-subtitles one. It is cleared and greyed out when the embedded track is chosen, since those subtitles already travel with the video, and it mirrors across linked jobs like the other per-job settings. RoundCheck grew a set_enabled() for that. The remove button moves into a column of its own spanning both rows, so it stays centred now that the second row is taller. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NkWDx9dVB7xBpqntvjvm6A --- lektor_ai_comfy.py | 174 +++++++++++++++++++++++++++++++++++++++++++-- ui.py | 71 ++++++++++++++++-- 2 files changed, 233 insertions(+), 12 deletions(-) diff --git a/lektor_ai_comfy.py b/lektor_ai_comfy.py index 62c9c32..185660e 100644 --- a/lektor_ai_comfy.py +++ b/lektor_ai_comfy.py @@ -819,6 +819,63 @@ def split_dialogue_blocks(blocks: list) -> list: return result +def save_voiceover_subs(srt_path: str, clips_dir: Path, start_sec: float, + end_sec: float) -> Path: + """Keep a copy of the subtitles this run speaks, timed to the output video. + + Written for every run, so a later --rebuild --include-subs still has them + when the original .srt is no longer on the command line. Cue text is copied + verbatim: this is the track a viewer reads, not the text handed to the + model, so dictionary replacements and spelled-out numbers stay out of it. + Timestamps are shifted by --start, because the video they are muxed into + begins there, and cues outside --start..--end are dropped. + """ + out = clips_dir / "voiceover_subs.srt" + text = Path(srt_path).read_text(encoding="utf-8-sig") + kept = [] + for block in re.split(r"\n{2,}", text.strip()): + lines = block.strip().splitlines() + ti = next((i for i, l in enumerate(lines) if "-->" in l), None) + if ti is None: + continue + parts = lines[ti].split("-->") + if len(parts) != 2: + continue + t_start = _parse_srt_time(parts[0]) + t_end = _parse_srt_time(parts[1]) + if t_start < start_sec or t_start > end_sec: + continue + body = [l for l in lines[ti + 1:] if l.strip()] + if not body: + continue + kept.append(f"{len(kept) + 1}\n" + f"{_fmt_srt_time(t_start - start_sec)} --> " + f"{_fmt_srt_time(max(0.0, t_end - start_sec))}\n" + + "\n".join(body) + "\n") + out.write_text("\n".join(kept), encoding="utf-8") + return out + + +def voiceover_subs_file(job: Path): + """The subtitle file --include-subs should mux, or None if there is none. + + Prefers the verbatim copy kept by the run itself. Runs made before that + copy existed still have tts_input.srt, which is the same timing but the + model's wording — better than no track at all, so it is used with a note. + """ + saved = job / "voiceover_subs.srt" + if saved.is_file() and saved.stat().st_size: + return saved + fallback = job / "tts_input.srt" + if fallback.is_file() and fallback.stat().st_size: + print(f" --include-subs: no {saved.name} from the original run — using " + f"{fallback.name} instead (the model's text, not the original wording).") + return fallback + print(" --include-subs: no subtitles found in the job folder — skipping " + "the extra track.", file=sys.stderr) + return None + + def blocks_to_srt(blocks: list) -> str: """Convert (start, end, text) tuples back to SRT string for inspection.""" lines = [] @@ -1196,12 +1253,48 @@ def get_media_duration(path: str) -> float: return 0.0 +_FONT_TAG_RE = re.compile(r']*>', re.IGNORECASE) + + +def extract_mov_text_srt(video_path: str, stream_index: int, out_path: Path) -> bool: + """Decode one mov_text track to a SubRip file, without its styling. + + ffmpeg's tx3g decoder turns the format's style records into markup, so + every cue comes out wrapped in . Players + that honour those tags then render the subtitles at whatever pixel size the + MP4 happened to specify — usually far too small on a 1080p screen — and the + ones that don't honour them show the tags as literal text. Neither is what + the source looked like, and the tags carry nothing worth keeping, so they + are dropped and the plain cue text is muxed instead. + + Returns False if the track holds no cues, which is common for the empty + placeholder track MP4 rips often carry. + """ + result = subprocess.run( + ["ffmpeg", "-y", "-v", "error", "-i", video_path, + "-map", f"0:{stream_index}", "-c:s", "srt", "-f", "srt", str(out_path)], + capture_output=True, text=True, + ) + if result.returncode != 0 or not out_path.exists(): + print(f" WARNING: could not extract subtitle stream #{stream_index}: " + f"{result.stderr.strip()[-300:]}", file=sys.stderr) + return False + text = out_path.read_text(encoding="utf-8", errors="replace") + cleaned = _FONT_TAG_RE.sub("", text) + if not cleaned.strip(): + return False + out_path.write_text(cleaned, encoding="utf-8") + return True + + def merge_voiceover(video_path: str, tts_path: str, output_path: str, - orig_volume: float = 0.60, tts_volume: float = 1.8): + orig_volume: float = 0.60, tts_volume: float = 1.8, + extra_subs: Path = None): """Produce a video with two audio tracks: track 0 — Original (first audio stream from source, stream-copied) track 1 — Polish (lektor mix: TTS at tts_volume + original at orig_volume) - plus every subtitle track from the source, stream-copied. + plus every subtitle track from the source, stream-copied, and — when + extra_subs is given — the subtitles that were spoken, as one more track. Done in two passes: 1. Render the lektor mix on its own to a temporary audio file. @@ -1253,6 +1346,7 @@ def merge_voiceover(video_path: str, tts_path: str, output_path: str, out = Path(output_path) mix_path = out.parent / f"{out.stem}_mix.m4a" + sub_tmp: list = [] # extracted .srt files, removed after the mux try: # ── Pass 1: render the lektor mix on its own ───────────────────────── _run_progress([ @@ -1269,15 +1363,66 @@ def merge_voiceover(video_path: str, tts_path: str, output_path: str, if sub_streams: print(f" Including {len(sub_streams)} subtitle track(s) in output") + # Subtitles cannot all be stream-copied: an MP4 source carries them as + # mov_text (tx3g), which the matroska muxer rejects outright — + # "Subtitle codec 94213 is not supported", and no output file is + # written at all. Those tracks are decoded to SubRip in a temporary + # file and muxed from there; everything else is still copied, which is + # what the image-based formats (PGS, VOBSUB) require since they cannot + # become text at all. cmd = [ "ffmpeg", "-y", "-i", video_path, "-i", str(mix_path), - "-map", "0:v", "-map", "0:a:0", "-map", "1:a:0", ] + maps = ["-map", "0:v", "-map", "0:a:0", "-map", "1:a:0"] + sub_codecs, sub_meta = [], [] + next_input = 2 + converted = 0 + out_idx = 0 # position among the OUTPUT subtitle streams, + # which a skipped track must not advance for s in sub_streams: - cmd += ["-map", f"0:{s['index']}"] + if s.get("codec_name") == "mov_text": + srt_path = out.parent / f"{out.stem}_sub{s['index']}.srt" + if not extract_mov_text_srt(video_path, s["index"], srt_path): + srt_path.unlink(missing_ok=True) + print(f" Subtitle stream #{s['index']} has no cues — skipping it") + continue + sub_tmp.append(srt_path) + cmd += ["-i", str(srt_path)] + maps += ["-map", f"{next_input}:0"] + # A standalone .srt carries no language, so the tag the source + # track had has to be re-applied by hand or the output track + # shows up as "undefined". + lang = (s.get("tags") or {}).get("language") + if lang: + sub_meta += [f"-metadata:s:s:{out_idx}", f"language={lang}"] + title = (s.get("tags") or {}).get("title") + if title: + sub_meta += [f"-metadata:s:s:{out_idx}", f"title={title}"] + next_input += 1 + converted += 1 + else: + maps += ["-map", f"0:{s['index']}"] + sub_codecs += [f"-c:s:{out_idx}", "copy"] + out_idx += 1 + # The subtitles the voiceover was made from, added as one more track + # so the file carries them even when the source had none of its own. + if extra_subs is not None: + print(f" Adding the voiceover subtitles as an extra track: " + f"{Path(extra_subs).name}") + cmd += ["-i", str(extra_subs)] + maps += ["-map", f"{next_input}:0"] + sub_codecs += [f"-c:s:{out_idx}", "copy"] + sub_meta += [f"-metadata:s:s:{out_idx}", "language=pol", + f"-metadata:s:s:{out_idx}", "title=Lektor"] + next_input += 1 + out_idx += 1 + + if converted: + print(f" Converted {converted} mov_text subtitle track(s) to SubRip — " + f"Matroska cannot store the MP4 subtitle format") + cmd += maps + ["-c:v", "copy", "-c:a", "copy"] + sub_codecs + sub_meta cmd += [ - "-c", "copy", # Must be an OUTPUT option (before the output path) — placed before # -i it silently configures the demuxer instead and does nothing. # Sparse subtitle tracks make the muxer give up on interleaving @@ -1295,6 +1440,8 @@ def merge_voiceover(video_path: str, tts_path: str, output_path: str, _run(cmd, "Muxing final video (2 audio tracks + subtitles)") finally: mix_path.unlink(missing_ok=True) + for p in sub_tmp: + p.unlink(missing_ok=True) # --------------------------------------------------------------------------- @@ -2008,7 +2155,8 @@ def _rebuild_video(args, out_dir: Path): print(f"\n Building 2-track video " f"(TTS {int(args.tts_volume*100)}% / orig {int(args.orig_volume*100)}%)…") merge_voiceover(video_src, str(assembled_tts), str(final_output), - orig_volume=args.orig_volume, tts_volume=args.tts_volume) + orig_volume=args.orig_volume, tts_volume=args.tts_volume, + extra_subs=voiceover_subs_file(job) if args.include_subs else None) print(f"\nDone! Output: {final_output}") @@ -2260,6 +2408,11 @@ def main(): "this video, then stop. Repairs runs where the model failed " "to stop and padded a clip out to --max-tokens, which pushes " "every later clip out of sync. Follow with --rebuild.") + parser.add_argument("--include-subs", action="store_true", + help="Also mux the subtitles the voiceover was made from into " + "the finished video, as an extra track alongside whatever " + "subtitle tracks the source already had. Works with " + "--rebuild too: the run keeps its own copy of them.") parser.add_argument("--embedded-sub-index", type=int, default=None, metavar="N", help="With --embedded-subs, pick this ffmpeg stream index instead " "of prompting. Used by ui.py and for scripted runs.") @@ -2400,6 +2553,11 @@ def main(): if end_sec is None: end_sec = 99 * 3600 + # Kept before any processing, and for every run: --rebuild --include-subs + # is given only the video, so this is the only place the original wording + # can be preserved from. + save_voiceover_subs(srt_path, clips_dir, start_sec, end_sec) + wanted = [] if args.dictionary.strip(): wanted.append(resolve_dictionary_path(args.dictionary)) @@ -2504,7 +2662,9 @@ def _run_generation(args, cfg, blocks, clips_dir, start_sec, end_sec): print(f"\n[4/4] Building 2-track video (Original + Polish lektor " f"TTS {int(args.tts_volume*100)}% / orig {int(args.orig_volume*100)}%)…") merge_voiceover(str(video_fragment), str(assembled_tts), str(final_output), - orig_volume=args.orig_volume, tts_volume=args.tts_volume) + orig_volume=args.orig_volume, tts_volume=args.tts_volume, + extra_subs=(voiceover_subs_file(clips_dir) + if args.include_subs else None)) print(f"\nDone! Output: {final_output}") diff --git a/ui.py b/ui.py index 910eb2a..b99e7f4 100644 --- a/ui.py +++ b/ui.py @@ -580,6 +580,7 @@ class RoundCheck(tk.Frame): def __init__(self, master, variable, text, font, command=None, box=22): super().__init__(master, bg=master.cget("background")) self.var, self.command = variable, command + self._enabled = True self.canvas = tk.Canvas(self, width=box, height=box, bg=master.cget("background"), highlightthickness=0, bd=0, cursor="hand2") @@ -599,6 +600,8 @@ class RoundCheck(tk.Frame): self.refresh() def _toggle(self, _=None): + if not self._enabled: + return self.var.set(not self.var.get()) self.refresh() if self.command: @@ -607,7 +610,19 @@ class RoundCheck(tk.Frame): def refresh(self): on = bool(self.var.get()) self.canvas.itemconfigure(self._tick, state="normal" if on else "hidden") - self.canvas.itemconfigure(self._box, fill="#ffffff" if on else FIELD) + if self._enabled: + fill = "#ffffff" if on else FIELD + else: + fill = "#8f8f8f" if on else SEP + self.canvas.itemconfigure(self._box, fill=fill) + + def set_enabled(self, on): + """Grey the box out and stop it responding to clicks.""" + self._enabled = bool(on) + cursor = "hand2" if self._enabled else "arrow" + self.canvas.configure(cursor=cursor) + self.label.configure(cursor=cursor, fg=FG if self._enabled else "#828282") + self.refresh() # Dropdown choices. The comboboxes are editable, so these are starting points # rather than limits. @@ -663,10 +678,18 @@ class JobBlock(tk.Frame): self.f_btn = (F, 9, "bold") self.f_num = (F, 13) + # The remove button gets a column of its own spanning both rows, so it + # stays vertically centred against the block however tall that grows. + close_col = tk.Frame(self, bg=BG) + close_col.pack(side="left", fill="y", padx=(0, 12)) + RoundButton(close_col, "\u00d7", self.remove, bg="#4a4a4a", fg=MUTED, + hover="#7a3a3a", font=(F, 11, "bold"), padx=7, pady=5 + ).pack(expand=True) + # Both rows share one grid, so column 0 gets a single width and the two # input fields line up exactly as they do in the mock-up. grid = tk.Frame(self, bg=BG) - grid.pack(fill="x") + grid.pack(side="left", fill="both", expand=True) grid.grid_columnconfigure(0, weight=1) # ── Row 1: video + the four numeric dropdowns ─────────────────────── @@ -687,9 +710,6 @@ class JobBlock(tk.Frame): if self.group is not None: tk.Label(r1, text="linked", bg=BG, fg="#7fa8c9", font=(F, 8) ).pack(side="left", padx=(0, 6)) - RoundButton(r1, "\u00d7", self.remove, bg="#4a4a4a", fg=MUTED, - hover="#7a3a3a", font=(F, 11, "bold"), padx=7, pady=5 - ).pack(side="left", padx=(10, 0)) # Mirror every numeric field to the other blocks in this group. A write # trace catches typed values as well as menu picks. @@ -713,6 +733,13 @@ class JobBlock(tk.Frame): font=(F, 9), command=self.toggle_embedded, box=22) self.emb_check.pack(side="left", padx=(0, 16)) + # Only meaningful for an external .srt — an embedded track is already + # in the file, so the option is cleared and greyed out for those. + self.include_subs = tk.BooleanVar(value=False) + self.inc_check = RoundCheck(r2, self.include_subs, "include the\nsubtitles", + font=(F, 9), command=self._mirror_include, box=22) + self.inc_check.pack(side="left", padx=(0, 16)) + # packed right-to-left so the group ends flush with the dropdowns above for label, cmd in (("RE-BUILD", self.do_rebuild), ("RESUME ...", self.ask_resume), @@ -786,6 +813,33 @@ class JobBlock(tk.Frame): finally: self.app._syncing = False + def _mirror_include(self): + """Copy the include-subtitles state to the rest of the group.""" + if self.app._syncing or self.group is None: + return + self.app._syncing = True + try: + want = self.include_subs.get() + for b in self._peers(): + if b.embedded.get() or b.include_subs.get() == want: + continue + b.include_subs.set(want) + b.inc_check.refresh() + finally: + self.app._syncing = False + + def _sync_include_state(self): + """Follow the embedded-subtitles checkbox. + + With an embedded track the subtitles already travel with the video, so + there is nothing to add — the option is cleared and greyed out. + """ + if self.embedded.get(): + self.include_subs.set(False) + self.inc_check.set_enabled(False) + else: + self.inc_check.set_enabled(True) + # ── file selection ────────────────────────────────────────────────────── def browse_video(self): p = choose_files("Select video", VIDEO_TYPES, @@ -820,6 +874,7 @@ class JobBlock(tk.Frame): if not video or not Path(video).is_file(): self.embedded.set(False) self.emb_check.refresh() + self._sync_include_state() messagebox.showwarning("No video", "Choose a video file first.") return self.sub_field.grid_remove() @@ -829,10 +884,12 @@ class JobBlock(tk.Frame): self.sub_combo["values"] = [] # ffprobe on a large file takes a moment; keep the UI responsive. threading.Thread(target=self._probe_subs, args=(video,), daemon=True).start() + self._sync_include_state() self._mirror_embedded() else: self.sub_combo.grid_remove() self.sub_field.grid() + self._sync_include_state() self._mirror_embedded() def _probe_subs(self, video): @@ -916,6 +973,10 @@ class JobBlock(tk.Frame): cmd += self.common_args() + # --regen only writes clips, so there is no mux for the track to join. + if self.include_subs.get() and not self.embedded.get() and mode != "regen": + cmd.append("--include-subs") + # The whole queue shares one extra dictionary, kept with the first # video. --regen and --rebuild work purely from tts_input.srt, so the # flag would do nothing there. A resume normally reuses that file too,