Install the whole backend from one script; queue log; UI fixes
install_comfyui.sh becomes install_backend.sh and now builds everything the app needs, not just the speech half: - Step 7 runs normalizer/setup.sh --model, which fetches both models: the open inflection model first, so the install gets as far as it can before anything can ask for credentials, then the gated fine-tune. - A missing Hugging Face account is no longer fatal. Accepting a licence is the one thing an installer cannot do, so the rest installs, the verification reports "fallback only (gated model missing)", and the summary prints the three commands that finish it. - --skip-normalizer, disk requirement 35 -> 55 GB, removal instructions and the step list updated to match. Queue logging — queue.log in the first queued video's job folder, which for a single-video run is its own: - Records what went wrong rather than what happened: warnings, tracebacks, ffmpeg errors, guard rejections, clips cut off mid-word or that crashed the backend, and the --regen line carrying their numbers. - ComfyUI failures *and* restarts, so the log shows the recovery rather than reading as though the run simply stopped. GUI-initiated backend actions go in too; they never pass through a job's pipe. - A failure also dumps the job's last 15 lines. Keyword matching alone is not enough — ffmpeg says "Invalid data found when processing input" and nothing that looks like an error. UI: - The output pane follows new output only while already at the bottom. Scroll up and it stays there; scroll back down and it resumes. Reading an earlier warning during a run was impossible while every ffmpeg progress frame yanked the view to the end. - The ComfyUI button's status dot is drawn as its own canvas item, so it can be green (answering) or amber (holds the port, does not answer). A "." inside the label could only ever be the same grey as the words. comfyui.log is capped at 120,000 lines, trimmed on backend start and every five minutes. Rewritten in place, never replaced: tee holds it open in append mode, so swapping the inode would leave the backend writing to an unlinked file. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -16,6 +16,7 @@ Needs tkinter (sudo apt install python3-tk). It also runs under
|
||||
to X11 bitmaps, so the window looks dated there; the UI detects this and says so.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import configparser
|
||||
import os
|
||||
import queue
|
||||
@@ -24,6 +25,7 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
@@ -61,6 +63,10 @@ PLUS = "#8f8f8f"
|
||||
LOG_BG = "#242424"
|
||||
LOG_FG = "#d0d0d0"
|
||||
|
||||
# Status dot on the ComfyUI button.
|
||||
COMFY_UP = "#4caf50" # answering
|
||||
COMFY_BUSY = "#d9a441" # holds the port but does not answer
|
||||
|
||||
# Rounding radius, in pixels, for fields and buttons.
|
||||
RADIUS = 9
|
||||
|
||||
@@ -113,6 +119,7 @@ class RoundButton(tk.Canvas):
|
||||
highlightthickness=0, bd=0)
|
||||
self._shape = round_rect(self, 1, 1, w - 1, h - 1, radius, fill=bg, outline="")
|
||||
self._label = self.create_text(w / 2, h / 2, text=text, fill=fg, font=f)
|
||||
self._dot = None
|
||||
self.bind("<Enter>", lambda _: self._paint(self._hover))
|
||||
self.bind("<Leave>", lambda _: self._paint(self._bg))
|
||||
self.bind("<Button-1>", self._click)
|
||||
@@ -131,6 +138,35 @@ class RoundButton(tk.Canvas):
|
||||
|
||||
def set_text(self, text):
|
||||
self.itemconfigure(self._label, text=text)
|
||||
if self._dot is not None:
|
||||
self._place_dot()
|
||||
|
||||
def set_indicator(self, colour):
|
||||
"""A status dot beside the label, or None for no dot.
|
||||
|
||||
Drawn as its own canvas item rather than a coloured character in the
|
||||
text: a canvas text item has one fill colour for the whole string, so
|
||||
a "●" in the label can only ever be the same grey as the words.
|
||||
"""
|
||||
if colour is None:
|
||||
if self._dot is not None:
|
||||
self.delete(self._dot)
|
||||
self._dot = None
|
||||
self.coords(self._label, int(self["width"]) / 2,
|
||||
int(self["height"]) / 2)
|
||||
return
|
||||
if self._dot is None:
|
||||
self._dot = self.create_oval(0, 0, 0, 0, fill=colour, outline="")
|
||||
self.itemconfigure(self._dot, fill=colour)
|
||||
self._place_dot()
|
||||
|
||||
def _place_dot(self):
|
||||
"""Keep label+dot centred together as the text changes length."""
|
||||
w, h, gap, r = int(self["width"]), int(self["height"]), 5, 4
|
||||
self.coords(self._label, w / 2 - (gap + r), h / 2)
|
||||
x1, _y1, x2, _y2 = self.bbox(self._label)
|
||||
cx, cy = x2 + gap + r, h / 2
|
||||
self.coords(self._dot, cx - r, cy - r, cx + r, cy + r)
|
||||
|
||||
|
||||
class RoundField(tk.Frame):
|
||||
@@ -925,7 +961,7 @@ class JobBlock(tk.Frame):
|
||||
except ValueError as exc:
|
||||
messagebox.showerror("Cannot run", str(exc))
|
||||
return
|
||||
self.app.enqueue([(self.label(), mode, cmd)])
|
||||
self.app.enqueue([(self.label(), mode, cmd, self.job_folder())])
|
||||
|
||||
def ask_regen(self):
|
||||
raw = SimplePrompt(self.app, "Regenerate clips",
|
||||
@@ -1036,7 +1072,9 @@ class LektorUI(tk.Tk):
|
||||
self.blocks = []
|
||||
self._group_seq = 0
|
||||
self._syncing = False
|
||||
self.jobs = [] # pending [(label, mode, cmd), …]
|
||||
self.jobs = [] # pending [(label, mode, cmd, job_folder), …]
|
||||
self.log_path = None # queue.log for the batch being run
|
||||
self._job_tail = collections.deque(maxlen=self.TAIL_LINES)
|
||||
self.proc = None
|
||||
self.running = False
|
||||
self.out_q = queue.Queue()
|
||||
@@ -1099,11 +1137,12 @@ class LektorUI(tk.Tk):
|
||||
RoundButton(top, "Queue dictionary", self.open_queue_dictionary,
|
||||
bg="#3a3a3a", fg=MUTED, hover="#4a4a4a", font=(F, 9),
|
||||
padx=13, pady=6).pack(side="right", padx=(0, 8))
|
||||
self.comfy_btn = RoundButton(top, "ComfyUI", self.toggle_comfy,
|
||||
self.comfy_btn = RoundButton(top, "Start ComfyUI", self.toggle_comfy,
|
||||
bg="#3a3a3a", fg=MUTED, hover="#4a4a4a",
|
||||
font=(F, 9), padx=13, pady=6)
|
||||
self.comfy_btn.pack(side="right", padx=(0, 8))
|
||||
self.after(500, self._refresh_comfy_button)
|
||||
self.after(60_000, self._trim_backend_log)
|
||||
|
||||
# scrollable list of job blocks
|
||||
mid = tk.Frame(self, bg=BG)
|
||||
@@ -1202,6 +1241,22 @@ class LektorUI(tk.Tk):
|
||||
# of the work is in lektor_ai_comfy so the terminal script and this button
|
||||
# behave identically.
|
||||
|
||||
def _trim_backend_log(self):
|
||||
"""Keep comfyui.log bounded while the backend runs.
|
||||
|
||||
Trimming only at startup is not enough: ComfyUI can stay up for days,
|
||||
and it is the running backend that fills the file. Every five minutes,
|
||||
and the call is cheap — it returns on a stat when the file is small.
|
||||
"""
|
||||
try:
|
||||
dropped = core.trim_comfy_log()
|
||||
if dropped:
|
||||
self.log(f"Trimmed {core.COMFY_LOG.name}: dropped {dropped} old "
|
||||
f"line(s), keeping the newest {core.COMFY_LOG_MAX_LINES}.")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.log(f"Could not trim {core.COMFY_LOG.name}: {exc}")
|
||||
self.after(300_000, self._trim_backend_log)
|
||||
|
||||
def _comfy_state(self):
|
||||
"""(running, healthy) — separate, because a backend can hold the port
|
||||
and the GPU while no longer answering."""
|
||||
@@ -1217,11 +1272,16 @@ class LektorUI(tk.Tk):
|
||||
def _refresh_comfy_button(self):
|
||||
running, healthy = self._comfy_state()
|
||||
if healthy:
|
||||
self.comfy_btn.set_text("ComfyUI ●")
|
||||
self.comfy_btn.set_text("ComfyUI")
|
||||
self.comfy_btn.set_indicator(COMFY_UP)
|
||||
elif running:
|
||||
self.comfy_btn.set_text("ComfyUI ◍") # up but not answering
|
||||
# Holding the port but not answering — the state that strands the
|
||||
# GPU, so it gets its own colour rather than looking healthy.
|
||||
self.comfy_btn.set_text("ComfyUI")
|
||||
self.comfy_btn.set_indicator(COMFY_BUSY)
|
||||
else:
|
||||
self.comfy_btn.set_text("Start ComfyUI")
|
||||
self.comfy_btn.set_indicator(None)
|
||||
self.after(5000, self._refresh_comfy_button)
|
||||
|
||||
def toggle_comfy(self):
|
||||
@@ -1240,17 +1300,17 @@ class LektorUI(tk.Tk):
|
||||
def _comfy_worker(self, running):
|
||||
try:
|
||||
if running:
|
||||
self.log("Stopping ComfyUI…")
|
||||
self.log_both("Stopping ComfyUI…")
|
||||
core.stop_comfy_backend()
|
||||
self.log("Starting ComfyUI…")
|
||||
self.log_both("Starting ComfyUI…")
|
||||
if not core.start_comfy_backend(core.COMFY_START_SCRIPT):
|
||||
self.log("Could not start ComfyUI — see the terminal window.")
|
||||
self.log_both("Could not start ComfyUI — see the terminal window.")
|
||||
return
|
||||
if core.wait_for_backend(180.0):
|
||||
self.log("ComfyUI is up.")
|
||||
self.log_both("ComfyUI is up.")
|
||||
else:
|
||||
self.log(f"ComfyUI did not answer within 180s. "
|
||||
f"Its output is in {core.COMFY_LOG}.")
|
||||
self.log_both(f"ComfyUI did not answer within 180s. "
|
||||
f"Its output is in {core.COMFY_LOG}.")
|
||||
finally:
|
||||
self.after(0, lambda: self.comfy_btn.set_enabled(True))
|
||||
self.after(0, self._refresh_comfy_button)
|
||||
@@ -1353,13 +1413,83 @@ class LektorUI(tk.Tk):
|
||||
jobs = []
|
||||
for b in self.blocks:
|
||||
try:
|
||||
jobs.append((b.label(), "generate", b.build_command("generate")))
|
||||
jobs.append((b.label(), "generate", b.build_command("generate"),
|
||||
b.job_folder()))
|
||||
except ValueError as exc:
|
||||
messagebox.showerror("Cannot start", f"{b.label()}:\n\n{exc}")
|
||||
return
|
||||
self.enqueue(jobs)
|
||||
|
||||
LOG_NAME = "queue.log"
|
||||
|
||||
# What counts as an issue. The pipeline is deliberately chatty on stdout —
|
||||
# every clip, every placement — and none of that belongs in a file you open
|
||||
# to find out what went wrong overnight. Matched case-insensitively because
|
||||
# the interesting lines come from three different sources: this script,
|
||||
# Python tracebacks, and ffmpeg, which capitalises nothing.
|
||||
ISSUE_RE = re.compile(
|
||||
r"error|warning|traceback|exception|fail|"
|
||||
r"cannot|could not|not found|no such|invalid|denied|"
|
||||
r"crash|killed comfyui|died|went away|unreachable|did not|"
|
||||
# Backend lifecycle: a restart is not an error, but a log that shows
|
||||
# the crash without the recovery reads like the run simply stopped.
|
||||
r"start(?:ing|ed) comfyui|stopping comfyui|comfyui is back|"
|
||||
r"comfyui is up|not answering|resubmitting|waiting for it|"
|
||||
r"rejected |safety check|skipping this clip|mid-word|"
|
||||
# The --regen line carries the clip numbers, and it follows its summary
|
||||
# on a line of its own — without it the log says three clips are broken
|
||||
# but not which three.
|
||||
r"--regen|"
|
||||
r"^\s*!", re.IGNORECASE)
|
||||
|
||||
# Kept per job so a failure can be explained even when its last words match
|
||||
# nothing above — an exit code with no context is not worth logging.
|
||||
TAIL_LINES = 15
|
||||
|
||||
def _open_log(self, jobs):
|
||||
"""Point the log at the first job of this batch.
|
||||
|
||||
One rule covers both cases the same way: the file goes in the first
|
||||
queued video's folder, which for a single-video run is simply its own.
|
||||
"""
|
||||
folder = next((f for _l, _m, _c, f in jobs if f is not None), None)
|
||||
if folder is None:
|
||||
self.log_path = None
|
||||
return
|
||||
self.log_path = folder / self.LOG_NAME
|
||||
try:
|
||||
self.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.log_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(f"\n===== queue started {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
f" — {len(jobs)} job(s) =====\n")
|
||||
except OSError as exc:
|
||||
self.log(f"Could not open {self.log_path}: {exc}")
|
||||
self.log_path = None
|
||||
|
||||
def _log_tail(self):
|
||||
"""The job's last lines, so a failure is explained even when its error
|
||||
text matches no keyword — ffmpeg in particular says things like
|
||||
"Invalid data found when processing input" and nothing else."""
|
||||
if not self.log_path or not self._job_tail:
|
||||
return
|
||||
self._log_issue(f" last {len(self._job_tail)} line(s) before it stopped:")
|
||||
for line in self._job_tail:
|
||||
self._log_issue(f" | {line}")
|
||||
|
||||
def _log_issue(self, text):
|
||||
"""Append one line. Appends rather than truncates, so the file is a
|
||||
record across runs and a crash never costs the previous session."""
|
||||
if not self.log_path:
|
||||
return
|
||||
try:
|
||||
with self.log_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(f"{time.strftime('%H:%M:%S')} {text.rstrip()}\n")
|
||||
except OSError:
|
||||
self.log_path = None # a broken log must not stop a run
|
||||
|
||||
def enqueue(self, jobs):
|
||||
if not self.running:
|
||||
self._open_log(jobs)
|
||||
self.jobs.extend(jobs)
|
||||
self.log(f"Queued {len(jobs)} job{'' if len(jobs) == 1 else 's'} "
|
||||
f"({len(self.jobs)} pending).")
|
||||
@@ -1372,13 +1502,16 @@ class LektorUI(tk.Tk):
|
||||
self._buttons_idle()
|
||||
self.status.configure(text="Idle")
|
||||
self.log("Queue finished.\n")
|
||||
self._log_issue("===== queue finished =====")
|
||||
return
|
||||
label, mode, cmd = self.jobs.pop(0)
|
||||
label, mode, cmd, folder = self.jobs.pop(0)
|
||||
self.running = True
|
||||
self.gen_btn.set_enabled(False)
|
||||
self.stop_btn.set_enabled(True)
|
||||
self.status.configure(text=f"{mode}: {label} ({len(self.jobs)} left)")
|
||||
self.log(f"\n=== {mode.upper()}: {label} ===")
|
||||
self._log_issue(f"--- {mode}: {label}")
|
||||
self._job_tail.clear()
|
||||
self.log("$ " + " ".join(f'"{c}"' if " " in c else c for c in cmd[1:]))
|
||||
threading.Thread(target=self._run, args=(cmd,), daemon=True).start()
|
||||
|
||||
@@ -1441,11 +1574,17 @@ class LektorUI(tk.Tk):
|
||||
self.proc = None
|
||||
if payload == 0:
|
||||
self.log(" OK - finished")
|
||||
self._log_issue(" finished OK")
|
||||
elif payload is not None and payload < 0:
|
||||
self.log(f" STOPPED (signal {-payload})")
|
||||
self._log_issue(f" STOPPED by signal {-payload} — "
|
||||
f"{len(self.jobs)} job(s) cancelled")
|
||||
self.jobs.clear()
|
||||
else:
|
||||
self.log(f" FAILED (exit {payload})")
|
||||
self._log_issue(f" FAILED with exit code {payload} — "
|
||||
f"{len(self.jobs)} job(s) cancelled")
|
||||
self._log_tail()
|
||||
self.jobs.clear()
|
||||
self.after(0, lambda c=payload: messagebox.showerror(
|
||||
"Job failed",
|
||||
@@ -1457,19 +1596,45 @@ class LektorUI(tk.Tk):
|
||||
pass
|
||||
self.after(80, self._drain_output)
|
||||
|
||||
def _at_bottom(self):
|
||||
"""True when the view is already showing the end of the log.
|
||||
|
||||
Scrolling to the end on every insert makes the pane unreadable during a
|
||||
run: output arrives several times a second, so any attempt to look back
|
||||
is yanked to the bottom before a line can be read. Following the tail
|
||||
only while the user is already at it means scrolling up pauses the
|
||||
follow, and scrolling back down resumes it, with no button to press.
|
||||
"""
|
||||
try:
|
||||
return self.log_txt.yview()[1] >= 0.999
|
||||
except tk.TclError: # widget gone during shutdown
|
||||
return True
|
||||
|
||||
def _append(self, text):
|
||||
follow = self._at_bottom()
|
||||
self.log_txt.configure(state="normal")
|
||||
self.log_txt.insert("end", text)
|
||||
self.log_txt.see("end")
|
||||
self.log_txt.configure(state="disabled")
|
||||
if follow:
|
||||
self.log_txt.see("end")
|
||||
|
||||
def _replace_last(self, text):
|
||||
"""Overwrite the last line, the way a terminal handles a carriage return."""
|
||||
follow = self._at_bottom()
|
||||
self.log_txt.configure(state="normal")
|
||||
self.log_txt.delete("end-1c linestart", "end-1c")
|
||||
self.log_txt.insert("end-1c", text)
|
||||
self.log_txt.see("end")
|
||||
self.log_txt.configure(state="disabled")
|
||||
if follow:
|
||||
self.log_txt.see("end")
|
||||
|
||||
def _record_if_issue(self, text):
|
||||
line = text.strip()
|
||||
if not line:
|
||||
return
|
||||
self._job_tail.append(line)
|
||||
if self.ISSUE_RE.search(line):
|
||||
self._log_issue(line)
|
||||
|
||||
def _emit_line(self, text):
|
||||
"""A \n-terminated segment.
|
||||
@@ -1479,6 +1644,7 @@ class LektorUI(tk.Tk):
|
||||
to overwrite the line rather than start a new one, or every update is
|
||||
appended and the bar marches down the pane instead of animating.
|
||||
"""
|
||||
self._record_if_issue(text)
|
||||
if self._in_redraw:
|
||||
self._replace_last(text)
|
||||
self._append("\n")
|
||||
@@ -1493,6 +1659,12 @@ class LektorUI(tk.Tk):
|
||||
def log(self, msg):
|
||||
self._append(msg + "\n")
|
||||
|
||||
def log_both(self, msg):
|
||||
"""Output pane and queue.log. For things the GUI does itself — starting
|
||||
or stopping the backend — which never pass through a job's pipe."""
|
||||
self.log(msg)
|
||||
self._log_issue(msg)
|
||||
|
||||
def _buttons_idle(self):
|
||||
self.gen_btn.set_enabled(True)
|
||||
self.stop_btn.set_enabled(False)
|
||||
@@ -1538,12 +1710,12 @@ class LektorUI(tk.Tk):
|
||||
try:
|
||||
if core.comfy_pid() is None:
|
||||
return
|
||||
self.log("Stopping ComfyUI…")
|
||||
self.log_both("Stopping ComfyUI on exit…")
|
||||
self.update_idletasks() # let the message paint before we block
|
||||
if not core.stop_comfy_backend():
|
||||
self.log("ComfyUI did not stop — check for a leftover process.")
|
||||
self.log_both("ComfyUI did not stop — check for a leftover process.")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.log(f"Could not stop ComfyUI: {exc}")
|
||||
self.log_both(f"Could not stop ComfyUI: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user