- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base workflow, serial render jobs, WS progress mirroring ComfyUI's push events, /api/view + ComfyUI-compatible /view, /upload/image, /easy-media/* aliases for the ported editor - dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI bridge (substitutes the plan into base_workflow.json, converts to the /prompt API format, polls history, downloads outputs) - timeline port: frontend/src/bench mounts the unmodified MultiTrack editor against a comfyAPI shim (fetchApi/toast/dialog/custom events); build.bench.ts emits bench/ui-dist served by the server - packaging: PyInstaller onedir spec + Tauri2 sidecar docs - tests: bench/tests contract suite + scripts/test_bench_pipeline.py e2e (API+WS+render); prebuilt bench UI committed
325 lines
12 KiB
Python
325 lines
12 KiB
Python
"""Self-contained mock backend.
|
|
|
|
Renders the fixed pipeline without torch/ComfyUI: deterministic synthetic
|
|
frames derived from (prompt, seed, frame index), a sine audio bed, PIL
|
|
lanczos "upscale" for the RTX branch, and ffmpeg h264 encode. Exercises the
|
|
exact same stage/progress contract as the ComfyUI bridge backend so the UI
|
|
and the API cannot tell the difference.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import math
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from ..tracks import RenderPlan, Segment
|
|
|
|
ProgressCb = Callable[[str, float, str, dict | None], None]
|
|
|
|
_VIDEO_CODECS = ("h264", "libx264", "mpeg4")
|
|
|
|
|
|
@dataclass
|
|
class RenderResult:
|
|
outputs: dict[str, dict] = field(default_factory=dict)
|
|
console: list[str] = field(default_factory=list)
|
|
|
|
def log(self, line: str) -> None:
|
|
self.console.append(line)
|
|
|
|
|
|
def _pick_encoder(codec: str) -> str:
|
|
probe = subprocess.run(
|
|
["ffmpeg", "-hide_banner", "-encoders"],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
names = probe.stdout.lower()
|
|
if codec == "h264" and "libx264" in names:
|
|
return "libx264"
|
|
for cand in _VIDEO_CODECS:
|
|
if cand in names:
|
|
return cand
|
|
return "mpeg4"
|
|
|
|
|
|
def _audio_encoder() -> str:
|
|
probe = subprocess.run(
|
|
["ffmpeg", "-hide_banner", "-encoders"],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
return "aac" if " aac" in probe.stdout.lower() else "libmp3lame"
|
|
|
|
|
|
def _segment_hue(segment: Segment, index: int) -> float:
|
|
digest = hashlib.sha256(f"{segment.prompt}|{index}".encode()).digest()
|
|
return digest[0] * 360.0 / 255.0
|
|
|
|
|
|
def _segment_freq(segment: Segment, index: int) -> float:
|
|
digest = hashlib.sha256(f"{segment.prompt}|{index}".encode()).digest()
|
|
return 280.0 + digest[1] % 5 * 55.0 # 280..500 Hz
|
|
|
|
|
|
def _seed_noise(seed: int, frame: int) -> float:
|
|
digest = hashlib.sha256(f"{seed}:{frame}".encode()).digest()
|
|
return digest[0] / 255.0 - 0.5
|
|
|
|
|
|
def _synthesize_chunk(
|
|
chunk_start: int,
|
|
frames: int,
|
|
plan: RenderPlan,
|
|
w: int,
|
|
h: int,
|
|
) -> bytes:
|
|
"""Deterministic RGB24 bytes for frames [chunk_start, chunk_start+frames)."""
|
|
import numpy as np
|
|
|
|
out = bytearray()
|
|
yy, xx = np.mgrid[0:h, 0:w].astype(np.float32)
|
|
xn = xx / max(1, w - 1)
|
|
yn = yy / max(1, h - 1)
|
|
seg_by_frame = _frame_to_segment_map(plan)
|
|
for f in range(chunk_start, chunk_start + frames):
|
|
seg, seg_index = seg_by_frame.get(f, (None, 0))
|
|
hue = _segment_hue(seg, seg_index) if seg else 210.0
|
|
t = f / max(1, plan.frame_rate)
|
|
# HSV-ish deterministic palette: hue rotates per segment prompt,
|
|
# luminance carries a moving wave so motion is visible.
|
|
base = np.zeros((h, w, 3), dtype=np.float32)
|
|
h1 = hue % 360.0
|
|
rgb = _hsv_to_rgb(h1 / 360.0, 0.55, 0.6)
|
|
wave = 0.25 * np.sin(6.0 * math.pi * (xn + 0.35 * t)) + 0.15 * np.sin(
|
|
4.0 * math.pi * (yn - 0.25 * t)
|
|
)
|
|
lum = 0.45 + wave + 0.06 * _seed_noise(plan.seed, f)
|
|
base[..., 0] = rgb[0] * lum
|
|
base[..., 1] = rgb[1] * lum
|
|
base[..., 2] = rgb[2] * lum
|
|
# progress bar strip at the bottom: frame position inside the timeline
|
|
pos = f / max(1, plan.total_length or 1)
|
|
bar_y = int(h * 0.95)
|
|
base[bar_y:, : int(w * pos), 1] = 0.95
|
|
out += (
|
|
np.clip(base * 255.0, 0, 255).astype(np.uint8).tobytes()
|
|
)
|
|
return bytes(out)
|
|
|
|
|
|
def _frame_to_segment_map(plan: RenderPlan) -> dict[int, tuple[Segment, int]]:
|
|
mapping: dict[int, tuple[Segment, int]] = {}
|
|
for idx, seg in enumerate(plan.segments):
|
|
for f in range(seg.start_frame, seg.end_frame):
|
|
mapping[f] = (seg, idx)
|
|
return mapping
|
|
|
|
|
|
def _hsv_to_rgb(h: float, s: float, v: float) -> tuple[float, float, float]:
|
|
import colorsys
|
|
|
|
r, g, b = colorsys.hsv_to_rgb(h % 1.0, s, v)
|
|
return r, g, b
|
|
|
|
|
|
def _s16le_audio(plan: RenderPlan, sample_rate: int = 44100) -> bytes:
|
|
"""Stereo s16le sine bed; per-segment tone, silence for muted segments."""
|
|
import numpy as np
|
|
|
|
total = int(sample_rate * plan.duration_seconds)
|
|
if total <= 0:
|
|
return b""
|
|
wave = np.zeros(total, dtype=np.float32)
|
|
sr = sample_rate
|
|
for idx, seg in enumerate(plan.segments):
|
|
if seg.muted or not seg.prompt:
|
|
continue
|
|
a, b = seg.start_frame, seg.end_frame
|
|
i0 = int(a / max(1, plan.frame_rate) * sr)
|
|
i1 = min(total, int(b / max(1, plan.frame_rate) * sr))
|
|
if i1 <= i0:
|
|
continue
|
|
t = np.arange(i1 - i0, dtype=np.float32) / sr
|
|
freq = _segment_freq(seg, idx)
|
|
gain = 0.12 * (10 ** (seg.volume_db / 20.0))
|
|
wave[i0:i1] += gain * np.sin(2 * math.pi * freq * t)
|
|
stereo = np.repeat(np.clip(wave, -1, 1)[:, None], 2, axis=1)
|
|
return (stereo * 32767.0).astype("<i2").tobytes()
|
|
|
|
|
|
def render(plan: RenderPlan, work_dir: Path, progress: ProgressCb) -> RenderResult:
|
|
"""Run the mock pipeline. Returns native + upscaled output descriptors."""
|
|
import numpy as np
|
|
from PIL import Image
|
|
|
|
result = RenderResult()
|
|
t0 = time.time()
|
|
work_dir.mkdir(parents=True, exist_ok=True)
|
|
if shutil.which("ffmpeg") is None:
|
|
raise RuntimeError("ffmpeg not found on PATH - the mock backend needs it")
|
|
|
|
w, h = plan.width, plan.height
|
|
frames = plan.total_length or sum(s.frame_count for s in plan.segments)
|
|
if frames <= 0:
|
|
raise RuntimeError("timeline has no frames to render")
|
|
|
|
def stage(name: str, pct: float, msg: str, data: dict | None = None) -> None:
|
|
progress(name, pct, msg, data)
|
|
|
|
# stages 1-3: loader / lora / preview (contract parity with the real chain)
|
|
stage("MoxieMultimediaLoader", 4.0, "mock: loading Moxie-Multimedia model set")
|
|
stage("LoraLoaderModelOnly", 9.0, f"mock: LoRA {plan.lora or 'none'} @ {plan.lora_strength}")
|
|
stage("MoxiePreviewOverride", 12.0, "mock: preview override active")
|
|
|
|
stage("easy multiTrackEditor", 15.0,
|
|
f"mock: timeline {w}x{h} @ {plan.frame_rate}fps, "
|
|
f"{len(plan.segments)} task segment(s)")
|
|
|
|
encoder = _pick_encoder(plan.codec)
|
|
aenc = _audio_encoder()
|
|
result.log(f"encoder: {encoder}, audio: {aenc}")
|
|
|
|
native_path = work_dir / "native.mp4"
|
|
upscaled_path = work_dir / "upscaled.mp4"
|
|
do_upscale = bool(plan.upscale_width and plan.upscale_height)
|
|
|
|
audio_bytes = _s16le_audio(plan)
|
|
|
|
def open_enc(out_path: Path, ow: int, oh: int) -> subprocess.Popen:
|
|
import os as _os
|
|
|
|
cmd = [
|
|
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
|
"-f", "rawvideo", "-pix_fmt", "rgb24",
|
|
"-s", f"{ow}x{oh}", "-r", str(plan.frame_rate), "-i", "pipe:0",
|
|
"-f", "s16le", "-ar", "44100", "-ac", "2",
|
|
]
|
|
r, w = _os.pipe()
|
|
# give ffmpeg the READ end (r); parent writes audio into w
|
|
cmd += ["-i", f"/dev/fd/{r}", "-map", "0:v", "-map", "1:a",
|
|
"-c:v", encoder, "-pix_fmt", "yuv420p",
|
|
"-movflags", "+faststart",
|
|
"-c:a", aenc, "-b:a", "128k", "-shortest",
|
|
str(out_path)]
|
|
proc = subprocess.Popen(
|
|
cmd, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.PIPE, pass_fds=(r,),
|
|
)
|
|
proc._audio_wfd = w # type: ignore[attr-defined]
|
|
_os.close(r)
|
|
return proc
|
|
|
|
def feed_audio(proc: subprocess.Popen) -> None:
|
|
# Audio may exceed the 64KB pipe buffer; pump it from a thread so the
|
|
# video stdin feed never deadlocks against ffmpeg's startup probing.
|
|
import os as _os
|
|
import threading as _threading
|
|
|
|
def _pump() -> None:
|
|
try:
|
|
view = memoryview(audio_bytes)
|
|
while view:
|
|
n = _os.write(proc._audio_wfd, view[:65536]) # type: ignore[attr-defined]
|
|
view = view[n:]
|
|
except (BrokenPipeError, OSError):
|
|
pass
|
|
finally:
|
|
try:
|
|
_os.close(proc._audio_wfd) # type: ignore[attr-defined]
|
|
except OSError:
|
|
pass
|
|
|
|
_threading.Thread(target=_pump, daemon=True).start()
|
|
|
|
seg_total = sum(s.frame_count for s in plan.segments)
|
|
seg_done = 0
|
|
chunk = 24
|
|
|
|
native_enc = open_enc(native_path, w, h)
|
|
upscale_enc = open_enc(upscaled_path, plan.upscale_width, plan.upscale_height) if do_upscale else None
|
|
|
|
def _enc_err(proc: subprocess.Popen) -> str:
|
|
try:
|
|
return proc.stderr.read().decode(errors="replace")[-500:]
|
|
except Exception: # noqa: BLE001
|
|
return "(stderr unavailable)"
|
|
|
|
try:
|
|
feed_audio(native_enc)
|
|
if upscale_enc:
|
|
feed_audio(upscale_enc)
|
|
|
|
f = 0
|
|
while f < frames:
|
|
n = min(chunk, frames - f)
|
|
raw = _synthesize_chunk(f, n, plan, w, h)
|
|
try:
|
|
native_enc.stdin.write(raw)
|
|
except BrokenPipeError as exc:
|
|
raise RuntimeError(f"native encoder died: {_enc_err(native_enc)}") from exc
|
|
if upscale_enc:
|
|
# RTX-branch stand-in: lanczos resize per frame to target dims
|
|
arr = np.frombuffer(raw, dtype=np.uint8).reshape(n, h, w, 3)
|
|
imgs = [Image.fromarray(fr).resize(
|
|
(plan.upscale_width, plan.upscale_height), Image.LANCZOS)
|
|
for fr in arr]
|
|
try:
|
|
upscale_enc.stdin.write(np.stack([np.asarray(im) for im in imgs]).tobytes())
|
|
except BrokenPipeError as exc:
|
|
raise RuntimeError(f"upscale encoder died: {_enc_err(upscale_enc)}") from exc
|
|
f += n
|
|
seg_done += n
|
|
pct = 15.0 + 60.0 * seg_done / seg_total
|
|
# per-stage sub-progress: emulate per-segment sampling steps
|
|
cur = next((s for s in plan.segments if s.start_frame <= f < s.end_frame), None)
|
|
stage("MoxieProject", pct,
|
|
f"mock: sampling frame {f}/{frames}"
|
|
+ (f" ({cur.prompt or 'untitled'})" if cur else ""),
|
|
{"frame": f, "frames": frames})
|
|
native_enc.stdin.close()
|
|
if upscale_enc:
|
|
upscale_enc.stdin.close()
|
|
rc = native_enc.wait(timeout=120)
|
|
if rc != 0:
|
|
raise RuntimeError(f"native encode failed: {native_enc.stderr.read().decode(errors='replace')[-400:]}")
|
|
if upscale_enc:
|
|
rc = upscale_enc.wait(timeout=180)
|
|
if rc != 0:
|
|
raise RuntimeError(f"upscale encode failed: {upscale_enc.stderr.read().decode(errors='replace')[-400:]}")
|
|
finally:
|
|
for proc in (native_enc, upscale_enc):
|
|
if proc and proc.poll() is None:
|
|
proc.kill()
|
|
|
|
stage("MoxieCombine", 78.0, "mock: combined images + audio + fps")
|
|
if do_upscale:
|
|
stage("RTXVideoSuperResolution", 88.0,
|
|
f"mock: upscale {w}x{h} -> {plan.upscale_width}x{plan.upscale_height} "
|
|
f"({plan.upscale_quality})")
|
|
|
|
stage("CreateVideo", 94.0, f"mock: encode {plan.codec}")
|
|
stage("SaveVideo", 99.0, "mock: saved outputs")
|
|
|
|
def describe(path: Path, ow: int, oh: int) -> dict:
|
|
return {
|
|
"filename": path.name,
|
|
"path": str(path),
|
|
"url": f"/api/view?filename={path.name}&type=output",
|
|
"width": ow, "height": oh,
|
|
"fps": plan.frame_rate,
|
|
"frames": frames,
|
|
"bytes": path.stat().st_size if path.exists() else 0,
|
|
}
|
|
|
|
result.outputs["native"] = describe(native_path, w, h)
|
|
if do_upscale:
|
|
result.outputs["upscaled"] = describe(
|
|
upscaled_path, plan.upscale_width, plan.upscale_height)
|
|
result.log(f"rendered {frames} frames in {time.time() - t0:.1f}s")
|
|
stage("SaveVideo", 100.0, "done", {"outputs": result.outputs})
|
|
return result
|