MMnode/bench/app/backends/mock.py
Z User eefca88d35 feat: Moxie bench - standalone python/web harness for the fixed pipeline
- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base
  workflow, serial render jobs, WS progress mirroring ComfyUI's push
  events, /api/view + ComfyUI-compatible /view, /upload/image,
  /easy-media/* aliases for the ported editor
- dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic
  per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI
  bridge (substitutes the plan into base_workflow.json, converts to the
  /prompt API format, polls history, downloads outputs)
- timeline port: frontend/src/bench mounts the unmodified MultiTrack
  editor against a comfyAPI shim (fetchApi/toast/dialog/custom events);
  build.bench.ts emits bench/ui-dist served by the server
- packaging: PyInstaller onedir spec + Tauri2 sidecar docs
- tests: bench/tests contract suite + scripts/test_bench_pipeline.py
  e2e (API+WS+render); prebuilt bench UI committed
2026-09-28 01:42:58 +00:00

325 lines
12 KiB
Python

"""Self-contained mock backend.
Renders the fixed pipeline without torch/ComfyUI: deterministic synthetic
frames derived from (prompt, seed, frame index), a sine audio bed, PIL
lanczos "upscale" for the RTX branch, and ffmpeg h264 encode. Exercises the
exact same stage/progress contract as the ComfyUI bridge backend so the UI
and the API cannot tell the difference.
"""
from __future__ import annotations
import hashlib
import math
import shutil
import subprocess
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable
from ..tracks import RenderPlan, Segment
ProgressCb = Callable[[str, float, str, dict | None], None]
_VIDEO_CODECS = ("h264", "libx264", "mpeg4")
@dataclass
class RenderResult:
outputs: dict[str, dict] = field(default_factory=dict)
console: list[str] = field(default_factory=list)
def log(self, line: str) -> None:
self.console.append(line)
def _pick_encoder(codec: str) -> str:
probe = subprocess.run(
["ffmpeg", "-hide_banner", "-encoders"],
capture_output=True, text=True, timeout=30,
)
names = probe.stdout.lower()
if codec == "h264" and "libx264" in names:
return "libx264"
for cand in _VIDEO_CODECS:
if cand in names:
return cand
return "mpeg4"
def _audio_encoder() -> str:
probe = subprocess.run(
["ffmpeg", "-hide_banner", "-encoders"],
capture_output=True, text=True, timeout=30,
)
return "aac" if " aac" in probe.stdout.lower() else "libmp3lame"
def _segment_hue(segment: Segment, index: int) -> float:
digest = hashlib.sha256(f"{segment.prompt}|{index}".encode()).digest()
return digest[0] * 360.0 / 255.0
def _segment_freq(segment: Segment, index: int) -> float:
digest = hashlib.sha256(f"{segment.prompt}|{index}".encode()).digest()
return 280.0 + digest[1] % 5 * 55.0 # 280..500 Hz
def _seed_noise(seed: int, frame: int) -> float:
digest = hashlib.sha256(f"{seed}:{frame}".encode()).digest()
return digest[0] / 255.0 - 0.5
def _synthesize_chunk(
chunk_start: int,
frames: int,
plan: RenderPlan,
w: int,
h: int,
) -> bytes:
"""Deterministic RGB24 bytes for frames [chunk_start, chunk_start+frames)."""
import numpy as np
out = bytearray()
yy, xx = np.mgrid[0:h, 0:w].astype(np.float32)
xn = xx / max(1, w - 1)
yn = yy / max(1, h - 1)
seg_by_frame = _frame_to_segment_map(plan)
for f in range(chunk_start, chunk_start + frames):
seg, seg_index = seg_by_frame.get(f, (None, 0))
hue = _segment_hue(seg, seg_index) if seg else 210.0
t = f / max(1, plan.frame_rate)
# HSV-ish deterministic palette: hue rotates per segment prompt,
# luminance carries a moving wave so motion is visible.
base = np.zeros((h, w, 3), dtype=np.float32)
h1 = hue % 360.0
rgb = _hsv_to_rgb(h1 / 360.0, 0.55, 0.6)
wave = 0.25 * np.sin(6.0 * math.pi * (xn + 0.35 * t)) + 0.15 * np.sin(
4.0 * math.pi * (yn - 0.25 * t)
)
lum = 0.45 + wave + 0.06 * _seed_noise(plan.seed, f)
base[..., 0] = rgb[0] * lum
base[..., 1] = rgb[1] * lum
base[..., 2] = rgb[2] * lum
# progress bar strip at the bottom: frame position inside the timeline
pos = f / max(1, plan.total_length or 1)
bar_y = int(h * 0.95)
base[bar_y:, : int(w * pos), 1] = 0.95
out += (
np.clip(base * 255.0, 0, 255).astype(np.uint8).tobytes()
)
return bytes(out)
def _frame_to_segment_map(plan: RenderPlan) -> dict[int, tuple[Segment, int]]:
mapping: dict[int, tuple[Segment, int]] = {}
for idx, seg in enumerate(plan.segments):
for f in range(seg.start_frame, seg.end_frame):
mapping[f] = (seg, idx)
return mapping
def _hsv_to_rgb(h: float, s: float, v: float) -> tuple[float, float, float]:
import colorsys
r, g, b = colorsys.hsv_to_rgb(h % 1.0, s, v)
return r, g, b
def _s16le_audio(plan: RenderPlan, sample_rate: int = 44100) -> bytes:
"""Stereo s16le sine bed; per-segment tone, silence for muted segments."""
import numpy as np
total = int(sample_rate * plan.duration_seconds)
if total <= 0:
return b""
wave = np.zeros(total, dtype=np.float32)
sr = sample_rate
for idx, seg in enumerate(plan.segments):
if seg.muted or not seg.prompt:
continue
a, b = seg.start_frame, seg.end_frame
i0 = int(a / max(1, plan.frame_rate) * sr)
i1 = min(total, int(b / max(1, plan.frame_rate) * sr))
if i1 <= i0:
continue
t = np.arange(i1 - i0, dtype=np.float32) / sr
freq = _segment_freq(seg, idx)
gain = 0.12 * (10 ** (seg.volume_db / 20.0))
wave[i0:i1] += gain * np.sin(2 * math.pi * freq * t)
stereo = np.repeat(np.clip(wave, -1, 1)[:, None], 2, axis=1)
return (stereo * 32767.0).astype("<i2").tobytes()
def render(plan: RenderPlan, work_dir: Path, progress: ProgressCb) -> RenderResult:
"""Run the mock pipeline. Returns native + upscaled output descriptors."""
import numpy as np
from PIL import Image
result = RenderResult()
t0 = time.time()
work_dir.mkdir(parents=True, exist_ok=True)
if shutil.which("ffmpeg") is None:
raise RuntimeError("ffmpeg not found on PATH - the mock backend needs it")
w, h = plan.width, plan.height
frames = plan.total_length or sum(s.frame_count for s in plan.segments)
if frames <= 0:
raise RuntimeError("timeline has no frames to render")
def stage(name: str, pct: float, msg: str, data: dict | None = None) -> None:
progress(name, pct, msg, data)
# stages 1-3: loader / lora / preview (contract parity with the real chain)
stage("MoxieMultimediaLoader", 4.0, "mock: loading Moxie-Multimedia model set")
stage("LoraLoaderModelOnly", 9.0, f"mock: LoRA {plan.lora or 'none'} @ {plan.lora_strength}")
stage("MoxiePreviewOverride", 12.0, "mock: preview override active")
stage("easy multiTrackEditor", 15.0,
f"mock: timeline {w}x{h} @ {plan.frame_rate}fps, "
f"{len(plan.segments)} task segment(s)")
encoder = _pick_encoder(plan.codec)
aenc = _audio_encoder()
result.log(f"encoder: {encoder}, audio: {aenc}")
native_path = work_dir / "native.mp4"
upscaled_path = work_dir / "upscaled.mp4"
do_upscale = bool(plan.upscale_width and plan.upscale_height)
audio_bytes = _s16le_audio(plan)
def open_enc(out_path: Path, ow: int, oh: int) -> subprocess.Popen:
import os as _os
cmd = [
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-f", "rawvideo", "-pix_fmt", "rgb24",
"-s", f"{ow}x{oh}", "-r", str(plan.frame_rate), "-i", "pipe:0",
"-f", "s16le", "-ar", "44100", "-ac", "2",
]
r, w = _os.pipe()
# give ffmpeg the READ end (r); parent writes audio into w
cmd += ["-i", f"/dev/fd/{r}", "-map", "0:v", "-map", "1:a",
"-c:v", encoder, "-pix_fmt", "yuv420p",
"-movflags", "+faststart",
"-c:a", aenc, "-b:a", "128k", "-shortest",
str(out_path)]
proc = subprocess.Popen(
cmd, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE, pass_fds=(r,),
)
proc._audio_wfd = w # type: ignore[attr-defined]
_os.close(r)
return proc
def feed_audio(proc: subprocess.Popen) -> None:
# Audio may exceed the 64KB pipe buffer; pump it from a thread so the
# video stdin feed never deadlocks against ffmpeg's startup probing.
import os as _os
import threading as _threading
def _pump() -> None:
try:
view = memoryview(audio_bytes)
while view:
n = _os.write(proc._audio_wfd, view[:65536]) # type: ignore[attr-defined]
view = view[n:]
except (BrokenPipeError, OSError):
pass
finally:
try:
_os.close(proc._audio_wfd) # type: ignore[attr-defined]
except OSError:
pass
_threading.Thread(target=_pump, daemon=True).start()
seg_total = sum(s.frame_count for s in plan.segments)
seg_done = 0
chunk = 24
native_enc = open_enc(native_path, w, h)
upscale_enc = open_enc(upscaled_path, plan.upscale_width, plan.upscale_height) if do_upscale else None
def _enc_err(proc: subprocess.Popen) -> str:
try:
return proc.stderr.read().decode(errors="replace")[-500:]
except Exception: # noqa: BLE001
return "(stderr unavailable)"
try:
feed_audio(native_enc)
if upscale_enc:
feed_audio(upscale_enc)
f = 0
while f < frames:
n = min(chunk, frames - f)
raw = _synthesize_chunk(f, n, plan, w, h)
try:
native_enc.stdin.write(raw)
except BrokenPipeError as exc:
raise RuntimeError(f"native encoder died: {_enc_err(native_enc)}") from exc
if upscale_enc:
# RTX-branch stand-in: lanczos resize per frame to target dims
arr = np.frombuffer(raw, dtype=np.uint8).reshape(n, h, w, 3)
imgs = [Image.fromarray(fr).resize(
(plan.upscale_width, plan.upscale_height), Image.LANCZOS)
for fr in arr]
try:
upscale_enc.stdin.write(np.stack([np.asarray(im) for im in imgs]).tobytes())
except BrokenPipeError as exc:
raise RuntimeError(f"upscale encoder died: {_enc_err(upscale_enc)}") from exc
f += n
seg_done += n
pct = 15.0 + 60.0 * seg_done / seg_total
# per-stage sub-progress: emulate per-segment sampling steps
cur = next((s for s in plan.segments if s.start_frame <= f < s.end_frame), None)
stage("MoxieProject", pct,
f"mock: sampling frame {f}/{frames}"
+ (f" ({cur.prompt or 'untitled'})" if cur else ""),
{"frame": f, "frames": frames})
native_enc.stdin.close()
if upscale_enc:
upscale_enc.stdin.close()
rc = native_enc.wait(timeout=120)
if rc != 0:
raise RuntimeError(f"native encode failed: {native_enc.stderr.read().decode(errors='replace')[-400:]}")
if upscale_enc:
rc = upscale_enc.wait(timeout=180)
if rc != 0:
raise RuntimeError(f"upscale encode failed: {upscale_enc.stderr.read().decode(errors='replace')[-400:]}")
finally:
for proc in (native_enc, upscale_enc):
if proc and proc.poll() is None:
proc.kill()
stage("MoxieCombine", 78.0, "mock: combined images + audio + fps")
if do_upscale:
stage("RTXVideoSuperResolution", 88.0,
f"mock: upscale {w}x{h} -> {plan.upscale_width}x{plan.upscale_height} "
f"({plan.upscale_quality})")
stage("CreateVideo", 94.0, f"mock: encode {plan.codec}")
stage("SaveVideo", 99.0, "mock: saved outputs")
def describe(path: Path, ow: int, oh: int) -> dict:
return {
"filename": path.name,
"path": str(path),
"url": f"/api/view?filename={path.name}&type=output",
"width": ow, "height": oh,
"fps": plan.frame_rate,
"frames": frames,
"bytes": path.stat().st_size if path.exists() else 0,
}
result.outputs["native"] = describe(native_path, w, h)
if do_upscale:
result.outputs["upscaled"] = describe(
upscaled_path, plan.upscale_width, plan.upscale_height)
result.log(f"rendered {frames} frames in {time.time() - t0:.1f}s")
stage("SaveVideo", 100.0, "done", {"outputs": result.outputs})
return result