MMnode/scripts/test_bench_pipeline.py
Z User eefca88d35 feat: Moxie bench - standalone python/web harness for the fixed pipeline
- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base
  workflow, serial render jobs, WS progress mirroring ComfyUI's push
  events, /api/view + ComfyUI-compatible /view, /upload/image,
  /easy-media/* aliases for the ported editor
- dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic
  per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI
  bridge (substitutes the plan into base_workflow.json, converts to the
  /prompt API format, polls history, downloads outputs)
- timeline port: frontend/src/bench mounts the unmodified MultiTrack
  editor against a comfyAPI shim (fetchApi/toast/dialog/custom events);
  build.bench.ts emits bench/ui-dist served by the server
- packaging: PyInstaller onedir spec + Tauri2 sidecar docs
- tests: bench/tests contract suite + scripts/test_bench_pipeline.py
  e2e (API+WS+render); prebuilt bench UI committed
2026-09-28 01:42:58 +00:00

161 lines
6.5 KiB
Python

#!/usr/bin/env python3
"""Standalone bench verification: API + WebSocket + mock pipeline end-to-end.
No ComfyUI, no torch. Exercises the FastAPI app exactly like a sidecar host
would: health/config, workflow & plan endpoints, a full render over the mock
backend (ffmpeg), WebSocket progress events, and output serving.
Run: python3 scripts/test_bench_pipeline.py
"""
from __future__ import annotations
import json
import os
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
BENCH = ROOT / "bench"
sys.path.insert(0, str(BENCH))
FAILED = []
def check(name: str, cond: bool, detail: str = "") -> None:
print(f"[{'PASS' if cond else 'FAIL'}] {name}" + (f" - {detail}" if detail and not cond else ""))
if not cond:
FAILED.append(name)
def main() -> int:
# py_compile every bench module first
import py_compile
py_files = sorted(BENCH.rglob("*.py"))
for f in py_files:
py_compile.compile(str(f), doraise=True)
check("py_compile all bench modules", True, f"({len(py_files)} files)")
from fastapi.testclient import TestClient
os.environ.setdefault("MOXIE_BENCH_WORK", str(BENCH / "work"))
from app.config import BenchConfig
from app.server import create_app
cfg = BenchConfig(backend="mock")
cfg.repo_root = ROOT
app = create_app(cfg)
with TestClient(app) as client: # context manager runs lifespan
r = client.get("/api/health")
check("GET /api/health", r.status_code == 200 and r.json()["ok"] is True)
check("health reports mock backend", r.json()["backend"] == "mock")
r = client.get("/api/config")
check("GET /api/config", r.status_code == 200 and "backends" in r.json())
r = client.get("/api/workflow")
body = r.json()
check("GET /api/workflow summary", r.status_code == 200
and body["summary"]["editor_resolution"] == {"width": 1920, "height": 1080})
track_data = json.loads(body["track_data"])
check("workflow carries editor track_data", track_data.get("total_length") == 240)
r = client.get("/api/plan")
plan = r.json()
check("GET /api/plan", r.status_code == 200
and [s["prompt"] for s in plan["segments"]] == ["scene1", "scene2"])
check("plan duration", plan["duration_seconds"] == 10.0, str(plan["duration_seconds"]))
# --- WebSocket + render e2e (small dims to keep it fast) ---
events = []
with client.websocket_connect("/api/ws") as ws:
# drain the initial connection by starting a render in parallel
render_payload = {
"track_data": body["track_data"],
"overrides": {
"width": 256, "height": 144, "frame_rate": 8,
"steps": 4, "seed": 42, "project_name": "MM-bench-e2e",
},
}
r = client.post("/api/render", json=render_payload)
check("POST /api/render accepted", r.status_code == 200, r.text[:200])
job_id = r.json()["job_id"]
deadline = time.time() + 120
got_success = False
while time.time() < deadline and not got_success:
msg = ws.receive_json()
events.append(msg)
if msg.get("job_id") != job_id:
continue
if msg.get("type") == "job_progress":
check("progress within pipeline stages", True) if len(events) == 2 else None
if msg.get("type") == "execution_success":
got_success = True
if msg.get("type") == "execution_error":
break
check("render completed over WebSocket", got_success,
f"events={ [e.get('type') for e in events][-6:] }")
r = client.get(f"/api/jobs/{job_id}")
job = r.json()
check("job result stored", job["status"] == "done" and job["result"] is not None)
outputs = (job["result"] or {}).get("outputs", {})
check("both branches produced", set(outputs) == {"native", "upscaled"}, str(list(outputs)))
for branch, out in outputs.items():
path = Path(out["path"])
check(f"{branch} file exists", path.is_file() and path.stat().st_size > 1000)
expected = {"native": (256, 144), "upscaled": (1920, 1080)}[branch]
check(f"{branch} served over /api/view", client.get(out["url"]).status_code == 200)
# output dims via ffprobe
import subprocess
probe = subprocess.run(
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams",
outputs["upscaled"]["path"]],
capture_output=True, text=True)
streams = json.loads(probe.stdout).get("streams") or []
vs = next((s for s in streams if s.get("codec_type") == "video"), {})
check("upscaled dims match RTX target",
(vs.get("width"), vs.get("height")) == (1920, 1080),
f"{vs.get('width')}x{vs.get('height')}")
vs_n = json.loads(subprocess.run(
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams",
outputs["native"]["path"]], capture_output=True, text=True).stdout
).get("streams", [{}])
vsn = next((s for s in vs_n if s.get("codec_type") == "video"), {})
check("native dims match editor canvas",
(vsn.get("width"), vsn.get("height")) == (256, 144),
f"{vsn.get('width')}x{vsn.get('height')}")
# progress stage ordering mirrors the pipeline
stages_seen = [e.get("stage") for e in events if e.get("type") == "job_progress"]
check("progress touches sampling stage", "MoxieProject" in stages_seen)
check("progress touches upscale stage", "RTXVideoSuperResolution" in stages_seen)
# /api/easy-media mirrors answer
r = client.get("/api/easy-media/media/list")
check("easy-media media/list mirror", r.status_code == 200 and "items" in r.json())
r = client.get("/api/easy-media/prompt/system-prompts")
check("easy-media system-prompts mirror", r.status_code == 200)
# static UI mount responds (built or explicit hint)
r = client.get("/")
check("root serves UI or build hint", r.status_code in (200, 404)
or "not built" in r.text)
print()
if FAILED:
print(f"FAILED: {len(FAILED)} -> {FAILED}")
return 1
print("ALL CHECKS PASSED")
return 0
if __name__ == "__main__":
raise SystemExit(main())