- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base workflow, serial render jobs, WS progress mirroring ComfyUI's push events, /api/view + ComfyUI-compatible /view, /upload/image, /easy-media/* aliases for the ported editor - dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI bridge (substitutes the plan into base_workflow.json, converts to the /prompt API format, polls history, downloads outputs) - timeline port: frontend/src/bench mounts the unmodified MultiTrack editor against a comfyAPI shim (fetchApi/toast/dialog/custom events); build.bench.ts emits bench/ui-dist served by the server - packaging: PyInstaller onedir spec + Tauri2 sidecar docs - tests: bench/tests contract suite + scripts/test_bench_pipeline.py e2e (API+WS+render); prebuilt bench UI committed
161 lines
6.5 KiB
Python
161 lines
6.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Standalone bench verification: API + WebSocket + mock pipeline end-to-end.
|
|
|
|
No ComfyUI, no torch. Exercises the FastAPI app exactly like a sidecar host
|
|
would: health/config, workflow & plan endpoints, a full render over the mock
|
|
backend (ffmpeg), WebSocket progress events, and output serving.
|
|
|
|
Run: python3 scripts/test_bench_pipeline.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
BENCH = ROOT / "bench"
|
|
sys.path.insert(0, str(BENCH))
|
|
|
|
FAILED = []
|
|
|
|
|
|
def check(name: str, cond: bool, detail: str = "") -> None:
|
|
print(f"[{'PASS' if cond else 'FAIL'}] {name}" + (f" - {detail}" if detail and not cond else ""))
|
|
if not cond:
|
|
FAILED.append(name)
|
|
|
|
|
|
def main() -> int:
|
|
# py_compile every bench module first
|
|
import py_compile
|
|
|
|
py_files = sorted(BENCH.rglob("*.py"))
|
|
for f in py_files:
|
|
py_compile.compile(str(f), doraise=True)
|
|
check("py_compile all bench modules", True, f"({len(py_files)} files)")
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
os.environ.setdefault("MOXIE_BENCH_WORK", str(BENCH / "work"))
|
|
from app.config import BenchConfig
|
|
from app.server import create_app
|
|
|
|
cfg = BenchConfig(backend="mock")
|
|
cfg.repo_root = ROOT
|
|
app = create_app(cfg)
|
|
|
|
with TestClient(app) as client: # context manager runs lifespan
|
|
r = client.get("/api/health")
|
|
check("GET /api/health", r.status_code == 200 and r.json()["ok"] is True)
|
|
check("health reports mock backend", r.json()["backend"] == "mock")
|
|
|
|
r = client.get("/api/config")
|
|
check("GET /api/config", r.status_code == 200 and "backends" in r.json())
|
|
|
|
r = client.get("/api/workflow")
|
|
body = r.json()
|
|
check("GET /api/workflow summary", r.status_code == 200
|
|
and body["summary"]["editor_resolution"] == {"width": 1920, "height": 1080})
|
|
track_data = json.loads(body["track_data"])
|
|
check("workflow carries editor track_data", track_data.get("total_length") == 240)
|
|
|
|
r = client.get("/api/plan")
|
|
plan = r.json()
|
|
check("GET /api/plan", r.status_code == 200
|
|
and [s["prompt"] for s in plan["segments"]] == ["scene1", "scene2"])
|
|
check("plan duration", plan["duration_seconds"] == 10.0, str(plan["duration_seconds"]))
|
|
|
|
# --- WebSocket + render e2e (small dims to keep it fast) ---
|
|
events = []
|
|
with client.websocket_connect("/api/ws") as ws:
|
|
# drain the initial connection by starting a render in parallel
|
|
render_payload = {
|
|
"track_data": body["track_data"],
|
|
"overrides": {
|
|
"width": 256, "height": 144, "frame_rate": 8,
|
|
"steps": 4, "seed": 42, "project_name": "MM-bench-e2e",
|
|
},
|
|
}
|
|
r = client.post("/api/render", json=render_payload)
|
|
check("POST /api/render accepted", r.status_code == 200, r.text[:200])
|
|
job_id = r.json()["job_id"]
|
|
|
|
deadline = time.time() + 120
|
|
got_success = False
|
|
while time.time() < deadline and not got_success:
|
|
msg = ws.receive_json()
|
|
events.append(msg)
|
|
if msg.get("job_id") != job_id:
|
|
continue
|
|
if msg.get("type") == "job_progress":
|
|
check("progress within pipeline stages", True) if len(events) == 2 else None
|
|
if msg.get("type") == "execution_success":
|
|
got_success = True
|
|
if msg.get("type") == "execution_error":
|
|
break
|
|
|
|
check("render completed over WebSocket", got_success,
|
|
f"events={ [e.get('type') for e in events][-6:] }")
|
|
|
|
r = client.get(f"/api/jobs/{job_id}")
|
|
job = r.json()
|
|
check("job result stored", job["status"] == "done" and job["result"] is not None)
|
|
outputs = (job["result"] or {}).get("outputs", {})
|
|
check("both branches produced", set(outputs) == {"native", "upscaled"}, str(list(outputs)))
|
|
for branch, out in outputs.items():
|
|
path = Path(out["path"])
|
|
check(f"{branch} file exists", path.is_file() and path.stat().st_size > 1000)
|
|
|
|
expected = {"native": (256, 144), "upscaled": (1920, 1080)}[branch]
|
|
check(f"{branch} served over /api/view", client.get(out["url"]).status_code == 200)
|
|
|
|
# output dims via ffprobe
|
|
import subprocess
|
|
probe = subprocess.run(
|
|
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams",
|
|
outputs["upscaled"]["path"]],
|
|
capture_output=True, text=True)
|
|
streams = json.loads(probe.stdout).get("streams") or []
|
|
vs = next((s for s in streams if s.get("codec_type") == "video"), {})
|
|
check("upscaled dims match RTX target",
|
|
(vs.get("width"), vs.get("height")) == (1920, 1080),
|
|
f"{vs.get('width')}x{vs.get('height')}")
|
|
vs_n = json.loads(subprocess.run(
|
|
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams",
|
|
outputs["native"]["path"]], capture_output=True, text=True).stdout
|
|
).get("streams", [{}])
|
|
vsn = next((s for s in vs_n if s.get("codec_type") == "video"), {})
|
|
check("native dims match editor canvas",
|
|
(vsn.get("width"), vsn.get("height")) == (256, 144),
|
|
f"{vsn.get('width')}x{vsn.get('height')}")
|
|
|
|
# progress stage ordering mirrors the pipeline
|
|
stages_seen = [e.get("stage") for e in events if e.get("type") == "job_progress"]
|
|
check("progress touches sampling stage", "MoxieProject" in stages_seen)
|
|
check("progress touches upscale stage", "RTXVideoSuperResolution" in stages_seen)
|
|
|
|
# /api/easy-media mirrors answer
|
|
r = client.get("/api/easy-media/media/list")
|
|
check("easy-media media/list mirror", r.status_code == 200 and "items" in r.json())
|
|
r = client.get("/api/easy-media/prompt/system-prompts")
|
|
check("easy-media system-prompts mirror", r.status_code == 200)
|
|
|
|
# static UI mount responds (built or explicit hint)
|
|
r = client.get("/")
|
|
check("root serves UI or build hint", r.status_code in (200, 404)
|
|
or "not built" in r.text)
|
|
|
|
print()
|
|
if FAILED:
|
|
print(f"FAILED: {len(FAILED)} -> {FAILED}")
|
|
return 1
|
|
print("ALL CHECKS PASSED")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|