MMnode/bench/tests/test_bench_contract.py
Z User eefca88d35 feat: Moxie bench - standalone python/web harness for the fixed pipeline
- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base
  workflow, serial render jobs, WS progress mirroring ComfyUI's push
  events, /api/view + ComfyUI-compatible /view, /upload/image,
  /easy-media/* aliases for the ported editor
- dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic
  per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI
  bridge (substitutes the plan into base_workflow.json, converts to the
  /prompt API format, polls history, downloads outputs)
- timeline port: frontend/src/bench mounts the unmodified MultiTrack
  editor against a comfyAPI shim (fetchApi/toast/dialog/custom events);
  build.bench.ts emits bench/ui-dist served by the server
- packaging: PyInstaller onedir spec + Tauri2 sidecar docs
- tests: bench/tests contract suite + scripts/test_bench_pipeline.py
  e2e (API+WS+render); prebuilt bench UI committed
2026-09-28 01:42:58 +00:00

197 lines
7.0 KiB
Python

"""Bench contract tests: no ComfyUI, no torch required.
Run: python3 -m pytest bench/tests -q
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
BENCH_DIR = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(BENCH_DIR))
from app.tracks import TrackDataError, build_render_plan # noqa: E402
from app.workflow import ( # noqa: E402
EDITOR_TYPE,
WorkflowError,
describe,
editor_track_data,
load_base_workflow,
parse_workflow_dict,
)
BASE = load_base_workflow(BENCH_DIR / "base_workflow.json")
# ---------------------------------------------------------------------------
# workflow parsing
# ---------------------------------------------------------------------------
def test_all_nine_stage_types_present():
types = set(BASE.stages)
assert {
"MoxieMultimediaLoader", "LoraLoaderModelOnly", "MoxiePreviewOverride",
EDITOR_TYPE, "MoxieProject", "MoxieCombine", "RTXVideoSuperResolution",
"CreateVideo",
} <= types
assert "CreateVideo#2" in types
def test_save_nodes_classified_by_branch():
assert set(BASE.save_nodes) == {"native", "upscaled"}
# native save is fed by CreateVideo #45, upscaled by #68
native_src = BASE.save_nodes["native"].link_inputs["video"][0]
upscaled_src = BASE.save_nodes["upscaled"].link_inputs["video"][0]
assert native_src == BASE.stages["CreateVideo"].node_id
assert upscaled_src == BASE.stages["CreateVideo#2"].node_id
def test_describe_summary():
d = describe(BASE)
assert d["editor_resolution"] == {"width": 1920, "height": 1080}
assert d["frame_rate"] == 24
assert d["steps"] == 4
assert d["seed"] == 803100406447983
assert d["project_name"] == "MM-girlwaving"
assert d["upscale"]["target"] == {"width": 1920, "height": 1080}
assert d["upscale"]["quality"] == "ULTRA"
assert d["lora"].endswith(".safetensors")
def test_editor_track_data_roundtrip():
td = editor_track_data(BASE)
assert td["frame_rate"] == 24
assert td["total_length"] == 240
task_tracks = [t for t in td["tracks"] if t["type"] == "task"]
assert len(task_tracks) == 1
assert len(task_tracks[0]["segments"]) == 2
def test_missing_stage_rejected():
raw = json.loads((BENCH_DIR / "base_workflow.json").read_text())
raw["nodes"] = [n for n in raw["nodes"] if n["type"] != "MoxieProject"]
with pytest.raises(WorkflowError):
parse_workflow_dict(raw)
# ---------------------------------------------------------------------------
# track_data -> RenderPlan
# ---------------------------------------------------------------------------
def _plan(**kw):
defaults = dict(width=1920, height=1080, frame_rate=24, steps=4, seed=7,
project_name="MM-girlwaving", upscale_width=1920,
upscale_height=1080)
defaults.update(kw)
return build_render_plan(editor_track_data(BASE), **defaults)
def test_plan_segments_ordered_and_bounded():
plan = _plan()
assert [(s.prompt, s.start_frame, s.end_frame) for s in plan.segments] == [
("scene1", 0, 120), ("scene2", 120, 240)]
assert plan.duration_seconds == pytest.approx(10.0)
def test_plan_honors_overrides():
plan = _plan(width=1088, height=1920, steps=8, seed=99)
assert (plan.width, plan.height) == (1088, 1920)
assert plan.steps == 8 and plan.seed == 99
def test_continuity_mode_carried_through():
plan = _plan()
modes = {s.prompt: s.continuity_mode for s in plan.segments}
assert modes == {"scene1": "shot", "scene2": "context"}
def test_invalid_track_data_raises():
with pytest.raises(TrackDataError):
build_render_plan("not json", width=1, height=1)
with pytest.raises(TrackDataError):
build_render_plan({"nope": 1}, width=1, height=1)
# ---------------------------------------------------------------------------
# /prompt conversion (comfy bridge)
# ---------------------------------------------------------------------------
def test_prompt_conversion_shape():
from app.backends.comfy import to_prompt_format
prompt = to_prompt_format(BASE.raw)
assert set(prompt) == {str(n["id"]) for n in BASE.raw["nodes"]}
loader = prompt["52"]
assert loader["class_type"] == "MoxieMultimediaLoader"
assert loader["inputs"]["diffusion_model"] == "Moxie-Multimedia.safetensors"
project = prompt["57"]
assert project["inputs"]["tracks_info"] == ["81", 0]
assert project["inputs"]["model"] == ["48", 0]
assert project["inputs"]["clip"] == ["52", 1]
assert "randomize" not in project["inputs"] # UI-only control widget
def test_prompt_conversion_wired_widget_wins():
from app.backends.comfy import to_prompt_format
prompt = to_prompt_format(BASE.raw)
# CreateVideo #45: fps widget is wired to MoxieCombine slot 2
create = prompt["45"]
assert create["inputs"]["fps"] == ["63", 2]
assert create["inputs"]["codec"] == "h264"
def test_substitution_updates_all_parameters():
from app.backends.comfy import substitute
plan = build_render_plan(
editor_track_data(BASE), width=720, height=1280, frame_rate=16,
steps=6, seed=1234, project_name="MM-bench", lora="MM3step.safetensors",
lora_strength=0.8, upscale_width=1440, upscale_height=2560,
)
raw = substitute(BASE, plan)
nodes = {n["id"]: n for n in raw["nodes"]}
editor = nodes[81]["widgets_values_named"]
assert json.loads(editor["track_data"])["frame_rate"] == 16
assert editor["resolution"] == "720 x 1280 (9:16)"
proj = nodes[57]["widgets_values_named"]
assert proj["project_name"] == "MM-bench" and proj["steps"] == 6
assert proj["seed"] == 1234
lora = nodes[66]["widgets_values_named"]
assert lora["lora_name"] == "MM3step.safetensors" and lora["strength_model"] == 0.8
rtx = nodes[67]["widgets_values_named"]
assert rtx["resize_type.width"] == 1440 and rtx["resize_type.height"] == 2560
combine = nodes[63]["widgets_values_named"]
pd = json.loads(combine["project_data"])
assert pd["width"] == 720 and pd["height"] == 1280 and pd["frame_rate"] == 16
saves = {n["id"]: n for n in raw["nodes"] if n["type"] == "SaveVideo"}
assert saves[46]["widgets_values_named"]["filename_prefix"] == "moxie-bench/native"
assert saves[69]["widgets_values_named"]["filename_prefix"] == "moxie-bench/upscaled"
def test_substitution_custom_resolution_uses_dict():
from app.backends.comfy import substitute
plan = build_render_plan(editor_track_data(BASE), width=998, height=742)
raw = substitute(BASE, plan)
editor = next(n for n in raw["nodes"] if n["id"] == 81)
assert editor["widgets_values_named"]["resolution"] == {"width": 998, "height": 742}
# ---------------------------------------------------------------------------
# mock backend end-to-end (ffmpeg required)
# ---------------------------------------------------------------------------
@pytest.mark.skipif(True, reason="e2e covered by scripts/test_bench_pipeline.py; keep pytest fast")
def test_mock_render_placeholder():
pass