- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base workflow, serial render jobs, WS progress mirroring ComfyUI's push events, /api/view + ComfyUI-compatible /view, /upload/image, /easy-media/* aliases for the ported editor - dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI bridge (substitutes the plan into base_workflow.json, converts to the /prompt API format, polls history, downloads outputs) - timeline port: frontend/src/bench mounts the unmodified MultiTrack editor against a comfyAPI shim (fetchApi/toast/dialog/custom events); build.bench.ts emits bench/ui-dist served by the server - packaging: PyInstaller onedir spec + Tauri2 sidecar docs - tests: bench/tests contract suite + scripts/test_bench_pipeline.py e2e (API+WS+render); prebuilt bench UI committed
197 lines
7.0 KiB
Python
197 lines
7.0 KiB
Python
"""Bench contract tests: no ComfyUI, no torch required.
|
|
|
|
Run: python3 -m pytest bench/tests -q
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
BENCH_DIR = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(BENCH_DIR))
|
|
|
|
from app.tracks import TrackDataError, build_render_plan # noqa: E402
|
|
from app.workflow import ( # noqa: E402
|
|
EDITOR_TYPE,
|
|
WorkflowError,
|
|
describe,
|
|
editor_track_data,
|
|
load_base_workflow,
|
|
parse_workflow_dict,
|
|
)
|
|
|
|
BASE = load_base_workflow(BENCH_DIR / "base_workflow.json")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# workflow parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_all_nine_stage_types_present():
|
|
types = set(BASE.stages)
|
|
assert {
|
|
"MoxieMultimediaLoader", "LoraLoaderModelOnly", "MoxiePreviewOverride",
|
|
EDITOR_TYPE, "MoxieProject", "MoxieCombine", "RTXVideoSuperResolution",
|
|
"CreateVideo",
|
|
} <= types
|
|
assert "CreateVideo#2" in types
|
|
|
|
|
|
def test_save_nodes_classified_by_branch():
|
|
assert set(BASE.save_nodes) == {"native", "upscaled"}
|
|
# native save is fed by CreateVideo #45, upscaled by #68
|
|
native_src = BASE.save_nodes["native"].link_inputs["video"][0]
|
|
upscaled_src = BASE.save_nodes["upscaled"].link_inputs["video"][0]
|
|
assert native_src == BASE.stages["CreateVideo"].node_id
|
|
assert upscaled_src == BASE.stages["CreateVideo#2"].node_id
|
|
|
|
|
|
def test_describe_summary():
|
|
d = describe(BASE)
|
|
assert d["editor_resolution"] == {"width": 1920, "height": 1080}
|
|
assert d["frame_rate"] == 24
|
|
assert d["steps"] == 4
|
|
assert d["seed"] == 803100406447983
|
|
assert d["project_name"] == "MM-girlwaving"
|
|
assert d["upscale"]["target"] == {"width": 1920, "height": 1080}
|
|
assert d["upscale"]["quality"] == "ULTRA"
|
|
assert d["lora"].endswith(".safetensors")
|
|
|
|
|
|
def test_editor_track_data_roundtrip():
|
|
td = editor_track_data(BASE)
|
|
assert td["frame_rate"] == 24
|
|
assert td["total_length"] == 240
|
|
task_tracks = [t for t in td["tracks"] if t["type"] == "task"]
|
|
assert len(task_tracks) == 1
|
|
assert len(task_tracks[0]["segments"]) == 2
|
|
|
|
|
|
def test_missing_stage_rejected():
|
|
raw = json.loads((BENCH_DIR / "base_workflow.json").read_text())
|
|
raw["nodes"] = [n for n in raw["nodes"] if n["type"] != "MoxieProject"]
|
|
with pytest.raises(WorkflowError):
|
|
parse_workflow_dict(raw)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# track_data -> RenderPlan
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _plan(**kw):
|
|
defaults = dict(width=1920, height=1080, frame_rate=24, steps=4, seed=7,
|
|
project_name="MM-girlwaving", upscale_width=1920,
|
|
upscale_height=1080)
|
|
defaults.update(kw)
|
|
return build_render_plan(editor_track_data(BASE), **defaults)
|
|
|
|
|
|
def test_plan_segments_ordered_and_bounded():
|
|
plan = _plan()
|
|
assert [(s.prompt, s.start_frame, s.end_frame) for s in plan.segments] == [
|
|
("scene1", 0, 120), ("scene2", 120, 240)]
|
|
assert plan.duration_seconds == pytest.approx(10.0)
|
|
|
|
|
|
def test_plan_honors_overrides():
|
|
plan = _plan(width=1088, height=1920, steps=8, seed=99)
|
|
assert (plan.width, plan.height) == (1088, 1920)
|
|
assert plan.steps == 8 and plan.seed == 99
|
|
|
|
|
|
def test_continuity_mode_carried_through():
|
|
plan = _plan()
|
|
modes = {s.prompt: s.continuity_mode for s in plan.segments}
|
|
assert modes == {"scene1": "shot", "scene2": "context"}
|
|
|
|
|
|
def test_invalid_track_data_raises():
|
|
with pytest.raises(TrackDataError):
|
|
build_render_plan("not json", width=1, height=1)
|
|
with pytest.raises(TrackDataError):
|
|
build_render_plan({"nope": 1}, width=1, height=1)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# /prompt conversion (comfy bridge)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_prompt_conversion_shape():
|
|
from app.backends.comfy import to_prompt_format
|
|
|
|
prompt = to_prompt_format(BASE.raw)
|
|
assert set(prompt) == {str(n["id"]) for n in BASE.raw["nodes"]}
|
|
loader = prompt["52"]
|
|
assert loader["class_type"] == "MoxieMultimediaLoader"
|
|
assert loader["inputs"]["diffusion_model"] == "Moxie-Multimedia.safetensors"
|
|
project = prompt["57"]
|
|
assert project["inputs"]["tracks_info"] == ["81", 0]
|
|
assert project["inputs"]["model"] == ["48", 0]
|
|
assert project["inputs"]["clip"] == ["52", 1]
|
|
assert "randomize" not in project["inputs"] # UI-only control widget
|
|
|
|
|
|
def test_prompt_conversion_wired_widget_wins():
|
|
from app.backends.comfy import to_prompt_format
|
|
|
|
prompt = to_prompt_format(BASE.raw)
|
|
# CreateVideo #45: fps widget is wired to MoxieCombine slot 2
|
|
create = prompt["45"]
|
|
assert create["inputs"]["fps"] == ["63", 2]
|
|
assert create["inputs"]["codec"] == "h264"
|
|
|
|
|
|
def test_substitution_updates_all_parameters():
|
|
from app.backends.comfy import substitute
|
|
|
|
plan = build_render_plan(
|
|
editor_track_data(BASE), width=720, height=1280, frame_rate=16,
|
|
steps=6, seed=1234, project_name="MM-bench", lora="MM3step.safetensors",
|
|
lora_strength=0.8, upscale_width=1440, upscale_height=2560,
|
|
)
|
|
raw = substitute(BASE, plan)
|
|
nodes = {n["id"]: n for n in raw["nodes"]}
|
|
|
|
editor = nodes[81]["widgets_values_named"]
|
|
assert json.loads(editor["track_data"])["frame_rate"] == 16
|
|
assert editor["resolution"] == "720 x 1280 (9:16)"
|
|
|
|
proj = nodes[57]["widgets_values_named"]
|
|
assert proj["project_name"] == "MM-bench" and proj["steps"] == 6
|
|
assert proj["seed"] == 1234
|
|
|
|
lora = nodes[66]["widgets_values_named"]
|
|
assert lora["lora_name"] == "MM3step.safetensors" and lora["strength_model"] == 0.8
|
|
|
|
rtx = nodes[67]["widgets_values_named"]
|
|
assert rtx["resize_type.width"] == 1440 and rtx["resize_type.height"] == 2560
|
|
|
|
combine = nodes[63]["widgets_values_named"]
|
|
pd = json.loads(combine["project_data"])
|
|
assert pd["width"] == 720 and pd["height"] == 1280 and pd["frame_rate"] == 16
|
|
|
|
saves = {n["id"]: n for n in raw["nodes"] if n["type"] == "SaveVideo"}
|
|
assert saves[46]["widgets_values_named"]["filename_prefix"] == "moxie-bench/native"
|
|
assert saves[69]["widgets_values_named"]["filename_prefix"] == "moxie-bench/upscaled"
|
|
|
|
|
|
def test_substitution_custom_resolution_uses_dict():
|
|
from app.backends.comfy import substitute
|
|
|
|
plan = build_render_plan(editor_track_data(BASE), width=998, height=742)
|
|
raw = substitute(BASE, plan)
|
|
editor = next(n for n in raw["nodes"] if n["id"] == 81)
|
|
assert editor["widgets_values_named"]["resolution"] == {"width": 998, "height": 742}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# mock backend end-to-end (ffmpeg required)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.mark.skipif(True, reason="e2e covered by scripts/test_bench_pipeline.py; keep pytest fast")
|
|
def test_mock_render_placeholder():
|
|
pass
|