- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base workflow, serial render jobs, WS progress mirroring ComfyUI's push events, /api/view + ComfyUI-compatible /view, /upload/image, /easy-media/* aliases for the ported editor - dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI bridge (substitutes the plan into base_workflow.json, converts to the /prompt API format, polls history, downloads outputs) - timeline port: frontend/src/bench mounts the unmodified MultiTrack editor against a comfyAPI shim (fetchApi/toast/dialog/custom events); build.bench.ts emits bench/ui-dist served by the server - packaging: PyInstaller onedir spec + Tauri2 sidecar docs - tests: bench/tests contract suite + scripts/test_bench_pipeline.py e2e (API+WS+render); prebuilt bench UI committed
352 lines
13 KiB
Python
352 lines
13 KiB
Python
"""ComfyUI bridge backend (real execution, phase-2 scope).
|
|
|
|
Drives a running ComfyUI over HTTP instead of reimplementing its executor
|
|
in-process:
|
|
|
|
1. take the base workflow (frontend export format, checked in at
|
|
bench/base_workflow.json),
|
|
2. substitute the RenderPlan parameters (track_data, resolution, steps,
|
|
seed, LoRA, upscale target, codecs, project_data),
|
|
3. convert to the /prompt API format using the input names carried by the
|
|
export itself (widget names via widgets_values_named, link inputs via
|
|
the node's inputs[] entries),
|
|
4. POST /prompt, poll /history/<id> for per-node progress and outputs,
|
|
5. download produced files via /view into the job dir.
|
|
|
|
Phase 3 replaces this with an in-process executor (same interface).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..tracks import RenderPlan
|
|
from ..workflow import EDITOR_TYPE, PipelineSpec
|
|
from .mock import ProgressCb, RenderResult
|
|
|
|
_CONTROL_WIDGET_NAMES = {"randomize", "control_after_generate", "control_before_generate"}
|
|
|
|
|
|
class ComfyBridgeError(RuntimeError):
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HTTP helpers (stdlib only)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _url(cfg_url: str, path: str, **params: Any) -> str:
|
|
base = cfg_url.rstrip("/")
|
|
url = f"{base}{path}"
|
|
if params:
|
|
url += "?" + urllib.parse.urlencode(params)
|
|
return url
|
|
|
|
|
|
def _get_json(url: str, timeout: float = 10.0) -> Any:
|
|
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode("utf-8"))
|
|
|
|
|
|
def _post_json(url: str, payload: dict, timeout: float = 30.0) -> Any:
|
|
data = json.dumps(payload).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
url, data=data, headers={"Content-Type": "application/json"}, method="POST"
|
|
)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode("utf-8"))
|
|
|
|
|
|
def comfy_bridge_available(cfg) -> bool:
|
|
"""True when a ComfyUI answers at cfg.comfyui_url."""
|
|
try:
|
|
_get_json(_url(cfg.comfyui_url, "/system_stats"), timeout=3.0)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Workflow substitution + /prompt conversion
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_RESOLUTION_PRESETS = {
|
|
(1920, 1080): "1920 x 1080 (16:9)",
|
|
(1088, 1920): "1088 x 1920 (9:16)",
|
|
(1080, 1920): "1080 x 1920 (9:16)",
|
|
(1280, 720): "1280 x 720 (16:9)",
|
|
(720, 1280): "720 x 1280 (9:16)",
|
|
(832, 480): "832 x 480 (16:9)",
|
|
(480, 832): "480 x 832 (9:16)",
|
|
(1024, 1024): "1024 x 1024 (1:1)",
|
|
}
|
|
|
|
|
|
def substitute(spec: PipelineSpec, plan: RenderPlan) -> dict[str, Any]:
|
|
"""Return a deep copy of the raw workflow with plan values applied."""
|
|
raw = copy.deepcopy(spec.raw)
|
|
nodes = {int(n["id"]): n for n in raw["nodes"]}
|
|
|
|
def named(nid: int) -> dict[str, Any]:
|
|
n = nodes[nid]
|
|
vals = n.setdefault("widgets_values_named", {})
|
|
vals.setdefault("widgets_values", None)
|
|
return vals
|
|
|
|
# editor: track_data + resolution
|
|
named(spec.editor_node_id)["track_data"] = json.dumps(
|
|
_track_data_for_submission(spec, plan)
|
|
)
|
|
preset = _RESOLUTION_PRESETS.get((plan.width, plan.height))
|
|
named(spec.editor_node_id)["resolution"] = (
|
|
preset if preset else {"width": plan.width, "height": plan.height}
|
|
)
|
|
|
|
# project: name, steps, seed
|
|
proj = spec.stages.get("MoxieProject")
|
|
if proj:
|
|
vals = named(proj.node_id)
|
|
vals["project_name"] = plan.project_name
|
|
vals["steps"] = plan.steps
|
|
vals["seed"] = plan.seed
|
|
|
|
# lora
|
|
lora = spec.stages.get("LoraLoaderModelOnly")
|
|
if lora and plan.lora:
|
|
vals = named(lora.node_id)
|
|
vals["lora_name"] = plan.lora
|
|
vals["strength_model"] = plan.lora_strength
|
|
|
|
# rtx upscale target
|
|
rtx = spec.stages.get("RTXVideoSuperResolution")
|
|
if rtx and plan.upscale_width:
|
|
vals = named(rtx.node_id)
|
|
vals["resize_type"] = "target dimensions"
|
|
vals["resize_type.width"] = plan.upscale_width
|
|
vals["resize_type.height"] = plan.upscale_height
|
|
vals["quality"] = plan.upscale_quality
|
|
|
|
# combine project_data carries the canvas + frame rate
|
|
combine = spec.stages.get("MoxieCombine")
|
|
if combine:
|
|
vals = named(combine.node_id)
|
|
pd_raw = vals.get("project_data", "{}")
|
|
pd = json.loads(pd_raw) if isinstance(pd_raw, str) else dict(pd_raw or {})
|
|
pd.update({
|
|
"project_name": plan.project_name,
|
|
"width": plan.width,
|
|
"height": plan.height,
|
|
"frame_rate": plan.frame_rate,
|
|
})
|
|
vals["project_data"] = json.dumps(pd)
|
|
|
|
# save prefixes get a per-run folder
|
|
for branch, stage in spec.save_nodes.items():
|
|
vals = named(stage.node_id)
|
|
if "filename_prefix" in vals:
|
|
vals["filename_prefix"] = f"moxie-bench/{branch}"
|
|
|
|
return raw
|
|
|
|
|
|
def _track_data_for_submission(spec: PipelineSpec, plan: RenderPlan) -> dict[str, Any]:
|
|
"""Track data as edited in the bench UI; the editor widget value wins."""
|
|
raw = spec.widget(EDITOR_TYPE, "track_data")
|
|
data = json.loads(raw) if isinstance(raw, str) else dict(raw or {})
|
|
data["frame_rate"] = plan.frame_rate
|
|
data.setdefault("total_length", plan.total_length)
|
|
return data
|
|
|
|
|
|
def to_prompt_format(raw: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
"""Frontend-format workflow -> /prompt API format.
|
|
|
|
Widget values come from widgets_values_named (names are the schema input
|
|
names); link inputs come from the node's inputs[] entries. A widget that
|
|
is wired to a link (marker key "widget" plus a link id) takes the link.
|
|
"""
|
|
nodes = raw.get("nodes") or []
|
|
links: dict[int, tuple[int, int]] = {}
|
|
for lk in raw.get("links") or []:
|
|
links[int(lk[0])] = (int(lk[1]), int(lk[2]))
|
|
|
|
prompt: dict[str, dict[str, Any]] = {}
|
|
for n in nodes:
|
|
nid = str(int(n["id"]))
|
|
entry: dict[str, Any] = {}
|
|
for inp in n.get("inputs") or []:
|
|
if not isinstance(inp, dict):
|
|
continue
|
|
name = inp.get("name")
|
|
link = inp.get("link")
|
|
if name is None or link is None:
|
|
continue
|
|
frm = links.get(int(link))
|
|
if frm is None:
|
|
continue
|
|
entry[name] = [str(frm[0]), frm[1]]
|
|
named = n.get("widgets_values_named") or {}
|
|
linked_widget_names = {
|
|
inp.get("widget", {}).get("name")
|
|
for inp in n.get("inputs") or []
|
|
if isinstance(inp, dict) and inp.get("link") is not None
|
|
and isinstance(inp.get("widget"), dict)
|
|
}
|
|
for name, value in named.items():
|
|
if name in ("widgets_values",) or name in _CONTROL_WIDGET_NAMES:
|
|
continue
|
|
if name in linked_widget_names and name in entry:
|
|
continue # wired input wins over the stored widget value
|
|
entry[name] = value
|
|
prompt[nid] = {"class_type": n.get("type"), "inputs": entry}
|
|
return prompt
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Render
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def render(
|
|
spec: PipelineSpec,
|
|
plan: RenderPlan,
|
|
cfg,
|
|
job_dir: Path,
|
|
progress: ProgressCb,
|
|
) -> RenderResult:
|
|
result = RenderResult()
|
|
raw = substitute(spec, plan)
|
|
prompt = to_prompt_format(raw)
|
|
node_types = {str(int(n["id"])): n.get("type", "") for n in raw["nodes"]}
|
|
|
|
progress("MoxieMultimediaLoader", 5.0, f"comfy: submitting prompt to {cfg.comfyui_url}")
|
|
try:
|
|
resp = _post_json(_url(cfg.comfyui_url, "/prompt"), {"prompt": prompt})
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
raise ComfyBridgeError(f"ComfyUI rejected the prompt ({exc.code}): {body[:600]}") from exc
|
|
except urllib.error.URLError as exc:
|
|
raise ComfyBridgeError(
|
|
f"Cannot reach ComfyUI at {cfg.comfyui_url} ({exc.reason}). "
|
|
"Start it with: python main.py --listen 127.0.0.1 --port 8188"
|
|
) from exc
|
|
prompt_id = resp.get("prompt_id")
|
|
if not prompt_id:
|
|
raise ComfyBridgeError(f"ComfyUI returned no prompt_id: {resp}")
|
|
|
|
progress("MoxieProject", 10.0, f"comfy: queued ({prompt_id})")
|
|
|
|
stage_by_node = {str(nid): ntype for nid, ntype in node_types.items()}
|
|
save_branches = {str(st.node_id): branch for branch, st in spec.save_nodes.items()}
|
|
|
|
deadline = time.time() + 60 * 60 # 1h safety stop
|
|
poll = 1.0
|
|
seen_progress_max = 0
|
|
while time.time() < deadline:
|
|
time.sleep(poll)
|
|
try:
|
|
hist = _get_json(_url(cfg.comfyui_url, f"/history/{prompt_id}"), timeout=15.0)
|
|
except Exception as exc: # transient network hiccups
|
|
result.log(f"history poll failed: {exc}")
|
|
continue
|
|
entry = hist.get(prompt_id)
|
|
if entry is None:
|
|
# still queued or running
|
|
continue
|
|
status = entry.get("status") or {}
|
|
messages = status.get("messages") or []
|
|
for msg in messages:
|
|
mtype, mdata = msg[0], (msg[1] if len(msg) > 1 else {})
|
|
if mtype == "executing" and mdata.get("node"):
|
|
ntype = stage_by_node.get(str(mdata["node"]), "")
|
|
if ntype:
|
|
progress(ntype, max(12.0, result_progress(ntype)), f"comfy: {ntype}")
|
|
elif mtype == "progress":
|
|
try:
|
|
val = float(mdata.get("value", 0)) / max(1.0, float(mdata.get("max", 1)))
|
|
except (TypeError, ValueError):
|
|
val = 0.0
|
|
seen_progress_max = max(seen_progress_max, val)
|
|
# sample/VAE progress mapped into the 15..92 window
|
|
progress("MoxieProject", 15.0 + 70.0 * val,
|
|
f"comfy: sampling {int(val * 100)}%",
|
|
{"step": mdata.get("value"), "steps": mdata.get("max")})
|
|
elif mtype == "execution_error":
|
|
raise ComfyBridgeError(
|
|
f"ComfyUI execution failed on node "
|
|
f"{mdata.get('node_type')} ({mdata.get('node_id')}): "
|
|
f"{mdata.get('exception_message')}"
|
|
)
|
|
if status.get("completed"):
|
|
break
|
|
if status.get("status_str") == "error":
|
|
raise ComfyBridgeError("ComfyUI reported an error for this prompt")
|
|
|
|
if not status.get("completed"):
|
|
raise ComfyBridgeError("ComfyUI did not finish in time")
|
|
|
|
# collect outputs
|
|
outputs = entry.get("outputs") or {}
|
|
files: dict[str, dict[str, Any]] = {}
|
|
for nid, node_out in outputs.items():
|
|
branch = save_branches.get(str(nid))
|
|
for key, items in node_out.items():
|
|
if not isinstance(items, list):
|
|
continue
|
|
for item in items:
|
|
if not isinstance(item, dict) or "filename" not in item:
|
|
continue
|
|
if branch:
|
|
files.setdefault(branch, dict(item))
|
|
else:
|
|
result.log(f"aux output from node {nid}: {key} {item.get('filename')}")
|
|
|
|
if not files:
|
|
raise ComfyBridgeError(f"ComfyUI produced no video outputs: outputs={list(outputs)}")
|
|
|
|
for branch, item in files.items():
|
|
params = {
|
|
"filename": item["filename"],
|
|
"subfolder": item.get("subfolder", ""),
|
|
"type": item.get("type", "output"),
|
|
}
|
|
data = _download(_url(cfg.comfyui_url, "/view", **params))
|
|
dst = job_dir / f"{branch}{Path(item['filename']).suffix or '.mp4'}"
|
|
dst.write_bytes(data)
|
|
result.outputs[branch] = {
|
|
"filename": dst.name,
|
|
"path": str(dst),
|
|
"url": f"/api/view?filename={dst.name}&type=output&job={job_dir.name}",
|
|
"source": item.get("subfolder", ""),
|
|
"bytes": len(data),
|
|
}
|
|
result.log(f"{branch}: {item['filename']} ({len(data)} bytes)")
|
|
|
|
progress("SaveVideo", 100.0, "done", {"outputs": result.outputs})
|
|
return result
|
|
|
|
|
|
def result_progress(stage: str) -> float:
|
|
order = {
|
|
"MoxieMultimediaLoader": 8.0,
|
|
"LoraLoaderModelOnly": 10.0,
|
|
"MoxiePreviewOverride": 12.0,
|
|
EDITOR_TYPE: 14.0,
|
|
"MoxieProject": 40.0,
|
|
"MoxieCombine": 78.0,
|
|
"RTXVideoSuperResolution": 85.0,
|
|
"CreateVideo": 92.0,
|
|
"SaveVideo": 97.0,
|
|
}
|
|
return order.get(stage, 20.0)
|
|
|
|
|
|
def _download(url: str, timeout: float = 120.0) -> bytes:
|
|
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
|
return resp.read()
|