MMnode/bench/app/backends/comfy.py
Z User eefca88d35 feat: Moxie bench - standalone python/web harness for the fixed pipeline
- bench/ FastAPI server: fixed 9-stage pipeline parsed from the base
  workflow, serial render jobs, WS progress mirroring ComfyUI's push
  events, /api/view + ComfyUI-compatible /view, /upload/image,
  /easy-media/* aliases for the ported editor
- dual backend: self-contained mock renderer (numpy+ffmpeg, deterministic
  per prompt/seed, native + RTX-upscaled mp4 branches) and a ComfyUI
  bridge (substitutes the plan into base_workflow.json, converts to the
  /prompt API format, polls history, downloads outputs)
- timeline port: frontend/src/bench mounts the unmodified MultiTrack
  editor against a comfyAPI shim (fetchApi/toast/dialog/custom events);
  build.bench.ts emits bench/ui-dist served by the server
- packaging: PyInstaller onedir spec + Tauri2 sidecar docs
- tests: bench/tests contract suite + scripts/test_bench_pipeline.py
  e2e (API+WS+render); prebuilt bench UI committed
2026-09-28 01:42:58 +00:00

352 lines
13 KiB
Python

"""ComfyUI bridge backend (real execution, phase-2 scope).
Drives a running ComfyUI over HTTP instead of reimplementing its executor
in-process:
1. take the base workflow (frontend export format, checked in at
bench/base_workflow.json),
2. substitute the RenderPlan parameters (track_data, resolution, steps,
seed, LoRA, upscale target, codecs, project_data),
3. convert to the /prompt API format using the input names carried by the
export itself (widget names via widgets_values_named, link inputs via
the node's inputs[] entries),
4. POST /prompt, poll /history/<id> for per-node progress and outputs,
5. download produced files via /view into the job dir.
Phase 3 replaces this with an in-process executor (same interface).
"""
from __future__ import annotations
import copy
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any
from ..tracks import RenderPlan
from ..workflow import EDITOR_TYPE, PipelineSpec
from .mock import ProgressCb, RenderResult
_CONTROL_WIDGET_NAMES = {"randomize", "control_after_generate", "control_before_generate"}
class ComfyBridgeError(RuntimeError):
pass
# ---------------------------------------------------------------------------
# HTTP helpers (stdlib only)
# ---------------------------------------------------------------------------
def _url(cfg_url: str, path: str, **params: Any) -> str:
base = cfg_url.rstrip("/")
url = f"{base}{path}"
if params:
url += "?" + urllib.parse.urlencode(params)
return url
def _get_json(url: str, timeout: float = 10.0) -> Any:
with urllib.request.urlopen(url, timeout=timeout) as resp:
return json.loads(resp.read().decode("utf-8"))
def _post_json(url: str, payload: dict, timeout: float = 30.0) -> Any:
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
url, data=data, headers={"Content-Type": "application/json"}, method="POST"
)
with urllib.request.urlopen(req, timeout=timeout) as resp:
return json.loads(resp.read().decode("utf-8"))
def comfy_bridge_available(cfg) -> bool:
"""True when a ComfyUI answers at cfg.comfyui_url."""
try:
_get_json(_url(cfg.comfyui_url, "/system_stats"), timeout=3.0)
return True
except Exception:
return False
# ---------------------------------------------------------------------------
# Workflow substitution + /prompt conversion
# ---------------------------------------------------------------------------
_RESOLUTION_PRESETS = {
(1920, 1080): "1920 x 1080 (16:9)",
(1088, 1920): "1088 x 1920 (9:16)",
(1080, 1920): "1080 x 1920 (9:16)",
(1280, 720): "1280 x 720 (16:9)",
(720, 1280): "720 x 1280 (9:16)",
(832, 480): "832 x 480 (16:9)",
(480, 832): "480 x 832 (9:16)",
(1024, 1024): "1024 x 1024 (1:1)",
}
def substitute(spec: PipelineSpec, plan: RenderPlan) -> dict[str, Any]:
"""Return a deep copy of the raw workflow with plan values applied."""
raw = copy.deepcopy(spec.raw)
nodes = {int(n["id"]): n for n in raw["nodes"]}
def named(nid: int) -> dict[str, Any]:
n = nodes[nid]
vals = n.setdefault("widgets_values_named", {})
vals.setdefault("widgets_values", None)
return vals
# editor: track_data + resolution
named(spec.editor_node_id)["track_data"] = json.dumps(
_track_data_for_submission(spec, plan)
)
preset = _RESOLUTION_PRESETS.get((plan.width, plan.height))
named(spec.editor_node_id)["resolution"] = (
preset if preset else {"width": plan.width, "height": plan.height}
)
# project: name, steps, seed
proj = spec.stages.get("MoxieProject")
if proj:
vals = named(proj.node_id)
vals["project_name"] = plan.project_name
vals["steps"] = plan.steps
vals["seed"] = plan.seed
# lora
lora = spec.stages.get("LoraLoaderModelOnly")
if lora and plan.lora:
vals = named(lora.node_id)
vals["lora_name"] = plan.lora
vals["strength_model"] = plan.lora_strength
# rtx upscale target
rtx = spec.stages.get("RTXVideoSuperResolution")
if rtx and plan.upscale_width:
vals = named(rtx.node_id)
vals["resize_type"] = "target dimensions"
vals["resize_type.width"] = plan.upscale_width
vals["resize_type.height"] = plan.upscale_height
vals["quality"] = plan.upscale_quality
# combine project_data carries the canvas + frame rate
combine = spec.stages.get("MoxieCombine")
if combine:
vals = named(combine.node_id)
pd_raw = vals.get("project_data", "{}")
pd = json.loads(pd_raw) if isinstance(pd_raw, str) else dict(pd_raw or {})
pd.update({
"project_name": plan.project_name,
"width": plan.width,
"height": plan.height,
"frame_rate": plan.frame_rate,
})
vals["project_data"] = json.dumps(pd)
# save prefixes get a per-run folder
for branch, stage in spec.save_nodes.items():
vals = named(stage.node_id)
if "filename_prefix" in vals:
vals["filename_prefix"] = f"moxie-bench/{branch}"
return raw
def _track_data_for_submission(spec: PipelineSpec, plan: RenderPlan) -> dict[str, Any]:
"""Track data as edited in the bench UI; the editor widget value wins."""
raw = spec.widget(EDITOR_TYPE, "track_data")
data = json.loads(raw) if isinstance(raw, str) else dict(raw or {})
data["frame_rate"] = plan.frame_rate
data.setdefault("total_length", plan.total_length)
return data
def to_prompt_format(raw: dict[str, Any]) -> dict[str, dict[str, Any]]:
"""Frontend-format workflow -> /prompt API format.
Widget values come from widgets_values_named (names are the schema input
names); link inputs come from the node's inputs[] entries. A widget that
is wired to a link (marker key "widget" plus a link id) takes the link.
"""
nodes = raw.get("nodes") or []
links: dict[int, tuple[int, int]] = {}
for lk in raw.get("links") or []:
links[int(lk[0])] = (int(lk[1]), int(lk[2]))
prompt: dict[str, dict[str, Any]] = {}
for n in nodes:
nid = str(int(n["id"]))
entry: dict[str, Any] = {}
for inp in n.get("inputs") or []:
if not isinstance(inp, dict):
continue
name = inp.get("name")
link = inp.get("link")
if name is None or link is None:
continue
frm = links.get(int(link))
if frm is None:
continue
entry[name] = [str(frm[0]), frm[1]]
named = n.get("widgets_values_named") or {}
linked_widget_names = {
inp.get("widget", {}).get("name")
for inp in n.get("inputs") or []
if isinstance(inp, dict) and inp.get("link") is not None
and isinstance(inp.get("widget"), dict)
}
for name, value in named.items():
if name in ("widgets_values",) or name in _CONTROL_WIDGET_NAMES:
continue
if name in linked_widget_names and name in entry:
continue # wired input wins over the stored widget value
entry[name] = value
prompt[nid] = {"class_type": n.get("type"), "inputs": entry}
return prompt
# ---------------------------------------------------------------------------
# Render
# ---------------------------------------------------------------------------
def render(
spec: PipelineSpec,
plan: RenderPlan,
cfg,
job_dir: Path,
progress: ProgressCb,
) -> RenderResult:
result = RenderResult()
raw = substitute(spec, plan)
prompt = to_prompt_format(raw)
node_types = {str(int(n["id"])): n.get("type", "") for n in raw["nodes"]}
progress("MoxieMultimediaLoader", 5.0, f"comfy: submitting prompt to {cfg.comfyui_url}")
try:
resp = _post_json(_url(cfg.comfyui_url, "/prompt"), {"prompt": prompt})
except urllib.error.HTTPError as exc:
body = exc.read().decode("utf-8", errors="replace")
raise ComfyBridgeError(f"ComfyUI rejected the prompt ({exc.code}): {body[:600]}") from exc
except urllib.error.URLError as exc:
raise ComfyBridgeError(
f"Cannot reach ComfyUI at {cfg.comfyui_url} ({exc.reason}). "
"Start it with: python main.py --listen 127.0.0.1 --port 8188"
) from exc
prompt_id = resp.get("prompt_id")
if not prompt_id:
raise ComfyBridgeError(f"ComfyUI returned no prompt_id: {resp}")
progress("MoxieProject", 10.0, f"comfy: queued ({prompt_id})")
stage_by_node = {str(nid): ntype for nid, ntype in node_types.items()}
save_branches = {str(st.node_id): branch for branch, st in spec.save_nodes.items()}
deadline = time.time() + 60 * 60 # 1h safety stop
poll = 1.0
seen_progress_max = 0
while time.time() < deadline:
time.sleep(poll)
try:
hist = _get_json(_url(cfg.comfyui_url, f"/history/{prompt_id}"), timeout=15.0)
except Exception as exc: # transient network hiccups
result.log(f"history poll failed: {exc}")
continue
entry = hist.get(prompt_id)
if entry is None:
# still queued or running
continue
status = entry.get("status") or {}
messages = status.get("messages") or []
for msg in messages:
mtype, mdata = msg[0], (msg[1] if len(msg) > 1 else {})
if mtype == "executing" and mdata.get("node"):
ntype = stage_by_node.get(str(mdata["node"]), "")
if ntype:
progress(ntype, max(12.0, result_progress(ntype)), f"comfy: {ntype}")
elif mtype == "progress":
try:
val = float(mdata.get("value", 0)) / max(1.0, float(mdata.get("max", 1)))
except (TypeError, ValueError):
val = 0.0
seen_progress_max = max(seen_progress_max, val)
# sample/VAE progress mapped into the 15..92 window
progress("MoxieProject", 15.0 + 70.0 * val,
f"comfy: sampling {int(val * 100)}%",
{"step": mdata.get("value"), "steps": mdata.get("max")})
elif mtype == "execution_error":
raise ComfyBridgeError(
f"ComfyUI execution failed on node "
f"{mdata.get('node_type')} ({mdata.get('node_id')}): "
f"{mdata.get('exception_message')}"
)
if status.get("completed"):
break
if status.get("status_str") == "error":
raise ComfyBridgeError("ComfyUI reported an error for this prompt")
if not status.get("completed"):
raise ComfyBridgeError("ComfyUI did not finish in time")
# collect outputs
outputs = entry.get("outputs") or {}
files: dict[str, dict[str, Any]] = {}
for nid, node_out in outputs.items():
branch = save_branches.get(str(nid))
for key, items in node_out.items():
if not isinstance(items, list):
continue
for item in items:
if not isinstance(item, dict) or "filename" not in item:
continue
if branch:
files.setdefault(branch, dict(item))
else:
result.log(f"aux output from node {nid}: {key} {item.get('filename')}")
if not files:
raise ComfyBridgeError(f"ComfyUI produced no video outputs: outputs={list(outputs)}")
for branch, item in files.items():
params = {
"filename": item["filename"],
"subfolder": item.get("subfolder", ""),
"type": item.get("type", "output"),
}
data = _download(_url(cfg.comfyui_url, "/view", **params))
dst = job_dir / f"{branch}{Path(item['filename']).suffix or '.mp4'}"
dst.write_bytes(data)
result.outputs[branch] = {
"filename": dst.name,
"path": str(dst),
"url": f"/api/view?filename={dst.name}&type=output&job={job_dir.name}",
"source": item.get("subfolder", ""),
"bytes": len(data),
}
result.log(f"{branch}: {item['filename']} ({len(data)} bytes)")
progress("SaveVideo", 100.0, "done", {"outputs": result.outputs})
return result
def result_progress(stage: str) -> float:
order = {
"MoxieMultimediaLoader": 8.0,
"LoraLoaderModelOnly": 10.0,
"MoxiePreviewOverride": 12.0,
EDITOR_TYPE: 14.0,
"MoxieProject": 40.0,
"MoxieCombine": 78.0,
"RTXVideoSuperResolution": 85.0,
"CreateVideo": 92.0,
"SaveVideo": 97.0,
}
return order.get(stage, 20.0)
def _download(url: str, timeout: float = 120.0) -> bytes:
with urllib.request.urlopen(url, timeout=timeout) as resp:
return resp.read()