MMnode/moxie_loader.py
Z User 0c4987d090 T2V resolves to the same value as R2V; loader shows a single model
- T2V selections (editor default mode, explicit [start-end,t2v] override
  tags, raw t2v values) now resolve to r2v in every task-type resolver
  (h3_task_type, _multitrack_task_type, frontend getMultiTrackTaskType);
  h3_generation_mode maps raw t2v to reference conditioning as belt and
  braces. Reference-mode with zero reference media renders text-only
  locally; the cloud path falls back to its native t2va request instead
  of raising. i2v/l2v/v2v/edit flows unchanged.
- MoxieMultimediaLoader now presents a single model picker: LoRA, video
  VAE, audio VAE, and CLIP widgets removed and auto-selected from the
  pack's models/ folders (first sorted file, loud error when missing,
  LoRA applied at strength 1.0 when bundled). Outputs and load order
  unchanged; saved workflows keep working (extra widget values ignored).

Verified: py_compile; scripts/test_t2v_r2v_and_loader.py 50/50;
frontend vitest 566/567 (one pre-existing env failure on HEAD);
bun run build; scripts/test_rtx_upscale_integration.py all pass.
2026-09-27 05:50:35 +00:00

274 lines
11 KiB
Python

import os
import logging
import torch
import folder_paths
import comfy.sd
import comfy.utils
import comfy.ldm.modules.attention
from comfy_api.latest import io
NODE_DIR = os.path.dirname(os.path.abspath(__file__))
MODELS_DIR = os.path.join(NODE_DIR, "models")
MODEL_EXTENSIONS = (".safetensors", ".sft", ".ckpt", ".pt", ".pth", ".bin")
SUBDIR_HINTS = {
"diffusion": "models/diffusion",
"vae": "models/vae",
"audio_vae": "models/audio_vae",
"clip": "models/clip",
"loras": "models/loras",
}
def _list_files(subdir):
"""List model files relative to the pack's models/<subdir> folder."""
directory = os.path.join(MODELS_DIR, subdir)
results = []
if os.path.isdir(directory):
for root, _dirs, files in os.walk(directory):
for filename in files:
if filename.lower().endswith(MODEL_EXTENSIONS):
rel = os.path.relpath(os.path.join(root, filename), directory)
results.append(rel.replace("\\", "/"))
results.sort()
if not results:
results.append("<no files found in {}>".format(SUBDIR_HINTS[subdir]))
return results
def _resolve(subdir, name):
if name.startswith("<no files found"):
raise RuntimeError(
"Moxie Multimedia Loader: no model files found in {}. "
"Place your models inside this folder and restart ComfyUI.".format(SUBDIR_HINTS[subdir])
)
path = os.path.join(MODELS_DIR, subdir, name)
if not os.path.isfile(path):
raise RuntimeError(
"Moxie Multimedia Loader: file '{}' not found in {} ({}).".format(name, SUBDIR_HINTS[subdir], path)
)
return path
# (path, sd, metadata) cache for the last parsed LoRA, mirroring the native
# LoraLoader. Kept at module level on purpose: newer ComfyUI builds lock node
# classes after registration, so writing class attributes on the node class
# itself raises "Cannot modify class attribute ... on locked class".
_loaded_lora = None
def _auto_pick(subdir):
"""Return the model file to use for a hidden picker.
End users only see the diffusion model widget; every auxiliary model
(video VAE, audio VAE, CLIP, LoRA) is auto-selected from the pack's
models/<subdir> folder - the first file in sorted order, which for the
published pack is exactly the bundled file. A missing file keeps the
loud, actionable error from _resolve.
"""
files = _list_files(subdir)
if files[0].startswith("<no files found"):
raise RuntimeError(
"Moxie Multimedia Loader: no model files found in {}. "
"Place your models inside this folder and restart ComfyUI.".format(SUBDIR_HINTS[subdir])
)
selected = files[0]
if len(files) > 1:
logging.info(
"[Moxie Multimedia Loader] Multiple files in %s; using '%s'.",
SUBDIR_HINTS[subdir],
selected,
)
return selected
def _auto_lora():
"""Bundled LoRA file, or ``None`` when models/loras is empty."""
files = _list_files("loras")
if files[0].startswith("<no files found"):
return None
return files[0]
def _h3_clip_type():
"""MiniMax H3 CLIP recipe. The Moxie model set is H3-only, so the text
encoder type is intentionally NOT a user-facing choice."""
clip_type_enum = getattr(comfy.sd.CLIPType, "MINIMAX", None)
if clip_type_enum is None:
raise RuntimeError(
"Moxie Multimedia Loader: this ComfyUI build does not expose "
"the CLIP recipe required for the bundled text encoders. "
"Please update ComfyUI."
)
return clip_type_enum
class MoxieMultimediaLoader(io.ComfyNode):
"""All-in-one loader that looks like it loads a single model.
The only visible picker is the diffusion model; the attention backend is
applied to it, the bundled LoRA (if any), the video VAE, the audio VAE
and the CLIP are auto-selected from the node pack's models/ folder.
The model set is MiniMax H3-only: the CLIP recipe is locked to
CLIPType.MINIMAX and outputs are exactly (model, clip, video_vae, audio_vae)
with the attention backend and LoRA already applied to the model."""
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="MoxieMultimediaLoader",
display_name="Moxie Multimedia Loader",
category="loaders/moxie",
description="Loads the Moxie diffusion model (with the attention backend and the bundled LoRA "
"already applied) plus the video VAE, audio VAE and text encoder that ship with "
"this custom node - one model picker, everything else is handled automatically. "
"Outputs model, clip, video_vae and audio_vae.",
inputs=[
io.Combo.Input(
"diffusion_model",
options=_list_files("diffusion"),
tooltip="Diffusion model file (inside the node pack's models/diffusion folder).",
),
io.Combo.Input(
"weight_dtype",
options=["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
default="default",
advanced=True,
tooltip="Weight precision of the diffusion model.",
),
io.Combo.Input(
"attention_backend",
options=["comfy_kitchen_attention", "pytorch_attention"],
default="comfy_kitchen_attention",
tooltip="Dense attention backend. comfy_kitchen_attention uses quantized INT8 attention (Nvidia/AMD GPUs only).",
),
],
outputs=[
io.Model.Output(display_name="model", tooltip="Diffusion model with attention backend and LoRA applied."),
io.Clip.Output(display_name="clip", tooltip="CLIP text encoder (with LoRA applied if present)."),
io.Vae.Output(display_name="video_vae", tooltip="Loaded video/image VAE."),
io.Vae.Output(display_name="audio_vae", tooltip="Loaded audio VAE."),
],
)
@classmethod
def _load_diffusion_model(cls, diffusion_model, weight_dtype):
model_options = {}
if weight_dtype == "fp8_e4m3fn":
model_options["dtype"] = torch.float8_e4m3fn
elif weight_dtype == "fp8_e4m3fn_fast":
model_options["dtype"] = torch.float8_e4m3fn
model_options["fp8_optimizations"] = True
elif weight_dtype == "fp8_e5m2":
model_options["dtype"] = torch.float8_e5m2
path = _resolve("diffusion", diffusion_model)
model = comfy.sd.load_diffusion_model(path, model_options=model_options)
if model is None:
raise RuntimeError("Moxie Multimedia Loader: failed to load diffusion model '{}'.".format(diffusion_model))
return model
@classmethod
def _apply_attention_backend(cls, model, attention_backend):
attention_name = {
"comfy_kitchen_attention": "comfy_kitchen_int8",
"pytorch_attention": "pytorch",
}.get(attention_backend, "pytorch")
attention_function = comfy.ldm.modules.attention.get_attention_function(attention_name, None)
if attention_function is None:
logging.warning(
"[Moxie Multimedia Loader] Attention backend '%s' is unavailable; using PyTorch attention.",
attention_backend,
)
attention_function = comfy.ldm.modules.attention.get_attention_function("pytorch")
m = model.clone()
m.set_model_optimized_attention(attention_function)
return m
@classmethod
def _apply_lora(cls, model, clip, lora_name, strength_model, strength_clip):
global _loaded_lora
if (
lora_name is None
or lora_name == "none"
or lora_name.startswith("<no files found")
or (strength_model == 0 and strength_clip == 0)
):
return model, clip
lora_path = _resolve("loras", lora_name)
lora = None
lora_metadata = None
if _loaded_lora is not None:
if _loaded_lora[0] == lora_path:
lora = _loaded_lora[1]
lora_metadata = _loaded_lora[2] if len(_loaded_lora) > 2 else None
else:
_loaded_lora = None
if lora is None:
lora, lora_metadata = comfy.utils.load_torch_file(lora_path, safe_load=True, return_metadata=True)
_loaded_lora = (lora_path, lora, lora_metadata)
model_lora, clip_lora = comfy.sd.load_lora_for_models(
model, clip, lora, strength_model, strength_clip, lora_metadata=lora_metadata
)
return model_lora, clip_lora
@classmethod
def _load_vae(cls, vae_name, subdir="vae"):
path = _resolve(subdir, vae_name)
sd, metadata = comfy.utils.load_torch_file(path, return_metadata=True)
if subdir == "audio_vae" and any(k.startswith("audio_vae.") for k in sd):
# Audio VAE checkpoint style (e.g. LTX audio VAE): rename keys to what comfy.sd.VAE expects.
sd = comfy.utils.state_dict_prefix_replace(
sd, {"audio_vae.": "autoencoder.", "vocoder.": "vocoder."}, filter_keys=True
)
vae = comfy.sd.VAE(sd=sd, metadata=metadata)
vae.throw_exception_if_invalid()
return vae
@classmethod
def _load_clip(cls, clip_name, device):
clip_type_enum = _h3_clip_type()
clip_path = _resolve("clip", clip_name)
model_options = {}
if device == "cpu":
model_options["load_device"] = model_options["offload_device"] = torch.device("cpu")
clip = comfy.sd.load_clip(
ckpt_paths=[clip_path],
embedding_directory=folder_paths.get_folder_paths("embeddings"),
clip_type=clip_type_enum,
model_options=model_options,
)
return clip
@classmethod
def execute(
cls,
diffusion_model,
weight_dtype,
attention_backend,
) -> io.NodeOutput:
# Chain: Load Diffusion Model -> Model Attention Backend -> Lora Loader (needs both model and clip)
model = cls._load_diffusion_model(diffusion_model, weight_dtype)
model = cls._apply_attention_backend(model, attention_backend)
# Hidden pickers: auxiliary models are auto-selected from the pack's
# models/ folder so end users only ever see the diffusion model.
clip_obj = cls._load_clip(_auto_pick("clip"), "default")
lora_name = _auto_lora()
model, clip_obj = cls._apply_lora(model, clip_obj, lora_name, 1.0, 1.0)
video_vae_obj = cls._load_vae(_auto_pick("vae"), "vae")
audio_vae_obj = cls._load_vae(_auto_pick("audio_vae"), "audio_vae")
return io.NodeOutput(model, clip_obj, video_vae_obj, audio_vae_obj)