Newer ComfyUI builds lock registered v3 node classes after registration (comfy_api internal __setattr__ guard), so _apply_lora writing cls._loaded_lora raised: Cannot modify class attribute '_loaded_lora' on locked class 'MoxieMultimediaLoaderClone'. The (path, sd, metadata) cache now lives at module level, which the locking mechanism does not touch. Test monkeypatch updated to match.
280 lines
12 KiB
Python
280 lines
12 KiB
Python
import os
|
|
import logging
|
|
|
|
import torch
|
|
|
|
import folder_paths
|
|
import comfy.sd
|
|
import comfy.utils
|
|
import comfy.ldm.modules.attention
|
|
from comfy_api.latest import io
|
|
|
|
NODE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
MODELS_DIR = os.path.join(NODE_DIR, "models")
|
|
|
|
MODEL_EXTENSIONS = (".safetensors", ".sft", ".ckpt", ".pt", ".pth", ".bin")
|
|
|
|
SUBDIR_HINTS = {
|
|
"diffusion": "models/diffusion",
|
|
"vae": "models/vae",
|
|
"audio_vae": "models/audio_vae",
|
|
"clip": "models/clip",
|
|
"loras": "models/loras",
|
|
}
|
|
|
|
|
|
def _list_files(subdir):
|
|
"""List model files relative to the pack's models/<subdir> folder."""
|
|
directory = os.path.join(MODELS_DIR, subdir)
|
|
results = []
|
|
if os.path.isdir(directory):
|
|
for root, _dirs, files in os.walk(directory):
|
|
for filename in files:
|
|
if filename.lower().endswith(MODEL_EXTENSIONS):
|
|
rel = os.path.relpath(os.path.join(root, filename), directory)
|
|
results.append(rel.replace("\\", "/"))
|
|
results.sort()
|
|
if not results:
|
|
results.append("<no files found in {}>".format(SUBDIR_HINTS[subdir]))
|
|
return results
|
|
|
|
|
|
def _resolve(subdir, name):
|
|
if name.startswith("<no files found"):
|
|
raise RuntimeError(
|
|
"Moxie Multimedia Loader: no model files found in {}. "
|
|
"Place your models inside this folder and restart ComfyUI.".format(SUBDIR_HINTS[subdir])
|
|
)
|
|
path = os.path.join(MODELS_DIR, subdir, name)
|
|
if not os.path.isfile(path):
|
|
raise RuntimeError(
|
|
"Moxie Multimedia Loader: file '{}' not found in {} ({}).".format(name, SUBDIR_HINTS[subdir], path)
|
|
)
|
|
return path
|
|
|
|
|
|
# (path, sd, metadata) cache for the last parsed LoRA, mirroring the native
|
|
# LoraLoader. Kept at module level on purpose: newer ComfyUI builds lock node
|
|
# classes after registration, so writing class attributes on the node class
|
|
# itself raises "Cannot modify class attribute ... on locked class".
|
|
_loaded_lora = None
|
|
|
|
|
|
def _h3_clip_type():
|
|
"""MiniMax H3 CLIP recipe. The Moxie model set is H3-only, so the text
|
|
encoder type is intentionally NOT a user-facing choice."""
|
|
clip_type_enum = getattr(comfy.sd.CLIPType, "MINIMAX", None)
|
|
if clip_type_enum is None:
|
|
raise RuntimeError(
|
|
"Moxie Multimedia Loader: this ComfyUI build does not expose "
|
|
"CLIPType.MINIMAX (required for MiniMax H3 Qwen3-VL text encoders). "
|
|
"Please update ComfyUI."
|
|
)
|
|
return clip_type_enum
|
|
|
|
|
|
class MoxieMultimediaLoader(io.ComfyNode):
|
|
"""All-in-one loader: diffusion model -> attention backend -> LoRA, plus
|
|
video VAE, audio VAE and CLIP, all read from the node pack's models/ folder.
|
|
|
|
The model set is MiniMax H3-only: the CLIP recipe is locked to
|
|
CLIPType.MINIMAX and outputs are exactly (model, clip, video_vae, audio_vae)
|
|
with the attention backend and LoRA already applied to the model."""
|
|
|
|
@classmethod
|
|
def define_schema(cls) -> io.Schema:
|
|
# The published pack ships its models inside the repo, so default to
|
|
# the bundled turbo LoRA out of the box; "none" opts out.
|
|
lora_files = _list_files("loras")
|
|
lora_default = "none" if lora_files[0].startswith("<no files found") else lora_files[0]
|
|
return io.Schema(
|
|
node_id="MoxieMultimediaLoader",
|
|
display_name="Moxie Multimedia Loader",
|
|
category="loaders/moxie",
|
|
description="Loads the MiniMax H3 diffusion model, applies the attention backend and a LoRA, and loads "
|
|
"a video VAE, an audio VAE and the H3 Qwen3-VL CLIP - all from the models folder bundled "
|
|
"with this custom node. Outputs model, clip, video_vae and audio_vae.",
|
|
inputs=[
|
|
io.Combo.Input(
|
|
"diffusion_model",
|
|
options=_list_files("diffusion"),
|
|
tooltip="Diffusion model file (inside the node pack's models/diffusion folder).",
|
|
),
|
|
io.Combo.Input(
|
|
"weight_dtype",
|
|
options=["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
|
default="default",
|
|
advanced=True,
|
|
tooltip="Weight precision of the diffusion model.",
|
|
),
|
|
io.Combo.Input(
|
|
"attention_backend",
|
|
options=["comfy_kitchen_attention", "pytorch_attention"],
|
|
default="comfy_kitchen_attention",
|
|
tooltip="Dense attention backend. comfy_kitchen_attention uses quantized INT8 attention (Nvidia/AMD GPUs only).",
|
|
),
|
|
io.Combo.Input(
|
|
"lora_name",
|
|
options=["none"] + lora_files,
|
|
default=lora_default,
|
|
tooltip="LoRA file from the pack's models/loras folder, applied to both the model and "
|
|
"the CLIP. Defaults to the bundled turbo LoRA; 'none' disables it.",
|
|
),
|
|
io.Float.Input("lora_strength_model", default=1.0, min=-100.0, max=100.0, step=0.01,
|
|
tooltip="How strongly the LoRA modifies the diffusion model."),
|
|
io.Float.Input("lora_strength_clip", default=1.0, min=-100.0, max=100.0, step=0.01,
|
|
tooltip="How strongly the LoRA modifies the CLIP model."),
|
|
io.Combo.Input(
|
|
"video_vae",
|
|
options=_list_files("vae"),
|
|
tooltip="Video/image VAE file (inside models/vae).",
|
|
),
|
|
io.Combo.Input(
|
|
"audio_vae",
|
|
options=_list_files("audio_vae"),
|
|
tooltip="Audio VAE file (inside models/audio_vae). Supports plain VAE checkpoints and "
|
|
"checkpoints with audio_vae./vocoder. key prefixes (e.g. LTX audio VAE).",
|
|
),
|
|
io.Combo.Input(
|
|
"clip",
|
|
options=_list_files("clip"),
|
|
tooltip="MiniMax H3 text encoder file (inside models/clip). Loaded with the locked "
|
|
"minimax CLIP recipe (Qwen3-VL).",
|
|
),
|
|
io.Combo.Input(
|
|
"clip_device",
|
|
options=["default", "cpu"],
|
|
default="default",
|
|
advanced=True,
|
|
tooltip="Force the CLIP onto CPU if needed.",
|
|
),
|
|
],
|
|
outputs=[
|
|
io.Model.Output(display_name="model", tooltip="Diffusion model with attention backend and LoRA applied."),
|
|
io.Clip.Output(display_name="clip", tooltip="CLIP text encoder (with LoRA applied if present)."),
|
|
io.Vae.Output(display_name="video_vae", tooltip="Loaded video/image VAE."),
|
|
io.Vae.Output(display_name="audio_vae", tooltip="Loaded audio VAE."),
|
|
],
|
|
)
|
|
|
|
@classmethod
|
|
def _load_diffusion_model(cls, diffusion_model, weight_dtype):
|
|
model_options = {}
|
|
if weight_dtype == "fp8_e4m3fn":
|
|
model_options["dtype"] = torch.float8_e4m3fn
|
|
elif weight_dtype == "fp8_e4m3fn_fast":
|
|
model_options["dtype"] = torch.float8_e4m3fn
|
|
model_options["fp8_optimizations"] = True
|
|
elif weight_dtype == "fp8_e5m2":
|
|
model_options["dtype"] = torch.float8_e5m2
|
|
|
|
path = _resolve("diffusion", diffusion_model)
|
|
model = comfy.sd.load_diffusion_model(path, model_options=model_options)
|
|
if model is None:
|
|
raise RuntimeError("Moxie Multimedia Loader: failed to load diffusion model '{}'.".format(diffusion_model))
|
|
return model
|
|
|
|
@classmethod
|
|
def _apply_attention_backend(cls, model, attention_backend):
|
|
attention_name = {
|
|
"comfy_kitchen_attention": "comfy_kitchen_int8",
|
|
"pytorch_attention": "pytorch",
|
|
}.get(attention_backend, "pytorch")
|
|
attention_function = comfy.ldm.modules.attention.get_attention_function(attention_name, None)
|
|
if attention_function is None:
|
|
logging.warning(
|
|
"[Moxie Multimedia Loader] Attention backend '%s' is unavailable; using PyTorch attention.",
|
|
attention_backend,
|
|
)
|
|
attention_function = comfy.ldm.modules.attention.get_attention_function("pytorch")
|
|
m = model.clone()
|
|
m.set_model_optimized_attention(attention_function)
|
|
return m
|
|
|
|
@classmethod
|
|
def _apply_lora(cls, model, clip, lora_name, strength_model, strength_clip):
|
|
global _loaded_lora
|
|
if (
|
|
lora_name is None
|
|
or lora_name == "none"
|
|
or lora_name.startswith("<no files found")
|
|
or (strength_model == 0 and strength_clip == 0)
|
|
):
|
|
return model, clip
|
|
|
|
lora_path = _resolve("loras", lora_name)
|
|
lora = None
|
|
lora_metadata = None
|
|
if _loaded_lora is not None:
|
|
if _loaded_lora[0] == lora_path:
|
|
lora = _loaded_lora[1]
|
|
lora_metadata = _loaded_lora[2] if len(_loaded_lora) > 2 else None
|
|
else:
|
|
_loaded_lora = None
|
|
|
|
if lora is None:
|
|
lora, lora_metadata = comfy.utils.load_torch_file(lora_path, safe_load=True, return_metadata=True)
|
|
_loaded_lora = (lora_path, lora, lora_metadata)
|
|
|
|
model_lora, clip_lora = comfy.sd.load_lora_for_models(
|
|
model, clip, lora, strength_model, strength_clip, lora_metadata=lora_metadata
|
|
)
|
|
return model_lora, clip_lora
|
|
|
|
@classmethod
|
|
def _load_vae(cls, vae_name, subdir="vae"):
|
|
path = _resolve(subdir, vae_name)
|
|
sd, metadata = comfy.utils.load_torch_file(path, return_metadata=True)
|
|
if subdir == "audio_vae" and any(k.startswith("audio_vae.") for k in sd):
|
|
# Audio VAE checkpoint style (e.g. LTX audio VAE): rename keys to what comfy.sd.VAE expects.
|
|
sd = comfy.utils.state_dict_prefix_replace(
|
|
sd, {"audio_vae.": "autoencoder.", "vocoder.": "vocoder."}, filter_keys=True
|
|
)
|
|
vae = comfy.sd.VAE(sd=sd, metadata=metadata)
|
|
vae.throw_exception_if_invalid()
|
|
return vae
|
|
|
|
@classmethod
|
|
def _load_clip(cls, clip_name, device):
|
|
clip_type_enum = _h3_clip_type()
|
|
clip_path = _resolve("clip", clip_name)
|
|
|
|
model_options = {}
|
|
if device == "cpu":
|
|
model_options["load_device"] = model_options["offload_device"] = torch.device("cpu")
|
|
|
|
clip = comfy.sd.load_clip(
|
|
ckpt_paths=[clip_path],
|
|
embedding_directory=folder_paths.get_folder_paths("embeddings"),
|
|
clip_type=clip_type_enum,
|
|
model_options=model_options,
|
|
)
|
|
return clip
|
|
|
|
@classmethod
|
|
def execute(
|
|
cls,
|
|
diffusion_model,
|
|
weight_dtype,
|
|
attention_backend,
|
|
lora_name,
|
|
lora_strength_model,
|
|
lora_strength_clip,
|
|
video_vae,
|
|
audio_vae,
|
|
clip,
|
|
clip_device,
|
|
) -> io.NodeOutput:
|
|
# Chain: Load Diffusion Model -> Model Attention Backend -> Lora Loader (needs both model and clip)
|
|
model = cls._load_diffusion_model(diffusion_model, weight_dtype)
|
|
model = cls._apply_attention_backend(model, attention_backend)
|
|
|
|
clip_obj = cls._load_clip(clip, clip_device)
|
|
model, clip_obj = cls._apply_lora(model, clip_obj, lora_name, lora_strength_model, lora_strength_clip)
|
|
|
|
video_vae_obj = cls._load_vae(video_vae, "vae")
|
|
audio_vae_obj = cls._load_vae(audio_vae, "audio_vae")
|
|
|
|
return io.NodeOutput(model, clip_obj, video_vae_obj, audio_vae_obj)
|
|
|