MMnode/moxie_preview.py
Z User e2ae630ec2 feat: lock Moxie loader to MiniMax H3 and drop model_loader bundle
The Moxie model set is MiniMax H3-only, so no other model choices are
exposed in the editor:

- Moxie Multimedia Loader: removed the clip_type dropdown (which listed
  every CLIPType recipe in ComfyUI). The H3 Qwen3-VL text encoder is now
  always loaded with CLIPType.MINIMAX; older ComfyUI builds raise a
  clear error instead of silently loading the wrong encoder.
- Moxie Multimedia Loader: outputs are now exactly model, clip,
  video_vae, audio_vae - the legacy FAST_MODEL_LOADER (model_loader)
  bundle output is removed. The model output keeps the attention
  backend and LoRA applied.
- Moxie MultiTrack Project: takes model, clip, video_vae and audio_vae
  as required direct inputs instead of a bundled model_loader; the
  fallback plumbing is removed while EasyMultiTrackProject keeps its
  model_loader input for non-Moxie workflows.
- Moxie Preview Override: dropped the model_loader pass-through slots
  and patches the model input directly.
- Moxie-consolidated.json reference workflow updated to the new slots.
- Added tests/test_moxie_nodes.py covering the schema contracts, the
  attention -> LoRA -> output chain, the MINIMAX lock and the project
  loader synthesis.
2026-09-26 19:06:50 +00:00

1113 lines
50 KiB
Python

import base64
import fractions
import io as pyio
import logging
import queue
import threading
import time
import numpy as np
import torch
import comfy.model_base
import comfy.model_management
import comfy.utils
import comfy.patcher_extension
try:
import comfy.model_prefetch
except Exception:
comfy.model_prefetch = None
import folder_paths
import latent_preview
from comfy_api.latest import io
from PIL import Image, ImageOps
from .tiny_vae import load_tiny_vae_decoder
try:
from .ltxv_nodes import WrappedPreviewer as _LTXWrappedPreviewer, get_ltx_rgb_factors as _ltx_rgb_factors
except Exception as e:
logging.warning(f"[KJ PreviewOverride] LTX preview helpers unavailable ({e}); LTX previews disabled.")
_LTXWrappedPreviewer = None
_ltx_rgb_factors = None
try:
from server import PromptServer
except ImportError:
PromptServer = None
def _suppressed_preview_image(self_, preview_format, x0):
return None
class _AsyncPreviewEncoder:
"""Off-thread encoder. Bounded FIFO drops-on-full so the sampler never blocks on us."""
_STOP = object()
def __init__(self, max_in_flight=2):
self.q = queue.Queue(maxsize=max_in_flight)
self.thread = threading.Thread(target=self._run, name="kj_preview_encoder", daemon=True)
self.thread.start()
def submit(self, fn):
try:
self.q.put_nowait(fn)
return True
except queue.Full:
return False
def _run(self):
while True:
item = self.q.get()
if item is self._STOP:
return
try:
item()
except Exception:
logging.exception("[KJ Preview Override] async encoder error")
def shutdown(self, drain_timeout=5.0):
try:
self.q.put(self._STOP, timeout=drain_timeout)
except queue.Full:
pass
self.thread.join(timeout=drain_timeout)
def _get_core_previewer(load_device, latent_format):
# Walk past custom-node hooks on get_previewer to reach the unwrapped core function.
fn = latent_preview.get_previewer
seen = set()
while hasattr(fn, "__wrapped__") and id(fn) not in seen:
seen.add(id(fn))
fn = fn.__wrapped__
return fn(load_device, latent_format)
def _decode_video_frames_l2rgb(x0, latent_format, max_frames, stride=1):
# Bulk-blocking GPU→CPU copy (not per-frame non_blocking) avoids torn frames at high res.
if x0.ndim != 5:
return []
rgb_factors = getattr(latent_format, "latent_rgb_factors", None)
if rgb_factors is None:
return []
try:
reshape = getattr(latent_format, "latent_rgb_factors_reshape", None)
if reshape is not None:
x0 = reshape(x0)
bias = getattr(latent_format, "latent_rgb_factors_bias", None)
factors = torch.tensor(rgb_factors, device=x0.device, dtype=x0.dtype).transpose(0, 1)
bias_t = torch.tensor(bias, device=x0.device, dtype=x0.dtype) if bias is not None else None
x = x0[0]
if stride > 1:
x = x[:, ::stride]
t_total = x.shape[1]
if max_frames > 0 and max_frames < t_total:
indices = np.linspace(0, t_total - 1, max_frames).round().astype(int).tolist()
x = x[:, indices]
x = x.movedim(0, -1)
rgb = torch.nn.functional.linear(x, factors, bias=bias_t)
rgb.add_(1.0).mul_(127.5).clamp_(0, 255)
rgb_cpu = rgb.to(torch.uint8).cpu().numpy()
return [Image.fromarray(rgb_cpu[i]) for i in range(rgb_cpu.shape[0])]
except Exception:
return []
def _probe_codec(name):
try:
import av # noqa
av.Codec(name, "w")
return True
except Exception:
return False
# --disable-api-nodes installs a CSP with no media-src
def _csp_blocks_video():
try:
from comfy.cli_args import args as _args
return bool(getattr(_args, "disable_api_nodes", False))
except Exception:
return False
# CPU x264 first: at ultrafast it encodes a preview clip in ~0.1s and needs no CUDA context, while
# NVENC's ffmpeg wrapper time-slices against the saturated sampling context and takes seconds per step.
# The candidates are (codec, options, min_w, min_h); NVENC rejects sub-145x49 inputs at avcodec_open2.
_MP4_CANDIDATES = [c for c in (
("libx264", {"preset": "ultrafast", "tune": "zerolatency", "crf": "28"}, 2, 2),
("h264_nvenc", {"preset": "p1", "rc": "vbr", "cq": "23"}, 145, 49),
("h264_nvenc", {"preset": "p1"}, 145, 49),
) if _probe_codec(c[0])]
_HAS_MP4 = bool(_MP4_CANDIDATES)
_MP4_AVAILABLE = _HAS_MP4 and not _csp_blocks_video()
if _HAS_MP4 and not _MP4_AVAILABLE:
logging.info("[KJ PreviewOverride] --disable-api-nodes blocks blob: video, using WebP for animated previews.")
_mp4_warned = False
def _encode_mp4(frames, fps, max_res, audio=None):
# Fragmented MP4 so the browser can decode mid-download. Returns (None, 0, 0) when every
# candidate fails (including too-small frames), so caller falls through to WebP.
# audio: optional ([2, L] float32 in [-1, 1], sample_rate) muxed as an AAC track; fps may be a
# Fraction so the clip's duration matches the audio
global _mp4_warned
if not frames:
return None, 0, 0
try:
import av
except Exception:
return None, 0, 0
# arrays arrive already at the preview size from _frames_to_arrays and go straight into the
# encoder plane; PIL frames from the other previewers are resized here
pil_frames = []
for f in frames:
if isinstance(f, np.ndarray):
pil_frames.append(f)
continue
pf = f if f.mode == "RGB" else f.convert("RGB")
if max_res and max_res > 0 and (pf.width > max_res or pf.height > max_res):
pf = ImageOps.contain(pf, (max_res, max_res), Image.LANCZOS)
pil_frames.append(pf)
# yuv420p requires even dimensions.
f0 = pil_frames[0]
w0, h0 = (f0.shape[1], f0.shape[0]) if isinstance(f0, np.ndarray) else (f0.width, f0.height)
out_w, out_h = w0 & ~1, h0 & ~1
if (out_w, out_h) != (w0, h0):
pil_frames = [pf[:out_h, :out_w] if isinstance(pf, np.ndarray) else pf.resize((out_w, out_h), Image.LANCZOS)
for pf in pil_frames]
last_err = None
for codec, opts, min_w, min_h in _MP4_CANDIDATES:
if out_w < min_w or out_h < min_h:
continue
buf = pyio.BytesIO()
try:
container = av.open(
buf, mode="w", format="mp4",
options={"movflags": "frag_keyframe+empty_moov+default_base_moof"},
)
rate = fps if isinstance(fps, fractions.Fraction) else int(max(1, fps))
stream = container.add_stream(codec, rate=rate)
stream.width = out_w
stream.height = out_h
stream.pix_fmt = "yuv420p"
stream.options = opts
# every stream must exist before the first mux writes the header
astream = container.add_stream("aac", rate=int(audio[1]), layout="stereo") if audio is not None else None
for pf in pil_frames:
vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(pf), format="rgb24") if isinstance(pf, np.ndarray) \
else av.VideoFrame.from_image(pf)
for pkt in stream.encode(vf):
container.mux(pkt)
for pkt in stream.encode():
container.mux(pkt)
if astream is not None:
_mux_aac(container, av, astream, audio[0], audio[1])
container.close()
return base64.b64encode(buf.getvalue()).decode("ascii"), out_w, out_h
except Exception as e:
last_err = e
continue
if not _mp4_warned and last_err is not None:
_mp4_warned = True
logging.warning(f"[KJ PreviewOverride] MP4 encode failed, using WebP fallback: {last_err}")
return None, 0, 0
def _mux_aac(container, av, astream, wave, sample_rate):
# wave [2, L] float32 in [-1, 1]; mirrors core's VideoFromComponents.save_to: one whole-clip
# frame at pts 0, the codec's own FIFO does the AAC framing
frame = av.AudioFrame.from_ndarray(np.ascontiguousarray(wave, dtype=np.float32), format="fltp", layout="stereo")
frame.sample_rate = int(sample_rate)
frame.pts = 0
container.mux(astream.encode(frame))
container.mux(astream.encode(None))
def _encode_animated_webp(frames, fps, quality, max_res):
if not frames:
return None, 0, 0
pil_frames = []
for f in frames:
pf = _as_pil(f)
if pf.mode != "RGB":
pf = pf.convert("RGB")
if max_res and max_res > 0 and (pf.width > max_res or pf.height > max_res):
pf = ImageOps.contain(pf, (max_res, max_res), Image.LANCZOS)
pil_frames.append(pf)
duration_ms = max(1, int(round(1000 / max(1, fps))))
buf = pyio.BytesIO()
try:
pil_frames[0].save(
buf,
format="WEBP",
save_all=True,
append_images=pil_frames[1:],
duration=duration_ms,
loop=0,
quality=quality,
method=4,
)
except Exception as e:
logging.warning(f"Animated WebP encode failed: {e}")
return None, 0, 0
return base64.b64encode(buf.getvalue()).decode("ascii"), pil_frames[0].width, pil_frames[0].height
def _interp_db_curve(t, xs, ys):
# Mirrors sampler_nodes._interp_curve.
if t <= xs[0]:
return ys[0]
if t >= xs[-1]:
return ys[-1]
for i in range(len(xs) - 1):
if xs[i] <= t <= xs[i + 1]:
span = xs[i + 1] - xs[i]
if span <= 0:
return ys[i]
f = (t - xs[i]) / span
return ys[i] + f * (ys[i + 1] - ys[i])
return 0.0
def _detect_detail_boost_curve(sampler, model_patcher, sigmas_list):
# Amount is already baked into ys by the editor, so peak ys == user-set amount.
try:
extra = getattr(sampler, "extra_options", None) or {}
xs = extra.get("db_curve_xs")
ys = extra.get("db_curve_ys")
if "db_wrapped_sampler" not in extra or not xs or not ys or len(xs) != len(ys) or len(xs) < 2:
return None
ms = model_patcher.get_model_object("model_sampling")
start_sigma = float(ms.percent_to_sigma(extra.get("db_start_percent", 0.0)))
end_sigma = float(ms.percent_to_sigma(extra.get("db_end_percent", 1.0)))
# None outside the gate so JS can distinguish "inactive" from "active with value 0".
out = []
for s in sigmas_list:
sig = float(s)
if sig <= 0 or start_sigma <= end_sigma or sig >= start_sigma or sig <= end_sigma:
out.append(None)
continue
t = (start_sigma - sig) / (start_sigma - end_sigma)
out.append(_interp_db_curve(t, xs, ys))
return out
except Exception as e:
logging.warning(f"[KJ PreviewOverride] DB curve detection failed: {e}")
return None
def _ltx_decode_to_pil(ltx_previewer, x0_5d, max_frames=None, stride=1):
# Pre-shape (B, C, T, H, W) → (B*T, C, H, W); WrappedPreviewer adds the sequence-batch dim.
if ltx_previewer is None or x0_5d.ndim != 5:
return []
if stride > 1:
x0_5d = x0_5d[:, :, ::stride]
x_moved = x0_5d.movedim(2, 1) # (B, T, C, H, W) — must take shape AFTER movedim
x_in = x_moved.reshape((-1,) + x_moved.shape[-3:])
rgb = ltx_previewer.decode_latent_to_preview(x_in)
if rgb is None:
return []
if rgb.ndim == 3:
rgb = rgb.unsqueeze(0)
if rgb.ndim != 4:
return []
t_total = rgb.shape[0]
if max_frames is not None and 0 < max_frames < t_total:
indices = np.linspace(0, t_total - 1, max_frames).round().astype(int).tolist()
rgb = rgb[indices]
u8 = (rgb * 255).clamp(0, 255).to(torch.uint8).cpu().numpy()
return [Image.fromarray(u8[i]) for i in range(u8.shape[0])]
def _ltx_full_vae_decode_to_pil(vae, x0_5d, max_frames=None, stride=1):
# vae.decode handles device + tiling. Slow vs TAEHV but full quality. Output shape
# varies by VAE; we accept (B, T, H, W, C) or (T, H, W, C) and normalize.
if vae is None or x0_5d.ndim != 5:
return []
if stride > 1:
x0_5d = x0_5d[:, :, ::stride]
try:
images = vae.decode(x0_5d)
except Exception as e:
logging.warning(f"[KJ PreviewOverride] LTX VAE decode failed: {e}")
return []
if images.ndim == 5:
images = images[0]
if images.ndim != 4:
return []
t_total = images.shape[0]
if max_frames is not None and 0 < max_frames < t_total:
indices = np.linspace(0, t_total - 1, max_frames).round().astype(int).tolist()
images = images[indices]
u8 = (images.float() * 255).clamp(0, 255).to(torch.uint8).cpu().numpy()
return [Image.fromarray(u8[i]) for i in range(u8.shape[0])]
def _frame_to_pil(frame):
# [3, H, W] on any device, float in [0, 1] or uint8. Copies before the in-place math so a view into
# a shared batch is never scaled twice; fp16 elementwise on the CPU is slow, so it goes through fp32
if frame.dtype != torch.uint8:
frame = frame.to(torch.float32, copy=True).clamp_(0, 1).mul_(255).to(torch.uint8)
return Image.fromarray(frame.movedim(0, -1).contiguous().cpu().numpy())
def _contain_size(w, h, max_res):
# ImageOps.contain sizing, rounded down to even so the yuv420p encode never needs a second resize;
# frames that already fit stay at native size
if not max_res or max_res <= 0 or (w <= max_res and h <= max_res):
return w, h
if w >= h:
tw, th = max_res, round(h / w * max_res)
else:
tw, th = round(w / h * max_res), max_res
return max(2, tw & ~1), max(2, th & ~1)
def _frames_to_arrays(frames, max_res, budget_bytes=256 << 20):
# [T, 3, H, W] on the CPU -> [H, W, 3] uint8 arrays at the preview size. One antialiased tensor
# resize per chunk replaces a LANCZOS pass per frame in PIL, which dominated the preview latency
# on long clips; the chunk follows a byte budget so peak RAM stays flat across resolutions
h, w = frames.shape[-2:]
tw, th = _contain_size(w, h, max_res)
resize = (tw, th) != (w, h)
chunk = max(1, budget_bytes // (3 * h * w * 4))
out = []
for i in range(0, frames.shape[0], chunk):
x = frames[i:i + chunk]
if x.dtype != torch.uint8 or resize:
# copy: the batch may be an inference tensor from the sampler thread, in-place would raise
x = x.to(torch.float32, copy=True)
if frames.dtype == torch.uint8:
x.div_(255)
if resize:
x = torch.nn.functional.interpolate(x, size=(th, tw), mode="bilinear", antialias=True, align_corners=False)
x = x.clamp_(0, 1).mul_(255).to(torch.uint8)
u8 = x.permute(0, 2, 3, 1).contiguous().numpy()
out.extend(u8[j] for j in range(u8.shape[0]))
return out
def _tiny_vae_decode_frames(decoder, x0, max_frames=None, compile_preview=False):
# Raises on failure so the caller can disable the decoder instead of retrying every step.
# Returns the whole [T, 3, H, W] clip on the CPU (uint8 from the 2D decoder, model dtype from
# TAEHV) or None. The compiler treats GPU allocations it did not plan as rogue and pays for them
# every step, so the decode joins the sampler thread's allocation graph and every GPU tensor is
# gone before the scope closes. Needs a core with per-thread graphs; older cores skip it.
if x0.ndim not in (4, 5):
return None, None
compiled = compile_preview and comfy.model_prefetch is not None and comfy.model_prefetch.malloc_graph_enabled(x0.device)
if compiled:
comfy.model_prefetch.malloc_graph_begin(x0.device)
try:
span = None
if x0.ndim == 4:
frames = decoder.decode(x0[:1])
else:
indices = list(range(x0.shape[2]))
if max_frames is not None and 0 < max_frames < len(indices):
picks = np.linspace(0, len(indices) - 1, max_frames).round().astype(int).tolist()
indices = [indices[i] for i in picks]
frames = decoder.decode_video(x0[:1], frame_indices=indices)
# the temporal decoder chains memblock state, so a partial request decodes the prefix instead
prefix = getattr(decoder, "decodes_prefix", False) and len(indices) < x0.shape[2]
span = (0, len(indices) - 1) if prefix else (indices[0], indices[-1])
frames = None if frames is None or frames.shape[0] == 0 else frames.cpu()
finally:
if compiled:
comfy.model_prefetch.malloc_graph_end()
return frames, span
def _materialize_frames(frames, max_res=0):
# a tensor batch comes from the tiny VAE and is converted here on the encoder thread into
# [H, W, 3] uint8 arrays; the other previewers already hand over PIL
if isinstance(frames, torch.Tensor):
return _frames_to_arrays(frames, max_res)
return list(frames)
def _as_pil(frame):
return frame if isinstance(frame, Image.Image) else Image.fromarray(frame)
def _tiny_vae_decode_to_pil(decoder, x0, max_frames=None, compile_preview=False):
frames, _ = _tiny_vae_decode_frames(decoder, x0, max_frames, compile_preview)
return [] if frames is None else [_as_pil(a) for a in _frames_to_arrays(frames, 0)]
def _is_ltx_latent_format(latent_format):
return "LTX" in type(latent_format).__name__
def _is_ltx2_diffusion_model(model_patcher):
# Same probe as ltxv_nodes.OuterSampleCallbackWrapper.
try:
dm = model_patcher.model.diffusion_model
return not getattr(dm, "caption_projection_first_linear", True)
except Exception:
return False
def _ltx_num_keyframes(guider):
try:
positive = guider.conds.get("positive") if hasattr(guider, "conds") else None
if positive and len(positive) > 0:
kf = positive[0].get("keyframe_idxs")
if kf is not None:
return int(torch.unique(kf[0, 0, :, 0]).numel())
except Exception:
pass
return 0
def _is_av_model(model):
# core marks audio+video models by model type; other packed latents (e.g. shape + camera) are not audio
flow_av = getattr(comfy.model_base.ModelType, "FLOW_AV", None)
return flow_av is not None and getattr(model, "model_type", None) == flow_av
def _packed_audio_latent(x0, latent_shapes):
# AV models pack every stream into one [B, 1, N] tensor; the audio stream is the last entry, as
# core's vae_decode_audio picks it
if x0.ndim != 3 or not latent_shapes or len(latent_shapes) < 2:
return None
try:
return comfy.utils.unpack_latents(x0, latent_shapes)[-1]
except Exception:
return None
def _frames_per_token(latent_format):
# pixel frames each latent token stands for, cycled: MiniMax H3 codes 17 frames per 5 tokens,
# the causal VAEs one frame then temporal_downscale_ratio per token
if type(latent_format).__name__.startswith("MiniMaxH3"):
return (1, 4, 4, 4, 4)
return (1, max(1, int(getattr(latent_format, "temporal_downscale_ratio", 1))))
def _token_span_fraction(span, n_tokens, pattern):
# fraction of the clip covered by latent tokens [first, last]; the audio latent spans the whole clip
if span is None or n_tokens <= 1:
return 0.0, 1.0
def frames(k):
if len(pattern) == 2:
return 0 if k <= 0 else 1 + (k - 1) * pattern[1]
return sum(pattern[i % len(pattern)] for i in range(k))
first, last = span
total = frames(n_tokens)
return frames(first) / total, min(1.0, frames(last + 1) / total)
_SPEC_LUT = None
_HANN = {}
def _spec_colormap():
# black -> purple -> orange -> yellow -> white, 256 entries
global _SPEC_LUT
if _SPEC_LUT is None:
anchors = np.array([[0, 0, 0], [60, 10, 90], [180, 50, 60], [240, 130, 30], [250, 210, 60], [255, 255, 230]], dtype=np.float32)
pos = np.linspace(0, 1, len(anchors))
x = np.linspace(0, 1, 256)
_SPEC_LUT = np.stack([np.interp(x, pos, anchors[:, c]) for c in range(3)], axis=1).astype(np.uint8)
return _SPEC_LUT
def _strip_to_png(mag, width=512, height=64):
# mag [rows, cols] >= 0, low rows = low frequency; drawn bottom-up, normalized per strip
mag = np.asarray(mag, dtype=np.float32)
if mag.ndim != 2 or mag.size == 0:
return None
hi = float(np.percentile(mag, 99.5))
norm = np.clip(mag / hi, 0, 1) ** 0.6 if hi > 0 else np.zeros_like(mag)
img = Image.fromarray(_spec_colormap()[(norm[::-1] * 255).astype(np.uint8)], "RGB")
img = img.resize((width, height), Image.BILINEAR)
buf = pyio.BytesIO()
img.save(buf, format="PNG", optimize=False)
return base64.b64encode(buf.getvalue()).decode("ascii")
def _wave_strip(wave, sample_rate, bands=64):
# [2, L] -> log-magnitude STFT folded into log-spaced bands
x = torch.as_tensor(wave, dtype=torch.float32).mean(0)
n_fft = 1024
hop = max(256, x.shape[0] // 800)
window = _HANN.get(n_fft)
if window is None:
window = _HANN[n_fft] = torch.hann_window(n_fft)
spec = torch.stft(x, n_fft, hop_length=hop, window=window, return_complex=True).abs()
edges = np.unique(np.geomspace(1, spec.shape[0] - 1, bands + 1).round().astype(int))
rows = [spec[edges[i]:max(edges[i + 1], edges[i] + 1)].mean(0) for i in range(len(edges) - 1)]
mag = torch.log1p(torch.stack(rows) * 10).numpy()
return _strip_to_png(mag)
def _normalize_packed_x0(x0, latent_shapes, num_keyframes):
# Restore standard video latents from flattened packs
if latent_shapes and len(latent_shapes) > 0:
target = latent_shapes[0]
if x0.ndim == 3 and len(target) >= 3:
cut = 1
for d in target[1:]:
cut *= int(d)
x0 = x0[:, :, :cut].reshape([x0.shape[0]] + list(target)[1:])
if num_keyframes > 0 and x0.ndim == 5:
x0 = x0[:, :, :-num_keyframes]
return x0
class _PreviewOverrideWrapper:
def __init__(self, max_resolution, node_id, jpeg_quality, suppress_default, preview_frames=1, preview_fps=12, vae=None, tiny_vae="none", audio_vae=None):
self.max_resolution = max_resolution
self.node_id = str(node_id) if node_id is not None else None
self.jpeg_quality = jpeg_quality
self.suppress_default = suppress_default
self.preview_frames = preview_frames
self.preview_fps = preview_fps
self.vae = vae
self.tiny_vae = tiny_vae
self.audio_vae = audio_vae
self.frames = []
def __call__(self, executor, noise, latent_image, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes):
guider = executor.class_obj
model_patcher = guider.model_patcher
is_ltx = _is_ltx_latent_format(model_patcher.model.latent_format)
is_ltx2 = is_ltx and _is_ltx2_diffusion_model(model_patcher)
num_keyframes = _ltx_num_keyframes(guider) if is_ltx else 0
# Tiny VAE from models/vae_approx
tiny_vae = None
if self.tiny_vae and self.tiny_vae != "none" and load_tiny_vae_decoder is not None:
tiny_vae = load_tiny_vae_decoder(self.tiny_vae)
if tiny_vae is not None and latent_shapes and len(latent_shapes[0]) >= 2:
channels = int(latent_shapes[0][1])
if channels != tiny_vae.latent_channels:
logging.warning(
f"[KJ PreviewOverride] '{self.tiny_vae}' decodes {tiny_vae.latent_channels}-channel "
f"latents but this model's are {channels}-channel; ignoring it."
)
tiny_vae = None
# LTX reuses the LTX-specific node's WrappedPreviewer; we call decode_latent_to_preview
# directly per step, bypassing its decode_latent_to_preview_image rate-limiting.
# If a non-TAEHV VAE is supplied, decode via vae.decode() for full quality (slower).
ltx_previewer = None
ltx_full_vae = None
vae_restore_device = None
if is_ltx:
try:
factors, bias = _ltx_rgb_factors(is_ltx2)
taeltx = None
if self.vae is not None:
if self.vae.first_stage_model.__class__.__name__ == "TAEHV":
# TAEHV-LTX decode needs the VAE on GPU; restored at end of __call__.
target_device = comfy.model_management.get_torch_device()
try:
for p in self.vae.first_stage_model.parameters():
vae_restore_device = p.device
break
self.vae.first_stage_model.to(target_device)
taeltx = self.vae
except Exception as e:
logging.warning(f"[KJ PreviewOverride] Could not move TAEHV-LTX to GPU, skipping: {e}")
else:
# Comfy VAE.decode manages its own device — no pin-to-GPU needed.
ltx_full_vae = self.vae
ltx_previewer = _LTXWrappedPreviewer(factors, bias, rate=8, taeltx=taeltx)
except Exception as e:
logging.warning(f"[KJ PreviewOverride] LTX previewer setup failed: {e}")
previewer = _get_core_previewer(model_patcher.load_device, model_patcher.model.latent_format)
# Latent2RGB fallback — used when the active previewer returns a non-PIL result
# (e.g. TAEHV/TAESD on a 5D latent). LTX skips this and goes through ltx_previewer.
fallback_previewer = None
try:
lf = model_patcher.model.latent_format
rgb_factors = getattr(lf, "latent_rgb_factors", None)
if rgb_factors is not None:
fallback_previewer = latent_preview.Latent2RGBPreviewer(
rgb_factors,
getattr(lf, "latent_rgb_factors_bias", None),
getattr(lf, "latent_rgb_factors_reshape", None),
)
except Exception:
pass
original_callback = callback
node_id = self.node_id
max_res = self.max_resolution
quality = self.jpeg_quality
self.frames = []
# N+1 boundaries for N steps: keep them all so the step marker advances through each.
sigmas_list = sigmas.detach().cpu().tolist() if sigmas is not None else []
# Pre-seed so step 1 has a measurable Δ (model's first transformation from noise → x0).
compile_preview = getattr(model_patcher.model.latent_format, "compile_preview", False)
initial_seed_cpu = None
try:
if sigmas is not None and len(sigmas) > 0:
# sigmas often lives on CPU while noise is on CUDA — align before the multiply.
s0 = sigmas[0].to(noise.device) if hasattr(sigmas[0], "to") else sigmas[0]
seeded = _normalize_packed_x0(noise * s0, latent_shapes, num_keyframes)
initial_seed_cpu = seeded.detach().float().cpu()
except Exception as e:
logging.warning(f"[KJ PreviewOverride] initial seed Δ pre-fill failed: {e}")
state = {"last_x0_cpu": initial_seed_cpu, "last_time": None, "step_ms_window": []}
total_steps_init = max(0, len(sigmas_list) - 1)
# Boundary-0 message: sigmas (required by JS hover handler) plus optional noise preview.
if node_id is not None and PromptServer is not None:
init_payload = {
"node_id": node_id,
"step": 0,
"total": total_steps_init,
"sigma": sigmas_list[0] if sigmas_list else None,
"sigmas": sigmas_list,
}
db_curve = _detect_detail_boost_curve(sampler, model_patcher, sigmas_list)
if db_curve is not None:
init_payload["db_curve"] = db_curve
# Use Latent2RGB (or LTX previewer) directly — the model's default previewer (TAEHV)
# slices to one temporal frame and returns a shape PIL can't render on raw noise.
try:
lf = model_patcher.model.latent_format
rgb_factors = getattr(lf, "latent_rgb_factors", None)
if sigmas is not None and len(sigmas) > 0:
s0 = sigmas[0].to(noise.device) if hasattr(sigmas[0], "to") else sigmas[0]
init_latent = noise * s0
else:
init_latent = noise
init_latent = _normalize_packed_x0(init_latent, latent_shapes, num_keyframes)
pil_init = None
if tiny_vae is not None:
pil_frames = _tiny_vae_decode_to_pil(tiny_vae, init_latent, max_frames=1, compile_preview=compile_preview)
pil_init = pil_frames[0] if pil_frames else None
elif ltx_previewer is not None and init_latent.ndim == 5:
pil_frames = _ltx_decode_to_pil(ltx_previewer, init_latent, max_frames=1)
pil_init = pil_frames[0] if pil_frames else None
elif rgb_factors is not None:
noise_previewer = latent_preview.Latent2RGBPreviewer(
rgb_factors,
getattr(lf, "latent_rgb_factors_bias", None),
getattr(lf, "latent_rgb_factors_reshape", None),
)
out = noise_previewer.decode_latent_to_preview(init_latent)
if isinstance(out, Image.Image):
pil_init = out
if pil_init is not None:
if pil_init.mode != "RGB":
pil_init = pil_init.convert("RGB")
if max_res and max_res > 0 and (pil_init.width > max_res or pil_init.height > max_res):
pil_init = ImageOps.contain(pil_init, (max_res, max_res), Image.LANCZOS)
ibuf = pyio.BytesIO()
pil_init.save(ibuf, format="JPEG", quality=quality)
init_payload["image"] = base64.b64encode(ibuf.getvalue()).decode("ascii")
init_payload["w"] = pil_init.width
init_payload["h"] = pil_init.height
except Exception as e:
logging.warning(f"Initial noise preview failed (sigmas still sent): {e}")
PromptServer.instance.send_sync("moxie_preview_override", init_payload, PromptServer.instance.client_id)
encoder = _AsyncPreviewEncoder()
animate_video = self.preview_frames > 1
anim_frames = self.preview_frames
anim_fps = self.preview_fps
# audio: with an audio VAE and an animated preview the audio latent is decoded each step,
# muxed into the MP4 in real time, and its spectrogram drawn under the graphs
has_audio_latent = _is_av_model(model_patcher.model) and _packed_audio_latent(noise, latent_shapes) is not None
audio_vae = self.audio_vae if (has_audio_latent and animate_video) else None
audio_rate = 0
frame_pattern = _frames_per_token(model_patcher.model.latent_format)
# the sampler carries the audio stream scaled by audio_scale and only process_latent_out undoes it
audio_scale = getattr(model_patcher.model, "audio_scale", None)
audio_scale = float(audio_scale()) if callable(audio_scale) else 1.0
if audio_vae is not None:
# core's vae_decode_audio prefers the output rate; the LTX audio VAE only sets that one
audio_rate = int(getattr(audio_vae, "audio_sample_rate_output", None) or getattr(audio_vae, "audio_sample_rate", 0) or 0)
if audio_rate <= 0:
logging.warning("[KJ PreviewOverride] audio_vae has no sample rate; audio preview disabled")
audio_vae = None
if audio_vae is not None:
# VAE.decode runs model management on every call (gc sweep, cache flush); load once here and
# go through the raw model per step instead
try:
audio_shape = tuple(_packed_audio_latent(noise, latent_shapes).shape)
comfy.model_management.load_models_gpu(
[audio_vae.patcher], memory_required=audio_vae.memory_used_decode(audio_shape, audio_vae.vae_dtype),
force_full_load=getattr(audio_vae, "disable_offload", False))
except Exception as e:
logging.warning(f"[KJ PreviewOverride] could not load audio_vae, audio preview disabled: {e}")
audio_vae = None
def decode_audio(audio_latent):
# mirrors VAE.decode minus the batching: [B, 32, 2, T] -> [B, 2, L] -> [2, L] float32 CPU, with
# core's VAEDecodeAudio level trim
z = (audio_latent / audio_scale).to(device=audio_vae.device, dtype=audio_vae.vae_dtype)
wave = audio_vae.process_output(audio_vae.first_stage_model.decode(z))
wave = wave[:1].float().cpu()
std = torch.std(wave, dim=[1, 2], keepdim=True) * 5.0
std[std < 1.0] = 1.0
return (wave / std)[0].clamp_(-1, 1).numpy()
def produce_frames(x0_view):
# (frames, span): a [T, 3, H, W] CPU tensor from the tiny VAE, else a PIL list from
# whichever previewer applies, else []; span = latent tokens (first, last) the frames cover
nonlocal tiny_vae
max_pil = anim_frames if animate_video else 1
if tiny_vae is not None:
try:
frames, span = _tiny_vae_decode_frames(tiny_vae, x0_view, max_frames=max_pil, compile_preview=compile_preview)
if frames is not None:
return frames, span
except Exception as e:
# OOM at 16x upscale is the likely cause — drop to the cheap paths for good.
logging.warning(f"[KJ PreviewOverride] tiny VAE decode failed, falling back: {e}")
tiny_vae = None
pil_frames = []
if ltx_full_vae is not None and x0_view.ndim == 5:
pil_frames = _ltx_full_vae_decode_to_pil(ltx_full_vae, x0_view, max_frames=max_pil)
if not pil_frames and ltx_previewer is not None and x0_view.ndim == 5:
try:
pil_frames = _ltx_decode_to_pil(ltx_previewer, x0_view, max_frames=max_pil)
except Exception as e:
logging.warning(f"LTX preview decode failed: {e}")
if not pil_frames and animate_video and x0_view.ndim == 5 and ltx_previewer is None:
pil_frames = _decode_video_frames_l2rgb(
x0_view, model_patcher.model.latent_format, anim_frames,
)
if not pil_frames:
for prev in (previewer, fallback_previewer):
if prev is None:
continue
try:
out = prev.decode_latent_to_preview(x0_view)
except Exception as e:
if prev is previewer:
logging.warning(f"Active previewer raised, trying Latent2RGB fallback: {e}")
continue
if isinstance(out, Image.Image):
pil_frames = [out]
break
elif prev is previewer:
logging.warning(
f"Preview override: {type(previewer).__name__} returned "
f"{type(out).__name__} instead of PIL.Image — falling back to Latent2RGB."
)
return pil_frames, None
def new_callback(step, x0, x, total_steps_):
if previewer is not None or fallback_previewer is not None or ltx_previewer is not None or tiny_vae is not None:
try:
# NEVER rebind x0 — the sampler reuses the same tensor downstream
# (unpack_latents reshapes it). Preview mutations stay on x0_view.
x0_view = _normalize_packed_x0(x0, latent_shapes, num_keyframes)
frames, span = produce_frames(x0_view)
if isinstance(frames, torch.Tensor):
pil_first = _frame_to_pil(frames[0])
elif frames:
pil_first = frames[0]
if pil_first.mode != "RGB":
pil_first = pil_first.convert("RGB")
frames[0] = pil_first
else:
if original_callback is not None:
original_callback(step, x0, x, total_steps_)
return
# Consumed by GetPreviewOverrideFramesKJ.
self.frames.append(pil_first)
if node_id is not None and PromptServer is not None:
# x0_view (not x0) so LTX keyframe padding doesn't dampen the Δ norm; copied
# off the GPU before the float cast so nothing is allocated on the device
x0_cpu_now = x0_view.detach().cpu().float()
prev_x0_cpu = state["last_x0_cpu"]
state["last_x0_cpu"] = x0_cpu_now
audio_wave = None
n_tokens = int(x0_view.shape[2]) if x0_view.ndim == 5 else 1
if audio_vae is not None:
audio_latent = _packed_audio_latent(x0, latent_shapes)
if audio_latent is not None:
try:
audio_wave = decode_audio(audio_latent)
except Exception as e:
logging.warning(f"[KJ PreviewOverride] audio decode failed: {e}")
now = time.perf_counter()
step_ms = None
if state["last_time"] is not None:
step_ms = (now - state["last_time"]) * 1000.0
w = state["step_ms_window"]
w.append(step_ms)
if len(w) > 8:
w.pop(0)
state["last_time"] = now
avg_step_ms = (sum(state["step_ms_window"]) / len(state["step_ms_window"])) if state["step_ms_window"] else None
sigma_val = sigmas_list[step] if 0 <= step < len(sigmas_list) else None
sent_step = step + 1
def _encode_and_send(
frames=frames, span=span, x0_cpu_now=x0_cpu_now, prev_x0_cpu=prev_x0_cpu,
audio_wave=audio_wave, n_tokens=n_tokens,
step_ms=step_ms, avg_step_ms=avg_step_ms, sigma_val=sigma_val,
sent_step=sent_step, total_steps_=total_steps_,
):
frames = _materialize_frames(frames, max_res)
frac = _token_span_fraction(span, n_tokens, frame_pattern)
audio = None
fps_out = anim_fps
if audio_wave is not None and len(frames) > 1:
total = audio_wave.shape[1]
s0 = int(frac[0] * total)
s1 = max(int(frac[1] * total), s0 + audio_rate // 10)
clip = audio_wave[:, s0:s1]
if clip.shape[1] > 0:
audio = (clip, audio_rate)
fps_out = fractions.Fraction(len(frames) * audio_rate, clip.shape[1]).limit_denominator(1000)
spec_b64 = None
if audio is not None:
try:
spec_b64 = _wave_strip(audio[0], audio_rate)
except Exception as e:
logging.warning(f"[KJ PreviewOverride] spectrogram failed: {e}")
if len(frames) > 1:
# MP4 is far faster and smaller than PIL WebP when an encoder is available.
b64, w_, h_, mime = None, 0, 0, None
if _MP4_AVAILABLE:
b64, w_, h_ = _encode_mp4(frames, fps_out, max_res, audio=audio)
if b64:
mime = "video/mp4"
if not b64:
audio = None
fps_out = anim_fps
b64, w_, h_ = _encode_animated_webp(frames, anim_fps, quality, max_res)
mime = "image/webp"
else:
pil_send = _as_pil(frames[0])
if max_res and max_res > 0 and (pil_send.width > max_res or pil_send.height > max_res):
pil_send = ImageOps.contain(pil_send, (max_res, max_res), Image.LANCZOS)
buf = pyio.BytesIO()
pil_send.save(buf, format="JPEG", quality=quality)
b64 = base64.b64encode(buf.getvalue()).decode("ascii")
w_, h_ = pil_send.width, pil_send.height
mime = "image/jpeg"
if not b64:
return
delta_v = None
if prev_x0_cpu is not None and prev_x0_cpu.shape == x0_cpu_now.shape:
diff = x0_cpu_now - prev_x0_cpu
delta_v = (diff.norm() / max(1, diff.numel()) ** 0.5).item()
PromptServer.instance.send_sync(
"moxie_preview_override",
{
"node_id": node_id,
"image": b64,
"mime": mime,
"w": w_,
"h": h_,
"step": sent_step,
"total": total_steps_,
"sigma": sigma_val,
"sigmas": None,
"delta": delta_v,
"step_ms": step_ms,
"avg_step_ms": avg_step_ms,
"fps": float(fps_out) if mime in ("video/mp4", "image/webp") else None,
"audio": audio is not None,
"audio_spec": spec_b64,
},
PromptServer.instance.client_id,
)
encoder.submit(_encode_and_send)
except Exception as e:
logging.warning(f"Preview override failed: {e}")
if original_callback is not None:
original_callback(step, x0, x, total_steps_)
# Patch every concrete decode_latent_to_preview_image — subclasses like VHS's
# WrappedPreviewer override it and would otherwise still emit previews of their own.
prev_methods = []
if self.suppress_default:
targets = [latent_preview.LatentPreviewer]
stack = list(latent_preview.LatentPreviewer.__subclasses__())
while stack:
cls = stack.pop()
targets.append(cls)
stack.extend(cls.__subclasses__())
for cls in targets:
if "decode_latent_to_preview_image" in cls.__dict__:
prev_methods.append((cls, cls.__dict__["decode_latent_to_preview_image"]))
cls.decode_latent_to_preview_image = _suppressed_preview_image
try:
# Seeds step 1's duration measurement (sampling-start → end of step 1).
state["last_time"] = time.perf_counter()
return executor(noise, latent_image, sampler, sigmas, denoise_mask, new_callback, disable_pbar, seed, latent_shapes=latent_shapes)
finally:
encoder.shutdown(drain_timeout=5.0)
for cls, prev in prev_methods:
cls.decode_latent_to_preview_image = prev
if torch.cuda.is_available():
# the decode grows torch's pool and the pool keeps it; under dynamic VRAM that reservation
# displaces weight pages for every later prompt, so hand it back before returning
torch.cuda.empty_cache()
if vae_restore_device is not None and self.vae is not None:
try:
self.vae.first_stage_model.to(vae_restore_device)
except Exception:
pass
class ModelPreviewOverrideKJ(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="MoxiePreviewOverride",
display_name="Moxie Preview Override",
category="sampling/moxie",
description=(
"Attaches the live-preview wrapper to the model produced by the "
"Moxie Multimedia Loader. "
"Sends high-res preview frames (and audio) to the node while sampling."
),
inputs=[
io.Model.Input(
"model",
tooltip="Diffusion model (with attention backend and LoRA already applied).",
),
io.Int.Input(
"max_resolution",
default=1024,
min=0,
max=8192,
step=8,
advanced=True,
tooltip="Max preview side in pixels for the live widget. 0 = full sampler resolution (no downscale).",
),
io.Int.Input(
"jpeg_quality",
default=80,
min=30,
max=100,
step=1,
advanced=True,
tooltip="JPEG quality for the live preview transport.",
),
io.Boolean.Input(
"suppress_default_preview",
default=True,
advanced=True,
tooltip="Suppress the standard sampler-node preview overlay while sampling, so only this node's frame updates.",
),
io.Int.Input(
"preview_frames",
default=48,
min=1,
max=1024,
step=1,
tooltip="Frames to sample from each video step's latent for animated preview. "
"1 = single frame (fastest). >1 = animated WebP at preview_fps (video models only).",
),
io.Int.Input(
"preview_fps",
default=24,
min=1,
max=60,
step=1,
tooltip="Playback FPS for the animated preview. Ignored when preview_frames=1, and when an "
"audio_vae is connected (the clip then plays in real time with its audio).",
),
io.Combo.Input(
"tiny_vae",
options=["none"] + folder_paths.get_filename_list("vae_approx"),
default="taeh3.safetensors",
tooltip="Tiny VAE decoder from models/vae_approx for true-RGB previews (e.g. taeh3.safetensors).",
),
],
outputs=[
io.Model.Output(
"model",
tooltip="Model with preview wrapper attached.",
),
],
hidden=[io.Hidden.unique_id],
is_experimental=True,
)
@classmethod
def execute(
cls,
model=None,
max_resolution=1024,
jpeg_quality=80,
suppress_default_preview=True,
preview_frames=48,
preview_fps=24,
tiny_vae="taeh3.safetensors",
) -> io.NodeOutput:
if model is None:
raise TypeError("Moxie Preview Override: 'model' input is required.")
patched_model = model.clone()
patched_model.add_wrapper_with_key(
comfy.patcher_extension.WrappersMP.OUTER_SAMPLE,
"moxie_preview_override",
_PreviewOverrideWrapper(
max_resolution, cls.hidden.unique_id, jpeg_quality, suppress_default_preview,
preview_frames, preview_fps, None, tiny_vae, None,
),
)
return io.NodeOutput(patched_model)
class GetPreviewOverrideFramesKJ(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="GetPreviewOverrideFramesKJ",
display_name="Get Preview Override Frames",
category="KJNodes/sampling",
description=(
"Returns the frames captured by Model Preview Override during the most recent sampling. "
"Wire 'model' from Model Preview Override (the same one feeding the sampler) and 'after_sample' "
"from after the sampler (LATENT/IMAGE) to enforce correct execution order."
),
inputs=[
io.Model.Input("model", tooltip="The model output by Model Preview Override (used to locate the captured frames)."),
io.MultiType.Input(
"after_sample",
[io.Latent, io.Image],
tooltip="Anything from after the sampler (LATENT or IMAGE). The value is ignored — it just forces this node to run after sampling.",
),
],
outputs=[io.Image.Output(display_name="frames")],
is_experimental=True,
)
@classmethod
def execute(cls, model, after_sample) -> io.NodeOutput:
wrappers = model.get_wrappers(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, "moxie_preview_override")
if not wrappers:
raise RuntimeError("Get Preview Override Frames: no Model Preview Override wrapper found on this model.")
frames = wrappers[-1].frames
if not frames:
raise RuntimeError("Get Preview Override Frames: no frames captured. Ensure the sampler ran with this model.")
tensors = []
for pil in frames:
arr = np.asarray(pil, dtype=np.float32) / 255.0
tensors.append(torch.from_numpy(arr))
return io.NodeOutput(torch.stack(tensors, dim=0))