MMnode/nodes/ltxv.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

296 lines
13 KiB
Python

from comfy_extras.nodes_lt import get_noise_mask, LTXVAddGuide, _append_guide_attention_entry, preprocess
import types
import math
from typing import Tuple
import comfy
from comfy_api.latest import io
import numpy as np
import torch
import logging
import comfy.model_management as mm
import comfy.ldm.modules.attention as _comfy_attn
from comfy.ldm.lightricks.model import apply_rotary_emb as _apply_rope
try:
from comfy.ldm.lightricks.model import GuideAttentionMask as _GuideAttentionMask, _attention_with_guide_mask as _ltx_attn_with_guide_mask
except ImportError:
_GuideAttentionMask = None
_ltx_attn_with_guide_mask = None
device = mm.get_torch_device()
import latent_preview
# code based on https://github.com/kijai/ComfyUI-KJNodes/blob/main/nodes/ltxv_nodes.py#L21
class LTXVAddGuidesFromBatchIndexes(LTXVAddGuide):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="LTXVAddGuidesFromBatchIndexes",
category="conditioning/ltxv",
description="Adds guide images or an image sequence from a batch to the latent at specified frame indices. Batches with 9 or more images are treated as one image sequence.",
inputs=[
io.Conditioning.Input("positive"),
io.Conditioning.Input("negative"),
io.Vae.Input("vae"),
io.Latent.Input("latent"),
io.Image.Input("images", tooltip="Batch of images. Batches with 9 or more images are treated as one image sequence."),
io.Int.Input(
id="img_compression", default=18, min=0, max=100, tooltip="Amount of compression to apply on image."
),
io.String.Input("image_indexes", default="", tooltip="Comma-separated frame indices, e.g. '0,61,121'. For image sequences, the first index is used as the sequence start frame."),
io.Float.Input("strength", default=1.0, min=0.0, max=1.0, step=0.01),
],
outputs=[
io.Conditioning.Output(display_name="positive"),
io.Conditioning.Output(display_name="negative"),
io.Latent.Output(display_name="latent"),
],
)
@classmethod
def execute(cls, positive, negative, vae, latent, images, img_compression, image_indexes, strength) -> io.NodeOutput:
scale_factors = vae.downscale_index_formula
latent_image = latent["samples"]
noise_mask = get_noise_mask(latent)
_, _, latent_length, latent_height, latent_width = latent_image.shape
parsed_indexes = [int(x.strip()) for x in image_indexes.split(",") if x.strip()]
# If no indexes provided, calculate evenly distributed frame indices
if not parsed_indexes:
# Find valid (non-black) images first
valid_count = 0
for i in range(images.shape[0]):
if images[i].max() > 0.001:
valid_count += 1
if valid_count > 0:
# Evenly distribute from 0 to latent_length-1
parsed_indexes = np.linspace(0, latent_length - 1, valid_count, dtype=int).tolist()
# Process each image in the batch
if img_compression > 0:
output_images = []
for i in range(images.shape[0]):
output_images.append(preprocess(images[i], img_compression))
batch_size = images.shape[0]
if batch_size >= 9:
if images.max() > 0.001:
f_idx = parsed_indexes[0] if parsed_indexes else 0
image_1, t = cls.encode(vae, latent_width, latent_height, images, scale_factors)
frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image_1), f_idx, scale_factors)
if latent_idx + t.shape[2] <= latent_length:
positive, negative, latent_image, noise_mask = cls.append_keyframe(
positive,
negative,
frame_idx,
latent_image,
noise_mask,
t,
strength,
scale_factors,
)
# Track this guide sequence for per-reference attention control.
pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4]
guide_latent_shape = list(t.shape[2:]) # [F, H, W]
positive, negative = _append_guide_attention_entry(positive, negative, pre_filter_count, guide_latent_shape, strength=strength)
else:
logging.warning("Skipping guide sequence - conditioning frames exceed latent sequence length")
return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})
for i in range(batch_size):
if i >= len(parsed_indexes):
break
img = images[i:i+1]
# If strength is a list/tuple, use corresponding value for this image, otherwise use the single strength value for all images
if isinstance(strength, (list, tuple)):
strength = strength[i] if i < len(strength) else strength[-1]
# Check if image is not black and use provided frame index
if img.max() > 0.001:
f_idx = parsed_indexes[i]
image_1, t = cls.encode(vae, latent_width, latent_height, img, scale_factors)
frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image_1), f_idx, scale_factors)
if latent_idx + t.shape[2] <= latent_length:
positive, negative, latent_image, noise_mask = cls.append_keyframe(
positive,
negative,
frame_idx,
latent_image,
noise_mask,
t,
strength,
scale_factors,
)
# Track this guide for per-reference attention control.
pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4]
guide_latent_shape = list(t.shape[2:]) # [F, H, W]
positive, negative = _append_guide_attention_entry(positive, negative, pre_filter_count, guide_latent_shape, strength=strength)
else:
logging.warning("Skipping guide at index %s - conditioning frames exceed latent sequence length", i)
return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})
# code based on https://github.com/liconstudio/ComfyUI-Licon-MSR/blob/main/licon_msr.py
class LTXVMakeRefVideo(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="LTXVMakeRefVideo",
display_name="LTXVMakeRefVideo",
category="image/ltxv",
description="Expands a batch of reference images into a reference video for IC-LoRA.",
is_input_list=True,
inputs=[
io.Image.Input("images"),
io.Int.Input("frame_count", default=17, min=17, step=8),
],
outputs=[
io.Image.Output("IMAGE"),
],
)
@classmethod
def execute(cls, images: list[torch.Tensor], frame_count: list[int]) -> io.NodeOutput:
try:
image_batch = cls._prepare_image_batch(images)
if not frame_count:
raise ValueError("frame_count must contain one value")
frames = cls._expand_frames(image_batch, int(frame_count[0]))
except Exception as exc:
raise RuntimeError("Failed to create IC-LoRA reference video frames") from exc
return io.NodeOutput(frames)
@staticmethod
def _prepare_image_batch(images: list[torch.Tensor]) -> torch.Tensor:
if not images:
raise ValueError("images must contain at least one image tensor")
if not all(isinstance(image, torch.Tensor) for image in images):
raise TypeError("images must contain only torch.Tensor values")
if any(image.ndim != 4 or image.shape[0] < 1 for image in images):
raise ValueError("Each image input must have shape [B, H, W, C] with B >= 1")
if len(images) == 1:
return images[0]
individual_images = [frame for batch in images for frame in batch.split(1, dim=0)]
background = individual_images[-1]
target_height, target_width, target_channels = background.shape[1:]
prepared = []
for image in individual_images[:-1]:
if image.shape[-1] != target_channels:
raise ValueError("All image inputs must have the same channel count as the background")
image = image.to(device=background.device, dtype=background.dtype)
source_height, source_width = image.shape[1:3]
if (source_height, source_width) == (target_height, target_width):
prepared.append(image)
continue
scale = min(target_width / source_width, target_height / source_height)
resized_width = max(1, min(target_width, round(source_width * scale)))
resized_height = max(1, min(target_height, round(source_height * scale)))
resized = torch.nn.functional.interpolate(
image.movedim(-1, 1),
size=(resized_height, resized_width),
mode="bilinear",
align_corners=False,
).movedim(1, -1)
canvas = torch.ones(
(1, target_height, target_width, target_channels),
device=background.device,
dtype=background.dtype,
)
top = (target_height - resized_height) // 2
left = (target_width - resized_width) // 2
canvas[:, top:top + resized_height, left:left + resized_width] = resized
prepared.append(canvas)
prepared.append(background)
return torch.cat(prepared, dim=0)
@staticmethod
def _expand_frames(images: torch.Tensor, frame_count: int) -> torch.Tensor:
if images.ndim != 4:
raise ValueError("images must be an IMAGE batch tensor with shape [B, H, W, C]")
image_count = images.shape[0]
if image_count <= 0:
raise ValueError("images batch must contain at least one image")
background = images[-1:]
frames = background.repeat(frame_count, 1, 1, 1)
subjects = images[:-1]
if subjects.shape[0] == 0:
return frames
latent_count = max(1, round((frame_count - 1) / 8) + 1)
subject_budget = max(0, latent_count - 1)
subject_count = subjects.shape[0]
if subject_budget >= subject_count:
counts = LTXVMakeRefVideo._allocate_subject_latent_counts(subject_count, subject_budget)
cursor = 0
for image, count in zip(subjects, counts):
start, end = LTXVMakeRefVideo._latent_to_frame_range(cursor, cursor + count - 1)
cursor += count
frames[max(0, start):min(frame_count - 1, end) + 1] = image
else:
_, subject_end = LTXVMakeRefVideo._latent_to_frame_range(0, max(0, latent_count - 2))
subject_frame_count = max(1, subject_end + 1)
for index, image in enumerate(subjects):
start = int(index * subject_frame_count / subject_count)
end = int((index + 1) * subject_frame_count / subject_count)
frames[start:min(frame_count, max(start + 1, end))] = image
return frames
@staticmethod
def _latent_to_frame_range(latent_start: int, latent_end: int) -> tuple[int, int]:
frame_start = 0 if latent_start <= 0 else 1 + (latent_start - 1) * 8
frame_end = 0 if latent_end <= 0 else latent_end * 8
return frame_start, frame_end
@staticmethod
def _allocate_subject_latent_counts(subject_count: int, subject_budget: int) -> list[int]:
counts = [1] * subject_count
extra = max(0, subject_budget - subject_count)
if extra > 0:
counts[0] += 1
extra -= 1
index = 1
while extra > 0 and subject_count > 1 and any(count < 2 for count in counts[1:]):
if counts[index] < 2:
counts[index] += 1
extra -= 1
index = index + 1 if index + 1 < subject_count else 1
if extra > 0 and counts[0] < 3:
counts[0] += 1
extra -= 1
index = 0
while extra > 0:
counts[index] += 1
extra -= 1
index = (index + 1) % subject_count
return counts