Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
296 lines
13 KiB
Python
296 lines
13 KiB
Python
from comfy_extras.nodes_lt import get_noise_mask, LTXVAddGuide, _append_guide_attention_entry, preprocess
|
|
import types
|
|
import math
|
|
from typing import Tuple
|
|
import comfy
|
|
from comfy_api.latest import io
|
|
import numpy as np
|
|
import torch
|
|
import logging
|
|
import comfy.model_management as mm
|
|
import comfy.ldm.modules.attention as _comfy_attn
|
|
from comfy.ldm.lightricks.model import apply_rotary_emb as _apply_rope
|
|
try:
|
|
from comfy.ldm.lightricks.model import GuideAttentionMask as _GuideAttentionMask, _attention_with_guide_mask as _ltx_attn_with_guide_mask
|
|
except ImportError:
|
|
_GuideAttentionMask = None
|
|
_ltx_attn_with_guide_mask = None
|
|
device = mm.get_torch_device()
|
|
import latent_preview
|
|
|
|
# code based on https://github.com/kijai/ComfyUI-KJNodes/blob/main/nodes/ltxv_nodes.py#L21
|
|
class LTXVAddGuidesFromBatchIndexes(LTXVAddGuide):
|
|
|
|
@classmethod
|
|
def define_schema(cls):
|
|
return io.Schema(
|
|
node_id="LTXVAddGuidesFromBatchIndexes",
|
|
category="conditioning/ltxv",
|
|
description="Adds guide images or an image sequence from a batch to the latent at specified frame indices. Batches with 9 or more images are treated as one image sequence.",
|
|
inputs=[
|
|
io.Conditioning.Input("positive"),
|
|
io.Conditioning.Input("negative"),
|
|
io.Vae.Input("vae"),
|
|
io.Latent.Input("latent"),
|
|
io.Image.Input("images", tooltip="Batch of images. Batches with 9 or more images are treated as one image sequence."),
|
|
io.Int.Input(
|
|
id="img_compression", default=18, min=0, max=100, tooltip="Amount of compression to apply on image."
|
|
),
|
|
io.String.Input("image_indexes", default="", tooltip="Comma-separated frame indices, e.g. '0,61,121'. For image sequences, the first index is used as the sequence start frame."),
|
|
io.Float.Input("strength", default=1.0, min=0.0, max=1.0, step=0.01),
|
|
],
|
|
outputs=[
|
|
io.Conditioning.Output(display_name="positive"),
|
|
io.Conditioning.Output(display_name="negative"),
|
|
io.Latent.Output(display_name="latent"),
|
|
],
|
|
)
|
|
|
|
@classmethod
|
|
def execute(cls, positive, negative, vae, latent, images, img_compression, image_indexes, strength) -> io.NodeOutput:
|
|
scale_factors = vae.downscale_index_formula
|
|
latent_image = latent["samples"]
|
|
noise_mask = get_noise_mask(latent)
|
|
|
|
_, _, latent_length, latent_height, latent_width = latent_image.shape
|
|
|
|
parsed_indexes = [int(x.strip()) for x in image_indexes.split(",") if x.strip()]
|
|
|
|
# If no indexes provided, calculate evenly distributed frame indices
|
|
if not parsed_indexes:
|
|
# Find valid (non-black) images first
|
|
valid_count = 0
|
|
for i in range(images.shape[0]):
|
|
if images[i].max() > 0.001:
|
|
valid_count += 1
|
|
|
|
if valid_count > 0:
|
|
# Evenly distribute from 0 to latent_length-1
|
|
parsed_indexes = np.linspace(0, latent_length - 1, valid_count, dtype=int).tolist()
|
|
|
|
# Process each image in the batch
|
|
if img_compression > 0:
|
|
output_images = []
|
|
for i in range(images.shape[0]):
|
|
output_images.append(preprocess(images[i], img_compression))
|
|
|
|
batch_size = images.shape[0]
|
|
|
|
if batch_size >= 9:
|
|
if images.max() > 0.001:
|
|
f_idx = parsed_indexes[0] if parsed_indexes else 0
|
|
|
|
image_1, t = cls.encode(vae, latent_width, latent_height, images, scale_factors)
|
|
|
|
frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image_1), f_idx, scale_factors)
|
|
|
|
if latent_idx + t.shape[2] <= latent_length:
|
|
positive, negative, latent_image, noise_mask = cls.append_keyframe(
|
|
positive,
|
|
negative,
|
|
frame_idx,
|
|
latent_image,
|
|
noise_mask,
|
|
t,
|
|
strength,
|
|
scale_factors,
|
|
)
|
|
|
|
# Track this guide sequence for per-reference attention control.
|
|
pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4]
|
|
guide_latent_shape = list(t.shape[2:]) # [F, H, W]
|
|
positive, negative = _append_guide_attention_entry(positive, negative, pre_filter_count, guide_latent_shape, strength=strength)
|
|
else:
|
|
logging.warning("Skipping guide sequence - conditioning frames exceed latent sequence length")
|
|
|
|
return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})
|
|
|
|
for i in range(batch_size):
|
|
if i >= len(parsed_indexes):
|
|
break
|
|
|
|
img = images[i:i+1]
|
|
|
|
# If strength is a list/tuple, use corresponding value for this image, otherwise use the single strength value for all images
|
|
if isinstance(strength, (list, tuple)):
|
|
strength = strength[i] if i < len(strength) else strength[-1]
|
|
|
|
# Check if image is not black and use provided frame index
|
|
if img.max() > 0.001:
|
|
f_idx = parsed_indexes[i]
|
|
|
|
image_1, t = cls.encode(vae, latent_width, latent_height, img, scale_factors)
|
|
|
|
frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image_1), f_idx, scale_factors)
|
|
|
|
if latent_idx + t.shape[2] <= latent_length:
|
|
positive, negative, latent_image, noise_mask = cls.append_keyframe(
|
|
positive,
|
|
negative,
|
|
frame_idx,
|
|
latent_image,
|
|
noise_mask,
|
|
t,
|
|
strength,
|
|
scale_factors,
|
|
)
|
|
|
|
# Track this guide for per-reference attention control.
|
|
pre_filter_count = t.shape[2] * t.shape[3] * t.shape[4]
|
|
guide_latent_shape = list(t.shape[2:]) # [F, H, W]
|
|
positive, negative = _append_guide_attention_entry(positive, negative, pre_filter_count, guide_latent_shape, strength=strength)
|
|
else:
|
|
logging.warning("Skipping guide at index %s - conditioning frames exceed latent sequence length", i)
|
|
|
|
return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})
|
|
|
|
# code based on https://github.com/liconstudio/ComfyUI-Licon-MSR/blob/main/licon_msr.py
|
|
class LTXVMakeRefVideo(io.ComfyNode):
|
|
@classmethod
|
|
def define_schema(cls):
|
|
return io.Schema(
|
|
node_id="LTXVMakeRefVideo",
|
|
display_name="LTXVMakeRefVideo",
|
|
category="image/ltxv",
|
|
description="Expands a batch of reference images into a reference video for IC-LoRA.",
|
|
is_input_list=True,
|
|
inputs=[
|
|
io.Image.Input("images"),
|
|
io.Int.Input("frame_count", default=17, min=17, step=8),
|
|
],
|
|
outputs=[
|
|
io.Image.Output("IMAGE"),
|
|
],
|
|
)
|
|
|
|
@classmethod
|
|
def execute(cls, images: list[torch.Tensor], frame_count: list[int]) -> io.NodeOutput:
|
|
try:
|
|
image_batch = cls._prepare_image_batch(images)
|
|
if not frame_count:
|
|
raise ValueError("frame_count must contain one value")
|
|
frames = cls._expand_frames(image_batch, int(frame_count[0]))
|
|
except Exception as exc:
|
|
raise RuntimeError("Failed to create IC-LoRA reference video frames") from exc
|
|
|
|
return io.NodeOutput(frames)
|
|
|
|
@staticmethod
|
|
def _prepare_image_batch(images: list[torch.Tensor]) -> torch.Tensor:
|
|
if not images:
|
|
raise ValueError("images must contain at least one image tensor")
|
|
if not all(isinstance(image, torch.Tensor) for image in images):
|
|
raise TypeError("images must contain only torch.Tensor values")
|
|
if any(image.ndim != 4 or image.shape[0] < 1 for image in images):
|
|
raise ValueError("Each image input must have shape [B, H, W, C] with B >= 1")
|
|
|
|
if len(images) == 1:
|
|
return images[0]
|
|
|
|
individual_images = [frame for batch in images for frame in batch.split(1, dim=0)]
|
|
background = individual_images[-1]
|
|
target_height, target_width, target_channels = background.shape[1:]
|
|
prepared = []
|
|
|
|
for image in individual_images[:-1]:
|
|
if image.shape[-1] != target_channels:
|
|
raise ValueError("All image inputs must have the same channel count as the background")
|
|
|
|
image = image.to(device=background.device, dtype=background.dtype)
|
|
source_height, source_width = image.shape[1:3]
|
|
if (source_height, source_width) == (target_height, target_width):
|
|
prepared.append(image)
|
|
continue
|
|
|
|
scale = min(target_width / source_width, target_height / source_height)
|
|
resized_width = max(1, min(target_width, round(source_width * scale)))
|
|
resized_height = max(1, min(target_height, round(source_height * scale)))
|
|
resized = torch.nn.functional.interpolate(
|
|
image.movedim(-1, 1),
|
|
size=(resized_height, resized_width),
|
|
mode="bilinear",
|
|
align_corners=False,
|
|
).movedim(1, -1)
|
|
canvas = torch.ones(
|
|
(1, target_height, target_width, target_channels),
|
|
device=background.device,
|
|
dtype=background.dtype,
|
|
)
|
|
top = (target_height - resized_height) // 2
|
|
left = (target_width - resized_width) // 2
|
|
canvas[:, top:top + resized_height, left:left + resized_width] = resized
|
|
prepared.append(canvas)
|
|
|
|
prepared.append(background)
|
|
return torch.cat(prepared, dim=0)
|
|
|
|
@staticmethod
|
|
def _expand_frames(images: torch.Tensor, frame_count: int) -> torch.Tensor:
|
|
if images.ndim != 4:
|
|
raise ValueError("images must be an IMAGE batch tensor with shape [B, H, W, C]")
|
|
|
|
image_count = images.shape[0]
|
|
if image_count <= 0:
|
|
raise ValueError("images batch must contain at least one image")
|
|
|
|
background = images[-1:]
|
|
frames = background.repeat(frame_count, 1, 1, 1)
|
|
subjects = images[:-1]
|
|
if subjects.shape[0] == 0:
|
|
return frames
|
|
|
|
latent_count = max(1, round((frame_count - 1) / 8) + 1)
|
|
subject_budget = max(0, latent_count - 1)
|
|
subject_count = subjects.shape[0]
|
|
|
|
if subject_budget >= subject_count:
|
|
counts = LTXVMakeRefVideo._allocate_subject_latent_counts(subject_count, subject_budget)
|
|
cursor = 0
|
|
for image, count in zip(subjects, counts):
|
|
start, end = LTXVMakeRefVideo._latent_to_frame_range(cursor, cursor + count - 1)
|
|
cursor += count
|
|
frames[max(0, start):min(frame_count - 1, end) + 1] = image
|
|
else:
|
|
_, subject_end = LTXVMakeRefVideo._latent_to_frame_range(0, max(0, latent_count - 2))
|
|
subject_frame_count = max(1, subject_end + 1)
|
|
for index, image in enumerate(subjects):
|
|
start = int(index * subject_frame_count / subject_count)
|
|
end = int((index + 1) * subject_frame_count / subject_count)
|
|
frames[start:min(frame_count, max(start + 1, end))] = image
|
|
|
|
return frames
|
|
|
|
@staticmethod
|
|
def _latent_to_frame_range(latent_start: int, latent_end: int) -> tuple[int, int]:
|
|
frame_start = 0 if latent_start <= 0 else 1 + (latent_start - 1) * 8
|
|
frame_end = 0 if latent_end <= 0 else latent_end * 8
|
|
return frame_start, frame_end
|
|
|
|
@staticmethod
|
|
def _allocate_subject_latent_counts(subject_count: int, subject_budget: int) -> list[int]:
|
|
counts = [1] * subject_count
|
|
extra = max(0, subject_budget - subject_count)
|
|
|
|
if extra > 0:
|
|
counts[0] += 1
|
|
extra -= 1
|
|
|
|
index = 1
|
|
while extra > 0 and subject_count > 1 and any(count < 2 for count in counts[1:]):
|
|
if counts[index] < 2:
|
|
counts[index] += 1
|
|
extra -= 1
|
|
index = index + 1 if index + 1 < subject_count else 1
|
|
|
|
if extra > 0 and counts[0] < 3:
|
|
counts[0] += 1
|
|
extra -= 1
|
|
|
|
index = 0
|
|
while extra > 0:
|
|
counts[index] += 1
|
|
extra -= 1
|
|
index = (index + 1) % subject_count
|
|
|
|
return counts
|