Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
233 lines
8.9 KiB
Python
233 lines
8.9 KiB
Python
import math
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
import comfy.utils
|
|
|
|
|
|
WAN_AUDIO_INPUT_FPS = 50
|
|
WAN_AUDIO_VIDEO_RATE = 30
|
|
WAN_AUDIO_FPS = 16
|
|
WAN_AUDIO_SAMPLE_RATE = 16000
|
|
WAN_VAE_SCALE = 8
|
|
WAN_PATCH_SPATIAL = 2
|
|
|
|
|
|
def ensure_float_image(image: torch.Tensor) -> torch.Tensor:
|
|
"""Convert an IMAGE tensor to floating point without mutating its input."""
|
|
if image.dtype == torch.uint8:
|
|
return image.float() / 255.0
|
|
if not image.is_floating_point():
|
|
return image.float()
|
|
return image
|
|
|
|
|
|
def resize_long_edge(image: torch.Tensor, max_size: int, stride: int = 16) -> torch.Tensor:
|
|
"""Preserve aspect ratio, cap the long edge, and align dimensions to a stride."""
|
|
image = ensure_float_image(image)
|
|
height, width = image.shape[1], image.shape[2]
|
|
scale = min(max_size / max(height, width), 1.0)
|
|
resized_height = max(stride, round(height * scale / stride) * stride)
|
|
resized_width = max(stride, round(width * scale / stride) * stride)
|
|
return comfy.utils.common_upscale(
|
|
image[:, :, :, :3].movedim(-1, 1),
|
|
resized_width,
|
|
resized_height,
|
|
"area",
|
|
"disabled",
|
|
).movedim(1, -1)
|
|
|
|
|
|
def build_context_latents(
|
|
vae: Any,
|
|
width: int,
|
|
height: int,
|
|
length: int,
|
|
source_video: torch.Tensor | None = None,
|
|
reference_video: torch.Tensor | None = None,
|
|
reference_images: dict[str, torch.Tensor | None] | None = None,
|
|
ref_max_size: int = 848,
|
|
) -> list[torch.Tensor]:
|
|
"""Encode Bernini context streams in their source-id order."""
|
|
context: list[torch.Tensor] = []
|
|
if source_video is not None:
|
|
source_video = ensure_float_image(source_video)
|
|
video = comfy.utils.common_upscale(
|
|
source_video[:length, :, :, :3].movedim(-1, 1),
|
|
width,
|
|
height,
|
|
"area",
|
|
"center",
|
|
).movedim(1, -1)
|
|
context.append(vae.encode(video[:, :, :, :3]))
|
|
|
|
if reference_video is not None:
|
|
resized_video = resize_long_edge(reference_video[:length], ref_max_size)
|
|
context.append(vae.encode(resized_video[:, :, :, :3]))
|
|
|
|
if reference_images:
|
|
for name in sorted(reference_images):
|
|
images = reference_images[name]
|
|
if images is None:
|
|
continue
|
|
for index in range(images.shape[0]):
|
|
image = resize_long_edge(images[index:index + 1], ref_max_size)
|
|
context.append(vae.encode(image[:, :, :, :3]))
|
|
return context
|
|
|
|
|
|
def _audio_feature(audio_encoder_output: dict[str, Any]) -> torch.Tensor:
|
|
try:
|
|
from comfy_extras.nodes_wan import linear_interpolation
|
|
except ImportError as exc:
|
|
raise RuntimeError("Wan S2V audio helpers are unavailable; update ComfyUI.") from exc
|
|
feature = torch.cat(audio_encoder_output["encoded_audio_all_layers"])
|
|
return linear_interpolation(feature, input_fps=WAN_AUDIO_INPUT_FPS, output_fps=WAN_AUDIO_VIDEO_RATE)
|
|
|
|
|
|
def audio_video_frames(audio_encoder_output: dict[str, Any], fps: int = WAN_AUDIO_FPS) -> int:
|
|
"""Return the approximate output-video duration of an encoded audio input."""
|
|
audio_samples = audio_encoder_output.get("audio_samples")
|
|
if audio_samples is not None:
|
|
return max(1, int(round(audio_samples / float(WAN_AUDIO_SAMPLE_RATE) * fps)))
|
|
feature = _audio_feature(audio_encoder_output)
|
|
return max(1, int(round(feature.shape[1] * fps / WAN_AUDIO_VIDEO_RATE)))
|
|
|
|
|
|
def resolve_audio_ranges(length: int, segments: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Resolve explicit or sequential speaker start frames against the latent duration."""
|
|
batch_frames = (((length - 1) // 4) + 1) * 4
|
|
resolved: list[dict[str, Any]] = []
|
|
automatic_cursor = 0
|
|
for segment in segments:
|
|
start_frame = segment.get("start_frame", -1)
|
|
start_frame = automatic_cursor if start_frame is None or start_frame < 0 else int(start_frame)
|
|
end_frame = min(
|
|
batch_frames,
|
|
start_frame + audio_video_frames(segment["audio_encoder_output"]),
|
|
)
|
|
resolved.append({**segment, "start_frame": start_frame, "end_frame": end_frame})
|
|
automatic_cursor = end_frame
|
|
return resolved
|
|
|
|
|
|
def build_audio_embed(length: int, segments: list[dict[str, Any]]) -> torch.Tensor:
|
|
"""Compose one or two speaker encodings onto the Wan S2V audio timeline."""
|
|
try:
|
|
from comfy_extras.nodes_wan import get_audio_embed_bucket_fps
|
|
except ImportError as exc:
|
|
raise RuntimeError("Wan S2V audio helpers are unavailable; update ComfyUI.") from exc
|
|
|
|
latent_frames = ((length - 1) // 4) + 1
|
|
batch_frames = latent_frames * 4
|
|
total_feature_frames = int(math.ceil(batch_frames * WAN_AUDIO_VIDEO_RATE / WAN_AUDIO_FPS))
|
|
composite: torch.Tensor | None = None
|
|
automatic_cursor = 0
|
|
|
|
for segment in segments:
|
|
feature = _audio_feature(segment["audio_encoder_output"])
|
|
if composite is None:
|
|
composite = torch.zeros(
|
|
feature.shape[0],
|
|
total_feature_frames,
|
|
feature.shape[2],
|
|
dtype=feature.dtype,
|
|
device=feature.device,
|
|
)
|
|
|
|
start_frame = segment.get("start_frame", -1)
|
|
start_frame = automatic_cursor if start_frame is None or start_frame < 0 else int(start_frame)
|
|
start_feature = int(round(start_frame * WAN_AUDIO_VIDEO_RATE / WAN_AUDIO_FPS))
|
|
if start_feature < total_feature_frames:
|
|
copy_length = min(feature.shape[1], total_feature_frames - start_feature)
|
|
if copy_length > 0:
|
|
composite[:, start_feature:start_feature + copy_length, :] = feature[:, :copy_length, :]
|
|
automatic_cursor = start_frame + audio_video_frames(segment["audio_encoder_output"])
|
|
|
|
if composite is None:
|
|
raise ValueError("At least one audio segment is required.")
|
|
|
|
bucket, _ = get_audio_embed_bucket_fps(
|
|
composite,
|
|
fps=WAN_AUDIO_FPS,
|
|
batch_frames=batch_frames,
|
|
m=0,
|
|
video_rate=WAN_AUDIO_VIDEO_RATE,
|
|
)
|
|
bucket = bucket.unsqueeze(0)
|
|
if bucket.ndim == 3:
|
|
bucket = bucket.permute(0, 2, 1)
|
|
elif bucket.ndim == 4:
|
|
bucket = bucket.permute(0, 2, 3, 1)
|
|
else:
|
|
raise ValueError(f"Unexpected Wan audio embedding rank: {bucket.ndim}.")
|
|
return bucket[:, :, :, :batch_frames]
|
|
|
|
|
|
def _padded_latent_dimension(pixels: int) -> int:
|
|
latent = pixels // WAN_VAE_SCALE
|
|
return latent + (WAN_PATCH_SPATIAL - latent % WAN_PATCH_SPATIAL) % WAN_PATCH_SPATIAL
|
|
|
|
|
|
def token_grid_size(width: int, height: int) -> tuple[int, int]:
|
|
"""Return the Wan S2V spatial token grid as height, width."""
|
|
return (
|
|
_padded_latent_dimension(height) // WAN_PATCH_SPATIAL,
|
|
_padded_latent_dimension(width) // WAN_PATCH_SPATIAL,
|
|
)
|
|
|
|
|
|
def mask_to_token_grid(mask_image: torch.Tensor, width: int, height: int) -> torch.Tensor:
|
|
"""Resize a ComfyUI MASK to a binary flattened Wan spatial token grid."""
|
|
token_height, token_width = token_grid_size(width, height)
|
|
mask = mask_image[0] if mask_image.ndim == 3 else mask_image
|
|
mask = mask.unsqueeze(0).unsqueeze(0).float()
|
|
mask = comfy.utils.common_upscale(mask, width, height, "area", "center")
|
|
mask = comfy.utils.common_upscale(mask, token_width, token_height, "area", "center")
|
|
return (mask > 0.5).to(dtype=torch.float32).flatten(2).squeeze(1)
|
|
|
|
|
|
def _frame_weight(video_frame: int, start_frame: int, end_frame: int, crossfade_frames: int) -> float:
|
|
if video_frame < start_frame or video_frame >= end_frame:
|
|
return 0.0
|
|
if crossfade_frames <= 0:
|
|
return 1.0
|
|
weight = 1.0
|
|
if video_frame < start_frame + crossfade_frames:
|
|
weight = min(weight, (video_frame - start_frame + 1) / crossfade_frames)
|
|
if video_frame >= end_frame - crossfade_frames:
|
|
weight = min(weight, (end_frame - video_frame) / crossfade_frames)
|
|
return max(0.0, weight)
|
|
|
|
|
|
def build_audio_inject_mask(
|
|
width: int,
|
|
height: int,
|
|
length: int,
|
|
segments: list[dict[str, Any]],
|
|
crossfade_frames: int = 0,
|
|
device: torch.device | str | None = None,
|
|
) -> torch.Tensor:
|
|
"""Build the temporal/spatial mask consumed by the patched S2V audio injector."""
|
|
latent_frames = ((length - 1) // 4) + 1
|
|
token_height, token_width = token_grid_size(width, height)
|
|
mask = torch.zeros(1, latent_frames, token_height * token_width, 1)
|
|
|
|
for segment in segments:
|
|
tokens = mask_to_token_grid(segment["mask_image"], width, height)
|
|
start_frame = int(segment["start_frame"])
|
|
end_frame = int(segment["end_frame"])
|
|
for latent_index in range(latent_frames):
|
|
weight = max(
|
|
_frame_weight(frame, start_frame, end_frame, crossfade_frames)
|
|
for frame in range(latent_index * 4, latent_index * 4 + 4)
|
|
)
|
|
if weight > 0.0:
|
|
mask[:, latent_index, :, 0] = torch.maximum(
|
|
mask[:, latent_index, :, 0],
|
|
tokens * weight,
|
|
)
|
|
return mask.to(device) if device is not None else mask
|
|
|