MMnode/utils/bernini_s2v.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

233 lines
8.9 KiB
Python

import math
from typing import Any
import torch
import comfy.utils
WAN_AUDIO_INPUT_FPS = 50
WAN_AUDIO_VIDEO_RATE = 30
WAN_AUDIO_FPS = 16
WAN_AUDIO_SAMPLE_RATE = 16000
WAN_VAE_SCALE = 8
WAN_PATCH_SPATIAL = 2
def ensure_float_image(image: torch.Tensor) -> torch.Tensor:
"""Convert an IMAGE tensor to floating point without mutating its input."""
if image.dtype == torch.uint8:
return image.float() / 255.0
if not image.is_floating_point():
return image.float()
return image
def resize_long_edge(image: torch.Tensor, max_size: int, stride: int = 16) -> torch.Tensor:
"""Preserve aspect ratio, cap the long edge, and align dimensions to a stride."""
image = ensure_float_image(image)
height, width = image.shape[1], image.shape[2]
scale = min(max_size / max(height, width), 1.0)
resized_height = max(stride, round(height * scale / stride) * stride)
resized_width = max(stride, round(width * scale / stride) * stride)
return comfy.utils.common_upscale(
image[:, :, :, :3].movedim(-1, 1),
resized_width,
resized_height,
"area",
"disabled",
).movedim(1, -1)
def build_context_latents(
vae: Any,
width: int,
height: int,
length: int,
source_video: torch.Tensor | None = None,
reference_video: torch.Tensor | None = None,
reference_images: dict[str, torch.Tensor | None] | None = None,
ref_max_size: int = 848,
) -> list[torch.Tensor]:
"""Encode Bernini context streams in their source-id order."""
context: list[torch.Tensor] = []
if source_video is not None:
source_video = ensure_float_image(source_video)
video = comfy.utils.common_upscale(
source_video[:length, :, :, :3].movedim(-1, 1),
width,
height,
"area",
"center",
).movedim(1, -1)
context.append(vae.encode(video[:, :, :, :3]))
if reference_video is not None:
resized_video = resize_long_edge(reference_video[:length], ref_max_size)
context.append(vae.encode(resized_video[:, :, :, :3]))
if reference_images:
for name in sorted(reference_images):
images = reference_images[name]
if images is None:
continue
for index in range(images.shape[0]):
image = resize_long_edge(images[index:index + 1], ref_max_size)
context.append(vae.encode(image[:, :, :, :3]))
return context
def _audio_feature(audio_encoder_output: dict[str, Any]) -> torch.Tensor:
try:
from comfy_extras.nodes_wan import linear_interpolation
except ImportError as exc:
raise RuntimeError("Wan S2V audio helpers are unavailable; update ComfyUI.") from exc
feature = torch.cat(audio_encoder_output["encoded_audio_all_layers"])
return linear_interpolation(feature, input_fps=WAN_AUDIO_INPUT_FPS, output_fps=WAN_AUDIO_VIDEO_RATE)
def audio_video_frames(audio_encoder_output: dict[str, Any], fps: int = WAN_AUDIO_FPS) -> int:
"""Return the approximate output-video duration of an encoded audio input."""
audio_samples = audio_encoder_output.get("audio_samples")
if audio_samples is not None:
return max(1, int(round(audio_samples / float(WAN_AUDIO_SAMPLE_RATE) * fps)))
feature = _audio_feature(audio_encoder_output)
return max(1, int(round(feature.shape[1] * fps / WAN_AUDIO_VIDEO_RATE)))
def resolve_audio_ranges(length: int, segments: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Resolve explicit or sequential speaker start frames against the latent duration."""
batch_frames = (((length - 1) // 4) + 1) * 4
resolved: list[dict[str, Any]] = []
automatic_cursor = 0
for segment in segments:
start_frame = segment.get("start_frame", -1)
start_frame = automatic_cursor if start_frame is None or start_frame < 0 else int(start_frame)
end_frame = min(
batch_frames,
start_frame + audio_video_frames(segment["audio_encoder_output"]),
)
resolved.append({**segment, "start_frame": start_frame, "end_frame": end_frame})
automatic_cursor = end_frame
return resolved
def build_audio_embed(length: int, segments: list[dict[str, Any]]) -> torch.Tensor:
"""Compose one or two speaker encodings onto the Wan S2V audio timeline."""
try:
from comfy_extras.nodes_wan import get_audio_embed_bucket_fps
except ImportError as exc:
raise RuntimeError("Wan S2V audio helpers are unavailable; update ComfyUI.") from exc
latent_frames = ((length - 1) // 4) + 1
batch_frames = latent_frames * 4
total_feature_frames = int(math.ceil(batch_frames * WAN_AUDIO_VIDEO_RATE / WAN_AUDIO_FPS))
composite: torch.Tensor | None = None
automatic_cursor = 0
for segment in segments:
feature = _audio_feature(segment["audio_encoder_output"])
if composite is None:
composite = torch.zeros(
feature.shape[0],
total_feature_frames,
feature.shape[2],
dtype=feature.dtype,
device=feature.device,
)
start_frame = segment.get("start_frame", -1)
start_frame = automatic_cursor if start_frame is None or start_frame < 0 else int(start_frame)
start_feature = int(round(start_frame * WAN_AUDIO_VIDEO_RATE / WAN_AUDIO_FPS))
if start_feature < total_feature_frames:
copy_length = min(feature.shape[1], total_feature_frames - start_feature)
if copy_length > 0:
composite[:, start_feature:start_feature + copy_length, :] = feature[:, :copy_length, :]
automatic_cursor = start_frame + audio_video_frames(segment["audio_encoder_output"])
if composite is None:
raise ValueError("At least one audio segment is required.")
bucket, _ = get_audio_embed_bucket_fps(
composite,
fps=WAN_AUDIO_FPS,
batch_frames=batch_frames,
m=0,
video_rate=WAN_AUDIO_VIDEO_RATE,
)
bucket = bucket.unsqueeze(0)
if bucket.ndim == 3:
bucket = bucket.permute(0, 2, 1)
elif bucket.ndim == 4:
bucket = bucket.permute(0, 2, 3, 1)
else:
raise ValueError(f"Unexpected Wan audio embedding rank: {bucket.ndim}.")
return bucket[:, :, :, :batch_frames]
def _padded_latent_dimension(pixels: int) -> int:
latent = pixels // WAN_VAE_SCALE
return latent + (WAN_PATCH_SPATIAL - latent % WAN_PATCH_SPATIAL) % WAN_PATCH_SPATIAL
def token_grid_size(width: int, height: int) -> tuple[int, int]:
"""Return the Wan S2V spatial token grid as height, width."""
return (
_padded_latent_dimension(height) // WAN_PATCH_SPATIAL,
_padded_latent_dimension(width) // WAN_PATCH_SPATIAL,
)
def mask_to_token_grid(mask_image: torch.Tensor, width: int, height: int) -> torch.Tensor:
"""Resize a ComfyUI MASK to a binary flattened Wan spatial token grid."""
token_height, token_width = token_grid_size(width, height)
mask = mask_image[0] if mask_image.ndim == 3 else mask_image
mask = mask.unsqueeze(0).unsqueeze(0).float()
mask = comfy.utils.common_upscale(mask, width, height, "area", "center")
mask = comfy.utils.common_upscale(mask, token_width, token_height, "area", "center")
return (mask > 0.5).to(dtype=torch.float32).flatten(2).squeeze(1)
def _frame_weight(video_frame: int, start_frame: int, end_frame: int, crossfade_frames: int) -> float:
if video_frame < start_frame or video_frame >= end_frame:
return 0.0
if crossfade_frames <= 0:
return 1.0
weight = 1.0
if video_frame < start_frame + crossfade_frames:
weight = min(weight, (video_frame - start_frame + 1) / crossfade_frames)
if video_frame >= end_frame - crossfade_frames:
weight = min(weight, (end_frame - video_frame) / crossfade_frames)
return max(0.0, weight)
def build_audio_inject_mask(
width: int,
height: int,
length: int,
segments: list[dict[str, Any]],
crossfade_frames: int = 0,
device: torch.device | str | None = None,
) -> torch.Tensor:
"""Build the temporal/spatial mask consumed by the patched S2V audio injector."""
latent_frames = ((length - 1) // 4) + 1
token_height, token_width = token_grid_size(width, height)
mask = torch.zeros(1, latent_frames, token_height * token_width, 1)
for segment in segments:
tokens = mask_to_token_grid(segment["mask_image"], width, height)
start_frame = int(segment["start_frame"])
end_frame = int(segment["end_frame"])
for latent_index in range(latent_frames):
weight = max(
_frame_weight(frame, start_frame, end_frame, crossfade_frames)
for frame in range(latent_index * 4, latent_index * 4 + 4)
)
if weight > 0.0:
mask[:, latent_index, :, 0] = torch.maximum(
mask[:, latent_index, :, 0],
tokens * weight,
)
return mask.to(device) if device is not None else mask