Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
476 lines
18 KiB
Python
476 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import inspect
|
|
import logging
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
|
|
def masked_audio_injector_forward(
|
|
original_forward: Callable[..., torch.Tensor],
|
|
injector: Any,
|
|
x: torch.Tensor,
|
|
block_id: int,
|
|
audio_embed: torch.Tensor,
|
|
audio_embed_global: torch.Tensor | None,
|
|
sequence_length: int,
|
|
scale: float = 1.0,
|
|
token_mask: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Run ComfyUI's injector globally, or apply its residual through a token mask."""
|
|
if token_mask is None:
|
|
return original_forward(
|
|
injector,
|
|
x,
|
|
block_id,
|
|
audio_embed,
|
|
audio_embed_global,
|
|
sequence_length,
|
|
scale=scale,
|
|
)
|
|
|
|
audio_attention_id = injector.injected_block_id.get(block_id)
|
|
if audio_attention_id is None:
|
|
return x
|
|
|
|
try:
|
|
from einops import rearrange
|
|
except ImportError as exc:
|
|
raise RuntimeError("Masked Bernini S2V audio requires einops.") from exc
|
|
|
|
frame_count = audio_embed.shape[1]
|
|
input_hidden_states = rearrange(
|
|
x[:, :sequence_length],
|
|
"b (t n) c -> (b t) n c",
|
|
t=frame_count,
|
|
)
|
|
if injector.enable_adain and injector.adain_mode == "attn_norm":
|
|
if audio_embed_global is None:
|
|
raise ValueError("Global audio embedding is required by the S2V AdaIN injector.")
|
|
global_embedding = rearrange(audio_embed_global, "b t n c -> (b t) n c")
|
|
attention_hidden_states = injector.injector_adain_layers[audio_attention_id](
|
|
input_hidden_states,
|
|
temb=global_embedding[:, 0],
|
|
)
|
|
else:
|
|
attention_hidden_states = injector.injector_pre_norm_feat[audio_attention_id](input_hidden_states)
|
|
|
|
if audio_embed.ndim == 3:
|
|
attention_audio = rearrange(audio_embed, "b t c -> (b t) 1 c", t=frame_count)
|
|
elif audio_embed.ndim == 4:
|
|
attention_audio = rearrange(audio_embed, "b t n c -> (b t) n c", t=frame_count)
|
|
else:
|
|
raise ValueError(f"Unexpected S2V audio embedding rank: {audio_embed.ndim}.")
|
|
|
|
residual = injector.injector[audio_attention_id](
|
|
x=attention_hidden_states,
|
|
context=attention_audio,
|
|
)
|
|
residual = rearrange(residual, "(b t) n c -> b (t n) c", t=frame_count)
|
|
flattened_mask = token_mask.flatten(1, 2) if token_mask.ndim == 4 else token_mask
|
|
if flattened_mask.shape[1] != residual.shape[1]:
|
|
LOGGER.warning(
|
|
"Bernini S2V audio mask has %s tokens, expected %s; falling back to global injection.",
|
|
flattened_mask.shape[1],
|
|
residual.shape[1],
|
|
)
|
|
else:
|
|
residual = residual * flattened_mask.to(device=residual.device, dtype=residual.dtype)
|
|
|
|
result = x.clone()
|
|
result[:, :sequence_length] = result[:, :sequence_length] + residual * scale
|
|
return result
|
|
|
|
|
|
def _patch_audio_injector() -> bool:
|
|
from comfy.ldm.wan.model import AudioInjector_WAN
|
|
|
|
current = AudioInjector_WAN.forward
|
|
if getattr(current, "__easy_bernini_s2v_mask_patch__", False):
|
|
return False
|
|
original = getattr(current, "__wan_bernini_s2v_masked_original__", current)
|
|
|
|
def forward(
|
|
self: Any,
|
|
x: torch.Tensor,
|
|
block_id: int,
|
|
audio_emb: torch.Tensor,
|
|
audio_emb_global: torch.Tensor | None,
|
|
seq_len: int,
|
|
scale: float = 1.0,
|
|
token_mask: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
return masked_audio_injector_forward(
|
|
original,
|
|
self,
|
|
x,
|
|
block_id,
|
|
audio_emb,
|
|
audio_emb_global,
|
|
seq_len,
|
|
scale,
|
|
token_mask,
|
|
)
|
|
|
|
forward.__easy_bernini_s2v_mask_patch__ = True
|
|
forward.__easy_bernini_s2v_original__ = original
|
|
AudioInjector_WAN.forward = forward
|
|
return True
|
|
|
|
|
|
def _patch_s2v_conditions() -> bool:
|
|
import comfy.conds
|
|
from comfy.model_base import WAN22_S2V
|
|
|
|
changed = False
|
|
current = WAN22_S2V.extra_conds
|
|
if not getattr(current, "__easy_bernini_s2v_condition_patch__", False):
|
|
original = getattr(current, "__wan_bernini_s2v_masked_original__", current)
|
|
|
|
def extra_conds(self: Any, **kwargs: Any) -> dict[str, Any]:
|
|
conditions = original(self, **kwargs)
|
|
context_latents = kwargs.get("context_latents")
|
|
if context_latents is not None:
|
|
conditions["context_latents"] = comfy.conds.CONDList([
|
|
self.process_latent_in(latent) for latent in context_latents
|
|
])
|
|
audio_mask = kwargs.get("audio_inject_mask")
|
|
if audio_mask is not None:
|
|
conditions["audio_inject_mask"] = comfy.conds.CONDRegular(audio_mask)
|
|
audio_scale = kwargs.get("audio_inject_scale")
|
|
if audio_scale is not None:
|
|
conditions["audio_inject_scale"] = comfy.conds.CONDRegular(
|
|
torch.tensor([audio_scale], dtype=torch.float32)
|
|
)
|
|
return conditions
|
|
|
|
extra_conds.__easy_bernini_s2v_condition_patch__ = True
|
|
extra_conds.__easy_bernini_s2v_original__ = original
|
|
WAN22_S2V.extra_conds = extra_conds
|
|
changed = True
|
|
|
|
current_resize = WAN22_S2V.resize_cond_for_context_window
|
|
if getattr(current_resize, "__easy_bernini_s2v_condition_patch__", False):
|
|
return changed
|
|
original_resize = getattr(current_resize, "__wan_bernini_s2v_masked_original__", current_resize)
|
|
|
|
def resize_cond_for_context_window(
|
|
self: Any,
|
|
cond_key: str,
|
|
cond_value: Any,
|
|
window: Any,
|
|
x_in: torch.Tensor,
|
|
device: torch.device,
|
|
retain_index_list: list[int] | None = None,
|
|
) -> Any:
|
|
if cond_key == "context_latents" and isinstance(getattr(cond_value, "cond", None), list):
|
|
dimension = window.dim
|
|
sliced = []
|
|
for latent in cond_value.cond:
|
|
if latent.ndim > dimension and latent.shape[dimension] > 1 and latent.shape[dimension] == x_in.shape[dimension]:
|
|
sliced.append(window.get_tensor(
|
|
latent,
|
|
device,
|
|
dim=dimension,
|
|
retain_index_list=[] if retain_index_list is None else retain_index_list,
|
|
))
|
|
else:
|
|
sliced.append(latent.to(device))
|
|
return cond_value._copy_with(sliced)
|
|
if cond_key == "audio_inject_mask":
|
|
mask = cond_value.cond
|
|
if mask.ndim == 4 and mask.shape[1] == x_in.shape[2]:
|
|
return cond_value._copy_with(window.get_tensor(mask, device, dim=1))
|
|
return original_resize(
|
|
self,
|
|
cond_key,
|
|
cond_value,
|
|
window,
|
|
x_in,
|
|
device,
|
|
retain_index_list=[] if retain_index_list is None else retain_index_list,
|
|
)
|
|
|
|
resize_cond_for_context_window.__easy_bernini_s2v_condition_patch__ = True
|
|
resize_cond_for_context_window.__easy_bernini_s2v_original__ = original_resize
|
|
WAN22_S2V.resize_cond_for_context_window = resize_cond_for_context_window
|
|
return True
|
|
|
|
|
|
def prepare_s2v_context(
|
|
model: Any,
|
|
context_latents: list[torch.Tensor] | None,
|
|
frequencies: torch.Tensor,
|
|
*,
|
|
device: torch.device,
|
|
dtype: torch.dtype,
|
|
transformer_options: dict[str, Any],
|
|
) -> tuple[dict[str, list[torch.Tensor]], torch.Tensor]:
|
|
"""Pad context streams and append one source-ID RoPE block for each stream."""
|
|
if not context_latents:
|
|
return {}, frequencies
|
|
|
|
import importlib
|
|
|
|
common_dit = importlib.import_module("comfy.ldm.common_dit")
|
|
padded = [
|
|
common_dit.pad_to_patch_size(latent, model.patch_size)
|
|
for latent in context_latents
|
|
]
|
|
for index, latent in enumerate(padded, start=1):
|
|
context_frequencies = model.rope_encode(
|
|
latent.shape[-3],
|
|
latent.shape[-2],
|
|
latent.shape[-1],
|
|
device=device,
|
|
dtype=dtype,
|
|
transformer_options=transformer_options,
|
|
source_id=index,
|
|
)
|
|
frequencies = torch.cat([frequencies, context_frequencies], dim=1)
|
|
return {"context_latents": padded}, frequencies
|
|
|
|
|
|
def s2v_forward_patch_ready(forward: Callable[..., torch.Tensor]) -> bool:
|
|
"""Return whether the inner S2V forward supports the outer context wrapper."""
|
|
return bool(getattr(forward, "__easy_bernini_s2v_forward_patch__", False))
|
|
|
|
|
|
def _patch_s2v_outer_forward() -> bool:
|
|
import comfy.ldm.common_dit
|
|
from comfy.ldm.wan.model import WanModel_S2V
|
|
|
|
current = WanModel_S2V._forward
|
|
if getattr(current, "__easy_bernini_s2v_outer_patch__", False):
|
|
return False
|
|
if not s2v_forward_patch_ready(WanModel_S2V.forward_orig):
|
|
LOGGER.warning("Skipping Bernini S2V outer forward patch because the inner forward is incompatible.")
|
|
return False
|
|
|
|
def outer_forward(
|
|
self: Any,
|
|
x: torch.Tensor,
|
|
timestep: torch.Tensor,
|
|
context: torch.Tensor,
|
|
clip_fea: torch.Tensor | None = None,
|
|
time_dim_concat: torch.Tensor | None = None,
|
|
transformer_options: dict[str, Any] | None = None,
|
|
**kwargs: Any,
|
|
) -> torch.Tensor:
|
|
transformer_options = {} if transformer_options is None else transformer_options
|
|
_, _, time, height, width = x.shape
|
|
x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size)
|
|
|
|
rope_time = time
|
|
if time_dim_concat is not None:
|
|
time_dim_concat = comfy.ldm.common_dit.pad_to_patch_size(time_dim_concat, self.patch_size)
|
|
x = torch.cat([x, time_dim_concat], dim=2)
|
|
rope_time = x.shape[2]
|
|
if self.ref_conv is not None and "reference_latent" in kwargs:
|
|
rope_time += 1
|
|
|
|
frequencies = self.rope_encode(
|
|
rope_time,
|
|
height,
|
|
width,
|
|
device=x.device,
|
|
dtype=x.dtype,
|
|
transformer_options=transformer_options,
|
|
)
|
|
context_kwargs, frequencies = prepare_s2v_context(
|
|
self,
|
|
kwargs.get("context_latents"),
|
|
frequencies,
|
|
device=x.device,
|
|
dtype=x.dtype,
|
|
transformer_options=transformer_options,
|
|
)
|
|
kwargs = {**kwargs, **context_kwargs}
|
|
return self.forward_orig(
|
|
x,
|
|
timestep,
|
|
context,
|
|
clip_fea=clip_fea,
|
|
freqs=frequencies,
|
|
transformer_options=transformer_options,
|
|
**kwargs,
|
|
)[:, :, :time, :height, :width]
|
|
|
|
outer_forward.__easy_bernini_s2v_outer_patch__ = True
|
|
outer_forward.__easy_bernini_s2v_original__ = current
|
|
WanModel_S2V._forward = outer_forward
|
|
return True
|
|
|
|
|
|
def _patch_s2v_forward() -> bool:
|
|
import comfy.model_management
|
|
from comfy.ldm.wan.model import WanModel_S2V, sinusoidal_embedding_1d
|
|
|
|
current = WanModel_S2V.forward_orig
|
|
if getattr(current, "__easy_bernini_s2v_forward_patch__", False):
|
|
return False
|
|
original = getattr(
|
|
current,
|
|
"__easy_bernini_s2v_original__",
|
|
getattr(current, "__wan_bernini_s2v_original__", current),
|
|
)
|
|
required = {"x", "t", "context", "audio_embed", "freqs", "transformer_options"}
|
|
if not required.issubset(inspect.signature(original).parameters):
|
|
LOGGER.warning("Skipping Bernini S2V forward patch because this ComfyUI version has an incompatible signature.")
|
|
return False
|
|
|
|
def forward_orig(
|
|
self: Any,
|
|
x: torch.Tensor,
|
|
t: torch.Tensor,
|
|
context: torch.Tensor,
|
|
audio_embed: torch.Tensor | None = None,
|
|
reference_latent: torch.Tensor | None = None,
|
|
control_video: torch.Tensor | None = None,
|
|
reference_motion: torch.Tensor | None = None,
|
|
clip_fea: torch.Tensor | None = None,
|
|
freqs: torch.Tensor | None = None,
|
|
transformer_options: dict[str, Any] | None = None,
|
|
**kwargs: Any,
|
|
) -> torch.Tensor:
|
|
del clip_fea
|
|
transformer_options = {} if transformer_options is None else transformer_options
|
|
if audio_embed is not None:
|
|
embed_count = x.shape[-3] * 4
|
|
audio_global, audio = self.casual_audio_encoder(audio_embed[:, :, :, :embed_count])
|
|
else:
|
|
audio = None
|
|
audio_global = None
|
|
|
|
_, _, time, _, _ = x.shape
|
|
x = self.patch_embedding(x.float()).to(x.dtype)
|
|
if control_video is not None:
|
|
x = x + self.cond_encoder(control_video)
|
|
if t.ndim == 1:
|
|
t = t.unsqueeze(1).repeat(1, x.shape[2])
|
|
|
|
grid_sizes = x.shape[2:]
|
|
x = x.flatten(2).transpose(1, 2)
|
|
sequence_length = x.size(1)
|
|
condition_weights = comfy.model_management.cast_to(
|
|
self.trainable_cond_mask.weight,
|
|
dtype=x.dtype,
|
|
device=x.device,
|
|
).unsqueeze(1).unsqueeze(1)
|
|
x = x + condition_weights[0]
|
|
|
|
for latent in kwargs.get("context_latents") or []:
|
|
context_tokens = self.patch_embedding(latent.float().to(x.device)).to(x.dtype)
|
|
x = torch.cat([x, context_tokens.flatten(2).transpose(1, 2)], dim=1)
|
|
|
|
if reference_latent is not None:
|
|
reference = self.patch_embedding(reference_latent.float()).to(x.dtype)
|
|
reference = reference.flatten(2).transpose(1, 2) + condition_weights[1]
|
|
x = torch.cat([x, reference], dim=1)
|
|
reference_freqs = self.rope_encode(
|
|
reference_latent.shape[-3],
|
|
reference_latent.shape[-2],
|
|
reference_latent.shape[-1],
|
|
t_start=max(30, time + 9),
|
|
device=x.device,
|
|
dtype=x.dtype,
|
|
)
|
|
if freqs is None:
|
|
raise ValueError("Wan S2V reference conditioning requires RoPE frequencies.")
|
|
freqs = torch.cat([freqs, reference_freqs], dim=1)
|
|
t = torch.cat([
|
|
t,
|
|
torch.zeros((t.shape[0], reference_latent.shape[-3]), device=t.device, dtype=t.dtype),
|
|
], dim=1)
|
|
|
|
if reference_motion is not None:
|
|
motion, motion_freqs = self.frame_packer(reference_motion, self)
|
|
x = torch.cat([x, motion + condition_weights[2]], dim=1)
|
|
if freqs is None:
|
|
raise ValueError("Wan S2V motion conditioning requires RoPE frequencies.")
|
|
freqs = torch.cat([freqs, motion_freqs], dim=1)
|
|
t = torch.repeat_interleave(t, 2, dim=1)
|
|
t = torch.cat([t, torch.zeros((t.shape[0], 3), device=t.device, dtype=t.dtype)], dim=1)
|
|
|
|
embedding = self.time_embedding(
|
|
sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x[0].dtype)
|
|
)
|
|
embedding = embedding.reshape(t.shape[0], -1, embedding.shape[-1])
|
|
projected_embedding = self.time_projection(embedding).unflatten(2, (6, self.dim))
|
|
context = self.text_embedding(context)
|
|
|
|
replacement_blocks = transformer_options.get("patches_replace", {}).get("dit", {})
|
|
transformer_options["total_blocks"] = len(self.blocks)
|
|
transformer_options["block_type"] = "double"
|
|
for index, block in enumerate(self.blocks):
|
|
transformer_options["block_index"] = index
|
|
if ("double_block", index) in replacement_blocks:
|
|
def block_wrapper(arguments: dict[str, Any]) -> dict[str, torch.Tensor]:
|
|
return {"img": block(
|
|
arguments["img"],
|
|
context=arguments["txt"],
|
|
e=arguments["vec"],
|
|
freqs=arguments["pe"],
|
|
transformer_options=arguments["transformer_options"],
|
|
)}
|
|
|
|
x = replacement_blocks[("double_block", index)](
|
|
{
|
|
"img": x,
|
|
"txt": context,
|
|
"vec": projected_embedding,
|
|
"pe": freqs,
|
|
"transformer_options": transformer_options,
|
|
},
|
|
{"original_block": block_wrapper},
|
|
)["img"]
|
|
else:
|
|
x = block(
|
|
x,
|
|
e=projected_embedding,
|
|
freqs=freqs,
|
|
context=context,
|
|
transformer_options=transformer_options,
|
|
)
|
|
if audio is not None:
|
|
scale = kwargs.get("audio_inject_scale", 1.0)
|
|
if isinstance(scale, torch.Tensor):
|
|
scale = float(scale.reshape(-1)[0].item())
|
|
x = self.audio_injector(
|
|
x,
|
|
index,
|
|
audio,
|
|
audio_global,
|
|
sequence_length,
|
|
scale=scale,
|
|
token_mask=kwargs.get("audio_inject_mask"),
|
|
)
|
|
|
|
x = self.head(x, embedding)
|
|
return self.unpatchify(x, grid_sizes)
|
|
|
|
forward_orig.__easy_bernini_s2v_forward_patch__ = True
|
|
forward_orig.__easy_bernini_s2v_original__ = original
|
|
WanModel_S2V.forward_orig = forward_orig
|
|
return True
|
|
|
|
|
|
def apply_bernini_s2v_model_patches() -> bool:
|
|
"""Install idempotent S2V context and masked-audio compatibility patches."""
|
|
try:
|
|
changed = _patch_s2v_forward()
|
|
changed = _patch_s2v_outer_forward() or changed
|
|
changed = _patch_audio_injector() or changed
|
|
changed = _patch_s2v_conditions() or changed
|
|
except (ImportError, AttributeError, TypeError, ValueError) as exc:
|
|
LOGGER.warning("Unable to apply Bernini S2V compatibility patches: %s", exc)
|
|
return False
|
|
if changed:
|
|
LOGGER.info("Applied Easy Media Bernini S2V compatibility patches.")
|
|
return changed
|