Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
416 lines
14 KiB
Python
416 lines
14 KiB
Python
import importlib.util
|
|
import json
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
|
|
class _Port:
|
|
def __init__(self, name=None, **kwargs):
|
|
self.name = name
|
|
self.kwargs = kwargs
|
|
|
|
|
|
class _PortType:
|
|
@staticmethod
|
|
def Input(name, **kwargs):
|
|
return _Port(name, **kwargs)
|
|
|
|
@staticmethod
|
|
def Output(name=None, **kwargs):
|
|
return _Port(name, **kwargs)
|
|
|
|
|
|
class _Autogrow(_PortType):
|
|
class TemplatePrefix:
|
|
def __init__(self, **kwargs):
|
|
self.kwargs = kwargs
|
|
|
|
|
|
class _NodeOutput:
|
|
def __init__(self, *values):
|
|
self.values = values
|
|
|
|
|
|
class _Schema:
|
|
def __init__(self, **kwargs):
|
|
self.__dict__.update(kwargs)
|
|
|
|
|
|
def _load_wan_module(monkeypatch):
|
|
io = types.SimpleNamespace(
|
|
AudioEncoderOutput=_PortType,
|
|
Autogrow=_Autogrow,
|
|
Conditioning=_PortType,
|
|
ComfyNode=object,
|
|
Float=_PortType,
|
|
Image=_PortType,
|
|
Int=_PortType,
|
|
Latent=_PortType,
|
|
Mask=_PortType,
|
|
Model=types.SimpleNamespace(Input=_PortType.Input, Output=_PortType.Output, Type=object),
|
|
NodeOutput=_NodeOutput,
|
|
Schema=_Schema,
|
|
Vae=_PortType,
|
|
)
|
|
comfy_api = types.ModuleType("comfy_api")
|
|
comfy_api_latest = types.ModuleType("comfy_api.latest")
|
|
comfy_api_latest.io = io
|
|
comfy_api.latest = comfy_api_latest
|
|
|
|
comfy = types.ModuleType("comfy")
|
|
comfy.__path__ = []
|
|
comfy_model_management = types.ModuleType("comfy.model_management")
|
|
comfy_model_management.intermediate_device = lambda: torch.device("cpu")
|
|
comfy_patcher_extension = types.ModuleType("comfy.patcher_extension")
|
|
comfy_patcher_extension.WrappersMP = types.SimpleNamespace(DIFFUSION_MODEL="diffusion_model")
|
|
comfy_utils = types.ModuleType("comfy.utils")
|
|
|
|
def common_upscale(image, width, height, *_args):
|
|
return torch.nn.functional.interpolate(image.float(), size=(height, width), mode="area")
|
|
|
|
comfy_utils.common_upscale = common_upscale
|
|
comfy.model_management = comfy_model_management
|
|
comfy.patcher_extension = comfy_patcher_extension
|
|
comfy.utils = comfy_utils
|
|
|
|
node_helpers = types.ModuleType("node_helpers")
|
|
|
|
def conditioning_set_values(conditioning, values):
|
|
return [(item[0], {**item[1], **values}) for item in conditioning]
|
|
|
|
node_helpers.conditioning_set_values = conditioning_set_values
|
|
|
|
package = types.ModuleType("easy_media")
|
|
package.__path__ = []
|
|
nodes_package = types.ModuleType("easy_media.nodes")
|
|
nodes_package.__path__ = []
|
|
utils_package = types.ModuleType("easy_media.utils")
|
|
utils_package.__path__ = []
|
|
|
|
monkeypatch.setitem(sys.modules, "comfy_api", comfy_api)
|
|
monkeypatch.setitem(sys.modules, "comfy_api.latest", comfy_api_latest)
|
|
monkeypatch.setitem(sys.modules, "comfy", comfy)
|
|
monkeypatch.setitem(sys.modules, "comfy.model_management", comfy_model_management)
|
|
monkeypatch.setitem(sys.modules, "comfy.patcher_extension", comfy_patcher_extension)
|
|
monkeypatch.setitem(sys.modules, "comfy.utils", comfy_utils)
|
|
monkeypatch.setitem(sys.modules, "node_helpers", node_helpers)
|
|
monkeypatch.setitem(sys.modules, "easy_media", package)
|
|
monkeypatch.setitem(sys.modules, "easy_media.nodes", nodes_package)
|
|
monkeypatch.setitem(sys.modules, "easy_media.utils", utils_package)
|
|
|
|
utils_path = Path(__file__).parents[1] / "utils" / "bernini_s2v.py"
|
|
utils_spec = importlib.util.spec_from_file_location("easy_media.utils.bernini_s2v", utils_path)
|
|
utils_module = importlib.util.module_from_spec(utils_spec)
|
|
monkeypatch.setitem(sys.modules, utils_spec.name, utils_module)
|
|
utils_spec.loader.exec_module(utils_module)
|
|
|
|
path = Path(__file__).parents[1] / "nodes" / "wan.py"
|
|
spec = importlib.util.spec_from_file_location("easy_media.nodes.wan", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
monkeypatch.setitem(sys.modules, spec.name, module)
|
|
spec.loader.exec_module(module)
|
|
return module, utils_module
|
|
|
|
|
|
def _load_model_patch_module():
|
|
path = Path(__file__).parents[1] / "utils" / "bernini_s2v_model_patch.py"
|
|
spec = importlib.util.spec_from_file_location("bernini_s2v_model_patch_under_test", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _conditioning():
|
|
return [(torch.zeros(1), {})]
|
|
|
|
|
|
def _audio(value, frames=8):
|
|
return {"audio_samples": frames * 1000, "encoded_audio_all_layers": [torch.full((1, frames, 1), value)]}
|
|
|
|
|
|
def _assert_complete_chinese_localization(schema, translation):
|
|
assert translation["display_name"]
|
|
assert translation["description"]
|
|
assert set(translation["inputs"]) == {input_.name for input_ in schema.inputs}
|
|
assert set(translation["outputs"]) == {str(index) for index in range(len(schema.outputs))}
|
|
|
|
for input_ in schema.inputs:
|
|
localized_input = translation["inputs"][input_.name]
|
|
assert localized_input["name"]
|
|
if input_.kwargs.get("tooltip"):
|
|
assert localized_input["tooltip"]
|
|
|
|
for output_index in range(len(schema.outputs)):
|
|
assert translation["outputs"][str(output_index)]["name"]
|
|
|
|
|
|
def test_schema_exposes_one_optional_primary_speaker_and_optional_second_speaker(monkeypatch):
|
|
module, _ = _load_wan_module(monkeypatch)
|
|
|
|
schema = module.EasyBerniniS2VConditioning.define_schema()
|
|
inputs = {input_.name: input_ for input_ in schema.inputs}
|
|
|
|
assert schema.node_id == "easy berniniS2VConditioning"
|
|
assert inputs["audio_0"].kwargs["optional"] is True
|
|
assert inputs["mask_0"].kwargs["optional"] is True
|
|
assert inputs["audio_1"].kwargs["optional"] is True
|
|
assert inputs["mask_1"].kwargs["optional"] is True
|
|
|
|
|
|
def test_single_audio_without_mask_preserves_global_v1_conditioning(monkeypatch):
|
|
module, helpers = _load_wan_module(monkeypatch)
|
|
monkeypatch.setattr(helpers, "build_audio_embed", lambda length, segments: torch.ones(1, 1, 1, 24))
|
|
|
|
output = module.EasyBerniniS2VConditioning.execute(
|
|
_conditioning(), _conditioning(), object(), 64, 48, 21, 1, audio_0=_audio(1),
|
|
)
|
|
positive, negative, latent = output.values
|
|
|
|
assert torch.all(positive[0][1]["audio_embed"] == 1)
|
|
assert torch.count_nonzero(negative[0][1]["audio_embed"]) == 0
|
|
assert "audio_inject_mask" not in positive[0][1]
|
|
assert "audio_inject_scale" not in positive[0][1]
|
|
assert latent["samples"].shape == (1, 16, 6, 6, 8)
|
|
|
|
|
|
def test_masked_single_speaker_uses_spatial_audio_injection(monkeypatch):
|
|
module, helpers = _load_wan_module(monkeypatch)
|
|
monkeypatch.setattr(helpers, "build_audio_embed", lambda length, segments: torch.ones(1, 1, 1, 24))
|
|
|
|
output = module.EasyBerniniS2VConditioning.execute(
|
|
_conditioning(), _conditioning(), object(), 64, 48, 21, 1,
|
|
audio_0=_audio(1), mask_0=torch.ones(1, 48, 64), audio_inject_scale=1.5,
|
|
)
|
|
positive, negative, _ = output.values
|
|
|
|
assert positive[0][1]["audio_inject_scale"] == 1.5
|
|
assert positive[0][1]["audio_inject_mask"].shape == (1, 6, 12, 1)
|
|
assert torch.count_nonzero(positive[0][1]["audio_inject_mask"]) > 0
|
|
assert torch.count_nonzero(negative[0][1]["audio_inject_mask"]) == 0
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("kwargs", "message"),
|
|
[
|
|
({"audio_1": _audio(2), "mask_0": torch.ones(1, 8, 8), "mask_1": torch.ones(1, 8, 8)}, "audio_0"),
|
|
({"audio_0": _audio(1), "audio_1": _audio(2), "mask_1": torch.ones(1, 8, 8)}, "mask_0"),
|
|
({"audio_0": _audio(1), "audio_1": _audio(2), "mask_0": torch.ones(1, 8, 8)}, "mask_1"),
|
|
],
|
|
)
|
|
def test_dual_speaker_inputs_require_a_complete_pair(monkeypatch, kwargs, message):
|
|
module, _ = _load_wan_module(monkeypatch)
|
|
|
|
with pytest.raises(ValueError, match=message):
|
|
module.EasyBerniniS2VConditioning.execute(
|
|
_conditioning(), _conditioning(), object(), 64, 48, 21, 1, **kwargs,
|
|
)
|
|
|
|
|
|
def test_context_order_is_source_video_then_reference_video_then_images(monkeypatch):
|
|
module, _ = _load_wan_module(monkeypatch)
|
|
|
|
class FakeVae:
|
|
def encode(self, image):
|
|
return torch.tensor([float(image.mean())])
|
|
|
|
output = module.EasyBerniniS2VConditioning.execute(
|
|
_conditioning(), _conditioning(), FakeVae(), 16, 16, 5, 1,
|
|
source_video=torch.full((1, 16, 16, 3), 0.1),
|
|
reference_video=torch.full((1, 16, 16, 3), 0.2),
|
|
reference_images={
|
|
"reference_image_1": torch.full((1, 16, 16, 3), 0.4),
|
|
"reference_image_0": torch.full((1, 16, 16, 3), 0.3),
|
|
},
|
|
ref_max_size=16,
|
|
)
|
|
context = output.values[0][0][1]["context_latents"]
|
|
|
|
assert [round(item.item(), 1) for item in context] == [0.1, 0.2, 0.3, 0.4]
|
|
|
|
|
|
def test_masked_audio_injector_limits_the_residual_to_selected_tokens():
|
|
module = _load_model_patch_module()
|
|
|
|
class Identity:
|
|
def __call__(self, value):
|
|
return value
|
|
|
|
class ConstantAttention:
|
|
def __call__(self, x, context):
|
|
return torch.ones_like(x)
|
|
|
|
injector = types.SimpleNamespace(
|
|
injected_block_id={0: 0},
|
|
enable_adain=False,
|
|
injector_pre_norm_feat=[Identity()],
|
|
injector=[ConstantAttention()],
|
|
)
|
|
original_calls = []
|
|
|
|
def original(*args, **kwargs):
|
|
original_calls.append((args, kwargs))
|
|
return torch.full_like(args[1], 9)
|
|
|
|
x = torch.zeros(1, 4, 1)
|
|
audio = torch.zeros(1, 2, 1, 1)
|
|
token_mask = torch.tensor([[[[1.0], [0.0]], [[0.0], [1.0]]]])
|
|
|
|
result = module.masked_audio_injector_forward(
|
|
original, injector, x, 0, audio, None, 4, scale=2.0, token_mask=token_mask,
|
|
)
|
|
|
|
assert original_calls == []
|
|
assert result.flatten().tolist() == [2.0, 0.0, 0.0, 2.0]
|
|
|
|
|
|
def test_audio_injector_without_mask_delegates_to_comfyui():
|
|
module = _load_model_patch_module()
|
|
expected = torch.ones(1, 1, 1)
|
|
|
|
def original(*args, **kwargs):
|
|
assert kwargs == {"scale": 1.25}
|
|
return expected
|
|
|
|
result = module.masked_audio_injector_forward(
|
|
original, object(), torch.zeros(1, 1, 1), 0, torch.zeros(1, 1, 1), None, 1,
|
|
scale=1.25, token_mask=None,
|
|
)
|
|
|
|
assert result is expected
|
|
|
|
|
|
def test_unified_node_has_complete_chinese_localization(monkeypatch):
|
|
module, _ = _load_wan_module(monkeypatch)
|
|
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
|
|
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
|
|
|
|
translation = node_defs["easy berniniS2VConditioning"]
|
|
|
|
_assert_complete_chinese_localization(
|
|
module.EasyBerniniS2VConditioning.define_schema(),
|
|
translation,
|
|
)
|
|
assert translation["display_name"] == "简易 Bernini S2V 条件"
|
|
assert set(translation["inputs"]) == {
|
|
"positive", "negative", "vae", "width", "height", "length", "batch_size",
|
|
"audio_0", "mask_0", "audio_1", "mask_1", "second_speaker_start_frame",
|
|
"source_video", "reference_video", "reference_images", "ref_max_size",
|
|
"mask_crossfade_frames", "audio_inject_scale",
|
|
}
|
|
assert translation["outputs"] == {
|
|
"0": {"name": "正向条件"},
|
|
"1": {"name": "负向条件"},
|
|
"2": {"name": "潜空间"},
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("class_name", "node_id"),
|
|
[
|
|
("BerniniModelPatch", "BerniniModelPatch"),
|
|
("BerniniConditioning", "BerniniConditioning"),
|
|
],
|
|
)
|
|
def test_bernini_nodes_have_complete_chinese_localization(monkeypatch, class_name, node_id):
|
|
module, _ = _load_wan_module(monkeypatch)
|
|
schema = getattr(module, class_name).define_schema()
|
|
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
|
|
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
|
|
|
|
assert schema.node_id == node_id
|
|
_assert_complete_chinese_localization(schema, node_defs[node_id])
|
|
|
|
|
|
def test_s2v_condition_patch_transports_context_latents_without_parent_support(monkeypatch):
|
|
module = _load_model_patch_module()
|
|
|
|
class Condition:
|
|
def __init__(self, cond):
|
|
self.cond = cond
|
|
|
|
def _copy_with(self, cond):
|
|
return Condition(cond)
|
|
|
|
class FakeS2V:
|
|
def process_latent_in(self, latent):
|
|
return latent + 1
|
|
|
|
def extra_conds(self, **kwargs):
|
|
return {}
|
|
|
|
def resize_cond_for_context_window(self, *args, **kwargs):
|
|
return "parent"
|
|
|
|
comfy = types.ModuleType("comfy")
|
|
comfy.__path__ = []
|
|
comfy_conds = types.ModuleType("comfy.conds")
|
|
comfy_conds.CONDList = Condition
|
|
comfy_conds.CONDRegular = Condition
|
|
comfy_model_base = types.ModuleType("comfy.model_base")
|
|
comfy_model_base.WAN22_S2V = FakeS2V
|
|
comfy.conds = comfy_conds
|
|
monkeypatch.setitem(sys.modules, "comfy", comfy)
|
|
monkeypatch.setitem(sys.modules, "comfy.conds", comfy_conds)
|
|
monkeypatch.setitem(sys.modules, "comfy.model_base", comfy_model_base)
|
|
|
|
assert module._patch_s2v_conditions() is True
|
|
result = FakeS2V().extra_conds(context_latents=[torch.tensor([2.0])])
|
|
|
|
assert len(result["context_latents"].cond) == 1
|
|
assert result["context_latents"].cond[0].item() == 3.0
|
|
|
|
|
|
def test_prepare_s2v_context_adds_one_source_id_rope_block_per_stream(monkeypatch):
|
|
module = _load_model_patch_module()
|
|
common_dit = types.ModuleType("comfy.ldm.common_dit")
|
|
common_dit.pad_to_patch_size = lambda latent, patch_size: latent
|
|
monkeypatch.setitem(sys.modules, "comfy.ldm.common_dit", common_dit)
|
|
|
|
class FakeModel:
|
|
patch_size = (1, 2, 2)
|
|
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def rope_encode(self, time, height, width, **kwargs):
|
|
self.calls.append((time, height, width, kwargs["source_id"]))
|
|
return torch.full((1, time * height * width, 1), kwargs["source_id"], dtype=torch.float32)
|
|
|
|
model = FakeModel()
|
|
base_freqs = torch.zeros(1, 2, 1)
|
|
context = [torch.zeros(1, 16, 1, 2, 2), torch.zeros(1, 16, 2, 1, 1)]
|
|
|
|
kwargs, freqs = module.prepare_s2v_context(
|
|
model,
|
|
context,
|
|
base_freqs,
|
|
device=torch.device("cpu"),
|
|
dtype=torch.float32,
|
|
transformer_options={},
|
|
)
|
|
|
|
assert model.calls == [(1, 2, 2, 1), (2, 1, 1, 2)]
|
|
assert kwargs["context_latents"] == context
|
|
assert freqs.shape[1] == 8
|
|
assert freqs[:, 2:6].unique().item() == 1
|
|
assert freqs[:, 6:].unique().item() == 2
|
|
|
|
|
|
def test_outer_forward_patch_requires_a_compatible_inner_forward():
|
|
module = _load_model_patch_module()
|
|
|
|
def unpatched():
|
|
pass
|
|
|
|
def patched():
|
|
pass
|
|
|
|
patched.__easy_bernini_s2v_forward_patch__ = True
|
|
|
|
assert module.s2v_forward_patch_ready(unpatched) is False
|
|
assert module.s2v_forward_patch_ready(patched) is True
|