MMnode/tests/test_h3_tiling.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

199 lines
6.4 KiB
Python

from __future__ import annotations
import importlib.util
import math
import sys
import types
from pathlib import Path
import pytest
import torch
class _PackedLayout:
def __init__(
self,
text_len,
latent_t,
latent_h,
latent_w,
audio_t,
keyframes=None,
refs=None,
):
segments = [("text", text_len)]
video_rows = latent_h // 2 * (latent_w // 2)
for keyframe in keyframes or []:
latent = keyframe.get("latent")
if latent is not None:
segments.append(("cond", latent.shape[2] * video_rows))
audio_latent = keyframe.get("audio_latent")
if audio_latent is not None:
segments.append(("cond_audio", audio_latent.shape[-1] * 2))
for reference in refs or []:
kind = reference["kind"]
if kind == "image":
rows = math.ceil(reference["latent_h"] / 2) * math.ceil(
reference["latent_w"] / 2
)
segments.append(("ref_img", rows))
elif kind == "audio":
segments.append(("ref_audio", reference["ref_audio_t"] * 2))
segments.extend(
[
("audio", audio_t * 2),
("video", latent_t * video_rows),
]
)
self.signature = (text_len, latent_t, latent_h, latent_w, audio_t)
self.segments = []
offset = 0
for kind, count in segments:
self.segments.append((offset, offset + count, kind))
offset += count
self.position_ids = torch.zeros(offset, 3, dtype=torch.float64)
def _load_tiling_module():
root = Path(__file__).resolve().parents[1]
comfy = types.ModuleType("comfy")
comfy.__path__ = []
ldm = types.ModuleType("comfy.ldm")
ldm.__path__ = []
common_dit = types.ModuleType("comfy.ldm.common_dit")
common_dit.pad_to_patch_size = lambda tensor, _patch: tensor
minimax = types.ModuleType("comfy.ldm.minimax")
minimax.__path__ = []
minimax_model = types.ModuleType("comfy.ldm.minimax.model")
minimax_model.PackedLayout = _PackedLayout
model_base = types.ModuleType("comfy.model_base")
model_base.MiniMaxH3 = type("MiniMaxH3", (), {})
patcher_extension = types.ModuleType("comfy.patcher_extension")
patcher_extension.WrappersMP = types.SimpleNamespace(
DIFFUSION_MODEL="diffusion_model",
PREPARE_SAMPLING="prepare_sampling",
)
sampler_helpers = types.ModuleType("comfy.sampler_helpers")
comfy.ldm = ldm
comfy.model_base = model_base
comfy.patcher_extension = patcher_extension
comfy.sampler_helpers = sampler_helpers
ldm.common_dit = common_dit
ldm.minimax = minimax
minimax.model = minimax_model
names = {
"comfy": comfy,
"comfy.ldm": ldm,
"comfy.ldm.common_dit": common_dit,
"comfy.ldm.minimax": minimax,
"comfy.ldm.minimax.model": minimax_model,
"comfy.model_base": model_base,
"comfy.patcher_extension": patcher_extension,
"comfy.sampler_helpers": sampler_helpers,
}
originals = {name: sys.modules.get(name) for name in names}
sys.modules.update(names)
spec = importlib.util.spec_from_file_location(
"h3_tiling_under_test",
root / "modules" / "selflift" / "h3_tiling.py",
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module)
finally:
for name, original in originals.items():
if original is None:
sys.modules.pop(name, None)
else:
sys.modules[name] = original
return module
tiling = _load_tiling_module()
def test_tile_payload_rebuilds_visual_conditions_with_refs_and_audio_keyframes():
video = torch.zeros(1, 24, 3, 8, 12)
audio = torch.zeros(1, 32, 2, 9)
context = torch.zeros(1, 5, 8)
visual_keyframe = {
"resolved_frame_index": 0,
"latent": torch.arange(video.numel(), dtype=torch.float32).reshape(video.shape),
}
audio_keyframe = {
"resolved_frame_index": 1,
"audio_latent": torch.ones(1, 32, 2, 4),
}
reference_latent = torch.ones(1, 24, 1, 4, 4)
reference = {
"kind": "image",
"latent_h": 4,
"latent_w": 4,
"latent": reference_latent,
}
payload = {
"keyframes": [visual_keyframe, audio_keyframe],
"refs": [reference],
"cond_video_latents": [visual_keyframe["latent"], reference_latent],
}
tiled = tiling._tile_payload(
payload,
context,
video,
audio,
axis=4,
start=4,
end=12,
)
expected_keyframe = visual_keyframe["latent"].narrow(4, 4, 8)
assert torch.equal(tiled["keyframes"][0]["latent"], expected_keyframe)
assert tiled["keyframes"][1] == audio_keyframe
assert torch.equal(tiled["cond_video_latents"][0], expected_keyframe)
assert tiled["cond_video_latents"][1] is reference_latent
def test_tiled_forward_crops_video_mask_and_averages_audio_predictions(monkeypatch):
video = torch.zeros(1, 24, 3, 8, 12)
audio = torch.zeros(1, 32, 2, 9)
mask = torch.arange(3 * 8 * 12, dtype=torch.float32).reshape(1, 1, 3, 8, 12)
captured_masks = []
calls = 0
monkeypatch.setattr(tiling, "_tile_payload", lambda *_args, **_kwargs: {})
def executor(streams, *_args, **kwargs):
nonlocal calls
calls += 1
captured_masks.append(kwargs["denoise_mask"].clone())
return [
torch.full_like(streams[0], float(calls)),
torch.full_like(streams[1], float(2 + 4 * (calls - 1))),
]
output_video, output_audio = tiling._tiled_forward(
executor,
[video, audio],
torch.tensor([1.0]),
torch.zeros(1, 5, 8),
{},
denoise_mask=mask,
n_tiles=2,
)
assert calls == 2
assert torch.equal(captured_masks[0], mask.narrow(4, 0, 8))
assert torch.equal(captured_masks[1], mask.narrow(4, 4, 8))
assert output_video.shape == video.shape
assert torch.all(output_audio == 4.0)
def test_tile_denoise_mask_rejects_mismatched_spatial_shape():
video = torch.zeros(1, 24, 3, 8, 12)
mask = torch.ones(1, 1, 3, 7, 12)
with pytest.raises(ValueError, match="temporal/spatial shape"):
tiling._tile_denoise_mask(mask, video, axis=4, start=0, end=8)