MMnode/tests/test_h3_motion_context.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

420 lines
14 KiB
Python

from __future__ import annotations
import sys
import types
from pathlib import Path
import pytest
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from modules.motion_context import core
from modules.motion_context import drift_control_av
class _NestedTensor:
is_nested = True
def __init__(self, tensors):
self.tensors = tuple(tensors)
def unbind(self):
return self.tensors
@pytest.fixture
def nested_tensor_module(monkeypatch):
comfy = sys.modules.get("comfy", types.ModuleType("comfy"))
nested_tensor = types.ModuleType("comfy.nested_tensor")
nested_tensor.NestedTensor = _NestedTensor
monkeypatch.setitem(sys.modules, "comfy", comfy)
monkeypatch.setitem(sys.modules, "comfy.nested_tensor", nested_tensor)
monkeypatch.setattr(comfy, "nested_tensor", nested_tensor, raising=False)
def _av_latent(
*,
video_steps: int = 7,
height: int = 2,
width: int = 2,
) -> dict:
video = torch.arange(
24 * video_steps * height * width,
dtype=torch.float32,
).reshape(1, 24, video_steps, height, width)
audio_steps = round(core._pixel_frames(video_steps) * core.FRAME_RESCALE)
audio = torch.arange(
32 * 2 * audio_steps,
dtype=torch.float32,
).reshape(1, 32, 2, audio_steps)
return {"samples": [video, audio]}
def _video_latent(
*,
video_steps: int = 2,
height: int = 2,
width: int = 2,
) -> dict:
video = torch.arange(
24 * video_steps * height * width,
dtype=torch.float32,
).reshape(1, 24, video_steps, height, width)
return {"samples": video}
def test_h3_motion_context_latent_path_uses_native_keyframes_and_keeps_refs():
target = _av_latent()
context = _av_latent()
references = [{"kind": "image", "marker": "unchanged"}]
output, trim_frames = core.apply_motion_context(
conditioning=[[torch.tensor([1.0]), {"minimax_refs": references}]],
vae=object(),
latent=target,
context_length="5",
audio_context_length=3,
context_latent=context,
)
metadata = output[0][1]
assert trim_frames == 5
assert len(metadata["minimax_keyframes"]) == 3
video_keyframes = [
keyframe
for keyframe in metadata["minimax_keyframes"]
if keyframe.get("latent") is not None
]
audio_keyframes = [
keyframe
for keyframe in metadata["minimax_keyframes"]
if keyframe.get("audio_latent") is not None
]
assert [
keyframe["resolved_frame_index"] for keyframe in video_keyframes
] == [0, 1]
assert len(audio_keyframes) == 1
assert audio_keyframes[0]["resolved_frame_index"] == pytest.approx(2.4)
assert audio_keyframes[0]["audio_latent"].shape[-1] == 5
assert metadata["minimax_refs"] is references
assert "minimax_frame_count" not in metadata
def test_h3_motion_context_rejects_latent_resolution_change():
with pytest.raises(ValueError, match="does not match target"):
core.apply_motion_context(
conditioning=[[torch.tensor([1.0]), {}]],
vae=object(),
latent=_av_latent(width=2),
context_length="5",
context_latent=_av_latent(width=3),
)
def test_h3_hard_context_copies_matching_video_and_audio_windows(
nested_tensor_module,
):
target = _av_latent()
context = _av_latent()
output = core._hard_av_latent(
target,
context,
video_frames=5,
video_transition_steps=1,
audio_transition_steps=2,
)
output_video, output_audio = output["samples"].tensors
context_video, context_audio = context["samples"]
video_mask, audio_mask = output["noise_mask"].tensors
assert isinstance(output["samples"], _NestedTensor)
assert isinstance(output["noise_mask"], _NestedTensor)
assert torch.equal(output_video[:, :, :2], context_video[:, :, -2:])
assert torch.equal(output_audio[..., :8], context_audio[..., -8:])
assert torch.all(video_mask[:, :, 0] == 0)
assert torch.allclose(
video_mask[:, :, 1],
torch.full_like(video_mask[:, :, 1], 0.5),
)
assert torch.all(audio_mask[..., :6] == 0)
assert torch.all(audio_mask[..., 8:] == 1)
def test_h3_hard_context_merges_existing_masks_into_official_nested_tensor(
nested_tensor_module,
):
target = _av_latent()
target_video, target_audio = target["samples"]
target["noise_mask"] = [
torch.full_like(target_video, 0.25),
torch.full_like(target_audio, 0.4),
]
output = core._hard_av_latent(
target,
_av_latent(),
video_frames=5,
video_transition_steps=1,
audio_transition_steps=2,
)
video_mask, audio_mask = output["noise_mask"].tensors
assert isinstance(output["noise_mask"], _NestedTensor)
assert torch.all(video_mask[:, :, 0] == 0)
assert torch.all(video_mask[:, :, 1:] == 0.25)
assert torch.all(audio_mask[..., :6] == 0)
assert torch.allclose(
audio_mask[..., 6],
torch.full_like(audio_mask[..., 6], 1 / 3),
)
assert torch.all(audio_mask[..., 7:] == 0.4)
def test_h3_reencoded_anchor_only_locks_last_five_frames_of_soft_context(
nested_tensor_module,
):
target = _av_latent(video_steps=12)
target_video, target_audio = target["samples"]
anchor = _video_latent(video_steps=2)
anchor["samples"] = anchor["samples"] + 10_000
output = core._reencoded_anchor_av_latent(
target,
anchor,
context_frames=22,
anchor_frames=5,
)
output_video, output_audio = output["samples"].tensors
video_mask, audio_mask = output["noise_mask"].tensors
assert torch.equal(output_video[:, :, :5], target_video[:, :, :5])
assert torch.equal(output_video[:, :, 5:7], anchor["samples"])
assert torch.equal(output_video[:, :, 7:], target_video[:, :, 7:])
assert torch.all(video_mask[:, :, :5] == 1)
assert torch.all(video_mask[:, :, 5:7] == 0)
assert torch.all(video_mask[:, :, 7:] == 1)
assert torch.equal(output_audio, target_audio)
assert torch.all(audio_mask == 1)
def test_h3_reencoded_anchor_keeps_long_native_motion_context(
nested_tensor_module,
):
context = _av_latent(video_steps=7)
context["anchor_samples"] = _video_latent(video_steps=2)["samples"]
conditioning, trim_frames, latent = core.apply_reencoded_anchor_motion_context(
conditioning=[[torch.tensor([1.0]), {}]],
vae=object(),
latent=_av_latent(video_steps=12),
context_latent=context,
)
assert trim_frames == 22
assert len(conditioning[0][1]["minimax_keyframes"]) == 8
video_mask, _ = latent["noise_mask"].tensors
assert torch.all(video_mask[:, :, :5] == 1)
assert torch.all(video_mask[:, :, 5:7] == 0)
def test_h3_hires_reencoded_anchor_freezes_audio_and_only_video_boundary(
nested_tensor_module,
):
context = _av_latent(video_steps=7)
context["anchor_samples"] = _video_latent(video_steps=2)["samples"]
output, trim_frames = core.apply_hires_anchor_continuity(
current_hires_latent=_av_latent(video_steps=12),
previous_hires_latent=context,
)
video_mask, audio_mask = output["noise_mask"].tensors
assert trim_frames == 22
assert torch.all(video_mask[:, :, :5] == 1)
assert torch.all(video_mask[:, :, 5:7] == 0)
assert torch.all(video_mask[:, :, 7:] == 1)
assert torch.all(audio_mask == 0)
def test_h3_anchor_falls_back_to_last_two_context_video_tokens():
context = _av_latent(video_steps=7)
anchor = core._video_anchor_from_context_latent(context)
assert torch.equal(anchor["samples"], context["samples"][0][:, :, -2:])
def test_context_swap_noise_is_disposable_deterministic_and_video_only(
nested_tensor_module,
):
context = _av_latent()
original_video, original_audio = context["samples"]
original_video = original_video.clone()
original_audio = original_audio.clone()
first = core.apply_context_swap_noise(context, context_length="5", seed=42)
second = core.apply_context_swap_noise(context, context_length="5", seed=42)
first_video, first_audio = first["samples"].tensors
second_video, second_audio = second["samples"].tensors
assert torch.equal(context["samples"][0], original_video)
assert torch.equal(context["samples"][1], original_audio)
assert torch.equal(first_video[:, :, :5], original_video[:, :, :5])
assert not torch.equal(first_video[:, :, 5:], original_video[:, :, 5:])
assert torch.equal(first_video, second_video)
assert torch.equal(first_audio, original_audio)
assert torch.equal(second_audio, original_audio)
def test_context_swap_dynamic_mask_tracks_selflift_stage_resolution():
state = drift_control_av._DriftControlMaskState(
(1, 24, 7, 8, 10),
torch.tensor([1.0, 0.5, 0.0]),
prefix_steps=2,
)
active_shapes = [(1, 24, 7, 4, 6), (1, 32, 2, 20)]
active_model = types.SimpleNamespace(
inner_model=types.SimpleNamespace(latent_shapes=active_shapes)
)
packed_size = 24 * 7 * 4 * 6 + 32 * 2 * 20
packed_mask = torch.ones(1, 1, packed_size)
output = state.denoise_mask_function(
torch.tensor(1.0),
packed_mask,
{"model": active_model},
)
assert output.shape == packed_mask.shape
assert state.current_video_mask is not None
assert state.current_video_mask.shape == (1, 1, 7, 4, 6)
def test_drift_control_reports_original_and_current_pixel_resolution():
context = _av_latent(video_steps=7, height=2, width=3)
target = _av_latent(video_steps=12, height=4, width=5)
with pytest.raises(ValueError) as error:
drift_control_av.prepare_context_swap_latent(
target,
context,
5,
continue_audio=True,
freeze_audio=False,
)
assert "original context 48x32 pixels" in str(error.value)
assert "current target 80x64 pixels" in str(error.value)
assert "(1, 24, 2, 2, 3)" in str(error.value)
assert "(1, 24, 12, 4, 5)" in str(error.value)
def test_drift_control_rejects_video_channel_mismatch_with_shapes():
context = _av_latent(video_steps=7)
context["samples"][0] = context["samples"][0][:, :23]
with pytest.raises(ValueError, match=r"shapes differ: .*23.*24"):
drift_control_av.prepare_context_swap_latent(
_av_latent(video_steps=12),
context,
5,
continue_audio=True,
freeze_audio=False,
)
def test_h3_hard_context_requires_native_video_and_audio_keyframes(monkeypatch):
monkeypatch.setattr(
core,
"apply_motion_context",
lambda **_kwargs: ([[torch.tensor([1.0]), {"minimax_keyframes": []}]], 5),
)
with pytest.raises(RuntimeError, match=r"Motion Context 0\.4\.0\+"):
core.apply_hard_motion_context(
conditioning=[[torch.tensor([1.0]), {}]],
vae=object(),
latent=_av_latent(),
context_latent=_av_latent(),
context_length="5",
)
def test_h3_hard_context_keeps_native_audio_and_uses_it_for_hard_lock(
nested_tensor_module,
):
references = [{"kind": "audio", "marker": "user-reference"}]
context = _av_latent()
conditioning, trim_frames, latent = core.apply_hard_motion_context(
conditioning=[[torch.tensor([1.0]), {"minimax_refs": references}]],
vae=object(),
latent=_av_latent(),
context_latent=context,
context_length="5",
video_transition_steps=1,
audio_transition_steps=2,
)
metadata = conditioning[0][1]
audio_keyframe = next(
keyframe
for keyframe in metadata["minimax_keyframes"]
if keyframe.get("audio_latent") is not None
)
_, hard_audio = latent["samples"].tensors
assert trim_frames == 5
assert metadata["minimax_refs"] is references
assert torch.equal(
hard_audio[..., : audio_keyframe["audio_latent"].shape[-1]],
audio_keyframe["audio_latent"],
)
def test_h3_hires_continuity_uses_previous_video_and_freezes_current_audio(
nested_tensor_module,
):
current = _av_latent()
previous = _av_latent()
output, trim_frames = core.apply_hires_continuity(
current,
previous,
context_length="5",
video_transition_steps=1,
)
output_video, output_audio = output["samples"].tensors
previous_video, _ = previous["samples"]
_, current_audio = current["samples"]
video_mask, audio_mask = output["noise_mask"].tensors
assert trim_frames == 5
assert torch.equal(output_video[:, :, :2], previous_video[:, :, -2:])
assert torch.equal(output_audio, current_audio)
assert torch.all(video_mask[:, :, 2:] == 1)
assert torch.all(audio_mask == 0)
def test_trim_motion_context_latent_keeps_only_detached_cpu_av_tail(
nested_tensor_module,
):
latent = _av_latent(video_steps=12)
source_video, source_audio = latent["samples"]
anchor_samples = _video_latent()["samples"]
latent["anchor_samples"] = anchor_samples
output = core.trim_motion_context_latent(latent, context_length="22")
video, audio = output["samples"].tensors
assert video.shape[2] == 7
assert audio.shape[-1] == 37
assert torch.equal(video, source_video[:, :, -7:])
assert torch.equal(audio, source_audio[..., -37:])
assert video.device.type == "cpu"
assert audio.device.type == "cpu"
assert torch.equal(output["anchor_samples"], anchor_samples)
assert output["anchor_samples"].untyped_storage().data_ptr() != (
anchor_samples.untyped_storage().data_ptr()
)
assert video.untyped_storage().data_ptr() != source_video.untyped_storage().data_ptr()
assert audio.untyped_storage().data_ptr() != source_audio.untyped_storage().data_ptr()