MMnode/tests/test_multitrack_info_output.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

6188 lines
210 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import importlib.util
import inspect
import io as stdlib_io
import json
import logging
import os
import shutil
import subprocess
import sys
import types
from fractions import Fraction
from pathlib import Path
import pytest
import torch
from PIL import Image
class _Port:
def __init__(self, name=None, **kwargs):
self.name = name
self.kwargs = kwargs
class _PortType:
@staticmethod
def Input(name, **kwargs):
return _Port(name, **kwargs)
@staticmethod
def Output(name=None, **kwargs):
return _Port(name, **kwargs)
class _DynamicCombo(_PortType):
@staticmethod
def Option(name, inputs):
return name, inputs
class _CustomType(_PortType):
pass
class _NodeOutput:
def __init__(self, *values, **kwargs):
self.values = values
self.kwargs = kwargs
self.expand = kwargs.get("expand")
def __getitem__(self, index):
return self.values[index]
class _GraphNode:
def __init__(self, class_type, node_id, inputs):
self.class_type = class_type
self.node_id = node_id
self.inputs = inputs
def out(self, index):
return [self.node_id, index]
class _GraphBuilder:
def __init__(self):
self.nodes = {}
def node(self, class_type, id=None, **inputs):
node_id = id or str(len(self.nodes))
node = _GraphNode(class_type, node_id, inputs)
self.nodes[node_id] = node
return node
def finalize(self):
return {
node_id: {"class_type": node.class_type, "inputs": node.inputs}
for node_id, node in self.nodes.items()
}
class _Schema:
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
class _VideoComponents:
def __init__(self, images, audio, frame_rate):
self.images = images
self.audio = audio
self.frame_rate = frame_rate
class _FakeVideo:
def __init__(self, components, source=None):
self.components = components
self.source = source
self.components_calls = 0
self.trim_calls = []
def get_components(self):
self.components_calls += 1
return self.components
def get_dimensions(self):
return self.components.images.shape[2], self.components.images.shape[1]
def get_stream_source(self):
return self.source
def get_duration(self):
return self.components.images.shape[0] / float(self.components.frame_rate)
def as_trimmed(self, start_time=0, duration=0, strict_duration=True):
self.trim_calls.append((start_time, duration, strict_duration))
return self
class _ProgressBar:
instances = []
def __init__(self, total):
self.total = total
self.current = 0
self.updates = []
self.instances.append(self)
def update_absolute(self, value, total=None, preview=None):
if total is not None:
self.total = total
self.current = value
self.updates.append(value)
class _InputImpl:
loaded_sources = []
rebuilt_components = []
@classmethod
def VideoFromFile(cls, source):
cls.loaded_sources.append(source)
return _FakeVideo(
_VideoComponents(
images=torch.zeros(2, 360, 640, 3),
audio=None,
frame_rate=Fraction(24),
),
source=source,
)
@classmethod
def VideoFromComponents(cls, components):
cls.rebuilt_components.append(components)
return _FakeVideo(components)
class _VideoContainer:
AUTO = "auto"
class _VideoCodec:
AUTO = "auto"
def _load_basic_module():
_ProgressBar.instances.clear()
_InputImpl.loaded_sources.clear()
_InputImpl.rebuilt_components.clear()
io = types.SimpleNamespace(
AnyType=_PortType,
Audio=_PortType,
Boolean=_PortType,
Clip=_PortType,
Combo=_PortType,
ComfyNode=object,
Custom=lambda **kwargs: _CustomType(),
DynamicCombo=_DynamicCombo,
Float=_PortType,
Hidden=types.SimpleNamespace(extra_pnginfo="EXTRA_PNGINFO"),
Image=_PortType,
Int=_PortType,
LatentUpscaleModel=_PortType,
Mask=_PortType,
Model=_PortType,
NodeOutput=_NodeOutput,
Schema=_Schema,
String=_PortType,
Video=_PortType,
)
comfy_api = types.ModuleType("comfy_api")
comfy_api_latest = types.ModuleType("comfy_api.latest")
comfy_api_latest.io = io
comfy_api_latest.InputImpl = _InputImpl
comfy_api_latest.Types = types.SimpleNamespace(
VideoComponents=_VideoComponents,
VideoContainer=_VideoContainer,
VideoCodec=_VideoCodec,
)
comfy_api.latest = comfy_api_latest
comfy = types.ModuleType("comfy")
comfy_utils = types.ModuleType("comfy.utils")
comfy_utils.ProgressBar = _ProgressBar
comfy.utils = comfy_utils
core_nodes = types.ModuleType("nodes")
core_nodes.NODE_CLASS_MAPPINGS = {}
comfy_execution = types.ModuleType("comfy_execution")
graph_utils = types.ModuleType("comfy_execution.graph_utils")
graph_utils.GraphBuilder = _GraphBuilder
graph_utils.is_link = lambda value: (
isinstance(value, list)
and len(value) == 2
and isinstance(value[0], str)
and isinstance(value[1], (int, float))
)
comfy_execution.graph_utils = graph_utils
package = types.ModuleType("easy_media")
package.__path__ = []
nodes_package = types.ModuleType("easy_media.nodes")
nodes_package.__path__ = []
utils_module = types.ModuleType("easy_media.utils")
utils_module.__path__ = []
utils_module.FFMPEG_RESIZE_METHODS = frozenset({
"stretch", "resize", "pad", "pad (white)", "crop",
})
for name in (
"audio_db_to_gain",
"audio_is_muted",
"audio_volume_db",
"burn_subtitles_with_ffmpeg",
"collect_multitrack_subtitle_segments",
"default_subtitle_filename",
"equirectangular_to_perspective",
"ffprobe_info",
"frames_to_seconds",
"load_audio_waveform",
"load_image_tensor",
"iter_valid_audio_inputs",
"merge_audio_inputs",
"merge_video_track_with_ffmpeg",
"canonicalize_multitrack_slot_content",
"multitrack_is_shared_reference",
"multitrack_is_muted_image",
"multitrack_media_identity",
"multitrack_shared_task_images",
"multitrack_segments_in_window",
"multitrack_slot_name",
"multitrack_slot_media_types",
"multitrack_task_images_with_shared",
"parse_subtitle_text",
"resize_image",
"resize_video_with_ffmpeg",
"resolve_video_path",
"silence",
"split_list_outputs",
"trim_audio",
"video_input_to_local_file",
"write_ass_file",
"write_srt_file",
):
setattr(utils_module, name, lambda *args, **kwargs: None)
utils_module.audio_db_to_gain = lambda value: 10 ** (float(value) / 20)
utils_module.audio_is_muted = lambda value: bool(value.get("muted", False))
utils_module.audio_volume_db = lambda value: float(value.get("volume_db", 0.0))
utils_module.default_subtitle_filename = lambda prefix="easy_multitrack_subtitles": f"{prefix}_20260704_120000"
utils_module.silence = lambda sample_rate, duration, channels=2: torch.zeros(
1, channels, max(1, int(sample_rate * duration)),
)
utils_module.split_list_outputs = lambda values, output_count=10: (
list(values[:output_count]) + [None] * max(0, output_count - len(values))
)
def trim_audio(audio, start_index, duration):
sample_rate = audio["sample_rate"]
start = round(start_index * sample_rate)
end = start + round(duration * sample_rate)
return {"waveform": audio["waveform"][..., start:end], "sample_rate": sample_rate}
utils_module.trim_audio = trim_audio
def iter_valid_audio_inputs(*values):
result = []
for value in values:
if isinstance(value, dict) and "waveform" in value:
result.append(value)
elif isinstance(value, list):
result.extend(iter_valid_audio_inputs(*value))
return result
utils_module.iter_valid_audio_inputs = iter_valid_audio_inputs
utils_module.video_input_to_local_file = lambda video, **kwargs: (video.get_stream_source(), [])
utils_module.log_node_info = lambda *args, **kwargs: None
utils_module.audio_data_uris = lambda values: []
utils_module.image_tensor_data_uris = lambda values, **kwargs: []
utils_module.video_data_uris = lambda values: []
utils_module.video_frame_data_uris = lambda values, **kwargs: []
utils_module.LLAMACPP_MODEL = "llama.cpp (本地)"
utils_module.MINIMAX_MODEL = "h3-context-ir (海螺官方)"
utils_module.PROMPT_ENHANCER_MODELS = [
utils_module.MINIMAX_MODEL,
"doubao (火山引擎)",
"glm (智谱)",
"doubao (RunningHub)",
"glm (RunningHub)",
utils_module.LLAMACPP_MODEL,
]
utils_module.PROMPT_ENHANCER_MAX_TOKENS = {
"doubao (火山引擎)": (4096, 131072),
"glm (智谱)": (65536, 131072),
"doubao (RunningHub)": (4096, 131072),
"glm (RunningHub)": (65536, 131072),
utils_module.LLAMACPP_MODEL: (512, 768),
}
utils_module.PromptEnhancerApiError = RuntimeError
utils_module.PromptEnhancerClient = object
utils_module.prompt_enhancer_supports_video_url = lambda model: False
utils_module.prompt_enhancer_video_inputs = lambda model, values: []
utils_module.minimax_length_to_seconds = lambda length: 5
folder_paths = types.ModuleType("folder_paths")
folder_paths.get_temp_directory = lambda: "/tmp"
folder_paths.get_output_directory = lambda: "/tmp"
sys.modules["folder_paths"] = folder_paths
prompt_override_path = Path(__file__).parents[1] / "utils" / "prompt_override.py"
prompt_override_spec = importlib.util.spec_from_file_location(
"easy_media.utils.prompt_override",
prompt_override_path,
)
prompt_override_module = importlib.util.module_from_spec(prompt_override_spec)
prompt_override_spec.loader.exec_module(prompt_override_module)
for name in prompt_override_module.__all__:
setattr(utils_module, name, getattr(prompt_override_module, name))
sys.modules.update({
"easy_media": package,
"easy_media.utils": utils_module,
})
multitrack_path = Path(__file__).parents[1] / "utils" / "multitrack.py"
multitrack_spec = importlib.util.spec_from_file_location(
"easy_media.utils.multitrack",
multitrack_path,
)
multitrack_module = importlib.util.module_from_spec(multitrack_spec)
sys.modules[multitrack_spec.name] = multitrack_module
multitrack_spec.loader.exec_module(multitrack_module)
# Load the real ``utils/video.py`` so its public helpers (especially
# ``extract_video_audio`` and ``ffmpeg_extract_audio``) are reachable via
# ``from ..utils.video import …`` from h3_project.py and nodes/basic.py
# during the test. Without this, those imports would resolve to whatever
# SimpleNamespace stub we ship and the audio-lock tests couldn't exercise
# the real fallback / cache behaviour.
media_module = types.SimpleNamespace(
AUDIO_EXTENSIONS=frozenset({
".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg",
}),
)
sys.modules["easy_media.utils.media"] = media_module
video_path = Path(__file__).parents[1] / "utils" / "video.py"
video_spec = importlib.util.spec_from_file_location(
"easy_media.utils.video",
video_path,
)
video_module = importlib.util.module_from_spec(video_spec)
sys.modules[video_spec.name] = video_module
video_spec.loader.exec_module(video_module)
utils_module.render_single_video_segment_with_ffmpeg = (
video_module.render_single_video_segment_with_ffmpeg
)
from contextlib import nullcontext
utils_module.log_stage_time = lambda *_args, **_kwargs: nullcontext()
utils_module.canonicalize_multitrack_slot_content = multitrack_module.canonicalize_multitrack_slot_content
utils_module.multitrack_is_shared_reference = multitrack_module.multitrack_is_shared_reference
utils_module.multitrack_is_muted_image = multitrack_module.multitrack_is_muted_image
utils_module.multitrack_media_identity = multitrack_module.multitrack_media_identity
utils_module.multitrack_shared_task_images = multitrack_module.multitrack_shared_task_images
utils_module.multitrack_segments_in_window = multitrack_module.multitrack_segments_in_window
utils_module.multitrack_slot_name = multitrack_module.multitrack_slot_name
utils_module.multitrack_slot_media_types = multitrack_module.multitrack_slot_media_types
utils_module.multitrack_task_images_with_shared = multitrack_module.multitrack_task_images_with_shared
sys.modules.update({
"comfy_api": comfy_api,
"comfy_api.latest": comfy_api_latest,
"folder_paths": folder_paths,
"easy_media": package,
"easy_media.utils": utils_module,
"easy_media.utils.audio_gain": types.SimpleNamespace(
audio_db_to_gain=utils_module.audio_db_to_gain,
audio_is_muted=utils_module.audio_is_muted,
audio_volume_db=utils_module.audio_volume_db,
),
"easy_media.utils.panorama": types.SimpleNamespace(
equirectangular_to_perspective=(
utils_module.equirectangular_to_perspective
),
),
})
h3_project_path = Path(__file__).parents[1] / "utils" / "h3_project.py"
h3_project_spec = importlib.util.spec_from_file_location(
"easy_media.utils.h3_project",
h3_project_path,
)
h3_project_module = importlib.util.module_from_spec(h3_project_spec)
sys.modules[h3_project_spec.name] = h3_project_module
h3_project_spec.loader.exec_module(h3_project_module)
prompt_builder_module = types.ModuleType("easy_media.utils.prompt_builder")
prompt_builder_module.calls = []
def build_prompt_request(task_type, user_prompt, **kwargs):
prompt_builder_module.calls.append((task_type, user_prompt, kwargs))
return f"system:{task_type}", f"api:{user_prompt}", False
prompt_builder_module.build_prompt_request = build_prompt_request
prompt_builder_module.build_llm_prompt = (
lambda system_prompt, user_prompt, json_mode=False:
f"llm:{system_prompt}:{user_prompt}:{json_mode}"
)
sys.modules.update({
"comfy_api": comfy_api,
"comfy_api.latest": comfy_api_latest,
"comfy": comfy,
"comfy.utils": comfy_utils,
"comfy_execution": comfy_execution,
"comfy_execution.graph_utils": graph_utils,
"nodes": core_nodes,
"easy_media": package,
"easy_media.nodes": nodes_package,
"easy_media.utils": utils_module,
"easy_media.utils.prompt_builder": prompt_builder_module,
"easy_media.utils.prompt_override": prompt_override_module,
"easy_media.utils.h3_project": h3_project_module,
})
path = Path(__file__).parents[1] / "nodes" / "basic.py"
spec = importlib.util.spec_from_file_location("easy_media.nodes.basic", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
module._h3_project_module = h3_project_module
module._multitrack_module = multitrack_module
return module
def _load_related_node_module(module_name):
_load_basic_module()
path = Path(__file__).parents[1] / "nodes" / f"{module_name}.py"
spec = importlib.util.spec_from_file_location(
f"easy_media.nodes.{module_name}",
path,
)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _load_image_module():
_load_basic_module()
path = Path(__file__).parents[1] / "nodes" / "image.py"
spec = importlib.util.spec_from_file_location("easy_media.nodes.image", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _load_video_utils_module(input_directory):
folder_paths = types.ModuleType("folder_paths")
folder_paths.get_annotated_filepath = lambda path: str(input_directory / path)
folder_paths.get_input_directory = lambda: str(input_directory)
folder_paths.get_output_directory = lambda: str(input_directory)
folder_paths.get_temp_directory = lambda: str(input_directory)
sys.modules["folder_paths"] = folder_paths
package = types.ModuleType("easy_media")
package.__path__ = []
utils_package = types.ModuleType("easy_media.utils")
utils_package.__path__ = []
media_module = types.ModuleType("easy_media.utils.media")
media_module.AUDIO_EXTENSIONS = frozenset({".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"})
sys.modules["easy_media"] = package
sys.modules["easy_media.utils"] = utils_package
sys.modules["easy_media.utils.media"] = media_module
path = Path(__file__).parents[1] / "utils" / "video.py"
spec = importlib.util.spec_from_file_location("easy_media.utils.video", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _load_prompt_builder_module():
path = Path(__file__).parents[1] / "utils" / "prompt_builder.py"
spec = importlib.util.spec_from_file_location("prompt_builder_under_test", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_multitrack_info_output_schema_has_only_required_outputs():
module = _load_basic_module()
schema = module.MultiTrackInfoOutput.define_schema()
assert [input_.name for input_ in schema.inputs] == ["tracks_info"]
assert [output.name for output in schema.outputs] == [
"WIDTH",
"HEIGHT",
"TOTAL_FRAMES",
"FPS",
"TASK_COUNT",
]
def test_multitrack_info_output_counts_task_segments():
module = _load_basic_module()
tracks_info = {
"width": 1280,
"height": 720,
"total_length": 97,
"frame_rate": 24,
"tracks": [
{"type": "task", "segments": [{"id": "task-1"}, {"id": "task-2"}]},
{"type": "video", "segments": [{"id": "video-1"}]},
{"type": "task", "segments": [{"id": "task-3"}, None]},
],
}
result = module.MultiTrackInfoOutput.execute(json.dumps(tracks_info))
assert result.values == (1280, 720, 97, 24.0, 3)
def test_multitrack_info_output_counts_marker_ranges_instead_of_task_segments():
module = _load_basic_module()
tracks_info = {
"width": 1280,
"height": 720,
"total_length": 13,
"frame_rate": 24,
"task_markers": [
{"id": "marker-8", "frame": 8},
{"id": "marker-4", "frame": 4},
],
"tracks": [{
"type": "task",
"segments": [{"id": "one", "start_frame": 0, "end_frame": 12}],
}],
}
result = module.MultiTrackInfoOutput.execute(tracks_info)
assert result.values[-1] == 3
def test_multitrack_info_output_treats_an_end_marker_as_marker_mode():
module = _load_basic_module()
tracks_info = {
"total_length": 12,
"frame_rate": 24,
"task_markers": [{"id": "end", "frame": 12}],
"tracks": [{
"type": "task",
"segments": [
{"id": "first", "start_frame": 0, "end_frame": 6},
{"id": "second", "start_frame": 6, "end_frame": 12},
],
}],
}
result = module.MultiTrackInfoOutput.execute(tracks_info)
assert result.values[-1] == 1
def test_multitrack_task_entries_keep_the_exclusive_timeline_end_without_an_end_marker():
module = _load_basic_module()
tracks_info = {
"timeline_total_length": 12,
"total_length": 13,
"frame_rate": 24,
"task_markers": [{"id": "split", "frame": 4}],
"tracks": [{
"type": "task",
"segments": [
{"id": "first", "start_frame": 0, "end_frame": 4},
{"id": "second", "start_frame": 4, "end_frame": 12},
],
}],
}
entries = module._multitrack_task_entries(tracks_info)
assert [
(entry["start_frame"], entry["end_frame"])
for entry in entries
] == [(0, 4), (4, 12)]
def test_multitrack_add_subtitle_to_video_saves_srt_to_output_srt(monkeypatch, tmp_path):
module = _load_basic_module()
output_dir = tmp_path / "output"
temp_dir = tmp_path / "temp"
output_dir.mkdir()
temp_dir.mkdir()
module.folder_paths.get_output_directory = lambda: str(output_dir)
module.folder_paths.get_temp_directory = lambda: str(temp_dir)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
video = _FakeVideo(
_VideoComponents(torch.zeros(24, 360, 640, 3), None, Fraction(24)),
source=str(source),
)
segment = types.SimpleNamespace(start=0.0, end=1.0, text="hello", style={})
calls = {}
monkeypatch.setattr(module, "collect_multitrack_subtitle_segments", lambda info: [segment])
monkeypatch.setattr(module, "default_subtitle_filename", lambda prefix="x": f"{prefix}_stamp")
def fake_write_srt(segments, path):
calls["srt"] = path
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("1\n00:00:00,000 --> 00:00:01,000\nhello\n", encoding="utf-8")
return path
def fake_write_ass(segments, path, width, height):
calls["ass"] = (path, width, height)
path.write_text("[Script Info]\n", encoding="utf-8")
return path
def fake_burn(video_path, subtitle_path, output_path):
calls["burn"] = (video_path, subtitle_path, output_path)
Path(output_path).write_bytes(b"subtitled")
return output_path
monkeypatch.setattr(module, "write_srt_file", fake_write_srt)
monkeypatch.setattr(module, "write_ass_file", fake_write_ass)
monkeypatch.setattr(module, "burn_subtitles_with_ffmpeg", fake_burn)
result = module.MultiTrackAddSubtitleToVideo.execute(
{"tracks": []},
video,
"output",
)
assert calls["srt"] == output_dir / "srt" / "source_stamp.srt"
assert calls["ass"][1:] == (640, 360)
assert calls["burn"][0] == str(source)
assert result.values[0].source == calls["burn"][2]
assert not calls["ass"][0].exists()
def test_add_subtitle_to_video_schema_accepts_multiline_text():
module = _load_basic_module()
schema = module.AddSubtitleToVideo.define_schema()
assert schema.node_id == "easy addSubtitleToVideo"
assert [input_.name for input_ in schema.inputs] == [
"subtitle_text",
"video",
"srt_save",
"font_size",
]
assert schema.inputs[0].kwargs["multiline"] is True
assert schema.inputs[3].kwargs == {"default": 16, "min": 8, "max": 96, "step": 1}
assert [output.name for output in schema.outputs] == ["VIDEO"]
def test_add_subtitle_to_video_parses_text_and_uses_multitrack_burn_pipeline(monkeypatch, tmp_path):
module = _load_basic_module()
output_dir = tmp_path / "output"
temp_dir = tmp_path / "temp"
output_dir.mkdir()
temp_dir.mkdir()
module.folder_paths.get_output_directory = lambda: str(output_dir)
module.folder_paths.get_temp_directory = lambda: str(temp_dir)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
video = _FakeVideo(
_VideoComponents(torch.zeros(24, 360, 640, 3), None, Fraction(24)),
source=str(source),
)
segment = types.SimpleNamespace(start=0.0, end=1.0, text="hello", style={})
calls = {}
def fake_parse(value, style=None):
calls["text"] = value
calls["style"] = style
return [segment]
def fake_write_srt(segments, path):
calls["srt"] = (segments, path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("srt", encoding="utf-8")
return path
def fake_write_ass(segments, path, width, height):
calls["ass"] = (segments, path, width, height)
path.write_text("ass", encoding="utf-8")
return path
def fake_burn(video_path, subtitle_path, output_path):
calls["burn"] = (video_path, subtitle_path, output_path)
Path(output_path).write_bytes(b"subtitled")
return output_path
monkeypatch.setattr(module, "parse_subtitle_text", fake_parse)
monkeypatch.setattr(module, "default_subtitle_filename", lambda prefix="x": f"{prefix}_stamp")
monkeypatch.setattr(module, "write_srt_file", fake_write_srt)
monkeypatch.setattr(module, "write_ass_file", fake_write_ass)
monkeypatch.setattr(module, "burn_subtitles_with_ffmpeg", fake_burn)
result = module.AddSubtitleToVideo.execute(
"[00:00.000 --> 00:01.000] hello",
video,
"output",
28,
)
assert calls["text"] == "[00:00.000 --> 00:01.000] hello"
assert calls["style"] == {"font_size": 28}
assert calls["srt"] == ([segment], output_dir / "srt" / "source_stamp.srt")
assert calls["ass"][0] == [segment]
assert calls["ass"][2:] == (640, 360)
assert calls["burn"][0] == str(source)
assert result.values[0].source == calls["burn"][2]
assert not calls["ass"][1].exists()
def test_multitrack_editor_includes_selected_dimensions_in_tracks_info():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)"},
"Wan",
{"total_length": 81, "frame_rate": 16, "tracks": []},
)
tracks_info = result.values[0]
assert tracks_info["width"] == 1280
assert tracks_info["height"] == 720
def test_multitrack_editor_calculates_megapixel_dimensions_on_the_format_grid():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{
"resolution": "width x height (megapixels)",
"aspect_ratio": "16:9 (Widescreen)",
"megapixels": 1.0,
},
"Wan",
{"total_length": 81, "frame_rate": 16, "tracks": []},
)
tracks_info = result.values[0]
assert (tracks_info["width"], tracks_info["height"]) == (1368, 768)
def test_timeline_editor_calculates_megapixel_dimensions_on_the_format_grid():
module = _load_basic_module()
result = module.TimelineEditor.execute(
{
"resolution": "width x height (megapixels)",
"aspect_ratio": "16:9 (Widescreen)",
"megapixels": 1.0,
},
"MiniMax",
{"total_length": 121, "frame_rate": 24, "tracks": []},
)
timeline_info = result.values[0]
assert (timeline_info["width"], timeline_info["height"]) == (1376, 768)
def test_timeline_editor_aligns_minimax_total_frames_to_model_grid():
module = _load_basic_module()
result = module.TimelineEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 121, "frame_rate": 24, "tracks": []},
)
assert result.values[0]["total_length"] == 124
assert result.values[0]["format"] == "MiniMax"
def test_multitrack_editor_aligns_minimax_output_frames_to_model_grid():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 120, "frame_rate": 24, "tracks": []},
)
assert result.values[0]["total_length"] == 124
assert result.values[0]["format"] == "MiniMax"
def test_multitrack_editor_outputs_none_for_empty_minimax_media_tracks():
module = _load_basic_module()
track_data = {
"total_length": 120,
"frame_rate": 24,
"tracks": [
{"id": "task-track", "type": "task", "segments": []},
{"id": "video-track", "type": "video", "segments": []},
{"id": "audio-track", "type": "audio", "segments": []},
],
}
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
track_data,
)
assert result.values[2] == [None]
assert result.values[3] == [None]
@pytest.mark.parametrize(
("format_name", "expected_total_length"),
[("MiniMax", 56), ("Wan", 49)],
)
def test_multitrack_editor_uses_short_segment_range_instead_of_stale_default_length(
format_name,
expected_total_length,
):
module = _load_basic_module()
track_data = {
"total_length": 120,
"frame_rate": 24,
"tracks": [{
"id": "task-track",
"type": "task",
"segments": [{
"id": "short-task",
"start_frame": 0,
"end_frame": 48,
"content": {"media_type": "none"},
}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
format_name,
track_data,
)
tracks_info = result.values[0]
assert tracks_info["total_length"] == expected_total_length
assert tracks_info["timeline_total_length"] == 48
def test_timeline_editor_aligns_minimax_frames_at_the_timeline_frame_rate():
module = _load_basic_module()
result = module.TimelineEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 65, "frame_rate": 16, "tracks": []},
)
timeline_info = result.values[0]
assert timeline_info["total_length"] == 56
assert timeline_info["frame_rate"] == 16
assert module.TimelineInfoOutput.execute(timeline_info, "default").values[4] == 16.0
def test_timeline_editor_keeps_source_duration_when_aligning_minimax_output():
module = _load_basic_module()
audio = {"waveform": torch.ones(1, 1, 200), "sample_rate": 16}
result = module.TimelineEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 65, "frame_rate": 16, "tracks": []},
prompt_override="four seconds",
audio=audio,
)
assert result.values[0]["total_length"] == 56
assert result.values[2]["waveform"].shape[-1] == 64
def test_multitrack_editor_does_not_add_one_before_minimax_alignment():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 107, "frame_rate": 24, "tracks": []},
)
assert result.values[0]["total_length"] == 107
@pytest.mark.parametrize(
("frame_rate", "total_length", "expected_total_length"),
[(16, 64, 56), (20, 80, 73), (24, 96, 90)],
)
def test_multitrack_editor_aligns_minimax_frames_at_the_timeline_frame_rate(
frame_rate,
total_length,
expected_total_length,
):
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": total_length, "frame_rate": frame_rate, "tracks": []},
)
tracks_info = result.values[0]
assert tracks_info["total_length"] == expected_total_length
assert tracks_info["frame_rate"] == float(frame_rate)
assert module.MultiTrackInfoOutput.execute(tracks_info).values[3] == float(frame_rate)
def test_multitrack_editor_minimax_prompt_override_does_not_add_a_timeline_frame():
module = _load_basic_module()
audio = {"waveform": torch.ones(1, 1, 120), "sample_rate": 24}
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 120, "frame_rate": 24, "tracks": []},
prompt_override="@音频1 four seconds [0-4s]",
audio=[audio],
)
tracks_info, _images, audio_out, video_out = result.values
assert tracks_info["total_length"] == 90
assert tracks_info["timeline_total_length"] == 96
assert audio_out[0]["waveform"].shape[-1] == 96
assert video_out == [None]
def test_multitrack_editor_converts_four_and_a_half_seconds_to_minimax_frames():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 108, "frame_rate": 24, "tracks": []},
)
assert result.values[0]["total_length"] == 107
def test_multitrack_editor_converts_six_seconds_to_nearest_minimax_frames():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 144, "frame_rate": 24, "tracks": []},
)
assert result.values[0]["total_length"] == 141
def test_multitrack_editor_minimax_total_length_sums_task_durations_and_skips_gaps():
module = _load_basic_module()
track_data = {
"total_length": 144,
"frame_rate": 24,
"tracks": [{
"id": "task-track",
"type": "task",
"segments": [
{"id": "task-0", "start_frame": 0, "end_frame": 48, "content": {"media_type": "none"}},
{"id": "task-1", "start_frame": 96, "end_frame": 144, "content": {"media_type": "none"}},
],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
track_data,
)
tracks_info = result.values[0]
assert tracks_info["timeline_total_length"] == 144
assert tracks_info["total_length"] == 90
def test_multitrack_editor_removes_legacy_volume_fields():
module = _load_basic_module()
track_data = {
"volume": 0.5,
"volume_db": -8,
"tracks": [{
"id": "audio-track",
"type": "audio",
"volume": 0,
"volume_db": -2,
"segments": [{
"id": "audio-segment",
"volume": 0,
"content": {"media_type": "none", "volume": 0, "volume_db": -3},
}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)"},
"Wan",
track_data,
)
tracks_info = result.values[0]
track = tracks_info["tracks"][0]
assert tracks_info["volume_db"] == -8
assert "volume" not in tracks_info
assert "volume" not in track
assert "volume" not in track["segments"][0]
assert "volume" not in track["segments"][0]["content"]
def test_multitrack_editor_uses_first_video_for_auto_and_rebuilds_all_videos():
module = _load_basic_module()
resize_calls = []
def fake_resize(images, width, height, method):
resize_calls.append((images.shape, width, height, method))
return torch.zeros(images.shape[0], height, width, images.shape[-1])
module._multitrack_module.resize_image = module.resize_image = fake_resize
audio = {"waveform": torch.ones(1, 2, 100), "sample_rate": 48000}
first_video = _FakeVideo(
_VideoComponents(torch.zeros(2, 360, 640, 3), audio, Fraction(30))
)
second_video = _FakeVideo(
_VideoComponents(torch.zeros(3, 480, 640, 3), None, Fraction(25))
)
track_data = {
"total_length": 5,
"frame_rate": 24,
"tracks": [{
"id": "video-track",
"type": "video",
"segments": [
{"id": "v1", "start_frame": 0, "end_frame": 2, "content": {"media_type": "video", "source_type": "slot", "slot_name": "video1"}},
{"id": "v2", "start_frame": 3, "end_frame": 5, "content": {"media_type": "video", "source_type": "slot", "slot_name": "video2"}},
],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)", "resize_method": "pad"},
"None",
track_data,
video=[first_video, second_video],
)
tracks_info, _images, _audio, videos = result.values
assert (tracks_info["width"], tracks_info["height"]) == (640, 360)
assert [video.get_dimensions() for video in videos] == [(640, 360)]
assert resize_calls == [
(torch.Size([3, 480, 640, 3]), 640, 360, "pad"),
]
assert first_video.components_calls == 1
assert _InputImpl.rebuilt_components[0].frame_rate == Fraction(25)
assert _ProgressBar.instances[-1].current == _ProgressBar.instances[-1].total
def test_multitrack_editor_scales_shortest_and_longest_from_first_video():
module = _load_basic_module()
assert module._resolve_configured_dimensions(
{"resolution": "width x height (shortest)", "resize_to_pixel": 320},
"None",
(1280, 720),
) == (569, 320)
assert module._resolve_configured_dimensions(
{"resolution": "width x height (longest)", "resize_to_pixel": 640},
"None",
(1280, 720),
) == (640, 360)
def test_multitrack_editor_outputs_task_images_as_unresized_list_items():
module = _load_basic_module()
image_one = torch.zeros(1, 10, 20, 3)
image_two = torch.zeros(1, 30, 40, 3)
track_data = {
"tracks": [{
"id": "task-track",
"type": "task",
"segments": [
{"id": "task-1", "content": {"media_type": "none", "images": [
{"id": "image-a", "source_type": "slot", "slot_name": "image1"},
]}},
{"id": "task-2", "content": {"media_type": "none", "images": [
{"id": "image-b", "source_type": "slot", "slot_name": "image2"},
]}},
],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "crop"},
"None",
track_data,
image=[image_one, image_two],
)
tracks_info, images, _audio, _videos = result.values
assert len(images) == 2
assert torch.equal(images[0], image_one)
assert torch.equal(images[1], image_two)
assert [tuple(image.shape) for image in images] == [(1, 10, 20, 3), (1, 30, 40, 3)]
task_images = [
image
for segment in tracks_info["tracks"][0]["segments"]
for image in segment["content"]["images"]
]
assert [image["media_index"] for image in task_images] == [0, 1]
def test_multitrack_editor_keeps_muted_task_image_metadata_but_skips_reference_output():
module = _load_basic_module()
muted_image = torch.zeros(1, 10, 20, 3)
active_image = torch.ones(1, 10, 20, 3)
track_data = {
"tracks": [{
"id": "task-track",
"type": "task",
"segments": [{
"id": "task-1",
"content": {"media_type": "none", "images": [
{"id": "muted", "source_type": "slot", "slot_name": "image1", "muted": True},
{"id": "active", "source_type": "slot", "slot_name": "image2"},
]},
}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "crop"},
"None",
track_data,
image=[muted_image, active_image],
)
tracks_info, images, _audio, _videos = result.values
image_items = tracks_info["tracks"][0]["segments"][0]["content"]["images"]
assert len(images) == 1
assert torch.equal(images[0], active_image)
assert image_items[0]["muted"] is True
assert "media_index" not in image_items[0]
assert image_items[1]["media_index"] == 0
def test_multitrack_editor_passes_task_markers_through_tracks_info():
module = _load_basic_module()
markers = [{"id": "marker-1", "frame": 24}]
result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)"},
"None",
{
"total_length": 48,
"frame_rate": 24,
"task_markers": markers,
"tracks": [{
"id": "task",
"type": "task",
"segments": [{"start_frame": 0, "end_frame": 48, "content": {"media_type": "none"}}],
}],
},
)
assert result.values[0]["task_markers"] == markers
def test_timeline_editor_empty_prompt_override_uses_original_timeline_data():
module = _load_basic_module()
result = module.TimelineEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{
"total_length": 8,
"frame_rate": 2,
"tracks": [{
"type": "maintain",
"segments": [{
"start_frame": 1,
"end_frame": 4,
"content": {"text": "original prompt", "images": [], "type": "flf"},
}],
}],
},
prompt_override="",
)
timeline_info = result.values[0]
assert timeline_info["segments"] == [{
"start_frame": 1,
"end_frame": 4,
"prompt": "original prompt",
"images": [],
}]
def test_timeline_editor_prompt_override_frame_range_total_length_is_not_incremented():
module = _load_basic_module()
result = module.TimelineEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{"total_length": 999, "frame_rate": 24, "tracks": []},
prompt_override="hello [0-120]|nice [120-240]",
)
timeline_info = result.values[0]
assert timeline_info["total_length"] == 241
assert [(segment["start_frame"], segment["end_frame"]) for segment in timeline_info["segments"]] == [
(0, 120),
(120, 240),
]
def test_timeline_editor_prompt_override_second_range_uses_exclusive_second_end():
module = _load_basic_module()
result = module.TimelineEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{"total_length": 999, "frame_rate": 24, "tracks": []},
prompt_override="hello [0-5s]|nice [5s-10s]",
)
timeline_info = result.values[0]
assert timeline_info["total_length"] == 241
assert [(segment["start_frame"], segment["end_frame"]) for segment in timeline_info["segments"]] == [
(0, 120),
(120, 240),
]
def test_multitrack_editor_prompt_override_builds_slot_audio_and_video_tracks():
module = _load_basic_module()
image = torch.zeros(1, 10, 20, 3)
audio_one = {"waveform": torch.ones(1, 1, 4), "sample_rate": 2}
audio_two = {"waveform": torch.full((1, 1, 4), 2.0), "sample_rate": 2}
video_one = _FakeVideo(_VideoComponents(torch.ones(2, 2, 2, 3), None, Fraction(2)))
video_two = _FakeVideo(_VideoComponents(torch.full((2, 2, 2, 3), 2.0), None, Fraction(2)))
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)", "resize_method": "stretch"},
"None",
{"total_length": 4, "frame_rate": 2, "tracks": [{"id": "old", "type": "task", "segments": []}]},
prompt_override="@image1 @audio2 @视频2 first [0-2,ref]|@video1 second [2-4]",
image=[image],
audio=[audio_one, audio_two],
video=[video_one, video_two],
)
tracks_info, images, audio, videos = result.values
assert tracks_info["media_loading"] == "eager"
assert len(images) == 1
assert torch.equal(images[0], image)
assert [track["type"] for track in tracks_info["tracks"]] == ["task", "video", "audio"]
task_track = tracks_info["tracks"][0]
assert [segment["content"]["text"] for segment in task_track["segments"]] == ["first", "second"]
assert task_track["segments"][0]["content"]["task_mode"] == "ref"
assert task_track["segments"][0]["content"]["images"][0]["slot_name"] == "image1"
assert task_track["segments"][0]["content"]["images"][0]["media_index"] == 0
video_track = tracks_info["tracks"][1]
assert [segment["content"]["slot_name"] for segment in video_track["segments"]] == ["video2", "video1"]
assert len(videos) == 1
frames = videos[0].get_components().images
assert [float(frames[index].mean()) for index in range(4)] == [2.0, 2.0, 1.0, 1.0]
audio_track = tracks_info["tracks"][2]
assert audio_track["segments"][0]["content"]["slot_name"] == "audio2"
assert len(audio) == 1
assert audio[0]["waveform"].flatten().tolist() == [2.0, 2.0, 0.0, 0.0]
def test_multitrack_editor_prompt_override_outputs_each_audio_slot_as_a_separate_track():
module = _load_basic_module()
audio_one = {"waveform": torch.ones(1, 1, 4), "sample_rate": 2}
audio_two = {"waveform": torch.full((1, 1, 4), 2.0), "sample_rate": 2}
result = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)"},
"None",
{"total_length": 4, "frame_rate": 2, "tracks": []},
prompt_override="@音频1 第一段 [0-2]|@音频2 第二段 [2-4]",
audio=[audio_one, audio_two],
)
tracks_info, _images, audio, _videos = result.values
audio_tracks = [track for track in tracks_info["tracks"] if track["type"] == "audio"]
assert len(audio_tracks) == 2
assert len(audio) == 2
assert audio[0]["waveform"].flatten().tolist() == [1.0, 1.0, 0.0, 0.0]
assert audio[1]["waveform"].flatten().tolist() == [0.0, 0.0, 2.0, 2.0]
def test_multitrack_editor_prompt_override_ranges_do_not_extend_total_length():
module = _load_basic_module()
frame_result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{"total_length": 999, "frame_rate": 24, "tracks": []},
prompt_override="hello [0-120]|nice [120-240]",
)
frame_tracks_info = frame_result.values[0]
assert frame_tracks_info["total_length"] == 241
assert [(segment["start_frame"], segment["end_frame"]) for segment in frame_tracks_info["tracks"][0]["segments"]] == [
(0, 120),
(120, 240),
]
seconds_result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{"total_length": 999, "frame_rate": 24, "tracks": []},
prompt_override="hello [0-5s]|nice [5s-10s]",
)
seconds_tracks_info = seconds_result.values[0]
assert seconds_tracks_info["total_length"] == 241
assert [(segment["start_frame"], segment["end_frame"]) for segment in seconds_tracks_info["tracks"][0]["segments"]] == [
(0, 120),
(120, 240),
]
def test_multitrack_editor_prompt_override_preserves_explicit_task_type():
module = _load_basic_module()
image = torch.zeros(1, 10, 20, 3)
editor_result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{"total_length": 50, "frame_rate": 24, "tracks": []},
prompt_override="@image1 text only generation [0-49,t2v]",
image=[image],
)
tracks_info, images, _audio, _videos = editor_result.values
task_content = tracks_info["tracks"][0]["segments"][0]["content"]
task_result = module.MultiTrackTaskOutput.execute(
[tracks_info],
[images],
[],
[],
[0],
["default"],
)
assert task_content["task_type"] == "t2v"
assert task_content["task_mode"] == "default"
assert task_result.values[2] == "t2v"
def test_multitrack_editor_prompt_override_outputs_custom_task_type_string():
module = _load_basic_module()
editor_result = module.MultiTrackEditor.execute(
{"resolution": "1280 x 720 (16:9)", "resize_method": "stretch"},
"None",
{"total_length": 50, "frame_rate": 24, "tracks": []},
prompt_override="custom model route [0-49,wan-2.2-fun]",
)
tracks_info, images, _audio, _videos = editor_result.values
task_content = tracks_info["tracks"][0]["segments"][0]["content"]
task_result = module.MultiTrackTaskOutput.execute(
[tracks_info],
[images],
[],
[],
[0],
["default"],
)
assert task_content["task_type"] == "wan-2.2-fun"
assert task_result.values[2] == "wan-2.2-fun"
def test_multitrack_editor_projects_panorama_images_to_video_dimensions_for_task_output():
module = _load_basic_module()
panorama = torch.zeros(1, 180, 360, 3)
video = _FakeVideo(
_VideoComponents(torch.zeros(2, 360, 640, 3), None, Fraction(24))
)
projection_calls = []
def fake_projection(image, view, width, height):
projection_calls.append((image, view, width, height))
return torch.full((1, height, width, 3), 0.25)
module.equirectangular_to_perspective = fake_projection
panorama_view = {
"version": 1,
"projection": "equirectangular",
"yaw": 30,
"pitch": -10,
"hfov": 75,
"aspect_ratio": 1.6,
}
track_data = {
"total_length": 2,
"frame_rate": 24,
"tracks": [
{
"id": "task-track",
"type": "task",
"segments": [{
"id": "task-1",
"start_frame": 0,
"end_frame": 2,
"content": {
"media_type": "none",
"images": [{
"id": "pano",
"source_type": "slot",
"slot_name": "image1",
"panorama_view": panorama_view,
}],
},
}],
},
{
"id": "video-track",
"type": "video",
"segments": [{
"id": "video-1",
"start_frame": 0,
"end_frame": 2,
"content": {
"media_type": "video",
"source_type": "slot",
"slot_name": "video1",
},
}],
},
],
}
editor_result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)", "resize_method": "stretch"},
"None",
track_data,
image=[panorama],
video=[video],
)
tracks_info, images, _audio, videos = editor_result.values
task_image = tracks_info["tracks"][0]["segments"][0]["content"]["images"][0]
assert len(projection_calls) == 1
assert torch.equal(projection_calls[0][0], panorama)
assert projection_calls[0][1] == panorama_view
assert projection_calls[0][2:] == (640, 360)
assert images[0].shape == (1, 360, 640, 3)
assert task_image["panorama_view"] == panorama_view
assert tracks_info["media"]["images"] == [images[0]]
task_result = module.MultiTrackTaskOutput.execute(
[tracks_info],
[images],
[],
[videos],
[0],
["default"],
)
assert task_result.values[4] == [images[0]]
assert task_result.values[4][0].shape == (1, 360, 640, 3)
def test_multitrack_editor_splits_connected_image_batches_into_list_items():
module = _load_basic_module()
image_batch = torch.zeros(2, 12, 18, 3)
track_data = {
"tracks": [{
"id": "task-track",
"type": "task",
"segments": [{"id": "task-1", "content": {
"media_type": "none",
"images": [
{"id": "image-a", "source_type": "slot", "slot_name": "image1"},
{"id": "image-b", "source_type": "slot", "slot_name": "image2"},
],
}}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)"},
"None",
track_data,
image=[image_batch],
)
images = result.values[1]
assert [tuple(image.shape) for image in images] == [(1, 12, 18, 3), (1, 12, 18, 3)]
def test_multitrack_editor_defers_path_video_loading():
module = _load_basic_module()
module._multitrack_module.resolve_video_path = module.resolve_video_path = (
lambda source_type, file_path, local_path, url: "resolved/video.mp4"
)
module._multitrack_module.resize_image = module.resize_image = (
lambda images, width, height, method: images
)
track_data = {
"tracks": [{
"id": "video-track",
"type": "video",
"segments": [{"id": "v1", "content": {
"media_type": "video",
"source_type": "input",
"file_path": "clip.mp4",
}}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)", "resize_method": "stretch"},
"None",
track_data,
)
assert result.values[0]["media_loading"] == "deferred"
assert result.values[3] == []
assert _InputImpl.loaded_sources == []
def test_multitrack_editor_probes_deferred_url_dimensions_without_downloading():
module = _load_basic_module()
probes = []
module.ffprobe_info = lambda source: probes.append(source) or {"width": 1280, "height": 720}
module._multitrack_module.resolve_video_path = module.resolve_video_path = (
lambda *args: pytest.fail("URL metadata must not use the download resolver")
)
track_data = {
"tracks": [{
"type": "video",
"segments": [{
"start_frame": 0,
"end_frame": 8,
"content": {
"media_type": "video",
"source_type": "url",
"url": "https://example.com/video.mp4",
},
}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)"},
"None",
track_data,
)
assert probes == ["https://example.com/video.mp4"]
assert (result.values[0]["width"], result.values[0]["height"]) == (1280, 720)
assert result.values[3] == []
def test_multitrack_editor_lazy_inputs_include_prompt_override_slots():
module = _load_basic_module()
schema = module.MultiTrackEditor.define_schema()
inputs = {input_port.name: input_port for input_port in schema.inputs}
assert inputs["image"].kwargs["lazy"] is True
assert inputs["audio"].kwargs["lazy"] is True
assert inputs["video"].kwargs["lazy"] is True
assert module.MultiTrackEditor.check_lazy_status(
{"resolution": "640 x 360"},
"None",
{"total_length": 8, "frame_rate": 2, "tracks": []},
prompt_override="@图片1 @音频1 @video1 prompt [0-8]",
) == ["audio", "image", "video"]
def test_multitrack_editor_lazy_inputs_include_minimax_prompt_override_slots():
module = _load_basic_module()
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["@图片2 first", "@视频1 second"],
"duration": "10,10",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
assert module.MultiTrackEditor.check_lazy_status(
{"resolution": "640 x 360"},
"MiniMax",
{"total_length": 8, "frame_rate": 2, "tracks": []},
prompt_override=payload,
) == ["image", "video"]
def test_multitrack_editor_materializes_only_the_referenced_audio_slot(monkeypatch):
module = _load_basic_module()
source_audio = {"waveform": torch.ones(1, 1, 4), "sample_rate": 2}
track_data = {
"total_length": 4,
"frame_rate": 2,
"tracks": [
{"id": "tasks", "type": "task", "segments": [{
"id": "task",
"start_frame": 0,
"end_frame": 4,
"content": {"media_type": "none", "images": [{
"id": "image",
"source_type": "input",
"file_path": "ordinary.png",
}]},
}]},
{"id": "audio", "type": "audio", "segments": [{
"id": "audio-slot",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "audio",
"source_type": "slot",
"slot_name": "audio1",
},
}]},
{"id": "video", "type": "video", "segments": [{
"id": "ordinary-video",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "ordinary.mp4",
},
}]},
],
}
assert module.MultiTrackEditor.check_lazy_status(
{"resolution": "32 x 32 (1:1)"},
"None",
track_data,
) == ["audio"]
monkeypatch.setattr(
module,
"_resolve_timeline_image_item",
lambda *_args, **_kwargs: pytest.fail("ordinary image loaded eagerly"),
)
monkeypatch.setattr(
module,
"_resolve_multitrack_video",
lambda *_args, **_kwargs: pytest.fail("ordinary video loaded eagerly"),
)
result = module.MultiTrackEditor.execute(
{"resolution": "32 x 32 (1:1)"},
"None",
track_data,
audio=[source_audio],
)
tracks_info, images, audio, videos = result.values
assert tracks_info["media_loading"] == "eager"
assert tracks_info["eager_media_types"] == ["audio"]
assert images == []
assert len(audio) == 1
assert videos == []
def test_multitrack_task_output_loads_only_deferred_task_video():
module = _load_basic_module()
module._multitrack_module.resolve_video_path = module.resolve_video_path = (
lambda source_type, file_path, local_path, url: f"resolved/{file_path}"
)
track_data = {
"total_length": 12,
"frame_rate": 2,
"tracks": [
{
"id": "task-track",
"type": "task",
"segments": [{
"id": "task",
"start_frame": 4,
"end_frame": 8,
"content": {"media_type": "none", "text": "task", "images": []},
}],
},
{
"id": "video-track",
"type": "video",
"segments": [{
"id": "video",
"start_frame": 0,
"end_frame": 12,
"origin_start_frame": 0,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "clip.mp4",
},
}],
},
],
}
editor_result = module.MultiTrackEditor.execute(
{"resolution": "640 x 360 (16:9)", "resize_method": "stretch"},
"None",
track_data,
)
tracks_info = editor_result.values[0]
assert tracks_info["media_loading"] == "deferred"
assert _InputImpl.loaded_sources == []
task_result = module.MultiTrackTaskOutput.execute(
tracks_info,
images=[None],
audio=[None],
video=[None],
task_index=0,
prompt_format="default",
)
assert _InputImpl.loaded_sources == ["resolved/clip.mp4"]
assert len(task_result.values[6]) == 1
assert task_result.values[6][0].get_components().images.shape[0] == 4
def test_multitrack_task_output_preserves_deferred_audio_crop_and_silence():
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(12, dtype=torch.float32).reshape(1, 1, 12),
"sample_rate": 2,
}
module._resolve_multitrack_audio = lambda content, audio_input: source_audio
track_data = {
"total_length": 8,
"frame_rate": 2,
"tracks": [
{
"id": "task-track",
"type": "task",
"segments": [{
"start_frame": 4,
"end_frame": 8,
"content": {"media_type": "none", "text": "task", "images": []},
}],
},
{
"id": "audio-track",
"type": "audio",
"segments": [{
"start_frame": 6,
"end_frame": 8,
"origin_start_frame": 0,
"content": {
"media_type": "audio",
"source_type": "input",
"file_path": "audio.wav",
},
}],
},
],
}
tracks_info = module.MultiTrackEditor.execute(
{"resolution": "640 x 360 (16:9)"},
"None",
track_data,
).values[0]
task_result = module.MultiTrackTaskOutput.execute(
tracks_info,
task_index=0,
prompt_format="default",
)
waveform = task_result.values[5][0]["waveform"].flatten().tolist()
assert waveform == [0.0, 0.0, 6.0, 7.0]
def test_multitrack_task_output_returns_the_locked_track_deferred_audio():
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
module._resolve_multitrack_audio = lambda content, audio_input: source_audio
tracks_info = {
"media_loading": "deferred",
"format": "MiniMax",
"width": 640,
"height": 360,
"total_length": 8,
"frame_rate": 2,
"tracks": [
{
"type": "task",
"segments": [{
"start_frame": 4,
"end_frame": 8,
"content": {"media_type": "none", "images": []},
}],
},
{
"type": "audio",
"audio_locked": True,
"segments": [{
"id": "locked-audio",
"start_frame": 4,
"end_frame": 8,
"origin_start_frame": 4,
"content": {
"media_type": "audio",
"source_type": "input",
"file_path": "audio.wav",
},
}],
},
],
}
result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=0)
assert result.values[8]["waveform"].flatten().tolist() == [0.0, 1.0, 2.0, 3.0]
def test_multitrack_task_output_returns_locked_audio_from_deferred_video():
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
source_frames = torch.zeros(8, 2, 2, 3)
source_video = _FakeVideo(_VideoComponents(source_frames, source_audio, Fraction(2)))
module._resolve_multitrack_video = lambda content, video_input: source_video
tracks_info = {
"media_loading": "deferred",
"format": "MiniMax",
"width": 2,
"height": 2,
"total_length": 4,
"frame_rate": 2,
"tracks": [
{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"media_type": "none", "images": []},
}],
},
{
"type": "video",
"audio_locked": True,
"segments": [{
"id": "locked-video",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "video.mp4",
},
}],
},
],
}
result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=0)
assert result.values[8]["waveform"].flatten().tolist() == [0.0, 1.0, 2.0, 3.0]
def test_multitrack_task_output_uses_the_locked_audio_track_index():
module = _load_basic_module()
first_audio = {"waveform": torch.ones(1, 1, 4), "sample_rate": 2}
locked_audio = {"waveform": torch.full((1, 1, 4), 7.0), "sample_rate": 2}
tracks_info = {
"frame_rate": 2,
"tracks": [
{"type": "task", "segments": [{
"start_frame": 0, "end_frame": 4,
"content": {"media_type": "none", "images": []},
}]},
{"type": "audio", "media_index": 0, "segments": [{
"start_frame": 0, "end_frame": 4,
"content": {"media_type": "audio", "media_index": 0},
}]},
{"type": "audio", "media_index": 1, "audio_locked": True, "segments": [{
"start_frame": 0, "end_frame": 4,
"content": {"media_type": "audio", "media_index": 1},
}]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[first_audio, locked_audio]], [], [0], ["default"]
)
assert torch.equal(result.values[8]["waveform"], locked_audio["waveform"])
def test_multitrack_task_output_crops_deferred_media_by_task_index():
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
source_frames = torch.arange(8, dtype=torch.float32).reshape(8, 1, 1, 1).expand(-1, 2, 2, 3)
source_video = _FakeVideo(_VideoComponents(source_frames, None, Fraction(2)))
module._resolve_multitrack_audio = lambda content, audio_input: source_audio
module._resolve_multitrack_video = lambda content, video_input: source_video
track_data = {
"total_length": 8,
"frame_rate": 2,
"tracks": [
{
"type": "task",
"segments": [
{"start_frame": 0, "end_frame": 4, "content": {"text": "first", "images": []}},
{"start_frame": 4, "end_frame": 8, "content": {"text": "second", "images": []}},
],
},
{
"type": "audio",
"segments": [{
"start_frame": 0,
"end_frame": 8,
"origin_start_frame": 0,
"content": {"media_type": "audio", "source_type": "input", "file_path": "audio.wav"},
}],
},
{
"type": "video",
"segments": [{
"start_frame": 0,
"end_frame": 8,
"origin_start_frame": 0,
"content": {"media_type": "video", "source_type": "input", "file_path": "video.mp4"},
}],
},
],
}
tracks_info = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)", "resize_method": "stretch"},
"None",
track_data,
).values[0]
first = module.MultiTrackTaskOutput.execute(tracks_info, task_index=0)
second = module.MultiTrackTaskOutput.execute(tracks_info, task_index=1)
assert first.values[5][0]["waveform"].flatten().tolist() == [0.0, 1.0, 2.0, 3.0]
assert second.values[5][0]["waveform"].flatten().tolist() == [4.0, 5.0, 6.0, 7.0]
assert first.values[6][0].get_components().images[:, 0, 0, 0].tolist() == [0.0, 1.0, 2.0, 3.0]
assert second.values[6][0].get_components().images[:, 0, 0, 0].tolist() == [4.0, 5.0, 6.0, 7.0]
def test_multitrack_task_output_passes_task_window_to_deferred_file_video_merge():
module = _load_basic_module()
source_video = _FakeVideo(
_VideoComponents(torch.zeros(8, 2, 2, 3), None, Fraction(2)),
source="source.mp4",
)
merge_calls = []
module._resolve_multitrack_video = lambda content, video_input: source_video
module.merge_video_track_with_ffmpeg = lambda segments, total_length, *args, **kwargs: (
merge_calls.append((segments, total_length)) or "task-window.mp4"
)
track_data = {
"total_length": 8,
"frame_rate": 2,
"tracks": [
{
"type": "task",
"segments": [
{"start_frame": 0, "end_frame": 4, "content": {"text": "first", "images": []}},
{"start_frame": 4, "end_frame": 8, "content": {"text": "second", "images": []}},
],
},
{
"type": "video",
"segments": [{
"start_frame": 0,
"end_frame": 8,
"origin_start_frame": 0,
"content": {"media_type": "video", "source_type": "input", "file_path": "video.mp4"},
}],
},
],
}
tracks_info = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)", "resize_method": "stretch"},
"None",
track_data,
).values[0]
result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=1)
assert result.values[6][0].source == "task-window.mp4"
assert merge_calls == [([{
"source": "source.mp4",
"start_frame": 0,
"end_frame": 4,
"source_start_frame": 4,
"audio_volume_db": 0.0,
"audio_muted": False,
}], 4)]
def test_paused_multitrack_preview_uses_output_frame_sampling(tmp_path, monkeypatch):
ffmpeg = shutil.which("ffmpeg")
if ffmpeg is None:
pytest.skip("FFmpeg is unavailable")
source = tmp_path / "30fps.mp4"
subprocess.run([
ffmpeg, "-v", "error", "-y",
"-f", "lavfi", "-i", "color=c=red:s=64x64:r=30:d=0.566667",
"-f", "lavfi", "-i", "color=c=blue:s=64x64:r=30:d=0.433333",
"-filter_complex", "[0:v][1:v]concat=n=2:v=1:a=0[v]",
"-map", "[v]", "-an", "-c:v", "mpeg4", str(source),
], check=True)
_load_basic_module()
video_module = sys.modules["easy_media.utils.video"]
before = Image.open(stdlib_io.BytesIO(
video_module.decode_multitrack_preview_frame(source, 13, 24)
)).convert("RGB").getpixel((32, 32))
after = Image.open(stdlib_io.BytesIO(
video_module.decode_multitrack_preview_frame(source, 14, 24)
)).convert("RGB").getpixel((32, 32))
assert before[0] > before[2]
assert after[2] > after[0]
monkeypatch.setattr(sys.modules["folder_paths"], "models_dir", str(tmp_path), raising=False)
monkeypatch.setitem(sys.modules, "easy_media.utils.models", types.SimpleNamespace(
require_model_path=lambda _name: source,
))
omnishotcut_path = Path(__file__).parents[1] / "modules" / "omnishotcut" / "__init__.py"
spec = importlib.util.spec_from_file_location(
"easy_media.modules.omnishotcut", omnishotcut_path,
submodule_search_locations=[str(omnishotcut_path.parent)],
)
omnishotcut = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, omnishotcut)
spec.loader.exec_module(omnishotcut)
sampled = omnishotcut._read_video(source, 16, 16, 24)
assert sampled[13, 8, 8, 0] > sampled[13, 8, 8, 2]
assert sampled[14, 8, 8, 2] > sampled[14, 8, 8, 0]
def test_paused_multitrack_preview_seek_matches_full_decode(tmp_path):
ffmpeg = shutil.which("ffmpeg")
if ffmpeg is None:
pytest.skip("FFmpeg is unavailable")
source = tmp_path / "30fps-offset.mp4"
subprocess.run([
ffmpeg, "-v", "error", "-y",
"-f", "lavfi", "-i", "testsrc2=s=64x64:r=30:d=6",
"-an", "-c:v", "mpeg4", "-output_ts_offset", "5", str(source),
], check=True)
_load_basic_module()
video_module = sys.modules["easy_media.utils.video"]
for frame in (96, 97, 120):
expected = subprocess.run([
ffmpeg, "-v", "error", "-i", str(source),
"-vf", f"setpts=PTS-STARTPTS,fps=fps=24:start_time=0,"
f"select=eq(n\\,{frame}),scale=w='min(960,iw)':h=-2",
"-frames:v", "1", "-an", "-f", "image2pipe", "-vcodec", "mjpeg", "-",
], capture_output=True, check=True, timeout=60).stdout
assert video_module.decode_multitrack_preview_frame(source, frame, 24) == expected
def test_multitrack_task_output_reports_media_processing_progress():
module = _load_basic_module()
tracks_info = {
"media_loading": "deferred",
"total_length": 4,
"timeline_total_length": 4,
"frame_rate": 2,
"tracks": [{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"text": "task", "images": []},
}],
}],
}
module.MultiTrackTaskOutput.execute(tracks_info, task_index=0)
progress = _ProgressBar.instances[-1]
assert progress.total == 2
assert progress.updates == [0, 1, 2]
def test_multitrack_editor_uses_ffmpeg_for_supported_file_video_resize():
module = _load_basic_module()
ffmpeg_calls = []
video = _FakeVideo(
_VideoComponents(torch.zeros(2, 360, 640, 3), None, Fraction(24)),
source="source.mp4",
)
def fake_ffmpeg(source, width, height, method, progress_callback=None):
ffmpeg_calls.append((source, width, height, method))
if progress_callback:
progress_callback(0.5)
progress_callback(1.0)
return "resized.mp4"
module._multitrack_module.resize_video_with_ffmpeg = (
module.resize_video_with_ffmpeg
) = fake_ffmpeg
track_data = {
"tracks": [{
"id": "video-track",
"type": "video",
"segments": [{"id": "v1", "content": {
"media_type": "video", "source_type": "slot", "slot_name": "video1",
}}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "320 x 180 (16:9)", "resize_method": "crop"},
"None",
track_data,
video=[video],
)
assert ffmpeg_calls == [("source.mp4", 320, 180, "crop")]
assert video.components_calls == 0
assert len(result.values[3]) == 1
def test_video_track_passes_audio_gain_and_mute_to_ffmpeg():
module = _load_basic_module()
calls = []
video = _FakeVideo(
_VideoComponents(torch.zeros(2, 360, 640, 3), None, Fraction(24)),
source="source.mp4",
)
module.merge_video_track_with_ffmpeg = lambda segments, *args: (
calls.append(segments) or "merged.mp4"
)
result = module._merge_video_track(
[({
"start_frame": 3,
"end_frame": 9,
"content": {"volume_db": -2.5, "muted": True},
}, video)],
12,
24,
640,
360,
base_volume_db=4,
)
assert result.source == "merged.mp4"
assert calls == [[{
"source": "source.mp4",
"start_frame": 3,
"end_frame": 9,
"audio_volume_db": 1.5,
"audio_muted": True,
}]]
def test_video_track_passes_source_trim_offset_to_ffmpeg():
module = _load_basic_module()
calls = []
video = _FakeVideo(
_VideoComponents(torch.zeros(48, 360, 640, 3), None, Fraction(24)),
source="source.mp4",
)
module.merge_video_track_with_ffmpeg = lambda segments, *args: (
calls.append(segments) or "merged.mp4"
)
module._merge_video_track(
[({
"start_frame": 24,
"end_frame": 48,
"origin_start_frame": 0,
"content": {},
}, video)],
48,
24,
640,
360,
)
assert calls[0][0]["source_start_frame"] == 24
def test_single_full_video_segment_uses_direct_render_when_resize_is_needed():
module = _load_basic_module()
video = _FakeVideo(
_VideoComponents(torch.zeros(4, 4, 4, 3), None, Fraction(2)),
source="source.mp4",
)
calls = []
module.render_single_video_segment_with_ffmpeg = lambda *args, **kwargs: (
calls.append((args, kwargs)) or "single.mp4"
)
module.merge_video_track_with_ffmpeg = lambda *_args, **_kwargs: (
pytest.fail("single full-window video must not use timeline overlay")
)
result = module._merge_video_track(
[({
"start_frame": 0,
"end_frame": 4,
"origin_start_frame": -2,
"content": {"volume_db": -3},
}, video)],
4, 2, 2, 2, resize_method="stretch",
)
assert result.source == "single.mp4"
assert calls == [(("source.mp4", 2, 4, 2, 2, 2, "stretch"), {
"audio_volume_db": -3,
"audio_muted": False,
})]
def test_ffmpeg_video_merge_applies_segment_audio_filters(tmp_path, monkeypatch):
module = _load_video_utils_module(tmp_path)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
module.folder_paths.get_temp_directory = lambda: str(tmp_path)
monkeypatch.setattr(module, "get_ffmpeg_path", lambda _name="ffmpeg": "ffmpeg")
monkeypatch.setattr(
module,
"ffprobe_info",
lambda _source: {"has_audio": True},
)
commands = []
def fake_run(command, capture_output, timeout):
commands.append(command)
assert timeout == 60
return types.SimpleNamespace(returncode=0, stderr=b"")
monkeypatch.setattr(module.subprocess, "run", fake_run)
output = module.merge_video_track_with_ffmpeg(
[{
"source": str(source),
"start_frame": 0,
"end_frame": 24,
"source_start_frame": 12,
"audio_volume_db": -3.5,
"audio_muted": False,
}],
24,
24,
640,
360,
resize_method="crop",
)
assert output is not None
filter_graph = commands[0][commands[0].index("-filter_complex") + 1]
assert "volume=-3.5dB" in filter_graph
assert "trim=start=0.5:duration=1.0" in filter_graph
assert "scale=640:360:force_original_aspect_ratio=increase,crop=640:360" in filter_graph
assert "atrim=start=0.5:duration=1.0" in filter_graph
def test_ffprobe_info_ignores_na_duration(tmp_path, monkeypatch):
module = _load_video_utils_module(tmp_path)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
monkeypatch.setattr(module, "get_ffmpeg_path", lambda _name="ffprobe": "ffprobe")
def fake_run(command, capture_output=False, text=False, timeout=None):
assert timeout == 30
return types.SimpleNamespace(
returncode=0,
stdout=json.dumps({
"format": {"duration": "N/A"},
"streams": [{
"codec_type": "video",
"width": 1920,
"height": 1080,
"avg_frame_rate": "24/1",
"nb_frames": "48",
}],
}),
)
monkeypatch.setattr(module.subprocess, "run", fake_run)
info = module.ffprobe_info(str(source))
assert info["duration"] is None
assert info["width"] == 1920
assert info["height"] == 1080
assert info["fps"] == 24.0
assert info["frame_count"] == 48
def test_ffprobe_timeout_reports_error(tmp_path, monkeypatch):
module = _load_video_utils_module(tmp_path)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
monkeypatch.setattr(module, "get_ffmpeg_path", lambda _name="ffprobe": "ffprobe")
def timed_out(command, capture_output=False, text=False, timeout=None):
raise subprocess.TimeoutExpired(cmd=command, timeout=timeout)
monkeypatch.setattr(module.subprocess, "run", timed_out)
with pytest.raises(RuntimeError, match="FFprobe timed out"):
module.ffprobe_info(str(source))
def test_ffmpeg_resize_skips_na_progress_and_outputs_standard_mp4(tmp_path, monkeypatch):
module = _load_video_utils_module(tmp_path)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
module.folder_paths.get_temp_directory = lambda: str(tmp_path)
monkeypatch.setattr(module, "get_ffmpeg_path", lambda _name="ffmpeg": "ffmpeg")
monkeypatch.setattr(module, "ffprobe_info", lambda _source: {"duration": 2.0})
commands = []
class FakePopen:
def __init__(self, command, stdout=None, stderr=None, text=False):
commands.append(command)
self.stdout = iter(["out_time_us=N/A\n", "out_time_us=1000000\n"])
def wait(self):
return 0
progress = []
monkeypatch.setattr(module.subprocess, "Popen", FakePopen)
output = module.resize_video_with_ffmpeg(
str(source),
1920,
1080,
"resize",
progress_callback=progress.append,
)
assert output is not None
assert output.endswith(".mp4")
command = commands[0]
assert command[command.index("-c:v") + 1] == "libx264"
assert command[command.index("-c:a") + 1] == "aac"
assert command[command.index("-pix_fmt") + 1] == "yuv420p"
assert "scale=1920:1080:force_original_aspect_ratio=decrease" in command[command.index("-vf") + 1]
assert progress == [0.0, 0.5, 1.0]
def test_extract_video_audio_to_temp_applies_trim_window(tmp_path, monkeypatch):
module = _load_video_utils_module(tmp_path)
source = tmp_path / "source.mp4"
source.write_bytes(b"video")
module.folder_paths.get_temp_directory = lambda: str(tmp_path)
monkeypatch.setattr(module, "get_ffmpeg_path", lambda _name="ffmpeg": "ffmpeg")
monkeypatch.setattr(module, "ffprobe_info", lambda _source: {"has_audio": True})
commands = []
def fake_run(command, capture_output=False, check=False):
commands.append(command)
Path(command[-1]).write_bytes(b"audio")
return types.SimpleNamespace(returncode=0, stderr=b"")
monkeypatch.setattr(module.subprocess, "run", fake_run)
output = module.extract_video_audio_to_temp(source, start_time=1.0, duration=2.0)
assert output.is_file()
assert "-ss" in commands[0]
assert float(commands[0][commands[0].index("-ss") + 1]) == 1.0
assert "-t" in commands[0]
assert float(commands[0][commands[0].index("-t") + 1]) == 2.0
def test_burn_subtitles_with_ffmpeg_maps_optional_audio(tmp_path, monkeypatch):
module = _load_video_utils_module(tmp_path)
source = tmp_path / "source.mp4"
subtitles_path = tmp_path / "subtitle file.ass"
output = tmp_path / "out.mp4"
source.write_bytes(b"video")
subtitles_path.write_text("[Script Info]\n", encoding="utf-8")
module.folder_paths.get_temp_directory = lambda: str(tmp_path)
monkeypatch.setattr(module, "get_ffmpeg_path", lambda _name="ffmpeg": "ffmpeg")
commands = []
def fake_run(command, capture_output):
commands.append(command)
output.write_bytes(b"done")
return types.SimpleNamespace(returncode=0, stderr=b"")
monkeypatch.setattr(module.subprocess, "run", fake_run)
assert module.burn_subtitles_with_ffmpeg(
str(source),
str(subtitles_path),
str(output),
) == str(output)
command = commands[0]
assert command[command.index("-vf") + 1].startswith("subtitles='")
assert "-map" in command
assert "0:a?" in command
assert command[-1] == str(output)
def test_multitrack_editor_reuses_cached_ffmpeg_result_for_duplicate_video():
module = _load_basic_module()
ffmpeg_calls = []
video = _FakeVideo(
_VideoComponents(torch.zeros(2, 360, 640, 3), None, Fraction(24)),
source="same-source.mp4",
)
def fake_ffmpeg(source, width, height, method, progress_callback=None):
ffmpeg_calls.append((source, width, height, method))
return "cached-resize.mp4"
module._multitrack_module.resize_video_with_ffmpeg = (
module.resize_video_with_ffmpeg
) = fake_ffmpeg
track_data = {
"tracks": [{
"id": "video-track",
"type": "video",
"segments": [
{"id": "v1", "content": {"media_type": "video", "source_type": "slot", "slot_name": "video1"}},
{"id": "v2", "content": {"media_type": "video", "source_type": "slot", "slot_name": "video1"}},
],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "320 x 180 (16:9)", "resize_method": "pad"},
"None",
track_data,
video=[video],
)
videos = result.values[3]
assert len(ffmpeg_calls) == 1
assert len(videos) == 1
def test_multitrack_editor_falls_back_to_tensor_for_unmapped_ffmpeg_method():
module = _load_basic_module()
video = _FakeVideo(
_VideoComponents(torch.zeros(2, 360, 640, 3), None, Fraction(24)),
source="source.mp4",
)
module._multitrack_module.resize_video_with_ffmpeg = module.resize_video_with_ffmpeg = lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("FFmpeg must not run for pillarbox_blur")
)
module._multitrack_module.resize_image = module.resize_image = lambda images, width, height, method: torch.zeros(
images.shape[0], height, width, images.shape[-1]
)
track_data = {
"tracks": [{
"id": "video-track",
"type": "video",
"segments": [{"id": "v1", "content": {
"media_type": "video", "source_type": "slot", "slot_name": "video1",
}}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "320 x 180 (16:9)", "resize_method": "pillarbox_blur"},
"None",
track_data,
video=[video],
)
assert video.components_calls == 1
assert result.values[3][0].get_dimensions() == (320, 180)
def test_multitrack_editor_does_not_create_progress_for_deferred_preset_video():
module = _load_basic_module()
track_data = {
"tracks": [{
"id": "video-track",
"type": "video",
"segments": [{"id": "preset", "content": {
"media_type": "video",
"source_type": "preset",
}}],
}],
}
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (auto)"},
"None",
track_data,
)
assert result.values[0]["media_loading"] == "deferred"
assert _ProgressBar.instances == []
def test_resolve_video_path_supports_comfy_input_files(tmp_path):
module = _load_video_utils_module(tmp_path)
video_path = tmp_path / "clip.mp4"
video_path.write_bytes(b"video")
assert module.resolve_video_path("input", "clip.mp4", None, None) == str(video_path)
def test_multitrack_editor_merges_video_segments_per_track_with_black_gaps():
module = _load_basic_module()
first = _FakeVideo(_VideoComponents(torch.ones(2, 2, 2, 3), None, Fraction(2)))
second = _FakeVideo(_VideoComponents(torch.full((2, 2, 2, 3), 2.0), None, Fraction(2)))
track_data = {
"total_length": 6,
"frame_rate": 2,
"tracks": [
{"id": "task", "type": "task", "segments": []},
{"id": "video-track", "type": "video", "segments": [
{"id": "v1", "start_frame": 1, "end_frame": 3, "content": {
"media_type": "video", "source_type": "slot", "slot_name": "video1",
}},
{"id": "v2", "start_frame": 4, "end_frame": 6, "content": {
"media_type": "video", "source_type": "slot", "slot_name": "video2",
}},
]},
],
}
result = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)"},
"None",
track_data,
video=[first, second],
)
tracks_info, _images, _audio, videos = result.values
assert len(videos) == 1
frames = videos[0].get_components().images
assert frames.shape == (6, 2, 2, 3)
assert [float(frames[index].mean()) for index in range(6)] == [0.0, 1.0, 1.0, 0.0, 2.0, 2.0]
video_track = tracks_info["tracks"][1]
assert video_track["media_index"] == 0
assert [segment["content"]["media_index"] for segment in video_track["segments"]] == [0, 0]
def test_multitrack_editor_merges_audio_segments_per_track_with_silence():
module = _load_basic_module()
first = {"waveform": torch.ones(1, 1, 4), "sample_rate": 4}
second = {"waveform": torch.full((1, 1, 2), 2.0), "sample_rate": 4}
track_data = {
"total_length": 6,
"frame_rate": 2,
"tracks": [
{"id": "task", "type": "task", "segments": []},
{"id": "audio-track", "type": "audio", "segments": [
{"id": "a1", "start_frame": 1, "end_frame": 3, "content": {
"media_type": "audio", "source_type": "slot", "slot_name": "audio1",
}},
{"id": "a2", "start_frame": 4, "end_frame": 5, "content": {
"media_type": "audio", "source_type": "slot", "slot_name": "audio2",
}},
]},
],
}
result = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)"},
"None",
track_data,
audio=[first, second],
)
tracks_info, _images, audio, _videos = result.values
assert len(audio) == 1
assert audio[0]["waveform"].flatten().tolist() == [0, 0, 1, 1, 1, 1, 0, 0, 2, 2]
audio_track = tracks_info["tracks"][1]
assert audio_track["media_index"] == 0
assert [segment["content"]["media_index"] for segment in audio_track["segments"]] == [0, 0]
def test_slot_media_flows_downstream_through_tracks_info_only():
module = _load_basic_module()
image = torch.ones(1, 2, 2, 3)
audio = {"waveform": torch.ones(1, 1, 4), "sample_rate": 2}
video = _FakeVideo(
_VideoComponents(torch.ones(4, 2, 2, 3), None, Fraction(2))
)
track_data = {
"total_length": 4,
"frame_rate": 2,
"tracks": [
{"id": "task", "type": "task", "segments": [{
"id": "task-1",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "none",
"images": [{
"id": "image-1",
"source_type": "slot",
"slot_name": "image1",
}],
},
}]},
{"id": "audio-track", "type": "audio", "segments": [{
"id": "audio-1",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "audio",
"source_type": "slot",
"slot_name": "audio1",
},
}]},
{"id": "video-track", "type": "video", "segments": [{
"id": "video-1",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "video",
"source_type": "slot",
"slot_name": "video1",
},
}]},
],
}
tracks_info = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)"},
"None",
track_data,
image=[image],
audio=[audio],
video=[video],
).values[0]
task_result = module.MultiTrackTaskOutput.execute(
tracks_info,
task_index=0,
prompt_format="default",
)
assert len(task_result.values[4]) == 1
assert torch.equal(task_result.values[4][0], image)
assert task_result.values[5][0]["waveform"].shape[-1] == 4
assert task_result.values[6][0].get_components().images.shape[0] == 4
audio_result = module.MultiTrackAudioOutput.execute(tracks_info, audio=(None,))
assert audio_result.values[1]["waveform"].shape[-1] == 4
def test_multitrack_editor_minimax_stops_media_at_each_track_last_segment():
module = _load_basic_module()
module._multitrack_module.resize_image = module.resize_image = lambda images, width, height, _method: torch.ones(
images.shape[0], height, width, images.shape[-1]
)
video = _FakeVideo(
_VideoComponents(torch.ones(5, 2, 2, 3), None, Fraction(2))
)
audio = {"waveform": torch.ones(1, 1, 7), "sample_rate": 2}
track_data = {
"total_length": 12,
"frame_rate": 2,
"tracks": [
{"id": "task", "type": "task", "segments": [{
"id": "task-1",
"start_frame": 0,
"end_frame": 12,
"content": {"media_type": "none"},
}]},
{"id": "video-track", "type": "video", "segments": [{
"id": "video-1",
"start_frame": 1,
"end_frame": 5,
"content": {
"media_type": "video",
"source_type": "slot",
"slot_name": "video1",
},
}]},
{"id": "audio-track", "type": "audio", "segments": [{
"id": "audio-1",
"start_frame": 1,
"end_frame": 7,
"content": {
"media_type": "audio",
"source_type": "slot",
"slot_name": "audio1",
},
}]},
],
}
result = module.MultiTrackEditor.execute(
{"resolution": "32 x 32 (1:1)"},
"MiniMax",
track_data,
audio=[audio],
video=[video],
)
_tracks_info, _images, audio_out, video_out = result.values
assert video_out[0].get_components().images.shape[0] == 5
assert audio_out[0]["waveform"].shape[-1] == 7
def test_multitrack_task_output_schema_and_task_media_selection():
module = _load_basic_module()
schema = module.MultiTrackTaskOutput.define_schema()
assert schema.is_input_list is True
assert [input_.name for input_ in schema.inputs] == [
"tracks_info", "images", "audio", "video", "task_index", "prompt_format",
"previous",
]
assert [output.name for output in schema.outputs] == [
"SYSTEM_PROMPT", "USER_PROMPT", "TYPE", "LENGTH", "IMAGES", "AUDIO", "VIDEO",
"IMAGE_INDEXES",
"LOCKED_AUDIO",
]
images = [torch.zeros(1, 2, 2, 3), torch.ones(1, 2, 2, 3), torch.full((1, 2, 2, 3), 2.0)]
audio_track = {"waveform": torch.arange(16).reshape(1, 1, 16), "sample_rate": 4}
video_track = _FakeVideo(_VideoComponents(torch.zeros(8, 2, 2, 3), None, Fraction(2)))
tracks_info = {
"total_length": 8,
"frame_rate": 2,
"tracks": [
{"id": "task", "type": "task", "segments": [{
"id": "task-1", "start_frame": 2, "end_frame": 6,
"content": {
"task_mode": "ref",
"user_prompt": "make it move",
"system_prompt": "custom template",
"images": [{"media_index": 1}, {"media_index": 2}],
},
}]},
{"id": "video-track", "type": "video", "media_index": 0, "segments": [{
"start_frame": 0, "end_frame": 8, "content": {"media_type": "video", "media_index": 0},
}]},
{"id": "audio-track", "type": "audio", "media_index": 0, "segments": [{
"start_frame": 0, "end_frame": 8, "content": {"media_type": "audio", "media_index": 0},
}]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info],
[images],
[[audio_track]],
[[video_track]],
[0],
["default"],
)
(
system_prompt,
user_prompt,
task_type,
length,
selected_images,
selected_audio,
selected_video,
image_indexes,
locked_audio,
) = result.values
assert system_prompt == ""
assert user_prompt == "make it move"
assert task_type == "rv2v"
assert length == 5
assert locked_audio is None
assert selected_images == [images[1], images[2]]
assert selected_audio[0]["waveform"].flatten().tolist() == list(range(4, 12))
assert selected_video == [video_track]
assert image_indexes == "0,-1"
assert video_track.trim_calls == [(1.0, 2.0, False)]
locked_track = tracks_info["tracks"][2]
locked_segment = locked_track["segments"][0]
locked_segment.update({"start_frame": 2, "end_frame": 6})
locked_track["audio_locked"] = True
locked_result = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [[audio_track]], [[video_track]], [0], ["default"]
)
assert locked_result.values[8]["waveform"].flatten().tolist() == list(range(4, 12))
def test_multitrack_task_output_prepends_project_preloaded_images():
module = _load_basic_module()
shared_image = torch.zeros(1, 2, 2, 3)
local_image = torch.ones(1, 2, 2, 3)
tracks_info = {
"frame_rate": 24,
"format": "MiniMax",
"_preloaded_media": {
"images": [shared_image],
"audio": [],
"video": [],
},
"tracks": [{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 120,
"content": {
"task_mode": "ref",
"images": [{"media_index": 0}],
},
}],
}],
}
result = module.MultiTrackTaskOutput.execute(
tracks_info,
images=[local_image],
task_index=0,
prompt_format="default",
)
assert result.values[4] == [shared_image, local_image]
def test_multitrack_task_output_restores_project_runtime_media_cache():
module = _load_basic_module()
shared_image = torch.zeros(1, 2, 2, 3)
tracks_info = {
"frame_rate": 24,
"format": "MiniMax",
"_easy_media_runtime_cache": {},
"_easy_media_cache_status": {
"project_media": "命中恢复缓存",
"segment_media": "命中恢复缓存",
},
"_preloaded_media": {
"images": [shared_image],
"audio": [],
"video": [],
},
"tracks": [{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 120,
"content": {"task_mode": "ref", "images": []},
}],
}],
}
first = module.MultiTrackTaskOutput.execute(
tracks_info,
task_index=0,
prompt_format="default",
)
second = module.MultiTrackTaskOutput.execute(
tracks_info,
task_index=0,
prompt_format="default",
)
assert second is first
assert tracks_info["_easy_media_cache_status"]["task_output"] == "命中恢复缓存"
def test_multitrack_task_output_uses_selected_user_prompt_variant():
module = _load_basic_module()
tracks_info = {
"total_length": 4,
"frame_rate": 1,
"tracks": [{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {
"user_prompt": "original A prompt",
"user_prompt_b": "reverse-engineered B prompt",
"user_prompt_variant": "b",
"images": [],
},
}],
}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["default"],
)
assert result.values[1] == "reverse-engineered B prompt"
def test_multitrack_task_output_defaults_existing_user_prompt_to_variant_a():
module = _load_basic_module()
assert module._selected_multitrack_user_prompt({
"user_prompt": "existing prompt",
"user_prompt_b": "unused B prompt",
}) == "existing prompt"
def test_multitrack_task_output_aligns_each_minimax_segment_without_plus_one():
module = _load_basic_module()
tracks_info = {
"format": "MiniMax",
"frame_rate": 24,
"tracks": [{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 107,
"content": {"user_prompt": "already aligned", "images": []},
}]}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["default"],
)
assert result.values[3] == 107
def test_multitrack_task_output_outputs_none_for_empty_minimax_media_tracks():
module = _load_basic_module()
tracks_info = {
"format": "MiniMax",
"frame_rate": 24,
"tracks": [
{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 120,
"content": {"user_prompt": "empty media", "images": []},
}]},
{"type": "video", "segments": []},
{"type": "audio", "segments": []},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["default"],
)
assert result.values[5] == [None]
assert result.values[6] == [None]
def test_minimax_speaker_reference_is_reused_in_every_task_and_capped_at_15_seconds(monkeypatch):
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(30, dtype=torch.float32).reshape(1, 1, 30),
"sample_rate": 1,
}
monkeypatch.setattr(module, "_resolve_multitrack_audio", lambda content, audio_input: source_audio)
tracks_info = {
"media_loading": "deferred",
"format": "MiniMax",
"frame_rate": 1,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 5, "content": {"user_prompt": "first"}},
{"start_frame": 20, "end_frame": 25, "content": {"user_prompt": "second"}},
]},
{"type": "audio", "segments": [
{
"id": "speaker",
"start_frame": 0,
"end_frame": 5,
"content": {"media_type": "audio", "speaker_reference": True},
},
{
"id": "other-audio",
"start_frame": 20,
"end_frame": 25,
"content": {"media_type": "audio"},
},
]},
],
}
first = module.MultiTrackTaskOutput.execute(tracks_info, task_index=0)
second = module.MultiTrackTaskOutput.execute(tracks_info, task_index=1)
expected = list(range(15))
assert first.values[5][0]["waveform"].flatten().tolist() == expected
assert second.values[5][0]["waveform"].flatten().tolist() == expected
def test_multitrack_editor_materializes_a_dedicated_minimax_speaker_reference():
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(30, dtype=torch.float32).reshape(1, 1, 30),
"sample_rate": 1,
}
track_data = {
"frame_rate": 1,
"total_length": 30,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 20, "end_frame": 25, "content": {"user_prompt": "late task"}},
]},
{"id": "voice-track", "type": "audio", "segments": [{
"id": "voice",
"start_frame": 0,
"end_frame": 5,
"content": {
"media_type": "audio",
"source_type": "slot",
"slot_name": "audio1",
"speaker_reference": True,
},
}]},
],
}
editor_result = module.MultiTrackEditor.execute(
"640 x 360 (16:9)",
"MiniMax",
track_data,
audio=[source_audio],
)
tracks_info, audio_output = editor_result.values[0], editor_result.values[2]
speaker_content = tracks_info["tracks"][1]["segments"][0]["content"]
assert speaker_content["shared_reference"] is True
assert speaker_content["shared_media_index"] == 1
assert "speaker_reference" not in speaker_content
assert audio_output[1]["waveform"].flatten().tolist() == list(range(15))
task_result = module.MultiTrackTaskOutput.execute(
tracks_info,
audio=audio_output,
task_index=0,
)
assert task_result.values[5][0]["waveform"].flatten().tolist() == list(range(15))
def test_shared_task_images_are_prefixed_deduplicated_and_reused():
module = _load_basic_module()
shared_image = torch.ones(1, 2, 2, 3)
first_local = torch.full((1, 2, 2, 3), 2.0)
second_local = torch.full((1, 2, 2, 3), 3.0)
track_data = {
"frame_rate": 1,
"total_length": 10,
"tracks": [{
"id": "tasks",
"type": "task",
"segments": [
{
"id": "first",
"start_frame": 0,
"end_frame": 5,
"content": {"media_type": "none", "images": [
{"id": "local-a", "source_type": "slot", "slot_name": "image2"},
{
"id": "shared-a",
"source_type": "slot",
"slot_name": "image1",
"shared_reference": True,
},
]},
},
{
"id": "second",
"start_frame": 5,
"end_frame": 10,
"content": {"media_type": "none", "images": [
{"id": "same-path", "source_type": "slot", "slot_name": "image1"},
{"id": "local-b", "source_type": "slot", "slot_name": "image3"},
]},
},
],
}],
}
editor_result = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)"},
"MiniMax",
track_data,
image=[shared_image, first_local, second_local],
)
tracks_info, images = editor_result.values[0], editor_result.values[1]
first_images = tracks_info["tracks"][0]["segments"][0]["content"]["images"]
second_images = tracks_info["tracks"][0]["segments"][1]["content"]["images"]
assert [item["slot_name"] for item in first_images] == ["image1", "image2"]
assert [item["slot_name"] for item in second_images] == ["image1", "image3"]
assert first_images[0]["shared_reference"] is True
assert second_images[0]["shared_reference"] is True
assert first_images[0]["media_index"] == second_images[0]["media_index"]
assert len(images) == 3
task_result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=1)
assert torch.equal(task_result.values[4][0], shared_image)
assert torch.equal(task_result.values[4][1], second_local)
def test_muted_shared_task_image_does_not_mute_other_tasks():
module = _load_basic_module()
shared_image = torch.ones(1, 2, 2, 3)
track_data = {"tracks": [{"type": "task", "segments": [
{"content": {"images": [{
"source_type": "slot", "slot_name": "image1",
"shared_reference": True, "muted": True,
}]}},
{"content": {"images": []}},
]}]}
result = module.MultiTrackEditor.execute(
{"resolution": "2 x 2 (1:1)"}, "MiniMax", track_data,
image=[shared_image],
)
tracks_info, images = result.values[0], result.values[1]
first, second = tracks_info["tracks"][0]["segments"]
assert first["content"]["images"][0]["muted"] is True
assert second["content"]["images"][0].get("muted") is not True
assert len(images) == 1
assert torch.equal(images[0], shared_image)
def test_shared_video_is_available_to_a_non_overlapping_task():
module = _load_basic_module()
source_video = _FakeVideo(
_VideoComponents(torch.ones(4, 32, 32, 3), None, Fraction(1))
)
track_data = {
"frame_rate": 1,
"total_length": 10,
"tracks": [
{"id": "tasks", "type": "task", "segments": [
{"id": "first", "start_frame": 0, "end_frame": 5, "content": {"media_type": "none"}},
{"id": "second", "start_frame": 5, "end_frame": 10, "content": {"media_type": "none"}},
]},
{"id": "videos", "type": "video", "segments": [{
"id": "shared-video",
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "video",
"source_type": "slot",
"slot_name": "video1",
"shared_reference": True,
},
}]},
],
}
editor_result = module.MultiTrackEditor.execute(
{"resolution": "32 x 32 (1:1)"},
"MiniMax",
track_data,
video=[source_video],
)
tracks_info, videos = editor_result.values[0], editor_result.values[3]
shared_content = tracks_info["tracks"][1]["segments"][0]["content"]
assert shared_content["shared_reference"] is True
assert shared_content["shared_media_index"] == 1
task_result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=1)
assert task_result.values[6][0] is videos[1]
def test_deferred_shared_video_is_resized_like_eager_shared_video(monkeypatch):
module = _load_basic_module()
source_video = object()
resized_video = object()
resize_calls = []
monkeypatch.setattr(
module,
"_resolve_multitrack_video",
lambda content, video_input: source_video,
)
def resize(video, width, height, method, cache, progress_callback):
resize_calls.append((video, width, height, method))
progress_callback(1.0)
return resized_video
monkeypatch.setattr(module, "_resize_multitrack_video", resize)
tracks_info = {
"media_loading": "deferred",
"eager_media_types": [],
"width": 640,
"height": 360,
"resize_method": "crop",
"frame_rate": 24,
"tracks": [
{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 24,
"content": {"user_prompt": "shared video"},
}]},
{"type": "video", "segments": [{
"id": "shared-video",
"start_frame": 0,
"end_frame": 12,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "clip.mp4",
"shared_reference": True,
},
}]},
],
}
result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=0)
assert resize_calls == [(source_video, 640, 360, "crop")]
assert result.values[6] == [resized_video]
def test_multitrack_task_output_keeps_minimax_audio_when_video_track_is_empty():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(120).reshape(1, 1, 120), "sample_rate": 24}
tracks_info = {
"format": "MiniMax",
"frame_rate": 24,
"tracks": [
{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 120,
"content": {"user_prompt": "audio only", "images": []},
}]},
{"type": "audio", "media_index": 0, "segments": [{
"start_frame": 0,
"end_frame": 120,
"content": {"media_type": "audio", "media_index": 0},
}]},
{"type": "video", "segments": []},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [], [0], ["default"],
)
assert result.values[5][0]["waveform"].shape[-1] == 120
assert result.values[6] == [None]
def test_multitrack_task_output_returns_none_when_selected_minimax_task_has_no_media_segments():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(10).reshape(1, 1, 10), "sample_rate": 1}
video_track = _FakeVideo(_VideoComponents(torch.zeros(10, 2, 2, 3), None, Fraction(1)))
tracks_info = {
"format": "MiniMax",
"frame_rate": 1,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 5, "content": {"user_prompt": "first"}},
{"start_frame": 5, "end_frame": 10, "content": {"user_prompt": "second"}},
]},
{"type": "audio", "media_index": 0, "segments": [{
"start_frame": 0,
"end_frame": 5,
"content": {"media_type": "audio", "media_index": 0},
}]},
{"type": "video", "media_index": 0, "segments": [{
"start_frame": 0,
"end_frame": 5,
"content": {"media_type": "video", "media_index": 0},
}]},
],
}
first_result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [[video_track]], [0], ["default"],
)
second_result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [[video_track]], [1], ["default"],
)
assert first_result.values[5][0]["waveform"].flatten().tolist() == list(range(5))
assert first_result.values[6] == [video_track]
assert second_result.values[5] == [None]
assert second_result.values[6] == [None]
assert video_track.trim_calls == [(0.0, 5.0, False)]
def test_multitrack_task_output_minimax_stops_audio_at_current_task_last_segment():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(7).reshape(1, 1, 7), "sample_rate": 1}
tracks_info = {
"format": "MiniMax",
"frame_rate": 1,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 5, "content": {"user_prompt": "first"}},
{"start_frame": 5, "end_frame": 10, "content": {"user_prompt": "second"}},
]},
{"type": "audio", "media_index": 0, "segments": [
{
"start_frame": 0,
"end_frame": 2,
"content": {"media_type": "audio", "media_index": 0},
},
{
"start_frame": 5,
"end_frame": 7,
"content": {"media_type": "audio", "media_index": 0},
},
]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [], [0], ["default"],
)
assert result.values[5][0]["waveform"].flatten().tolist() == [0, 1]
def test_multitrack_task_output_minimax_crops_media_at_next_task_start_across_a_gap():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(15).reshape(1, 1, 15), "sample_rate": 1}
video_track = _FakeVideo(_VideoComponents(torch.zeros(15, 2, 2, 3), None, Fraction(1)))
tracks_info = {
"format": "MiniMax",
"frame_rate": 1,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 5, "content": {"user_prompt": "first"}},
{"start_frame": 10, "end_frame": 12, "content": {"user_prompt": "second"}},
]},
{"type": "audio", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 15, "content": {"media_type": "audio"}},
]},
{"type": "video", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 15, "content": {"media_type": "video"}},
]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [[video_track]], [0], ["default"],
)
assert result.values[5][0]["waveform"].flatten().tolist() == list(range(10))
assert result.values[6] == [video_track]
assert video_track.trim_calls == [(0.0, 10.0, False)]
def test_multitrack_task_output_minimax_trims_the_last_task_from_its_start_to_media_end():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(15).reshape(1, 1, 15), "sample_rate": 1}
video_track = _FakeVideo(_VideoComponents(torch.zeros(15, 2, 2, 3), None, Fraction(1)))
tracks_info = {
"format": "MiniMax",
"frame_rate": 1,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 5, "content": {"user_prompt": "first"}},
{"start_frame": 10, "end_frame": 15, "content": {"user_prompt": "last"}},
]},
{"type": "audio", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 15, "content": {"media_type": "audio"}},
]},
{"type": "video", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 15, "content": {"media_type": "video"}},
]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [[video_track]], [1], ["default"],
)
assert result.values[5][0]["waveform"].flatten().tolist() == list(range(10, 15))
assert result.values[6] == [video_track]
assert video_track.trim_calls == [(10.0, 5.0, False)]
def test_multitrack_task_output_minimax_stops_at_each_track_last_trimmed_segment():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(15).reshape(1, 1, 15), "sample_rate": 1}
video_track = _FakeVideo(_VideoComponents(torch.zeros(15, 2, 2, 3), None, Fraction(1)))
tracks_info = {
"format": "MiniMax",
"frame_rate": 1,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 10, "end_frame": 15, "content": {"user_prompt": "last"}},
]},
{"type": "audio", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 12, "content": {"media_type": "audio"}},
]},
{"type": "video", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 13, "content": {"media_type": "video"}},
]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [[video_track]], [0], ["default"],
)
assert result.values[5][0]["waveform"].flatten().tolist() == [10, 11]
assert result.values[6] == [video_track]
assert video_track.trim_calls == [(10.0, 3.0, False)]
def test_timeline_segment_output_selects_nearest_frames_for_four_second_minimax_segment():
module = _load_basic_module()
timeline_info = {
"format": "MiniMax",
"frame_rate": 24,
"segments": [{
"start_frame": 0,
"end_frame": 96,
"prompt": "four seconds",
"images": [{}],
}],
}
result = module.TimelineSegmentOutput.execute(
timeline_info,
"default",
0,
)
assert result.values[4] == 90
def test_multitrack_task_output_evenly_distributes_regular_task_image_indexes():
module = _load_basic_module()
images = [torch.full((1, 2, 2, 3), float(index)) for index in range(4)]
tracks_info = {
"frame_rate": 24,
"tracks": [{"type": "task", "segments": [{
"start_frame": 10,
"end_frame": 22,
"content": {
"user_prompt": "four conditions",
"images": [{"media_index": index} for index in range(4)],
},
}]}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [], [], [0], ["default"],
)
assert result.values[7] == "0,4,8,-1"
def test_multitrack_task_output_uses_relative_segment_starts_for_marker_image_indexes():
module = _load_basic_module()
images = [torch.zeros(1, 2, 2, 3), torch.ones(1, 2, 2, 3)]
tracks_info = {
"total_length": 12,
"frame_rate": 24,
"task_markers": [
{"id": "middle", "frame": 6},
{"id": "end", "frame": 12},
],
"tracks": [{"type": "task", "segments": [
{
"start_frame": 2,
"end_frame": 8,
"content": {"user_prompt": "first", "images": [{"media_index": 0}]},
},
{
"start_frame": 8,
"end_frame": 10,
"content": {"user_prompt": "no image", "images": []},
},
{
"start_frame": 10,
"end_frame": 12,
"content": {"user_prompt": "last", "images": [{"media_index": 1}]},
},
]}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [], [], [1], ["default"],
)
assert result.values[4] == [images[0], images[1]]
assert result.values[7] == "0,4"
def test_multitrack_task_output_uses_marker_ranges_and_overlapping_task_content():
module = _load_basic_module()
audio_track = {"waveform": torch.arange(20).reshape(1, 1, 20), "sample_rate": 2}
video_track = _FakeVideo(_VideoComponents(torch.zeros(10, 2, 2, 3), None, Fraction(2)))
tracks_info = {
"total_length": 11,
"frame_rate": 2,
"task_markers": [{"id": "marker", "frame": 6}],
"tracks": [
{"type": "task", "segments": [
{"id": "first", "start_frame": 0, "end_frame": 2, "content": {"user_prompt": "first", "images": []}},
{"id": "second", "start_frame": 2, "end_frame": 10, "content": {"user_prompt": "second", "images": []}},
]},
{"type": "video", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 10, "content": {"media_type": "video"}},
]},
{"type": "audio", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 10, "content": {"media_type": "audio"}},
]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [[audio_track]], [[video_track]], [1], ["default"],
)
assert result.values[1] == "second"
assert result.values[3] == 5
assert result.values[5][0]["waveform"].flatten().tolist() == list(range(6, 10))
assert video_track.trim_calls == [(3.0, 2.0, False)]
def test_multitrack_task_output_uses_task_markers_for_full_media_split_ranges():
module = _load_basic_module()
images = [torch.zeros(1, 2, 2, 3), torch.ones(1, 2, 2, 3)]
audio_track = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 1}
video_track = _FakeVideo(_VideoComponents(torch.zeros(12, 2, 2, 3), None, Fraction(1)))
tracks_info = {
"total_length": 12,
"frame_rate": 1,
"task_markers": [
{"id": "first-label", "frame": 4},
{"id": "second-label", "frame": 12},
],
"tracks": [
{"type": "task", "segments": [
{
"start_frame": 2,
"end_frame": 6,
"content": {"user_prompt": "first", "images": [{"media_index": 0}]},
},
{
"start_frame": 6,
"end_frame": 10,
"content": {"user_prompt": "second", "images": [{"media_index": 1}]},
},
]},
{"type": "audio", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 12, "content": {"media_type": "audio"}},
]},
{"type": "video", "media_index": 0, "segments": [
{"start_frame": 0, "end_frame": 12, "content": {"media_type": "video"}},
]},
],
}
first = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [[audio_track]], [[video_track]], [0], ["default"],
)
second = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [[audio_track]], [[video_track]], [1], ["default"],
)
assert first.values[3] == 5
assert first.values[4] == [images[0]]
assert first.values[5][0]["waveform"].flatten().tolist() == [0, 1, 2, 3]
assert second.values[3] == 9
assert second.values[4] == [images[0], images[1]]
assert second.values[5][0]["waveform"].flatten().tolist() == list(range(4, 12))
assert video_track.trim_calls == [(0.0, 4.0, False), (4.0, 8.0, False)]
def test_multitrack_task_output_combines_images_from_all_segments_inside_task_label():
module = _load_basic_module()
images = [
torch.zeros(1, 2, 2, 3),
torch.ones(1, 2, 2, 3),
torch.full((1, 2, 2, 3), 2.0),
]
tracks_info = {
"total_length": 12,
"frame_rate": 1,
"task_markers": [
{"id": "first-label", "frame": 8},
{"id": "second-label", "frame": 12},
],
"tracks": [{
"type": "task",
"segments": [
{
"start_frame": 0,
"end_frame": 4,
"content": {"user_prompt": "first", "images": [{"media_index": 0}]},
},
{
"start_frame": 4,
"end_frame": 8,
"content": {"user_prompt": "second", "images": [{"media_index": 1}]},
},
{
"start_frame": 8,
"end_frame": 12,
"content": {"user_prompt": "third", "images": [{"media_index": 2}]},
},
],
}],
}
first = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [], [], [0], ["default"],
)
second = module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [], [], [1], ["default"],
)
assert first.values[4] == [images[0], images[1]]
assert second.values[4] == [images[2]]
def test_multitrack_task_output_minus_one_materializes_the_complete_deferred_timeline():
module = _load_basic_module()
first_image = torch.zeros(1, 2, 2, 3)
second_image = torch.ones(1, 2, 2, 3)
source_audio = {
"waveform": torch.arange(4, dtype=torch.float32).reshape(1, 1, 4),
"sample_rate": 2,
}
source_video = _FakeVideo(
_VideoComponents(torch.ones(2, 2, 2, 3), None, Fraction(2))
)
module._resolve_timeline_image_item = lambda item, image_input: (
first_image if item["id"] == "first-image" else second_image
)
module._resolve_multitrack_audio = lambda content, audio_input: source_audio
module._resolve_multitrack_video = lambda content, video_input: source_video
module.merge_video_track_with_ffmpeg = lambda *args, **kwargs: None
module._multitrack_module.resize_image = module.resize_image = (
lambda images, width, height, method: images
)
tracks_info = {
"media_loading": "deferred",
"format": "None",
"width": 2,
"height": 2,
"resize_method": "stretch",
"frame_rate": 2,
"timeline_total_length": 6,
"tracks": [
{"type": "task", "segments": [
{
"start_frame": 0,
"end_frame": 3,
"content": {
"user_prompt": "first",
"images": [{"id": "first-image"}],
},
},
{
"start_frame": 3,
"end_frame": 6,
"content": {
"user_prompt": "second",
"images": [{"id": "second-image"}],
},
},
]},
{"type": "audio", "segments": [{
"start_frame": 2,
"end_frame": 4,
"content": {"media_type": "audio"},
}]},
{"type": "video", "segments": [{
"start_frame": 4,
"end_frame": 6,
"content": {"media_type": "video"},
}]},
],
}
result = module.MultiTrackTaskOutput.execute(
tracks_info,
task_index=-1,
prompt_format="default",
)
assert result.values[3] == 7
assert result.values[4] == [first_image, second_image]
assert result.values[7] == "0,3"
assert result.values[5][0]["waveform"].flatten().tolist() == [
0.0, 0.0, 0.0, 1.0, 0.0, 0.0,
]
frames = result.values[6][0].get_components().images
assert [float(frame.mean()) for frame in frames] == [0.0, 0.0, 0.0, 0.0, 1.0, 1.0]
def test_multitrack_task_output_minus_one_reuses_eager_full_timeline_media():
module = _load_basic_module()
image = torch.ones(1, 2, 2, 3)
audio = {"waveform": torch.ones(1, 1, 4), "sample_rate": 2}
video_audio = {"waveform": torch.zeros(1, 1, 4), "sample_rate": 2}
video = _FakeVideo(
_VideoComponents(torch.ones(4, 2, 2, 3), video_audio, Fraction(2))
)
tracks_info = {
"media_loading": "eager",
"frame_rate": 2,
"timeline_total_length": 4,
"tracks": [
{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"user_prompt": "full", "images": [{"media_index": 0}]},
}]},
{"type": "video", "media_index": 0, "audio_locked": True, "segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"media_type": "video", "media_index": 0},
}]},
{"type": "audio", "media_index": 0, "audio_locked": True, "segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"media_type": "audio", "media_index": 0},
}]},
],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [image], [[audio]], [[video]], [-1], ["default"],
)
assert result.values[4] == [image]
assert result.values[5] == [audio]
assert result.values[6] == [video]
assert result.values[8] is audio
assert video.trim_calls == []
def test_multitrack_task_output_minus_one_does_not_stop_at_the_next_minimax_task():
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 1,
}
module._resolve_multitrack_audio = lambda content, audio_input: source_audio
tracks_info = {
"media_loading": "deferred",
"format": "MiniMax",
"width": 2,
"height": 2,
"frame_rate": 1,
"timeline_total_length": 8,
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 4, "content": {"user_prompt": "first"}},
{"start_frame": 4, "end_frame": 8, "content": {"user_prompt": "second"}},
]},
{"type": "audio", "audio_locked": True, "segments": [{
"start_frame": 0,
"end_frame": 8,
"content": {"media_type": "audio"},
}]},
],
}
result = module.MultiTrackTaskOutput.execute(tracks_info, task_index=-1)
assert result.values[5][0]["waveform"].flatten().tolist() == list(range(8))
assert result.values[8]["waveform"].flatten().tolist() == list(range(8))
def test_prepare_multitrack_project_media_extracts_shared_and_locked_audio(monkeypatch):
_load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
shared_image = torch.ones(1, 2, 2, 3)
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
monkeypatch.setattr(
project_module,
"_resolve_timeline_image_item",
lambda *_args: shared_image,
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_audio",
lambda *_args: source_audio,
)
tracks_info = {
"media_loading": "deferred",
"frame_rate": 2,
"timeline_total_length": 8,
"width": 2,
"height": 2,
"tracks": [
{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"images": [{
"file_path": "shared.png",
"shared_reference": True,
}]},
}]},
{"type": "audio", "segments": [{
"start_frame": 0,
"end_frame": 8,
"content": {
"media_type": "audio",
"file_path": "shared.wav",
"shared_reference": True,
},
}]},
{"type": "audio", "audio_locked": True, "segments": [{
"start_frame": 0,
"end_frame": 8,
"content": {"media_type": "audio", "file_path": "locked.wav"},
}]},
],
}
task_info, images, audios, videos, locked = (
project_module.prepare_multitrack_project_media(tracks_info)
)
assert images == [shared_image]
assert audios[0]["waveform"].flatten().tolist() == list(range(8))
assert videos == []
assert locked["waveform"].flatten().tolist() == list(range(8))
assert task_info["tracks"][0]["segments"][0]["content"]["images"] == []
assert task_info["tracks"][1]["segments"] == []
assert task_info["tracks"][2]["segments"] == []
def test_project_shared_image_respects_each_task_image_mute(monkeypatch):
module = _load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
shared_image = torch.ones(1, 2, 2, 3)
monkeypatch.setattr(
project_module,
"_resolve_timeline_image_item",
lambda *_args: shared_image,
)
tracks_info = {
"frame_rate": 1,
"format": "MiniMax",
"tracks": [{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 4, "content": {"images": [{
"file_path": "shared.png", "shared_reference": True, "muted": True,
}]}},
{"start_frame": 4, "end_frame": 8, "content": {"images": [{
"file_path": "shared.png", "shared_reference": True,
}]}},
{"start_frame": 8, "end_frame": 12, "content": {"images": []}},
]}],
}
task_base, shared_images, _audio, _video, _locked = (
project_module.prepare_multitrack_project_media(tracks_info)
)
assert shared_images == [shared_image]
assert task_base["tracks"][0]["segments"][0]["content"]["images"][0]["muted"] is True
entries = project_module.h3_task_entries(task_base)
outputs = []
for index, entry in enumerate(entries):
task_info = project_module.prepare_multitrack_project_task_info(
task_base, shared_images, [], [], task_entry=entry,
)
outputs.append(module.MultiTrackTaskOutput.execute(task_info, task_index=index).values[4])
assert outputs[0] == []
assert outputs[1] == [shared_image]
assert outputs[2] == [shared_image]
def test_prepare_multitrack_project_media_prefers_audio_lock_and_keeps_locked_video_reference(
monkeypatch,
):
module = _load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
video_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
audio_only = {
"waveform": torch.full((1, 1, 8), 9.0),
"sample_rate": 2,
}
video = _FakeVideo(
_VideoComponents(torch.zeros(8, 2, 2, 3), video_audio, Fraction(2)),
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_video",
lambda *_args: video,
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_audio",
lambda *_args: audio_only,
)
tracks_info = {
"frame_rate": 2,
"timeline_total_length": 8,
"media": {"audio": [audio_only], "video": [video]},
"tracks": [
{"type": "task", "segments": [{
"start_frame": 2,
"end_frame": 6,
"content": {"media_type": "none", "images": []},
}]},
{
"type": "video",
"media_index": 0,
"audio_locked": True,
"segments": [{
"id": "locked-video",
"start_frame": 0,
"end_frame": 8,
"content": {"media_type": "video", "media_index": 0},
}],
},
{
"type": "audio",
"media_index": 0,
"audio_locked": True,
"segments": [{
"id": "locked-audio",
"start_frame": 0,
"end_frame": 8,
"content": {"media_type": "audio", "media_index": 0},
}],
},
],
}
task_info, _images, _audios, _videos, locked = (
project_module.prepare_multitrack_project_media(tracks_info)
)
assert locked["waveform"].flatten().tolist() == [9.0] * 8
assert task_info["tracks"][1] == tracks_info["tracks"][1]
assert task_info["tracks"][2]["segments"] == []
assert "media_index" not in task_info["tracks"][2]
task_result = module.MultiTrackTaskOutput.execute(task_info, task_index=0)
assert task_result.values[5] == []
assert task_result.values[6] == [video]
assert video.trim_calls[-1] == (1.0, 2.0, False)
def test_prepare_multitrack_project_media_audio_lock_dedupes_video_decode(monkeypatch):
"""Many segments sharing one source video must not re-decode it per segment."""
_load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
video_audio = {
"waveform": torch.arange(16, dtype=torch.float32).reshape(1, 1, 16),
"sample_rate": 2,
}
video = _FakeVideo(
_VideoComponents(torch.zeros(16, 2, 2, 3), video_audio, Fraction(2)),
source="locked.mp4",
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_video",
lambda *_args, **_kwargs: video,
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_audio",
lambda *_args, **_kwargs: None,
)
segments = []
for index in range(40):
start = index
end = index + 1
segments.append({
"id": f"seg-{index}",
"start_frame": start,
"end_frame": end,
"origin_start_frame": start,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "locked.mp4",
},
})
tracks_info = {
"frame_rate": 2,
"timeline_total_length": 40,
"tracks": [
{"type": "video", "audio_locked": True, "segments": segments},
],
}
_task_info, _images, _audios, _videos, locked = (
project_module.prepare_multitrack_project_media(tracks_info)
)
# The whole point of the cache: decode the long video once, not 40 times.
assert video.components_calls == 1
assert locked is not None
# The merge step reshapes the audio to ``timeline_total_length *
# sample_rate / frame_rate`` samples, regardless of source length.
expected_samples = round(40 * 2 / 2)
assert locked["waveform"].shape[-1] == expected_samples
assert locked["sample_rate"] == video_audio["sample_rate"]
def test_prepare_multitrack_project_media_audio_lock_prefers_ffmpeg(monkeypatch):
"""When the locked video is file-backed, FFmpeg should extract the audio,
leaving PyAV's full-frame decode on the cold path."""
_load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
ffmpeg_audio = {
"waveform": torch.full((1, 1, 16), 3.0),
"sample_rate": 2,
}
pyav_audio = {
"waveform": torch.full((1, 1, 16), 9.0),
"sample_rate": 2,
}
video = _FakeVideo(
_VideoComponents(torch.zeros(16, 2, 2, 3), pyav_audio, Fraction(2)),
source="locked.mp4",
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_video",
lambda *_args, **_kwargs: video,
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_audio",
lambda *_args, **_kwargs: None,
)
ffmpeg_calls = []
def fake_ffmpeg(source_path):
ffmpeg_calls.append(source_path)
return ffmpeg_audio
monkeypatch.setattr(
sys.modules["easy_media.utils.video"], "ffmpeg_extract_audio", fake_ffmpeg,
)
monkeypatch.setattr(
sys.modules["easy_media.utils.video"],
"ffprobe_info",
lambda _path: {"has_audio": True},
)
monkeypatch.setattr(os.path, "isfile", lambda _path: True)
tracks_info = {
"frame_rate": 2,
"timeline_total_length": 8,
"tracks": [
{"type": "video", "audio_locked": True, "segments": [
{
"id": "seg-0",
"start_frame": 0,
"end_frame": 4,
"origin_start_frame": 0,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "locked.mp4",
},
},
{
"id": "seg-1",
"start_frame": 4,
"end_frame": 8,
"origin_start_frame": 4,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "locked.mp4",
},
},
]},
],
}
_task_info, _images, _audios, _videos, locked = (
project_module.prepare_multitrack_project_media(tracks_info)
)
# FFmpeg wins and is hit once thanks to the cache; PyAV never gets called.
assert ffmpeg_calls == ["locked.mp4"]
assert video.components_calls == 0
assert locked is not None
assert locked["waveform"].flatten().tolist() == [3.0] * 8
def test_prepare_multitrack_project_media_audio_lock_falls_back_to_pyav(
monkeypatch, caplog,
):
"""When FFmpeg cannot extract audio, the locked audio still resolves via
PyAV so the existing audio-locked workflow continues to function. The
fallback path should also emit a ``[Warning]`` log so users notice."""
_load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
pyav_audio = {
"waveform": torch.full((1, 1, 8), 4.0),
"sample_rate": 2,
}
video = _FakeVideo(
_VideoComponents(torch.zeros(8, 2, 2, 3), pyav_audio, Fraction(2)),
source="locked.mp4",
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_video",
lambda *_args, **_kwargs: video,
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_audio",
lambda *_args, **_kwargs: None,
)
def fake_ffmpeg(_source_path):
return None
monkeypatch.setattr(
sys.modules["easy_media.utils.video"], "ffmpeg_extract_audio", fake_ffmpeg,
)
monkeypatch.setattr(
sys.modules["easy_media.utils.video"],
"ffprobe_info",
lambda _path: {"has_audio": True},
)
# ``os.path.isfile`` would normally return False for ``"locked.mp4"`` in
# this test environment. Patch it to True so the helper actually attempts
# FFmpeg before falling back — that's the path we want to exercise.
monkeypatch.setattr(os.path, "isfile", lambda _path: True)
tracks_info = {
"frame_rate": 2,
"timeline_total_length": 8,
"tracks": [
{"type": "video", "audio_locked": True, "segments": [
{
"id": "seg-0",
"start_frame": 0,
"end_frame": 8,
"origin_start_frame": 0,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "locked.mp4",
},
},
]},
],
}
caplog.set_level(logging.WARNING, logger="utils.video")
_task_info, _images, _audios, _videos, locked = (
project_module.prepare_multitrack_project_media(tracks_info)
)
assert video.components_calls == 1
assert locked is not None
assert locked["waveform"].flatten().tolist() == [4.0] * 8
assert "[extract_video_audio]" in caplog.text
assert "falling back to PyAV" in caplog.text
def test_multitrack_task_output_audio_lock_prefers_ffmpeg(monkeypatch):
"""``MultiTrackTaskOutput`` should route locked-audio video tracks through
the shared ``extract_video_audio`` helper, so file-backed sources go
straight to FFmpeg and never reach PyAV's full-frame decode."""
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
video = _FakeVideo(
_VideoComponents(torch.zeros(8, 2, 2, 3), source_audio, Fraction(2)),
source="locked.mp4",
)
module._resolve_multitrack_video = lambda content, video_input: video
ffmpeg_calls = []
def fake_ffmpeg(source_path):
ffmpeg_calls.append(source_path)
return source_audio
monkeypatch.setattr(
sys.modules["easy_media.utils.video"], "ffmpeg_extract_audio", fake_ffmpeg,
)
monkeypatch.setattr(
sys.modules["easy_media.utils.video"],
"ffprobe_info",
lambda _path: {"has_audio": True},
)
monkeypatch.setattr(os.path, "isfile", lambda _path: True)
# ``output_full_timeline=True`` routes through ``video_items[media_index]``
# (site 3056) instead of the per-track deferred branch.
tracks_info = {
"media_loading": "eager",
"format": "MiniMax",
"width": 2,
"height": 2,
"total_length": 8,
"frame_rate": 2,
"tracks": [
{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 8,
"content": {"media_type": "none", "images": []},
}],
},
{
"type": "video",
"media_index": 0,
"audio_locked": True,
"segments": [{
"id": "locked-video",
"start_frame": 0,
"end_frame": 8,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "locked.mp4",
},
}],
},
],
}
result = module.MultiTrackTaskOutput.execute(
tracks_info, task_index=-1, video=[video],
)
assert ffmpeg_calls == ["locked.mp4"]
# FFmpeg-first path means PyAV is never reached for the locked audio.
assert video.components_calls == 0
assert result is not None
def test_extract_video_audio_caches_by_source_path(monkeypatch):
"""Two different ``VideoInput`` wrappers around the same source file must
share a single extraction via the cache dict the helper accepts."""
_load_basic_module()
video_module = sys.modules["easy_media.utils.video"]
audio = {
"waveform": torch.full((1, 1, 4), 5.0),
"sample_rate": 2,
}
def fake_ffmpeg(source_path):
return audio
monkeypatch.setattr(video_module, "ffmpeg_extract_audio", fake_ffmpeg)
monkeypatch.setattr(
video_module,
"ffprobe_info",
lambda _path: {"has_audio": True},
)
monkeypatch.setattr(os.path, "isfile", lambda _path: True)
# Two distinct _FakeVideo instances but same ``source`` path.
video_a = _FakeVideo(
_VideoComponents(torch.zeros(4, 2, 2, 3), audio, Fraction(2)),
source="locked.mp4",
)
video_b = _FakeVideo(
_VideoComponents(torch.zeros(4, 2, 2, 3), audio, Fraction(2)),
source="locked.mp4",
)
cache: dict = {}
first = video_module.extract_video_audio(video_a, cache=cache)
second = video_module.extract_video_audio(video_b, cache=cache)
assert first is audio
assert second is audio
# Cache key matches the source path so two distinct wrappers dedupe.
assert cache == {"locked.mp4": audio}
# Neither wrapper paid the PyAV decode cost.
assert video_a.components_calls == 0
assert video_b.components_calls == 0
def test_extract_video_audio_caches_missing_audio(monkeypatch):
"""A source confirmed to have no audio must not be probed per segment."""
_load_basic_module()
video_module = sys.modules["easy_media.utils.video"]
video = _FakeVideo(
_VideoComponents(torch.zeros(4, 2, 2, 3), None, Fraction(2)),
source="silent.mp4",
)
ffmpeg_calls = []
monkeypatch.setattr(os.path, "isfile", lambda _path: True)
monkeypatch.setattr(video_module, "ffprobe_info", lambda _path: {"has_audio": False})
monkeypatch.setattr(
video_module,
"ffmpeg_extract_audio",
lambda path: ffmpeg_calls.append(path),
)
cache: dict = {}
assert video_module.extract_video_audio(video, cache=cache) is None
assert video_module.extract_video_audio(video, cache=cache) is None
assert cache == {"silent.mp4": None}
assert ffmpeg_calls == []
assert video.components_calls == 0
def test_extract_video_audio_timeout_does_not_start_full_frame_decode(
monkeypatch,
):
_load_basic_module()
video_module = sys.modules["easy_media.utils.video"]
video = _FakeVideo(
_VideoComponents(torch.zeros(4, 2, 2, 3), None, Fraction(2)),
source="locked.mp4",
)
monkeypatch.setattr(os.path, "isfile", lambda _path: True)
monkeypatch.setattr(video_module, "ffprobe_info", lambda _path: {"has_audio": True})
def timed_out(_path):
raise subprocess.TimeoutExpired(cmd="ffmpeg", timeout=60)
monkeypatch.setattr(video_module, "ffmpeg_extract_audio", timed_out)
with pytest.raises(RuntimeError, match="FFmpeg audio extraction timed out"):
video_module.extract_video_audio(video)
assert video.components_calls == 0
def test_prepare_multitrack_project_media_muted_video_does_not_lock_audio(
monkeypatch,
):
_load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
video = _FakeVideo(
_VideoComponents(
torch.zeros(4, 2, 2, 3),
{"waveform": torch.ones(1, 1, 4), "sample_rate": 2},
Fraction(2),
),
)
monkeypatch.setattr(
project_module,
"_resolve_multitrack_video",
lambda *_args, **_kwargs: video,
)
tracks_info = {
"frame_rate": 2,
"timeline_total_length": 4,
"tracks": [{
"type": "video",
"audio_locked": True,
"muted": True,
"segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"media_type": "video"},
}],
}],
}
locked = project_module.prepare_multitrack_project_media(tracks_info)[-1]
assert locked is None
assert video.components_calls == 0
def test_crop_multitrack_project_media_crops_reused_audio_and_video():
_load_basic_module()
project_module = sys.modules["easy_media.utils.h3_project"]
shared_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 2,
}
shared_video = _FakeVideo(
_VideoComponents(torch.ones(8, 2, 2, 3), None, Fraction(2))
)
audios, videos, locked = project_module.crop_multitrack_project_media(
[shared_audio],
[shared_video],
shared_audio,
2,
4,
2,
)
assert audios[0]["waveform"].flatten().tolist() == [0, 1, 2, 3]
assert shared_video.trim_calls == [(0.0, 2.0, False)]
assert videos
assert locked["waveform"].flatten().tolist() == [2, 3, 4, 5]
def test_multitrack_audio_output_schema_is_basic_and_exposes_mode_and_two_tracks():
module = _load_basic_module()
schema = module.MultiTrackAudioOutput.define_schema()
assert schema.node_id == "easy multiTrackAudioOutput"
assert schema.category == "EasyUse/MultiTrackEditor"
assert schema.is_input_list is True
assert [input_.name for input_ in schema.inputs] == ["tracks_info", "audio", "mode", "task_index"]
assert schema.inputs[2].kwargs["options"] == ["default", "crop"]
assert schema.inputs[2].kwargs["default"] == "default"
assert schema.inputs[3].kwargs["default"] == 0
assert schema.inputs[3].kwargs["min"] == 0
assert [output.name for output in schema.outputs] == [
"combine_audio", "audio_0", "audio_0_start", "audio_1", "audio_1_start",
]
def test_multitrack_audio_output_chinese_options_match_schema():
module = _load_basic_module()
schema = module.MultiTrackAudioOutput.define_schema()
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
translation = node_defs[schema.node_id]
mode_options = translation["inputs"]["mode"]["options"]
assert translation["display_name"]
assert translation["description"]
assert set(translation["inputs"]) == {input_.name for input_ in schema.inputs}
assert set(translation["outputs"]) == {str(index) for index in range(len(schema.outputs))}
for input_ in schema.inputs:
assert translation["inputs"][input_.name]["name"]
if input_.kwargs.get("tooltip"):
assert translation["inputs"][input_.name]["tooltip"]
for output_index in range(len(schema.outputs)):
assert translation["outputs"][str(output_index)]["name"]
assert set(mode_options) == set(schema.inputs[2].kwargs["options"])
assert mode_options["crop"] == "S2V 裁剪"
def test_multitrack_audio_output_crop_merges_audio_and_crops_to_track_frame_ranges(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 4}
second = {"waveform": torch.arange(20, 32).reshape(1, 1, 12), "sample_rate": 4}
calls = []
def fake_merge(audios, method="add"):
calls.append((audios, method))
return {"waveform": torch.tensor([[[0.75, 0.75]]]), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", fake_merge)
tracks_info = {
"frame_rate": 4,
"tracks": [
{"type": "video", "segments": [{"start_frame": 1}]},
{"type": "audio", "segments": [
{"start_frame": 6, "end_frame": 9},
{"start_frame": 2, "end_frame": 4},
]},
{"type": "audio", "segments": [
{"start_time": 1.5, "end_time": 2.0},
{"start_time": 0.5, "end_time": 1.0},
]},
],
}
result = module.MultiTrackAudioOutput.execute([tracks_info], [[first, second]], ["crop"], [-1])
assert calls == [([first, second], "add")]
assert result.values[1]["waveform"].flatten().tolist() == list(range(2, 9))
assert result.values[2] == 2
assert result.values[3]["waveform"].flatten().tolist() == list(range(22, 28))
assert result.values[4] == 2
def test_multitrack_audio_output_crop_uses_minus_one_for_missing_tracks_or_segments(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.ones(1, 1, 2), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
result = module.MultiTrackAudioOutput.execute(
[{"frame_rate": 24, "tracks": [{"type": "audio", "segments": []}]}],
[[first]],
["crop"],
[-1],
)
assert result.values == (first, None, -1, None, -1)
def test_multitrack_audio_output_default_returns_full_tracks_with_zero_starts(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 4}
second = {"waveform": torch.arange(20, 32).reshape(1, 1, 12), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
result = module.MultiTrackAudioOutput.execute(
[{"frame_rate": 4, "tracks": []}],
[[first, second]],
["default"],
[-1],
)
assert result.values == (first, first, 0, second, 0)
def test_multitrack_audio_output_materializes_deferred_file_tracks(monkeypatch):
module = _load_basic_module()
source_audio = {
"waveform": torch.arange(8, dtype=torch.float32).reshape(1, 1, 8),
"sample_rate": 1,
}
monkeypatch.setattr(module, "_resolve_multitrack_audio", lambda content, audio_input: source_audio)
track_data = {
"total_length": 6,
"frame_rate": 1,
"tracks": [
{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 6,
"content": {"media_type": "none", "images": []},
}],
},
{
"type": "audio",
"segments": [{
"start_frame": 2,
"end_frame": 4,
"origin_start_frame": 0,
"content": {
"media_type": "audio",
"source_type": "input",
"file_path": "audio.wav",
},
}],
},
],
}
tracks_info = module.MultiTrackEditor.execute(
{"resolution": "640 x 360 (16:9)"},
"None",
track_data,
).values[0]
result = module.MultiTrackAudioOutput.execute(tracks_info, None, "default", 0)
assert result.values[1]["waveform"].flatten().tolist() == [
0.0, 0.0, 2.0, 3.0, 0.0, 0.0,
]
def test_multitrack_audio_output_default_task_index_returns_full_tracks(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 4}
second = {"waveform": torch.arange(20, 32).reshape(1, 1, 12), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
tracks_info = {
"frame_rate": 4,
"tracks": [
{"type": "task", "segments": [{"start_frame": 3, "end_frame": 8}]},
{"type": "audio", "segments": [{"start_frame": 3, "end_frame": 8}]},
{"type": "audio", "segments": [{"start_frame": 4, "end_frame": 7}]},
],
}
result = module.MultiTrackAudioOutput.execute(
[tracks_info], [[first, second]], ["default"],
)
assert result.values == (first, first, 0, second, 0)
def test_multitrack_audio_output_crop_task_index_uses_track_segments_relative_to_task_start(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 4}
second = {"waveform": torch.arange(20, 32).reshape(1, 1, 12), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
tracks_info = {
"frame_rate": 4,
"tracks": [{
"type": "task",
"segments": [
{"start_frame": 0, "end_frame": 2},
{"start_frame": 3, "end_frame": 8},
],
}, {
"type": "audio",
"segments": [{"start_frame": 0, "end_frame": 2}],
}, {
"type": "audio",
"segments": [{"start_frame": 5, "end_frame": 7}],
}],
}
result = module.MultiTrackAudioOutput.execute(
[tracks_info], [[first, second]], ["crop"], [1],
)
task_zero_result = module.MultiTrackAudioOutput.execute(
[tracks_info], [[first, second]], ["crop"], [0],
)
assert result.values[0] is first
assert result.values[1]["sample_rate"] == 4
assert result.values[1]["waveform"].flatten().tolist() == [0, 0, 0, 0, 0]
assert result.values[2] == -1
assert result.values[3]["waveform"].flatten().tolist() == [25, 26]
assert result.values[4] == 2
assert task_zero_result.values[1]["waveform"].flatten().tolist() == [0, 1]
assert task_zero_result.values[2] == 0
assert task_zero_result.values[4] == -1
def test_multitrack_audio_output_crop_uses_marker_ranges_before_task_segments(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
tracks_info = {
"frame_rate": 4,
"total_length": 11,
"task_markers": [{"id": "marker", "frame": 6}],
"tracks": [
{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 2},
{"start_frame": 2, "end_frame": 10},
]},
{"type": "audio", "segments": [{"start_frame": 5, "end_frame": 9}]},
],
}
result = module.MultiTrackAudioOutput.execute(
[tracks_info], [[first]], ["crop"], [1],
)
assert result.values[1]["waveform"].flatten().tolist() == [6, 7, 8]
assert result.values[2] == 0
def test_multitrack_audio_output_task_index_clips_overlapping_track_to_task_start(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(12).reshape(1, 1, 12), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
tracks_info = {
"frame_rate": 4,
"tracks": [
{"type": "task", "segments": [{"start_frame": 3, "end_frame": 8}]},
{"type": "audio", "segments": [{"start_frame": 1, "end_frame": 5}]},
],
}
result = module.MultiTrackAudioOutput.execute(
[tracks_info], [[first]], ["crop"], [0],
)
assert result.values[1]["waveform"].flatten().tolist() == [3, 4]
assert result.values[2] == 0
def test_multitrack_audio_output_invalid_task_index_returns_empty_track_outputs(monkeypatch):
module = _load_basic_module()
first = {"waveform": torch.arange(8).reshape(1, 1, 8), "sample_rate": 4}
monkeypatch.setattr(module, "merge_audio_inputs", lambda audios, method="add": first)
result = module.MultiTrackAudioOutput.execute(
[{"frame_rate": 4, "tracks": []}], [[first]], ["crop"], [0],
)
assert result.values == (first, None, -1, None, -1)
def test_multitrack_task_output_supports_prompt_formats_and_non_overlapping_ranges():
module = _load_basic_module()
images = [torch.zeros(1, 2, 2, 3), torch.ones(1, 2, 2, 3)]
tracks_info = {
"frame_rate": 24,
"tracks": [{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 121,
"content": {
"task_mode": "default",
"text": "first | second",
"images": [{"media_index": 0}, {"media_index": 1}],
},
}]}],
}
def execute(prompt_format):
return module.MultiTrackTaskOutput.execute(
[tracks_info], [images], [], [], [0], [prompt_format]
).values
default = execute("default")
relay = execute("promptRelay")
api = execute("api")
llm = execute("llm")
assert default[0] == ""
assert relay[0] == ""
assert api[0] == llm[0] == "api:first | second"
assert default[1] == "first | second"
assert relay[1] == "first [0-61] | second [61-121]"
assert api[1] == "first | second"
assert llm[1] == "llm:api:first | second:first | second:False"
def test_multitrack_task_output_prompt_relay_uses_task_range_without_images():
module = _load_basic_module()
tracks_info = {
"frame_rate": 24,
"tracks": [{"type": "task", "segments": [{
"start_frame": 5,
"end_frame": 10,
"content": {"task_mode": "default", "user_prompt": "single prompt", "images": []},
}]}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["promptRelay"]
)
assert result.values[1] == "single prompt [5-10]"
def test_multitrack_task_output_filters_at_signs_from_prompt_outputs():
module = _load_basic_module()
prompt_builder_stub = sys.modules["easy_media.utils.prompt_builder"]
tracks_info = {
"frame_rate": 24,
"tracks": [{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 12,
"content": {
"task_mode": "default",
"user_prompt": "Use @图片1 and <Picture 1>",
"system_prompt": "Keep @resource references",
"images": [],
},
}]}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["api"],
)
assert result.values[1] == "Use 图片1 and <Picture 1>"
assert prompt_builder_stub.calls[-1][1] == "Use 图片1 and <Picture 1>"
assert prompt_builder_stub.calls[-1][2]["custom_system_prompt"] == "Keep resource references"
def test_multitrack_task_output_prompt_relay_joins_task_fragments_inside_marker_ranges():
module = _load_basic_module()
tracks_info = {
"total_length": 12,
"frame_rate": 24,
"task_markers": [
{"id": "middle", "frame": 6},
{"id": "end", "frame": 12},
],
"tracks": [{"type": "task", "segments": [
{"start_frame": 0, "end_frame": 4, "content": {"user_prompt": "first"}},
{"start_frame": 4, "end_frame": 8, "content": {"user_prompt": "second"}},
{"start_frame": 8, "end_frame": 12, "content": {"user_prompt": "third"}},
]}],
}
first = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["promptRelay"],
)
second = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [1], ["promptRelay"],
)
assert first.values[1] == "first [0-4] | second [4-6]"
assert second.values[1] == "second [6-8] | third [8-12]"
def test_prompt_builder_supports_t2v_and_i2v_tasks():
module = _load_prompt_builder_module()
t2v_system, t2v_user, _ = module.build_prompt_request("t2v", "a person walks")
i2v_system, i2v_user, _ = module.build_prompt_request(
"i2v",
"a person walks",
images=[torch.zeros(1, 2, 2, 3)],
)
assert t2v_system == module.T2V_TEMPLATE
assert t2v_user == "a person walks"
assert i2v_system == module.SYSTEM_PROMPTS["default"]
assert "a person walks" in i2v_user
assert "1 reference image(s)" in i2v_user
def test_multitrack_task_output_uses_l2v_and_minimax_base_system_prompt():
module = _load_basic_module()
prompt_builder_stub = sys.modules["easy_media.utils.prompt_builder"]
tracks_info = {
"format": "MiniMax",
"frame_rate": 24,
"tracks": [{"type": "task", "segments": [{
"start_frame": 0,
"end_frame": 120,
"content": {
"task_mode": "l2v",
"user_prompt": "finish on this frame",
"images": [],
},
}]}],
}
result = module.MultiTrackTaskOutput.execute(
[tracks_info], [], [], [], [0], ["api"],
)
assert result.values[0] == "api:finish on this frame"
assert result.values[1] == "finish on this frame"
assert result.values[2] == "l2v"
assert prompt_builder_stub.calls[-1][2]["video_format"] == "MiniMax"
assert prompt_builder_stub.calls[-1][2]["task_mode"] == "l2v"
def test_multitrack_prompt_enhancer_schema_exposes_requested_inputs_and_outputs():
module = _load_basic_module()
schema = module.MultiTrackPromptEnhancer.define_schema()
inputs = {input_port.name: input_port for input_port in schema.inputs}
assert schema.node_id == "easy multiTrackPromptEnhancer"
assert schema.is_input_list is True
assert schema.not_idempotent is True
assert schema.enable_expand is True
assert inputs["enabled"].kwargs["default"] is True
assert inputs["system_prompt"].kwargs["force_input"] is True
assert inputs["user_prompt"].kwargs["force_input"] is True
assert inputs["llama_model"].kwargs["lazy"] is True
assert inputs["llama_model"].kwargs["raw_link"] is True
assert schema.inputs[-1].name == "api_account"
assert "socketless" not in inputs["api_account"].kwargs
model_options = dict(inputs["model"].kwargs["options"])
h3_inputs = {port.name: port for port in model_options[module.MINIMAX_MODEL]}
assert h3_inputs["ratio"].kwargs["options"] == [
"adaptive",
"21:9",
"16:9",
"4:3",
"1:1",
"3:4",
"9:16",
]
assert h3_inputs["return_async"].kwargs["tooltip"] == (
"Only effective for h3-context-ir. When enabled, return the task ID "
"without polling the task status."
)
assert set(h3_inputs) == {"apikey", "ratio", "return_async"}
for model_name, (default, maximum) in module.PROMPT_ENHANCER_MAX_TOKENS.items():
child_inputs = {port.name: port for port in model_options[model_name]}
expected_inputs = (
{"inference_mode", "force_offload", "max_size"}
if model_name == module.LLAMACPP_MODEL
else {"apikey", "max_tokens"}
)
assert set(child_inputs) == expected_inputs
value_name = "max_size" if model_name == module.LLAMACPP_MODEL else "max_tokens"
assert child_inputs[value_name].kwargs["default"] == default
assert child_inputs[value_name].kwargs["max"] == maximum
assert all(input_port.kwargs.get("tooltip") for input_port in schema.inputs)
assert all(output_port.kwargs.get("tooltip") for output_port in schema.outputs)
assert [output.name for output in schema.outputs] == [
"PROMPT",
"TASK_ID",
"FILE_IDS",
]
execute_parameters = set(inspect.signature(module.MultiTrackPromptEnhancer.execute).parameters)
assert "api_account" in execute_parameters
assert execute_parameters.isdisjoint({
"apikey",
"ratio",
"return_async",
"inference_mode",
"max_frames",
"force_offload",
"max_tokens",
})
def test_multitrack_prompt_enhance_to_project_schema_reuses_enhancer_widgets():
module = _load_basic_module()
schema = module.MultiTrackPromptEnhanceToProject.define_schema()
inputs = {input_port.name: input_port for input_port in schema.inputs}
assert schema.node_id == "easy multitrackPromptEnhanceToProject"
assert schema.is_input_list is True
assert schema.not_idempotent is True
assert schema.enable_expand is True
assert set(inputs) == {
"tracks_info",
"llama_model",
"model",
"seed",
"enabled",
"api_account",
}
assert inputs["llama_model"].kwargs["lazy"] is True
assert inputs["llama_model"].kwargs["raw_link"] is True
assert dict(inputs["model"].kwargs["options"]).keys() == dict(
module.MultiTrackPromptEnhancer._model_options()
).keys()
project_h3_inputs = dict(inputs["model"].kwargs["options"])[module.MINIMAX_MODEL]
project_return_async = {
port.name: port for port in project_h3_inputs
}["return_async"]
assert "always waits" in project_return_async.kwargs["tooltip"]
assert [output.name for output in schema.outputs] == ["TRACKS_INFO", "PROMPTS"]
assert "is_output_list" not in schema.outputs[1].kwargs
assert all(input_port.kwargs.get("tooltip") for input_port in schema.inputs)
assert all(output_port.kwargs.get("tooltip") for output_port in schema.outputs)
def test_multitrack_prompt_enhance_to_project_does_not_inherit_enhancer_types():
module = _load_basic_module()
# ComfyUI V3 caches RETURN_TYPES on the base node class. Keeping this node
# as a sibling instead of a subclass prevents it from inheriting the
# enhancer's STRING outputs for its TRACKS_INFO output.
assert not issubclass(
module.MultiTrackPromptEnhanceToProject,
module.MultiTrackPromptEnhancer,
)
def test_multitrack_prompt_enhance_to_project_updates_each_task_in_time_order(
monkeypatch,
):
module = _load_basic_module()
calls = []
image = torch.zeros(1, 2, 2, 3)
video = _FakeVideo(
_VideoComponents(torch.zeros(240, 2, 2, 3), None, Fraction(24))
)
class PublicNodeOutput:
def __init__(self, *args, expand=None):
self.args = args
self.expand = expand
def __getitem__(self, index):
return self.args[index]
def fake_enhance(cls, **kwargs):
calls.append(kwargs)
return PublicNodeOutput(f"enhanced-{len(calls)}", "", "")
monkeypatch.setattr(
module.MultiTrackPromptEnhancer,
"execute",
classmethod(fake_enhance),
)
tracks_info = {
"frame_rate": 24,
"format": "MiniMax",
"media_loading": "eager",
"eager_media_types": ["image", "video"],
"media": {"images": [image], "audio": [None], "video": [video]},
"tracks": [
{
"type": "task",
"segments": [
{
"id": "later",
"start_frame": 120,
"end_frame": 240,
"content": {
"user_prompt_variant": "b",
"user_prompt": "unused-a",
"user_prompt_b": "later @prompt",
"task_mode": "ref",
},
},
{
"id": "earlier",
"start_frame": 0,
"end_frame": 120,
"content": {
"user_prompt": "earlier @prompt",
"images": [{"media_index": 0}],
},
},
],
},
{
"type": "audio",
"segments": [{
"start_frame": 0,
"end_frame": 240,
"content": {"user_prompt": "must not change"},
}],
},
{
"type": "video",
"media_index": 0,
"segments": [{
"start_frame": 120,
"end_frame": 240,
"content": {"media_type": "video", "media_index": 0},
}],
},
],
}
output = module.MultiTrackPromptEnhanceToProject.execute(
tracks_info=[tracks_info],
model=[{"model": module.MINIMAX_MODEL, "return_async": True}],
seed=[7],
)
updated_info, prompts = output.values
updated_tasks = module._multitrack_task_segments(updated_info)
assert prompts == ["enhanced-1", "enhanced-2"]
assert updated_tasks[0]["content"]["user_prompt"] == "enhanced-1"
assert updated_tasks[1]["content"]["user_prompt_b"] == "enhanced-2"
assert updated_tasks[1]["content"]["user_prompt"] == "unused-a"
assert updated_info["tracks"][1]["segments"][0]["content"]["user_prompt"] == (
"must not change"
)
assert tracks_info["tracks"][0]["segments"][1]["content"]["user_prompt"] == (
"earlier @prompt"
)
assert [call["user_prompt"] for call in calls] == [
["earlier prompt"],
["later prompt"],
]
assert [call["type"] for call in calls] == [["i2v"], ["rv2v"]]
prompt_builder_calls = sys.modules["easy_media.utils.prompt_builder"].calls
assert [call[0] for call in prompt_builder_calls[-2:]] == ["i2v", "rv2v"]
assert all(call["model"][0]["return_async"] is False for call in calls)
assert calls[0]["images"] == [image]
assert calls[0]["video"] == [None]
assert calls[1]["images"] == []
assert calls[1]["video"] == [video]
assert all(call["audio"] == [None] for call in calls)
segment_progress = _ProgressBar.instances[-1]
assert segment_progress.total == 2
assert segment_progress.updates == [0, 1, 2]
def test_multitrack_prompt_enhance_to_project_disabled_preserves_prompts(monkeypatch):
module = _load_basic_module()
monkeypatch.setattr(
module.MultiTrackPromptEnhancer,
"execute",
classmethod(lambda cls, **kwargs: pytest.fail("enhancer must not run")),
)
tracks_info = {
"tracks": [{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 24,
"content": {"user_prompt": "Keep @reference unchanged"},
}],
}],
}
output = module.MultiTrackPromptEnhanceToProject.execute(
tracks_info=[tracks_info],
enabled=[False],
)
assert output.values[1] == ["Keep @reference unchanged"]
assert output.values[0] is not tracks_info
assert output.values[0]["tracks"] is not tracks_info["tracks"]
assert output.values[0]["tracks"][0]["segments"][0]["content"][
"user_prompt"
] == "Keep @reference unchanged"
def test_multitrack_prompt_enhance_to_project_apply_updates_expanded_prompts():
module = _load_basic_module()
apply_schema = module.MultiTrackPromptEnhanceToProjectApply.define_schema()
assert "is_output_list" not in apply_schema.outputs[1].kwargs
tracks_info = {
"tracks": [{
"type": "task",
"segments": [
{"start_frame": 0, "end_frame": 24, "content": {"user_prompt": "a"}},
{"start_frame": 24, "end_frame": 48, "content": {"user_prompt": "b"}},
],
}],
}
output = module.MultiTrackPromptEnhanceToProjectApply.execute(
tracks_info,
2,
prompt_0="first",
prompt_1="second",
)
assert output.values[1] == ["first", "second"]
assert [
task["content"]["user_prompt"]
for task in module._multitrack_task_segments(output.values[0])
] == ["first", "second"]
def test_multitrack_prompt_enhance_to_project_collects_local_expansion(monkeypatch):
module = _load_basic_module()
call_count = 0
calls = []
def fake_expand(cls, **kwargs):
nonlocal call_count
calls.append(kwargs)
node_id = f"local-prompt-{call_count}"
call_count += 1
return module.io.NodeOutput(
[node_id, 0],
"",
"",
expand={node_id: {"class_type": "FakeLocalLlama", "inputs": {}}},
)
monkeypatch.setattr(
module.MultiTrackPromptEnhancer,
"execute",
classmethod(fake_expand),
)
tracks_info = {
"tracks": [{
"type": "task",
"segments": [
{"start_frame": 0, "end_frame": 24, "content": {"user_prompt": "a"}},
{"start_frame": 24, "end_frame": 48, "content": {"user_prompt": "b"}},
],
}],
}
output = module.MultiTrackPromptEnhanceToProject.execute(
tracks_info=[tracks_info],
model=[{"model": module.LLAMACPP_MODEL, "force_offload": True}],
)
apply_node = output.expand["multitrack_prompt_project_apply"]
assert output.values == (
["multitrack_prompt_project_apply", 0],
["multitrack_prompt_project_apply", 1],
)
assert apply_node["class_type"] == module.MULTITRACK_PROMPT_PROJECT_APPLY_NODE_ID
assert apply_node["inputs"]["prompt_0"] == ["local-prompt-0", 0]
assert apply_node["inputs"]["prompt_1"] == ["local-prompt-1", 0]
assert set(output.expand) == {
"local-prompt-0",
"local-prompt-1",
"multitrack_prompt_project_apply",
}
assert [call["model"][0]["force_offload"] for call in calls] == [False, True]
def test_multitrack_prompt_enhance_to_project_has_chinese_localization():
module = _load_basic_module()
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
translation = node_defs["easy multitrackPromptEnhanceToProject"]
assert "媒体" in translation["description"]
assert "纯文本" not in json.dumps(translation, ensure_ascii=False)
assert set(translation["inputs"]) == {
"tracks_info",
"llama_model",
"model",
"inference_mode",
"force_offload",
"ratio",
"return_async",
"apikey",
"max_size",
"max_tokens",
"seed",
"enabled",
"api_account",
}
assert set(translation["inputs"]).isdisjoint(
{"images", "video", "audio", "files"}
)
assert all(
input_translation.get("tooltip")
for input_translation in translation["inputs"].values()
)
assert set(translation["outputs"]) == {"0", "1"}
assert module.MULTITRACK_PROMPT_PROJECT_APPLY_NODE_ID in node_defs
def test_multitrack_prompt_enhancer_returns_user_prompt_unchanged_when_disabled(
monkeypatch,
):
module = _load_basic_module()
monkeypatch.setattr(
module,
"PromptEnhancerClient",
lambda *_args, **_kwargs: pytest.fail("disabled enhancer must not create a client"),
)
output = module.MultiTrackPromptEnhancer.execute(
system_prompt=["System"],
user_prompt=["Keep this prompt unchanged"],
model=["invalid model configuration is intentionally ignored"],
enabled=[False],
)
assert output.values == ("Keep this prompt unchanged", "", "")
assert output.expand is None
def test_multitrack_prompt_enhance_to_project_resolves_deferred_reference_video(
monkeypatch,
):
module = _load_basic_module()
calls = []
source_video = _FakeVideo(
_VideoComponents(torch.zeros(4, 2, 2, 3), None, Fraction(2))
)
class PublicNodeOutput:
expand = None
def __getitem__(self, index):
return ("enhanced", "", "")[index]
monkeypatch.setattr(
module,
"_resolve_multitrack_video",
lambda content, video_input: source_video,
)
def fake_enhance(cls, **kwargs):
calls.append(kwargs)
return PublicNodeOutput()
monkeypatch.setattr(
module.MultiTrackPromptEnhancer,
"execute",
classmethod(fake_enhance),
)
tracks_info = {
"format": "MiniMax",
"media_loading": "deferred",
"frame_rate": 2,
"width": 2,
"height": 2,
"total_length": 4,
"tracks": [
{
"type": "task",
"segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {"task_mode": "ref", "user_prompt": "move"},
}],
},
{
"type": "video",
"segments": [{
"start_frame": 0,
"end_frame": 4,
"content": {
"media_type": "video",
"source_type": "input",
"file_path": "reference.mp4",
},
}],
},
],
}
output = module.MultiTrackPromptEnhanceToProject.execute(
tracks_info=tracks_info,
model={"model": module.MINIMAX_MODEL},
)
assert output.values[1] == ["enhanced"]
assert calls[0]["type"] == ["rv2v"]
assert len(calls[0]["video"]) == 1
assert calls[0]["video"][0] is not None
def test_multitrack_prompt_enhancer_passes_connected_llama_model_link_to_expansion():
module = _load_basic_module()
module.comfy_nodes.NODE_CLASS_MAPPINGS[module.LLAMA_CPP_INSTRUCT_NODE_ID] = object
output = module.MultiTrackPromptEnhancer.execute(
user_prompt=["Enhance this prompt"],
llama_model=[["llama-loader", 0]],
model=[{"model": module.LLAMACPP_MODEL}],
)
expanded = output.expand["local_llama_prompt_enhancer"]
assert expanded["inputs"]["llama_model"] == ["llama-loader", 0]
assert "check_lazy_status" not in module.MultiTrackPromptEnhancer.__dict__
def test_multitrack_prompt_enhancer_expands_local_llama_instruct_node():
module = _load_basic_module()
module.comfy_nodes.NODE_CLASS_MAPPINGS[module.LLAMA_CPP_INSTRUCT_NODE_ID] = object
image = torch.zeros(1, 2, 2, 3)
llama_model = {"model": "local.gguf"}
output = module.MultiTrackPromptEnhancer.execute(
system_prompt=["System"],
user_prompt=["Enhance this prompt"],
images=[image],
llama_model=[llama_model],
model=[{
"model": module.LLAMACPP_MODEL,
"inference_mode": "images",
"max_frames": 48,
"force_offload": True,
"max_size": 768,
}],
seed=[9],
)
image_bridge = output.expand["local_llama_images"]
expanded = output.expand["local_llama_prompt_enhancer"]
assert output.values == (["local_llama_prompt_start_switch", 0], "", "")
assert image_bridge == {
"class_type": module.LLAMA_CPP_IMAGE_LIST_BRIDGE_NODE_ID,
"inputs": {
"images": [image],
"inference_mode": "images",
"max_size": 768,
},
}
assert expanded["class_type"] == module.LLAMA_CPP_INSTRUCT_NODE_ID
assert expanded["inputs"]["llama_model"] == llama_model
assert expanded["inputs"]["preset_prompt"] == "Empty - Nothing"
assert expanded["inputs"]["custom_prompt"] == "Enhance this prompt"
assert expanded["inputs"]["system_prompt"] == "System"
assert expanded["inputs"]["inference_mode"] == "images"
assert expanded["inputs"]["max_frames"] == 24
assert expanded["inputs"]["max_size"] == 768
assert expanded["inputs"]["seed"] == 9
assert expanded["inputs"]["force_offload"] is True
assert expanded["inputs"]["save_states"] is False
assert expanded["inputs"]["images"] == ["local_llama_images", 0]
assert output.expand["local_llama_prompt_starts_with_text_fence"] == {
"class_type": module.STRING_COMPARE_NODE_ID,
"inputs": {
"string_a": ["local_llama_prompt_trim", 0],
"string_b": "```text",
"mode": "Starts With",
"case_sensitive": True,
},
}
assert output.expand["local_llama_prompt_ends_with_fence"] == {
"class_type": module.STRING_COMPARE_NODE_ID,
"inputs": {
"string_a": ["local_llama_prompt_trim", 0],
"string_b": "```",
"mode": "Ends With",
"case_sensitive": True,
},
}
assert output.expand["local_llama_prompt_remove_text_fence"]["class_type"] == (
module.STRING_REPLACE_NODE_ID
)
assert output.expand["local_llama_prompt_remove_closing_fence"]["class_type"] == (
module.STRING_REPLACE_NODE_ID
)
assert output.expand["local_llama_prompt_start_switch"] == {
"class_type": module.SWITCH_NODE_ID,
"inputs": {
"switch": ["local_llama_prompt_starts_with_text_fence", 0],
"on_false": ["local_llama_prompt_trim", 0],
"on_true": ["local_llama_prompt_end_switch", 0],
},
}
def test_multitrack_prompt_enhancer_unwraps_tuple_wrapped_llama_model():
module = _load_basic_module()
module.comfy_nodes.NODE_CLASS_MAPPINGS[module.LLAMA_CPP_INSTRUCT_NODE_ID] = object
llama_model = {"model": "local.gguf", "mmproj": "local-mmproj.gguf"}
output = module.MultiTrackPromptEnhancer.execute(
user_prompt=["Enhance this prompt"],
llama_model=[(llama_model,)],
model=[{"model": module.LLAMACPP_MODEL}],
)
expanded = output.expand["local_llama_prompt_enhancer"]
assert expanded["inputs"]["llama_model"] == llama_model
def test_multitrack_prompt_enhancer_reports_missing_local_llama_node():
module = _load_basic_module()
with pytest.raises(RuntimeError) as error:
module.MultiTrackPromptEnhancer.execute(
user_prompt=["Prompt"],
llama_model=[object()],
model=[{"model": module.LLAMACPP_MODEL}],
)
assert module.LLAMA_CPP_INSTRUCT_NODE_ID in str(error.value)
assert module.LLAMA_CPP_INSTALL_URL in str(error.value)
def test_multitrack_prompt_enhancer_executes_with_progress_and_h3_task_id(monkeypatch):
module = _load_basic_module()
calls = []
class FakeClient:
def __init__(self, model, api_key):
calls.append((model, api_key))
def enhance(self, **kwargs):
calls.append(kwargs)
for _ in range(20):
kwargs["poll_callback"]("running")
return types.SimpleNamespace(
prompt="Enhanced prompt",
task_id="task-789",
file_ids="101,202,303",
)
monkeypatch.setattr(module, "PromptEnhancerClient", FakeClient)
monkeypatch.setattr(
module,
"image_tensor_data_uris",
lambda values, **kwargs: ["image"],
)
monkeypatch.setattr(
module,
"video_data_uris",
lambda values, **kwargs: calls.append(("video_data", kwargs)) or ["video"],
)
monkeypatch.setattr(module, "audio_data_uris", lambda values: ["audio"])
monkeypatch.setattr(module, "minimax_length_to_seconds", lambda length: 6)
output = module.MultiTrackPromptEnhancer.execute(
system_prompt=["System"],
user_prompt=["User"],
type=["r2v"],
length=[124],
images=[torch.zeros(1, 2, 2, 3)],
video=[object()],
audio=[{"waveform": torch.zeros(1, 1, 16), "sample_rate": 16}],
files=["reference.pdf", None],
model=[{
"model": module.MINIMAX_MODEL,
"ratio": "16:9",
"return_async": True,
"apikey": "secret",
}],
seed=[9],
)
assert output.values == ("Enhanced prompt", "task-789", "101,202,303")
assert calls[0] == (module.MINIMAX_MODEL, "secret")
assert ("video_data", {"max_duration": 15}) in calls
request_call = calls[-1]
assert request_call["duration"] == 6
assert request_call["poll_interval"] == 5.0
assert request_call["file_count"] == 1
assert request_call["request_logger"] is module.log_node_info
progress_updates = _ProgressBar.instances[-1].updates
assert progress_updates[:3] == [0, 10, 20]
assert max(progress_updates[:-1]) == 95
assert progress_updates[-1] == 100
def test_multitrack_prompt_enhancer_prepares_third_party_video_as_24_frames(
monkeypatch,
):
module = _load_basic_module()
calls = []
class FakeClient:
def __init__(self, model, api_key):
calls.append((model, api_key))
def enhance(self, **kwargs):
calls.append(kwargs)
return types.SimpleNamespace(prompt="Enhanced", task_id="ignored", file_ids="")
monkeypatch.setattr(module, "PromptEnhancerClient", FakeClient)
monkeypatch.setattr(
module,
"image_tensor_data_uris",
lambda values, **kwargs: calls.append(("images", kwargs)) or ["image"],
)
monkeypatch.setattr(
module,
"prompt_enhancer_video_inputs",
lambda model, values: calls.append(("video_inputs", model)) or ["frame"],
)
monkeypatch.setattr(
module,
"video_data_uris",
lambda values: pytest.fail("third-party providers must not upload a video file"),
)
monkeypatch.setattr(
module,
"audio_data_uris",
lambda values: pytest.fail("third-party providers must not receive audio"),
)
output = module.MultiTrackPromptEnhancer.execute(
system_prompt=["System"],
user_prompt=["User"],
type=["rv2v"],
length=[124],
images=[torch.zeros(1, 2, 2, 3)],
video=[object()],
audio=[{"waveform": torch.zeros(1, 1, 16), "sample_rate": 16}],
model=[
{
"model": "third-party",
"apikey": "secret",
"max_tokens": 1234,
}
],
seed=[9],
)
assert output.values == ("Enhanced", "", "")
assert ("images", {"max_pixels": 2_000_000}) in calls
assert ("video_inputs", "third-party") in calls
assert calls[0] == ("third-party", "secret")
assert calls[-1]["max_tokens"] == 1234
assert calls[-1]["video_urls"] == ["frame"]
assert calls[-1]["audio_urls"] == []
def test_multitrack_prompt_enhancer_keeps_video_for_native_video_url_provider(
monkeypatch,
):
module = _load_basic_module()
calls = []
class FakeClient:
def __init__(self, model, api_key):
pass
def enhance(self, **kwargs):
calls.append(kwargs)
return types.SimpleNamespace(prompt="Enhanced", task_id="", file_ids="")
monkeypatch.setattr(module, "PromptEnhancerClient", FakeClient)
monkeypatch.setattr(
module,
"prompt_enhancer_video_inputs",
lambda model, values: ["video"],
)
monkeypatch.setattr(
module,
"image_tensor_data_uris",
lambda values, **kwargs: [],
)
monkeypatch.setattr(module, "video_data_uris", lambda values: ["video"])
output = module.MultiTrackPromptEnhancer.execute(
system_prompt=["System"],
user_prompt=["User"],
type=["v2v"],
length=[124],
video=[object()],
model=[{
"model": "native-video-provider",
"apikey": "secret",
"max_tokens": 4096,
}],
seed=[9],
)
assert output.values == ("Enhanced", "", "")
assert calls[-1]["video_urls"] == ["video"]
def test_multitrack_prompt_enhancer_has_complete_chinese_localization():
module = _load_basic_module()
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
translation = node_defs["easy multiTrackPromptEnhancer"]
assert set(translation["inputs"]) == {
"system_prompt",
"user_prompt",
"type",
"length",
"images",
"video",
"audio",
"files",
"llama_model",
"model",
"inference_mode",
"force_offload",
"ratio",
"return_async",
"apikey",
"max_size",
"max_tokens",
"seed",
"enabled",
"api_account",
}
assert all("tooltip" in input_translation for input_translation in translation["inputs"].values())
assert module.LLAMA_CPP_INSTALL_URL in translation["inputs"]["llama_model"]["tooltip"]
assert translation["outputs"] == {
"0": {"name": "提示词", "tooltip": "增强后的视频提示词。"},
"1": {
"name": "任务 ID",
"tooltip": "MiniMax 任务 ID;其他厂商返回空字符串。",
},
"2": {
"name": "文件 ID",
"tooltip": (
"MiniMax 已上传图像、视频和音频的文件 ID,多个 ID 使用英文逗号分隔;"
"没有上传媒体时返回空字符串。"
),
},
}
bridge_translation = node_defs[
module.LLAMA_CPP_IMAGE_LIST_BRIDGE_NODE_ID
]
bridge_schema = module.MultiTrackPromptEnhancerImageListBridge.define_schema()
assert bridge_schema.is_dev_only is True
assert bridge_translation["inputs"]["images"]["tooltip"]
assert bridge_translation["outputs"]["0"]["tooltip"]
assert all(port.kwargs.get("tooltip") for port in bridge_schema.inputs)
assert all(port.kwargs.get("tooltip") for port in bridge_schema.outputs)
def test_multitrack_prompt_enhancer_image_bridge_limits_single_image_long_edge():
module = _load_basic_module()
image = torch.zeros(1, 1200, 800, 3)
output = module.MultiTrackPromptEnhancerImageListBridge.execute(
images=[image],
max_size=512,
)
assert len(output.values[0]) == 1
assert output.values[0][0].shape == (1, 512, 341, 3)
def test_multitrack_prompt_enhancer_image_bridge_preserves_small_batches():
module = _load_basic_module()
image_batch = torch.zeros(2, 64, 96, 3)
output = module.MultiTrackPromptEnhancerImageListBridge.execute(
images=[image_batch],
max_size=512,
)
assert len(output.values[0]) == 1
assert output.values[0][0] is image_batch
def test_multitrack_prompt_enhancer_image_bridge_preserves_images_mode_references():
module = _load_basic_module()
images = [torch.zeros(1, 512, 512, 3), torch.ones(1, 512, 512, 3)]
output = module.MultiTrackPromptEnhancerImageListBridge.execute(
images=images,
max_size=512,
inference_mode="images",
)
assert len(output.values[0]) == 2
assert output.values[0][0].shape == (1, 362, 362, 3)
assert output.values[0][1].shape == (1, 362, 362, 3)
def test_multitrack_prompt_enhancer_image_bridge_caps_all_inference_modes():
module = _load_basic_module()
image = torch.zeros(1, 1024, 1024, 3)
for inference_mode in ("one by one", "images", "video"):
output = module.MultiTrackPromptEnhancerImageListBridge.execute(
images=[image],
max_size=8192,
inference_mode=inference_mode,
)
assert output.values[0][0].shape == (1, 768, 768, 3)
def test_match_line_returns_first_containing_line_index():
module = _load_related_node_module("common")
schema = module.MatchLine.define_schema()
result = module.MatchLine.execute("alpha\r\nbeta target\r\ntarget again", "target")
assert [input_.name for input_ in schema.inputs] == ["text", "match"]
assert schema.inputs[0].kwargs["multiline"] is True
assert "multiline" not in schema.inputs[1].kwargs
assert result.values == (1,)
def test_match_line_returns_minus_one_for_empty_or_missing_match():
module = _load_related_node_module("common")
assert module.MatchLine.execute("alpha\nbeta", "missing").values == (-1,)
assert module.MatchLine.execute("alpha\nbeta", "").values == (-1,)
def test_match_line_has_chinese_localization():
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
translation = node_defs["easy matchLine"]
assert translation["display_name"] == "匹配行"
assert set(translation["inputs"]) == {"text", "match"}
assert translation["outputs"] == {"0": {"name": "行索引"}}
def test_workflow_format_gate_skips_input_for_workflow_metadata():
module = _load_related_node_module("common")
module.APIWorkflowGate.hidden = types.SimpleNamespace(
extra_pnginfo={"workflow": {"nodes": []}},
)
assert module.APIWorkflowGate.check_lazy_status() == []
assert module.APIWorkflowGate.execute("ignored").values == (None, [])
def test_workflow_format_gate_requests_input_for_api_prompt():
module = _load_related_node_module("common")
module.APIWorkflowGate.hidden = types.SimpleNamespace(extra_pnginfo={})
assert module.APIWorkflowGate.check_lazy_status() == ["value"]
assert module.APIWorkflowGate.check_lazy_status("payload") == []
assert module.APIWorkflowGate.execute("payload").values == ("payload", [])
def test_workflow_format_gate_passes_list_values_through_list_output():
module = _load_related_node_module("common")
module.APIWorkflowGate.hidden = types.SimpleNamespace(extra_pnginfo={})
value = ["a", "b"]
assert module.APIWorkflowGate.execute(value).values == (None, value)
def test_workflow_format_gate_schema_has_list_output():
module = _load_related_node_module("common")
schema = module.APIWorkflowGate.define_schema()
assert schema.outputs[0].name == "VALUE"
assert schema.outputs[1].name == "VALUES"
assert schema.outputs[1].kwargs == {"is_output_list": True}
def test_workflow_format_gate_detects_nested_workflow_metadata():
module = _load_related_node_module("common")
assert module._is_workflow_format({"extra": [{"workflow": {"version": 1}}]}) is True
assert module._is_workflow_format({"prompt": {"1": {"class_type": "Node"}}}) is False
def test_workflow_format_gate_has_chinese_localization():
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
translation = node_defs["easy apiWorkflowGate"]
assert translation["display_name"] == "API工作流阀门"
assert set(translation["inputs"]) == {"value"}
assert translation["outputs"] == {"0": {"name": "输出"}, "1": {"name": "列表输出"}}
def test_split_images_splits_a_single_batched_tensor_into_single_images():
module = _load_image_module()
batch = torch.arange(3 * 2 * 2 * 3).reshape(3, 2, 2, 3)
result = module.SplitImages.execute([batch])
assert len(result.values) == 10
assert all(image.shape == (1, 2, 2, 3) for image in result.values[:3])
assert torch.equal(result.values[0], batch[0:1])
assert torch.equal(result.values[2], batch[2:3])
assert result.values[3:] == (None,) * 7
def test_split_images_uses_multiple_list_items_without_batch_splitting():
module = _load_image_module()
images = [torch.full((1, 2, 2, 3), value) for value in (1, 2)]
schema = module.SplitImages.define_schema()
result = module.SplitImages.execute(images)
assert schema.is_input_list is True
assert len(schema.outputs) == 10
assert torch.equal(result.values[0], images[0])
assert torch.equal(result.values[1], images[1])
assert result.values[2:] == (None,) * 8
def test_split_images_has_chinese_localization():
locale_path = Path(__file__).parents[1] / "locales" / "zh" / "nodeDefs.json"
node_defs = json.loads(locale_path.read_text(encoding="utf-8"))
translation = node_defs["easy splitImages"]
assert translation["display_name"] == "图像拆分V2"
assert set(translation["inputs"]) == {"images"}
assert set(translation["outputs"]) == {str(index) for index in range(10)}
def test_split_audios_expands_a_list_into_ten_independent_outputs():
module = _load_related_node_module("audio")
audios = [
{"waveform": torch.full((1, 1, 4), value), "sample_rate": 16000}
for value in (1.0, 2.0)
]
schema = module.SplitAudios.define_schema()
result = module.SplitAudios.execute(audios)
assert schema.is_input_list is True
assert len(schema.outputs) == 10
assert result.values[:2] == tuple(audios)
assert result.values[2:] == (None,) * 8
def test_split_audios_rejects_an_empty_list():
module = _load_related_node_module("audio")
with pytest.raises(ValueError, match="at least one audio"):
module.SplitAudios.execute([])
def test_multitrack_editor_preserves_audio_only_custom_resolution():
module = _load_basic_module()
result = module.MultiTrackEditor.execute(
{"resolution": "width x height (custom)", "width": 32, "height": 32},
"MiniMax",
{"total_length": 120, "frame_rate": 24, "tracks": []},
)
assert (result.values[0]["width"], result.values[0]["height"]) == (32, 32)
def test_minimax_prompt_override_renumbers_and_skips_unreferenced_slots():
module = _load_basic_module()
images = [torch.full((1, 2, 2, 3), float(value)) for value in (1, 2, 3)]
audios = [
{"waveform": torch.full((1, 1, 4), float(value)), "sample_rate": 2}
for value in (1, 2, 3)
]
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["@图片2 @音频2 first", "@图片3 @音频3 second", "third"],
"duration": "2,2,2",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
result = module.MultiTrackEditor.execute(
{"resolution": "1344 x 768 (16:9)"},
"MiniMax",
{"total_length": 1, "frame_rate": 2, "tracks": []},
prompt_override=payload,
image=images,
audio=audios,
)
tracks_info, image_out, audio_out, _video_out = result.values
task_track = tracks_info["tracks"][0]
assert [segment["content"]["text"] for segment in task_track["segments"]] == [
"<Picture 1> <Audio 1> first",
"<Picture 2> <Audio 2> second",
"third",
]
assert [
image_info["media_index"]
for segment in task_track["segments"]
for image_info in segment["content"]["images"]
] == [0, 1]
assert len(image_out) == 2
assert torch.equal(image_out[0], images[1])
assert torch.equal(image_out[1], images[2])
audio_tracks = [
track for track in tracks_info["tracks"] if track["type"] == "audio"
]
assert [track["name"] for track in audio_tracks] == ["Audio 2", "Audio 3"]
assert len(audio_out) == 4
assert audio_tracks[0]["segments"][0]["content"]["shared_reference"] is True
assert audio_tracks[1]["segments"][0]["content"]["shared_reference"] is True
assert audio_tracks[0]["segments"][0]["content"]["shared_media_index"] == 1
assert audio_tracks[1]["segments"][0]["content"]["shared_media_index"] == 3
assert audio_out[1]["waveform"].flatten().tolist() == [2.0, 2.0, 2.0, 2.0]
assert audio_out[3]["waveform"].flatten().tolist() == [3.0, 3.0, 3.0, 3.0]
assert audio_out[0]["waveform"].flatten().tolist() == [2.0, 2.0, 2.0, 2.0]
assert audio_out[2]["waveform"].flatten().tolist() == [
0.0, 0.0, 0.0, 0.0, 3.0, 3.0, 3.0, 3.0,
]
def test_minimax_prompt_override_builds_locked_media_tracks():
module = _load_basic_module()
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["@视频1 @音频1 first", "@视频2 @音频2 second"],
"duration": "10,10",
"video_track_lock": 2,
"audio_track_lock": 1,
},
ensure_ascii=False,
)
data = module.build_minimax_multitrack_data_from_prompt_override(
{"total_length": 1, "frame_rate": 24, "tracks": []},
payload,
)
video_tracks = [track for track in data["tracks"] if track["type"] == "video"]
audio_tracks = [track for track in data["tracks"] if track["type"] == "audio"]
assert [track["name"] for track in video_tracks] == ["Video 1", "Video 2"]
assert video_tracks[0].get("audio_locked") is not True
assert video_tracks[1]["audio_locked"] is True
assert [track["name"] for track in audio_tracks] == ["Audio 1", "Audio 2"]
assert audio_tracks[0]["audio_locked"] is True
assert audio_tracks[1].get("audio_locked") is not True
def test_minimax_prompt_override_applies_generation_and_continuity_modes():
module = _load_basic_module()
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["first", "second", "third"],
"duration": "10,5,10",
"generation_type": "r2v,i2v,l2v",
"continuity_mode": "shot,context,context",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
data = module.build_minimax_multitrack_data_from_prompt_override(
{"total_length": 1, "frame_rate": 24, "tracks": []},
payload,
)
task_segments = data["tracks"][0]["segments"]
assert [
(
segment["content"]["task_type"],
segment["content"]["task_mode"],
segment["content"]["continuity_mode"],
)
for segment in task_segments
] == [
("r2v", "ref", "shot"),
("i2v", "default", "context"),
("l2v", "l2v", "context"),
]
def test_minimax_prompt_override_duration_values_are_bounded_to_2_through_15():
module = _load_basic_module()
def build_with_duration(duration):
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["single prompt"],
"duration": duration,
"generation_type": "r2v",
"continuity_mode": "shot",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
return module.build_minimax_multitrack_data_from_prompt_override(
{"total_length": 1, "frame_rate": 24, "tracks": []},
payload,
)
assert build_with_duration("2")["tracks"][0]["segments"][0]["end_frame"] == 48
assert build_with_duration("15")["tracks"][0]["segments"][0]["end_frame"] == 360
for invalid in ("1", "16", "10,1", "10,16"):
with pytest.raises(ValueError, match="between 2 and 15"):
build_with_duration(invalid)
def test_minimax_prompt_override_applies_system_prompt_when_present():
module = _load_basic_module()
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["first", "second"],
"duration": "10,5",
"generation_type": "r2v,i2v",
"continuity_mode": "shot,context",
"system_prompt": "custom system prompt",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
data = module.build_minimax_multitrack_data_from_prompt_override(
{"total_length": 1, "frame_rate": 24, "tracks": []},
payload,
)
task_segments = data["tracks"][0]["segments"]
assert [
segment["content"]["system_prompt"]
for segment in task_segments
] == ["custom system prompt", "custom system prompt"]
def test_minimax_prompt_override_video_reference_keeps_source_duration():
module = _load_basic_module()
source_video = _FakeVideo(
_VideoComponents(
torch.zeros(4, 32, 32, 3),
None,
Fraction(1),
)
)
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["@视频1 first"],
"duration": "2",
"generation_type": "r2v",
"continuity_mode": "shot",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
result = module.MultiTrackEditor.execute(
{"resolution": "32 x 32 (1:1)"},
"MiniMax",
{"total_length": 1, "frame_rate": 1, "tracks": []},
prompt_override=payload,
video=[source_video],
)
tracks_info, _images, _audio, videos = result.values
video_track = next(
track for track in tracks_info["tracks"] if track["type"] == "video"
)
shared_content = video_track["segments"][0]["content"]
assert shared_content["shared_reference"] is True
assert shared_content["shared_media_index"] == 1
task_result = module.MultiTrackTaskOutput.execute(
tracks_info,
video=videos,
task_index=0,
)
assert task_result.values[6][0] is videos[1]
assert task_result.values[6][0].get_components().images.shape[0] == 4
def test_build_shared_reference_video_caps_source_to_15_seconds():
module = _load_basic_module()
video = _FakeVideo(
_VideoComponents(
torch.zeros(20, 2, 2, 3),
None,
Fraction(1),
)
)
result = module._build_shared_reference_video(video)
assert result is video
assert video.trim_calls == [(0.0, 15.0, False)]
def test_minimax_prompt_override_media_types_only_includes_referenced_media():
module = _load_basic_module()
payload = json.dumps(
{
"type": "minimax_prompt_override",
"version": 1,
"prompts": ["@图片2 first", "@视频1 second"],
"duration": "10,10",
"video_track_lock": 0,
"audio_track_lock": 0,
},
ensure_ascii=False,
)
assert module.minimax_prompt_override_media_types(payload) == {"image", "video"}
assert module.is_minimax_prompt_override(payload) is True
assert module.is_minimax_prompt_override("@图片1 legacy override") is False
def test_multi_images_loader_resizes_ordered_image_list(monkeypatch):
module = _load_basic_module()
sources = {
"first.png": torch.zeros(1, 40, 80, 3),
"second.png": torch.ones(1, 100, 50, 3),
}
monkeypatch.setattr(
module,
"_resolve_timeline_image_item",
lambda item, unused: sources[item["file_path"]],
)
calls = []
def resize(image, width, height, method):
calls.append((width, height, method))
return torch.full((1, height, width, 3), float(len(calls)))
monkeypatch.setattr(module, "resize_image", resize)
image_data = {"images": [
{"source_type": "input", "file_path": "first.png"},
{"source_type": "input", "file_path": "second.png"},
]}
result = module.MultiImagesLoader.execute(
{"resolution": "width x height (longest)", "resize_to_pixel": 120, "resize_method": "crop"},
json.dumps(image_data),
)
assert calls == [(120, 60, "crop"), (60, 120, "crop")]
assert len(result[0]) == 2
assert result[0][0].shape == (1, 60, 120, 3)
assert result[0][1].shape == (1, 120, 60, 3)
def test_multi_images_loader_rejects_more_than_25_images():
module = _load_basic_module()
with pytest.raises(ValueError, match="at most 25"):
module.MultiImagesLoader.execute("width x height (auto)", {
"images": [{"source_type": "input", "file_path": "x.png"}] * 26,
})