Console, errors, tooltips, descriptions and display names no longer reveal the pack heritage or the underlying model family: - [EasyMedia]/[Easy Media] console prefixes -> [Moxie] (11 files) - New console privacy filter (utils/log.py, installed at pack import): rewrites MiniMax variants in every displayed logging record, including ComfyUI core lines like 'Model MiniMaxH3 prepared for dynamic VRAM loading', %-args and matching tracebacks. api.minimaxi.com is protected so cloud-API errors stay accurate. - Node display names rebranded (node_ids untouched, workflows safe): 'MiniMax H3 To Video' -> 'Moxie To Video', etc. - ~40 error/tooltip/description strings reworded to Moxie terms. - CLIP error no longer names CLIPType.MINIMAX; test match updated. Unchanged on purpose (breaking if touched): node_ids/class names, combo value 'MiniMax' (workflow payloads), internal dict keys (minimax_keyframes/minimax_refs), llm_api cloud provider strings, class-name logic checks, source-only docstrings.
445 lines
16 KiB
Python
445 lines
16 KiB
Python
"""Schedule-matched Drift-Control AV for Easy-Media H3 context_swap.
|
|
|
|
Code based on ethanfel/ComfyUI-MiniMaxH3-Contex-Loop - Drift-Control AV.
|
|
Modified integration: 2026-09-07.
|
|
|
|
Unlike the previous Easy-Media context_swap implementation, this module never adds random noise to the saved predecessor latent.
|
|
It copies the predecessor AV tail into a disposable target prefix, keeps that prefix clean, and changes only the live denoise mask during sampling according to the active sigma schedule.
|
|
|
|
---
|
|
Motion stability is better at the cost of slightly reduced clarity vs. the noise-addition scheme (in my testing). The predecessor latent is copied verbatim — no denoise-rebuild cycle means no motion detail is lost — but the model cannot further enhance the copied portion. This trade-off is preferred for subject replacement and motion transfer where preserving the source motion fidelity is the primary goal over visual fidelity improvement.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from typing import Any, Iterable
|
|
|
|
import torch
|
|
|
|
from .core import (
|
|
_audio_tail_from_latent,
|
|
_merge_noise_mask,
|
|
_noise_mask_streams,
|
|
_official_nested_tensor,
|
|
_streams_from_latent,
|
|
_video_tail_from_latent,
|
|
)
|
|
|
|
|
|
DRIFT_CONTROL_TAPER_STEPS = 4
|
|
DRIFT_CONTROL_AUDIO_RELEASE_STEPS = 8
|
|
_WRAPPER_KEY = "easy_media_h3_context_swap_drift_control_av"
|
|
|
|
|
|
def _schedule_values(sigmas: Any) -> tuple[float, ...]:
|
|
"""Return finite, non-negative scheduler values in descending order."""
|
|
if torch.is_tensor(sigmas):
|
|
values: Iterable[Any] = sigmas.detach().float().reshape(-1).cpu()
|
|
else:
|
|
values = sigmas or ()
|
|
normalized: list[float] = []
|
|
for value in values:
|
|
number = float(value)
|
|
if math.isfinite(number) and number >= 0.0:
|
|
normalized.append(number)
|
|
return tuple(sorted(set(normalized), reverse=True))
|
|
|
|
|
|
def drift_control_step_count(sigmas: Any) -> int:
|
|
return max(0, len(_schedule_values(sigmas)) - 1)
|
|
|
|
|
|
def next_schedule_sigma(current_sigma: float, sigmas: Any) -> float:
|
|
current = float(current_sigma)
|
|
if not math.isfinite(current) or current <= 0.0:
|
|
return 0.0
|
|
tolerance = max(1e-7, abs(current) * 1e-6)
|
|
for candidate in _schedule_values(sigmas):
|
|
if candidate < current - tolerance:
|
|
return candidate
|
|
return 0.0
|
|
|
|
|
|
def matched_noise_ratio(current_sigma: float, sigmas: Any) -> float:
|
|
current = float(current_sigma)
|
|
if not math.isfinite(current) or current <= 0.0:
|
|
return 0.0
|
|
return max(
|
|
0.0,
|
|
min(1.0, next_schedule_sigma(current, sigmas) / current),
|
|
)
|
|
|
|
|
|
def temporal_prefix_weights(
|
|
prefix_steps: int,
|
|
taper_steps: int = DRIFT_CONTROL_TAPER_STEPS,
|
|
) -> tuple[float, ...]:
|
|
"""Full release at the disposable side, tapered to zero at the seam side."""
|
|
count = int(prefix_steps)
|
|
taper = min(int(taper_steps), count)
|
|
if count < 1:
|
|
raise ValueError("Moxie Drift-Control AV prefix steps must be positive")
|
|
if taper < 1:
|
|
raise ValueError("Moxie Drift-Control AV taper steps must be positive")
|
|
weights = [1.0] * (count - taper)
|
|
weights.extend(
|
|
float(taper - offset - 1) / float(taper)
|
|
for offset in range(taper)
|
|
)
|
|
return tuple(weights)
|
|
|
|
|
|
def apply_dynamic_prefix_mask(
|
|
packed_mask: torch.Tensor,
|
|
video_shape: tuple[int, ...],
|
|
ratio: float,
|
|
prefix_steps: int,
|
|
taper_steps: int = DRIFT_CONTROL_TAPER_STEPS,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Apply the live schedule-matched mask to the copied video prefix."""
|
|
if not torch.is_tensor(packed_mask) or packed_mask.ndim != 3:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV expects packed denoise mask [B,1,N]"
|
|
)
|
|
|
|
shape = tuple(int(value) for value in video_shape)
|
|
if len(shape) != 5 or shape[0] != int(packed_mask.shape[0]):
|
|
raise ValueError(
|
|
f"Moxie Drift-Control AV expects video latent [B,C,T,H,W], got {shape}"
|
|
)
|
|
prefix_steps = int(prefix_steps)
|
|
if prefix_steps < 1 or prefix_steps >= shape[2]:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV prefix must fit before generated video"
|
|
)
|
|
|
|
video_elements = math.prod(shape[1:])
|
|
if int(packed_mask.shape[-1]) < video_elements:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV packed mask is shorter than its video stream"
|
|
)
|
|
|
|
output = packed_mask.clone()
|
|
video_mask = output[..., :video_elements].reshape(shape)
|
|
weights = torch.tensor(
|
|
temporal_prefix_weights(prefix_steps, taper_steps),
|
|
device=video_mask.device,
|
|
dtype=video_mask.dtype,
|
|
).mul_(float(max(0.0, min(1.0, ratio))))
|
|
video_mask[:, :, :prefix_steps] = weights.view(
|
|
1, 1, prefix_steps, 1, 1
|
|
)
|
|
|
|
# Match ComfyUI H3 token-mask quantization so sampler blending and the
|
|
# model's per-row timestep labels describe the same live state.
|
|
h3_video_mask = torch.ceil(video_mask[:, :1].float() * 256.0) / 256.0
|
|
return output, h3_video_mask
|
|
|
|
|
|
def _soft_audio_release(
|
|
mask: torch.Tensor,
|
|
prefix_steps: int,
|
|
release_steps: int = DRIFT_CONTROL_AUDIO_RELEASE_STEPS,
|
|
) -> list[float]:
|
|
"""Keep copied audio locked, then half-cosine release at the seam."""
|
|
prefix_steps = max(0, min(int(prefix_steps), int(mask.shape[-1])))
|
|
if prefix_steps == 0:
|
|
return []
|
|
mask[..., :prefix_steps] = 0.0
|
|
release = min(max(0, int(release_steps)), prefix_steps)
|
|
if release == 0:
|
|
return []
|
|
indices = torch.arange(
|
|
1,
|
|
release + 1,
|
|
dtype=mask.dtype,
|
|
device=mask.device,
|
|
)
|
|
values = 0.5 - 0.5 * torch.cos(torch.pi * indices / float(release))
|
|
mask[..., prefix_steps - release : prefix_steps] = values.view(
|
|
*([1] * (mask.ndim - 1)), release
|
|
)
|
|
return [round(float(value), 4) for value in values.detach().cpu()]
|
|
|
|
|
|
def prepare_context_swap_latent(
|
|
target_latent: dict[str, Any],
|
|
context_latent: dict[str, Any],
|
|
context_frames: int,
|
|
*,
|
|
continue_audio: bool,
|
|
freeze_audio: bool,
|
|
audio_release_steps: int = DRIFT_CONTROL_AUDIO_RELEASE_STEPS,
|
|
) -> tuple[dict[str, Any], int, int]:
|
|
"""Copy a clean predecessor prefix without modifying the predecessor latent.
|
|
|
|
First-pass context_swap uses ``continue_audio=True`` and gets the same
|
|
half-cosine soft AV release used by TimelineDirector.
|
|
|
|
Hi-res second pass uses ``continue_audio=False, freeze_audio=True`` to keep
|
|
Easy-Media's existing policy of freezing the current audio latent while the
|
|
video prefix is being resampled.
|
|
"""
|
|
if continue_audio and freeze_audio:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV cannot continue and freeze audio simultaneously"
|
|
)
|
|
|
|
existing_video_mask, existing_audio_mask = _noise_mask_streams(target_latent)
|
|
target_parts = _streams_from_latent(target_latent)
|
|
if len(target_parts) < 2:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV target latent has no audio stream"
|
|
)
|
|
target_video, target_audio = target_parts[:2]
|
|
if target_video.ndim == 4:
|
|
target_video = target_video.unsqueeze(0)
|
|
if target_audio.ndim == 3:
|
|
target_audio = target_audio.unsqueeze(0)
|
|
if (
|
|
target_video.ndim != 5
|
|
or target_audio.ndim != 4
|
|
or int(target_audio.shape[2]) != 2
|
|
):
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV expects matching video/audio latent streams"
|
|
)
|
|
|
|
blocks, _, covered = _video_tail_from_latent(
|
|
context_latent,
|
|
int(context_frames),
|
|
)
|
|
copied_video = torch.cat(blocks, dim=2)
|
|
video_steps = int(copied_video.shape[2])
|
|
if covered != int(context_frames):
|
|
raise RuntimeError("Moxie Drift-Control AV video context span changed")
|
|
if video_steps < 1 or video_steps >= int(target_video.shape[2]):
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV copied video prefix must be shorter than target"
|
|
)
|
|
if (
|
|
copied_video.shape[0] != target_video.shape[0]
|
|
or copied_video.shape[1] != target_video.shape[1]
|
|
or copied_video.shape[3:] != target_video.shape[3:]
|
|
):
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV context and target video latent shapes "
|
|
f"differ: original context {copied_video.shape[4] * 16}x{copied_video.shape[3] * 16} pixels, "
|
|
f"current target {target_video.shape[4] * 16}x{target_video.shape[3] * 16} pixels "
|
|
"(width x height); "
|
|
f"copied context prefix [B,C,T,H,W]={tuple(copied_video.shape)}, "
|
|
f"target [B,C,T,H,W]={tuple(target_video.shape)}"
|
|
)
|
|
|
|
output_video = target_video.clone()
|
|
output_video[:, :, :video_steps] = copied_video.to(
|
|
device=output_video.device,
|
|
dtype=output_video.dtype,
|
|
)
|
|
video_mask = torch.ones_like(output_video[:, :1], dtype=torch.float32)
|
|
video_mask[:, :, :video_steps] = 0.0
|
|
video_mask = _merge_noise_mask(
|
|
video_mask,
|
|
existing_video_mask,
|
|
"video",
|
|
)
|
|
|
|
output_audio = target_audio.clone()
|
|
audio_mask = torch.ones_like(output_audio[:, :1], dtype=torch.float32)
|
|
audio_steps = 0
|
|
|
|
if continue_audio:
|
|
copied_audio, audio_steps, _overhang = _audio_tail_from_latent(
|
|
context_latent,
|
|
int(context_frames),
|
|
)
|
|
if audio_steps < 1 or audio_steps >= int(target_audio.shape[-1]):
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV copied audio prefix must be shorter than target"
|
|
)
|
|
if copied_audio.shape[:3] != target_audio.shape[:3]:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV context and target audio latent shapes differ"
|
|
)
|
|
output_audio[..., :audio_steps] = copied_audio.to(
|
|
device=output_audio.device,
|
|
dtype=output_audio.dtype,
|
|
)
|
|
_soft_audio_release(
|
|
audio_mask,
|
|
audio_steps,
|
|
audio_release_steps,
|
|
)
|
|
audio_mask = _merge_noise_mask(
|
|
audio_mask,
|
|
existing_audio_mask,
|
|
"audio",
|
|
)
|
|
elif freeze_audio:
|
|
audio_mask.zero_()
|
|
elif existing_audio_mask is not None:
|
|
audio_mask = _merge_noise_mask(
|
|
audio_mask,
|
|
existing_audio_mask,
|
|
"audio",
|
|
)
|
|
|
|
output = target_latent.copy()
|
|
output["samples"] = _official_nested_tensor((output_video, output_audio))
|
|
output["noise_mask"] = _official_nested_tensor((video_mask, audio_mask))
|
|
return output, int(covered), video_steps
|
|
|
|
|
|
class _DriftControlMaskState:
|
|
def __init__(
|
|
self,
|
|
video_shape: tuple[int, ...],
|
|
sigmas: Any,
|
|
prefix_steps: int,
|
|
):
|
|
self.video_shape = tuple(int(value) for value in video_shape)
|
|
self.sigmas = _schedule_values(sigmas)
|
|
self.prefix_steps = int(prefix_steps)
|
|
self.current_video_mask: torch.Tensor | None = None
|
|
|
|
def denoise_mask_function(
|
|
self,
|
|
sigma: torch.Tensor,
|
|
denoise_mask: torch.Tensor,
|
|
extra_options: dict[str, Any] | None = None,
|
|
) -> torch.Tensor:
|
|
current = float(torch.as_tensor(sigma).detach().float().reshape(-1)[0])
|
|
schedule = self.sigmas or _schedule_values(
|
|
(extra_options or {}).get("sigmas", ())
|
|
)
|
|
active_video_shape = self.video_shape
|
|
active_model = (extra_options or {}).get("model")
|
|
active_inner_model = getattr(active_model, "inner_model", None)
|
|
active_shapes = getattr(active_inner_model, "latent_shapes", None)
|
|
if (
|
|
isinstance(active_shapes, (tuple, list))
|
|
and active_shapes
|
|
and len(active_shapes[0]) == 5
|
|
):
|
|
active_video_shape = tuple(int(value) for value in active_shapes[0])
|
|
output, video_mask = apply_dynamic_prefix_mask(
|
|
denoise_mask,
|
|
active_video_shape,
|
|
matched_noise_ratio(current, schedule),
|
|
prefix_steps=self.prefix_steps,
|
|
taper_steps=min(DRIFT_CONTROL_TAPER_STEPS, self.prefix_steps),
|
|
)
|
|
self.current_video_mask = video_mask
|
|
return output
|
|
|
|
def apply_model_wrapper(self, executor, *args, **kwargs):
|
|
if self.current_video_mask is not None:
|
|
kwargs["denoise_mask"] = self.current_video_mask
|
|
return executor(*args, **kwargs)
|
|
|
|
|
|
def install_drift_control_av_model(
|
|
model: Any,
|
|
latent: dict[str, Any],
|
|
sigmas: Any,
|
|
prefix_steps: int,
|
|
):
|
|
"""Clone the H3 model and install sampler/model dynamic-mask hooks."""
|
|
if drift_control_step_count(sigmas) < 1:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV requires at least one sigma sampling step"
|
|
)
|
|
if model is None or not callable(getattr(model, "clone", None)):
|
|
raise ValueError("Moxie Drift-Control AV requires a ComfyUI MODEL input")
|
|
|
|
inner = getattr(model, "model", None)
|
|
model_type = str(getattr(getattr(inner, "model_type", None), "name", ""))
|
|
if model_type != "FLOW_AV" and inner.__class__.__name__ != "MiniMaxH3":
|
|
# Some patcher wrappers expose MiniMaxH3 under model_config instead.
|
|
model_config = getattr(inner, "model_config", None)
|
|
if model_config.__class__.__name__ != "MiniMaxH3":
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV requires a Moxie AV model"
|
|
)
|
|
|
|
samples = latent.get("samples") if isinstance(latent, dict) else None
|
|
if hasattr(samples, "unbind"):
|
|
streams = list(samples.unbind())
|
|
elif hasattr(samples, "tensors"):
|
|
streams = list(samples.tensors)
|
|
elif isinstance(samples, (tuple, list)):
|
|
streams = list(samples)
|
|
else:
|
|
streams = []
|
|
if not streams or not torch.is_tensor(streams[0]) or streams[0].ndim != 5:
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV requires a Moxie AV latent"
|
|
)
|
|
|
|
prefix_steps = int(prefix_steps)
|
|
if prefix_steps < 1 or prefix_steps >= int(streams[0].shape[2]):
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV prefix must fit before newly generated video"
|
|
)
|
|
|
|
patched = model.clone()
|
|
options = getattr(patched, "model_options", None)
|
|
if not isinstance(options, dict):
|
|
raise ValueError("The connected MODEL has no model_options dictionary")
|
|
if callable(options.get("denoise_mask_function")):
|
|
raise ValueError(
|
|
"Moxie Drift-Control AV cannot combine with another dynamic denoise-mask patch"
|
|
)
|
|
if not callable(getattr(patched, "set_model_denoise_mask_function", None)):
|
|
raise RuntimeError(
|
|
"Moxie Drift-Control AV requires current ComfyUI dynamic denoise-mask support"
|
|
)
|
|
if not callable(getattr(patched, "add_wrapper_with_key", None)):
|
|
raise RuntimeError(
|
|
"Moxie Drift-Control AV requires ComfyUI apply-model wrappers"
|
|
)
|
|
|
|
from comfy.patcher_extension import WrappersMP
|
|
|
|
state = _DriftControlMaskState(
|
|
tuple(streams[0].shape),
|
|
sigmas,
|
|
prefix_steps,
|
|
)
|
|
patched.set_model_denoise_mask_function(state.denoise_mask_function)
|
|
patched.add_wrapper_with_key(
|
|
WrappersMP.APPLY_MODEL,
|
|
_WRAPPER_KEY,
|
|
state.apply_model_wrapper,
|
|
)
|
|
patched.model_options[_WRAPPER_KEY] = state
|
|
return patched
|
|
|
|
|
|
def apply_context_swap_drift_control(
|
|
model: Any,
|
|
target_latent: dict[str, Any],
|
|
context_latent: dict[str, Any],
|
|
sigmas: Any,
|
|
context_length: int | str = "22",
|
|
*,
|
|
continue_audio: bool = True,
|
|
freeze_audio: bool = False,
|
|
) -> tuple[Any, dict[str, Any], int]:
|
|
"""Prepare the disposable AV prefix and return a Drift-Control patched model."""
|
|
prepared, trim_frames, video_steps = prepare_context_swap_latent(
|
|
target_latent,
|
|
context_latent,
|
|
int(context_length),
|
|
continue_audio=bool(continue_audio),
|
|
freeze_audio=bool(freeze_audio),
|
|
)
|
|
patched_model = install_drift_control_av_model(
|
|
model,
|
|
prepared,
|
|
sigmas,
|
|
prefix_steps=video_steps,
|
|
)
|
|
return patched_model, prepared, trim_frames
|