MMnode/modules/omnishotcut/__init__.py
Z User 27041a8a9f feat: scrub EasyMedia/MiniMax branding from all user-facing output
Console, errors, tooltips, descriptions and display names no longer
reveal the pack heritage or the underlying model family:

- [EasyMedia]/[Easy Media] console prefixes -> [Moxie] (11 files)
- New console privacy filter (utils/log.py, installed at pack import):
  rewrites MiniMax variants in every displayed logging record, including
  ComfyUI core lines like 'Model MiniMaxH3 prepared for dynamic VRAM
  loading', %-args and matching tracebacks. api.minimaxi.com is protected
  so cloud-API errors stay accurate.
- Node display names rebranded (node_ids untouched, workflows safe):
  'MiniMax H3 To Video' -> 'Moxie To Video', etc.
- ~40 error/tooltip/description strings reworded to Moxie terms.
- CLIP error no longer names CLIPType.MINIMAX; test match updated.

Unchanged on purpose (breaking if touched): node_ids/class names,
combo value 'MiniMax' (workflow payloads), internal dict keys
(minimax_keyframes/minimax_refs), llm_api cloud provider strings,
class-name logic checks, source-only docstrings.
2026-09-26 23:05:24 +00:00

264 lines
10 KiB
Python

"""Minimal OmniShotCut model loading and shot detection helpers."""
from __future__ import annotations
import gc
import math
import os
import subprocess
from pathlib import Path
from typing import Any
import folder_paths
import numpy as np
import torch
from ...utils.models import require_model_path
from ...utils.video import get_ffmpeg_path
MODEL_CATEGORY = "checkpoints"
MODEL_DIRECTORY = Path(folder_paths.models_dir) / MODEL_CATEGORY
def _load_implementation() -> tuple[Any, Any, Any, Any, dict[str, int], dict[str, int]]:
from .architecture.backbone import build_backbone
from .architecture.model import OmniShotCut
from .architecture.transformer import build_transformer
from .config.label_correspondence import unique_inter_label_mapping, unique_intra_label_mapping
from .datasets.transforms import Video_Augmentation_Transform
return (
build_backbone,
build_transformer,
OmniShotCut,
Video_Augmentation_Transform,
unique_intra_label_mapping,
unique_inter_label_mapping,
)
def find_checkpoint() -> Path:
"""Return the configured OmniShotCut checkpoint path."""
configured = os.environ.get("OMNISHOTCUT_CHECKPOINT")
if configured:
checkpoint = Path(configured).expanduser().resolve()
if checkpoint.is_file():
return checkpoint
raise FileNotFoundError(f"OmniShotCut checkpoint not found: {checkpoint}")
checkpoint_names = folder_paths.get_filename_list(MODEL_CATEGORY)
preferred_names = ["OmniShotCut_ckpt.pth", "omnishotcut_ckpt.pth"]
model_name = next((name for name in preferred_names if name in checkpoint_names), None)
if model_name is None:
model_name = next((
name for name in checkpoint_names
if "omnishotcut" in name.lower().replace("_", "").replace("-", "")
and name.lower().endswith(".pth")
), None)
if model_name is not None:
checkpoint_path = folder_paths.get_full_path(MODEL_CATEGORY, model_name)
if checkpoint_path:
return Path(checkpoint_path)
return require_model_path("omnishotcut")
def _load_model(checkpoint_path: Path) -> tuple[torch.nn.Module, Any, torch.device]:
build_backbone, build_transformer, model_class, _, _, _ = _load_implementation()
try:
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
except TypeError:
checkpoint = torch.load(checkpoint_path, map_location="cpu")
if not isinstance(checkpoint, dict) or "args" not in checkpoint or "model" not in checkpoint:
raise ValueError("OmniShotCut checkpoint must contain 'args' and 'model'")
args = checkpoint["args"]
model = model_class(
build_backbone(args),
build_transformer(args),
num_intra_relation_classes=args.num_intra_relation_classes,
num_inter_relation_classes=args.num_inter_relation_classes,
num_frames=args.max_process_window_length,
num_queries=args.num_queries,
aux_loss=args.aux_loss,
)
model.load_state_dict(checkpoint["model"], strict=True)
del checkpoint
try:
import comfy.model_management as model_management
device = model_management.get_torch_device()
except (ImportError, RuntimeError):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
try:
model.to(device).eval()
except (torch.cuda.OutOfMemoryError, RuntimeError) as error:
print(f"[Moxie] OmniShotCut could not load on {device}; falling back to CPU: {error}")
model.to("cpu").eval()
device = torch.device("cpu")
return model, args, torch.device(device)
def _read_video(video_path: Path, width: int, height: int, fps: float) -> np.ndarray:
"""Decode the same numbered timeline frames used by the video-track merger."""
if not math.isfinite(fps) or fps <= 0 or width <= 0 or height <= 0:
raise ValueError("fps, width, and height must be positive")
ffmpeg = get_ffmpeg_path()
if ffmpeg is None:
raise RuntimeError("FFmpeg is required for frame-aligned smart split")
frame_size = width * height * 3
result = subprocess.run(
[ffmpeg, "-v", "error", "-i", str(video_path),
"-vf", f"setpts=PTS-STARTPTS,fps=fps={fps}:start_time=0,scale={width}:{height}",
"-an", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
capture_output=True, check=False, timeout=600,
)
if result.returncode != 0 or not result.stdout or len(result.stdout) % frame_size:
raise RuntimeError(
f"Unable to decode timeline frames: {result.stderr.decode(errors='replace')[-600:]}"
)
return np.frombuffer(result.stdout, dtype=np.uint8).reshape(-1, height, width, 3).copy()
def _split_video(video: np.ndarray, window_size: int, context_frames: int) -> list[np.ndarray]:
if context_frames < 0 or context_frames * 2 >= window_size:
raise ValueError("num_context_frames must be less than half the inference window")
height, width, channels = video.shape[1:]
padded = np.concatenate(
[np.zeros((context_frames, height, width, channels), dtype=video.dtype), video], axis=0
)
stride = window_size - 2 * context_frames
clips: list[np.ndarray] = []
for start in range(0, len(video), stride):
clip = padded[start:start + window_size]
if len(clip) < window_size:
clip = np.concatenate(
[clip, np.zeros((window_size - len(clip), height, width, channels), dtype=video.dtype)],
axis=0,
)
clips.append(clip)
return clips
def _clip_predictions(
outputs: dict[str, torch.Tensor],
valid_frames: int,
context_frames: int,
window_size: int,
) -> tuple[list[list[int]], list[int], list[int]]:
intra = outputs["intra_clip_logits"].softmax(-1)[0, :, :-1].argmax(-1)
inter = outputs["inter_clip_logits"].softmax(-1)[0, :, :-1].argmax(-1)
ends = outputs["pred_shot_logits"].softmax(-1)[0, :, :-1].argmax(-1)
ranges: list[list[int]] = []
intra_labels: list[int] = []
inter_labels: list[int] = []
start = 0
content_end = min(window_size - context_frames, context_frames + valid_frames)
for index in range(len(ends)):
end = int(ends[index].detach().cpu())
if end <= start:
continue
clipped_start = max(start, context_frames) - context_frames
clipped_end = min(end, content_end) - context_frames
start = end
if clipped_end <= clipped_start:
continue
ranges.append([clipped_start, clipped_end])
intra_labels.append(int(intra[index].detach().cpu()))
inter_labels.append(int(inter[index].detach().cpu()))
if end >= content_end:
break
return ranges, intra_labels, inter_labels
def _merge_predictions(
target_ranges: list[list[int]],
target_intra: list[int],
target_inter: list[int],
ranges: list[list[int]],
intra: list[int],
inter: list[int],
new_start_label: int,
) -> None:
if not ranges:
return
last_frame = target_ranges[-1][1] if target_ranges else 0
first_index = 0
if target_ranges and target_intra[-1] == intra[0] and inter[0] == new_start_label:
target_ranges[-1][1] = last_frame + ranges[0][1]
first_index = 1
for index in range(first_index, len(ranges)):
start, end = ranges[index]
target_ranges.append([last_frame + start, last_frame + end])
target_intra.append(intra[index])
target_inter.append(inter[index])
def _release_model(model: torch.nn.Module | None) -> None:
if model is not None:
try:
model.to("cpu")
except (RuntimeError, AttributeError) as error:
print(f"[Moxie] Failed to move OmniShotCut model to CPU: {error}")
gc.collect()
try:
import comfy.model_management as model_management
model_management.soft_empty_cache()
except (ImportError, RuntimeError, AttributeError) as error:
print(f"[Moxie] OmniShotCut cache cleanup fallback: {error}")
if torch.cuda.is_available():
torch.cuda.empty_cache()
def detect_shots(
video_path: str | Path,
fps: float,
mode: str = "clean_shot",
checkpoint_path: str | Path | None = None,
num_context_frames: int = 5,
) -> list[list[int]]:
"""Detect shots and return frame ranges remapped to the requested FPS."""
if mode not in {"default", "clean_shot"}:
raise ValueError(f"unsupported OmniShotCut mode: {mode}")
model: torch.nn.Module | None = None
try:
checkpoint = Path(checkpoint_path) if checkpoint_path else find_checkpoint()
model, args, device = _load_model(checkpoint)
video = _read_video(Path(video_path), args.process_width, args.process_height, fps)
_, _, _, transform_class, intra_mapping, inter_mapping = _load_implementation()
transform = transform_class(set_type="val")
window_size = int(args.max_process_window_length)
stride = window_size - 2 * num_context_frames
ranges: list[list[int]] = []
intra_labels: list[int] = []
inter_labels: list[int] = []
for index, clip in enumerate(_split_video(video, window_size, num_context_frames)):
valid_frames = min(stride, len(video) - index * stride)
tensor: torch.Tensor | None = None
outputs: dict[str, torch.Tensor] | None = None
try:
tensor = transform(clip).unsqueeze(0).to(device)
with torch.inference_mode():
outputs = model(tensor)
clip_ranges, clip_intra, clip_inter = _clip_predictions(
outputs, valid_frames, num_context_frames, window_size
)
_merge_predictions(
ranges, intra_labels, inter_labels,
clip_ranges, clip_intra, clip_inter,
inter_mapping["new_start"],
)
finally:
del outputs, tensor
if mode == "clean_shot":
general_label = intra_mapping["general"]
ranges = [
shot_range
for shot_range, intra_label in zip(ranges, intra_labels, strict=True)
if intra_label == general_label
]
return ranges
finally:
_release_model(model)