Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
264 lines
10 KiB
Python
264 lines
10 KiB
Python
"""Minimal OmniShotCut model loading and shot detection helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import gc
|
|
import math
|
|
import os
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import folder_paths
|
|
import numpy as np
|
|
import torch
|
|
|
|
from ...utils.models import require_model_path
|
|
from ...utils.video import get_ffmpeg_path
|
|
|
|
MODEL_CATEGORY = "checkpoints"
|
|
MODEL_DIRECTORY = Path(folder_paths.models_dir) / MODEL_CATEGORY
|
|
|
|
|
|
def _load_implementation() -> tuple[Any, Any, Any, Any, dict[str, int], dict[str, int]]:
|
|
from .architecture.backbone import build_backbone
|
|
from .architecture.model import OmniShotCut
|
|
from .architecture.transformer import build_transformer
|
|
from .config.label_correspondence import unique_inter_label_mapping, unique_intra_label_mapping
|
|
from .datasets.transforms import Video_Augmentation_Transform
|
|
|
|
return (
|
|
build_backbone,
|
|
build_transformer,
|
|
OmniShotCut,
|
|
Video_Augmentation_Transform,
|
|
unique_intra_label_mapping,
|
|
unique_inter_label_mapping,
|
|
)
|
|
|
|
|
|
def find_checkpoint() -> Path:
|
|
"""Return the configured OmniShotCut checkpoint path."""
|
|
configured = os.environ.get("OMNISHOTCUT_CHECKPOINT")
|
|
if configured:
|
|
checkpoint = Path(configured).expanduser().resolve()
|
|
if checkpoint.is_file():
|
|
return checkpoint
|
|
raise FileNotFoundError(f"OmniShotCut checkpoint not found: {checkpoint}")
|
|
|
|
checkpoint_names = folder_paths.get_filename_list(MODEL_CATEGORY)
|
|
preferred_names = ["OmniShotCut_ckpt.pth", "omnishotcut_ckpt.pth"]
|
|
model_name = next((name for name in preferred_names if name in checkpoint_names), None)
|
|
if model_name is None:
|
|
model_name = next((
|
|
name for name in checkpoint_names
|
|
if "omnishotcut" in name.lower().replace("_", "").replace("-", "")
|
|
and name.lower().endswith(".pth")
|
|
), None)
|
|
if model_name is not None:
|
|
checkpoint_path = folder_paths.get_full_path(MODEL_CATEGORY, model_name)
|
|
if checkpoint_path:
|
|
return Path(checkpoint_path)
|
|
return require_model_path("omnishotcut")
|
|
|
|
|
|
def _load_model(checkpoint_path: Path) -> tuple[torch.nn.Module, Any, torch.device]:
|
|
build_backbone, build_transformer, model_class, _, _, _ = _load_implementation()
|
|
try:
|
|
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
|
|
except TypeError:
|
|
checkpoint = torch.load(checkpoint_path, map_location="cpu")
|
|
if not isinstance(checkpoint, dict) or "args" not in checkpoint or "model" not in checkpoint:
|
|
raise ValueError("OmniShotCut checkpoint must contain 'args' and 'model'")
|
|
|
|
args = checkpoint["args"]
|
|
model = model_class(
|
|
build_backbone(args),
|
|
build_transformer(args),
|
|
num_intra_relation_classes=args.num_intra_relation_classes,
|
|
num_inter_relation_classes=args.num_inter_relation_classes,
|
|
num_frames=args.max_process_window_length,
|
|
num_queries=args.num_queries,
|
|
aux_loss=args.aux_loss,
|
|
)
|
|
model.load_state_dict(checkpoint["model"], strict=True)
|
|
del checkpoint
|
|
|
|
try:
|
|
import comfy.model_management as model_management
|
|
|
|
device = model_management.get_torch_device()
|
|
except (ImportError, RuntimeError):
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
try:
|
|
model.to(device).eval()
|
|
except (torch.cuda.OutOfMemoryError, RuntimeError) as error:
|
|
print(f"[Easy Media] OmniShotCut could not load on {device}; falling back to CPU: {error}")
|
|
model.to("cpu").eval()
|
|
device = torch.device("cpu")
|
|
return model, args, torch.device(device)
|
|
|
|
|
|
def _read_video(video_path: Path, width: int, height: int, fps: float) -> np.ndarray:
|
|
"""Decode the same numbered timeline frames used by the video-track merger."""
|
|
if not math.isfinite(fps) or fps <= 0 or width <= 0 or height <= 0:
|
|
raise ValueError("fps, width, and height must be positive")
|
|
ffmpeg = get_ffmpeg_path()
|
|
if ffmpeg is None:
|
|
raise RuntimeError("FFmpeg is required for frame-aligned smart split")
|
|
frame_size = width * height * 3
|
|
result = subprocess.run(
|
|
[ffmpeg, "-v", "error", "-i", str(video_path),
|
|
"-vf", f"setpts=PTS-STARTPTS,fps=fps={fps}:start_time=0,scale={width}:{height}",
|
|
"-an", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
|
|
capture_output=True, check=False, timeout=600,
|
|
)
|
|
if result.returncode != 0 or not result.stdout or len(result.stdout) % frame_size:
|
|
raise RuntimeError(
|
|
f"Unable to decode timeline frames: {result.stderr.decode(errors='replace')[-600:]}"
|
|
)
|
|
return np.frombuffer(result.stdout, dtype=np.uint8).reshape(-1, height, width, 3).copy()
|
|
|
|
|
|
def _split_video(video: np.ndarray, window_size: int, context_frames: int) -> list[np.ndarray]:
|
|
if context_frames < 0 or context_frames * 2 >= window_size:
|
|
raise ValueError("num_context_frames must be less than half the inference window")
|
|
height, width, channels = video.shape[1:]
|
|
padded = np.concatenate(
|
|
[np.zeros((context_frames, height, width, channels), dtype=video.dtype), video], axis=0
|
|
)
|
|
stride = window_size - 2 * context_frames
|
|
clips: list[np.ndarray] = []
|
|
for start in range(0, len(video), stride):
|
|
clip = padded[start:start + window_size]
|
|
if len(clip) < window_size:
|
|
clip = np.concatenate(
|
|
[clip, np.zeros((window_size - len(clip), height, width, channels), dtype=video.dtype)],
|
|
axis=0,
|
|
)
|
|
clips.append(clip)
|
|
return clips
|
|
|
|
|
|
def _clip_predictions(
|
|
outputs: dict[str, torch.Tensor],
|
|
valid_frames: int,
|
|
context_frames: int,
|
|
window_size: int,
|
|
) -> tuple[list[list[int]], list[int], list[int]]:
|
|
intra = outputs["intra_clip_logits"].softmax(-1)[0, :, :-1].argmax(-1)
|
|
inter = outputs["inter_clip_logits"].softmax(-1)[0, :, :-1].argmax(-1)
|
|
ends = outputs["pred_shot_logits"].softmax(-1)[0, :, :-1].argmax(-1)
|
|
ranges: list[list[int]] = []
|
|
intra_labels: list[int] = []
|
|
inter_labels: list[int] = []
|
|
start = 0
|
|
content_end = min(window_size - context_frames, context_frames + valid_frames)
|
|
for index in range(len(ends)):
|
|
end = int(ends[index].detach().cpu())
|
|
if end <= start:
|
|
continue
|
|
clipped_start = max(start, context_frames) - context_frames
|
|
clipped_end = min(end, content_end) - context_frames
|
|
start = end
|
|
if clipped_end <= clipped_start:
|
|
continue
|
|
ranges.append([clipped_start, clipped_end])
|
|
intra_labels.append(int(intra[index].detach().cpu()))
|
|
inter_labels.append(int(inter[index].detach().cpu()))
|
|
if end >= content_end:
|
|
break
|
|
return ranges, intra_labels, inter_labels
|
|
|
|
|
|
def _merge_predictions(
|
|
target_ranges: list[list[int]],
|
|
target_intra: list[int],
|
|
target_inter: list[int],
|
|
ranges: list[list[int]],
|
|
intra: list[int],
|
|
inter: list[int],
|
|
new_start_label: int,
|
|
) -> None:
|
|
if not ranges:
|
|
return
|
|
last_frame = target_ranges[-1][1] if target_ranges else 0
|
|
first_index = 0
|
|
if target_ranges and target_intra[-1] == intra[0] and inter[0] == new_start_label:
|
|
target_ranges[-1][1] = last_frame + ranges[0][1]
|
|
first_index = 1
|
|
for index in range(first_index, len(ranges)):
|
|
start, end = ranges[index]
|
|
target_ranges.append([last_frame + start, last_frame + end])
|
|
target_intra.append(intra[index])
|
|
target_inter.append(inter[index])
|
|
|
|
|
|
def _release_model(model: torch.nn.Module | None) -> None:
|
|
if model is not None:
|
|
try:
|
|
model.to("cpu")
|
|
except (RuntimeError, AttributeError) as error:
|
|
print(f"[Easy Media] Failed to move OmniShotCut model to CPU: {error}")
|
|
gc.collect()
|
|
try:
|
|
import comfy.model_management as model_management
|
|
|
|
model_management.soft_empty_cache()
|
|
except (ImportError, RuntimeError, AttributeError) as error:
|
|
print(f"[Easy Media] OmniShotCut cache cleanup fallback: {error}")
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
|
|
|
|
def detect_shots(
|
|
video_path: str | Path,
|
|
fps: float,
|
|
mode: str = "clean_shot",
|
|
checkpoint_path: str | Path | None = None,
|
|
num_context_frames: int = 5,
|
|
) -> list[list[int]]:
|
|
"""Detect shots and return frame ranges remapped to the requested FPS."""
|
|
if mode not in {"default", "clean_shot"}:
|
|
raise ValueError(f"unsupported OmniShotCut mode: {mode}")
|
|
model: torch.nn.Module | None = None
|
|
try:
|
|
checkpoint = Path(checkpoint_path) if checkpoint_path else find_checkpoint()
|
|
model, args, device = _load_model(checkpoint)
|
|
video = _read_video(Path(video_path), args.process_width, args.process_height, fps)
|
|
_, _, _, transform_class, intra_mapping, inter_mapping = _load_implementation()
|
|
transform = transform_class(set_type="val")
|
|
window_size = int(args.max_process_window_length)
|
|
stride = window_size - 2 * num_context_frames
|
|
ranges: list[list[int]] = []
|
|
intra_labels: list[int] = []
|
|
inter_labels: list[int] = []
|
|
for index, clip in enumerate(_split_video(video, window_size, num_context_frames)):
|
|
valid_frames = min(stride, len(video) - index * stride)
|
|
tensor: torch.Tensor | None = None
|
|
outputs: dict[str, torch.Tensor] | None = None
|
|
try:
|
|
tensor = transform(clip).unsqueeze(0).to(device)
|
|
with torch.inference_mode():
|
|
outputs = model(tensor)
|
|
clip_ranges, clip_intra, clip_inter = _clip_predictions(
|
|
outputs, valid_frames, num_context_frames, window_size
|
|
)
|
|
_merge_predictions(
|
|
ranges, intra_labels, inter_labels,
|
|
clip_ranges, clip_intra, clip_inter,
|
|
inter_mapping["new_start"],
|
|
)
|
|
finally:
|
|
del outputs, tensor
|
|
if mode == "clean_shot":
|
|
general_label = intra_mapping["general"]
|
|
ranges = [
|
|
shot_range
|
|
for shot_range, intra_label in zip(ranges, intra_labels, strict=True)
|
|
if intra_label == general_label
|
|
]
|
|
return ranges
|
|
finally:
|
|
_release_model(model)
|