MMnode/utils/rtx_vsr.py
Z User bd26656732 feat: hidden half-res sampling with RTX VSR output restore
Single-pass MultiTrack projects now condition and sample at half the
editor-selected resolution; Moxie Project Video Combine restores the
exact editor resolution on the combined frames before export, so the
editor resolution remains what the user sees and gets.

- utils/rtx_vsr.py (new): RTX Video Super Resolution core adapted from
  Comfy-Org/Nvidia_RTX_Nodes_ComfyUI (Apache 2.0) driving the optional
  nvidia-vfx package per frame, with automatic Lanczos fallback and
  exact-size correction; owns the sampling-dimension policy and the
  project_data gating so only half-res signatures are upscaled
- nodes/project.py: gate half-res conditioning/sampling to single-pass
  mode (dual-pass, SelfLift, passthrough, audio-only unchanged)
- moxie_combine.py: restore editor resolution right before the images
  output; VIDEO preview output passthrough unchanged
- scripts/test_rtx_upscale_integration.py: standalone sanity checks
  (no ComfyUI/torch needed) for the dimension policy and gating
2026-09-27 00:27:37 +00:00

262 lines
9.3 KiB
Python

"""Hidden half-res sampling output restore for Moxie MultiTrack projects.
Single-pass project sampling runs at half the editor-selected resolution so
the sampling chain stays small and fast. This module owns the other half of
that contract: combined frames are restored to the exact editor-selected
resolution before the final export, with no user-visible wiring anywhere.
Hardware path adapted from NVIDIA's RTX Video Super Resolution ComfyUI node
(Apache License 2.0):
https://github.com/Comfy-Org/Nvidia_RTX_Nodes_ComfyUI
Copyright 2025 NVIDIA Corporation
It drives the ``nvidia-vfx`` pip package (RTX Video Super Resolution) per
frame via DLPack. When the package or a supported GPU is unavailable, the
module falls back to ComfyUI's Lanczos resize so the delivered frame size is
always exact and rendering never fails because of the upscaler.
"""
from __future__ import annotations
import json
from typing import Any
__all__ = [
"h3_halfres_sampling_dimensions",
"resolve_output_upscale_target",
"rtx_vsr_upscale",
"upscale_images_to_output_resolution",
]
def h3_halfres_sampling_dimensions(
width: int, height: int
) -> tuple[int, int] | None:
"""Sampling canvas for hidden half-res sampling, or ``None`` when disabled.
Editor dimensions are always a multiple of 8 (MultiTrack Editor widgets
and presets), so the normal case is an exact 2x: half of an 8-aligned
size rounded up to the 16px sampling grid returns the editor size exactly
(544x960 -> 272x480 -> 544x960) and RTX VSR's internal /8 snap becomes a
no-op. The ceil policy also keeps hand-written resolutions working:
sampling rounds up to the 16px latent grid (H3 video latents are //16,
and even pixels keep the staging video codec happy) while the exact
target upscale still lands on the editor selection.
Returns ``None`` for invalid sizes, for the audio-only sentinel (32x32),
and for tiny canvases whose half would collapse onto that sentinel.
"""
try:
width = int(width)
height = int(height)
except (TypeError, ValueError, OverflowError):
return None
if width <= 0 or height <= 0 or (width, height) == (32, 32):
return None
def _half_up(value: int) -> int:
return max(32, -(-value // 2 // 16) * 16)
sampling_width = _half_up(width)
sampling_height = _half_up(height)
if (sampling_width, sampling_height) == (32, 32):
return None
return sampling_width, sampling_height
def _output_resolution_from_project_data(
project_data: Any,
) -> tuple[int, int] | None:
"""Editor-selected output resolution carried by the project widget data."""
data = project_data
if isinstance(data, str):
try:
data = json.loads(data)
except json.JSONDecodeError:
return None
if not isinstance(data, dict):
return None
try:
width = int(data.get("width"))
height = int(data.get("height"))
except (TypeError, ValueError, OverflowError):
return None
if width <= 0 or height <= 0:
return None
return width, height
def resolve_output_upscale_target(
frame_size: tuple[int, int],
project_data: Any,
) -> tuple[int, int] | None:
"""Output size to restore, or ``None`` when the frames need no upscale.
Fires only for the exact signature of the hidden half-res pipeline: the
combined frames match the half-res sampling canvas derived from the
project's editor resolution. Everything else passes through untouched -
full-res renders, dual-pass and SelfLift outputs, passthrough media,
legacy projects without the sampling contract, and foreign videos that
happen to share a project's name.
"""
output = _output_resolution_from_project_data(project_data)
if output is None:
return None
output_width, output_height = output
sampling = h3_halfres_sampling_dimensions(output_width, output_height)
if sampling is None:
return None
sampling_width, sampling_height = sampling
if frame_size == (sampling_width, sampling_height) and frame_size != (
output_width,
output_height,
):
return output_width, output_height
return None
def _log(message: str) -> None:
try:
from .log import log_node_info
except Exception: # pragma: no cover - standalone tooling without the pack
print(f"[Moxie] {message}", flush=True) # noqa: T201
return
log_node_info("MultiTrack Output", message)
_FALLBACK_NOTICE_SHOWN = False
def _notice_fallback(message: str) -> None:
global _FALLBACK_NOTICE_SHOWN
if _FALLBACK_NOTICE_SHOWN:
return
_FALLBACK_NOTICE_SHOWN = True
_log(message)
def _lanczos_upscale(images: Any, output_width: int, output_height: int) -> Any:
import comfy.utils
samples = images[..., :3].movedim(-1, 1)
upscaled = comfy.utils.common_upscale(
samples, output_width, output_height, "lanczos", "disabled"
)
return upscaled.movedim(1, -1)
def rtx_vsr_upscale(
images: Any,
output_width: int,
output_height: int,
quality: str = "ULTRA",
) -> Any:
"""Upscale an IMAGE batch to the exact target size, hardware first.
Frames are processed per frame through RTX Video Super Resolution
(adapted from Comfy-Org/Nvidia_RTX_Nodes_ComfyUI, Apache 2.0). Any
failure - missing package, unsupported GPU, VRAM pressure - degrades to
the Lanczos fallback instead of breaking the render, and the delivered
size is corrected to the exact target whenever the hardware snap misses.
"""
import torch
frame_count, in_height, in_width, channels = images.shape
if frame_count == 0 or (in_height, in_width) == (output_height, output_width):
return images
hardware_ready = channels == 3
if hardware_ready:
try:
import nvvfx
except Exception:
hardware_ready = False
_notice_fallback(
"Hardware upscale unavailable (install nvidia-vfx for RTX "
"acceleration); using Lanczos fallback."
)
if hardware_ready:
# /8 snap from the NVIDIA node; editor selections are 8-aligned so
# this is an identity operation for real project resolutions.
snapped_width = max(8, round(output_width / 8) * 8)
snapped_height = max(8, round(output_height / 8) * 8)
max_batch_pixels = 1024 * 1024 * 16
batch_size = max(
1, max_batch_pixels // max(1, snapped_width * snapped_height)
)
quality_levels = {
"LOW": nvvfx.effects.QualityLevel.LOW,
"MEDIUM": nvvfx.effects.QualityLevel.MEDIUM,
"HIGH": nvvfx.effects.QualityLevel.HIGH,
"ULTRA": nvvfx.effects.QualityLevel.ULTRA,
}
selected_quality = quality_levels.get(
str(quality).upper(), nvvfx.effects.QualityLevel.HIGH
)
out_tensor = torch.empty(
(frame_count, snapped_height, snapped_width, channels),
device=images.device,
dtype=images.dtype,
)
try:
with nvvfx.VideoSuperRes(selected_quality) as super_res:
super_res.output_width = snapped_width
super_res.output_height = snapped_height
super_res.load()
for start in range(0, frame_count, batch_size):
batch = images[start : start + batch_size]
batch_cuda = (
batch.cuda().permute(0, 3, 1, 2).float().contiguous()
)
for offset in range(batch_cuda.shape[0]):
result = super_res.run(batch_cuda[offset]).image
out_tensor[start + offset : start + offset + 1] = (
torch.from_dlpack(result).movedim(0, -1).unsqueeze(0)
)
except Exception as error:
hardware_ready = False
_notice_fallback(
f"Hardware upscale failed ({error}); using Lanczos fallback."
)
else:
if (snapped_height, snapped_width) != (output_height, output_width):
return _lanczos_upscale(out_tensor, output_width, output_height)
return out_tensor
return _lanczos_upscale(images, output_width, output_height)
def upscale_images_to_output_resolution(
images: Any,
project_data: Any,
) -> Any:
"""Restore the editor resolution on combined half-res frames.
Used by Moxie Project Video Combine right before the ``images`` output:
frames carrying the hidden half-res sampling signature are upscaled back
to the editor-selected resolution, everything else is returned untouched.
"""
try:
import torch
except ImportError: # pragma: no cover - torch is required in ComfyUI
return images
if (
not isinstance(images, torch.Tensor)
or images.ndim != 4
or images.shape[0] == 0
):
return images
target = resolve_output_upscale_target(
(int(images.shape[2]), int(images.shape[1])), project_data
)
if target is None:
return images
output_width, output_height = target
_log(
f"Output upscale: {int(images.shape[2])}x{int(images.shape[1])} -> "
f"{output_width}x{output_height}"
)
return rtx_vsr_upscale(images, output_width, output_height)