From bd26656732fd0bd78b70ac7a787c68371845cd20 Mon Sep 17 00:00:00 2001 From: Z User Date: Sun, 27 Sep 2026 00:27:37 +0000 Subject: [PATCH] feat: hidden half-res sampling with RTX VSR output restore Single-pass MultiTrack projects now condition and sample at half the editor-selected resolution; Moxie Project Video Combine restores the exact editor resolution on the combined frames before export, so the editor resolution remains what the user sees and gets. - utils/rtx_vsr.py (new): RTX Video Super Resolution core adapted from Comfy-Org/Nvidia_RTX_Nodes_ComfyUI (Apache 2.0) driving the optional nvidia-vfx package per frame, with automatic Lanczos fallback and exact-size correction; owns the sampling-dimension policy and the project_data gating so only half-res signatures are upscaled - nodes/project.py: gate half-res conditioning/sampling to single-pass mode (dual-pass, SelfLift, passthrough, audio-only unchanged) - moxie_combine.py: restore editor resolution right before the images output; VIDEO preview output passthrough unchanged - scripts/test_rtx_upscale_integration.py: standalone sanity checks (no ComfyUI/torch needed) for the dimension policy and gating --- CHANGELOG.md | 1 + moxie_combine.py | 7 +- nodes/project.py | 34 ++- scripts/test_rtx_upscale_integration.py | 84 ++++++++ utils/rtx_vsr.py | 261 ++++++++++++++++++++++++ 5 files changed, 378 insertions(+), 9 deletions(-) create mode 100644 scripts/test_rtx_upscale_integration.py create mode 100644 utils/rtx_vsr.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e877a18..3128b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### ⚡ Improvements +- **Moxie MultiTrack Project**: Single-pass project sampling now runs internally at half the editor-selected resolution, and Moxie Project Video Combine restores the exact editor-selected resolution on the combined frames before export. Output size, frame timing, and audio are identical to the editor settings and no workflow wiring changes; upscaling is hardware-accelerated when the optional `nvidia-vfx` package is installed (NVIDIA RTX GPUs), with an automatic Lanczos fallback everywhere else. Dual-pass, SelfLift, passthrough, and audio-only flows are unchanged. - **Moxie Multimedia Loader**: The `lora_name` combo now defaults to the bundled turbo LoRA (`MM3step.safetensors`) so a fresh clone of the pack works out of the box, with a `none` entry to disable LoRA. Previously the default was a `""` placeholder, which silently skipped LoRA application even when a LoRA was bundled. - **Moxie Preview Override**: The `tiny_vae` combo now lists files from the node pack's own `models/vae_approx` folder first (shipping `MM-preview.safetensors` for MiniMax H3 previews as the default), falling back to ComfyUI's standard `models/vae_approx` directory. The folder is rescanned on every schema refresh, so newly dropped files appear without restarting ComfyUI. - **Moxie Multimedia Loader**: Locked the loader to the MiniMax H3 model set. The `clip_type` dropdown (which previously exposed every CLIP recipe in ComfyUI) was removed; the H3 Qwen3-VL text encoder is now always loaded with the `minimax` CLIP recipe. Any older ComfyUI build without `CLIPType.MINIMAX` raises a clear startup error instead of silently loading the wrong encoder. diff --git a/moxie_combine.py b/moxie_combine.py index 0aa3d02..6c1915c 100644 --- a/moxie_combine.py +++ b/moxie_combine.py @@ -4,6 +4,7 @@ from comfy_api.latest import io from comfy_execution.graph_utils import ExecutionBlocker from .nodes.project import EasyMultiTrackProjectVideoCombine, TYPE_PROJECT_DATA +from .utils.rtx_vsr import upscale_images_to_output_resolution class MoxieCombine(EasyMultiTrackProjectVideoCombine): @@ -49,10 +50,14 @@ class MoxieCombine(EasyMultiTrackProjectVideoCombine): return io.NodeOutput(video, video, video, video, video) components = video.get_components() + # Hidden half-res sampling contract: restore the editor-selected + # resolution right before the frames leave this node. Frames without + # the half-res signature pass through untouched. + images = upscale_images_to_output_resolution(components.images, project_data) return io.NodeOutput( video, prefix, - components.images, + images, components.audio, float(components.frame_rate), ) diff --git a/nodes/project.py b/nodes/project.py index 53a751e..60dc014 100644 --- a/nodes/project.py +++ b/nodes/project.py @@ -36,6 +36,7 @@ from ..utils.h3_project import ( validate_h3_project_outputs, ) from ..utils.models import detect_turbo_lora_from_prompt, detect_turbo_model +from ..utils.rtx_vsr import h3_halfres_sampling_dimensions from ..utils.project_memory import ( BOUNDARY_META, SEGMENT_META, @@ -1169,16 +1170,33 @@ class EasyMultiTrackProject(io.ComfyNode): "upscale_by must be a finite value greater than or equal to 1" ) - first_pass_width = int(info["width"]) - first_pass_height = int(info["height"]) - audio_only = (first_pass_width, first_pass_height) == (32, 32) + editor_width = int(info["width"]) + editor_height = int(info["height"]) + audio_only = (editor_width, editor_height) == (32, 32) fps = float(info["frame_rate"]) - target_width, target_height = h3_second_pass_dimensions( - first_pass_width, - first_pass_height, - run_second_pass, - upscale_by, + # Hidden half-res sampling: single-pass projects condition and sample + # at half the editor resolution; the video combine step restores the + # editor resolution in pixel space (utils/rtx_vsr.py) before export. + halfres_sampling = h3_halfres_sampling_dimensions( + editor_width, editor_height ) + use_halfres_sampling = ( + halfres_sampling is not None + and sampling_mode == "single" + and not audio_only + ) + if use_halfres_sampling: + first_pass_width, first_pass_height = halfres_sampling + target_width, target_height = editor_width, editor_height + else: + first_pass_width = editor_width + first_pass_height = editor_height + target_width, target_height = h3_second_pass_dimensions( + editor_width, + editor_height, + run_second_pass, + upscale_by, + ) # Keep task inputs at the configured size; exports use the final size. output_info = {**info, "width": target_width, "height": target_height} initialize_h3_project( diff --git a/scripts/test_rtx_upscale_integration.py b/scripts/test_rtx_upscale_integration.py new file mode 100644 index 0000000..76c2db6 --- /dev/null +++ b/scripts/test_rtx_upscale_integration.py @@ -0,0 +1,84 @@ +"""Standalone sanity checks for the hidden half-res output upscale contract. + +Loads utils/rtx_vsr.py directly (no ComfyUI, no torch) and verifies the +dimension policy and the project_data gating that MoxieCombine relies on. +The torch/nvvfx execution path is covered by the user's on-machine tests. + +Run: python scripts/test_rtx_upscale_integration.py +""" +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +PACK_ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = PACK_ROOT / "utils" / "rtx_vsr.py" + +spec = importlib.util.spec_from_file_location("rtx_vsr_standalone", MODULE_PATH) +rtx_vsr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rtx_vsr) + +assert "torch" not in sys.modules, "rtx_vsr must import torch lazily" + +failures = [] + + +def check(name, got, expected): + ok = got == expected + print(f"{'PASS' if ok else 'FAIL'} {name}: got={got!r} expected={expected!r}") + if not ok: + failures.append(name) + + +# --- dimension policy --- +cases = [ + ((544, 960), (272, 480)), + ((960, 544), (480, 272)), + ((1088, 1920), (544, 960)), + ((1344, 768), (672, 384)), + ((552, 968), (288, 496)), + ((552, 488), (288, 256)), + ((256, 256), (128, 128)), + ((32, 32), None), # audio-only sentinel + ((64, 64), None), # half would collapse onto the sentinel + ((32, 64), None), + ((33, 33), None), + ((0, 100), None), + ((100, -5), None), + (("abc", 100), None), +] +for (w, h), expected in cases: + got = rtx_vsr.h3_halfres_sampling_dimensions(w, h) + check(f"dims({w}x{h})", got, expected) + +# --- round trip: sampling canvas restores the editor selection --- +for w, h in [(544, 960), (960, 544), (1088, 1920), (1344, 768), (552, 968), (272, 480)]: + sampling = rtx_vsr.h3_halfres_sampling_dimensions(w, h) + if sampling is None: + continue + target = rtx_vsr.resolve_output_upscale_target( + sampling, {"width": w, "height": h} + ) + check(f"round-trip {w}x{h}", target, (w, h)) + +# --- gating (MoxieCombine contract) --- +G = rtx_vsr.resolve_output_upscale_target +data = {"width": 960, "height": 544} +check("fires on half-res signature", G((480, 272), data), (960, 544)) +check("no-op at target size", G((960, 544), data), None) +check("no-op for foreign size", G((500, 300), data), None) +check("no-op for dual-pass size", G((1664, 960), data), None) +check("json string project data", G((272, 480), '{"width": 544, "height": 960}'), (544, 960)) +check("bad json", G((272, 480), "{oops"), None) +check("empty dict", G((272, 480), {}), None) +check("none data", G((272, 480), None), None) +check("missing height", G((272, 480), {"width": 960}), None) +check("zero dims", G((272, 480), {"width": 0, "height": 0}), None) +check("sentinel project", G((32, 32), {"width": 32, "height": 32}), None) + +print() +if failures: + print(f"{len(failures)} FAILED: {failures}") + sys.exit(1) +print("All sanity checks passed.") diff --git a/utils/rtx_vsr.py b/utils/rtx_vsr.py new file mode 100644 index 0000000..f383faf --- /dev/null +++ b/utils/rtx_vsr.py @@ -0,0 +1,261 @@ +"""Hidden half-res sampling output restore for Moxie MultiTrack projects. + +Single-pass project sampling runs at half the editor-selected resolution so +the sampling chain stays small and fast. This module owns the other half of +that contract: combined frames are restored to the exact editor-selected +resolution before the final export, with no user-visible wiring anywhere. + +Hardware path adapted from NVIDIA's RTX Video Super Resolution ComfyUI node +(Apache License 2.0): + + https://github.com/Comfy-Org/Nvidia_RTX_Nodes_ComfyUI + Copyright 2025 NVIDIA Corporation + +It drives the ``nvidia-vfx`` pip package (RTX Video Super Resolution) per +frame via DLPack. When the package or a supported GPU is unavailable, the +module falls back to ComfyUI's Lanczos resize so the delivered frame size is +always exact and rendering never fails because of the upscaler. +""" + +from __future__ import annotations + +import json +from typing import Any + +__all__ = [ + "h3_halfres_sampling_dimensions", + "resolve_output_upscale_target", + "rtx_vsr_upscale", + "upscale_images_to_output_resolution", +] + + +def h3_halfres_sampling_dimensions( + width: int, height: int +) -> tuple[int, int] | None: + """Sampling canvas for hidden half-res sampling, or ``None`` when disabled. + + Editor dimensions are always a multiple of 8 (MultiTrack Editor widgets + and presets), so the normal case is an exact 2x: half of an 8-aligned + size rounded up to the 16px sampling grid returns the editor size exactly + (544x960 -> 272x480 -> 544x960) and RTX VSR's internal /8 snap becomes a + no-op. The ceil policy also keeps hand-written resolutions working: + sampling rounds up to the 16px latent grid (H3 video latents are //16, + and even pixels keep the staging video codec happy) while the exact + target upscale still lands on the editor selection. + + Returns ``None`` for invalid sizes, for the audio-only sentinel (32x32), + and for tiny canvases whose half would collapse onto that sentinel. + """ + try: + width = int(width) + height = int(height) + except (TypeError, ValueError, OverflowError): + return None + if width <= 0 or height <= 0 or (width, height) == (32, 32): + return None + + def _half_up(value: int) -> int: + return max(32, -(-value // 2 // 16) * 16) + + sampling_width = _half_up(width) + sampling_height = _half_up(height) + if (sampling_width, sampling_height) == (32, 32): + return None + return sampling_width, sampling_height + + +def _output_resolution_from_project_data( + project_data: Any, +) -> tuple[int, int] | None: + """Editor-selected output resolution carried by the project widget data.""" + data = project_data + if isinstance(data, str): + try: + data = json.loads(data) + except json.JSONDecodeError: + return None + if not isinstance(data, dict): + return None + try: + width = int(data.get("width")) + height = int(data.get("height")) + except (TypeError, ValueError, OverflowError): + return None + if width <= 0 or height <= 0: + return None + return width, height + + +def resolve_output_upscale_target( + frame_size: tuple[int, int], + project_data: Any, +) -> tuple[int, int] | None: + """Output size to restore, or ``None`` when the frames need no upscale. + + Fires only for the exact signature of the hidden half-res pipeline: the + combined frames match the half-res sampling canvas derived from the + project's editor resolution. Everything else passes through untouched - + full-res renders, dual-pass and SelfLift outputs, passthrough media, + legacy projects without the sampling contract, and foreign videos that + happen to share a project's name. + """ + output = _output_resolution_from_project_data(project_data) + if output is None: + return None + output_width, output_height = output + sampling = h3_halfres_sampling_dimensions(output_width, output_height) + if sampling is None: + return None + sampling_width, sampling_height = sampling + if frame_size == (sampling_width, sampling_height) and frame_size != ( + output_width, + output_height, + ): + return output_width, output_height + return None + + +def _log(message: str) -> None: + try: + from .log import log_node_info + except Exception: # pragma: no cover - standalone tooling without the pack + print(f"[Moxie] {message}", flush=True) # noqa: T201 + return + log_node_info("MultiTrack Output", message) + + +_FALLBACK_NOTICE_SHOWN = False + + +def _notice_fallback(message: str) -> None: + global _FALLBACK_NOTICE_SHOWN + if _FALLBACK_NOTICE_SHOWN: + return + _FALLBACK_NOTICE_SHOWN = True + _log(message) + + +def _lanczos_upscale(images: Any, output_width: int, output_height: int) -> Any: + import comfy.utils + + samples = images[..., :3].movedim(-1, 1) + upscaled = comfy.utils.common_upscale( + samples, output_width, output_height, "lanczos", "disabled" + ) + return upscaled.movedim(1, -1) + + +def rtx_vsr_upscale( + images: Any, + output_width: int, + output_height: int, + quality: str = "ULTRA", +) -> Any: + """Upscale an IMAGE batch to the exact target size, hardware first. + + Frames are processed per frame through RTX Video Super Resolution + (adapted from Comfy-Org/Nvidia_RTX_Nodes_ComfyUI, Apache 2.0). Any + failure - missing package, unsupported GPU, VRAM pressure - degrades to + the Lanczos fallback instead of breaking the render, and the delivered + size is corrected to the exact target whenever the hardware snap misses. + """ + import torch + + frame_count, in_height, in_width, channels = images.shape + if frame_count == 0 or (in_height, in_width) == (output_height, output_width): + return images + + hardware_ready = channels == 3 + if hardware_ready: + try: + import nvvfx + except Exception: + hardware_ready = False + _notice_fallback( + "Hardware upscale unavailable (install nvidia-vfx for RTX " + "acceleration); using Lanczos fallback." + ) + + if hardware_ready: + # /8 snap from the NVIDIA node; editor selections are 8-aligned so + # this is an identity operation for real project resolutions. + snapped_width = max(8, round(output_width / 8) * 8) + snapped_height = max(8, round(output_height / 8) * 8) + max_batch_pixels = 1024 * 1024 * 16 + batch_size = max( + 1, max_batch_pixels // max(1, snapped_width * snapped_height) + ) + quality_levels = { + "LOW": nvvfx.effects.QualityLevel.LOW, + "MEDIUM": nvvfx.effects.QualityLevel.MEDIUM, + "HIGH": nvvfx.effects.QualityLevel.HIGH, + "ULTRA": nvvfx.effects.QualityLevel.ULTRA, + } + selected_quality = quality_levels.get( + str(quality).upper(), nvvfx.effects.QualityLevel.HIGH + ) + out_tensor = torch.empty( + (frame_count, snapped_height, snapped_width, channels), + device=images.device, + dtype=images.dtype, + ) + try: + with nvvfx.VideoSuperRes(selected_quality) as super_res: + super_res.output_width = snapped_width + super_res.output_height = snapped_height + super_res.load() + for start in range(0, frame_count, batch_size): + batch = images[start : start + batch_size] + batch_cuda = ( + batch.cuda().permute(0, 3, 1, 2).float().contiguous() + ) + for offset in range(batch_cuda.shape[0]): + result = super_res.run(batch_cuda[offset]).image + out_tensor[start + offset : start + offset + 1] = ( + torch.from_dlpack(result).movedim(0, -1).unsqueeze(0) + ) + except Exception as error: + hardware_ready = False + _notice_fallback( + f"Hardware upscale failed ({error}); using Lanczos fallback." + ) + else: + if (snapped_height, snapped_width) != (output_height, output_width): + return _lanczos_upscale(out_tensor, output_width, output_height) + return out_tensor + + return _lanczos_upscale(images, output_width, output_height) + + +def upscale_images_to_output_resolution( + images: Any, + project_data: Any, +) -> Any: + """Restore the editor resolution on combined half-res frames. + + Used by Moxie Project Video Combine right before the ``images`` output: + frames carrying the hidden half-res sampling signature are upscaled back + to the editor-selected resolution, everything else is returned untouched. + """ + try: + import torch + except ImportError: # pragma: no cover - torch is required in ComfyUI + return images + if ( + not isinstance(images, torch.Tensor) + or images.ndim != 4 + or images.shape[0] == 0 + ): + return images + target = resolve_output_upscale_target( + (int(images.shape[2]), int(images.shape[1])), project_data + ) + if target is None: + return images + output_width, output_height = target + _log( + f"Output upscale: {int(images.shape[2])}x{int(images.shape[1])} -> " + f"{output_width}x{output_height}" + ) + return rtx_vsr_upscale(images, output_width, output_height)