MMnode/modules/selflift/__init__.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

133 lines
5.5 KiB
Python

"""SelfLift-zero (Artifact-Aware Consistency Lift) for progressive-resolution sampling.
Implements the training-free transition from the SelfLift paper (arXiv:2609.02036),
Algorithm 1 with mode=zero:
1. predict the clean endpoint at the transition state (Eq. 3)
2. build paired lifts: direct latent upsample vs pixel-VAE re-encode (Eqs. 4-5)
3. use their residual as a localized artifact-risk signal and correction direction
(Eqs. 6-8), correcting only the top-rho inconsistent locations (Eq. 9)
4. re-noise the corrected clean latent at the transition sigma (Eq. 10)
"""
import torch
import torch.nn.functional
import comfy.utils
def paired_lifts(
z0_low,
vae,
out_hw,
latent_mode="nearest",
latent_lifter=None,
need_lat=True,
need_pix=True,
temporal_split=None,
):
"""Direct latent lift and pixel-VAE re-encode of the low-res clean endpoint (Eqs. 4-5).
z0_low: [B, C, T, h, w] video or [B, C, h, w] image clean latent prediction in the
VAE's native latent space. latent_lifter: optional learned upsampler callable
(z0_low, out_hw, temporal_split=...) -> latent, used for the direct lift
on MiniMax H3. ``temporal_split`` separates a disposable context prefix
from the retained generated suffix for temporal upscalers.
need_lat/need_pix skip branches the correction weights would discard: rho=0 needs
no pixel anchor, the pure anchor (rho=w=1) needs no direct lift.
Returns (z_lat, z_pix) at the target resolution, None for skipped branches.
"""
H, W = out_hw
if z0_low.ndim == 4: # image latent
z_lat = None
if need_lat:
z_lat = torch.nn.functional.interpolate(z0_low.float(), size=(H, W),
mode="nearest" if latent_mode == "nearest" else "bilinear")
z_pix = None
if need_pix:
img = vae.decode(z0_low) # [B, h*ratio, w*ratio, 3]
ratio = img.shape[1] // z0_low.shape[-2]
up = comfy.utils.common_upscale(img.movedim(-1, 1), W * ratio, H * ratio, "lanczos", "disabled").movedim(1, -1)
del img
z_pix = vae.encode(up).float()
if z_lat is not None:
z_lat = z_lat.to(z_pix.device)
return z_lat, z_pix
T = z0_low.shape[2]
z_lat = None
if need_lat:
if latent_lifter is not None:
z_lat = latent_lifter(
z0_low,
(H, W),
temporal_split=temporal_split,
)
else:
if latent_mode == "bilinear":
latent_mode = "trilinear" # same method family, 5D name
z_lat = torch.nn.functional.interpolate(z0_low.float(), size=(T, H, W), mode=latent_mode)
z_pix = None
if need_pix:
z_pix = _pixel_anchor_video(z0_low, vae, (H, W))
if z_lat is not None:
z_lat = z_lat.to(z_pix.device)
return z_lat, z_pix
def _pixel_anchor_video(z0_low, vae, out_hw):
"""Build pixel anchors independently for each video in the latent batch."""
if z0_low.shape[0] == 1:
return _pixel_anchor_video_single(z0_low, vae, out_hw)
return torch.cat([
_pixel_anchor_video_single(sample, vae, out_hw)
for sample in z0_low.split(1)
], dim=0)
def _pixel_anchor_video_single(z0_low, vae, out_hw):
"""Pixel-VAE anchor for video: decode low-res, upscale, re-encode (Eq. 5).
The upscale runs in fp16 GPU frame chunks into a preallocated buffer: a naive
whole-video lanczos pass materializes several fp32 [F, H*ratio, W*ratio, 3] copies
on the CPU (tens of GB on long clips), which pushes an already full RAM into swap
and stalls the transition for minutes.
"""
H, W = out_hw
frames = vae.decode(z0_low)
if frames.ndim == 5: # VAEDecode convention: video pixels are frames-as-batch [F, H, W, C]
frames = frames.reshape(-1, frames.shape[-3], frames.shape[-2], frames.shape[-1])
ratio = frames.shape[1] // z0_low.shape[-2]
Hp, Wp = H * ratio, W * ratio
work_dtype = vae.vae_dtype if vae.vae_dtype in (torch.float16, torch.bfloat16, torch.float32) else torch.float32
if vae.device.type == "cpu":
work_dtype = torch.float32
n = frames.shape[0]
up = torch.empty((n, Hp, Wp, frames.shape[-1]), dtype=work_dtype)
for i in range(0, n, 32):
chunk = frames[i:i + 32].movedim(-1, 1).to(device=vae.device, dtype=work_dtype)
chunk = torch.nn.functional.interpolate(chunk, size=(Hp, Wp), mode="bicubic", antialias=True)
up[i:i + 32] = chunk.movedim(1, -1).to(up.device)
del chunk
del frames
return vae.encode(up).float() # the wrapper turns the frame batch back into the time dim
def artifact_aware_consistency_lift(z_lat, z_pix, rho, w_min, w_max):
"""Selective correction of the direct lift toward the pixel-VAE anchor (Eqs. 6-9)."""
if rho <= 0.0 or w_max <= 0.0:
return z_lat
if rho >= 1.0 and w_min >= 1.0 and w_max >= 1.0:
return z_pix
delta = z_pix - z_lat
s = delta.abs().mean(dim=1) # per-location inconsistency, [B, (T,) H, W]
view = (-1,) + (1,) * (s.ndim - 1)
flat = s.flatten(1)
thr = torch.quantile(flat, 1.0 - rho, dim=1).view(view)
mask = s >= thr
s_min = s.masked_fill(~mask, float("inf")).flatten(1).amin(dim=1).view(view)
s_max = s.masked_fill(~mask, float("-inf")).flatten(1).amax(dim=1).view(view)
w = w_min + (w_max - w_min) * (s - s_min) / (s_max - s_min + 1e-8)
w = torch.where(mask, w, torch.zeros_like(w)).unsqueeze(1)
return z_lat + w * delta