MMnode/modules/selflift/h3_latent_upscale.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

504 lines
20 KiB
Python

"""Learned latent-space lifter for MiniMax H3 (3D-conv upscaler).
Bundles the inference architecture of the Minimax H3 Latent Upscaler
(LBH-123-AI/Minimax_h3_latent_Upscaler, architecture drawing on the LTX 2.3
spatial upscaler) as an optional experimental replacement for SelfLift-zero's
nearest-neighbor latent lift. This external model is not the SelfLift-rich
lifter described by the paper. The checkpoint is expected under
ComfyUI/models/latent_upscale_models/.
"""
import logging
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import comfy.model_management
import comfy.model_patcher
import folder_paths
from comfy.ldm.minimax.vae import LATENTS_MEAN, LATENTS_STD
from .diagnostics import log_memory
def _normalization(channels):
return nn.GroupNorm(32, channels)
def _zero_module(module):
for p in module.parameters():
p.detach().zero_()
return module
def _temporal_windows(length, chunk, overlap):
for start in range(0, length, chunk):
end = min(length, start + chunk)
yield start, end, max(0, start - overlap), min(length, end + overlap)
class AttnBlock3D(nn.Module):
def __init__(self, in_channels):
super().__init__()
self.norm = _normalization(in_channels)
self.q = nn.Conv3d(in_channels, in_channels, 1)
self.k = nn.Conv3d(in_channels, in_channels, 1)
self.v = nn.Conv3d(in_channels, in_channels, 1)
self.proj_out = nn.Conv3d(in_channels, in_channels, 1)
def forward(self, x):
h = self.norm(x)
q = self.q(h).flatten(2).movedim(-1, 1).unsqueeze(1)
k = self.k(h).flatten(2).movedim(-1, 1).unsqueeze(1)
v = self.v(h).flatten(2).movedim(-1, 1).unsqueeze(1)
h = F.scaled_dot_product_attention(q, k, v)
h = h.squeeze(1).movedim(1, -1).reshape(x.shape)
return x + self.proj_out(h)
class ResBlockEmb3D(nn.Module):
def __init__(self, channels, emb_channels, dropout=0, out_channels=None):
super().__init__()
self.out_channels = out_channels or channels
self.in_layers = nn.Sequential(
_normalization(channels), nn.SiLU(),
nn.Conv3d(channels, self.out_channels, 3, padding=1),
)
self.emb_layers = nn.Sequential(
nn.SiLU(), nn.Linear(emb_channels, 2 * self.out_channels),
)
self.out_norm = _normalization(self.out_channels)
self.out_layers = nn.Sequential(
nn.SiLU(), nn.Dropout(p=dropout),
_zero_module(nn.Conv3d(self.out_channels, self.out_channels, 3, padding=1)),
)
self.skip = (
nn.Conv3d(channels, self.out_channels, 1)
if self.out_channels != channels else nn.Identity()
)
def forward(self, x, emb):
h = self.in_layers(x)
emb_out = self.emb_layers(emb).type(h.dtype)
while len(emb_out.shape) < len(h.shape):
emb_out = emb_out[..., None]
scale, shift = torch.chunk(emb_out, 2, dim=1)
h = self.out_norm(h) * (1 + scale) + shift
h = self.out_layers(h)
return self.skip(x) + h
class TemporalConv(nn.Module):
def __init__(self, channels, kernel_size=5):
super().__init__()
padding = kernel_size // 2
self.norm = _normalization(channels)
self.dwconv = nn.Conv3d(channels, channels,
kernel_size=(kernel_size, 1, 1),
padding=(padding, 0, 0),
groups=channels)
self.pwconv = nn.Conv3d(channels, channels, kernel_size=1)
nn.init.zeros_(self.pwconv.weight)
nn.init.zeros_(self.pwconv.bias)
def forward(self, x):
identity = x
h = self.norm(x)
h = F.silu(h)
h = self.dwconv(h)
h = self.pwconv(h)
return identity + h
class LatentResizer3D(nn.Module):
"""Pure-3D upscaler backbone with optional temporal chunking (LBH-123-AI architecture)."""
def __init__(self, in_channels=24, in_blocks=12, out_blocks=12,
channels=512, dropout=0.1, attn=False,
temporal_every=2, temporal_kernel=5):
super().__init__()
self.conv_in = nn.Conv3d(in_channels, channels, 3, padding=1)
embed_dim = 64
self.embed = nn.Sequential(
nn.Linear(1, embed_dim), nn.SiLU(), nn.Linear(embed_dim, embed_dim))
self.in_blocks = nn.ModuleList()
for b in range(in_blocks):
if (b == 1 or b == in_blocks - 1) and attn:
self.in_blocks.append(AttnBlock3D(channels))
self.in_blocks.append(ResBlockEmb3D(channels, embed_dim, dropout))
if temporal_every > 0 and b % temporal_every == 0:
self.in_blocks.append(TemporalConv(channels, temporal_kernel))
self.out_blocks = nn.ModuleList()
for b in range(out_blocks):
if (b == 1 or b == out_blocks - 1) and attn:
self.out_blocks.append(AttnBlock3D(channels))
self.out_blocks.append(ResBlockEmb3D(channels, embed_dim, dropout))
if temporal_every > 0 and b % temporal_every == 0:
self.out_blocks.append(TemporalConv(channels, temporal_kernel))
self.norm_out = _normalization(channels)
self.conv_out = nn.Conv3d(channels, in_channels, 3, padding=1)
def temporal_chunk_settings(self):
for block in self.in_blocks:
if isinstance(block, TemporalConv):
return 32, block.dwconv.weight.shape[2]
return 32, 0
def temporal_convolution_radius(self) -> int:
"""Return the convolution-only radius used to size split context.
GroupNorm uses statistics over the whole input segment, so this is
not an exact receptive field and splitting changes model output.
"""
radius = int(self.conv_in.kernel_size[0]) // 2
for block in (*self.in_blocks, *self.out_blocks):
if isinstance(block, TemporalConv):
radius += int(block.dwconv.kernel_size[0]) // 2
elif isinstance(block, ResBlockEmb3D):
radius += sum(
int(layer.kernel_size[0]) // 2
for layer in (*block.in_layers, *block.out_layers)
if isinstance(layer, nn.Conv3d)
)
radius += int(self.conv_out.kernel_size[0]) // 2
return radius
def temporal_window_budget(self, length):
chunk, overlap = self.temporal_chunk_settings()
return length if length <= chunk else min(length + 2 * overlap, chunk + 4 * overlap)
def forward(self, x, scale=None, target_size=None, enable_chunking=True):
if target_size is not None:
size = target_size
elif scale is not None:
size = tuple(int(round(s * scale)) for s in x.shape[-3:])
else:
return x
if size == x.shape[-3:]:
return x
B, C, T, H, W = x.shape
chunk, overlap = self.temporal_chunk_settings()
if not enable_chunking or T <= chunk:
return self._forward_seg(x, scale, size)
x_padded = F.pad(x, (0, 0, 0, 0, overlap, overlap), mode='replicate')
out_full = torch.zeros(B, C, T, size[-2], size[-1], device=x.device, dtype=x.dtype)
weight_full = torch.zeros(1, 1, T, 1, 1, device=x.device, dtype=x.dtype)
for seg_start, seg_end, out_start, out_end in _temporal_windows(T, chunk, overlap):
lo = out_start
hi = out_end + 2 * overlap
seg = x_padded[:, :, lo:hi].contiguous()
seg_size = (hi - lo, size[-2], size[-1])
seg_out = self._forward_seg(seg, scale, seg_size)
s0 = (out_start + overlap) - lo
s1 = s0 + (out_end - out_start)
valid_out = seg_out[:, :, s0:s1]
n_valid = out_end - out_start
weight = torch.ones(n_valid, device=x.device, dtype=x.dtype)
if seg_start > out_start:
blend_len = seg_start - out_start
weight[:blend_len] = torch.arange(1, blend_len + 1, device=x.device, dtype=x.dtype) / (blend_len + 1)
if out_end > seg_end:
blend_len = out_end - seg_end
weight[-blend_len:] = torch.arange(blend_len, 0, -1, device=x.device, dtype=x.dtype) / (blend_len + 1)
out_full[:, :, out_start:out_end] += valid_out * weight.view(1, 1, n_valid, 1, 1)
weight_full[:, :, out_start:out_end] += weight.view(1, 1, n_valid, 1, 1)
del seg, seg_out, valid_out
return out_full / weight_full.clamp(min=1e-8)
def _forward_seg(self, x, scale, size):
scale_emb = torch.tensor(
[scale - 1 if scale is not None else 0.0],
dtype=x.dtype, device=x.device).unsqueeze(0)
emb = self.embed(scale_emb)
x = self.conv_in(x)
for b in self.in_blocks:
if isinstance(b, ResBlockEmb3D):
x = b(x, emb.expand(x.shape[0], -1))
else:
x = b(x)
x = F.interpolate(x, size=size, mode="trilinear", align_corners=False)
for b in self.out_blocks:
if isinstance(b, ResBlockEmb3D):
x = b(x, emb.expand(x.shape[0], -1))
else:
x = b(x)
x = self.norm_out(x)
x = F.silu(x)
return self.conv_out(x)
_model_cache = {}
def _detect_arch(sd):
import re
cfg = {"in_channels": 24, "in_blocks": 12, "out_blocks": 12, "channels": 512,
"dropout": 0.1, "attn": False, "temporal_every": 2, "temporal_kernel": 5}
if 'conv_in.weight' in sd:
cfg["in_channels"] = sd['conv_in.weight'].shape[1]
cfg["channels"] = sd['conv_in.weight'].shape[0]
in_ids, out_ids, tin, tout = set(), set(), set(), set()
for k in sd:
m = re.match(r'in_blocks\.(\d+)\.in_layers\.', k)
if m:
in_ids.add(int(m.group(1)))
m = re.match(r'out_blocks\.(\d+)\.in_layers\.', k)
if m:
out_ids.add(int(m.group(1)))
m = re.match(r'in_blocks\.(\d+)\.dwconv\.weight', k)
if m:
tin.add(int(m.group(1)))
m = re.match(r'out_blocks\.(\d+)\.dwconv\.weight', k)
if m:
tout.add(int(m.group(1)))
if in_ids:
cfg["in_blocks"] = len(in_ids)
if out_ids:
cfg["out_blocks"] = len(out_ids)
if tin or tout:
cfg["temporal_every"] = 2
for k in sd:
if k.endswith('dwconv.weight'):
cfg["temporal_kernel"] = sd[k].shape[2]
break
else:
cfg["temporal_every"] = 0
cfg["attn"] = False
return cfg
def _normalize_checkpoint_dtype(state_dict):
weight = state_dict.get("conv_in.weight")
if weight is None:
raise ValueError("SelfLift: upscaler checkpoint is missing conv_in.weight")
dtype = weight.dtype
if str(dtype).startswith("torch.float8_"):
dtype = torch.bfloat16 if any(value.dtype == torch.bfloat16 for value in state_dict.values()) else torch.float16
if dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
raise ValueError(f"SelfLift: unsupported upscaler weight dtype {dtype}")
if any(value.is_floating_point() and value.dtype != dtype for value in state_dict.values()):
logging.info("SelfLift: normalizing mixed upscaler floating-point tensors to %s", dtype)
return {name: value.to(dtype=dtype) if value.is_floating_point() else value
for name, value in state_dict.items()}
def _load_model(model_name, device):
key = (model_name, str(device))
if key in _model_cache:
return _model_cache[key]
try:
path = folder_paths.get_full_path('latent_upscale_models', model_name)
except FileNotFoundError:
raise FileNotFoundError(f"latent upscaler model not found: {model_name} (place it under ComfyUI/models/latent_upscale_models/)")
import comfy.utils
sd = comfy.utils.load_torch_file(path)
if isinstance(sd, dict) and 'model' in sd:
sd = sd['model']
if any(k.startswith("upscaler.") for k in sd):
sd = {k[len("upscaler."):]: v for k, v in sd.items() if k.startswith("upscaler.")}
sd = _normalize_checkpoint_dtype(sd)
cfg = _detect_arch(sd)
with torch.device("meta"):
model = LatentResizer3D(
in_channels=cfg["in_channels"], in_blocks=cfg["in_blocks"], out_blocks=cfg["out_blocks"],
channels=cfg["channels"], dropout=cfg["dropout"], attn=cfg["attn"],
temporal_every=cfg["temporal_every"], temporal_kernel=cfg["temporal_kernel"],
)
model.load_state_dict(sd, strict=True, assign=True)
model = model.eval().requires_grad_(False)
patcher = comfy.model_patcher.CoreModelPatcher(
model, load_device=device,
offload_device=comfy.model_management.unet_offload_device())
_model_cache[key] = patcher
return patcher
def _inference_memory_required(
model,
z0_low,
out_hw,
enable_temporal_chunking=True,
temporal_length=None,
):
H, W = out_hw
T = int(temporal_length or z0_low.shape[2])
temporal_window = model.temporal_window_budget(T) if enable_temporal_chunking else T
feature_elements = z0_low.shape[0] * model.conv_in.out_channels * temporal_window * H * W
return feature_elements * model.conv_in.weight.element_size() * 8
def learned_latent_lift(
z0_low,
out_hw,
model_name,
device=None,
*,
enable_temporal_chunking=True,
force_unload=False,
temporal_split=None,
):
"""2D/3D learned upsample of the low-res clean endpoint to the target latent size.
z0_low: [B, 24, T, h, w] H3 video latent in VAE space. Returns [B, 24, T, H, W].
"""
H, W = out_hw
if device is None:
device = comfy.model_management.get_torch_device()
h, w = z0_low.shape[-2], z0_low.shape[-1]
scale = (H / h + W / w) / 2.0
log_memory("upscaler before_load", device)
patcher = _load_model(model_name, device)
model = patcher.model
length = int(z0_low.shape[2])
split = int(temporal_split or 0)
split_halo = 0
inference_length = length
if 0 < split < length:
candidate_halo = max(2, int(model.temporal_convolution_radius()))
candidate_length = max(split, length - split) + 2 * candidate_halo
# Keep the original model windowing when segment padding would process
# at least as many frames and add another normalization boundary.
if candidate_length < length:
split_halo = candidate_halo
inference_length = candidate_length
memory_required = _inference_memory_required(
model,
z0_low,
(H, W),
enable_temporal_chunking=enable_temporal_chunking,
temporal_length=inference_length,
)
chunk, overlap = model.temporal_chunk_settings()
identity = (H, W) == (h, w)
chunked = enable_temporal_chunking and not identity and inference_length > chunk
windows = list(_temporal_windows(inference_length, chunk, overlap)) if chunked else []
actual_window = max(end - start + 2 * overlap for _, _, start, end in windows) if chunked else inference_length
budget_window = model.temporal_window_budget(inference_length) if enable_temporal_chunking else inference_length
logging.info("[SelfLift upscaler] model=%s dtype=%s input=%s target_hw=%s "
"spatial_lift=(%.4f, %.4f) scale_embedding=%.4f mode=%s "
"chunk=%d overlap=%d windows=%d max_input_window=%d budget_window=%d "
"estimated_workspace=%.2f MiB",
model_name, model.conv_in.weight.dtype, tuple(z0_low.shape), (H, W),
H / h, W / w, scale - 1.0, "identity" if identity else "chunked" if chunked else "full",
chunk, overlap if chunked else 0, len(windows) if chunked else int(not identity),
actual_window if not identity else 0, budget_window, memory_required / 2**20)
try:
comfy.model_management.load_models_gpu([patcher], memory_required=memory_required)
log_memory("upscaler after_load", device)
dtype = model.conv_in.weight.dtype
mean = torch.tensor(LATENTS_MEAN, dtype=dtype, device=device).view(1, -1, 1, 1, 1)
std = torch.tensor(LATENTS_STD, dtype=dtype, device=device).view(1, -1, 1, 1, 1)
intermediate_device = comfy.model_management.intermediate_device()
with torch.no_grad():
if split_halo:
# GroupNorm still sees each segment separately. The halo
# supplies convolution context, not whole-clip statistics.
halo = split_halo
prefix = z0_low[:, :, :split].to(device=device, dtype=dtype)
prefix_length = int(prefix.shape[2])
padded_prefix = F.pad(
prefix,
(0, 0, 0, 0, halo, halo),
mode="replicate",
)
del prefix
padded_prefix = (padded_prefix - mean) / std
lifted_prefix = model(
padded_prefix,
scale=scale,
target_size=(prefix_length + 2 * halo, H, W),
enable_chunking=enable_temporal_chunking,
)[:, :, halo : halo + prefix_length]
del padded_prefix
# Stage each result on CPU before the next model invocation.
# This avoids retaining both large outputs on the GPU.
lifted_prefix = (lifted_prefix * std + mean).float().to("cpu")
suffix = z0_low[:, :, split:].to(device=device, dtype=dtype)
suffix_length = int(suffix.shape[2])
left_context = z0_low[:, :, max(0, split - halo) : split].to(
device=device, dtype=dtype
)
if int(left_context.shape[2]) < halo:
left_context = F.pad(
left_context,
(0, 0, 0, 0, halo - int(left_context.shape[2]), 0),
mode="replicate",
)
right_context = suffix[:, :, -1:].expand(
-1, -1, halo, -1, -1
)
padded_suffix = torch.cat(
(left_context, suffix, right_context),
dim=2,
)
del suffix, left_context, right_context
padded_suffix = (padded_suffix - mean) / std
lifted_suffix = model(
padded_suffix,
scale=scale,
target_size=(suffix_length + 2 * halo, H, W),
enable_chunking=enable_temporal_chunking,
)[:, :, halo : halo + suffix_length]
del padded_suffix
lifted_suffix = (lifted_suffix * std + mean).float().to("cpu")
out = torch.cat((lifted_prefix, lifted_suffix), dim=2).to(
intermediate_device
)
del lifted_prefix, lifted_suffix
logging.info(
"[SelfLift upscaler] continuation boundary token=%d uses "
"a real low-resolution left-context halo=%d; GroupNorm "
"statistics remain segment-local",
split,
halo,
)
else:
x = z0_low.to(device=device, dtype=dtype)
x = (x - mean) / std
out = model(
x,
scale=scale,
target_size=(z0_low.shape[2], H, W),
enable_chunking=enable_temporal_chunking,
)
out = (out * std + mean).float().to(intermediate_device)
log_memory("upscaler end", device)
return out
finally:
if force_unload:
comfy.model_management.unload_model_and_clones(
patcher,
unload_additional_models=False,
)
comfy.model_management.soft_empty_cache()