"""Learned latent-space lifter for MiniMax H3 (3D-conv upscaler). Bundles the inference architecture of the Minimax H3 Latent Upscaler (LBH-123-AI/Minimax_h3_latent_Upscaler, architecture drawing on the LTX 2.3 spatial upscaler) as an optional experimental replacement for SelfLift-zero's nearest-neighbor latent lift. This external model is not the SelfLift-rich lifter described by the paper. The checkpoint is expected under ComfyUI/models/latent_upscale_models/. """ import logging import os import torch import torch.nn as nn import torch.nn.functional as F import comfy.model_management import comfy.model_patcher import folder_paths from comfy.ldm.minimax.vae import LATENTS_MEAN, LATENTS_STD from .diagnostics import log_memory def _normalization(channels): return nn.GroupNorm(32, channels) def _zero_module(module): for p in module.parameters(): p.detach().zero_() return module def _temporal_windows(length, chunk, overlap): for start in range(0, length, chunk): end = min(length, start + chunk) yield start, end, max(0, start - overlap), min(length, end + overlap) class AttnBlock3D(nn.Module): def __init__(self, in_channels): super().__init__() self.norm = _normalization(in_channels) self.q = nn.Conv3d(in_channels, in_channels, 1) self.k = nn.Conv3d(in_channels, in_channels, 1) self.v = nn.Conv3d(in_channels, in_channels, 1) self.proj_out = nn.Conv3d(in_channels, in_channels, 1) def forward(self, x): h = self.norm(x) q = self.q(h).flatten(2).movedim(-1, 1).unsqueeze(1) k = self.k(h).flatten(2).movedim(-1, 1).unsqueeze(1) v = self.v(h).flatten(2).movedim(-1, 1).unsqueeze(1) h = F.scaled_dot_product_attention(q, k, v) h = h.squeeze(1).movedim(1, -1).reshape(x.shape) return x + self.proj_out(h) class ResBlockEmb3D(nn.Module): def __init__(self, channels, emb_channels, dropout=0, out_channels=None): super().__init__() self.out_channels = out_channels or channels self.in_layers = nn.Sequential( _normalization(channels), nn.SiLU(), nn.Conv3d(channels, self.out_channels, 3, padding=1), ) self.emb_layers = nn.Sequential( nn.SiLU(), nn.Linear(emb_channels, 2 * self.out_channels), ) self.out_norm = _normalization(self.out_channels) self.out_layers = nn.Sequential( nn.SiLU(), nn.Dropout(p=dropout), _zero_module(nn.Conv3d(self.out_channels, self.out_channels, 3, padding=1)), ) self.skip = ( nn.Conv3d(channels, self.out_channels, 1) if self.out_channels != channels else nn.Identity() ) def forward(self, x, emb): h = self.in_layers(x) emb_out = self.emb_layers(emb).type(h.dtype) while len(emb_out.shape) < len(h.shape): emb_out = emb_out[..., None] scale, shift = torch.chunk(emb_out, 2, dim=1) h = self.out_norm(h) * (1 + scale) + shift h = self.out_layers(h) return self.skip(x) + h class TemporalConv(nn.Module): def __init__(self, channels, kernel_size=5): super().__init__() padding = kernel_size // 2 self.norm = _normalization(channels) self.dwconv = nn.Conv3d(channels, channels, kernel_size=(kernel_size, 1, 1), padding=(padding, 0, 0), groups=channels) self.pwconv = nn.Conv3d(channels, channels, kernel_size=1) nn.init.zeros_(self.pwconv.weight) nn.init.zeros_(self.pwconv.bias) def forward(self, x): identity = x h = self.norm(x) h = F.silu(h) h = self.dwconv(h) h = self.pwconv(h) return identity + h class LatentResizer3D(nn.Module): """Pure-3D upscaler backbone with optional temporal chunking (LBH-123-AI architecture).""" def __init__(self, in_channels=24, in_blocks=12, out_blocks=12, channels=512, dropout=0.1, attn=False, temporal_every=2, temporal_kernel=5): super().__init__() self.conv_in = nn.Conv3d(in_channels, channels, 3, padding=1) embed_dim = 64 self.embed = nn.Sequential( nn.Linear(1, embed_dim), nn.SiLU(), nn.Linear(embed_dim, embed_dim)) self.in_blocks = nn.ModuleList() for b in range(in_blocks): if (b == 1 or b == in_blocks - 1) and attn: self.in_blocks.append(AttnBlock3D(channels)) self.in_blocks.append(ResBlockEmb3D(channels, embed_dim, dropout)) if temporal_every > 0 and b % temporal_every == 0: self.in_blocks.append(TemporalConv(channels, temporal_kernel)) self.out_blocks = nn.ModuleList() for b in range(out_blocks): if (b == 1 or b == out_blocks - 1) and attn: self.out_blocks.append(AttnBlock3D(channels)) self.out_blocks.append(ResBlockEmb3D(channels, embed_dim, dropout)) if temporal_every > 0 and b % temporal_every == 0: self.out_blocks.append(TemporalConv(channels, temporal_kernel)) self.norm_out = _normalization(channels) self.conv_out = nn.Conv3d(channels, in_channels, 3, padding=1) def temporal_chunk_settings(self): for block in self.in_blocks: if isinstance(block, TemporalConv): return 32, block.dwconv.weight.shape[2] return 32, 0 def temporal_convolution_radius(self) -> int: """Return the convolution-only radius used to size split context. GroupNorm uses statistics over the whole input segment, so this is not an exact receptive field and splitting changes model output. """ radius = int(self.conv_in.kernel_size[0]) // 2 for block in (*self.in_blocks, *self.out_blocks): if isinstance(block, TemporalConv): radius += int(block.dwconv.kernel_size[0]) // 2 elif isinstance(block, ResBlockEmb3D): radius += sum( int(layer.kernel_size[0]) // 2 for layer in (*block.in_layers, *block.out_layers) if isinstance(layer, nn.Conv3d) ) radius += int(self.conv_out.kernel_size[0]) // 2 return radius def temporal_window_budget(self, length): chunk, overlap = self.temporal_chunk_settings() return length if length <= chunk else min(length + 2 * overlap, chunk + 4 * overlap) def forward(self, x, scale=None, target_size=None, enable_chunking=True): if target_size is not None: size = target_size elif scale is not None: size = tuple(int(round(s * scale)) for s in x.shape[-3:]) else: return x if size == x.shape[-3:]: return x B, C, T, H, W = x.shape chunk, overlap = self.temporal_chunk_settings() if not enable_chunking or T <= chunk: return self._forward_seg(x, scale, size) x_padded = F.pad(x, (0, 0, 0, 0, overlap, overlap), mode='replicate') out_full = torch.zeros(B, C, T, size[-2], size[-1], device=x.device, dtype=x.dtype) weight_full = torch.zeros(1, 1, T, 1, 1, device=x.device, dtype=x.dtype) for seg_start, seg_end, out_start, out_end in _temporal_windows(T, chunk, overlap): lo = out_start hi = out_end + 2 * overlap seg = x_padded[:, :, lo:hi].contiguous() seg_size = (hi - lo, size[-2], size[-1]) seg_out = self._forward_seg(seg, scale, seg_size) s0 = (out_start + overlap) - lo s1 = s0 + (out_end - out_start) valid_out = seg_out[:, :, s0:s1] n_valid = out_end - out_start weight = torch.ones(n_valid, device=x.device, dtype=x.dtype) if seg_start > out_start: blend_len = seg_start - out_start weight[:blend_len] = torch.arange(1, blend_len + 1, device=x.device, dtype=x.dtype) / (blend_len + 1) if out_end > seg_end: blend_len = out_end - seg_end weight[-blend_len:] = torch.arange(blend_len, 0, -1, device=x.device, dtype=x.dtype) / (blend_len + 1) out_full[:, :, out_start:out_end] += valid_out * weight.view(1, 1, n_valid, 1, 1) weight_full[:, :, out_start:out_end] += weight.view(1, 1, n_valid, 1, 1) del seg, seg_out, valid_out return out_full / weight_full.clamp(min=1e-8) def _forward_seg(self, x, scale, size): scale_emb = torch.tensor( [scale - 1 if scale is not None else 0.0], dtype=x.dtype, device=x.device).unsqueeze(0) emb = self.embed(scale_emb) x = self.conv_in(x) for b in self.in_blocks: if isinstance(b, ResBlockEmb3D): x = b(x, emb.expand(x.shape[0], -1)) else: x = b(x) x = F.interpolate(x, size=size, mode="trilinear", align_corners=False) for b in self.out_blocks: if isinstance(b, ResBlockEmb3D): x = b(x, emb.expand(x.shape[0], -1)) else: x = b(x) x = self.norm_out(x) x = F.silu(x) return self.conv_out(x) _model_cache = {} def _detect_arch(sd): import re cfg = {"in_channels": 24, "in_blocks": 12, "out_blocks": 12, "channels": 512, "dropout": 0.1, "attn": False, "temporal_every": 2, "temporal_kernel": 5} if 'conv_in.weight' in sd: cfg["in_channels"] = sd['conv_in.weight'].shape[1] cfg["channels"] = sd['conv_in.weight'].shape[0] in_ids, out_ids, tin, tout = set(), set(), set(), set() for k in sd: m = re.match(r'in_blocks\.(\d+)\.in_layers\.', k) if m: in_ids.add(int(m.group(1))) m = re.match(r'out_blocks\.(\d+)\.in_layers\.', k) if m: out_ids.add(int(m.group(1))) m = re.match(r'in_blocks\.(\d+)\.dwconv\.weight', k) if m: tin.add(int(m.group(1))) m = re.match(r'out_blocks\.(\d+)\.dwconv\.weight', k) if m: tout.add(int(m.group(1))) if in_ids: cfg["in_blocks"] = len(in_ids) if out_ids: cfg["out_blocks"] = len(out_ids) if tin or tout: cfg["temporal_every"] = 2 for k in sd: if k.endswith('dwconv.weight'): cfg["temporal_kernel"] = sd[k].shape[2] break else: cfg["temporal_every"] = 0 cfg["attn"] = False return cfg def _normalize_checkpoint_dtype(state_dict): weight = state_dict.get("conv_in.weight") if weight is None: raise ValueError("SelfLift: upscaler checkpoint is missing conv_in.weight") dtype = weight.dtype if str(dtype).startswith("torch.float8_"): dtype = torch.bfloat16 if any(value.dtype == torch.bfloat16 for value in state_dict.values()) else torch.float16 if dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64): raise ValueError(f"SelfLift: unsupported upscaler weight dtype {dtype}") if any(value.is_floating_point() and value.dtype != dtype for value in state_dict.values()): logging.info("SelfLift: normalizing mixed upscaler floating-point tensors to %s", dtype) return {name: value.to(dtype=dtype) if value.is_floating_point() else value for name, value in state_dict.items()} def _load_model(model_name, device): key = (model_name, str(device)) if key in _model_cache: return _model_cache[key] try: path = folder_paths.get_full_path('latent_upscale_models', model_name) except FileNotFoundError: raise FileNotFoundError(f"latent upscaler model not found: {model_name} (place it under ComfyUI/models/latent_upscale_models/)") import comfy.utils sd = comfy.utils.load_torch_file(path) if isinstance(sd, dict) and 'model' in sd: sd = sd['model'] if any(k.startswith("upscaler.") for k in sd): sd = {k[len("upscaler."):]: v for k, v in sd.items() if k.startswith("upscaler.")} sd = _normalize_checkpoint_dtype(sd) cfg = _detect_arch(sd) with torch.device("meta"): model = LatentResizer3D( in_channels=cfg["in_channels"], in_blocks=cfg["in_blocks"], out_blocks=cfg["out_blocks"], channels=cfg["channels"], dropout=cfg["dropout"], attn=cfg["attn"], temporal_every=cfg["temporal_every"], temporal_kernel=cfg["temporal_kernel"], ) model.load_state_dict(sd, strict=True, assign=True) model = model.eval().requires_grad_(False) patcher = comfy.model_patcher.CoreModelPatcher( model, load_device=device, offload_device=comfy.model_management.unet_offload_device()) _model_cache[key] = patcher return patcher def _inference_memory_required( model, z0_low, out_hw, enable_temporal_chunking=True, temporal_length=None, ): H, W = out_hw T = int(temporal_length or z0_low.shape[2]) temporal_window = model.temporal_window_budget(T) if enable_temporal_chunking else T feature_elements = z0_low.shape[0] * model.conv_in.out_channels * temporal_window * H * W return feature_elements * model.conv_in.weight.element_size() * 8 def learned_latent_lift( z0_low, out_hw, model_name, device=None, *, enable_temporal_chunking=True, force_unload=False, temporal_split=None, ): """2D/3D learned upsample of the low-res clean endpoint to the target latent size. z0_low: [B, 24, T, h, w] H3 video latent in VAE space. Returns [B, 24, T, H, W]. """ H, W = out_hw if device is None: device = comfy.model_management.get_torch_device() h, w = z0_low.shape[-2], z0_low.shape[-1] scale = (H / h + W / w) / 2.0 log_memory("upscaler before_load", device) patcher = _load_model(model_name, device) model = patcher.model length = int(z0_low.shape[2]) split = int(temporal_split or 0) split_halo = 0 inference_length = length if 0 < split < length: candidate_halo = max(2, int(model.temporal_convolution_radius())) candidate_length = max(split, length - split) + 2 * candidate_halo # Keep the original model windowing when segment padding would process # at least as many frames and add another normalization boundary. if candidate_length < length: split_halo = candidate_halo inference_length = candidate_length memory_required = _inference_memory_required( model, z0_low, (H, W), enable_temporal_chunking=enable_temporal_chunking, temporal_length=inference_length, ) chunk, overlap = model.temporal_chunk_settings() identity = (H, W) == (h, w) chunked = enable_temporal_chunking and not identity and inference_length > chunk windows = list(_temporal_windows(inference_length, chunk, overlap)) if chunked else [] actual_window = max(end - start + 2 * overlap for _, _, start, end in windows) if chunked else inference_length budget_window = model.temporal_window_budget(inference_length) if enable_temporal_chunking else inference_length logging.info("[SelfLift upscaler] model=%s dtype=%s input=%s target_hw=%s " "spatial_lift=(%.4f, %.4f) scale_embedding=%.4f mode=%s " "chunk=%d overlap=%d windows=%d max_input_window=%d budget_window=%d " "estimated_workspace=%.2f MiB", model_name, model.conv_in.weight.dtype, tuple(z0_low.shape), (H, W), H / h, W / w, scale - 1.0, "identity" if identity else "chunked" if chunked else "full", chunk, overlap if chunked else 0, len(windows) if chunked else int(not identity), actual_window if not identity else 0, budget_window, memory_required / 2**20) try: comfy.model_management.load_models_gpu([patcher], memory_required=memory_required) log_memory("upscaler after_load", device) dtype = model.conv_in.weight.dtype mean = torch.tensor(LATENTS_MEAN, dtype=dtype, device=device).view(1, -1, 1, 1, 1) std = torch.tensor(LATENTS_STD, dtype=dtype, device=device).view(1, -1, 1, 1, 1) intermediate_device = comfy.model_management.intermediate_device() with torch.no_grad(): if split_halo: # GroupNorm still sees each segment separately. The halo # supplies convolution context, not whole-clip statistics. halo = split_halo prefix = z0_low[:, :, :split].to(device=device, dtype=dtype) prefix_length = int(prefix.shape[2]) padded_prefix = F.pad( prefix, (0, 0, 0, 0, halo, halo), mode="replicate", ) del prefix padded_prefix = (padded_prefix - mean) / std lifted_prefix = model( padded_prefix, scale=scale, target_size=(prefix_length + 2 * halo, H, W), enable_chunking=enable_temporal_chunking, )[:, :, halo : halo + prefix_length] del padded_prefix # Stage each result on CPU before the next model invocation. # This avoids retaining both large outputs on the GPU. lifted_prefix = (lifted_prefix * std + mean).float().to("cpu") suffix = z0_low[:, :, split:].to(device=device, dtype=dtype) suffix_length = int(suffix.shape[2]) left_context = z0_low[:, :, max(0, split - halo) : split].to( device=device, dtype=dtype ) if int(left_context.shape[2]) < halo: left_context = F.pad( left_context, (0, 0, 0, 0, halo - int(left_context.shape[2]), 0), mode="replicate", ) right_context = suffix[:, :, -1:].expand( -1, -1, halo, -1, -1 ) padded_suffix = torch.cat( (left_context, suffix, right_context), dim=2, ) del suffix, left_context, right_context padded_suffix = (padded_suffix - mean) / std lifted_suffix = model( padded_suffix, scale=scale, target_size=(suffix_length + 2 * halo, H, W), enable_chunking=enable_temporal_chunking, )[:, :, halo : halo + suffix_length] del padded_suffix lifted_suffix = (lifted_suffix * std + mean).float().to("cpu") out = torch.cat((lifted_prefix, lifted_suffix), dim=2).to( intermediate_device ) del lifted_prefix, lifted_suffix logging.info( "[SelfLift upscaler] continuation boundary token=%d uses " "a real low-resolution left-context halo=%d; GroupNorm " "statistics remain segment-local", split, halo, ) else: x = z0_low.to(device=device, dtype=dtype) x = (x - mean) / std out = model( x, scale=scale, target_size=(z0_low.shape[2], H, W), enable_chunking=enable_temporal_chunking, ) out = (out * std + mean).float().to(intermediate_device) log_memory("upscaler end", device) return out finally: if force_unload: comfy.model_management.unload_model_and_clones( patcher, unload_additional_models=False, ) comfy.model_management.soft_empty_cache()