"""Experimental spatial tiling of H3 high-resolution model evaluations.""" from functools import partial import inspect import itertools import logging import math import torch import comfy.ldm.common_dit from comfy.ldm.minimax.model import PackedLayout import comfy.model_base import comfy.patcher_extension import comfy.sampler_helpers _H3_TILING_EVAL_COUNTER = itertools.count(1) def _cuda_memory_snapshot(device): try: device = torch.device(device) except (TypeError, RuntimeError): return None if device.type != "cuda" or not torch.cuda.is_available(): return None mib = 1024 ** 2 return { "allocated": torch.cuda.memory_allocated(device) / mib, "peak": torch.cuda.max_memory_allocated(device) / mib, "reserved": torch.cuda.memory_reserved(device) / mib, "peak_reserved": torch.cuda.max_memory_reserved(device) / mib, } def _reset_cuda_peak(device): try: device = torch.device(device) except (TypeError, RuntimeError): return if device.type == "cuda" and torch.cuda.is_available(): torch.cuda.reset_peak_memory_stats(device) def _log_cuda_memory(prefix, eval_id, device, tile_index=None, tile_total=None): stats = _cuda_memory_snapshot(device) if stats is None: return tile_suffix = "" if tile_index is not None: tile_suffix = f" tile={tile_index}/{tile_total}" logging.info( "%s eval=%d%s allocated=%.2f MiB peak=%.2f MiB " "reserved=%.2f MiB peak_reserved=%.2f MiB", prefix, eval_id, tile_suffix, stats["allocated"], stats["peak"], stats["reserved"], stats["peak_reserved"], ) def _regions(length, tile_count=2): patches = (length + 1) // 2 tile_count = max(1, min(tile_count, max(1, patches // 2))) if patches < 4 or tile_count <= 1: return [(0, length)] overlap = min(4, max(1, patches // 8)) regions = [] for index in range(tile_count): start = max(0, index * patches // tile_count - overlap) * 2 end = min(patches, (index + 1) * patches // tile_count + overlap) * 2 end = min(end, length) regions.append((start, end)) regions[-1] = (regions[-1][0], length) return regions def _packed_layout(signature, payload): options = {"keyframes": payload.get("keyframes"), "refs": payload.get("refs")} if "frame_count" in inspect.signature(PackedLayout).parameters: options["frame_count"] = payload.get("frame_count") return PackedLayout(*signature, **options) def _tile_payload(payload, context, video, audio, axis, start, end): height, width = video.shape[-2:] padded_height, padded_width = (height + 1) // 2 * 2, (width + 1) // 2 * 2 full_layout = payload.get("layout") signature = (context.shape[1], video.shape[2], padded_height, padded_width, audio.shape[-1]) if full_layout is None or full_layout.signature != signature: full_layout = _packed_layout(signature, payload) tiled = payload.copy() if payload.get("keyframes"): keyframes = [] for keyframe in payload["keyframes"]: latent = keyframe.get("latent") if latent is None: keyframes.append(dict(keyframe)) continue if latent.shape[-2:] != (height, width): raise ValueError( "SelfLift: tiled H3 keyframes must match the target latent " "height and width" ) region = latent.narrow(axis, start, end - start) keyframes.append( { **keyframe, "latent": comfy.ldm.common_dit.pad_to_patch_size( region, (1, 2, 2), ).contiguous(), } ) tiled["keyframes"] = keyframes tiled["cond_video_latents"] = [ keyframe["latent"] for keyframe in keyframes if keyframe.get("latent") is not None ] + [ reference["latent"] for reference in payload.get("refs") or [] if reference.get("latent") is not None ] tile_height = end - start if axis == 3 else height tile_width = end - start if axis == 4 else width layout = _packed_layout((context.shape[1], video.shape[2], (tile_height + 1) // 2 * 2, (tile_width + 1) // 2 * 2, audio.shape[-1]), tiled) for (source_start, source_end, kind), (target_start, target_end, _) in zip(full_layout.segments, layout.segments): positions = full_layout.position_ids[source_start:source_end] if kind in ("cond", "video"): positions = positions.reshape(-1, padded_height // 2, padded_width // 2, 3) positions = positions.narrow(axis - 2, start // 2, (end - start + 1) // 2).reshape(-1, 3) layout.position_ids[target_start:target_end].copy_(positions) tiled["layout"] = layout return tiled def _tile_denoise_mask(mask, video, axis, start, end): """Crop a spatial H3 denoise mask to the same region as its video tile.""" if mask is None or not isinstance(mask, torch.Tensor): return mask if mask.ndim != video.ndim: raise ValueError( "SelfLift: tiled H3 video denoise mask must have the same number " "of dimensions as the video latent" ) if mask.shape[2] != video.shape[2] or any( mask.shape[spatial_axis] not in (1, video.shape[spatial_axis]) for spatial_axis in (3, 4) ): raise ValueError( "SelfLift: tiled H3 video denoise mask does not match the target " "latent temporal/spatial shape" ) if mask.shape[axis] == 1: return mask return mask.narrow(axis, start, end - start).contiguous() def _tiled_forward(executor, streams, timestep, context, transformer_options, minimax_payload=None, n_tiles=2, plan=None, **kwargs): video, audio = streams axis = 3 if (video.shape[3] + 1) // 2 >= (video.shape[4] + 1) // 2 else 4 length = video.shape[axis] regions = _regions(length, plan['tiles'] if plan is not None else n_tiles) eval_id = next(_H3_TILING_EVAL_COUNTER) axis_name = "H" if axis == 3 else "W" _reset_cuda_peak(video.device) logging.info( "[H3 tiled forward] eval=%d axis=%s tiles=%d full_video=%s audio=%s", eval_id, axis_name, len(regions), tuple(video.shape), tuple(audio.shape), ) _log_cuda_memory("[H3 tiled forward memory/start]", eval_id, video.device) if len(regions) == 1: logging.info( "[H3 tiled forward] eval=%d bypassed effective_tiles=1", eval_id, ) result = executor( streams, timestep, context, transformer_options, minimax_payload=minimax_payload, **kwargs, ) _log_cuda_memory("[H3 tiled forward memory/bypass]", eval_id, video.device) return result if kwargs.get("control") is not None: raise ValueError("SelfLift: high-resolution H3 tiling does not support ControlNet") # Keep the stitched accumulator on CPU so previous tiles do not remain on # the accelerator while the next tile is evaluated. video_output = torch.zeros(video.shape, dtype=torch.float32, device="cpu") audio_output = torch.zeros(audio.shape, dtype=torch.float32, device="cpu") audio_weight_total = 0.0 weights = torch.zeros(length, dtype=torch.float32, device="cpu") for index, (start, end) in enumerate(regions): tile = video.narrow(axis, start, end - start).contiguous() logging.info( "[H3 tile] eval=%d tile=%d/%d axis=%s range=%d:%d tile_video=%s full_audio=%s", eval_id, index + 1, len(regions), axis_name, start, end, tuple(tile.shape), tuple(audio.shape), ) payload = _tile_payload(minimax_payload or {}, context, video, audio, axis, start, end) tile_kwargs = kwargs.copy() tile_kwargs["denoise_mask"] = _tile_denoise_mask( tile_kwargs.get("denoise_mask"), video, axis, start, end, ) predicted_video, predicted_audio = executor( [tile, audio], timestep, context, transformer_options.copy(), minimax_payload=payload, **tile_kwargs, ) _log_cuda_memory( "[H3 tile memory/after-forward]", eval_id, video.device, tile_index=index + 1, tile_total=len(regions), ) window = torch.ones(end - start, device=video.device, dtype=torch.float32) if index > 0: overlap = min(end, regions[index - 1][1]) - start window[:overlap] *= (torch.arange(overlap, device=video.device, dtype=torch.float32) + 0.5) / overlap if index + 1 < len(regions): overlap = end - regions[index + 1][0] window[-overlap:] *= 1.0 - (torch.arange(overlap, device=video.device, dtype=torch.float32) + 0.5) / overlap predicted_video = predicted_video.float().cpu() window = window.cpu() weights[start:end].add_(window) audio_weight = float(window.sum().item()) audio_output.add_(predicted_audio.float().cpu(), alpha=audio_weight) audio_weight_total += audio_weight window_shape = [1] * video.ndim window_shape[axis] = end - start video_output.narrow(axis, start, end - start).addcmul_(predicted_video.float(), window.view(window_shape)) del tile, payload, predicted_video, predicted_audio, window _log_cuda_memory( "[H3 tile memory/after-release]", eval_id, video.device, tile_index=index + 1, tile_total=len(regions), ) window_shape[axis] = length video_output.div_(weights.view(window_shape)) audio_output.div_(audio_weight_total) logging.info( "[H3 tiled forward done] eval=%d axis=%s tiles=%d", eval_id, axis_name, len(regions), ) _log_cuda_memory("[H3 tiled forward memory/done]", eval_id, video.device) return [ video_output.to(device=video.device, dtype=video.dtype), audio_output.to(device=audio.device, dtype=audio.dtype), ] def _condition_elements(condition, tile_height, tile_width, channels): text = condition.get("cross_attn") elements = text.shape[-2] * channels * 4 if text is not None else 0 for keyframe in condition.get("minimax_keyframes") or []: shape = keyframe["latent"].shape elements += shape[1] * shape[2] * tile_height * tile_width for reference in condition.get("minimax_refs") or []: latent = reference.get("latent") if latent is not None: shape = latent.shape elements += shape[1] * shape[2] * ((shape[3] + 1) // 2 * 2) * ((shape[4] + 1) // 2 * 2) audio = reference.get("audio_latent") if audio is not None: elements += math.prod(audio.shape[1:]) return elements def _budget(model, noise_shape, conds, latent_shapes, regions, axis): video_shape, audio_shape = latent_shapes full_elements = math.prod(video_shape[1:]) + math.prod(audio_shape[1:]) tile_shape = list(video_shape) tile_shape[axis] = max(end - start for start, end in regions) tile_shape[3] = (tile_shape[3] + 1) // 2 * 2 tile_shape[4] = (tile_shape[4] + 1) // 2 * 2 tile_elements = math.prod(tile_shape[1:]) + math.prod(audio_shape[1:]) condition_elements = max((_condition_elements(condition, tile_shape[3], tile_shape[4], video_shape[1]) for group in conds.values() for condition in (group or [])), default=0) buffer_bytes = full_elements * 4 * 8 bytes_per_element = model.model.memory_required((1, 1, 1)) budget_elements = tile_elements + condition_elements + math.ceil(buffer_bytes / bytes_per_element) budget_shape = (noise_shape[0], 1, budget_elements) preferred, minimum = comfy.sampler_helpers.estimate_memory(model, budget_shape, conds) return budget_shape, tuple(tile_shape), buffer_bytes, preferred, minimum def _prepare_tiled_sampling(executor, model, noise_shape, conds, model_options=None, force_full_load=False, force_offload=False, *, latent_shapes, plan=None): video_shape, audio_shape = latent_shapes axis = 3 if (video_shape[3] + 1) // 2 >= (video_shape[4] + 1) // 2 else 4 full_elements = math.prod(video_shape[1:]) + math.prod(audio_shape[1:]) if tuple(noise_shape) != (video_shape[0], 1, full_elements): raise ValueError("SelfLift: tiled memory planning received a different latent shape than the sampling input") count = 2 requested = int(plan.get("tiles", 0) or 0) if plan is not None else 0 if requested > 0: count = max(2, min(8, requested)) regions = _regions(video_shape[axis], count) count = len(regions) plan["tiles"] = count logging.info( "[H3 tiling plan] axis=%s tiles=%d mode=manual", "H" if axis == 3 else "W", count, ) elif plan is not None: available = _available_workspace(model) for count in range(1, 9): regions = _regions(video_shape[axis], count) _, _, _, _, minimum = _budget(model, noise_shape, conds, latent_shapes, regions, axis) if minimum <= available: break count = len(regions) plan['tiles'] = count logging.info("[SelfLift tiling plan] axis=%s tiles=%d target=%.2f MiB estimate_fits=%s", 'H' if axis == 3 else 'W', plan['tiles'], available / 2**20, minimum <= available) regions = _regions(video_shape[axis], count) if len(regions) == 1 or force_offload: return executor(model, noise_shape, conds, model_options=model_options, force_full_load=force_full_load, force_offload=force_offload) budget_shape, tile_shape, buffer_bytes, preferred, minimum = _budget( model, noise_shape, conds, latent_shapes, regions, axis) logging.info("[SelfLift tiling memory] largest_tile=%s full_audio=%s full_state_buffers=%.2f MiB " "minimum=%.2f MiB preferred=%.2f MiB (ComfyUI estimates; additional models and reserves excluded)", tuple(tile_shape), audio_shape, buffer_bytes * noise_shape[0] / 2**20, minimum / 2**20, preferred / 2**20) return executor(model, budget_shape, conds, model_options=model_options, force_full_load=force_full_load, force_offload=force_offload) def _available_workspace(model): manager = comfy.model_management free = manager.get_free_memory(model.load_device) reclaimable = 0 seen = set() for loaded in manager.loaded_models(): patcher = loaded if callable(getattr(loaded, "loaded_size", None)) else loaded.model load_device = getattr(patcher, "load_device", None) identity = id(patcher.model) if load_device == model.load_device and identity not in seen: seen.add(identity) size_fn = getattr(patcher, "loaded_size", None) if callable(size_fn): reclaimable += size_fn() pool = min(manager.get_total_memory(model.load_device), free + reclaimable) weights = min(model.model_size(), pool * manager.MIN_WEIGHT_MEMORY_RATIO) available = max(0, pool - weights - manager.minimum_inference_memory()) logging.info("[SelfLift tiling capacity] free=%.2f MiB reclaimable_weights=%.2f MiB " "weight_allowance=%.2f MiB workspace=%.2f MiB", free / 2**20, reclaimable / 2**20, weights / 2**20, available / 2**20) return available def tiled_model(model, latent_shapes, tile_count: int = 0): if not isinstance(model.model, comfy.model_base.MiniMaxH3): raise ValueError("SelfLift: high-resolution tiling requires a MiniMax H3 model") if len(latent_shapes) != 2 or len(latent_shapes[0]) != 5 or len(latent_shapes[1]) != 4: raise ValueError("SelfLift: high-resolution tiling requires H3 video and audio latent streams") requested_tiles = int(tile_count or 0) if requested_tiles not in (0, 1) and not 2 <= requested_tiles <= 8: raise ValueError( "H3 high-resolution tile_count must be 0/auto or between 2 and 8" ) plan = {} if requested_tiles <= 1 else {"tiles": requested_tiles} patched = model.clone() patched.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, "selflift_high_resolution_tiling", partial(_tiled_forward, plan=plan)) patched.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.PREPARE_SAMPLING, "selflift_high_resolution_tiling", partial(_prepare_tiled_sampling, latent_shapes=tuple(tuple(shape) for shape in latent_shapes), plan=plan)) return patched