Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
403 lines
17 KiB
Python
403 lines
17 KiB
Python
"""Experimental spatial tiling of H3 high-resolution model evaluations."""
|
|
|
|
from functools import partial
|
|
import inspect
|
|
import itertools
|
|
import logging
|
|
import math
|
|
|
|
import torch
|
|
|
|
import comfy.ldm.common_dit
|
|
from comfy.ldm.minimax.model import PackedLayout
|
|
import comfy.model_base
|
|
import comfy.patcher_extension
|
|
import comfy.sampler_helpers
|
|
|
|
|
|
_H3_TILING_EVAL_COUNTER = itertools.count(1)
|
|
|
|
|
|
def _cuda_memory_snapshot(device):
|
|
try:
|
|
device = torch.device(device)
|
|
except (TypeError, RuntimeError):
|
|
return None
|
|
if device.type != "cuda" or not torch.cuda.is_available():
|
|
return None
|
|
mib = 1024 ** 2
|
|
return {
|
|
"allocated": torch.cuda.memory_allocated(device) / mib,
|
|
"peak": torch.cuda.max_memory_allocated(device) / mib,
|
|
"reserved": torch.cuda.memory_reserved(device) / mib,
|
|
"peak_reserved": torch.cuda.max_memory_reserved(device) / mib,
|
|
}
|
|
|
|
|
|
def _reset_cuda_peak(device):
|
|
try:
|
|
device = torch.device(device)
|
|
except (TypeError, RuntimeError):
|
|
return
|
|
if device.type == "cuda" and torch.cuda.is_available():
|
|
torch.cuda.reset_peak_memory_stats(device)
|
|
|
|
|
|
def _log_cuda_memory(prefix, eval_id, device, tile_index=None, tile_total=None):
|
|
stats = _cuda_memory_snapshot(device)
|
|
if stats is None:
|
|
return
|
|
tile_suffix = ""
|
|
if tile_index is not None:
|
|
tile_suffix = f" tile={tile_index}/{tile_total}"
|
|
logging.info(
|
|
"%s eval=%d%s allocated=%.2f MiB peak=%.2f MiB "
|
|
"reserved=%.2f MiB peak_reserved=%.2f MiB",
|
|
prefix,
|
|
eval_id,
|
|
tile_suffix,
|
|
stats["allocated"],
|
|
stats["peak"],
|
|
stats["reserved"],
|
|
stats["peak_reserved"],
|
|
)
|
|
|
|
|
|
def _regions(length, tile_count=2):
|
|
patches = (length + 1) // 2
|
|
tile_count = max(1, min(tile_count, max(1, patches // 2)))
|
|
if patches < 4 or tile_count <= 1:
|
|
return [(0, length)]
|
|
overlap = min(4, max(1, patches // 8))
|
|
regions = []
|
|
for index in range(tile_count):
|
|
start = max(0, index * patches // tile_count - overlap) * 2
|
|
end = min(patches, (index + 1) * patches // tile_count + overlap) * 2
|
|
end = min(end, length)
|
|
regions.append((start, end))
|
|
regions[-1] = (regions[-1][0], length)
|
|
return regions
|
|
|
|
|
|
def _packed_layout(signature, payload):
|
|
options = {"keyframes": payload.get("keyframes"), "refs": payload.get("refs")}
|
|
if "frame_count" in inspect.signature(PackedLayout).parameters:
|
|
options["frame_count"] = payload.get("frame_count")
|
|
return PackedLayout(*signature, **options)
|
|
|
|
|
|
def _tile_payload(payload, context, video, audio, axis, start, end):
|
|
height, width = video.shape[-2:]
|
|
padded_height, padded_width = (height + 1) // 2 * 2, (width + 1) // 2 * 2
|
|
full_layout = payload.get("layout")
|
|
signature = (context.shape[1], video.shape[2], padded_height, padded_width, audio.shape[-1])
|
|
if full_layout is None or full_layout.signature != signature:
|
|
full_layout = _packed_layout(signature, payload)
|
|
tiled = payload.copy()
|
|
if payload.get("keyframes"):
|
|
keyframes = []
|
|
for keyframe in payload["keyframes"]:
|
|
latent = keyframe.get("latent")
|
|
if latent is None:
|
|
keyframes.append(dict(keyframe))
|
|
continue
|
|
if latent.shape[-2:] != (height, width):
|
|
raise ValueError(
|
|
"SelfLift: tiled H3 keyframes must match the target latent "
|
|
"height and width"
|
|
)
|
|
region = latent.narrow(axis, start, end - start)
|
|
keyframes.append(
|
|
{
|
|
**keyframe,
|
|
"latent": comfy.ldm.common_dit.pad_to_patch_size(
|
|
region,
|
|
(1, 2, 2),
|
|
).contiguous(),
|
|
}
|
|
)
|
|
tiled["keyframes"] = keyframes
|
|
tiled["cond_video_latents"] = [
|
|
keyframe["latent"]
|
|
for keyframe in keyframes
|
|
if keyframe.get("latent") is not None
|
|
] + [
|
|
reference["latent"]
|
|
for reference in payload.get("refs") or []
|
|
if reference.get("latent") is not None
|
|
]
|
|
tile_height = end - start if axis == 3 else height
|
|
tile_width = end - start if axis == 4 else width
|
|
layout = _packed_layout((context.shape[1], video.shape[2], (tile_height + 1) // 2 * 2,
|
|
(tile_width + 1) // 2 * 2, audio.shape[-1]), tiled)
|
|
for (source_start, source_end, kind), (target_start, target_end, _) in zip(full_layout.segments, layout.segments):
|
|
positions = full_layout.position_ids[source_start:source_end]
|
|
if kind in ("cond", "video"):
|
|
positions = positions.reshape(-1, padded_height // 2, padded_width // 2, 3)
|
|
positions = positions.narrow(axis - 2, start // 2, (end - start + 1) // 2).reshape(-1, 3)
|
|
layout.position_ids[target_start:target_end].copy_(positions)
|
|
tiled["layout"] = layout
|
|
return tiled
|
|
|
|
|
|
def _tile_denoise_mask(mask, video, axis, start, end):
|
|
"""Crop a spatial H3 denoise mask to the same region as its video tile."""
|
|
if mask is None or not isinstance(mask, torch.Tensor):
|
|
return mask
|
|
if mask.ndim != video.ndim:
|
|
raise ValueError(
|
|
"SelfLift: tiled H3 video denoise mask must have the same number "
|
|
"of dimensions as the video latent"
|
|
)
|
|
if mask.shape[2] != video.shape[2] or any(
|
|
mask.shape[spatial_axis] not in (1, video.shape[spatial_axis])
|
|
for spatial_axis in (3, 4)
|
|
):
|
|
raise ValueError(
|
|
"SelfLift: tiled H3 video denoise mask does not match the target "
|
|
"latent temporal/spatial shape"
|
|
)
|
|
if mask.shape[axis] == 1:
|
|
return mask
|
|
return mask.narrow(axis, start, end - start).contiguous()
|
|
|
|
|
|
def _tiled_forward(executor, streams, timestep, context, transformer_options, minimax_payload=None,
|
|
n_tiles=2, plan=None, **kwargs):
|
|
video, audio = streams
|
|
axis = 3 if (video.shape[3] + 1) // 2 >= (video.shape[4] + 1) // 2 else 4
|
|
length = video.shape[axis]
|
|
regions = _regions(length, plan['tiles'] if plan is not None else n_tiles)
|
|
eval_id = next(_H3_TILING_EVAL_COUNTER)
|
|
axis_name = "H" if axis == 3 else "W"
|
|
_reset_cuda_peak(video.device)
|
|
logging.info(
|
|
"[H3 tiled forward] eval=%d axis=%s tiles=%d full_video=%s audio=%s",
|
|
eval_id,
|
|
axis_name,
|
|
len(regions),
|
|
tuple(video.shape),
|
|
tuple(audio.shape),
|
|
)
|
|
_log_cuda_memory("[H3 tiled forward memory/start]", eval_id, video.device)
|
|
if len(regions) == 1:
|
|
logging.info(
|
|
"[H3 tiled forward] eval=%d bypassed effective_tiles=1",
|
|
eval_id,
|
|
)
|
|
result = executor(
|
|
streams,
|
|
timestep,
|
|
context,
|
|
transformer_options,
|
|
minimax_payload=minimax_payload,
|
|
**kwargs,
|
|
)
|
|
_log_cuda_memory("[H3 tiled forward memory/bypass]", eval_id, video.device)
|
|
return result
|
|
if kwargs.get("control") is not None:
|
|
raise ValueError("SelfLift: high-resolution H3 tiling does not support ControlNet")
|
|
# Keep the stitched accumulator on CPU so previous tiles do not remain on
|
|
# the accelerator while the next tile is evaluated.
|
|
video_output = torch.zeros(video.shape, dtype=torch.float32, device="cpu")
|
|
audio_output = torch.zeros(audio.shape, dtype=torch.float32, device="cpu")
|
|
audio_weight_total = 0.0
|
|
weights = torch.zeros(length, dtype=torch.float32, device="cpu")
|
|
for index, (start, end) in enumerate(regions):
|
|
tile = video.narrow(axis, start, end - start).contiguous()
|
|
logging.info(
|
|
"[H3 tile] eval=%d tile=%d/%d axis=%s range=%d:%d tile_video=%s full_audio=%s",
|
|
eval_id,
|
|
index + 1,
|
|
len(regions),
|
|
axis_name,
|
|
start,
|
|
end,
|
|
tuple(tile.shape),
|
|
tuple(audio.shape),
|
|
)
|
|
payload = _tile_payload(minimax_payload or {}, context, video, audio, axis, start, end)
|
|
tile_kwargs = kwargs.copy()
|
|
tile_kwargs["denoise_mask"] = _tile_denoise_mask(
|
|
tile_kwargs.get("denoise_mask"),
|
|
video,
|
|
axis,
|
|
start,
|
|
end,
|
|
)
|
|
predicted_video, predicted_audio = executor(
|
|
[tile, audio],
|
|
timestep,
|
|
context,
|
|
transformer_options.copy(),
|
|
minimax_payload=payload,
|
|
**tile_kwargs,
|
|
)
|
|
_log_cuda_memory(
|
|
"[H3 tile memory/after-forward]",
|
|
eval_id,
|
|
video.device,
|
|
tile_index=index + 1,
|
|
tile_total=len(regions),
|
|
)
|
|
window = torch.ones(end - start, device=video.device, dtype=torch.float32)
|
|
if index > 0:
|
|
overlap = min(end, regions[index - 1][1]) - start
|
|
window[:overlap] *= (torch.arange(overlap, device=video.device, dtype=torch.float32) + 0.5) / overlap
|
|
if index + 1 < len(regions):
|
|
overlap = end - regions[index + 1][0]
|
|
window[-overlap:] *= 1.0 - (torch.arange(overlap, device=video.device, dtype=torch.float32) + 0.5) / overlap
|
|
predicted_video = predicted_video.float().cpu()
|
|
window = window.cpu()
|
|
weights[start:end].add_(window)
|
|
audio_weight = float(window.sum().item())
|
|
audio_output.add_(predicted_audio.float().cpu(), alpha=audio_weight)
|
|
audio_weight_total += audio_weight
|
|
window_shape = [1] * video.ndim
|
|
window_shape[axis] = end - start
|
|
video_output.narrow(axis, start, end - start).addcmul_(predicted_video.float(), window.view(window_shape))
|
|
del tile, payload, predicted_video, predicted_audio, window
|
|
_log_cuda_memory(
|
|
"[H3 tile memory/after-release]",
|
|
eval_id,
|
|
video.device,
|
|
tile_index=index + 1,
|
|
tile_total=len(regions),
|
|
)
|
|
window_shape[axis] = length
|
|
video_output.div_(weights.view(window_shape))
|
|
audio_output.div_(audio_weight_total)
|
|
logging.info(
|
|
"[H3 tiled forward done] eval=%d axis=%s tiles=%d",
|
|
eval_id,
|
|
axis_name,
|
|
len(regions),
|
|
)
|
|
_log_cuda_memory("[H3 tiled forward memory/done]", eval_id, video.device)
|
|
return [
|
|
video_output.to(device=video.device, dtype=video.dtype),
|
|
audio_output.to(device=audio.device, dtype=audio.dtype),
|
|
]
|
|
|
|
|
|
def _condition_elements(condition, tile_height, tile_width, channels):
|
|
text = condition.get("cross_attn")
|
|
elements = text.shape[-2] * channels * 4 if text is not None else 0
|
|
for keyframe in condition.get("minimax_keyframes") or []:
|
|
shape = keyframe["latent"].shape
|
|
elements += shape[1] * shape[2] * tile_height * tile_width
|
|
for reference in condition.get("minimax_refs") or []:
|
|
latent = reference.get("latent")
|
|
if latent is not None:
|
|
shape = latent.shape
|
|
elements += shape[1] * shape[2] * ((shape[3] + 1) // 2 * 2) * ((shape[4] + 1) // 2 * 2)
|
|
audio = reference.get("audio_latent")
|
|
if audio is not None:
|
|
elements += math.prod(audio.shape[1:])
|
|
return elements
|
|
|
|
|
|
def _budget(model, noise_shape, conds, latent_shapes, regions, axis):
|
|
video_shape, audio_shape = latent_shapes
|
|
full_elements = math.prod(video_shape[1:]) + math.prod(audio_shape[1:])
|
|
tile_shape = list(video_shape)
|
|
tile_shape[axis] = max(end - start for start, end in regions)
|
|
tile_shape[3] = (tile_shape[3] + 1) // 2 * 2
|
|
tile_shape[4] = (tile_shape[4] + 1) // 2 * 2
|
|
tile_elements = math.prod(tile_shape[1:]) + math.prod(audio_shape[1:])
|
|
condition_elements = max((_condition_elements(condition, tile_shape[3], tile_shape[4], video_shape[1])
|
|
for group in conds.values() for condition in (group or [])), default=0)
|
|
buffer_bytes = full_elements * 4 * 8
|
|
bytes_per_element = model.model.memory_required((1, 1, 1))
|
|
budget_elements = tile_elements + condition_elements + math.ceil(buffer_bytes / bytes_per_element)
|
|
budget_shape = (noise_shape[0], 1, budget_elements)
|
|
preferred, minimum = comfy.sampler_helpers.estimate_memory(model, budget_shape, conds)
|
|
return budget_shape, tuple(tile_shape), buffer_bytes, preferred, minimum
|
|
|
|
|
|
def _prepare_tiled_sampling(executor, model, noise_shape, conds, model_options=None,
|
|
force_full_load=False, force_offload=False, *, latent_shapes, plan=None):
|
|
video_shape, audio_shape = latent_shapes
|
|
axis = 3 if (video_shape[3] + 1) // 2 >= (video_shape[4] + 1) // 2 else 4
|
|
full_elements = math.prod(video_shape[1:]) + math.prod(audio_shape[1:])
|
|
if tuple(noise_shape) != (video_shape[0], 1, full_elements):
|
|
raise ValueError("SelfLift: tiled memory planning received a different latent shape than the sampling input")
|
|
count = 2
|
|
requested = int(plan.get("tiles", 0) or 0) if plan is not None else 0
|
|
if requested > 0:
|
|
count = max(2, min(8, requested))
|
|
regions = _regions(video_shape[axis], count)
|
|
count = len(regions)
|
|
plan["tiles"] = count
|
|
logging.info(
|
|
"[H3 tiling plan] axis=%s tiles=%d mode=manual",
|
|
"H" if axis == 3 else "W",
|
|
count,
|
|
)
|
|
elif plan is not None:
|
|
available = _available_workspace(model)
|
|
for count in range(1, 9):
|
|
regions = _regions(video_shape[axis], count)
|
|
_, _, _, _, minimum = _budget(model, noise_shape, conds, latent_shapes, regions, axis)
|
|
if minimum <= available:
|
|
break
|
|
count = len(regions)
|
|
plan['tiles'] = count
|
|
logging.info("[SelfLift tiling plan] axis=%s tiles=%d target=%.2f MiB estimate_fits=%s",
|
|
'H' if axis == 3 else 'W', plan['tiles'], available / 2**20, minimum <= available)
|
|
regions = _regions(video_shape[axis], count)
|
|
if len(regions) == 1 or force_offload:
|
|
return executor(model, noise_shape, conds, model_options=model_options,
|
|
force_full_load=force_full_load, force_offload=force_offload)
|
|
budget_shape, tile_shape, buffer_bytes, preferred, minimum = _budget(
|
|
model, noise_shape, conds, latent_shapes, regions, axis)
|
|
logging.info("[SelfLift tiling memory] largest_tile=%s full_audio=%s full_state_buffers=%.2f MiB "
|
|
"minimum=%.2f MiB preferred=%.2f MiB (ComfyUI estimates; additional models and reserves excluded)",
|
|
tuple(tile_shape), audio_shape, buffer_bytes * noise_shape[0] / 2**20,
|
|
minimum / 2**20, preferred / 2**20)
|
|
return executor(model, budget_shape, conds, model_options=model_options,
|
|
force_full_load=force_full_load, force_offload=force_offload)
|
|
|
|
|
|
def _available_workspace(model):
|
|
manager = comfy.model_management
|
|
free = manager.get_free_memory(model.load_device)
|
|
reclaimable = 0
|
|
seen = set()
|
|
for loaded in manager.loaded_models():
|
|
patcher = loaded if callable(getattr(loaded, "loaded_size", None)) else loaded.model
|
|
load_device = getattr(patcher, "load_device", None)
|
|
identity = id(patcher.model)
|
|
if load_device == model.load_device and identity not in seen:
|
|
seen.add(identity)
|
|
size_fn = getattr(patcher, "loaded_size", None)
|
|
if callable(size_fn):
|
|
reclaimable += size_fn()
|
|
pool = min(manager.get_total_memory(model.load_device), free + reclaimable)
|
|
weights = min(model.model_size(), pool * manager.MIN_WEIGHT_MEMORY_RATIO)
|
|
available = max(0, pool - weights - manager.minimum_inference_memory())
|
|
logging.info("[SelfLift tiling capacity] free=%.2f MiB reclaimable_weights=%.2f MiB "
|
|
"weight_allowance=%.2f MiB workspace=%.2f MiB",
|
|
free / 2**20, reclaimable / 2**20, weights / 2**20, available / 2**20)
|
|
return available
|
|
|
|
|
|
def tiled_model(model, latent_shapes, tile_count: int = 0):
|
|
if not isinstance(model.model, comfy.model_base.MiniMaxH3):
|
|
raise ValueError("SelfLift: high-resolution tiling requires a MiniMax H3 model")
|
|
if len(latent_shapes) != 2 or len(latent_shapes[0]) != 5 or len(latent_shapes[1]) != 4:
|
|
raise ValueError("SelfLift: high-resolution tiling requires H3 video and audio latent streams")
|
|
requested_tiles = int(tile_count or 0)
|
|
if requested_tiles not in (0, 1) and not 2 <= requested_tiles <= 8:
|
|
raise ValueError(
|
|
"H3 high-resolution tile_count must be 0/auto or between 2 and 8"
|
|
)
|
|
plan = {} if requested_tiles <= 1 else {"tiles": requested_tiles}
|
|
patched = model.clone()
|
|
patched.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL,
|
|
"selflift_high_resolution_tiling", partial(_tiled_forward, plan=plan))
|
|
patched.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.PREPARE_SAMPLING,
|
|
"selflift_high_resolution_tiling",
|
|
partial(_prepare_tiled_sampling, latent_shapes=tuple(tuple(shape) for shape in latent_shapes), plan=plan))
|
|
return patched
|