Console, errors, tooltips, descriptions and display names no longer reveal the pack heritage or the underlying model family: - [EasyMedia]/[Easy Media] console prefixes -> [Moxie] (11 files) - New console privacy filter (utils/log.py, installed at pack import): rewrites MiniMax variants in every displayed logging record, including ComfyUI core lines like 'Model MiniMaxH3 prepared for dynamic VRAM loading', %-args and matching tracebacks. api.minimaxi.com is protected so cloud-API errors stay accurate. - Node display names rebranded (node_ids untouched, workflows safe): 'MiniMax H3 To Video' -> 'Moxie To Video', etc. - ~40 error/tooltip/description strings reworded to Moxie terms. - CLIP error no longer names CLIPType.MINIMAX; test match updated. Unchanged on purpose (breaking if touched): node_ids/class names, combo value 'MiniMax' (workflow payloads), internal dict keys (minimax_keyframes/minimax_refs), llm_api cloud provider strings, class-name logic checks, source-only docstrings.
657 lines
24 KiB
Python
657 lines
24 KiB
Python
"""MiniMax H3 progressive sampling adapted from facok/comfyui-SelfLift.
|
|
|
|
Upstream reference: https://github.com/facok/comfyui-SelfLift
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from typing import Any, Callable
|
|
|
|
import torch
|
|
|
|
from . import artifact_aware_consistency_lift, paired_lifts
|
|
from .diagnostics import log_memory
|
|
from ...utils.minimax import selflift_transition_step
|
|
|
|
|
|
class _StageTimer:
|
|
"""Match the upstream SelfLift stage timing diagnostics."""
|
|
|
|
def __init__(self, stage: str, device: Any) -> None:
|
|
self.stage = stage
|
|
self.device = torch.device(device)
|
|
self.synchronize = (
|
|
os.environ.get("SELFLIFT_TIMING_SYNC", "0") == "1"
|
|
and self.device.type == "cuda"
|
|
)
|
|
self.started = self.previous = self._now()
|
|
logging.info(
|
|
"[SelfLift timing] %s start (%s)",
|
|
stage,
|
|
(
|
|
"CUDA-synchronized wall time"
|
|
if self.synchronize
|
|
else "wall time; no forced CUDA sync"
|
|
),
|
|
)
|
|
|
|
def _now(self) -> float:
|
|
if self.synchronize:
|
|
torch.cuda.synchronize(self.device)
|
|
return time.perf_counter()
|
|
|
|
def mark(self, label: str) -> None:
|
|
current = self._now()
|
|
logging.info(
|
|
"[SelfLift timing] %s %s: %.3fs",
|
|
self.stage,
|
|
label,
|
|
current - self.previous,
|
|
)
|
|
self.previous = current
|
|
|
|
def finish(self) -> None:
|
|
current = self._now()
|
|
logging.info(
|
|
"[SelfLift timing] %s total: %.3fs; after last mark: %.3fs",
|
|
self.stage,
|
|
current - self.started,
|
|
current - self.previous,
|
|
)
|
|
|
|
|
|
def _streams(samples: Any) -> tuple[list[torch.Tensor], bool]:
|
|
if getattr(samples, "is_nested", False):
|
|
return list(samples.unbind()), True
|
|
return [samples], False
|
|
|
|
|
|
def _pack(streams: list[torch.Tensor], nested: bool) -> Any:
|
|
if not nested:
|
|
return streams[0]
|
|
import comfy.nested_tensor
|
|
|
|
return comfy.nested_tensor.NestedTensor(streams)
|
|
|
|
|
|
def _prepare_sampling_callback(
|
|
model: Any,
|
|
step_count: int,
|
|
preview_callback: Callable[[int, Any, Any, int], None] | None,
|
|
) -> Callable[[int, Any, Any, int], Any]:
|
|
"""Use native preview normally, or progress-only when custom preview is active."""
|
|
import comfy.utils
|
|
import latent_preview
|
|
|
|
if preview_callback is None:
|
|
return latent_preview.prepare_callback(model, step_count)
|
|
|
|
progress = comfy.utils.ProgressBar(step_count)
|
|
|
|
def callback(step: int, _x0: Any, _state: Any, total: int) -> None:
|
|
progress.update_absolute(step + 1, total)
|
|
|
|
return callback
|
|
|
|
|
|
def _highres_sampling_model(
|
|
model: Any,
|
|
streams: list[torch.Tensor],
|
|
highres_tiling: bool,
|
|
tile_count: int = 0,
|
|
) -> Any:
|
|
"""Patch only the high-resolution H3 stage when tiling is requested."""
|
|
if not highres_tiling:
|
|
return model
|
|
from .h3_tiling import tiled_model
|
|
|
|
shapes = [tuple(stream.shape) for stream in streams]
|
|
if tile_count > 0:
|
|
return tiled_model(model, shapes, tile_count=tile_count)
|
|
return tiled_model(model, shapes)
|
|
|
|
|
|
def _validate_schedule(sigmas: torch.Tensor, transition_step: int) -> None:
|
|
if sigmas.ndim != 1 or not sigmas.is_floating_point():
|
|
raise ValueError(
|
|
"SelfLift: sigmas must be a one-dimensional floating-point tensor"
|
|
)
|
|
if not torch.isfinite(sigmas).all() or (sigmas < 0).any():
|
|
raise ValueError("SelfLift: sigmas must be finite and nonnegative")
|
|
if sigmas.numel() < 3:
|
|
raise ValueError("SelfLift: an active schedule needs at least two steps")
|
|
if not 1 <= transition_step <= sigmas.numel() - 2:
|
|
raise ValueError(
|
|
f"SelfLift: transition_step {transition_step} is outside the active schedule"
|
|
)
|
|
if (sigmas[1:] > sigmas[:-1]).any():
|
|
raise ValueError("SelfLift: sigmas must be non-increasing")
|
|
if (sigmas[:-1] <= 0).any():
|
|
raise ValueError("SelfLift: only the final sigma may be zero")
|
|
if sigmas[transition_step] >= 1:
|
|
raise ValueError(
|
|
"SelfLift: the high-resolution starting sigma must be less than 1"
|
|
)
|
|
|
|
|
|
def _validate_latent_input(latent_image: dict[str, Any]) -> None:
|
|
streams, _ = _streams(latent_image["samples"])
|
|
if len(streams) != 2 or streams[0].ndim != 5 or streams[1].ndim != 4:
|
|
raise ValueError("SelfLift: expected Moxie video and audio latent streams")
|
|
for stream in streams:
|
|
if any(size == 0 for size in stream.shape) or stream.shape[0] != streams[0].shape[0]:
|
|
raise ValueError("SelfLift: latent streams must be nonempty with equal batches")
|
|
|
|
noise_mask = latent_image.get("noise_mask")
|
|
if noise_mask is None:
|
|
return
|
|
masks, _ = _streams(noise_mask)
|
|
if not masks or len(masks) > len(streams) or any(
|
|
not isinstance(mask, torch.Tensor) for mask in masks
|
|
):
|
|
raise ValueError("SelfLift: noise_mask contains invalid H3 streams")
|
|
expected_dimensions = (5, 4)
|
|
for index, mask in enumerate(masks):
|
|
if mask.ndim != expected_dimensions[index]:
|
|
raise ValueError(
|
|
f"SelfLift: noise_mask stream {index} has invalid shape "
|
|
f"{tuple(mask.shape)}"
|
|
)
|
|
if mask.shape[0] != streams[index].shape[0]:
|
|
raise ValueError("SelfLift: latent and noise_mask batch sizes differ")
|
|
|
|
|
|
def _resize_video_spatial(
|
|
tensor: torch.Tensor,
|
|
height: int,
|
|
width: int,
|
|
*,
|
|
mode: str,
|
|
) -> torch.Tensor:
|
|
"""Resize only the spatial axes of an H3 video latent or mask."""
|
|
if tensor.ndim != 5:
|
|
raise ValueError(
|
|
f"SelfLift: expected a 5D video tensor, got shape {tuple(tensor.shape)}"
|
|
)
|
|
if tensor.shape[-2:] == (height, width):
|
|
return tensor
|
|
kwargs: dict[str, Any] = {
|
|
"size": (tensor.shape[2], height, width),
|
|
"mode": mode,
|
|
}
|
|
if mode != "nearest":
|
|
kwargs["align_corners"] = False
|
|
return torch.nn.functional.interpolate(tensor.float(), **kwargs).to(tensor)
|
|
|
|
|
|
def _low_resolution_inputs(
|
|
samples: Any,
|
|
noise_mask: Any | None,
|
|
height: int,
|
|
width: int,
|
|
device: Any,
|
|
low_context_samples: Any | None = None,
|
|
) -> tuple[Any, Any | None]:
|
|
"""Build matching low-resolution H3 latent and denoise-mask streams."""
|
|
streams, nested = _streams(samples)
|
|
low_streams = [
|
|
_resize_video_spatial(streams[0], height, width, mode="trilinear").to(device)
|
|
] + [stream.to(device) for stream in streams[1:]]
|
|
low_mask = None
|
|
if noise_mask is not None:
|
|
masks, _ = _streams(noise_mask)
|
|
low_masks = [
|
|
_resize_video_spatial(masks[0], height, width, mode="nearest").to(device)
|
|
] + [mask.to(device) for mask in masks[1:]]
|
|
low_mask = _pack(low_masks, nested or len(low_masks) > 1)
|
|
if low_context_samples is not None:
|
|
context_streams, _ = _streams(low_context_samples)
|
|
if not context_streams or context_streams[0].ndim != 5:
|
|
raise ValueError("SelfLift: low context has no H3 video stream")
|
|
context_video = _resize_video_spatial(
|
|
context_streams[0], height, width, mode="trilinear"
|
|
).to(device=device, dtype=low_streams[0].dtype)
|
|
target_video = low_streams[0]
|
|
if context_video.shape[:2] != target_video.shape[:2]:
|
|
raise ValueError(
|
|
"SelfLift: low context video batch/channels do not match the target"
|
|
)
|
|
temporal_mask = (low_masks[0] < 1.0 - 1e-6).any(
|
|
dim=(0, 1, 3, 4)
|
|
)
|
|
masked_steps = torch.nonzero(
|
|
temporal_mask, as_tuple=False
|
|
).flatten()
|
|
if masked_steps.numel() > 0:
|
|
prefix_steps = min(
|
|
int(masked_steps[-1].item()) + 1,
|
|
int(context_video.shape[2]),
|
|
int(target_video.shape[2]),
|
|
)
|
|
target_video = target_video.clone()
|
|
low_streams[0] = target_video
|
|
target_video[:, :, :prefix_steps] = context_video[
|
|
:, :, -prefix_steps:
|
|
]
|
|
return _pack(low_streams, nested), low_mask
|
|
|
|
|
|
def _video_context_prefix_steps(noise_mask: Any | None) -> int | None:
|
|
"""Return the temporal extent of a partially preserved video prefix."""
|
|
if noise_mask is None:
|
|
return None
|
|
masks, _ = _streams(noise_mask)
|
|
if not masks:
|
|
return None
|
|
video_mask = masks[0]
|
|
if not isinstance(video_mask, torch.Tensor) or video_mask.ndim != 5:
|
|
return None
|
|
preserved = (video_mask < 1.0 - 1e-6).any(dim=(0, 1, 3, 4))
|
|
preserved_steps = torch.nonzero(preserved, as_tuple=False).flatten()
|
|
if preserved_steps.numel() == 0:
|
|
return None
|
|
return int(preserved_steps[-1].item()) + 1
|
|
|
|
|
|
def _resume_noise_for_anchor(
|
|
model_sampling: Any,
|
|
desired_streams: list[torch.Tensor],
|
|
anchor_streams: list[torch.Tensor],
|
|
sigma: torch.Tensor,
|
|
) -> list[torch.Tensor]:
|
|
"""Solve CONST noise scaling while retaining the clean input as mask anchor."""
|
|
scale = float(getattr(model_sampling, "noise_scale", 1.0))
|
|
if not torch.isfinite(torch.as_tensor(scale)) or scale == 0.0:
|
|
raise ValueError("SelfLift: model noise_scale must be finite and nonzero")
|
|
output: list[torch.Tensor] = []
|
|
for desired, anchor in zip(desired_streams, anchor_streams):
|
|
stream_sigma = sigma.to(device=desired.device, dtype=desired.dtype)
|
|
stream_sigma = stream_sigma.reshape(
|
|
stream_sigma.shape[:1] + (1,) * (desired.ndim - 1)
|
|
if stream_sigma.ndim > 0 and stream_sigma.numel() > 1
|
|
else ()
|
|
)
|
|
output.append(
|
|
(desired - (1.0 - stream_sigma) * anchor.to(desired))
|
|
/ (stream_sigma * scale)
|
|
)
|
|
return output
|
|
|
|
|
|
def _anchor_masked_clean_video(
|
|
clean_video: torch.Tensor,
|
|
anchor_video: torch.Tensor,
|
|
noise_mask: Any | None,
|
|
) -> torch.Tensor:
|
|
"""Make the lifted clean endpoint agree with the full-size masked anchor."""
|
|
if noise_mask is None:
|
|
return clean_video
|
|
masks, _ = _streams(noise_mask)
|
|
if not masks:
|
|
return clean_video
|
|
mask = masks[0]
|
|
if mask.ndim != 5 or mask.shape[0] != clean_video.shape[0]:
|
|
raise ValueError("SelfLift: video noise_mask cannot anchor the lifted endpoint")
|
|
if mask.shape[-3:] != clean_video.shape[-3:]:
|
|
raise ValueError(
|
|
"SelfLift: video noise_mask temporal/spatial shape does not match "
|
|
"the target latent"
|
|
)
|
|
mask = mask.to(device=clean_video.device, dtype=clean_video.dtype).clamp_(0.0, 1.0)
|
|
anchor = anchor_video.to(device=clean_video.device, dtype=clean_video.dtype)
|
|
try:
|
|
return torch.lerp(anchor, clean_video, mask)
|
|
except RuntimeError as error:
|
|
raise ValueError(
|
|
"SelfLift: video noise_mask channels cannot broadcast to the target latent"
|
|
) from error
|
|
|
|
|
|
def _validate_euler_sampler(sampler: Any) -> None:
|
|
import comfy.k_diffusion.sampling
|
|
import comfy.samplers
|
|
|
|
if (
|
|
not isinstance(sampler, comfy.samplers.KSAMPLER)
|
|
or sampler.sampler_function is not comfy.k_diffusion.sampling.sample_euler
|
|
or sampler.extra_options.get("s_churn", 0.0) != 0.0
|
|
):
|
|
raise ValueError("SelfLift requires standard Euler with s_churn=0")
|
|
|
|
|
|
def _euler_step(
|
|
state: torch.Tensor,
|
|
denoised: torch.Tensor,
|
|
sigma: torch.Tensor,
|
|
sigma_next: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
step = ((sigma_next - sigma) / sigma).to(device=state.device, dtype=state.dtype)
|
|
return state + (state - denoised.to(state)) * step
|
|
|
|
|
|
def _resize_keyframes(conditioning: Any, height: int, width: int) -> Any:
|
|
"""Resize H3 keyframe latents for the low-resolution sampling prefix."""
|
|
resized_conditioning = []
|
|
for embedding, metadata in conditioning:
|
|
keyframes = metadata.get("minimax_keyframes")
|
|
if keyframes is None:
|
|
resized_conditioning.append((embedding, metadata))
|
|
continue
|
|
updated_metadata = metadata.copy()
|
|
updated_keyframes = []
|
|
for keyframe in keyframes:
|
|
updated = dict(keyframe)
|
|
latent = updated.get("latent")
|
|
if latent is not None and latent.shape[-2:] != (height, width):
|
|
updated["latent"] = torch.nn.functional.interpolate(
|
|
latent.float(),
|
|
size=(latent.shape[2], height, width),
|
|
mode="trilinear",
|
|
align_corners=False,
|
|
).to(latent)
|
|
updated_keyframes.append(updated)
|
|
updated_metadata["minimax_keyframes"] = updated_keyframes
|
|
resized_conditioning.append((embedding, updated_metadata))
|
|
return resized_conditioning
|
|
|
|
|
|
def progressive_sample_h3(
|
|
model: Any,
|
|
positive: Any,
|
|
vae: Any,
|
|
latent_image: dict[str, Any],
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
transition_ratio: float,
|
|
lowres_scale: float,
|
|
*,
|
|
highres_model: Any | None = None,
|
|
latent_lifter: Callable[[torch.Tensor, tuple[int, int]], torch.Tensor] | None = None,
|
|
low_context_latent: dict[str, Any] | None = None,
|
|
rho: float = 0.0,
|
|
w_min: float = 0.5,
|
|
w_max: float = 1.0,
|
|
highres_tiling: bool = False,
|
|
tile_count: int = 0,
|
|
preview_callback: Callable[[int, Any, Any, int], None] | None = None,
|
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
"""Run the original NFE-preserving H3 SelfLift transition and resume flow."""
|
|
import comfy.model_management
|
|
import comfy.model_sampling
|
|
import comfy.sample
|
|
import comfy.samplers
|
|
import comfy.utils
|
|
|
|
if not 0.25 <= lowres_scale <= 1.0:
|
|
raise ValueError("SelfLift: lowres_scale must be between 0.25 and 1")
|
|
if not 0.0 <= rho <= 1.0 or not 0.0 <= w_min <= w_max <= 1.0:
|
|
raise ValueError("SelfLift: invalid artifact-correction parameters")
|
|
|
|
step_count = int(sigmas.numel()) - 1
|
|
transition_step = selflift_transition_step(step_count, transition_ratio)
|
|
_validate_schedule(sigmas, transition_step)
|
|
_validate_latent_input(latent_image)
|
|
model_sampling = model.get_model_object("model_sampling")
|
|
if not isinstance(model_sampling, comfy.model_sampling.CONST):
|
|
raise ValueError("SelfLift requires a rectified-flow model")
|
|
highres_model = highres_model if highres_model is not None else model
|
|
highres_model_sampling = highres_model.get_model_object("model_sampling")
|
|
if not isinstance(highres_model_sampling, comfy.model_sampling.CONST):
|
|
raise ValueError("SelfLift high-resolution model requires rectified flow")
|
|
sampler = comfy.samplers.sampler_object("euler")
|
|
_validate_euler_sampler(sampler)
|
|
|
|
fixed_samples = comfy.sample.fix_empty_latent_channels(
|
|
model,
|
|
latent_image["samples"],
|
|
latent_image.get("downscale_ratio_spacial"),
|
|
latent_image.get("downscale_ratio_temporal"),
|
|
)
|
|
streams, nested = _streams(fixed_samples)
|
|
high_model = _highres_sampling_model(
|
|
highres_model,
|
|
streams,
|
|
highres_tiling,
|
|
tile_count=tile_count,
|
|
)
|
|
batch, channels, frames, target_height, target_width = streams[0].shape
|
|
low_height = max(2, round(target_height * lowres_scale / 2) * 2)
|
|
low_width = max(2, round(target_width * lowres_scale / 2) * 2)
|
|
low_shape = (batch, channels, frames, low_height, low_width)
|
|
logging.info(
|
|
"[SelfLift plan] low_latent=%s target_latent=%s low_nfe=%d high_nfe=%d "
|
|
"transition_ratio=%.4f sigma_prediction=%.8g sigma_resume=%.8g "
|
|
"direct_lift=%s pixel_anchor=%s",
|
|
low_shape,
|
|
tuple(streams[0].shape),
|
|
transition_step,
|
|
step_count - transition_step,
|
|
transition_ratio,
|
|
sigmas[transition_step - 1].item(),
|
|
sigmas[transition_step].item(),
|
|
"external" if latent_lifter is not None else "nearest",
|
|
rho > 0.0 and w_max > 0.0,
|
|
)
|
|
|
|
device = comfy.model_management.intermediate_device()
|
|
low_latent, low_noise_mask = _low_resolution_inputs(
|
|
fixed_samples,
|
|
latent_image.get("noise_mask"),
|
|
low_height,
|
|
low_width,
|
|
device,
|
|
(
|
|
low_context_latent.get("samples")
|
|
if isinstance(low_context_latent, dict)
|
|
else None
|
|
),
|
|
)
|
|
noise_low = comfy.sample.prepare_noise(
|
|
low_latent, seed, latent_image.get("batch_index")
|
|
)
|
|
callback = _prepare_sampling_callback(model, step_count, preview_callback)
|
|
disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED
|
|
positive_low = _resize_keyframes(positive, low_height, low_width)
|
|
|
|
transition: dict[str, Any] = {}
|
|
low_evaluations = 0
|
|
|
|
def callback_low(step: int, x0: Any, state: Any, total: int) -> Any:
|
|
del step, total
|
|
nonlocal low_evaluations
|
|
current = low_evaluations
|
|
low_evaluations += 1
|
|
if low_evaluations > transition_step:
|
|
raise RuntimeError("SelfLift: too many low-resolution Euler callbacks")
|
|
if current == transition_step - 1:
|
|
transition["state"] = state
|
|
transition["x0"] = x0
|
|
result = callback(current, x0, state, step_count)
|
|
if preview_callback is not None:
|
|
preview_callback(current, x0, state, step_count)
|
|
low_timer.mark(
|
|
f"step {current + 1}/{transition_step}"
|
|
+ (" (includes setup)" if current == 0 else "")
|
|
)
|
|
return result
|
|
|
|
low_timer = _StageTimer("low_resolution", model.load_device)
|
|
log_memory("low_resolution start", model.load_device)
|
|
comfy.samplers.sample(
|
|
model,
|
|
noise_low,
|
|
positive_low,
|
|
[],
|
|
1.0,
|
|
model.load_device,
|
|
sampler,
|
|
sigmas[: transition_step + 1],
|
|
model.model_options,
|
|
latent_image=low_latent,
|
|
denoise_mask=low_noise_mask,
|
|
callback=callback_low,
|
|
disable_pbar=disable_pbar,
|
|
seed=seed,
|
|
)
|
|
if low_evaluations != transition_step:
|
|
raise RuntimeError(
|
|
f"SelfLift: expected {transition_step} low-resolution callbacks, "
|
|
f"received {low_evaluations}"
|
|
)
|
|
low_timer.finish()
|
|
log_memory("low_resolution end", model.load_device)
|
|
|
|
transition_timer = _StageTimer("transition", model.load_device)
|
|
low_streams, nested = _streams(transition.pop("state"))
|
|
x0_streams, _ = _streams(transition.pop("x0"))
|
|
low_clean = model.model.process_latent_out(_pack(x0_streams, nested))
|
|
low_result = latent_image.copy()
|
|
low_result["samples"] = low_clean.to(
|
|
device=comfy.model_management.intermediate_device(),
|
|
dtype=comfy.model_management.intermediate_dtype(),
|
|
)
|
|
if low_noise_mask is not None:
|
|
low_result["noise_mask"] = low_noise_mask
|
|
else:
|
|
low_result.pop("noise_mask", None)
|
|
sigma_prediction = sigmas[transition_step - 1]
|
|
sigma_resume = sigmas[transition_step]
|
|
auxiliary_next = [
|
|
_euler_step(state.to(device), denoised.to(device), sigma_prediction, sigma_resume)
|
|
for state, denoised in zip(low_streams[1:], x0_streams[1:])
|
|
]
|
|
latent_format = model.get_model_object("latent_format")
|
|
clean_low_vae = latent_format.process_out(x0_streams[0].float()).to(device)
|
|
del low_latent, noise_low, low_streams, x0_streams, positive_low
|
|
transition_timer.mark("prepare_endpoint")
|
|
log_memory("transition endpoint_ready", model.load_device)
|
|
|
|
need_pixel_anchor = rho > 0.0 and w_max > 0.0
|
|
need_direct_lift = not (rho >= 1.0 and w_min >= 1.0 and w_max >= 1.0)
|
|
temporal_split = _video_context_prefix_steps(latent_image.get("noise_mask"))
|
|
direct_vae, pixel_vae = paired_lifts(
|
|
clean_low_vae,
|
|
vae,
|
|
(target_height, target_width),
|
|
"nearest",
|
|
latent_lifter,
|
|
need_lat=need_direct_lift,
|
|
need_pix=need_pixel_anchor,
|
|
temporal_split=temporal_split,
|
|
)
|
|
transition_timer.mark("paired_lifts")
|
|
log_memory("transition lifts_ready", model.load_device)
|
|
direct = latent_format.process_in(direct_vae) if direct_vae is not None else None
|
|
pixel = latent_format.process_in(pixel_vae) if pixel_vae is not None else None
|
|
clean_high = artifact_aware_consistency_lift(direct, pixel, rho, w_min, w_max)
|
|
if clean_high is None:
|
|
raise RuntimeError("SelfLift transition produced no high-resolution endpoint")
|
|
clean_high = clean_high.to(device)
|
|
del clean_low_vae, direct_vae, pixel_vae, direct, pixel
|
|
transition_timer.mark("correction_and_debug")
|
|
log_memory("transition correction_ready", model.load_device)
|
|
|
|
target_shapes = [tuple(stream.shape) for stream in streams]
|
|
packed_anchor, _ = comfy.utils.pack_latents(streams)
|
|
highres_model.model.latent_shapes = target_shapes
|
|
processed_anchor = highres_model.model.process_latent_in(packed_anchor)
|
|
anchor_streams = list(comfy.utils.unpack_latents(processed_anchor, target_shapes))
|
|
clean_high = _anchor_masked_clean_video(
|
|
clean_high,
|
|
anchor_streams[0],
|
|
latent_image.get("noise_mask"),
|
|
)
|
|
transition_timer.mark("mask_anchor")
|
|
|
|
video_noise = comfy.sample.prepare_noise(
|
|
clean_high,
|
|
(int(seed) + 1) % (1 << 64),
|
|
latent_image.get("batch_index"),
|
|
).to(clean_high)
|
|
video_state = highres_model_sampling.noise_scaling(
|
|
sigma_prediction, video_noise, clean_high
|
|
)
|
|
next_streams = [
|
|
_euler_step(video_state, clean_high, sigma_prediction, sigma_resume)
|
|
] + auxiliary_next
|
|
del clean_high, video_noise, video_state, auxiliary_next
|
|
# A regular masked sample needs two independent values: the current noisy
|
|
# state and the clean latent used as its preserved-region anchor. SelfLift's
|
|
# former zero-noise resume encoded both into latent_image, which only works
|
|
# when that latent is all zero and unmasked. Solve the CONST/RF noise term
|
|
# instead so the lifted state resumes exactly while the original full-size
|
|
# AV latent remains available to ComfyUI's inpaint-mask machinery.
|
|
resume_noise = _pack(
|
|
_resume_noise_for_anchor(
|
|
highres_model_sampling,
|
|
next_streams,
|
|
anchor_streams,
|
|
sigma_resume,
|
|
),
|
|
nested,
|
|
)
|
|
resume_latent = fixed_samples
|
|
del next_streams
|
|
transition_timer.mark("renoise")
|
|
transition_timer.finish()
|
|
log_memory("transition end / high_resolution start", model.load_device)
|
|
|
|
high_evaluations = 0
|
|
|
|
def callback_high(step: int, x0: Any, state: Any, total: int) -> Any:
|
|
del step, total
|
|
nonlocal high_evaluations
|
|
current = high_evaluations
|
|
high_evaluations += 1
|
|
if high_evaluations > step_count - transition_step:
|
|
raise RuntimeError("SelfLift: too many high-resolution Euler callbacks")
|
|
result = callback(current + transition_step, x0, state, step_count)
|
|
if preview_callback is not None:
|
|
preview_callback(current + transition_step, x0, state, step_count)
|
|
high_timer.mark(
|
|
f"step {current + 1}/{step_count - transition_step}"
|
|
+ (" (includes setup)" if current == 0 else "")
|
|
)
|
|
return result
|
|
|
|
high_timer = _StageTimer("high_resolution", high_model.load_device)
|
|
if highres_tiling:
|
|
logging.info(
|
|
"[SelfLift plan] automatic high-resolution tiling enabled; "
|
|
"preparation selects the tile count"
|
|
)
|
|
sampled = comfy.samplers.sample(
|
|
high_model,
|
|
resume_noise,
|
|
positive,
|
|
[],
|
|
1.0,
|
|
high_model.load_device,
|
|
sampler,
|
|
sigmas[transition_step:],
|
|
high_model.model_options,
|
|
latent_image=resume_latent,
|
|
denoise_mask=latent_image.get("noise_mask"),
|
|
callback=callback_high,
|
|
disable_pbar=disable_pbar,
|
|
seed=seed,
|
|
)
|
|
del resume_latent, resume_noise
|
|
if high_evaluations != step_count - transition_step:
|
|
raise RuntimeError(
|
|
f"SelfLift: expected {step_count - transition_step} high-resolution "
|
|
f"callbacks, received {high_evaluations}"
|
|
)
|
|
|
|
result = latent_image.copy()
|
|
result["samples"] = sampled.to(
|
|
device=comfy.model_management.intermediate_device(),
|
|
dtype=comfy.model_management.intermediate_dtype(),
|
|
)
|
|
high_timer.finish()
|
|
log_memory("high_resolution end", high_model.load_device)
|
|
return result, low_result
|