MMnode/modules/asr/whisper_asr.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

223 lines
8.0 KiB
Python

from __future__ import annotations
from collections.abc import Iterable
import importlib.util
from pathlib import Path
import re
try:
from ...utils.model_memory import cleanup_model_memory
from ...utils.subtitles import (
_detect_language,
_detect_text,
_restore_subtitle_punctuation,
_summarize_value,
normalize_subtitle_segments,
normalize_subtitle_tokens,
)
except ImportError:
from utils.model_memory import cleanup_model_memory
from utils.subtitles import (
_detect_language,
_detect_text,
_restore_subtitle_punctuation,
_summarize_value,
normalize_subtitle_segments,
normalize_subtitle_tokens,
)
def _module_available(name: str) -> bool:
try:
return importlib.util.find_spec(name) is not None
except (ImportError, AttributeError, ValueError):
return False
def missing_dependencies() -> list[str]:
missing: list[str] = []
if not _module_available("torch"):
missing.append("torch")
if not _module_available("whisper"):
missing.append("openai-whisper")
return missing
def _count_indexed_layers(keys: Iterable[str], prefix: str) -> int:
pattern = re.compile(rf"^{re.escape(prefix)}\.(\d+)\.")
return len({int(match.group(1)) for key in keys if (match := pattern.match(key))})
def _infer_whisper_head_count(state_size: int) -> int:
if state_size % 64 == 0:
return state_size // 64
known = {
384: 6,
512: 8,
768: 12,
1024: 16,
1280: 20,
}
if state_size in known:
return known[state_size]
raise RuntimeError(f"Cannot infer Whisper attention head count from state size {state_size}")
def _replace_hf_whisper_block_key(key: str) -> str:
replacements = (
(".self_attn.q_proj.", ".attn.query."),
(".self_attn.k_proj.", ".attn.key."),
(".self_attn.v_proj.", ".attn.value."),
(".self_attn.out_proj.", ".attn.out."),
(".encoder_attn.q_proj.", ".cross_attn.query."),
(".encoder_attn.k_proj.", ".cross_attn.key."),
(".encoder_attn.v_proj.", ".cross_attn.value."),
(".encoder_attn.out_proj.", ".cross_attn.out."),
(".self_attn_layer_norm.", ".attn_ln."),
(".encoder_attn_layer_norm.", ".cross_attn_ln."),
(".fc1.", ".mlp.0."),
(".fc2.", ".mlp.2."),
(".final_layer_norm.", ".mlp_ln."),
)
value = key
for old, new in replacements:
value = value.replace(old, new)
return value
def _openai_whisper_key_from_hf(key: str) -> str | None:
direct_replacements = {
"model.encoder.embed_positions.weight": "encoder.positional_embedding",
"model.encoder.layer_norm.weight": "encoder.ln_post.weight",
"model.encoder.layer_norm.bias": "encoder.ln_post.bias",
"model.decoder.embed_tokens.weight": "decoder.token_embedding.weight",
"model.decoder.embed_positions.weight": "decoder.positional_embedding",
"model.decoder.layer_norm.weight": "decoder.ln.weight",
"model.decoder.layer_norm.bias": "decoder.ln.bias",
}
if key in direct_replacements:
return direct_replacements[key]
if key.startswith("model.encoder.conv"):
return key.replace("model.encoder.", "encoder.", 1)
if key.startswith("model.encoder.layers."):
return _replace_hf_whisper_block_key(key.replace("model.encoder.layers.", "encoder.blocks.", 1))
if key.startswith("model.decoder.layers."):
return _replace_hf_whisper_block_key(key.replace("model.decoder.layers.", "decoder.blocks.", 1))
if key.startswith("proj_out."):
return None
return None
def _convert_hf_whisper_state_dict(sd: dict[str, object]) -> tuple[dict[str, object], object]:
from whisper.model import ModelDimensions # type: ignore[import]
required_keys = (
"model.encoder.conv1.weight",
"model.encoder.embed_positions.weight",
"model.decoder.embed_tokens.weight",
"model.decoder.embed_positions.weight",
)
missing_keys = [key for key in required_keys if key not in sd]
if missing_keys:
raise RuntimeError(f"Whisper safetensors file is missing required weights: {', '.join(missing_keys)}")
encoder_conv1 = sd["model.encoder.conv1.weight"]
encoder_positions = sd["model.encoder.embed_positions.weight"]
decoder_tokens = sd["model.decoder.embed_tokens.weight"]
decoder_positions = sd["model.decoder.embed_positions.weight"]
n_audio_state = int(encoder_conv1.shape[0]) # type: ignore[attr-defined]
n_mels = int(encoder_conv1.shape[1]) # type: ignore[attr-defined]
n_audio_ctx = int(encoder_positions.shape[0]) # type: ignore[attr-defined]
n_vocab = int(decoder_tokens.shape[0]) # type: ignore[attr-defined]
n_text_state = int(decoder_tokens.shape[1]) # type: ignore[attr-defined]
n_text_ctx = int(decoder_positions.shape[0]) # type: ignore[attr-defined]
keys = tuple(sd.keys())
dims = ModelDimensions(
n_mels=n_mels,
n_audio_ctx=n_audio_ctx,
n_audio_state=n_audio_state,
n_audio_head=_infer_whisper_head_count(n_audio_state),
n_audio_layer=_count_indexed_layers(keys, "model.encoder.layers"),
n_vocab=n_vocab,
n_text_ctx=n_text_ctx,
n_text_state=n_text_state,
n_text_head=_infer_whisper_head_count(n_text_state),
n_text_layer=_count_indexed_layers(keys, "model.decoder.layers"),
)
converted: dict[str, object] = {}
unexpected: list[str] = []
for key, value in sd.items():
next_key = _openai_whisper_key_from_hf(key)
if next_key is None:
if not key.startswith("proj_out."):
unexpected.append(key)
continue
converted[next_key] = value
if unexpected:
raise RuntimeError(
"Whisper safetensors file contains unsupported weights: "
+ ", ".join(unexpected[:8])
+ ("..." if len(unexpected) > 8 else "")
)
return converted, dims
def _load_openai_whisper_from_audio_encoder(model_path: Path, device: str) -> object:
import comfy.utils
from whisper.model import Whisper # type: ignore[import]
state_dict = comfy.utils.load_torch_file(str(model_path), safe_load=True)
converted_state_dict, dims = _convert_hf_whisper_state_dict(state_dict)
model = Whisper(dims)
missing, unexpected = model.load_state_dict(converted_state_dict, strict=False)
if missing or unexpected:
details = []
if missing:
details.append(f"missing={missing[:8]}")
if unexpected:
details.append(f"unexpected={unexpected[:8]}")
raise RuntimeError("Failed to load Whisper safetensors weights: " + "; ".join(details))
return model.to(device)
def recognize_subtitle_segments(
audio_path: Path,
model_path: Path,
unload_model: bool = True,
) -> list[dict]:
import torch
import whisper # type: ignore[import]
device = "cuda" if torch.cuda.is_available() else "cpu"
model = None
try:
model = _load_openai_whisper_from_audio_encoder(model_path, device)
result = model.transcribe(
str(audio_path),
verbose=False,
word_timestamps=True,
)
transcript = _detect_text(result)
detected_language = _detect_language(result)
timed_units = normalize_subtitle_tokens(result)
if timed_units:
return _restore_subtitle_punctuation(
timed_units,
transcript,
detected_language,
add_fallback=False,
add_final_fallback=True,
)
segments = normalize_subtitle_segments(result)
if segments:
return _restore_subtitle_punctuation(segments, transcript, detected_language)
raise RuntimeError(
"Whisper Large V3 did not return timestamped subtitle text. "
f"transcribe={_summarize_value(result)}"
)
finally:
if unload_model:
cleanup_model_memory(model)