MMnode/modules/asr/qwen_asr.py
turtle89431 4ea8431cd0
Some checks are pending
Publish to Comfy registry / Publish Custom Node to registry (push) Waiting to run
init
2026-09-26 11:11:23 -07:00

234 lines
6.9 KiB
Python

from __future__ import annotations
import importlib.util
from pathlib import Path
from typing import Any
try:
from ...utils.model_memory import cleanup_model_memory
from ...utils.subtitles import (
_detect_language,
_detect_text,
_restore_subtitle_punctuation,
_summarize_value,
normalize_subtitle_segments,
normalize_subtitle_tokens,
)
except ImportError:
from utils.model_memory import cleanup_model_memory
from utils.subtitles import (
_detect_language,
_detect_text,
_restore_subtitle_punctuation,
_summarize_value,
normalize_subtitle_segments,
normalize_subtitle_tokens,
)
_ALIGNER_LANGUAGE_ALIASES = {
"zh": "Chinese",
"cn": "Chinese",
"chinese": "Chinese",
"mandarin": "Chinese",
"en": "English",
"eng": "English",
"english": "English",
"yue": "Cantonese",
"cantonese": "Cantonese",
"fr": "French",
"fra": "French",
"fre": "French",
"french": "French",
"de": "German",
"deu": "German",
"ger": "German",
"german": "German",
"it": "Italian",
"ita": "Italian",
"italian": "Italian",
"ja": "Japanese",
"jp": "Japanese",
"jpn": "Japanese",
"japanese": "Japanese",
"ko": "Korean",
"kor": "Korean",
"korean": "Korean",
"pt": "Portuguese",
"por": "Portuguese",
"portuguese": "Portuguese",
"ru": "Russian",
"rus": "Russian",
"russian": "Russian",
"es": "Spanish",
"spa": "Spanish",
"spanish": "Spanish",
}
def _module_available(name: str) -> bool:
try:
return importlib.util.find_spec(name) is not None
except (ImportError, AttributeError, ValueError):
return False
def missing_dependencies() -> list[str]:
missing: list[str] = []
if not _module_available("qwen_asr"):
missing.append("qwen-asr")
if not _module_available("torchaudio"):
missing.append("torchaudio")
if not _module_available("pkg_resources"):
missing.append("setuptools")
return missing
def _normalize_aligner_language(language: str | None) -> str | None:
if not language:
return None
value = language.strip()
if not value:
return None
lowered = value.lower().replace("_", "-")
mapped = _ALIGNER_LANGUAGE_ALIASES.get(lowered)
if mapped:
return mapped
for key, mapped_language in _ALIGNER_LANGUAGE_ALIASES.items():
if f"({key})" in lowered:
return mapped_language
return None
def _infer_aligner_language_from_text(text: str | None) -> str | None:
if not text:
return None
if any("\u4e00" <= char <= "\u9fff" for char in text):
return "Chinese"
if any(char.isascii() and char.isalpha() for char in text):
return "English"
return None
def _transcribe(model: object, audio_path: Path, language: str | None) -> Any:
return model.transcribe(
audio=str(audio_path),
language=language,
return_time_stamps=True,
)
def _align_subtitle_segments(
audio_path: Path,
text: str,
language: str,
aligner_model_dir: Path,
dtype: object,
device: str,
unload_model: bool = True,
) -> list[dict]:
from qwen_asr import Qwen3ForcedAligner # type: ignore[import]
aligner = None
try:
aligner = Qwen3ForcedAligner.from_pretrained(
str(aligner_model_dir),
dtype=dtype,
device_map=device,
)
result = aligner.align(
audio=str(audio_path),
text=text,
language=language,
)
return normalize_subtitle_tokens(result) or normalize_subtitle_segments(result)
finally:
if unload_model:
cleanup_model_memory(aligner)
def recognize_subtitle_segments(
audio_path: Path,
asr_model_dir: Path,
aligner_model_dir: Path,
unload_model: bool = True,
) -> list[dict]:
import torch
from qwen_asr import Qwen3ASRModel # type: ignore[import]
device = "cuda:0" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32
model_kwargs: dict[str, object] = {
"dtype": dtype,
"device_map": device,
"max_inference_batch_size": 1,
"max_new_tokens": 4096,
"forced_aligner": str(aligner_model_dir),
"forced_aligner_kwargs": {
"dtype": dtype,
"device_map": device,
},
}
model = None
try:
model = Qwen3ASRModel.from_pretrained(str(asr_model_dir), **model_kwargs)
result = _transcribe(model, audio_path, None)
result_summaries = [f"transcribe={_summarize_value(result)}"]
detected_language = _detect_language(result)
transcript = _detect_text(result)
segments = normalize_subtitle_tokens(result) or normalize_subtitle_segments(result)
if segments:
return _restore_subtitle_punctuation(
segments,
transcript,
detected_language,
add_fallback=False,
add_final_fallback=True,
)
retry_language = _normalize_aligner_language(detected_language) or detected_language
aligner_language = _normalize_aligner_language(detected_language) or _infer_aligner_language_from_text(transcript)
if retry_language:
retry_result = _transcribe(model, audio_path, retry_language)
result_summaries.append(f"retry={_summarize_value(retry_result)}")
transcript = _detect_text(retry_result) or transcript
retry_segments = normalize_subtitle_tokens(retry_result) or normalize_subtitle_segments(retry_result)
if retry_segments:
return _restore_subtitle_punctuation(
retry_segments,
transcript,
detected_language,
add_fallback=False,
add_final_fallback=True,
)
aligner_language = _normalize_aligner_language(detected_language) or _infer_aligner_language_from_text(transcript)
if transcript and aligner_language:
align_segments = _align_subtitle_segments(
audio_path,
transcript,
aligner_language,
aligner_model_dir,
dtype,
device,
unload_model,
)
result_summaries.append(f"align={_summarize_value(align_segments)}")
if align_segments:
return _restore_subtitle_punctuation(
align_segments,
transcript,
aligner_language,
add_fallback=False,
add_final_fallback=True,
)
raise RuntimeError(
"Qwen3-ASR did not return timestamped subtitle text. "
+ "; ".join(result_summaries)
)
finally:
if unload_model:
cleanup_model_memory(model)