MM-prompter/scripts/validate_outputs.py

296 lines
12 KiB
Python

#!/usr/bin/env python3
"""Validator for mm-prompter skill outputs.
Checks the output contract described in SKILL.md:
- segments.txt: n segments, n-1 "|" separators, six sections per segment in order
- global gapless Subject numbering, <Picture N> paired with <Subject N>
- image-map.txt manifest covers every subject; Image-N.json valid (two keys, 16:9)
- no <Video> labels; lyrics (optional file) never appear in the prompts
- audio segments: exact summary prefix, exact audio template, lip-sync phrase,
fully_copy retention line
- [Shot 1] has no timestamp; shot numbers contiguous; timestamps only on N>=2
Usage:
python3 scripts/validate_outputs.py --segments segments.txt [--expected-n 3] \
[--dir .] [--manifest image-map.txt] [--lyrics-file lyrics.txt] [--expect-audio]
Exit code 0 = all checks passed, 1 = failures listed.
"""
import argparse
import json
import re
import sys
from pathlib import Path
SECTION_ORDER = [
"subject_definitions",
"summary",
"retention_analysis",
"detailed_description",
"overall_soundscape",
"non_diegetic_music",
]
AUDIO_TEMPLATE_RE = re.compile(
r"overall_soundscape:\s*"
r"(?:<Subject (\d+)> is Singing the words of <Audio 1>\s*)+"
r"non_diegetic_music:\s*N/A\s*$"
)
LIPSYNC_RE = re.compile(r"<Subject (\d+)> lips are synced to the audio in <Audio 1>")
AUDIO_RETENTION_RE = re.compile(
r"<Audio 1>: fully_copy - <Audio 1> is reused 1:1 as the target video's "
r"complete final audio track\."
)
SHOT_RE = re.compile(r"\[Shot (\d+)\]( At (\d{2}):(\d{2})\.(\d{3}))?")
SUBJECT_RE = re.compile(r"<Subject (\d+)>")
PICTURE_RE = re.compile(r"<Picture (\d+)>")
def fail(errors, seg_no, msg):
errors.append(f"[segment {seg_no}] {msg}" if seg_no else msg)
def split_sections(segment_text, seg_no, errors):
"""Return {name: content} or None if headings are missing/out of order."""
positions = []
for name in SECTION_ORDER:
matches = list(re.finditer(rf"^{name}:\s*$", segment_text, re.MULTILINE))
if len(matches) != 1:
errors.append(f"[segment {seg_no}] section '{name}:' must appear exactly "
f"once on its own line (found {len(matches)})")
return None
positions.append((name, matches[0].start(), matches[0].end()))
for i in range(1, len(positions)):
if positions[i][1] <= positions[i - 1][1]:
errors.append(f"[segment {seg_no}] sections out of order at '{positions[i][0]}'")
return None
sections = {}
for i, (name, start, end) in enumerate(positions):
next_start = positions[i + 1][1] if i + 1 < len(positions) else len(segment_text)
sections[name] = segment_text[end:next_start].strip()
return sections
def normalize(text):
return re.sub(r"\s+", " ", text).strip().lower()
def check_segment(seg_text, seg_no, errors):
"""Per-segment checks. Returns (defined_subjects, has_audio)."""
sections = split_sections(seg_text, seg_no, errors)
if sections is None:
return set(), False
defs = sections["subject_definitions"]
body_after_defs = seg_text.split("detailed_description:", 1)[1] \
if "detailed_description:" in seg_text else ""
defined = set(SUBJECT_RE.findall(defs))
used_after_defs = set(SUBJECT_RE.findall(body_after_defs))
if defined and not used_after_defs:
fail(errors, seg_no, "no subject label is used after subject_definitions; "
"defined subjects must appear in the body sections")
# Picture pairing: each definition line must cite <Picture N> with N == subject N
for line in [ln for ln in defs.splitlines() if ln.strip()]:
if line.lstrip().startswith("<Audio"):
continue
m = re.match(r"\s*<Subject (\d+)>", line)
if not m:
fail(errors, seg_no, f"subject_definitions line does not start with a "
f"<Subject N> label: {line.strip()[:60]}...")
continue
n = m.group(1)
pics = PICTURE_RE.findall(line)
if pics != [n]:
fail(errors, seg_no, f"<Subject {n}> definition must cite exactly "
f"<Picture {n}> (found {pics})")
# Summary prefix
summary = sections["summary"]
has_audio = "<Audio" in seg_text
if has_audio:
if not summary.startswith("[reference generation + audio reuse]"):
fail(errors, seg_no, "audio segment summary must start with "
"'[reference generation + audio reuse]'")
if not AUDIO_TEMPLATE_RE.search(seg_text):
fail(errors, seg_no, "audio segment must end with the exact fixed audio "
"template: 'overall_soundscape:' / '<Subject N> is "
"Singing the words of <Audio 1>' / 'non_diegetic_music:' / 'N/A'")
if not LIPSYNC_RE.search(sections["detailed_description"]):
fail(errors, seg_no, "audio segment detailed_description must contain "
"'<Subject N> lips are synced to the audio in <Audio 1>'")
if not AUDIO_RETENTION_RE.search(sections["retention_analysis"]):
fail(errors, seg_no, "audio segment retention_analysis must contain the "
"exact fully_copy line for <Audio 1>")
if "<d>" in sections["detailed_description"]:
print(f"WARN: [segment {seg_no}] <d> block inside an audio segment; song "
f"lyrics must never be written out", file=sys.stderr)
else:
if not summary.startswith("[reference generation]"):
fail(errors, seg_no, "summary must start with '[reference generation]'")
if "<Audio" in seg_text:
fail(errors, seg_no, "segment mixes <Audio> usage with non-audio prefix")
# Shots
shots = SHOT_RE.findall(sections["detailed_description"])
shot_lines = re.findall(r"^\[Shot (\d+)\].*$", sections["detailed_description"],
re.MULTILINE)
if shot_lines:
nums = [int(n) for n in shot_lines]
if nums != list(range(1, len(nums) + 1)):
fail(errors, seg_no, f"shot numbers must start at [Shot 1] and be "
f"contiguous (found {nums})")
prev_ts = 0.0
for m in SHOT_RE.finditer(sections["detailed_description"]):
n = int(m.group(1))
if m.group(2):
if n == 1:
fail(errors, seg_no, "[Shot 1] must not carry a timestamp")
ts = int(m.group(3)) * 60 + int(m.group(4)) + int(m.group(5)) / 1000.0
if ts < prev_ts:
fail(errors, seg_no, f"timestamp on [Shot {n}] goes backwards")
prev_ts = ts
elif n > 1:
fail(errors, seg_no, f"[Shot {n}] must carry an 'At MM:SS.mmm' timestamp")
if not shots:
fail(errors, seg_no, "detailed_description contains no [Shot N] block")
return defined, has_audio
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--segments", required=True)
ap.add_argument("--expected-n", type=int, default=None)
ap.add_argument("--dir", default=".", help="directory containing Image-N.json files")
ap.add_argument("--manifest", default=None, help="default: <dir>/image-map.txt")
ap.add_argument("--lyrics-file", default=None)
ap.add_argument("--expect-audio", action="store_true",
help="fail unless at least one segment references <Audio>")
args = ap.parse_args()
errors = []
seg_path = Path(args.segments)
if not seg_path.exists():
print(f"FAIL: {seg_path} not found")
return 1
raw = seg_path.read_text(encoding="utf-8")
parts = raw.split("|")
n = len(parts)
if args.expected_n is not None and n != args.expected_n:
errors.append(f"segments.txt has {n} segments, expected {args.expected_n}")
if raw.endswith("|") or raw.startswith("|"):
errors.append("segments.txt must not start or end with a '|' separator")
for i, part in enumerate(parts, 1):
if not part.strip():
errors.append(f"segment {i} is empty")
subjects, audio_flags = set(), []
for i, part in enumerate(parts, 1):
defined, has_audio = check_segment(part.strip(), i, errors)
subjects |= defined
audio_flags.append(has_audio)
if "<Video" in raw:
errors.append("segments.txt contains a <Video> label; this skill never uses "
"video references")
all_nums = set(int(x) for x in SUBJECT_RE.findall(raw))
if all_nums and all_nums != set(range(1, max(all_nums) + 1)):
errors.append(f"global Subject numbering must be gapless 1..{max(all_nums)} "
f"(found {sorted(all_nums)})")
# Manifest
manifest_path = Path(args.manifest) if args.manifest else Path(args.dir) / "image-map.txt"
mapping = {}
if not manifest_path.exists():
errors.append(f"manifest not found: {manifest_path}")
else:
for ln_no, line in enumerate(manifest_path.read_text(encoding="utf-8")
.splitlines(), 1):
line = line.strip()
if not line:
continue
m = re.match(r"^Subject (\d+) - (.+)$", line)
if not m:
errors.append(f"manifest line {ln_no} does not match "
f"'Subject N - file': {line}")
continue
num, fname = int(m.group(1)), m.group(2).strip()
if num in mapping:
errors.append(f"manifest lists Subject {num} twice")
mapping[num] = fname
for num in sorted(all_nums):
if num not in mapping:
errors.append(f"Subject {num} has no manifest line")
out_dir = Path(args.dir)
if out_dir.exists():
orphans = {p.name for p in out_dir.glob("Image-*.json")} - {
f for f in mapping.values() if re.fullmatch(r"Image-\d+\.json", f)
}
if orphans:
errors.append(f"orphan Image-*.json files not listed in the manifest: "
f"{sorted(orphans)}")
for num, fname in sorted(mapping.items()):
if re.fullmatch(r"Image-\d+\.json", fname):
m = re.fullmatch(r"Image-(\d+)\.json", fname)
if m and int(m.group(1)) != num:
errors.append(f"manifest maps Subject {num} to {fname}; the sheet "
f"number must follow the subject number")
fpath = out_dir / fname
if not fpath.exists():
errors.append(f"generated sheet missing: {fpath}")
continue
try:
obj = json.loads(fpath.read_text(encoding="utf-8"))
except json.JSONDecodeError as e:
errors.append(f"{fname} is not valid JSON: {e}")
continue
if set(obj.keys()) != {"expanded_caption", "aspect_ratio"}:
errors.append(f"{fname} must have exactly the two keys "
f"expanded_caption and aspect_ratio (found "
f"{sorted(obj.keys())})")
continue
if obj["aspect_ratio"] != "16:9":
errors.append(f"{fname} aspect_ratio must be \"16:9\" "
f"(found {obj['aspect_ratio']!r})")
if not isinstance(obj["expanded_caption"], str) or \
len(obj["expanded_caption"].strip()) < 40:
errors.append(f"{fname} expanded_caption is missing or too short")
# Lyrics must never appear
if args.lyrics_file:
lpath = Path(args.lyrics_file)
if not lpath.exists():
errors.append(f"lyrics file not found: {lpath}")
else:
norm_raw = normalize(raw)
for line in lpath.read_text(encoding="utf-8").splitlines():
words = normalize(line)
if len(words.split()) >= 4 and words in norm_raw:
errors.append(f"lyric line leaked into the prompts: "
f"{words[:60]}...")
if args.expect_audio and not any(audio_flags):
errors.append("no segment references <Audio>; --expect-audio was set")
if errors:
print(f"FAIL: {len(errors)} problem(s)")
for e in errors:
print(f" - {e}")
return 1
print(f"PASS: {n} segment(s), subjects {sorted(subjects)}, "
f"audio={'yes' if any(audio_flags) else 'no'}")
return 0
if __name__ == "__main__":
sys.exit(main())