MM-prompter/scripts/validate_outputs.py

508 lines
23 KiB
Python

#!/usr/bin/env python3
"""Validator for mm-prompter skill outputs.
Checks the output contract described in SKILL.md:
- segments.txt: n segments, n-1 "|" separators
- ref2va mode (default): six sections per segment in order, global gapless
Subject numbering, <Picture N> paired with <Subject N>, summary prefixes,
audio template / lip-sync phrase / fully_copy retention line,
[Shot 1] timestamp rules
- fl2va mode (opt-in): exact instruction line (final shot index + two-decimal
duration), the three fields in order, no reference labels, no ref2va
section labels, shot rules, fixed song sentences when a song is reused
- image-map.txt manifest covers every subject; Image-N.json valid (two keys,
16:9, four-panel photorealistic sheet language)
- Segment-img-N.json per segment (both modes) and Segment-img-N-last.json
per segment (fl2va only): two keys, 16:9, single-frame photorealistic
language, no banned vocabulary
- no <Video> labels; lyrics (optional file) never appear in the prompts
Usage:
python3 scripts/validate_outputs.py --segments segments.txt [--expected-n 3] \
[--mode ref2va|fl2va] [--total-seconds 21] [--dir .] [--manifest image-map.txt] \
[--lyrics-file lyrics.txt] [--expect-audio]
Exit code 0 = all checks passed, 1 = failures listed.
"""
import argparse
import json
import re
import sys
from pathlib import Path
SECTION_ORDER = [
"subject_definitions",
"summary",
"retention_analysis",
"detailed_description",
"overall_soundscape",
"non_diegetic_music",
]
AUDIO_TEMPLATE_RE = re.compile(
r"overall_soundscape:\s*"
r"(?:<Subject (\d+)> is Singing the words of <Audio 1>\s*)+"
r"non_diegetic_music:\s*N/A\s*$"
)
LIPSYNC_RE = re.compile(r"<Subject (\d+)> lips are synced to the audio in <Audio 1>")
AUDIO_RETENTION_RE = re.compile(
r"<Audio 1>: fully_copy - <Audio 1> is reused 1:1 as the target video's "
r"complete final audio track\."
)
SHOT_RE = re.compile(r"\[Shot (\d+)\]( At (\d{2}):(\d{2})\.(\d{3}))?")
SUBJECT_RE = re.compile(r"<Subject (\d+)>")
PICTURE_RE = re.compile(r"<Picture (\d+)>")
FL2VA_INSTRUCTION_RE = re.compile(
r"How the reference pictures align with the target video "
r"— Picture 1 \(from Shot 1\) aligns with the 0\.00-second mark of the "
r"target video; Picture 2 \(from Shot (\d+)\) aligns with the "
r"(\d+)\.(\d{2})-second mark of the target video\."
)
FL2VA_FIELDS = [
"integrated_multimodal_description",
"overall_soundscape",
"non_diegetic_music",
]
FL2VA_SONG_SENTENCE_RE = re.compile(
r"overall_soundscape:\s*"
r"(?:The supplied song plays as the segment's complete final audio track, "
r"with \(S\d+\) performing its vocals live\.\s*)+$"
)
FL2VA_LIPSYNC_RE = re.compile(r"\(S(\d+)\) lips stay synced to the song's vocals")
BANNED_WORDS = ["cartoon", "anime", "manga", "illustration", "stylized",
"cgi", "3d render", "mascot", "avatar", "name tag",
"swatch", "palette strip", "design board", "drafting paper"]
def fail(errors, seg_no, msg):
errors.append(f"[segment {seg_no}] {msg}" if seg_no else msg)
def normalize(text):
return re.sub(r"\s+", " ", text).strip().lower()
def split_sections(segment_text, seg_no, errors):
"""Return {name: content} or None if headings are missing/out of order."""
positions = []
for name in SECTION_ORDER:
matches = list(re.finditer(rf"^{name}:\s*$", segment_text, re.MULTILINE))
if len(matches) != 1:
errors.append(f"[segment {seg_no}] section '{name}:' must appear exactly "
f"once on its own line (found {len(matches)})")
return None
positions.append((name, matches[0].start(), matches[0].end()))
for i in range(1, len(positions)):
if positions[i][1] <= positions[i - 1][1]:
errors.append(f"[segment {seg_no}] sections out of order at '{positions[i][0]}'")
return None
sections = {}
for i, (name, start, end) in enumerate(positions):
next_start = positions[i + 1][1] if i + 1 < len(positions) else len(segment_text)
sections[name] = segment_text[end:next_start].strip()
return sections
def check_shots(body, seg_no, errors):
"""Shared shot checks: [Shot 1] no timestamp, contiguous numbers, increasing."""
shot_nums = [int(m.group(1)) for m in SHOT_RE.finditer(body)]
shot_lines = re.findall(r"^\[Shot (\d+)\]", body, re.MULTILINE)
if shot_lines:
nums = [int(x) for x in shot_lines]
if nums != list(range(1, len(nums) + 1)):
fail(errors, seg_no, f"shot numbers must start at [Shot 1] and be "
f"contiguous (found {nums})")
elif shot_nums and shot_nums != list(range(1, max(shot_nums) + 1)):
fail(errors, seg_no, f"shot numbers must start at [Shot 1] and be "
f"contiguous (found {shot_nums})")
prev_ts = 0.0
for m in SHOT_RE.finditer(body):
n = int(m.group(1))
if m.group(2):
if n == 1:
fail(errors, seg_no, "[Shot 1] must not carry a timestamp")
ts = int(m.group(3)) * 60 + int(m.group(4)) + int(m.group(5)) / 1000.0
if ts < prev_ts:
fail(errors, seg_no, f"timestamp on [Shot {n}] goes backwards")
prev_ts = ts
elif n > 1:
fail(errors, seg_no, f"[Shot {n}] must carry an 'At MM:SS.mmm' timestamp")
if not shot_nums:
fail(errors, seg_no, "description contains no [Shot N] block")
return shot_nums
def check_segment_ref2va(seg_text, seg_no, errors):
"""Per-segment ref2va checks. Returns (defined_subjects, has_audio)."""
sections = split_sections(seg_text, seg_no, errors)
if sections is None:
return set(), False
if "integrated_multimodal_description" in seg_text:
fail(errors, seg_no, "ref2va prompts must not use the FL2VA field "
"'integrated_multimodal_description'")
defs = sections["subject_definitions"]
body_after_defs = seg_text.split("detailed_description:", 1)[1] \
if "detailed_description:" in seg_text else ""
defined = set(SUBJECT_RE.findall(defs))
used_after_defs = set(SUBJECT_RE.findall(body_after_defs))
if defined and not used_after_defs:
fail(errors, seg_no, "no subject label is used after subject_definitions; "
"defined subjects must appear in the body sections")
# Picture pairing: each definition line must cite <Picture N> with N == subject N
for line in [ln for ln in defs.splitlines() if ln.strip()]:
if line.lstrip().startswith("<Audio"):
continue
m = re.match(r"\s*<Subject (\d+)>", line)
if not m:
fail(errors, seg_no, f"subject_definitions line does not start with a "
f"<Subject N> label: {line.strip()[:60]}...")
continue
n = m.group(1)
pics = PICTURE_RE.findall(line)
if pics != [n]:
fail(errors, seg_no, f"<Subject {n}> definition must cite exactly "
f"<Picture {n}> (found {pics})")
# Summary prefix
summary = sections["summary"]
has_audio = "<Audio" in seg_text
if has_audio:
if not summary.startswith("[reference generation + audio reuse]"):
fail(errors, seg_no, "audio segment summary must start with "
"'[reference generation + audio reuse]'")
if not AUDIO_TEMPLATE_RE.search(seg_text):
fail(errors, seg_no, "audio segment must end with the exact fixed audio "
"template: 'overall_soundscape:' / '<Subject N> is "
"Singing the words of <Audio 1>' / 'non_diegetic_music:' / 'N/A'")
if not LIPSYNC_RE.search(sections["detailed_description"]):
fail(errors, seg_no, "audio segment detailed_description must contain "
"'<Subject N> lips are synced to the audio in <Audio 1>'")
if not AUDIO_RETENTION_RE.search(sections["retention_analysis"]):
fail(errors, seg_no, "audio segment retention_analysis must contain the "
"exact fully_copy line for <Audio 1>")
if "<d>" in sections["detailed_description"]:
print(f"WARN: [segment {seg_no}] <d> block inside an audio segment; song "
f"lyrics must never be written out", file=sys.stderr)
else:
if not summary.startswith("[reference generation]"):
fail(errors, seg_no, "summary must start with '[reference generation]'")
if "<Audio" in seg_text:
fail(errors, seg_no, "segment mixes <Audio> usage with non-audio prefix")
# Shots
check_shots(sections["detailed_description"], seg_no, errors)
return defined, has_audio
def check_segment_fl2va(seg_text, seg_no, errors, expected_duration=None):
"""Per-segment FL2VA checks. Returns has_audio."""
lines = seg_text.split("\n")
first = lines[0].strip() if lines else ""
m = FL2VA_INSTRUCTION_RE.fullmatch(first)
cited_shot = None
if not m:
fail(errors, seg_no, "first line must be the exact FL2VA instruction: 'How "
"the reference pictures align with the target video — Picture 1 (from "
"Shot 1) aligns with the 0.00-second mark of the target video; Picture "
"2 (from Shot N) aligns with the S.SS-second mark of the target video.'")
else:
cited_shot = int(m.group(1))
secs = int(m.group(2)) + int(m.group(3)) / 100.0
if expected_duration is not None and abs(secs - expected_duration) > 0.0051:
fail(errors, seg_no, f"instruction pins Picture 2 to the {secs:.2f}-second "
f"mark, but the segment duration is "
f"{expected_duration:.2f} seconds (total/n, two decimals)")
if len(lines) < 2 or lines[1].strip() != "":
fail(errors, seg_no, "the instruction line must be followed by exactly "
"one blank line")
# The three fields, in order, inline
spans = []
for name in FL2VA_FIELDS:
matches = list(re.finditer(rf"^{name}: ", seg_text, re.MULTILINE))
if len(matches) != 1:
fail(errors, seg_no, f"field '{name}:' must appear exactly once as an "
f"inline field (found {len(matches)})")
spans.append(None)
else:
spans.append(matches[0])
integ = ""
if all(s is not None for s in spans):
starts = [s.start() for s in spans]
if starts != sorted(starts):
fail(errors, seg_no, "fields out of order; expected "
"integrated_multimodal_description, overall_soundscape, "
"non_diegetic_music")
integ = seg_text[spans[0].end():spans[1].start()].strip()
soundscape = seg_text[spans[1].end():spans[2].start()].strip()
music = seg_text[spans[2].end():].strip()
if not integ:
fail(errors, seg_no, "integrated_multimodal_description is empty")
elif not integ.startswith("[Shot 1]"):
fail(errors, seg_no, "integrated_multimodal_description must start "
"with '[Shot 1]'")
if not soundscape:
fail(errors, seg_no, "overall_soundscape is empty")
if not music:
fail(errors, seg_no, "non_diegetic_music is empty")
# ref2va machinery must not leak into FL2VA prompts
for name in SECTION_ORDER:
if re.search(rf"^{name}:\s*$", seg_text, re.MULTILINE):
fail(errors, seg_no, f"FL2VA prompts must not contain the ref2va section "
f"label '{name}:'")
for label in ("<Subject", "<Picture", "<Video", "<Audio"):
if label in seg_text:
fail(errors, seg_no, f"FL2VA prompts must not contain full-reference "
f"labels ({label} N)")
# Shots
shot_nums = check_shots(integ or seg_text, seg_no, errors)
if cited_shot is not None and shot_nums:
if max(shot_nums) != cited_shot:
fail(errors, seg_no, f"instruction cites Picture 2 from Shot {cited_shot} "
f"but the highest shot in the description is "
f"Shot {max(shot_nums)}")
# Song reuse
has_song = bool(FL2VA_SONG_SENTENCE_RE.search(seg_text))
lipsync = FL2VA_LIPSYNC_RE.search(seg_text)
if has_song:
if not lipsync:
fail(errors, seg_no, "FL2VA singing segment must contain "
"'(S1) lips stay synced to the song's vocals' inside "
"integrated_multimodal_description")
if not re.search(r"non_diegetic_music:\s*N/A", seg_text):
fail(errors, seg_no, "FL2VA singing segment must set non_diegetic_music "
"to N/A (the song is the complete final audio track)")
if "<d>" in seg_text:
print(f"WARN: [segment {seg_no}] <d> block inside a singing FL2VA "
f"segment; song lyrics must never be written out", file=sys.stderr)
elif lipsync:
fail(errors, seg_no, "'(S1) lips stay synced to the song's vocals' present but "
"the fixed song sentence is missing from overall_soundscape: "
"'The supplied song plays as the segment's complete final "
"audio track, with (S1) performing its vocals live.'")
return has_song
def load_two_key_json(fpath, errors):
"""Return the parsed object or None; logs contract errors."""
try:
obj = json.loads(fpath.read_text(encoding="utf-8"))
except json.JSONDecodeError as e:
errors.append(f"{fpath.name} is not valid JSON: {e}")
return None
if set(obj.keys()) != {"expanded_caption", "aspect_ratio"}:
errors.append(f"{fpath.name} must have exactly the two keys "
f"expanded_caption and aspect_ratio (found {sorted(obj.keys())})")
return None
if obj["aspect_ratio"] != "16:9":
errors.append(f"{fpath.name} aspect_ratio must be \"16:9\" "
f"(found {obj['aspect_ratio']!r})")
if not isinstance(obj["expanded_caption"], str) or \
len(obj["expanded_caption"].strip()) < 40:
errors.append(f"{fpath.name} expanded_caption is missing or too short")
return None
return obj
def check_frame_caption(fname, obj, errors):
"""Frame-file caption checks (single full-bleed film still)."""
cap = obj["expanded_caption"].lower()
hits = [w for w in BANNED_WORDS if w in cap]
if hits:
errors.append(f"{fname} caption contains illustration/design-board vocabulary "
f"that produces cartoons: {hits}")
if "photorealistic" not in cap and "photograph" not in cap:
errors.append(f"{fname} caption must pin the photorealistic photographic "
f"style for the frame")
if "four panels" in cap:
errors.append(f"{fname} caption describes a multi-panel sheet; a segment "
f"frame must be one single full-bleed film still")
if "supplied character reference images" not in cap and \
"supplied reference images" not in cap:
errors.append(f"{fname} caption must anchor its characters to the supplied "
f"character reference images")
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--segments", required=True)
ap.add_argument("--expected-n", type=int, default=None)
ap.add_argument("--mode", choices=["ref2va", "fl2va"], default="ref2va",
help="ref2va (default) or fl2va first-last mode")
ap.add_argument("--total-seconds", type=float, default=None,
help="total video length; enables the S.SS duration check in "
"fl2va mode")
ap.add_argument("--dir", default=".", help="directory containing the image files")
ap.add_argument("--manifest", default=None, help="default: <dir>/image-map.txt")
ap.add_argument("--lyrics-file", default=None)
ap.add_argument("--expect-audio", action="store_true",
help="fail unless at least one segment reuses audio")
args = ap.parse_args()
errors = []
seg_path = Path(args.segments)
if not seg_path.exists():
print(f"FAIL: {seg_path} not found")
return 1
raw = seg_path.read_text(encoding="utf-8")
parts = raw.split("|")
n = len(parts)
if args.expected_n is not None and n != args.expected_n:
errors.append(f"segments.txt has {n} segments, expected {args.expected_n}")
if raw.endswith("|") or raw.startswith("|"):
errors.append("segments.txt must not start or end with a '|' separator")
for i, part in enumerate(parts, 1):
if not part.strip():
errors.append(f"segment {i} is empty")
expected_duration = None
if args.mode == "fl2va" and args.total_seconds:
expected_duration = round(args.total_seconds / n, 2)
subjects, audio_flags = set(), []
for i, part in enumerate(parts, 1):
if args.mode == "fl2va":
has_audio = check_segment_fl2va(part.strip(), i, errors, expected_duration)
else:
defined, has_audio = check_segment_ref2va(part.strip(), i, errors)
subjects |= defined
audio_flags.append(has_audio)
if "<Video" in raw:
errors.append("segments.txt contains a <Video> label; this skill never uses "
"video references")
all_nums = set(int(x) for x in SUBJECT_RE.findall(raw))
if args.mode == "ref2va" and all_nums and \
all_nums != set(range(1, max(all_nums) + 1)):
errors.append(f"global Subject numbering must be gapless 1..{max(all_nums)} "
f"(found {sorted(all_nums)})")
# Manifest
manifest_path = Path(args.manifest) if args.manifest else Path(args.dir) / "image-map.txt"
mapping = {}
if not manifest_path.exists():
errors.append(f"manifest not found: {manifest_path}")
else:
for ln_no, line in enumerate(manifest_path.read_text(encoding="utf-8")
.splitlines(), 1):
line = line.strip()
if not line:
continue
m = re.match(r"^Subject (\d+) - (.+)$", line)
if not m:
errors.append(f"manifest line {ln_no} does not match "
f"'Subject N - file': {line}")
continue
num, fname = int(m.group(1)), m.group(2).strip()
if num in mapping:
errors.append(f"manifest lists Subject {num} twice")
mapping[num] = fname
if mapping and set(mapping) != set(range(1, max(mapping) + 1)):
errors.append(f"manifest subject numbering must be gapless 1.."
f"{max(mapping)} (found {sorted(mapping)})")
for num in sorted(all_nums):
if num not in mapping:
errors.append(f"Subject {num} has no manifest line")
out_dir = Path(args.dir)
if out_dir.exists():
orphans = {p.name for p in out_dir.glob("Image-*.json")} - {
f for f in mapping.values() if re.fullmatch(r"Image-\d+\.json", f)
}
if orphans:
errors.append(f"orphan Image-*.json files not listed in the manifest: "
f"{sorted(orphans)}")
for num, fname in sorted(mapping.items()):
if re.fullmatch(r"Image-\d+\.json", fname):
m = re.fullmatch(r"Image-(\d+)\.json", fname)
if m and int(m.group(1)) != num:
errors.append(f"manifest maps Subject {num} to {fname}; the sheet "
f"number must follow the subject number")
fpath = out_dir / fname
if not fpath.exists():
errors.append(f"generated sheet missing: {fpath}")
continue
obj = load_two_key_json(fpath, errors)
if obj is None:
continue
cap = obj["expanded_caption"].lower()
if "four panels" not in cap:
errors.append(f"{fname} caption must describe the four-panel "
f"turnaround sheet (face close-up, front, side, back)")
hits = [w for w in BANNED_WORDS if w in cap]
if hits:
errors.append(f"{fname} caption contains illustration/design-board "
f"vocabulary that produces cartoons: {hits}")
if "photorealistic" not in cap and "photograph" not in cap:
errors.append(f"{fname} caption must pin the photorealistic "
f"photographic style for the sheet")
# Frame image prompts
if out_dir.exists():
expected_frames = {f"Segment-img-{i}.json" for i in range(1, n + 1)}
if args.mode == "fl2va":
expected_frames |= {f"Segment-img-{i}-last.json" for i in range(1, n + 1)}
for fname in sorted(expected_frames):
fpath = out_dir / fname
if not fpath.exists():
kind = "last-frame" if "-last." in fname else "first-frame"
errors.append(f"{kind} prompt missing: {fpath}")
continue
obj = load_two_key_json(fpath, errors)
if obj is None:
continue
check_frame_caption(fname, obj, errors)
stray = {p.name for p in out_dir.glob("Segment-img-*.json")} - expected_frames
if stray:
hint = (" (Segment-img-N-last.json files exist only in fl2va mode)"
if args.mode == "ref2va" else "")
errors.append(f"unexpected Segment-img files not matching {n} segment(s): "
f"{sorted(stray)}{hint}")
# Lyrics must never appear
if args.lyrics_file:
lpath = Path(args.lyrics_file)
if not lpath.exists():
errors.append(f"lyrics file not found: {lpath}")
else:
norm_raw = normalize(raw)
for line in lpath.read_text(encoding="utf-8").splitlines():
words = normalize(line)
if len(words.split()) >= 4 and words in norm_raw:
errors.append(f"lyric line leaked into the prompts: "
f"{words[:60]}...")
if args.expect_audio and not any(audio_flags):
errors.append("no segment reuses audio; --expect-audio was set")
if errors:
print(f"FAIL: {len(errors)} problem(s)")
for e in errors:
print(f" - {e}")
return 1
print(f"PASS: mode={args.mode}, {n} segment(s), "
f"subjects {sorted(subjects) if subjects else 'per manifest'}, "
f"audio={'yes' if any(audio_flags) else 'no'}")
return 0
if __name__ == "__main__":
sys.exit(main())