- h3_scene_agent_system_prompt.md: outline turn + one-scene-per-turn writer with a truthful _handoff chain (style_and_cast frozen at outline) - h3_plan_assembler_system_prompt.md: verbatim-copy LLM merge step (deterministic alternative: build_plan.py) - h3_plan_json_system_prompt.md: single-pass variant for short 2-4 scene stories; prompt_prefix fixed to [""], native H3 three-line shot template - verify_plan_rules.py: node-replica validator (plan + fragment modes, handoff-chain checks, embedded pass/fail suite) - build_plan.py: deterministic fragment merger with --check for existing plans - h3_agent_workflow_examples/: worked 3-scene greenhouse courier run (outline -> 3 fragments -> validated plan.json, 1042 frames = 43.42s)
29 lines
891 B
Python
29 lines
891 B
Python
"""Count o200k tokens (proxy for Llama/Qwen tokenizers) of every ```text fence
|
|
in the given markdown files; .json args are counted as-is.
|
|
|
|
Usage: python count_prompt_tokens.py FILE.md [FILE.md ...] [FILE.json ...]
|
|
"""
|
|
import re
|
|
import sys
|
|
|
|
import tiktoken
|
|
|
|
enc = tiktoken.get_encoding("o200k_base")
|
|
|
|
for path in sys.argv[1:]:
|
|
with open(path, encoding="utf-8") as fh:
|
|
text = fh.read()
|
|
print(path)
|
|
if path.endswith(".json"):
|
|
n = len(enc.encode(text))
|
|
print(f" whole file: {n:5d} tok ({len(text)} chars)")
|
|
continue
|
|
fences = re.findall(r"```text\n(.*?)\n```", text, re.S)
|
|
total = 0
|
|
for i, fence in enumerate(fences, 1):
|
|
t = len(enc.encode(fence))
|
|
total += t
|
|
first = fence.lstrip().splitlines()[0][:60]
|
|
print(f" fence {i}: {t:5d} tok ({len(fence):6d} chars) | {first}")
|
|
print(f" total: {total} tok")
|