"""Count o200k tokens (proxy for Llama/Qwen tokenizers) of every ```text fence in the given markdown files; .json args are counted as-is. Usage: python count_prompt_tokens.py FILE.md [FILE.md ...] [FILE.json ...] """ import re import sys import tiktoken enc = tiktoken.get_encoding("o200k_base") for path in sys.argv[1:]: with open(path, encoding="utf-8") as fh: text = fh.read() print(path) if path.endswith(".json"): n = len(enc.encode(text)) print(f" whole file: {n:5d} tok ({len(text)} chars)") continue fences = re.findall(r"```text\n(.*?)\n```", text, re.S) total = 0 for i, fence in enumerate(fences, 1): t = len(enc.encode(fence)) total += t first = fence.lstrip().splitlines()[0][:60] print(f" fence {i}: {t:5d} tok ({len(fence):6d} chars) | {first}") print(f" total: {total} tok")