parity harness, pinned to the python tokeniser over 10k lines

parity/oracle/tokens.py

@@ -0,0 +1,26 @@
+"""Token counts from the Python tokeniser, one per line of the sample.
+
+Reads the sample on stdin as UTF-8 bytes and splits it on newlines itself,
+rather than iterating the stream, so that universal-newline translation
+cannot silently change how many lines there are.
+
+    python3 oracle/tokens.py < sample.txt > golden/tokens.counts
+"""
+
+import sys
+
+import tiktoken
+
+ENC = tiktoken.get_encoding("cl100k_base")
+
+
+def main() -> int:
+    text = sys.stdin.buffer.read().decode("utf-8")
+    out = sys.stdout
+    for line in text.split("\n"):
+        out.write(f"{len(ENC.encode(line, disallowed_special=()))}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())