mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 19:27:14 +00:00
27 lines
680 B
Python
27 lines
680 B
Python
"""Token counts from the Python tokeniser, one per line of the sample.
|
|
|
|
Reads the sample on stdin as UTF-8 bytes and splits it on newlines itself,
|
|
rather than iterating the stream, so that universal-newline translation
|
|
cannot silently change how many lines there are.
|
|
|
|
python3 oracle/tokens.py < sample.txt > golden/tokens.counts
|
|
"""
|
|
|
|
import sys
|
|
|
|
import tiktoken
|
|
|
|
ENC = tiktoken.get_encoding("cl100k_base")
|
|
|
|
|
|
def main() -> int:
|
|
text = sys.stdin.buffer.read().decode("utf-8")
|
|
out = sys.stdout
|
|
for line in text.split("\n"):
|
|
out.write(f"{len(ENC.encode(line, disallowed_special=()))}\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|