parity harness, pinned to the python tokeniser over 10k lines

Cargo.lock

@@ -94,6 +94,13 @@ version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
+[[package]]
+name = "parity"
+version = "0.1.0"
+dependencies = [
+ "textkit",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.107"

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "3"
-members = ["crates/*"]
+members = ["crates/*", "parity"]
 
 [workspace.package]
 version = "0.1.0"

parity/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "parity"
+description = "Differential tests against the Python implementations these tools were ported from."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
+
+[features]
+# Re-runs the Python oracle during the test instead of trusting the golden
+# files. Off by default so the suite passes on a machine with no Python.
+python = []
+
+[dependencies]
+textkit.workspace = true

parity/KNOWN.md

@@ -0,0 +1,21 @@
+# Known divergences
+
+Places where the Rust and the Python deliberately do not agree, each with the
+reason it is acceptable. A divergence that is not written down here is a bug,
+and loosening a test instead of adding an entry here is how a port rots.
+
+Format:
+
+```
+## <function> — <one-line summary>
+
+**Python:** what it does.
+**Rust:** what it does instead.
+**Why this is acceptable:** the argument, including what input class is
+affected and how often it occurs in practice.
+**Measured:** how many records out of how many differ.
+```
+
+---
+
+None recorded yet.

parity/README.md

@@ -0,0 +1,53 @@
+# parity
+
+Differential tests against the Python implementations these tools were ported
+from.
+
+A reimplementation that is 98% right is worse than no reimplementation,
+because the missing 2% is silent: a chunk boundary moves, a verdict flips on
+one class of file, and nothing fails — the index is just quietly worse. These
+tests exist to make that loud.
+
+## How it works
+
+Each ported function has a **golden file** under `golden/`, holding what the
+Python produced for a fixture. The default test run compares Rust against
+those files and needs nothing but Rust:
+
+```sh
+cargo test -p parity
+```
+
+With an interpreter that has the Python dependencies installed, the oracles
+can be re-run live, which catches a golden file drifting away from a Python
+implementation that has moved on:
+
+```sh
+CATENA_PARITY_PYTHON=/path/to/venv/bin/python cargo test -p parity --features python
+```
+
+## Regenerating
+
+Golden files are never regenerated by a test — a golden file that rewrites
+itself when it fails is a test that cannot fail. Regenerate deliberately,
+read the diff, and commit it with the reason:
+
+```sh
+CATENA_PARITY_PYTHON=/path/to/venv/bin/python cargo run -p parity --bin regen
+```
+
+## Fixtures
+
+Generated, synthetic, or public domain — nothing else. `src/sample.rs`
+generates its fixture deterministically from a fixed seed, so both sides
+demonstrably read the same bytes and the clone stays small.
+
+Where a heuristic was tuned against copyrighted text, commit the **derived
+signal** and not the text: the page character counts, the ASCII scores, the
+word-shape tuples, the cue timings. A parity test over a text-layer classifier
+needs the score inputs, not the book.
+
+## Divergences
+
+Where Rust and Python legitimately differ, record it in `KNOWN.md` with the
+reason. Do not loosen a test to accommodate one.

parity/oracle/tokens.py

@@ -0,0 +1,26 @@
+"""Token counts from the Python tokeniser, one per line of the sample.
+
+Reads the sample on stdin as UTF-8 bytes and splits it on newlines itself,
+rather than iterating the stream, so that universal-newline translation
+cannot silently change how many lines there are.
+
+    python3 oracle/tokens.py < sample.txt > golden/tokens.counts
+"""
+
+import sys
+
+import tiktoken
+
+ENC = tiktoken.get_encoding("cl100k_base")
+
+
+def main() -> int:
+    text = sys.stdin.buffer.read().decode("utf-8")
+    out = sys.stdout
+    for line in text.split("\n"):
+        out.write(f"{len(ENC.encode(line, disallowed_special=()))}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -0,0 +1,35 @@
+//! Regenerates the golden files by running the Python oracles.
+//!
+//! Never run as part of a test: a golden file that regenerates itself when
+//! it fails is a test that cannot fail. Run it deliberately, read the diff,
+//! and commit the change with the reason.
+//!
+//!     cargo run -p parity --bin regen
+
+use std::process::ExitCode;
+
+fn main() -> ExitCode {
+    match regen_tokens() {
+        Ok(path) => {
+            eprintln!("wrote {path}");
+            ExitCode::SUCCESS
+        }
+        Err(e) => {
+            eprintln!("regen: {e}");
+            ExitCode::FAILURE
+        }
+    }
+}
+
+fn regen_tokens() -> Result<String, String> {
+    let header = format!(
+        "# cl100k_base token counts, one per line of parity::sample::standard()\n\
+         # seed={:#018x} lines={}\n",
+        parity::sample::SEED,
+        parity::sample::LINES,
+    );
+    let counts = parity::run_oracle("tokens.py", &parity::sample::standard())?;
+    let path = parity::golden("tokens.counts");
+    std::fs::write(&path, format!("{header}{counts}")).map_err(|e| e.to_string())?;
+    Ok(path.display().to_string())
+}

parity/src/lib.rs

@@ -0,0 +1,88 @@
+//! Differential tests against the Python implementations these tools were
+//! ported from.
+//!
+//! The risk in this port is not that Rust is hard. It is that the heuristics
+//! were tuned against a real corpus over a long time, and a reimplementation
+//! that is 98% right is worse than no port at all, because the 2% is silent.
+//!
+//! So every ported function is pinned to what the Python produced, by golden
+//! files under `parity/golden/`. The default test run compares Rust against
+//! those files and needs nothing but Rust. With `--features python` the
+//! oracle is re-run live, which catches the golden files drifting away from
+//! a Python implementation that has moved on.
+//!
+//! Fixtures are generated, synthetic, or public domain. No corpus text is
+//! committed here, and for copyrighted sources only the derived signal — the
+//! scores, the counts, the timings — is ever checked in.
+
+pub mod sample;
+
+use std::path::{Path, PathBuf};
+
+/// The `parity/` directory, wherever this crate was checked out to.
+pub fn dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf()
+}
+
+/// Path to a golden file.
+pub fn golden(name: &str) -> PathBuf {
+    dir().join("golden").join(name)
+}
+
+/// Path to an oracle script.
+pub fn oracle(name: &str) -> PathBuf {
+    dir().join("oracle").join(name)
+}
+
+/// The Python interpreter to run an oracle with.
+///
+/// Taken from `CATENA_PARITY_PYTHON`, because the interpreter that has the
+/// pipeline's dependencies installed is a fact about one machine and must not
+/// be compiled in.
+pub fn python() -> String {
+    std::env::var("CATENA_PARITY_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+/// Run an oracle script over `input` and return its stdout.
+///
+/// The interpreter comes from the environment and the script from this
+/// directory, so nothing about where a Python pipeline lives on a particular
+/// machine is compiled in.
+pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
+    use std::io::Write as _;
+    use std::process::{Command, Stdio};
+
+    let script = oracle(script);
+    let mut child = Command::new(python())
+        .arg(&script)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .map_err(|e| format!("running {} with {}: {e}", script.display(), python()))?;
+    child
+        .stdin
+        .take()
+        .expect("stdin was piped")
+        .write_all(input.as_bytes())
+        .map_err(|e| format!("writing to {}: {e}", script.display()))?;
+    let out = child
+        .wait_with_output()
+        .map_err(|e| format!("waiting for {}: {e}", script.display()))?;
+    if !out.status.success() {
+        return Err(format!("{} exited with {}", script.display(), out.status));
+    }
+    String::from_utf8(out.stdout)
+        .map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
+}
+
+/// Read a golden file, dropping its `#` header lines.
+pub fn read_golden(name: &str) -> Vec<String> {
+    let path = golden(name);
+    let text = std::fs::read_to_string(&path)
+        .unwrap_or_else(|e| panic!("reading {}: {e}", path.display()));
+    text.lines()
+        .filter(|line| !line.starts_with('#'))
+        .map(str::to_string)
+        .collect()
+}

parity/src/sample.rs

@@ -0,0 +1,212 @@
+//! A deterministic sample of awkward text.
+//!
+//! Generated rather than committed, so the fixture costs the clone nothing,
+//! and generated the same way on every machine, so Rust and Python are
+//! demonstrably reading the same bytes. Change anything in here and every
+//! golden file derived from it has to be regenerated.
+//!
+//! The lines are chosen to be hostile to a tokeniser rather than
+//! representative of prose: scripts that are not Latin, combining marks,
+//! zero-width characters, the literal spelling of a control token, numbers,
+//! URLs, markdown, and runs of punctuation. Ordinary English is the easy case
+//! and is well covered by the corpus itself.
+
+/// Lines in the standard sample.
+pub const LINES: usize = 10_000;
+
+/// The seed the standard sample is generated from.
+pub const SEED: u64 = 0x636174656e610001;
+
+/// The standard sample: [`LINES`] lines, newline-separated, no trailing
+/// newline, no carriage returns.
+///
+/// Carriage returns are excluded deliberately: Python's universal-newline
+/// translation would turn a lone `\r` into a line break and the two sides
+/// would then disagree about how many lines there even are.
+pub fn standard() -> String {
+    generate(SEED, LINES)
+}
+
+/// Generate `lines` lines from `seed`.
+pub fn generate(seed: u64, lines: usize) -> String {
+    let mut rng = Rng::new(seed);
+    let mut out = String::new();
+    for i in 0..lines {
+        if i > 0 {
+            out.push('\n');
+        }
+        out.push_str(&line(&mut rng));
+    }
+    out
+}
+
+fn line(rng: &mut Rng) -> String {
+    match rng.below(10) {
+        0 => words(rng, 3, 14),
+        1 => format!("{} {}", pick(rng, SCRIPTS), words(rng, 1, 6)),
+        2 => format!("# {}", words(rng, 2, 7)),
+        3 => format!(
+            "- [{}]({}) {}",
+            words(rng, 1, 4),
+            pick(rng, URLS),
+            words(rng, 0, 5)
+        ),
+        4 => format!("{} {} {}", numbers(rng), pick(rng, PUNCT), numbers(rng)),
+        5 => format!("{}{}", pick(rng, AWKWARD), words(rng, 0, 6)),
+        6 => format!("    {}", pick(rng, CODE)),
+        7 => format!(
+            "{}{}",
+            words(rng, 1, 5),
+            pick(rng, PUNCT).repeat(rng.below(4) + 1)
+        ),
+        8 => words(rng, 20, 60),
+        _ => String::new(),
+    }
+}
+
+fn words(rng: &mut Rng, least: usize, most: usize) -> String {
+    let count = least + rng.below(most.saturating_sub(least) + 1);
+    let mut out = String::new();
+    for i in 0..count {
+        if i > 0 {
+            out.push(' ');
+        }
+        out.push_str(pick(rng, WORDS));
+    }
+    out
+}
+
+fn numbers(rng: &mut Rng) -> String {
+    match rng.below(4) {
+        0 => format!("{}", rng.below(10)),
+        1 => format!("{}", rng.below(100_000)),
+        2 => format!("{}.{}", rng.below(1000), rng.below(1000)),
+        _ => format!("{}-{}", rng.below(2100), rng.below(13)),
+    }
+}
+
+fn pick<'a>(rng: &mut Rng, from: &[&'a str]) -> &'a str {
+    from[rng.below(from.len())]
+}
+
+const WORDS: &[&str] = &[
+    "the",
+    "second",
+    "estate",
+    "manuscript",
+    "marginalia",
+    "colophon",
+    "quire",
+    "recto",
+    "verso",
+    "palimpsest",
+    "incunabulum",
+    "folio",
+    "gloss",
+    "catena",
+    "scholion",
+    "lacuna",
+    "obelus",
+    "supralinear",
+    "hapax",
+    "stemma",
+    "apparatus",
+    "witness",
+    "conjecture",
+    "lemma",
+];
+
+const SCRIPTS: &[&str] = &[
+    "café",
+    "naïve",
+    "Москва",
+    "東京",
+    "مرحبا",
+    "Ἀθῆναι",
+    "עברית",
+    "ॐ",
+    "Ægypt",
+    "first",
+    "Straße",
+];
+
+const URLS: &[&str] = &[
+    "https://example.org/a/b?c=d&e=f",
+    "http://localhost:8080/x",
+    "#anchor-name",
+    "mailto:someone@example.invalid",
+];
+
+const PUNCT: &[&str] = &[".", "!", "?", "…", "—", ";", ":", ",", ")", "\"", "'"];
+
+const CODE: &[&str] = &[
+    "fn main() { println!(\"{}\", 1 + 1); }",
+    "SELECT * FROM t WHERE id = $1;",
+    "{\"v\":1,\"idx\":0}",
+    "<!-- Page 41 -->",
+];
+
+/// Characters that have broken something before: a zero-width joiner, a
+/// non-breaking space, a combining acute, the literal spelling of a control
+/// token, and a tab.
+const AWKWARD: &[&str] = &[
+    "\u{200d}",
+    "\u{00a0}",
+    "e\u{0301}",
+    "<|endoftext|>",
+    "\t",
+    "\u{feff}",
+    "🙂",
+    "\u{1c}",
+];
+
+/// xorshift64*, so the sample is the same on every machine and every run.
+struct Rng(u64);
+
+impl Rng {
+    fn new(seed: u64) -> Self {
+        Self(if seed == 0 { 1 } else { seed })
+    }
+
+    fn next(&mut self) -> u64 {
+        let mut x = self.0;
+        x ^= x >> 12;
+        x ^= x << 25;
+        x ^= x >> 27;
+        self.0 = x;
+        x.wrapping_mul(0x2545_f491_4f6c_dd1d)
+    }
+
+    fn below(&mut self, n: usize) -> usize {
+        if n == 0 {
+            0
+        } else {
+            (self.next() % n as u64) as usize
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_sample_is_the_same_every_time() {
+        assert_eq!(generate(SEED, 50), generate(SEED, 50));
+    }
+
+    #[test]
+    fn a_shorter_sample_is_a_prefix_of_a_longer_one() {
+        let long = generate(SEED, 50);
+        let short = generate(SEED, 10);
+        assert!(long.starts_with(&short));
+    }
+
+    #[test]
+    fn the_standard_sample_has_the_advertised_shape() {
+        let text = standard();
+        assert_eq!(text.split('\n').count(), LINES);
+        assert!(!text.contains('\r'));
+        assert!(!text.ends_with('\n'));
+    }
+}

parity/tests/tokens.rs

@@ -0,0 +1,62 @@
+//! Token counts must agree with the Python tokeniser exactly.
+//!
+//! A count that is one off moves a chunk boundary, and a chunk boundary that
+//! moves means every passage after it in the document is a different passage
+//! than the one already embedded in an index.
+
+#[test]
+fn every_line_counts_the_same_as_python() {
+    let golden = parity::read_golden("tokens.counts");
+    let sample = parity::sample::standard();
+    let lines: Vec<&str> = sample.split('\n').collect();
+
+    assert_eq!(
+        golden.len(),
+        lines.len(),
+        "the golden file has {} counts for {} lines — regenerate it",
+        golden.len(),
+        lines.len()
+    );
+
+    let mut wrong = Vec::new();
+    for (at, (line, expected)) in lines.iter().zip(&golden).enumerate() {
+        let expected: usize = expected
+            .parse()
+            .unwrap_or_else(|e| panic!("golden line {}: {e}", at + 1));
+        let got = textkit::count(line);
+        if got != expected {
+            wrong.push(format!(
+                "line {}: python {expected}, rust {got}, text {line:?}",
+                at + 1
+            ));
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} of {} lines disagree:\n{}",
+        wrong.len(),
+        lines.len(),
+        wrong
+            .iter()
+            .take(20)
+            .cloned()
+            .collect::<Vec<_>>()
+            .join("\n")
+    );
+}
+
+/// Catches the golden file drifting away from a Python that has moved on.
+/// Needs an interpreter with `tiktoken`, named by `CATENA_PARITY_PYTHON`.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let live = parity::run_oracle("tokens.py", &parity::sample::standard())
+        .expect("the oracle should run");
+    let live: Vec<&str> = live.lines().collect();
+    let golden = parity::read_golden("tokens.counts");
+    assert_eq!(live.len(), golden.len(), "line count moved");
+    for (at, (live, golden)) in live.iter().zip(&golden).enumerate() {
+        assert_eq!(live, golden, "golden file is stale at line {}", at + 1);
+    }
+}