mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
parity harness, pinned to the python tokeniser over 10k lines
This commit is contained in:
funkpower
@@ -0,0 +1,35 @@
|
||||
//! Regenerates the golden files by running the Python oracles.
|
||||
//!
|
||||
//! Never run as part of a test: a golden file that regenerates itself when
|
||||
//! it fails is a test that cannot fail. Run it deliberately, read the diff,
|
||||
//! and commit the change with the reason.
|
||||
//!
|
||||
//! cargo run -p parity --bin regen
|
||||
|
||||
use std::process::ExitCode;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
match regen_tokens() {
|
||||
Ok(path) => {
|
||||
eprintln!("wrote {path}");
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("regen: {e}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn regen_tokens() -> Result<String, String> {
|
||||
let header = format!(
|
||||
"# cl100k_base token counts, one per line of parity::sample::standard()\n\
|
||||
# seed={:#018x} lines={}\n",
|
||||
parity::sample::SEED,
|
||||
parity::sample::LINES,
|
||||
);
|
||||
let counts = parity::run_oracle("tokens.py", &parity::sample::standard())?;
|
||||
let path = parity::golden("tokens.counts");
|
||||
std::fs::write(&path, format!("{header}{counts}")).map_err(|e| e.to_string())?;
|
||||
Ok(path.display().to_string())
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
//! Differential tests against the Python implementations these tools were
|
||||
//! ported from.
|
||||
//!
|
||||
//! The risk in this port is not that Rust is hard. It is that the heuristics
|
||||
//! were tuned against a real corpus over a long time, and a reimplementation
|
||||
//! that is 98% right is worse than no port at all, because the 2% is silent.
|
||||
//!
|
||||
//! So every ported function is pinned to what the Python produced, by golden
|
||||
//! files under `parity/golden/`. The default test run compares Rust against
|
||||
//! those files and needs nothing but Rust. With `--features python` the
|
||||
//! oracle is re-run live, which catches the golden files drifting away from
|
||||
//! a Python implementation that has moved on.
|
||||
//!
|
||||
//! Fixtures are generated, synthetic, or public domain. No corpus text is
|
||||
//! committed here, and for copyrighted sources only the derived signal — the
|
||||
//! scores, the counts, the timings — is ever checked in.
|
||||
|
||||
pub mod sample;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// The `parity/` directory, wherever this crate was checked out to.
|
||||
pub fn dir() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf()
|
||||
}
|
||||
|
||||
/// Path to a golden file.
|
||||
pub fn golden(name: &str) -> PathBuf {
|
||||
dir().join("golden").join(name)
|
||||
}
|
||||
|
||||
/// Path to an oracle script.
|
||||
pub fn oracle(name: &str) -> PathBuf {
|
||||
dir().join("oracle").join(name)
|
||||
}
|
||||
|
||||
/// The Python interpreter to run an oracle with.
|
||||
///
|
||||
/// Taken from `CATENA_PARITY_PYTHON`, because the interpreter that has the
|
||||
/// pipeline's dependencies installed is a fact about one machine and must not
|
||||
/// be compiled in.
|
||||
pub fn python() -> String {
|
||||
std::env::var("CATENA_PARITY_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
/// Run an oracle script over `input` and return its stdout.
|
||||
///
|
||||
/// The interpreter comes from the environment and the script from this
|
||||
/// directory, so nothing about where a Python pipeline lives on a particular
|
||||
/// machine is compiled in.
|
||||
pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
|
||||
use std::io::Write as _;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
let script = oracle(script);
|
||||
let mut child = Command::new(python())
|
||||
.arg(&script)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.map_err(|e| format!("running {} with {}: {e}", script.display(), python()))?;
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.expect("stdin was piped")
|
||||
.write_all(input.as_bytes())
|
||||
.map_err(|e| format!("writing to {}: {e}", script.display()))?;
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("waiting for {}: {e}", script.display()))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!("{} exited with {}", script.display(), out.status));
|
||||
}
|
||||
String::from_utf8(out.stdout)
|
||||
.map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
|
||||
}
|
||||
|
||||
/// Read a golden file, dropping its `#` header lines.
|
||||
pub fn read_golden(name: &str) -> Vec<String> {
|
||||
let path = golden(name);
|
||||
let text = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|e| panic!("reading {}: {e}", path.display()));
|
||||
text.lines()
|
||||
.filter(|line| !line.starts_with('#'))
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//! A deterministic sample of awkward text.
|
||||
//!
|
||||
//! Generated rather than committed, so the fixture costs the clone nothing,
|
||||
//! and generated the same way on every machine, so Rust and Python are
|
||||
//! demonstrably reading the same bytes. Change anything in here and every
|
||||
//! golden file derived from it has to be regenerated.
|
||||
//!
|
||||
//! The lines are chosen to be hostile to a tokeniser rather than
|
||||
//! representative of prose: scripts that are not Latin, combining marks,
|
||||
//! zero-width characters, the literal spelling of a control token, numbers,
|
||||
//! URLs, markdown, and runs of punctuation. Ordinary English is the easy case
|
||||
//! and is well covered by the corpus itself.
|
||||
|
||||
/// Lines in the standard sample.
|
||||
pub const LINES: usize = 10_000;
|
||||
|
||||
/// The seed the standard sample is generated from.
|
||||
pub const SEED: u64 = 0x636174656e610001;
|
||||
|
||||
/// The standard sample: [`LINES`] lines, newline-separated, no trailing
|
||||
/// newline, no carriage returns.
|
||||
///
|
||||
/// Carriage returns are excluded deliberately: Python's universal-newline
|
||||
/// translation would turn a lone `\r` into a line break and the two sides
|
||||
/// would then disagree about how many lines there even are.
|
||||
pub fn standard() -> String {
|
||||
generate(SEED, LINES)
|
||||
}
|
||||
|
||||
/// Generate `lines` lines from `seed`.
|
||||
pub fn generate(seed: u64, lines: usize) -> String {
|
||||
let mut rng = Rng::new(seed);
|
||||
let mut out = String::new();
|
||||
for i in 0..lines {
|
||||
if i > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str(&line(&mut rng));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn line(rng: &mut Rng) -> String {
|
||||
match rng.below(10) {
|
||||
0 => words(rng, 3, 14),
|
||||
1 => format!("{} {}", pick(rng, SCRIPTS), words(rng, 1, 6)),
|
||||
2 => format!("# {}", words(rng, 2, 7)),
|
||||
3 => format!(
|
||||
"- [{}]({}) {}",
|
||||
words(rng, 1, 4),
|
||||
pick(rng, URLS),
|
||||
words(rng, 0, 5)
|
||||
),
|
||||
4 => format!("{} {} {}", numbers(rng), pick(rng, PUNCT), numbers(rng)),
|
||||
5 => format!("{}{}", pick(rng, AWKWARD), words(rng, 0, 6)),
|
||||
6 => format!(" {}", pick(rng, CODE)),
|
||||
7 => format!(
|
||||
"{}{}",
|
||||
words(rng, 1, 5),
|
||||
pick(rng, PUNCT).repeat(rng.below(4) + 1)
|
||||
),
|
||||
8 => words(rng, 20, 60),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn words(rng: &mut Rng, least: usize, most: usize) -> String {
|
||||
let count = least + rng.below(most.saturating_sub(least) + 1);
|
||||
let mut out = String::new();
|
||||
for i in 0..count {
|
||||
if i > 0 {
|
||||
out.push(' ');
|
||||
}
|
||||
out.push_str(pick(rng, WORDS));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn numbers(rng: &mut Rng) -> String {
|
||||
match rng.below(4) {
|
||||
0 => format!("{}", rng.below(10)),
|
||||
1 => format!("{}", rng.below(100_000)),
|
||||
2 => format!("{}.{}", rng.below(1000), rng.below(1000)),
|
||||
_ => format!("{}-{}", rng.below(2100), rng.below(13)),
|
||||
}
|
||||
}
|
||||
|
||||
fn pick<'a>(rng: &mut Rng, from: &[&'a str]) -> &'a str {
|
||||
from[rng.below(from.len())]
|
||||
}
|
||||
|
||||
const WORDS: &[&str] = &[
|
||||
"the",
|
||||
"second",
|
||||
"estate",
|
||||
"manuscript",
|
||||
"marginalia",
|
||||
"colophon",
|
||||
"quire",
|
||||
"recto",
|
||||
"verso",
|
||||
"palimpsest",
|
||||
"incunabulum",
|
||||
"folio",
|
||||
"gloss",
|
||||
"catena",
|
||||
"scholion",
|
||||
"lacuna",
|
||||
"obelus",
|
||||
"supralinear",
|
||||
"hapax",
|
||||
"stemma",
|
||||
"apparatus",
|
||||
"witness",
|
||||
"conjecture",
|
||||
"lemma",
|
||||
];
|
||||
|
||||
const SCRIPTS: &[&str] = &[
|
||||
"café",
|
||||
"naïve",
|
||||
"Москва",
|
||||
"東京",
|
||||
"مرحبا",
|
||||
"Ἀθῆναι",
|
||||
"עברית",
|
||||
"ॐ",
|
||||
"Ægypt",
|
||||
"first",
|
||||
"Straße",
|
||||
];
|
||||
|
||||
const URLS: &[&str] = &[
|
||||
"https://example.org/a/b?c=d&e=f",
|
||||
"http://localhost:8080/x",
|
||||
"#anchor-name",
|
||||
"mailto:someone@example.invalid",
|
||||
];
|
||||
|
||||
const PUNCT: &[&str] = &[".", "!", "?", "…", "—", ";", ":", ",", ")", "\"", "'"];
|
||||
|
||||
const CODE: &[&str] = &[
|
||||
"fn main() { println!(\"{}\", 1 + 1); }",
|
||||
"SELECT * FROM t WHERE id = $1;",
|
||||
"{\"v\":1,\"idx\":0}",
|
||||
"<!-- Page 41 -->",
|
||||
];
|
||||
|
||||
/// Characters that have broken something before: a zero-width joiner, a
|
||||
/// non-breaking space, a combining acute, the literal spelling of a control
|
||||
/// token, and a tab.
|
||||
const AWKWARD: &[&str] = &[
|
||||
"\u{200d}",
|
||||
"\u{00a0}",
|
||||
"e\u{0301}",
|
||||
"<|endoftext|>",
|
||||
"\t",
|
||||
"\u{feff}",
|
||||
"🙂",
|
||||
"\u{1c}",
|
||||
];
|
||||
|
||||
/// xorshift64*, so the sample is the same on every machine and every run.
|
||||
struct Rng(u64);
|
||||
|
||||
impl Rng {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self(if seed == 0 { 1 } else { seed })
|
||||
}
|
||||
|
||||
fn next(&mut self) -> u64 {
|
||||
let mut x = self.0;
|
||||
x ^= x >> 12;
|
||||
x ^= x << 25;
|
||||
x ^= x >> 27;
|
||||
self.0 = x;
|
||||
x.wrapping_mul(0x2545_f491_4f6c_dd1d)
|
||||
}
|
||||
|
||||
fn below(&mut self, n: usize) -> usize {
|
||||
if n == 0 {
|
||||
0
|
||||
} else {
|
||||
(self.next() % n as u64) as usize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_sample_is_the_same_every_time() {
|
||||
assert_eq!(generate(SEED, 50), generate(SEED, 50));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_shorter_sample_is_a_prefix_of_a_longer_one() {
|
||||
let long = generate(SEED, 50);
|
||||
let short = generate(SEED, 10);
|
||||
assert!(long.starts_with(&short));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_standard_sample_has_the_advertised_shape() {
|
||||
let text = standard();
|
||||
assert_eq!(text.split('\n').count(), LINES);
|
||||
assert!(!text.contains('\r'));
|
||||
assert!(!text.ends_with('\n'));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user