chunkmd: page and chapter aware chunking, roles, footnote flag

parity/src/bin/regen.rs

@@ -4,24 +4,40 @@
 //! it fails is a test that cannot fail. Run it deliberately, read the diff,
 //! and commit the change with the reason.
 //!
+//! ```text
+//! CATENA_PARITY_PYTHON=... CATENA_PARITY_PYTHONPATH=... \
 //!     cargo run -p parity --bin regen
+//! ```
 
 use std::process::ExitCode;
 
+/// What one oracle run produces: the golden files it wrote, or why it could
+/// not.
+type Written = Result<Vec<String>, String>;
+
+/// One oracle and the name it is reported under.
+type Job = (&'static str, fn() -> Written);
+
 fn main() -> ExitCode {
-    match regen_tokens() {
-        Ok(path) => {
-            eprintln!("wrote {path}");
-            ExitCode::SUCCESS
-        }
-        Err(e) => {
-            eprintln!("regen: {e}");
-            ExitCode::FAILURE
+    let jobs: [Job; 2] = [("tokens", tokens), ("chunkmd", chunkmd)];
+
+    for (name, job) in jobs {
+        match job() {
+            Ok(written) => {
+                for path in written {
+                    eprintln!("wrote {path}");
+                }
+            }
+            Err(e) => {
+                eprintln!("regen {name}: {e}");
+                return ExitCode::FAILURE;
+            }
         }
     }
+    ExitCode::SUCCESS
 }
 
-fn regen_tokens() -> Result<String, String> {
+fn tokens() -> Written {
     let header = format!(
         "# cl100k_base token counts, one per line of parity::sample::standard()\n\
          # seed={:#018x} lines={}\n",
@@ -31,5 +47,26 @@ fn regen_tokens() -> Result<String, String> {
     let counts = parity::run_oracle("tokens.py", &parity::sample::standard())?;
     let path = parity::golden("tokens.counts");
     std::fs::write(&path, format!("{header}{counts}")).map_err(|e| e.to_string())?;
-    Ok(path.display().to_string())
+    Ok(vec![path.display().to_string()])
+}
+
+fn chunkmd() -> Written {
+    let dir = parity::golden("chunkmd");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let mut written = Vec::new();
+    for case in parity::cases("chunkmd") {
+        let markdown = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
+        for (set, budgets) in parity::budgets::SETS {
+            let records = parity::run_oracle_with(
+                "chunkmd.py",
+                &parity::budgets::as_args(budgets),
+                &markdown,
+            )?;
+            let path = dir.join(format!("{}.{set}.ndjson", parity::case_name(&case)));
+            std::fs::write(&path, records).map_err(|e| e.to_string())?;
+            written.push(path.display().to_string());
+        }
+    }
+    Ok(written)
 }

parity/src/budgets.rs

@@ -0,0 +1,43 @@
+//! The budget sets the chunker is pinned at.
+//!
+//! Two of them: the defaults, and a deliberately small set that forces the
+//! flush, overlap and hard-maximum paths to run on a fixture small enough to
+//! read. Most of the interesting behaviour only appears at a boundary, and
+//! at the default budgets a fixture would have to be pages long to have one.
+
+use chunkmd::Budgets;
+
+/// Named budget sets, in the order goldens are generated.
+pub const SETS: &[(&str, Budgets)] = &[
+    (
+        "default",
+        Budgets {
+            target: 512,
+            overlap: 64,
+            min_tokens: 50,
+            hard_max: 768,
+        },
+    ),
+    (
+        "small",
+        Budgets {
+            target: 60,
+            overlap: 12,
+            min_tokens: 5,
+            hard_max: 90,
+        },
+    ),
+];
+
+/// The arguments the oracle takes for a budget set.
+pub fn as_args(budgets: &Budgets) -> Vec<String> {
+    [
+        budgets.target,
+        budgets.overlap,
+        budgets.min_tokens,
+        budgets.hard_max,
+    ]
+    .iter()
+    .map(usize::to_string)
+    .collect()
+}

parity/src/lib.rs

@@ -15,6 +15,7 @@
 //! committed here, and for copyrighted sources only the derived signal — the
 //! scores, the counts, the timings — is ever checked in.
 
+pub mod budgets;
 pub mod sample;
 
 use std::path::{Path, PathBuf};
@@ -49,12 +50,18 @@ pub fn python() -> String {
 /// directory, so nothing about where a Python pipeline lives on a particular
 /// machine is compiled in.
 pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
+    run_oracle_with(script, &[], input)
+}
+
+/// Run an oracle script with arguments.
+pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<String, String> {
     use std::io::Write as _;
     use std::process::{Command, Stdio};
 
     let script = oracle(script);
     let mut child = Command::new(python())
         .arg(&script)
+        .args(args)
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::inherit())
@@ -76,6 +83,33 @@ pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
         .map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
 }
 
+/// Every case file for a tool, in a fixed order.
+pub fn cases(tool: &str) -> Vec<PathBuf> {
+    let dir = dir().join("cases").join(tool);
+    let mut found: Vec<PathBuf> = std::fs::read_dir(&dir)
+        .unwrap_or_else(|e| panic!("reading {}: {e}", dir.display()))
+        .filter_map(Result::ok)
+        .map(|entry| entry.path())
+        .filter(|path| path.extension().is_some_and(|ext| ext == "md"))
+        .collect();
+    found.sort();
+    found
+}
+
+/// The stem of a case path, which names its golden file.
+pub fn case_name(path: &Path) -> String {
+    path.file_stem()
+        .expect("a case file has a name")
+        .to_string_lossy()
+        .into_owned()
+}
+
+/// Read a golden file whole.
+pub fn read_golden_text(name: &str) -> String {
+    let path = golden(name);
+    std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display()))
+}
+
 /// Read a golden file, dropping its `#` header lines.
 pub fn read_golden(name: &str) -> Vec<String> {
     let path = golden(name);