pdftriage: text layer verdicts, including the reflected-font recovery

Cargo.lock

@@ -247,10 +247,23 @@ dependencies = [
  "chunkmd",
  "corpus-core",
  "mdscrub",
+ "pdftriage",
  "serde_json",
  "textkit",
 ]
 
+[[package]]
+name = "pdftriage"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "corpus-core",
+ "regex",
+ "serde_json",
+ "textkit",
+ "unicode-general-category",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.107"
@@ -462,6 +475,12 @@ dependencies = [
  "rustc-hash",
 ]
 
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.26"

Cargo.toml

@@ -14,14 +14,19 @@ corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
 textkit = { path = "crates/textkit", version = "0.1.0" }
 chunkmd = { path = "crates/chunkmd", version = "0.1.0" }
 mdscrub = { path = "crates/mdscrub", version = "0.1.0" }
+pdftriage = { path = "crates/pdftriage", version = "0.1.0" }
 
 clap = { version = "4", features = ["derive"] }
 fancy-regex = "0.16"
 regex = "1"
 schemars = "1"
 serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+# float_roundtrip: without it a float read back from JSON can be one unit in
+# the last place away from the one that was written, which would make a
+# record that round-trips through a pipe not the record it started as.
+serde_json = { version = "1", features = ["float_roundtrip"] }
 thiserror = "2"
+unicode-general-category = "1"
 tiktoken-rs = "0.12"
 
 [workspace.lints.rust]

crates/corpus-core/src/ndjson.rs

@@ -328,6 +328,17 @@ mod tests {
         assert_eq!(back[0].text, "two\nlines");
     }
 
+    #[test]
+    fn a_float_survives_the_round_trip_bit_for_bit() {
+        for score in [0.1, 0.9740932642487047, 1.0 / 3.0, f64::MIN_POSITIVE, 1e308] {
+            let hit = Hit::new("a", Channel::Vector, 1, score);
+            let mut out = Vec::new();
+            write_line(&mut out, &hit).unwrap();
+            let back: Vec<Hit> = read(out.as_slice()).collect::<Result<_, _>>().unwrap();
+            assert_eq!(back[0].score.to_bits(), score.to_bits(), "{score}");
+        }
+    }
+
     #[test]
     fn what_is_written_reads_back_identical() {
         let hit = Hit::new("a", Channel::Vector, 3, 0.125);

crates/pdftriage/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "pdftriage"
+description = "Says whether a PDF's text layer is worth reading, or whether it has to be OCRed."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+regex.workspace = true
+serde_json.workspace = true
+textkit.workspace = true
+unicode-general-category.workspace = true
+
+[lints]
+workspace = true

crates/pdftriage/src/decode.rs

@@ -0,0 +1,71 @@
+//! Reading what a PDF text extractor wrote.
+
+/// Decode the bytes a PDF text extractor produced.
+///
+/// Poppler defaults to UTF-8, but a broken character map or a legacy Latin
+/// text layer often emits bytes that are not valid UTF-8 at all — a lone
+/// `0xed` is the usual one. Strict UTF-8 first, then Windows-1252, which
+/// covers the Western European bytes without failing. Bytes that Windows-1252
+/// does not define become the replacement character, exactly as the Python
+/// this was ported from does.
+pub fn decode(bytes: &[u8]) -> String {
+    match std::str::from_utf8(bytes) {
+        Ok(text) => text.to_string(),
+        Err(_) => bytes.iter().map(|byte| windows_1252(*byte)).collect(),
+    }
+}
+
+fn windows_1252(byte: u8) -> char {
+    if byte < 0x80 {
+        return byte as char;
+    }
+    HIGH[byte as usize - 0x80]
+}
+
+/// Windows-1252 for the bytes above ASCII. Generated from the codec itself.
+static HIGH: [char; 128] = [
+    '\u{20ac}', '\u{fffd}', '\u{201a}', '\u{192}', '\u{201e}', '\u{2026}', '\u{2020}', '\u{2021}',
+    '\u{2c6}', '\u{2030}', '\u{160}', '\u{2039}', '\u{152}', '\u{fffd}', '\u{17d}', '\u{fffd}',
+    '\u{fffd}', '\u{2018}', '\u{2019}', '\u{201c}', '\u{201d}', '\u{2022}', '\u{2013}', '\u{2014}',
+    '\u{2dc}', '\u{2122}', '\u{161}', '\u{203a}', '\u{153}', '\u{fffd}', '\u{17e}', '\u{178}',
+    '\u{a0}', '\u{a1}', '\u{a2}', '\u{a3}', '\u{a4}', '\u{a5}', '\u{a6}', '\u{a7}', '\u{a8}',
+    '\u{a9}', '\u{aa}', '\u{ab}', '\u{ac}', '\u{ad}', '\u{ae}', '\u{af}', '\u{b0}', '\u{b1}',
+    '\u{b2}', '\u{b3}', '\u{b4}', '\u{b5}', '\u{b6}', '\u{b7}', '\u{b8}', '\u{b9}', '\u{ba}',
+    '\u{bb}', '\u{bc}', '\u{bd}', '\u{be}', '\u{bf}', '\u{c0}', '\u{c1}', '\u{c2}', '\u{c3}',
+    '\u{c4}', '\u{c5}', '\u{c6}', '\u{c7}', '\u{c8}', '\u{c9}', '\u{ca}', '\u{cb}', '\u{cc}',
+    '\u{cd}', '\u{ce}', '\u{cf}', '\u{d0}', '\u{d1}', '\u{d2}', '\u{d3}', '\u{d4}', '\u{d5}',
+    '\u{d6}', '\u{d7}', '\u{d8}', '\u{d9}', '\u{da}', '\u{db}', '\u{dc}', '\u{dd}', '\u{de}',
+    '\u{df}', '\u{e0}', '\u{e1}', '\u{e2}', '\u{e3}', '\u{e4}', '\u{e5}', '\u{e6}', '\u{e7}',
+    '\u{e8}', '\u{e9}', '\u{ea}', '\u{eb}', '\u{ec}', '\u{ed}', '\u{ee}', '\u{ef}', '\u{f0}',
+    '\u{f1}', '\u{f2}', '\u{f3}', '\u{f4}', '\u{f5}', '\u{f6}', '\u{f7}', '\u{f8}', '\u{f9}',
+    '\u{fa}', '\u{fb}', '\u{fc}', '\u{fd}', '\u{fe}', '\u{ff}',
+];
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn valid_utf8_is_read_as_utf8() {
+        assert_eq!(
+            decode("caf\u{e9} \u{4e2d}".as_bytes()),
+            "caf\u{e9} \u{4e2d}"
+        );
+    }
+
+    #[test]
+    fn a_lone_high_byte_falls_back_to_windows_1252() {
+        assert_eq!(decode(&[b'a', 0xed, b'b']), "a\u{ed}b");
+        assert_eq!(decode(&[0x93, b'x', 0x94]), "\u{201c}x\u{201d}");
+    }
+
+    #[test]
+    fn an_undefined_byte_becomes_the_replacement_character() {
+        assert_eq!(decode(&[0xff, 0x81]), "\u{ff}\u{fffd}");
+    }
+
+    #[test]
+    fn the_table_covers_every_high_byte() {
+        assert_eq!(HIGH.len(), 128);
+    }
+}

crates/pdftriage/src/lib.rs

@@ -0,0 +1,333 @@
+//! Says whether a PDF's text layer is worth reading, or whether it has to be
+//! OCRed.
+//!
+//! OCR is the expensive step in any ingest: minutes of GPU time a page, and a
+//! result that is worse than a text layer the file already had. So the
+//! question "is this layer any good" is worth answering carefully, and the
+//! thresholds below are measurements rather than guesses — each one is the
+//! middle of a gap observed across three thousand real PDFs, with the reason
+//! recorded beside it.
+//!
+//! Everything here is a function over already-extracted text. Running
+//! `pdftotext` is the binary's job, so that the judgement can be tested
+//! against the numbers without shipping anybody's books.
+
+pub mod decode;
+pub mod reflect;
+pub mod score;
+
+use corpus_core::TextLayer;
+
+/// Nothing legitimate carries more than two per cent of its characters in the
+/// Latin-1 band, and the lowest genuine letter share measured was 0.39 —
+/// chess notation, recipe lists, a page index. Both gates sit clear of that
+/// by a wide margin, because the cost of a false positive is a needless
+/// re-OCR of a book that already reads.
+const DEAD_LETTER_SHARE: f64 = 0.25;
+const DEAD_LATIN1_SHARE: f64 = 0.35;
+
+/// Non-space characters a page below which a text layer is not really there.
+///
+/// Chosen from the measured gap, not from a round number. The broken books
+/// sit at 3, 86, 94 and 132 characters a page — a scan whose layer holds only
+/// the page numbers and the running heads. The lowest-density legitimate
+/// books are small-format e-reader PDFs at 258 to 382. Two hundred sits in
+/// the middle of that gap with margin on both sides.
+const SPARSE_CHARS_PER_PAGE: f64 = 200.0;
+
+/// Function words that make up a large, stable share of any English prose.
+///
+/// A damaged synthetic-font layer still emits letters, so counting letters
+/// proves nothing — but it cannot reproduce these. Healthy layers score 0.25
+/// to 0.40; garbled ones score near zero. A tenth sits in the gap with room
+/// on both sides.
+const COMMON_WORDS: &[&str] = &[
+    "a", "all", "an", "and", "are", "as", "at", "be", "been", "but", "by", "for", "from", "had",
+    "have", "his", "in", "is", "it", "more", "not", "of", "on", "one", "or", "that", "the",
+    "there", "they", "this", "to", "was", "were", "which", "with", "you",
+];
+const HEALTHY_WORD_RATIO: f64 = 0.10;
+
+/// Characters stripped from the ends of a word before it is looked up.
+const WORD_TRIM: &[char] = &['.', ',', ';', ':', '!', '?', '(', ')', '[', ']', '"', '\''];
+
+/// How much text has to be there before a judgement means anything.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Limits {
+    /// Non-space characters in the opening sample, below which the layer is
+    /// treated as absent.
+    pub min_chars_present: usize,
+    /// Non-space characters in the middle sample, below which damage cannot
+    /// be judged.
+    pub min_chars_to_judge: usize,
+    /// Words needed before the English prose check means anything.
+    pub min_words: usize,
+    /// Pages below which density says nothing — a handful of pages cannot be
+    /// judged that way.
+    pub min_pages_for_density: u32,
+}
+
+impl Default for Limits {
+    fn default() -> Self {
+        Self {
+            min_chars_present: 200,
+            min_chars_to_judge: 400,
+            min_words: 80,
+            min_pages_for_density: 8,
+        }
+    }
+}
+
+/// What was read off a file, and what it means.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Triage {
+    /// The verdict.
+    pub layer: TextLayer,
+    /// Whether the file has to be OCRed to become searchable.
+    pub needs_ocr: bool,
+    /// What the numbers were, in the order they were taken.
+    pub notes: Vec<String>,
+}
+
+/// The text a judgement is made from.
+#[derive(Debug, Clone, Copy)]
+pub struct Sample<'a> {
+    /// Text extracted from the opening pages, which answers whether there is
+    /// a layer at all.
+    pub opening: &'a str,
+    /// Text extracted from the middle of the document, which answers whether
+    /// the layer is any good. The middle, because front matter is often a
+    /// bare scan in a book whose body carries perfectly good text — and is
+    /// often cleanly typeset boilerplate in a book whose body is damaged.
+    pub middle: &'a str,
+    /// How many pages the middle sample covers.
+    pub middle_pages: u32,
+    /// Pages in the whole document, when they could be counted.
+    pub pages: Option<u32>,
+}
+
+/// Judge a text layer.
+///
+/// The order is not arbitrary. A layer that is not there at all cannot be
+/// judged for damage; a layer that is only permuted is recoverable in full
+/// and must not be sent for OCR; damage is a stronger statement than
+/// thinness, so it is tested first.
+pub fn triage(sample: &Sample<'_>, limits: &Limits) -> Triage {
+    let mut notes = Vec::new();
+
+    let opening_chars = score::body(sample.opening).len();
+    notes.push(format!("{opening_chars} characters on the opening pages"));
+    if opening_chars < limits.min_chars_present {
+        return Triage {
+            layer: TextLayer::Absent,
+            needs_ocr: true,
+            notes,
+        };
+    }
+
+    let middle = score::body(sample.middle);
+    let ascii = score::ascii_text_score(sample.middle);
+    notes.push(format!(
+        "{} characters over {} sample pages, ascii text score {ascii:.2}",
+        middle.len(),
+        sample.middle_pages
+    ));
+
+    if reflect::undo(sample.middle).is_some() {
+        notes.push("reflecting the character codes recovers ordinary text".into());
+        return Triage {
+            layer: TextLayer::ReflectedFont,
+            needs_ocr: false,
+            notes,
+        };
+    }
+
+    if middle.len() >= limits.min_chars_to_judge {
+        let letters = score::letter_share(&middle);
+        let latin1 = score::latin1_share(&middle);
+        notes.push(format!("letters {letters:.2}, latin-1 {latin1:.2}"));
+        if letters < DEAD_LETTER_SHARE || latin1 > DEAD_LATIN1_SHARE {
+            return Triage {
+                layer: TextLayer::Garbled,
+                needs_ocr: true,
+                notes,
+            };
+        }
+    }
+
+    if let Some(pages) = sample.pages
+        && pages >= limits.min_pages_for_density
+    {
+        let density = middle.len() as f64 / sample.middle_pages.max(1) as f64;
+        notes.push(format!("{density:.0} characters a page"));
+        if density < SPARSE_CHARS_PER_PAGE {
+            return Triage {
+                layer: TextLayer::Sparse,
+                needs_ocr: true,
+                notes,
+            };
+        }
+    }
+
+    if let Some(share) = english_prose_share(sample.middle, limits.min_words) {
+        notes.push(format!("english function words {share:.2}"));
+        if share < HEALTHY_WORD_RATIO {
+            notes.push("not English, or not prose — judged on shape alone".into());
+        }
+    }
+
+    Triage {
+        layer: TextLayer::Healthy,
+        needs_ocr: false,
+        notes,
+    }
+}
+
+/// Share of words that are common English function words, or `None` when
+/// there is too little text to judge.
+///
+/// English-only on purpose, and so never the whole verdict: it cannot judge
+/// the German, Greek and Church Slavonic books that sit beside the English
+/// ones. It is reported because when it does apply it is the most direct
+/// evidence there is that a layer reads as prose.
+pub fn english_prose_share(text: &str, min_words: usize) -> Option<f64> {
+    let words: Vec<String> = text
+        .split(textkit::is_space)
+        .map(|word| word.trim_matches(WORD_TRIM).to_lowercase())
+        .filter(|word| !word.is_empty())
+        .collect();
+    if words.len() < min_words {
+        return None;
+    }
+    let common = words
+        .iter()
+        .filter(|word| COMMON_WORDS.binary_search(&word.as_str()).is_ok())
+        .count();
+    Some(common as f64 / words.len() as f64)
+}
+
+/// The pages to judge a layer by: the middle of the document.
+///
+/// A fixed window near the front is only ever the middle of a pamphlet. On a
+/// real book it reads front matter, which is cleanly typeset boilerplate even
+/// when the body is damaged — a 424-page commentary scored healthy on pages
+/// 10 to 14 and garbled on both 160 to 164 and 300 to 304.
+pub fn middle_window(pages: Option<u32>, span: u32) -> (u32, u32) {
+    let Some(pages) = pages.filter(|pages| *pages >= 1) else {
+        // The old window, for a document whose length is unknown.
+        return (10, 14);
+    };
+    if pages <= span {
+        return (1, pages);
+    }
+    let start = (pages / 2).saturating_sub(span / 2).max(1);
+    (start, pages.min(start + span - 1))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn prose() -> String {
+        "The second estate held its land in exchange for service, and the service was \
+         military. That much is agreed; what follows is not. The court rolls do not say \
+         how often the obligation was tested, because the clerk who recorded it had no \
+         vocabulary for the difference between a rebellion and a bad harvest. This is the \
+         central difficulty of the sources, and everything that follows depends on reading \
+         around it rather than through it."
+            .repeat(3)
+    }
+
+    fn sample<'a>(opening: &'a str, middle: &'a str, pages: Option<u32>) -> Sample<'a> {
+        Sample {
+            opening,
+            middle,
+            middle_pages: 5,
+            pages,
+        }
+    }
+
+    #[test]
+    fn the_common_word_list_is_sorted_so_it_can_be_searched() {
+        assert!(COMMON_WORDS.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
+    #[test]
+    fn a_book_that_reads_is_healthy() {
+        let text = prose();
+        let found = triage(&sample(&text, &text, Some(400)), &Limits::default());
+        assert_eq!(found.layer, TextLayer::Healthy);
+        assert!(!found.needs_ocr);
+    }
+
+    #[test]
+    fn a_scan_with_no_layer_is_absent() {
+        let found = triage(&sample("  \n 12 \n", "", Some(400)), &Limits::default());
+        assert_eq!(found.layer, TextLayer::Absent);
+        assert!(found.needs_ocr);
+    }
+
+    #[test]
+    fn a_layer_of_latin1_soup_is_garbled() {
+        // Scrambled with some key other than the reflection, so that the
+        // recovery pass declines it and the damage stands.
+        let soup: String = (0..800)
+            .map(|i: u32| char::from_u32(0xa1 + (i * 7) % 0x5f).expect("in the latin-1 band"))
+            .collect();
+        let opening = prose();
+        let found = triage(&sample(&opening, &soup, Some(400)), &Limits::default());
+        assert_eq!(found.layer, TextLayer::Garbled, "{:#?}", found.notes);
+        assert!(found.needs_ocr);
+    }
+
+    #[test]
+    fn a_layer_holding_only_running_heads_is_sparse() {
+        let thin = "The Second Estate 41\n".repeat(8);
+        let opening = prose();
+        let found = triage(&sample(&opening, &thin, Some(400)), &Limits::default());
+        assert_eq!(found.layer, TextLayer::Sparse, "{:#?}", found.notes);
+        assert!(found.needs_ocr);
+    }
+
+    #[test]
+    fn a_short_document_is_not_judged_on_density() {
+        let thin = "Page 3\n".repeat(8);
+        let found = triage(&sample(&prose(), &thin, Some(4)), &Limits::default());
+        assert_ne!(found.layer, TextLayer::Sparse);
+    }
+
+    #[test]
+    fn a_permuted_layer_is_recovered_rather_than_sent_for_ocr() {
+        let stored: String = prose()
+            .chars()
+            .map(|c| {
+                let reflected = 0x120 - c as u32;
+                if (0xa1..=0xff).contains(&reflected) {
+                    char::from_u32(reflected).expect("in range")
+                } else {
+                    c
+                }
+            })
+            .collect();
+        let found = triage(&sample(&stored, &stored, Some(400)), &Limits::default());
+        assert_eq!(found.layer, TextLayer::ReflectedFont, "{:#?}", found.notes);
+        assert!(!found.needs_ocr, "a recoverable layer must not be OCRed");
+    }
+
+    #[test]
+    fn the_window_is_the_middle_of_the_book() {
+        assert_eq!(middle_window(Some(400), 5), (198, 202));
+        assert_eq!(middle_window(Some(3), 5), (1, 3));
+        assert_eq!(middle_window(Some(5), 5), (1, 5));
+        assert_eq!(middle_window(None, 5), (10, 14));
+        assert_eq!(middle_window(Some(0), 5), (10, 14));
+    }
+
+    #[test]
+    fn english_prose_is_recognised_and_soup_is_not() {
+        let share = english_prose_share(&prose(), 80).expect("enough words");
+        assert!(share > 0.25, "{share}");
+        let soup = "xqz ".repeat(200);
+        assert!(english_prose_share(&soup, 80).expect("enough words") < 0.01);
+    }
+}

crates/pdftriage/src/main.rs

@@ -0,0 +1,196 @@
+//! Says whether a PDF's text layer is worth reading, or whether it has to be
+//! OCRed.
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::{Command, ExitCode};
+
+use clap::Parser;
+use corpus_core::{Doc, Kind, TextLayer, exit};
+use pdftriage::{Limits, Sample};
+
+/// Pages sampled from the middle of the document.
+const SPAN: u32 = 5;
+
+/// Judge a PDF's text layer: healthy, garbled, sparse, reflected-font, or
+/// absent — and so whether OCR is worth its cost.
+///
+/// Takes a path rather than a stream, because it reads two specific page
+/// ranges rather than the whole file.
+///
+/// Shells out to `pdftotext`, and to `pdfinfo` for the page count; both ship
+/// with poppler. Without `pdftotext` there is nothing to judge and the tool
+/// exits 69. It never runs OCR, never writes to the file, and imposes no
+/// timeout of its own — a caller that needs one owns it, which is half the
+/// reason this is a separate process.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    long_about = None,
+    after_help = "\
+Exit codes: 0 the layer needs OCR, 1 it does not, 2 could not tell,
+69 pdftotext is not installed, 74 the file could not be read.
+
+Example:
+
+    for f in *.pdf; do pdftriage --quiet \"$f\" && ocrmypdf \"$f\" \"ocr/$f\"; done"
+)]
+struct Args {
+    /// The PDF to judge.
+    path: PathBuf,
+
+    /// Emit a doc record instead of a human-readable verdict.
+    #[arg(long)]
+    json: bool,
+
+    /// Say nothing; answer with the exit code alone.
+    #[arg(long, short)]
+    quiet: bool,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+
+    let bytes = match std::fs::metadata(&args.path) {
+        Ok(meta) => meta.len() as i64,
+        Err(e) => {
+            eprintln!("pdftriage: {}: {e}", args.path.display());
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    if which("pdftotext").is_none() {
+        eprintln!("pdftriage: pdftotext is not on PATH (it ships with poppler)");
+        return ExitCode::from(exit::UNAVAILABLE);
+    }
+
+    let pages = page_count(&args.path);
+    let (first, last) = pdftriage::middle_window(pages, SPAN);
+
+    let opening = extract(&args.path, 1, SPAN);
+    let middle = extract(&args.path, first, last);
+    let (Some(opening), Some(middle)) = (opening, middle) else {
+        eprintln!(
+            "pdftriage: pdftotext could not read {}",
+            args.path.display()
+        );
+        if args.json && !args.quiet {
+            let mut doc = unjudged(&args.path, bytes, pages);
+            doc.note("pdftotext could not read the file");
+            let _ = emit_json(&doc);
+        }
+        return ExitCode::from(exit::INDETERMINATE);
+    };
+
+    let found = pdftriage::triage(
+        &Sample {
+            opening: &opening,
+            middle: &middle,
+            middle_pages: last.saturating_sub(first) + 1,
+            pages,
+        },
+        &Limits::default(),
+    );
+
+    if !args.quiet {
+        let wrote = if args.json {
+            let mut doc = unjudged(&args.path, bytes, pages);
+            doc.text_layer = Some(found.layer);
+            doc.needs_ocr = Some(found.needs_ocr);
+            doc.notes = found.notes.clone();
+            emit_json(&doc)
+        } else {
+            emit_report(&found)
+        };
+        if let Err(e) = wrote {
+            if e.kind() == std::io::ErrorKind::BrokenPipe {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("pdftriage: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+
+    ExitCode::from(if found.needs_ocr { exit::OK } else { exit::NO })
+}
+
+fn unjudged(path: &Path, bytes: i64, pages: Option<u32>) -> Doc {
+    let mut doc = Doc::new(path.display().to_string(), Kind::Pdf, bytes);
+    doc.declared_ext = path
+        .extension()
+        .map(|ext| ext.to_string_lossy().to_lowercase());
+    doc.pages = pages;
+    doc
+}
+
+fn emit_json(doc: &Doc) -> std::io::Result<()> {
+    let mut out = std::io::stdout().lock();
+    let json = serde_json::to_string(doc).expect("a doc record serialises");
+    writeln!(out, "{json}")?;
+    out.flush()
+}
+
+fn emit_report(found: &pdftriage::Triage) -> std::io::Result<()> {
+    let mut out = std::io::stdout().lock();
+    let verdict = match found.layer {
+        TextLayer::Healthy => "healthy",
+        TextLayer::Garbled => "garbled",
+        TextLayer::Sparse => "sparse",
+        TextLayer::ReflectedFont => "reflected-font",
+        TextLayer::Absent => "absent",
+    };
+    let advice = if found.needs_ocr {
+        "needs OCR"
+    } else {
+        "no OCR needed"
+    };
+    writeln!(out, "{verdict}, {advice}")?;
+    for note in &found.notes {
+        writeln!(out, "  {note}")?;
+    }
+    out.flush()
+}
+
+/// Extract a page range as text, or `None` when the extractor could not.
+fn extract(path: &Path, first: u32, last: u32) -> Option<String> {
+    let out = Command::new("pdftotext")
+        .args(["-f", &first.to_string(), "-l", &last.to_string(), "-layout"])
+        .arg(path)
+        .arg("-")
+        .output()
+        .ok()?;
+    out.status
+        .success()
+        .then(|| pdftriage::decode::decode(&out.stdout))
+}
+
+/// The document's page count, when `pdfinfo` can say.
+fn page_count(path: &Path) -> Option<u32> {
+    let out = Command::new("pdfinfo").arg(path).output().ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    pdftriage::decode::decode(&out.stdout)
+        .lines()
+        .find_map(|line| line.strip_prefix("Pages:"))
+        .and_then(|count| count.trim().parse().ok())
+}
+
+fn which(program: &str) -> Option<PathBuf> {
+    let path = std::env::var_os("PATH")?;
+    std::env::split_paths(&path)
+        .map(|dir| dir.join(program))
+        .find(|candidate| candidate.is_file())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_program_that_exists_is_found_and_one_that_does_not_is_not() {
+        assert!(which("sh").is_some() || which("sh.exe").is_some());
+        assert!(which("definitely-not-a-program-93a1").is_none());
+    }
+}

crates/pdftriage/src/reflect.rs

@@ -0,0 +1,187 @@
+//! Recovering a text layer that is not damaged, only permuted.
+
+use crate::score::{ascii_text_score, word_shape};
+
+/// Codes are stored reflected about this point.
+const ABOUT: u32 = 0x120;
+/// The band the reflection applies to.
+const LOW: u32 = 0xa1;
+const HIGH: u32 = 0xff;
+
+/// Below this, the text as extracted does not read as ordinary text at all,
+/// which is the precondition for trying the decode.
+const LOOKS_DEAD: f64 = 0.35;
+/// The decode has to improve the score by at least this much to be believed.
+const MUST_IMPROVE: f64 = 0.30;
+/// Landing in the ASCII range is not the same as landing in a language.
+/// Measured on the two damaged books in the library this came from: tidy-case
+/// 0.999 for the recovered text, 0.224 for a wrong-key decode that also
+/// reflects into ASCII but into nonsense — `zgW$c_`, `iXYc$f\KO`.
+const MIN_TIDY_CASE: f64 = 0.90;
+const MIN_VOWELLED: f64 = 0.95;
+/// Words needed before word shape means anything.
+const MIN_WORDS: usize = 40;
+
+/// Glyph slots outside the reflected band.
+///
+/// These are the ligatures, the quotes and dashes, and the accented letters.
+/// Each was read off decoded context — `Univer<>ity` is an s, `Scienti<>c` an
+/// fi, `Wolfenb<>ttel` a u-umlaut — rather than guessed. A slot with no
+/// determined reading is left exactly as it stands: a rare glyph that still
+/// looks odd is fixable later, and a wrong guess is not.
+const SLOTS: &[(char, &str)] = &[
+    ('-', "s"),
+    ('A', "fl"),
+    ('B', "fi"),
+    ('H', "ij"),
+    ('K', "\u{2019}"),
+    ('L', "\u{201c}"),
+    ('M', "\u{201d}"),
+    ('N', "\u{2018}"),
+    ('O', "\u{2014}"),
+    ('P', "\u{2013}"),
+    ('U', "\u{c0}"),
+    ('a', "\u{f8}"),
+    ('w', "\u{a9}"),
+    ('\u{152}', "\u{ee}"),
+    ('\u{160}', "\u{f1}"),
+    ('\u{178}', "\u{c5}"),
+    ('\u{17d}', "\u{ed}"),
+    ('\u{2c6}', "\u{f2}"),
+    ('\u{2dc}', "\u{e0}"),
+    ('\u{2013}', "\u{e4}"),
+    ('\u{2014}', "\u{e2}"),
+    ('\u{2018}', "\u{e8}"),
+    ('\u{2019}', "\u{e9}"),
+    ('\u{201c}', "\u{e7}"),
+    ('\u{201e}', "\u{fa}"),
+    ('\u{2020}', "\u{f6}"),
+    ('\u{2021}', "\u{f4}"),
+    ('\u{2022}', "\u{fc}"),
+    ('\u{2030}', "\u{f3}"),
+    ('\u{2039}', "\u{ef}"),
+    ('\u{203a}', "\u{d6}"),
+    ('\u{2122}', "\u{e1}"),
+];
+
+/// Reverse a reflected-ASCII font layer, or `None` if that is not what this
+/// is.
+///
+/// Some PDFs store every printable ASCII code reflected about a fixed point,
+/// so that the text extracts as Latin-1 punctuation soup: `ײ¬®±¼«½¬·±²` is
+/// "Introduction". The mapping is exact, so the text comes back in full —
+/// which is strictly better than an OCR pass that could only approximate it.
+///
+/// Deliberately self-validating rather than gated on which program produced
+/// the file. The transform is applied only when the text as extracted does
+/// not read as ordinary text at all *and* reflecting it makes it read as
+/// ordinary text. A healthy book fails the first test, so it can never be
+/// mangled by this; a Greek or Cyrillic book fails the second, because its
+/// code points lie outside the band and the reflection changes nothing.
+pub fn undo(text: &str) -> Option<String> {
+    let before = ascii_text_score(text);
+    if before >= LOOKS_DEAD {
+        return None;
+    }
+
+    let mut decoded = String::with_capacity(text.len());
+    for c in text.chars() {
+        let code = c as u32;
+        if (LOW..=HIGH).contains(&code) {
+            decoded.push(char::from_u32(ABOUT - code).expect("the band reflects into ASCII"));
+        } else {
+            match slot(c) {
+                Some(reading) => decoded.push_str(reading),
+                None => decoded.push(c),
+            }
+        }
+    }
+
+    if ascii_text_score(&decoded) - before < MUST_IMPROVE {
+        return None;
+    }
+    // Declining here leaves the file for a re-OCR, which is the right answer
+    // when the key is not this one.
+    let (vowelled, tidy_case) = word_shape(&decoded, MIN_WORDS)?;
+    if tidy_case < MIN_TIDY_CASE || vowelled < MIN_VOWELLED {
+        return None;
+    }
+    Some(decoded)
+}
+
+fn slot(c: char) -> Option<&'static str> {
+    SLOTS
+        .binary_search_by_key(&c, |(glyph, _)| *glyph)
+        .ok()
+        .map(|at| SLOTS[at].1)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Encode text the way such a PDF stores it, so a test can be written
+    /// without shipping one of the books this was measured on.
+    fn reflect(text: &str) -> String {
+        text.chars()
+            .map(|c| {
+                let code = c as u32;
+                let reflected = ABOUT - code;
+                if (LOW..=HIGH).contains(&reflected) {
+                    char::from_u32(reflected).expect("in range")
+                } else {
+                    c
+                }
+            })
+            .collect()
+    }
+
+    fn prose() -> String {
+        "The second estate held its land in exchange for service and the service was \
+         military That much is agreed and what follows is not The court rolls do not \
+         say how often the obligation was tested because the clerk who recorded it had \
+         no vocabulary for the difference between a rebellion and a bad harvest"
+            .to_string()
+    }
+
+    #[test]
+    fn a_reflected_layer_comes_back_whole() {
+        let stored = reflect(&prose());
+        assert!(
+            ascii_text_score(&stored) < 0.1,
+            "the fixture is not reflected"
+        );
+        assert_eq!(undo(&stored).as_deref(), Some(prose().as_str()));
+    }
+
+    #[test]
+    fn healthy_text_is_never_touched() {
+        assert_eq!(undo(&prose()), None);
+    }
+
+    #[test]
+    fn a_greek_layer_is_left_alone() {
+        let greek = "Ἀθῆναι καὶ Μόσχα καὶ τὰ λοιπά ".repeat(20);
+        assert_eq!(undo(&greek), None);
+    }
+
+    #[test]
+    fn nonsense_that_reflects_into_ascii_is_refused() {
+        // A layer scrambled with some other key reflects into ASCII too, but
+        // into words with no vowels and no case pattern.
+        let nonsense: String = (0..400)
+            .map(|i| char::from_u32(0xa1 + (i * 7) % 0x5f).expect("in band"))
+            .collect();
+        assert_eq!(undo(&nonsense), None);
+    }
+
+    #[test]
+    fn too_little_text_to_judge_is_refused() {
+        assert_eq!(undo(&reflect("Introduction")), None);
+    }
+
+    #[test]
+    fn the_slot_table_is_sorted_so_it_can_be_searched() {
+        assert!(SLOTS.windows(2).all(|pair| pair[0].0 < pair[1].0));
+    }
+}

crates/pdftriage/src/score.rs

@@ -0,0 +1,170 @@
+//! Cheap, script-blind measurements of whether text is text.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+use unicode_general_category::{GeneralCategory, get_general_category};
+
+/// Characters that ordinary Latin-script text is made of.
+///
+/// Prose in any Latin-script language sits near 1.0 on the share of its
+/// non-space characters that are in here; a text layer stored through a
+/// reflected font sits near 0.0. The em dash is in the set despite not being
+/// ASCII, because it is ordinary in a book and its absence would penalise
+/// well-typeset text.
+const ASCII_TEXT: &str = concat!(
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
+    ".,;:!?'\"()[]{}-\u{2014}/&%$#@*+=<>|~^_`\\"
+);
+
+/// Words of three letters or more, which is enough to have a shape.
+static WORDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[A-Za-z]{3,}").expect("valid"));
+
+static VOWEL: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[aeiouyAEIOUY]").expect("valid"));
+
+/// Whether a character is a letter, by Python's definition.
+///
+/// Python's `str.isalpha` is true for the letter categories and nothing else.
+/// Rust's [`char::is_alphabetic`] is true for the Unicode `Alphabetic`
+/// property, which also covers some seventeen hundred combining marks —
+/// including `U+0363`–`U+036F`, the medieval superscript letters that appear
+/// in exactly the scanned manuscripts this has to judge. The thresholds below
+/// were measured against the narrower definition.
+pub fn is_letter(c: char) -> bool {
+    matches!(
+        get_general_category(c),
+        GeneralCategory::UppercaseLetter
+            | GeneralCategory::LowercaseLetter
+            | GeneralCategory::TitlecaseLetter
+            | GeneralCategory::ModifierLetter
+            | GeneralCategory::OtherLetter
+    )
+}
+
+/// The non-space characters of a text, which every share below is taken over.
+pub fn body(text: &str) -> Vec<char> {
+    text.chars().filter(|c| !textkit::is_space(*c)).collect()
+}
+
+/// Share of non-space characters that are ordinary Latin-script text.
+pub fn ascii_text_score(text: &str) -> f64 {
+    let body = body(text);
+    if body.is_empty() {
+        return 0.0;
+    }
+    let ordinary = body.iter().filter(|c| ASCII_TEXT.contains(**c)).count();
+    ordinary as f64 / body.len() as f64
+}
+
+/// Share of the non-space characters that are letters, in any alphabet.
+pub fn letter_share(body: &[char]) -> f64 {
+    if body.is_empty() {
+        return 0.0;
+    }
+    body.iter().filter(|c| is_letter(**c)).count() as f64 / body.len() as f64
+}
+
+/// Share of the non-space characters sitting in the Latin-1 band, which is
+/// where a broken character map falls back to.
+pub fn latin1_share(body: &[char]) -> f64 {
+    if body.is_empty() {
+        return 0.0;
+    }
+    let banded = body
+        .iter()
+        .filter(|c| (0xa1..=0xff).contains(&(**c as u32)))
+        .count();
+    banded as f64 / body.len() as f64
+}
+
+/// The share of words carrying a vowel, and the share in a sane case
+/// pattern. `None` when there are too few words to judge.
+///
+/// A cheap, script-blind proxy for "is this a language", used to tell a real
+/// decode from one done with the wrong key.
+pub fn word_shape(text: &str, min_words: usize) -> Option<(f64, f64)> {
+    let words: Vec<&str> = WORDS.find_iter(text).map(|found| found.as_str()).collect();
+    if words.len() < min_words {
+        return None;
+    }
+    let vowelled = words.iter().filter(|word| VOWEL.is_match(word)).count();
+    let tidy = words.iter().filter(|word| tidy_case(word)).count();
+    Some((
+        vowelled as f64 / words.len() as f64,
+        tidy as f64 / words.len() as f64,
+    ))
+}
+
+/// Whether a word is all lower case, Capitalised, or ALL CAPS — which real
+/// words are, and `zgW$c_` is not.
+fn tidy_case(word: &str) -> bool {
+    let mut chars = word.chars();
+    let Some(first) = chars.next() else {
+        return false;
+    };
+    let rest = chars.as_str();
+    let all_lower = word.chars().all(|c| c.is_ascii_lowercase());
+    let all_upper = word.chars().all(|c| c.is_ascii_uppercase());
+    let title = first.is_ascii_uppercase() && rest.chars().all(|c| c.is_ascii_lowercase());
+    all_lower || all_upper || title
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn prose_scores_near_one() {
+        let score = ascii_text_score("The second estate held its land in exchange for service.");
+        assert!(score > 0.95, "{score}");
+    }
+
+    #[test]
+    fn a_latin1_soup_scores_near_zero() {
+        let score = ascii_text_score("\u{d7}\u{b2}\u{ac}\u{ae}\u{b1}\u{bc}\u{ab}\u{bd}");
+        assert_eq!(score, 0.0);
+    }
+
+    #[test]
+    fn empty_text_scores_zero_rather_than_dividing_by_nothing() {
+        assert_eq!(ascii_text_score(""), 0.0);
+        assert_eq!(ascii_text_score("   \n  "), 0.0);
+        assert_eq!(letter_share(&[]), 0.0);
+        assert_eq!(latin1_share(&[]), 0.0);
+    }
+
+    #[test]
+    fn a_medieval_combining_letter_is_not_a_letter_here() {
+        // Python's isalpha says no; Rust's is_alphabetic says yes.
+        assert!(!is_letter('\u{363}'));
+        assert!('\u{363}'.is_alphabetic());
+        assert!(is_letter('a'));
+        assert!(is_letter('\u{3b1}'));
+        assert!(!is_letter('1'));
+    }
+
+    #[test]
+    fn word_shape_needs_enough_words_to_judge() {
+        assert_eq!(word_shape("one two three", 40), None);
+    }
+
+    #[test]
+    fn real_words_are_vowelled_and_tidily_cased() {
+        let text = "The second estate held its land in exchange for service and the service \
+                    was military That much is agreed what follows is not The rolls do not say \
+                    how often the obligation was tested and the clerk had no vocabulary for it "
+            .repeat(3);
+        let (vowelled, tidy) = word_shape(&text, 40).expect("enough words");
+        assert!(vowelled > 0.95, "{vowelled}");
+        assert!(tidy > 0.95, "{tidy}");
+    }
+
+    #[test]
+    fn the_case_test_is_what_it_says() {
+        assert!(tidy_case("lower"));
+        assert!(tidy_case("Title"));
+        assert!(tidy_case("UPPER"));
+        assert!(!tidy_case("mIxEd"));
+        assert!(!tidy_case("aBc"));
+    }
+}

crates/pdftriage/tests/cli.rs

@@ -0,0 +1,196 @@
+//! The tool as a caller meets it: a path in, a verdict and an exit code out.
+//!
+//! The PDFs are built here rather than committed, so the suite ships no
+//! book and no binary fixture. Every test is skipped when `pdftotext` is not
+//! installed, because without it there is nothing to judge and the tool says
+//! so with exit 69.
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+
+const PROSE: &str = "The second estate held its land in exchange for service, and
+the service was military. That much is agreed; what follows is
+not. The court rolls do not say how often the obligation was
+tested, because the clerk who recorded it had no vocabulary for
+the difference between a rebellion and a bad harvest, and so the
+record flattens the two into one offence with one penalty.";
+
+fn pdftotext_installed() -> bool {
+    Command::new("pdftotext")
+        .arg("-v")
+        .output()
+        .is_ok_and(|out| out.status.success() || !out.stderr.is_empty())
+}
+
+fn run(args: &[&str]) -> Output {
+    Command::new(env!("CARGO_BIN_EXE_pdftriage"))
+        .args(args)
+        .output()
+        .expect("the binary should run")
+}
+
+/// Write a PDF with `pages` pages, each holding `text`.
+///
+/// Hand-assembled rather than generated by a library: a PDF is an object
+/// table and a cross-reference index, and twenty lines of it here save a
+/// dependency and a committed fixture.
+fn write_pdf(path: &Path, text: &str, pages: usize) {
+    let mut objects: Vec<(usize, Vec<u8>)> = Vec::new();
+    let page_ids: Vec<usize> = (0..pages).map(|at| 4 + 2 * at).collect();
+
+    let kids: Vec<String> = page_ids.iter().map(|id| format!("{id} 0 R")).collect();
+    objects.push((1, b"<</Type/Catalog/Pages 2 0 R>>".to_vec()));
+    objects.push((
+        2,
+        format!("<</Type/Pages/Kids[{}]/Count {pages}>>", kids.join(" ")).into_bytes(),
+    ));
+    objects.push((
+        3,
+        b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
+    ));
+
+    for id in &page_ids {
+        let contents_id = id + 1;
+        objects.push((
+            *id,
+            format!(
+                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]\
+                 /Contents {contents_id} 0 R/Resources<</Font<</F1 3 0 R>>>>>>"
+            )
+            .into_bytes(),
+        ));
+        let mut stream = String::new();
+        let mut y = 720;
+        for line in text.lines() {
+            let escaped = line
+                .replace('\\', r"\\")
+                .replace('(', r"\(")
+                .replace(')', r"\)");
+            stream.push_str(&format!("BT /F1 11 Tf 72 {y} Td ({escaped}) Tj ET\n"));
+            y -= 14;
+        }
+        let mut object = format!("<</Length {}>>\nstream\n", stream.len()).into_bytes();
+        object.extend_from_slice(stream.as_bytes());
+        object.extend_from_slice(b"\nendstream");
+        objects.push((contents_id, object));
+    }
+
+    objects.sort_by_key(|(id, _)| *id);
+    let mut out: Vec<u8> = b"%PDF-1.4\n".to_vec();
+    let mut offsets: Vec<(usize, usize)> = Vec::new();
+    for (id, body) in &objects {
+        offsets.push((*id, out.len()));
+        out.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
+        out.extend_from_slice(body);
+        out.extend_from_slice(b"\nendobj\n");
+    }
+
+    let xref_at = out.len();
+    let top = objects.last().expect("at least one object").0 + 1;
+    out.extend_from_slice(format!("xref\n0 {top}\n0000000000 65535 f \n").as_bytes());
+    for id in 1..top {
+        let at = offsets
+            .iter()
+            .find(|(object, _)| *object == id)
+            .map_or(0, |(_, at)| *at);
+        out.extend_from_slice(format!("{at:010} 00000 n \n").as_bytes());
+    }
+    out.extend_from_slice(
+        format!("trailer\n<</Size {top}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n").as_bytes(),
+    );
+
+    let mut file = std::fs::File::create(path).expect("a writable path");
+    file.write_all(&out).expect("the PDF should write");
+}
+
+fn temp(name: &str) -> PathBuf {
+    let dir = std::env::temp_dir().join("pdftriage-tests");
+    std::fs::create_dir_all(&dir).expect("a writable temp directory");
+    dir.join(format!("{name}.pdf"))
+}
+
+#[test]
+fn a_book_that_reads_is_not_sent_for_ocr() {
+    if !pdftotext_installed() {
+        return;
+    }
+    let path = temp("healthy");
+    write_pdf(&path, PROSE, 20);
+
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(1), "a healthy layer must exit 1");
+    let said = String::from_utf8(out.stdout).expect("utf-8");
+    assert!(said.starts_with("healthy, no OCR needed"), "{said}");
+    assert!(said.contains("characters a page"), "{said}");
+}
+
+#[test]
+fn a_scan_with_no_layer_is_sent_for_ocr() {
+    if !pdftotext_installed() {
+        return;
+    }
+    let path = temp("absent");
+    write_pdf(&path, "", 20);
+
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(0), "a dead layer must exit 0");
+    assert!(String::from_utf8_lossy(&out.stdout).starts_with("absent, needs OCR"));
+}
+
+#[test]
+fn the_json_form_is_a_doc_record() {
+    if !pdftotext_installed() {
+        return;
+    }
+    let path = temp("json");
+    write_pdf(&path, PROSE, 20);
+
+    let out = run(&["--json", path.to_str().expect("utf-8 path")]);
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["v"], 1);
+    assert_eq!(doc["kind"], "pdf");
+    assert_eq!(doc["declared_ext"], "pdf");
+    assert_eq!(doc["pages"], 20);
+    assert_eq!(doc["text_layer"], "healthy");
+    assert_eq!(doc["needs_ocr"], false);
+    assert!(
+        doc["notes"]
+            .as_array()
+            .is_some_and(|notes| !notes.is_empty())
+    );
+}
+
+#[test]
+fn quiet_says_nothing_at_all() {
+    if !pdftotext_installed() {
+        return;
+    }
+    let path = temp("quiet");
+    write_pdf(&path, PROSE, 20);
+
+    let out = run(&["--quiet", path.to_str().expect("utf-8 path")]);
+    assert!(out.stdout.is_empty(), "stdout was not empty");
+    assert!(out.stderr.is_empty(), "stderr was not empty");
+    assert_eq!(out.status.code(), Some(1));
+}
+
+#[test]
+fn a_file_that_is_not_there_is_an_io_error() {
+    let out = run(&["/definitely/not/here.pdf"]);
+    assert_eq!(out.status.code(), Some(74));
+    assert!(out.stdout.is_empty(), "a diagnostic reached stdout");
+    assert!(!out.stderr.is_empty());
+}
+
+#[test]
+fn a_file_that_is_not_a_pdf_cannot_be_judged() {
+    if !pdftotext_installed() {
+        return;
+    }
+    let path = temp("not-a-pdf");
+    std::fs::write(&path, b"this is not a PDF at all\n").expect("a writable path");
+
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(2), "could not tell must exit 2");
+}

parity/Cargo.toml

@@ -16,6 +16,7 @@ python = []
 [dependencies]
 chunkmd.workspace = true
 mdscrub.workspace = true
+pdftriage.workspace = true
 corpus-core.workspace = true
 serde_json.workspace = true
 textkit.workspace = true

parity/cases/pdftriage/samples.ndjson

@@ -0,0 +1,24 @@
+{"name": "empty", "text": ""}
+{"name": "whitespace", "text": "   \n\t  \n"}
+{"name": "prose-short", "text": "The mill stood at the northern end."}
+{"name": "prose", "text": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. "}
+{"name": "prose-reflected", "text": "̸» ­»½±²¼ »­¬¿¬» ¸»´¼ ·¬­ ´¿²¼ ·² »¨½¸¿²¹» º±® ­»®ª·½»ô ¿²¼ ¬¸» ­»®ª·½» ©¿­ ³·´·¬¿®§ò ̸¿¬ ³«½¸ ·­ ¿¹®»»¼å ©¸¿¬ º±´´±©­ ·­ ²±¬ò ̸» ½±«®¬ ®±´´­ ¼± ²±¬ ­¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿­ ¬»­¬»¼ô ¾»½¿«­» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»­¬ò ̸·­ ·­ ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ­±«®½»­ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±©­ ¼»°»²¼­ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò ̸» ­»½±²¼ »­¬¿¬» ¸»´¼ ·¬­ ´¿²¼ ·² »¨½¸¿²¹» º±® ­»®ª·½»ô ¿²¼ ¬¸» ­»®ª·½» ©¿­ ³·´·¬¿®§ò ̸¿¬ ³«½¸ ·­ ¿¹®»»¼å ©¸¿¬ º±´´±©­ ·­ ²±¬ò ̸» ½±«®¬ ®±´´­ ¼± ²±¬ ­¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿­ ¬»­¬»¼ô ¾»½¿«­» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»­¬ò ̸·­ ·­ ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ­±«®½»­ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±©­ ¼»°»²¼­ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò ̸» ­»½±²¼ »­¬¿¬» ¸»´¼ ·¬­ ´¿²¼ ·² »¨½¸¿²¹» º±® ­»®ª·½»ô ¿²¼ ¬¸» ­»®ª·½» ©¿­ ³·´·¬¿®§ò ̸¿¬ ³«½¸ ·­ ¿¹®»»¼å ©¸¿¬ º±´´±©­ ·­ ²±¬ò ̸» ½±«®¬ ®±´´­ ¼± ²±¬ ­¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿­ ¬»­¬»¼ô ¾»½¿«­» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»­¬ò ̸·­ ·­ ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ­±«®½»­ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±©­ ¼»°»²¼­ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò "}
+{"name": "prose-reflected-short", "text": "̸» ­»½±²¼ »­¬¿¬» ¸»´¼ ·¬­ ´¿²¼ ·² »¨½¸¿²¹» º±® ­»®ª·½»ô ¿²¼ ¬¸» ­»®ª·½» ©¿­ ³·´·¬¿®§ò ̸¿¬ ³«½¸ ·­ ¿¹®»»¼å ©¸¿¬ º±´´±©­ ·­ ²±¬ò ̸» ½±«®¬ ®±´´­ ¼± ²±¬ ­¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿­ ¬»­¬»¼ô ¾»½¿«­» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»­¬ò ̸·­ ·­ ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ­±«®½»­ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±©­ ¼»°»²¼­ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò "}
+{"name": "german", "text": "Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikte schweigen die Quellen. "}
+{"name": "german-reflected", "text": "Ü·» Ù®«²¼¸»®®­½¸¿º¬ ©¿® µ»·² Û·¹»²¬«³ ·³ ®ö³·­½¸»² Í·²²»ô ­±²¼»®² »·² Þü²¼»´ ª±² ß²­°®ü½¸»²ò Ü·» Þ¿«»®² ­½¸«´¼»¬»² ß¾¹¿¾»² «²¼ Ú®±²¼·»²­¬»å ¼»® Ø»®® ­½¸«´¼»¬» ͽ¸«¬¦ò ܾ»® ¼·» Ø䫺·¹µ»·¬ ¼»® Õ±²º´·µ¬» ­½¸©»·¹»² ¼·» Ï«»´´»²ò Ü·» Ù®«²¼¸»®®­½¸¿º¬ ©¿® µ»·² Û·¹»²¬«³ ·³ ®ö³·­½¸»² Í·²²»ô ­±²¼»®² »·² Þü²¼»´ ª±² ß²­°®ü½¸»²ò Ü·» Þ¿«»®² ­½¸«´¼»¬»² ß¾¹¿¾»² «²¼ Ú®±²¼·»²­¬»å ¼»® Ø»®® ­½¸«´¼»¬» ͽ¸«¬¦ò ܾ»® ¼·» Ø䫺·¹µ»·¬ ¼»® Õ±²º´·µ¬» ­½¸©»·¹»² ¼·» Ï«»´´»²ò Ü·» Ù®«²¼¸»®®­½¸¿º¬ ©¿® µ»·² Û·¹»²¬«³ ·³ ®ö³·­½¸»² Í·²²»ô ­±²¼»®² »·² Þü²¼»´ ª±² ß²­°®ü½¸»²ò Ü·» Þ¿«»®² ­½¸«´¼»¬»² ß¾¹¿¾»² «²¼ Ú®±²¼·»²­¬»å ¼»® Ø»®® ­½¸«´¼»¬» ͽ¸«¬¦ò ܾ»® ¼·» Ø䫺·¹µ»·¬ ¼»® Õ±²º´·µ¬» ­½¸©»·¹»² ¼·» Ï«»´´»²ò "}
+{"name": "greek", "text": "Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. "}
+{"name": "cyrillic", "text": "Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. "}
+{"name": "chess", "text": "1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O "}
+{"name": "running-heads", "text": "The Second Estate   41\nThe Second Estate   41\nThe Second Estate   41\nThe Second Estate   41\nThe Second Estate   41\nThe Second Estate   41\nThe Second Estate   41\nThe Second Estate   41\n"}
+{"name": "combining-letters", "text": "wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. "}
+{"name": "nonsense-latin1", "text": "¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíô"}
+{"name": "nonsense-reflected", "text": "¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦­´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíô"}
+{"name": "mixed-scripts", "text": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, becauseἈθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikteτὰ 東京 ١٢٣ The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, becauseἈθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikteτὰ 東京 ١٢٣ "}
+{"name": "all-caps", "text": "THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY "}
+{"name": "mixed-case-junk", "text": "zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x "}
+{"name": "digits", "text": "1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 "}
+{"name": "punctuation", "text": ".,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- "}
+{"name": "emdash-prose", "text": "The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. "}
+{"name": "nbsp-prose", "text": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. "}
+{"name": "control-chars", "text": "text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c "}
+{"name": "ligatures", "text": "The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. "}

parity/golden/pdftriage/samples.ndjson

@@ -0,0 +1,24 @@
+{"ascii": 0.0, "body": 0, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "empty", "tidy": null, "vowelled": null}
+{"ascii": 0.0, "body": 0, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "whitespace", "tidy": null, "vowelled": null}
+{"ascii": 1.0, "body": 29, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.9655172413793104, "name": "prose-short", "tidy": null, "vowelled": null}
+{"ascii": 1.0, "body": 1104, "decoded": null, "english": 0.42105263157894735, "latin1": 0.0, "letters": 0.9782608695652174, "name": "prose", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 0.0, "body": 1104, "decoded": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. ", "english": 0.0, "latin1": 1.0, "letters": 0.07608695652173914, "name": "prose-reflected", "tidy": null, "vowelled": null}
+{"ascii": 0.0, "body": 368, "decoded": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. ", "english": null, "latin1": 1.0, "letters": 0.07608695652173914, "name": "prose-reflected-short", "tidy": null, "vowelled": null}
+{"ascii": 0.9740932642487047, "body": 579, "decoded": null, "english": 0.0, "latin1": 0.025906735751295335, "letters": 0.9740932642487047, "name": "german", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 0.0, "body": 579, "decoded": "Die Grundherrschaft war kein Eigentum im r*mischen Sinne, sondern ein B$ndel von Anspr$chen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Dber die H<ufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im r*mischen Sinne, sondern ein B$ndel von Anspr$chen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Dber die H<ufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im r*mischen Sinne, sondern ein B$ndel von Anspr$chen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Dber die H<ufigkeit der Konflikte schweigen die Quellen. ", "english": 0.0, "latin1": 1.0, "letters": 0.16580310880829016, "name": "german-reflected", "tidy": null, "vowelled": null}
+{"ascii": 0.025, "body": 240, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.975, "name": "greek", "tidy": null, "vowelled": null}
+{"ascii": 0.125, "body": 240, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.875, "name": "cyrillic", "tidy": null, "vowelled": null}
+{"ascii": 1.0, "body": 348, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.4482758620689655, "name": "chess", "tidy": null, "vowelled": null}
+{"ascii": 1.0, "body": 136, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.8823529411764706, "name": "running-heads", "tidy": null, "vowelled": null}
+{"ascii": 0.7878787878787878, "body": 660, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.8181818181818182, "name": "combining-letters", "tidy": null, "vowelled": null}
+{"ascii": 0.0, "body": 800, "decoded": null, "english": null, "latin1": 1.0, "letters": 0.685, "name": "nonsense-latin1", "tidy": null, "vowelled": null}
+{"ascii": 0.0, "body": 800, "decoded": null, "english": null, "latin1": 1.0, "letters": 0.685, "name": "nonsense-reflected", "tidy": null, "vowelled": null}
+{"ascii": 0.8675324675324675, "body": 770, "decoded": null, "english": 0.1917808219178082, "latin1": 0.012987012987012988, "letters": 0.9662337662337662, "name": "mixed-scripts", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 1.0, "body": 280, "decoded": null, "english": null, "latin1": 0.0, "letters": 1.0, "name": "all-caps", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 1.0, "body": 1080, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.7777777777777778, "name": "mixed-case-junk", "tidy": 0.0, "vowelled": 0.25}
+{"ascii": 1.0, "body": 400, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "digits", "tidy": null, "vowelled": null}
+{"ascii": 1.0, "body": 520, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "punctuation", "tidy": null, "vowelled": null}
+{"ascii": 1.0, "body": 564, "decoded": null, "english": 0.4166666666666667, "latin1": 0.0, "letters": 0.9361702127659575, "name": "emdash-prose", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 1.0, "body": 736, "decoded": null, "english": 0.42105263157894735, "latin1": 0.0, "letters": 0.9782608695652174, "name": "nbsp-prose", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 0.9523809523809523, "body": 630, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.9523809523809523, "name": "control-chars", "tidy": 1.0, "vowelled": 1.0}
+{"ascii": 0.8823529411764706, "body": 680, "decoded": null, "english": 0.5, "latin1": 0.0, "letters": 0.9705882352941176, "name": "ligatures", "tidy": 1.0, "vowelled": 0.875}

parity/oracle/pdftriage.py

@@ -0,0 +1,70 @@
+"""Measure text layers the way the Python pipeline measures them.
+
+Reads one `{"name", "text"}` JSON object per line and writes back the numbers
+each gate is decided on, so the judgement can be compared without shipping any
+book. Two of the measurements — the letter share and the Latin-1 share — are
+written inline in the Python's garbled gate rather than in a function of their
+own, so they are spelled out again here, copied line for line.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+        python3 oracle/pdftriage.py < cases/pdftriage/samples.ndjson
+"""
+
+import json
+import os
+import sys
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from book_pipeline import convert  # noqa: E402
+
+
+def letter_share(body: list[str]) -> float:
+    if not body:
+        return 0.0
+    return sum(1 for c in body if c.isalpha()) / len(body)
+
+
+def latin1_share(body: list[str]) -> float:
+    if not body:
+        return 0.0
+    return sum(1 for c in body if 0xA1 <= ord(c) <= 0xFF) / len(body)
+
+
+def english_share(text: str, min_words: int = 80) -> float | None:
+    words = [w.strip(".,;:!?()[]\"'").lower() for w in text.split()]
+    words = [w for w in words if w]
+    if len(words) < min_words:
+        return None
+    return sum(1 for w in words if w in convert._COMMON_WORDS) / len(words)
+
+
+def main() -> int:
+    out = sys.stdout
+    for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+        if not line.strip():
+            continue
+        case = json.loads(line)
+        text = case["text"]
+        body = [c for c in text if not c.isspace()]
+        shape = convert._word_shape(text)
+        decoded = convert.undo_reflected_font(text)
+        record = {
+            "name": case["name"],
+            "body": len(body),
+            "ascii": convert._ascii_text_score(text),
+            "letters": letter_share(body),
+            "latin1": latin1_share(body),
+            "vowelled": None if shape is None else shape[0],
+            "tidy": None if shape is None else shape[1],
+            "english": english_share(text),
+            "decoded": decoded,
+        }
+        out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,11 +19,12 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 4] = [
+    let jobs: [Job; 5] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
         ("mdscrub", mdscrub),
+        ("pdftriage", pdftriage),
     ];
 
     for (name, job) in jobs {
@@ -67,6 +68,18 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn pdftriage() -> Written {
+    let dir = parity::golden("pdftriage");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/pdftriage/samples.ndjson");
+    let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let measured = parity::run_oracle("pdftriage.py", &text)?;
+    let path = dir.join("samples.ndjson");
+    std::fs::write(&path, measured).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn mdscrub() -> Written {
     let dir = parity::golden("mdscrub");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/pdftriage.rs

@@ -0,0 +1,124 @@
+//! Every number a text-layer verdict is decided on must match the Python
+//! exactly.
+//!
+//! The thresholds these feed were measured against three thousand real PDFs,
+//! so a score that is a little different is a verdict that is occasionally
+//! different — and a wrong "garbled" costs an hour of GPU time while a wrong
+//! "healthy" puts unreadable text in the index for good. The samples are
+//! synthetic: a parity test over a text-layer classifier needs the score
+//! inputs, not anybody's book.
+
+use serde_json::Value;
+
+/// Both sides compute a count divided by a count, so agreement is exact or
+/// the counts differ.
+fn same(ours: f64, theirs: &Value) -> bool {
+    match theirs {
+        Value::Null => false,
+        value => value.as_f64() == Some(ours),
+    }
+}
+
+fn cases() -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join("cases/pdftriage/samples.ndjson"))
+        .expect("a readable case file");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+#[test]
+fn every_measurement_matches_python() {
+    let golden: Vec<Value> = parity::read_golden_text("pdftriage/samples.ndjson")
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect();
+    let cases = cases();
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    let mut wrong: Vec<String> = Vec::new();
+    for (case, theirs) in cases.iter().zip(&golden) {
+        let name = case["name"].as_str().expect("a name");
+        assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
+        let text = case["text"].as_str().expect("text");
+        let body = pdftriage::score::body(text);
+
+        let mut note = |field: &str, ours: String, theirs: &Value| {
+            wrong.push(format!("{name}.{field}: python {theirs}, rust {ours}"));
+        };
+
+        if !same(body.len() as f64, &theirs["body"]) {
+            note("body", body.len().to_string(), &theirs["body"]);
+        }
+        if !same(pdftriage::score::ascii_text_score(text), &theirs["ascii"]) {
+            let ours = pdftriage::score::ascii_text_score(text);
+            note("ascii", ours.to_string(), &theirs["ascii"]);
+        }
+        if !same(pdftriage::score::letter_share(&body), &theirs["letters"]) {
+            let ours = pdftriage::score::letter_share(&body);
+            note("letters", ours.to_string(), &theirs["letters"]);
+        }
+        if !same(pdftriage::score::latin1_share(&body), &theirs["latin1"]) {
+            let ours = pdftriage::score::latin1_share(&body);
+            note("latin1", ours.to_string(), &theirs["latin1"]);
+        }
+
+        let shape = pdftriage::score::word_shape(text, 40);
+        match (shape, theirs["vowelled"].as_f64(), theirs["tidy"].as_f64()) {
+            (None, None, None) => {}
+            (Some((vowelled, tidy)), Some(their_vowelled), Some(their_tidy)) => {
+                if vowelled != their_vowelled {
+                    note("vowelled", vowelled.to_string(), &theirs["vowelled"]);
+                }
+                if tidy != their_tidy {
+                    note("tidy", tidy.to_string(), &theirs["tidy"]);
+                }
+            }
+            (ours, _, _) => note("word_shape", format!("{ours:?}"), &theirs["vowelled"]),
+        }
+
+        let english = pdftriage::english_prose_share(text, 80);
+        match (english, theirs["english"].as_f64()) {
+            (None, None) => {}
+            (Some(ours), Some(theirs_value)) if ours == theirs_value => {}
+            (ours, _) => note("english", format!("{ours:?}"), &theirs["english"]),
+        }
+
+        let decoded = pdftriage::reflect::undo(text);
+        let their_decoded = theirs["decoded"].as_str();
+        if decoded.as_deref() != their_decoded {
+            note(
+                "decoded",
+                match &decoded {
+                    None => "none".to_string(),
+                    Some(text) => format!("{} characters", text.chars().count()),
+                },
+                &theirs["decoded"],
+            );
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} measurements differ:\n{}",
+        wrong.len(),
+        wrong
+            .iter()
+            .take(20)
+            .map(|line| line.chars().take(200).collect::<String>())
+            .collect::<Vec<_>>()
+            .join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/pdftriage/samples.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("pdftriage.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("pdftriage/samples.ndjson"));
+}