funkpower/catena
1
0
Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-10-22 18:57:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

pdftriage: text layer verdicts, including the reflected-font recovery

Browse Source
This commit is contained in:
funkpower
2026-09-19 11:46:34 -04:00
parent bd0ae11d01
commit 33c2e40a1a
16 changed files with 1465 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/corpus-core/src/ndjson.rs
+11
View File
@@ -328,6 +328,17 @@ mod tests {
assert_eq!(back[0].text, "two\nlines");
}
#[test]
fn a_float_survives_the_round_trip_bit_for_bit() {
for score in [0.1, 0.9740932642487047, 1.0 / 3.0, f64::MIN_POSITIVE, 1e308] {
let hit = Hit::new("a", Channel::Vector, 1, score);
let mut out = Vec::new();
write_line(&mut out, &hit).unwrap();
let back: Vec<Hit> = read(out.as_slice()).collect::<Result<_, _>>().unwrap();
assert_eq!(back[0].score.to_bits(), score.to_bits(), "{score}");
}
}
#[test]
fn what_is_written_reads_back_identical() {
let hit = Hit::new("a", Channel::Vector, 3, 0.125);
crates/pdftriage/Cargo.toml
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "pdftriage"
description = "Says whether a PDF's text layer is worth reading, or whether it has to be OCRed."
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
clap.workspace = true
corpus-core.workspace = true
regex.workspace = true
serde_json.workspace = true
textkit.workspace = true
unicode-general-category.workspace = true
[lints]
workspace = true
crates/pdftriage/src/decode.rs
+71
View File
@@ -0,0 +1,71 @@
//! Reading what a PDF text extractor wrote.
/// Decode the bytes a PDF text extractor produced.
///
/// Poppler defaults to UTF-8, but a broken character map or a legacy Latin
/// text layer often emits bytes that are not valid UTF-8 at all — a lone
/// `0xed` is the usual one. Strict UTF-8 first, then Windows-1252, which
/// covers the Western European bytes without failing. Bytes that Windows-1252
/// does not define become the replacement character, exactly as the Python
/// this was ported from does.
pub fn decode(bytes: &[u8]) -> String {
match std::str::from_utf8(bytes) {
Ok(text) => text.to_string(),
Err(_) => bytes.iter().map(|byte| windows_1252(*byte)).collect(),
}
}
fn windows_1252(byte: u8) -> char {
if byte < 0x80 {
return byte as char;
}
HIGH[byte as usize - 0x80]
}
/// Windows-1252 for the bytes above ASCII. Generated from the codec itself.
static HIGH: [char; 128] = [
'\u{20ac}', '\u{fffd}', '\u{201a}', '\u{192}', '\u{201e}', '\u{2026}', '\u{2020}', '\u{2021}',
'\u{2c6}', '\u{2030}', '\u{160}', '\u{2039}', '\u{152}', '\u{fffd}', '\u{17d}', '\u{fffd}',
'\u{fffd}', '\u{2018}', '\u{2019}', '\u{201c}', '\u{201d}', '\u{2022}', '\u{2013}', '\u{2014}',
'\u{2dc}', '\u{2122}', '\u{161}', '\u{203a}', '\u{153}', '\u{fffd}', '\u{17e}', '\u{178}',
'\u{a0}', '\u{a1}', '\u{a2}', '\u{a3}', '\u{a4}', '\u{a5}', '\u{a6}', '\u{a7}', '\u{a8}',
'\u{a9}', '\u{aa}', '\u{ab}', '\u{ac}', '\u{ad}', '\u{ae}', '\u{af}', '\u{b0}', '\u{b1}',
'\u{b2}', '\u{b3}', '\u{b4}', '\u{b5}', '\u{b6}', '\u{b7}', '\u{b8}', '\u{b9}', '\u{ba}',
'\u{bb}', '\u{bc}', '\u{bd}', '\u{be}', '\u{bf}', '\u{c0}', '\u{c1}', '\u{c2}', '\u{c3}',
'\u{c4}', '\u{c5}', '\u{c6}', '\u{c7}', '\u{c8}', '\u{c9}', '\u{ca}', '\u{cb}', '\u{cc}',
'\u{cd}', '\u{ce}', '\u{cf}', '\u{d0}', '\u{d1}', '\u{d2}', '\u{d3}', '\u{d4}', '\u{d5}',
'\u{d6}', '\u{d7}', '\u{d8}', '\u{d9}', '\u{da}', '\u{db}', '\u{dc}', '\u{dd}', '\u{de}',
'\u{df}', '\u{e0}', '\u{e1}', '\u{e2}', '\u{e3}', '\u{e4}', '\u{e5}', '\u{e6}', '\u{e7}',
'\u{e8}', '\u{e9}', '\u{ea}', '\u{eb}', '\u{ec}', '\u{ed}', '\u{ee}', '\u{ef}', '\u{f0}',
'\u{f1}', '\u{f2}', '\u{f3}', '\u{f4}', '\u{f5}', '\u{f6}', '\u{f7}', '\u{f8}', '\u{f9}',
'\u{fa}', '\u{fb}', '\u{fc}', '\u{fd}', '\u{fe}', '\u{ff}',
];
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn valid_utf8_is_read_as_utf8() {
assert_eq!(
decode("caf\u{e9} \u{4e2d}".as_bytes()),
"caf\u{e9} \u{4e2d}"
);
}
#[test]
fn a_lone_high_byte_falls_back_to_windows_1252() {
assert_eq!(decode(&[b'a', 0xed, b'b']), "a\u{ed}b");
assert_eq!(decode(&[0x93, b'x', 0x94]), "\u{201c}x\u{201d}");
}
#[test]
fn an_undefined_byte_becomes_the_replacement_character() {
assert_eq!(decode(&[0xff, 0x81]), "\u{ff}\u{fffd}");
}
#[test]
fn the_table_covers_every_high_byte() {
assert_eq!(HIGH.len(), 128);
}
}
crates/pdftriage/src/lib.rs
+333
View File
@@ -0,0 +1,333 @@
//! Says whether a PDF's text layer is worth reading, or whether it has to be
//! OCRed.
//!
//! OCR is the expensive step in any ingest: minutes of GPU time a page, and a
//! result that is worse than a text layer the file already had. So the
//! question "is this layer any good" is worth answering carefully, and the
//! thresholds below are measurements rather than guesses — each one is the
//! middle of a gap observed across three thousand real PDFs, with the reason
//! recorded beside it.
//!
//! Everything here is a function over already-extracted text. Running
//! `pdftotext` is the binary's job, so that the judgement can be tested
//! against the numbers without shipping anybody's books.
pub mod decode;
pub mod reflect;
pub mod score;
use corpus_core::TextLayer;
/// Nothing legitimate carries more than two per cent of its characters in the
/// Latin-1 band, and the lowest genuine letter share measured was 0.39 —
/// chess notation, recipe lists, a page index. Both gates sit clear of that
/// by a wide margin, because the cost of a false positive is a needless
/// re-OCR of a book that already reads.
const DEAD_LETTER_SHARE: f64 = 0.25;
const DEAD_LATIN1_SHARE: f64 = 0.35;
/// Non-space characters a page below which a text layer is not really there.
///
/// Chosen from the measured gap, not from a round number. The broken books
/// sit at 3, 86, 94 and 132 characters a page — a scan whose layer holds only
/// the page numbers and the running heads. The lowest-density legitimate
/// books are small-format e-reader PDFs at 258 to 382. Two hundred sits in
/// the middle of that gap with margin on both sides.
const SPARSE_CHARS_PER_PAGE: f64 = 200.0;
/// Function words that make up a large, stable share of any English prose.
///
/// A damaged synthetic-font layer still emits letters, so counting letters
/// proves nothing — but it cannot reproduce these. Healthy layers score 0.25
/// to 0.40; garbled ones score near zero. A tenth sits in the gap with room
/// on both sides.
const COMMON_WORDS: &[&str] = &[
"a", "all", "an", "and", "are", "as", "at", "be", "been", "but", "by", "for", "from", "had",
"have", "his", "in", "is", "it", "more", "not", "of", "on", "one", "or", "that", "the",
"there", "they", "this", "to", "was", "were", "which", "with", "you",
];
const HEALTHY_WORD_RATIO: f64 = 0.10;
/// Characters stripped from the ends of a word before it is looked up.
const WORD_TRIM: &[char] = &['.', ',', ';', ':', '!', '?', '(', ')', '[', ']', '"', '\''];
/// How much text has to be there before a judgement means anything.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Limits {
/// Non-space characters in the opening sample, below which the layer is
/// treated as absent.
pub min_chars_present: usize,
/// Non-space characters in the middle sample, below which damage cannot
/// be judged.
pub min_chars_to_judge: usize,
/// Words needed before the English prose check means anything.
pub min_words: usize,
/// Pages below which density says nothing — a handful of pages cannot be
/// judged that way.
pub min_pages_for_density: u32,
}
impl Default for Limits {
fn default() -> Self {
Self {
min_chars_present: 200,
min_chars_to_judge: 400,
min_words: 80,
min_pages_for_density: 8,
}
}
}
/// What was read off a file, and what it means.
#[derive(Debug, Clone, PartialEq)]
pub struct Triage {
/// The verdict.
pub layer: TextLayer,
/// Whether the file has to be OCRed to become searchable.
pub needs_ocr: bool,
/// What the numbers were, in the order they were taken.
pub notes: Vec<String>,
}
/// The text a judgement is made from.
#[derive(Debug, Clone, Copy)]
pub struct Sample<'a> {
/// Text extracted from the opening pages, which answers whether there is
/// a layer at all.
pub opening: &'a str,
/// Text extracted from the middle of the document, which answers whether
/// the layer is any good. The middle, because front matter is often a
/// bare scan in a book whose body carries perfectly good text — and is
/// often cleanly typeset boilerplate in a book whose body is damaged.
pub middle: &'a str,
/// How many pages the middle sample covers.
pub middle_pages: u32,
/// Pages in the whole document, when they could be counted.
pub pages: Option<u32>,
}
/// Judge a text layer.
///
/// The order is not arbitrary. A layer that is not there at all cannot be
/// judged for damage; a layer that is only permuted is recoverable in full
/// and must not be sent for OCR; damage is a stronger statement than
/// thinness, so it is tested first.
pub fn triage(sample: &Sample<'_>, limits: &Limits) -> Triage {
let mut notes = Vec::new();
let opening_chars = score::body(sample.opening).len();
notes.push(format!("{opening_chars} characters on the opening pages"));
if opening_chars < limits.min_chars_present {
return Triage {
layer: TextLayer::Absent,
needs_ocr: true,
notes,
};
}
let middle = score::body(sample.middle);
let ascii = score::ascii_text_score(sample.middle);
notes.push(format!(
"{} characters over {} sample pages, ascii text score {ascii:.2}",
middle.len(),
sample.middle_pages
));
if reflect::undo(sample.middle).is_some() {
notes.push("reflecting the character codes recovers ordinary text".into());
return Triage {
layer: TextLayer::ReflectedFont,
needs_ocr: false,
notes,
};
}
if middle.len() >= limits.min_chars_to_judge {
let letters = score::letter_share(&middle);
let latin1 = score::latin1_share(&middle);
notes.push(format!("letters {letters:.2}, latin-1 {latin1:.2}"));
if letters < DEAD_LETTER_SHARE || latin1 > DEAD_LATIN1_SHARE {
return Triage {
layer: TextLayer::Garbled,
needs_ocr: true,
notes,
};
}
}
if let Some(pages) = sample.pages
&& pages >= limits.min_pages_for_density
{
let density = middle.len() as f64 / sample.middle_pages.max(1) as f64;
notes.push(format!("{density:.0} characters a page"));
if density < SPARSE_CHARS_PER_PAGE {
return Triage {
layer: TextLayer::Sparse,
needs_ocr: true,
notes,
};
}
}
if let Some(share) = english_prose_share(sample.middle, limits.min_words) {
notes.push(format!("english function words {share:.2}"));
if share < HEALTHY_WORD_RATIO {
notes.push("not English, or not prose — judged on shape alone".into());
}
}
Triage {
layer: TextLayer::Healthy,
needs_ocr: false,
notes,
}
}
/// Share of words that are common English function words, or `None` when
/// there is too little text to judge.
///
/// English-only on purpose, and so never the whole verdict: it cannot judge
/// the German, Greek and Church Slavonic books that sit beside the English
/// ones. It is reported because when it does apply it is the most direct
/// evidence there is that a layer reads as prose.
pub fn english_prose_share(text: &str, min_words: usize) -> Option<f64> {
let words: Vec<String> = text
.split(textkit::is_space)
.map(|word| word.trim_matches(WORD_TRIM).to_lowercase())
.filter(|word| !word.is_empty())
.collect();
if words.len() < min_words {
return None;
}
let common = words
.iter()
.filter(|word| COMMON_WORDS.binary_search(&word.as_str()).is_ok())
.count();
Some(common as f64 / words.len() as f64)
}
/// The pages to judge a layer by: the middle of the document.
///
/// A fixed window near the front is only ever the middle of a pamphlet. On a
/// real book it reads front matter, which is cleanly typeset boilerplate even
/// when the body is damaged — a 424-page commentary scored healthy on pages
/// 10 to 14 and garbled on both 160 to 164 and 300 to 304.
pub fn middle_window(pages: Option<u32>, span: u32) -> (u32, u32) {
let Some(pages) = pages.filter(|pages| *pages >= 1) else {
// The old window, for a document whose length is unknown.
return (10, 14);
};
if pages <= span {
return (1, pages);
}
let start = (pages / 2).saturating_sub(span / 2).max(1);
(start, pages.min(start + span - 1))
}
#[cfg(test)]
mod tests {
use super::*;
fn prose() -> String {
"The second estate held its land in exchange for service, and the service was \
military. That much is agreed; what follows is not. The court rolls do not say \
how often the obligation was tested, because the clerk who recorded it had no \
vocabulary for the difference between a rebellion and a bad harvest. This is the \
central difficulty of the sources, and everything that follows depends on reading \
around it rather than through it."
.repeat(3)
}
fn sample<'a>(opening: &'a str, middle: &'a str, pages: Option<u32>) -> Sample<'a> {
Sample {
opening,
middle,
middle_pages: 5,
pages,
}
}
#[test]
fn the_common_word_list_is_sorted_so_it_can_be_searched() {
assert!(COMMON_WORDS.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn a_book_that_reads_is_healthy() {
let text = prose();
let found = triage(&sample(&text, &text, Some(400)), &Limits::default());
assert_eq!(found.layer, TextLayer::Healthy);
assert!(!found.needs_ocr);
}
#[test]
fn a_scan_with_no_layer_is_absent() {
let found = triage(&sample(" \n 12 \n", "", Some(400)), &Limits::default());
assert_eq!(found.layer, TextLayer::Absent);
assert!(found.needs_ocr);
}
#[test]
fn a_layer_of_latin1_soup_is_garbled() {
// Scrambled with some key other than the reflection, so that the
// recovery pass declines it and the damage stands.
let soup: String = (0..800)
.map(|i: u32| char::from_u32(0xa1 + (i * 7) % 0x5f).expect("in the latin-1 band"))
.collect();
let opening = prose();
let found = triage(&sample(&opening, &soup, Some(400)), &Limits::default());
assert_eq!(found.layer, TextLayer::Garbled, "{:#?}", found.notes);
assert!(found.needs_ocr);
}
#[test]
fn a_layer_holding_only_running_heads_is_sparse() {
let thin = "The Second Estate 41\n".repeat(8);
let opening = prose();
let found = triage(&sample(&opening, &thin, Some(400)), &Limits::default());
assert_eq!(found.layer, TextLayer::Sparse, "{:#?}", found.notes);
assert!(found.needs_ocr);
}
#[test]
fn a_short_document_is_not_judged_on_density() {
let thin = "Page 3\n".repeat(8);
let found = triage(&sample(&prose(), &thin, Some(4)), &Limits::default());
assert_ne!(found.layer, TextLayer::Sparse);
}
#[test]
fn a_permuted_layer_is_recovered_rather_than_sent_for_ocr() {
let stored: String = prose()
.chars()
.map(|c| {
let reflected = 0x120 - c as u32;
if (0xa1..=0xff).contains(&reflected) {
char::from_u32(reflected).expect("in range")
} else {
c
}
})
.collect();
let found = triage(&sample(&stored, &stored, Some(400)), &Limits::default());
assert_eq!(found.layer, TextLayer::ReflectedFont, "{:#?}", found.notes);
assert!(!found.needs_ocr, "a recoverable layer must not be OCRed");
}
#[test]
fn the_window_is_the_middle_of_the_book() {
assert_eq!(middle_window(Some(400), 5), (198, 202));
assert_eq!(middle_window(Some(3), 5), (1, 3));
assert_eq!(middle_window(Some(5), 5), (1, 5));
assert_eq!(middle_window(None, 5), (10, 14));
assert_eq!(middle_window(Some(0), 5), (10, 14));
}
#[test]
fn english_prose_is_recognised_and_soup_is_not() {
let share = english_prose_share(&prose(), 80).expect("enough words");
assert!(share > 0.25, "{share}");
let soup = "xqz ".repeat(200);
assert!(english_prose_share(&soup, 80).expect("enough words") < 0.01);
}
}
crates/pdftriage/src/main.rs
+196
View File
@@ -0,0 +1,196 @@
//! Says whether a PDF's text layer is worth reading, or whether it has to be
//! OCRed.
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode};
use clap::Parser;
use corpus_core::{Doc, Kind, TextLayer, exit};
use pdftriage::{Limits, Sample};
/// Pages sampled from the middle of the document.
const SPAN: u32 = 5;
/// Judge a PDF's text layer: healthy, garbled, sparse, reflected-font, or
/// absent — and so whether OCR is worth its cost.
///
/// Takes a path rather than a stream, because it reads two specific page
/// ranges rather than the whole file.
///
/// Shells out to `pdftotext`, and to `pdfinfo` for the page count; both ship
/// with poppler. Without `pdftotext` there is nothing to judge and the tool
/// exits 69. It never runs OCR, never writes to the file, and imposes no
/// timeout of its own — a caller that needs one owns it, which is half the
/// reason this is a separate process.
#[derive(Parser)]
#[command(
version,
about,
long_about = None,
after_help = "\
Exit codes: 0 the layer needs OCR, 1 it does not, 2 could not tell,
69 pdftotext is not installed, 74 the file could not be read.
Example:
for f in *.pdf; do pdftriage --quiet \"$f\" && ocrmypdf \"$f\" \"ocr/$f\"; done"
)]
struct Args {
/// The PDF to judge.
path: PathBuf,
/// Emit a doc record instead of a human-readable verdict.
#[arg(long)]
json: bool,
/// Say nothing; answer with the exit code alone.
#[arg(long, short)]
quiet: bool,
}
fn main() -> ExitCode {
let args = Args::parse();
let bytes = match std::fs::metadata(&args.path) {
Ok(meta) => meta.len() as i64,
Err(e) => {
eprintln!("pdftriage: {}: {e}", args.path.display());
return ExitCode::from(exit::IO);
}
};
if which("pdftotext").is_none() {
eprintln!("pdftriage: pdftotext is not on PATH (it ships with poppler)");
return ExitCode::from(exit::UNAVAILABLE);
}
let pages = page_count(&args.path);
let (first, last) = pdftriage::middle_window(pages, SPAN);
let opening = extract(&args.path, 1, SPAN);
let middle = extract(&args.path, first, last);
let (Some(opening), Some(middle)) = (opening, middle) else {
eprintln!(
"pdftriage: pdftotext could not read {}",
args.path.display()
);
if args.json && !args.quiet {
let mut doc = unjudged(&args.path, bytes, pages);
doc.note("pdftotext could not read the file");
let _ = emit_json(&doc);
}
return ExitCode::from(exit::INDETERMINATE);
};
let found = pdftriage::triage(
&Sample {
opening: &opening,
middle: &middle,
middle_pages: last.saturating_sub(first) + 1,
pages,
},
&Limits::default(),
);
if !args.quiet {
let wrote = if args.json {
let mut doc = unjudged(&args.path, bytes, pages);
doc.text_layer = Some(found.layer);
doc.needs_ocr = Some(found.needs_ocr);
doc.notes = found.notes.clone();
emit_json(&doc)
} else {
emit_report(&found)
};
if let Err(e) = wrote {
if e.kind() == std::io::ErrorKind::BrokenPipe {
return ExitCode::from(exit::OK);
}
eprintln!("pdftriage: {e}");
return ExitCode::from(exit::IO);
}
}
ExitCode::from(if found.needs_ocr { exit::OK } else { exit::NO })
}
fn unjudged(path: &Path, bytes: i64, pages: Option<u32>) -> Doc {
let mut doc = Doc::new(path.display().to_string(), Kind::Pdf, bytes);
doc.declared_ext = path
.extension()
.map(|ext| ext.to_string_lossy().to_lowercase());
doc.pages = pages;
doc
}
fn emit_json(doc: &Doc) -> std::io::Result<()> {
let mut out = std::io::stdout().lock();
let json = serde_json::to_string(doc).expect("a doc record serialises");
writeln!(out, "{json}")?;
out.flush()
}
fn emit_report(found: &pdftriage::Triage) -> std::io::Result<()> {
let mut out = std::io::stdout().lock();
let verdict = match found.layer {
TextLayer::Healthy => "healthy",
TextLayer::Garbled => "garbled",
TextLayer::Sparse => "sparse",
TextLayer::ReflectedFont => "reflected-font",
TextLayer::Absent => "absent",
};
let advice = if found.needs_ocr {
"needs OCR"
} else {
"no OCR needed"
};
writeln!(out, "{verdict}, {advice}")?;
for note in &found.notes {
writeln!(out, " {note}")?;
}
out.flush()
}
/// Extract a page range as text, or `None` when the extractor could not.
fn extract(path: &Path, first: u32, last: u32) -> Option<String> {
let out = Command::new("pdftotext")
.args(["-f", &first.to_string(), "-l", &last.to_string(), "-layout"])
.arg(path)
.arg("-")
.output()
.ok()?;
out.status
.success()
.then(|| pdftriage::decode::decode(&out.stdout))
}
/// The document's page count, when `pdfinfo` can say.
fn page_count(path: &Path) -> Option<u32> {
let out = Command::new("pdfinfo").arg(path).output().ok()?;
if !out.status.success() {
return None;
}
pdftriage::decode::decode(&out.stdout)
.lines()
.find_map(|line| line.strip_prefix("Pages:"))
.and_then(|count| count.trim().parse().ok())
}
fn which(program: &str) -> Option<PathBuf> {
let path = std::env::var_os("PATH")?;
std::env::split_paths(&path)
.map(|dir| dir.join(program))
.find(|candidate| candidate.is_file())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_program_that_exists_is_found_and_one_that_does_not_is_not() {
assert!(which("sh").is_some() || which("sh.exe").is_some());
assert!(which("definitely-not-a-program-93a1").is_none());
}
}
crates/pdftriage/src/reflect.rs
+187
View File
@@ -0,0 +1,187 @@
//! Recovering a text layer that is not damaged, only permuted.
use crate::score::{ascii_text_score, word_shape};
/// Codes are stored reflected about this point.
const ABOUT: u32 = 0x120;
/// The band the reflection applies to.
const LOW: u32 = 0xa1;
const HIGH: u32 = 0xff;
/// Below this, the text as extracted does not read as ordinary text at all,
/// which is the precondition for trying the decode.
const LOOKS_DEAD: f64 = 0.35;
/// The decode has to improve the score by at least this much to be believed.
const MUST_IMPROVE: f64 = 0.30;
/// Landing in the ASCII range is not the same as landing in a language.
/// Measured on the two damaged books in the library this came from: tidy-case
/// 0.999 for the recovered text, 0.224 for a wrong-key decode that also
/// reflects into ASCII but into nonsense — `zgW$c_`, `iXYc$f\KO`.
const MIN_TIDY_CASE: f64 = 0.90;
const MIN_VOWELLED: f64 = 0.95;
/// Words needed before word shape means anything.
const MIN_WORDS: usize = 40;
/// Glyph slots outside the reflected band.
///
/// These are the ligatures, the quotes and dashes, and the accented letters.
/// Each was read off decoded context — `Univer<>ity` is an s, `Scienti<>c` an
/// fi, `Wolfenb<>ttel` a u-umlaut — rather than guessed. A slot with no
/// determined reading is left exactly as it stands: a rare glyph that still
/// looks odd is fixable later, and a wrong guess is not.
const SLOTS: &[(char, &str)] = &[
('-', "s"),
('A', "fl"),
('B', "fi"),
('H', "ij"),
('K', "\u{2019}"),
('L', "\u{201c}"),
('M', "\u{201d}"),
('N', "\u{2018}"),
('O', "\u{2014}"),
('P', "\u{2013}"),
('U', "\u{c0}"),
('a', "\u{f8}"),
('w', "\u{a9}"),
('\u{152}', "\u{ee}"),
('\u{160}', "\u{f1}"),
('\u{178}', "\u{c5}"),
('\u{17d}', "\u{ed}"),
('\u{2c6}', "\u{f2}"),
('\u{2dc}', "\u{e0}"),
('\u{2013}', "\u{e4}"),
('\u{2014}', "\u{e2}"),
('\u{2018}', "\u{e8}"),
('\u{2019}', "\u{e9}"),
('\u{201c}', "\u{e7}"),
('\u{201e}', "\u{fa}"),
('\u{2020}', "\u{f6}"),
('\u{2021}', "\u{f4}"),
('\u{2022}', "\u{fc}"),
('\u{2030}', "\u{f3}"),
('\u{2039}', "\u{ef}"),
('\u{203a}', "\u{d6}"),
('\u{2122}', "\u{e1}"),
];
/// Reverse a reflected-ASCII font layer, or `None` if that is not what this
/// is.
///
/// Some PDFs store every printable ASCII code reflected about a fixed point,
/// so that the text extracts as Latin-1 punctuation soup: `ײ¬®±¼«½¬·±²` is
/// "Introduction". The mapping is exact, so the text comes back in full —
/// which is strictly better than an OCR pass that could only approximate it.
///
/// Deliberately self-validating rather than gated on which program produced
/// the file. The transform is applied only when the text as extracted does
/// not read as ordinary text at all *and* reflecting it makes it read as
/// ordinary text. A healthy book fails the first test, so it can never be
/// mangled by this; a Greek or Cyrillic book fails the second, because its
/// code points lie outside the band and the reflection changes nothing.
pub fn undo(text: &str) -> Option<String> {
let before = ascii_text_score(text);
if before >= LOOKS_DEAD {
return None;
}
let mut decoded = String::with_capacity(text.len());
for c in text.chars() {
let code = c as u32;
if (LOW..=HIGH).contains(&code) {
decoded.push(char::from_u32(ABOUT - code).expect("the band reflects into ASCII"));
} else {
match slot(c) {
Some(reading) => decoded.push_str(reading),
None => decoded.push(c),
}
}
}
if ascii_text_score(&decoded) - before < MUST_IMPROVE {
return None;
}
// Declining here leaves the file for a re-OCR, which is the right answer
// when the key is not this one.
let (vowelled, tidy_case) = word_shape(&decoded, MIN_WORDS)?;
if tidy_case < MIN_TIDY_CASE || vowelled < MIN_VOWELLED {
return None;
}
Some(decoded)
}
fn slot(c: char) -> Option<&'static str> {
SLOTS
.binary_search_by_key(&c, |(glyph, _)| *glyph)
.ok()
.map(|at| SLOTS[at].1)
}
#[cfg(test)]
mod tests {
use super::*;
/// Encode text the way such a PDF stores it, so a test can be written
/// without shipping one of the books this was measured on.
fn reflect(text: &str) -> String {
text.chars()
.map(|c| {
let code = c as u32;
let reflected = ABOUT - code;
if (LOW..=HIGH).contains(&reflected) {
char::from_u32(reflected).expect("in range")
} else {
c
}
})
.collect()
}
fn prose() -> String {
"The second estate held its land in exchange for service and the service was \
military That much is agreed and what follows is not The court rolls do not \
say how often the obligation was tested because the clerk who recorded it had \
no vocabulary for the difference between a rebellion and a bad harvest"
.to_string()
}
#[test]
fn a_reflected_layer_comes_back_whole() {
let stored = reflect(&prose());
assert!(
ascii_text_score(&stored) < 0.1,
"the fixture is not reflected"
);
assert_eq!(undo(&stored).as_deref(), Some(prose().as_str()));
}
#[test]
fn healthy_text_is_never_touched() {
assert_eq!(undo(&prose()), None);
}
#[test]
fn a_greek_layer_is_left_alone() {
let greek = "Ἀθῆναι καὶ Μόσχα καὶ τὰ λοιπά ".repeat(20);
assert_eq!(undo(&greek), None);
}
#[test]
fn nonsense_that_reflects_into_ascii_is_refused() {
// A layer scrambled with some other key reflects into ASCII too, but
// into words with no vowels and no case pattern.
let nonsense: String = (0..400)
.map(|i| char::from_u32(0xa1 + (i * 7) % 0x5f).expect("in band"))
.collect();
assert_eq!(undo(&nonsense), None);
}
#[test]
fn too_little_text_to_judge_is_refused() {
assert_eq!(undo(&reflect("Introduction")), None);
}
#[test]
fn the_slot_table_is_sorted_so_it_can_be_searched() {
assert!(SLOTS.windows(2).all(|pair| pair[0].0 < pair[1].0));
}
}
crates/pdftriage/src/score.rs
+170
View File
@@ -0,0 +1,170 @@
//! Cheap, script-blind measurements of whether text is text.
use std::sync::LazyLock;
use regex::Regex;
use unicode_general_category::{GeneralCategory, get_general_category};
/// Characters that ordinary Latin-script text is made of.
///
/// Prose in any Latin-script language sits near 1.0 on the share of its
/// non-space characters that are in here; a text layer stored through a
/// reflected font sits near 0.0. The em dash is in the set despite not being
/// ASCII, because it is ordinary in a book and its absence would penalise
/// well-typeset text.
const ASCII_TEXT: &str = concat!(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
".,;:!?'\"()[]{}-\u{2014}/&%$#@*+=<>|~^_`\\"
);
/// Words of three letters or more, which is enough to have a shape.
static WORDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[A-Za-z]{3,}").expect("valid"));
static VOWEL: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[aeiouyAEIOUY]").expect("valid"));
/// Whether a character is a letter, by Python's definition.
///
/// Python's `str.isalpha` is true for the letter categories and nothing else.
/// Rust's [`char::is_alphabetic`] is true for the Unicode `Alphabetic`
/// property, which also covers some seventeen hundred combining marks —
/// including `U+0363`–`U+036F`, the medieval superscript letters that appear
/// in exactly the scanned manuscripts this has to judge. The thresholds below
/// were measured against the narrower definition.
pub fn is_letter(c: char) -> bool {
matches!(
get_general_category(c),
GeneralCategory::UppercaseLetter
| GeneralCategory::LowercaseLetter
| GeneralCategory::TitlecaseLetter
| GeneralCategory::ModifierLetter
| GeneralCategory::OtherLetter
)
}
/// The non-space characters of a text, which every share below is taken over.
pub fn body(text: &str) -> Vec<char> {
text.chars().filter(|c| !textkit::is_space(*c)).collect()
}
/// Share of non-space characters that are ordinary Latin-script text.
pub fn ascii_text_score(text: &str) -> f64 {
let body = body(text);
if body.is_empty() {
return 0.0;
}
let ordinary = body.iter().filter(|c| ASCII_TEXT.contains(**c)).count();
ordinary as f64 / body.len() as f64
}
/// Share of the non-space characters that are letters, in any alphabet.
pub fn letter_share(body: &[char]) -> f64 {
if body.is_empty() {
return 0.0;
}
body.iter().filter(|c| is_letter(**c)).count() as f64 / body.len() as f64
}
/// Share of the non-space characters sitting in the Latin-1 band, which is
/// where a broken character map falls back to.
pub fn latin1_share(body: &[char]) -> f64 {
if body.is_empty() {
return 0.0;
}
let banded = body
.iter()
.filter(|c| (0xa1..=0xff).contains(&(**c as u32)))
.count();
banded as f64 / body.len() as f64
}
/// The share of words carrying a vowel, and the share in a sane case
/// pattern. `None` when there are too few words to judge.
///
/// A cheap, script-blind proxy for "is this a language", used to tell a real
/// decode from one done with the wrong key.
pub fn word_shape(text: &str, min_words: usize) -> Option<(f64, f64)> {
let words: Vec<&str> = WORDS.find_iter(text).map(|found| found.as_str()).collect();
if words.len() < min_words {
return None;
}
let vowelled = words.iter().filter(|word| VOWEL.is_match(word)).count();
let tidy = words.iter().filter(|word| tidy_case(word)).count();
Some((
vowelled as f64 / words.len() as f64,
tidy as f64 / words.len() as f64,
))
}
/// Whether a word is all lower case, Capitalised, or ALL CAPS — which real
/// words are, and `zgW$c_` is not.
fn tidy_case(word: &str) -> bool {
let mut chars = word.chars();
let Some(first) = chars.next() else {
return false;
};
let rest = chars.as_str();
let all_lower = word.chars().all(|c| c.is_ascii_lowercase());
let all_upper = word.chars().all(|c| c.is_ascii_uppercase());
let title = first.is_ascii_uppercase() && rest.chars().all(|c| c.is_ascii_lowercase());
all_lower || all_upper || title
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn prose_scores_near_one() {
let score = ascii_text_score("The second estate held its land in exchange for service.");
assert!(score > 0.95, "{score}");
}
#[test]
fn a_latin1_soup_scores_near_zero() {
let score = ascii_text_score("\u{d7}\u{b2}\u{ac}\u{ae}\u{b1}\u{bc}\u{ab}\u{bd}");
assert_eq!(score, 0.0);
}
#[test]
fn empty_text_scores_zero_rather_than_dividing_by_nothing() {
assert_eq!(ascii_text_score(""), 0.0);
assert_eq!(ascii_text_score(" \n "), 0.0);
assert_eq!(letter_share(&[]), 0.0);
assert_eq!(latin1_share(&[]), 0.0);
}
#[test]
fn a_medieval_combining_letter_is_not_a_letter_here() {
// Python's isalpha says no; Rust's is_alphabetic says yes.
assert!(!is_letter('\u{363}'));
assert!('\u{363}'.is_alphabetic());
assert!(is_letter('a'));
assert!(is_letter('\u{3b1}'));
assert!(!is_letter('1'));
}
#[test]
fn word_shape_needs_enough_words_to_judge() {
assert_eq!(word_shape("one two three", 40), None);
}
#[test]
fn real_words_are_vowelled_and_tidily_cased() {
let text = "The second estate held its land in exchange for service and the service \
was military That much is agreed what follows is not The rolls do not say \
how often the obligation was tested and the clerk had no vocabulary for it "
.repeat(3);
let (vowelled, tidy) = word_shape(&text, 40).expect("enough words");
assert!(vowelled > 0.95, "{vowelled}");
assert!(tidy > 0.95, "{tidy}");
}
#[test]
fn the_case_test_is_what_it_says() {
assert!(tidy_case("lower"));
assert!(tidy_case("Title"));
assert!(tidy_case("UPPER"));
assert!(!tidy_case("mIxEd"));
assert!(!tidy_case("aBc"));
}
}
crates/pdftriage/tests/cli.rs
+196
View File
@@ -0,0 +1,196 @@
//! The tool as a caller meets it: a path in, a verdict and an exit code out.
//!
//! The PDFs are built here rather than committed, so the suite ships no
//! book and no binary fixture. Every test is skipped when `pdftotext` is not
//! installed, because without it there is nothing to judge and the tool says
//! so with exit 69.
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
const PROSE: &str = "The second estate held its land in exchange for service, and
the service was military. That much is agreed; what follows is
not. The court rolls do not say how often the obligation was
tested, because the clerk who recorded it had no vocabulary for
the difference between a rebellion and a bad harvest, and so the
record flattens the two into one offence with one penalty.";
fn pdftotext_installed() -> bool {
Command::new("pdftotext")
.arg("-v")
.output()
.is_ok_and(|out| out.status.success() || !out.stderr.is_empty())
}
fn run(args: &[&str]) -> Output {
Command::new(env!("CARGO_BIN_EXE_pdftriage"))
.args(args)
.output()
.expect("the binary should run")
}
/// Write a PDF with `pages` pages, each holding `text`.
///
/// Hand-assembled rather than generated by a library: a PDF is an object
/// table and a cross-reference index, and twenty lines of it here save a
/// dependency and a committed fixture.
fn write_pdf(path: &Path, text: &str, pages: usize) {
let mut objects: Vec<(usize, Vec<u8>)> = Vec::new();
let page_ids: Vec<usize> = (0..pages).map(|at| 4 + 2 * at).collect();
let kids: Vec<String> = page_ids.iter().map(|id| format!("{id} 0 R")).collect();
objects.push((1, b"<</Type/Catalog/Pages 2 0 R>>".to_vec()));
objects.push((
2,
format!("<</Type/Pages/Kids[{}]/Count {pages}>>", kids.join(" ")).into_bytes(),
));
objects.push((
3,
b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
));
for id in &page_ids {
let contents_id = id + 1;
objects.push((
*id,
format!(
"<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]\
/Contents {contents_id} 0 R/Resources<</Font<</F1 3 0 R>>>>>>"
)
.into_bytes(),
));
let mut stream = String::new();
let mut y = 720;
for line in text.lines() {
let escaped = line
.replace('\\', r"\\")
.replace('(', r"\(")
.replace(')', r"\)");
stream.push_str(&format!("BT /F1 11 Tf 72 {y} Td ({escaped}) Tj ET\n"));
y -= 14;
}
let mut object = format!("<</Length {}>>\nstream\n", stream.len()).into_bytes();
object.extend_from_slice(stream.as_bytes());
object.extend_from_slice(b"\nendstream");
objects.push((contents_id, object));
}
objects.sort_by_key(|(id, _)| *id);
let mut out: Vec<u8> = b"%PDF-1.4\n".to_vec();
let mut offsets: Vec<(usize, usize)> = Vec::new();
for (id, body) in &objects {
offsets.push((*id, out.len()));
out.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
out.extend_from_slice(body);
out.extend_from_slice(b"\nendobj\n");
}
let xref_at = out.len();
let top = objects.last().expect("at least one object").0 + 1;
out.extend_from_slice(format!("xref\n0 {top}\n0000000000 65535 f \n").as_bytes());
for id in 1..top {
let at = offsets
.iter()
.find(|(object, _)| *object == id)
.map_or(0, |(_, at)| *at);
out.extend_from_slice(format!("{at:010} 00000 n \n").as_bytes());
}
out.extend_from_slice(
format!("trailer\n<</Size {top}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n").as_bytes(),
);
let mut file = std::fs::File::create(path).expect("a writable path");
file.write_all(&out).expect("the PDF should write");
}
fn temp(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join("pdftriage-tests");
std::fs::create_dir_all(&dir).expect("a writable temp directory");
dir.join(format!("{name}.pdf"))
}
#[test]
fn a_book_that_reads_is_not_sent_for_ocr() {
if !pdftotext_installed() {
return;
}
let path = temp("healthy");
write_pdf(&path, PROSE, 20);
let out = run(&[path.to_str().expect("utf-8 path")]);
assert_eq!(out.status.code(), Some(1), "a healthy layer must exit 1");
let said = String::from_utf8(out.stdout).expect("utf-8");
assert!(said.starts_with("healthy, no OCR needed"), "{said}");
assert!(said.contains("characters a page"), "{said}");
}
#[test]
fn a_scan_with_no_layer_is_sent_for_ocr() {
if !pdftotext_installed() {
return;
}
let path = temp("absent");
write_pdf(&path, "", 20);
let out = run(&[path.to_str().expect("utf-8 path")]);
assert_eq!(out.status.code(), Some(0), "a dead layer must exit 0");
assert!(String::from_utf8_lossy(&out.stdout).starts_with("absent, needs OCR"));
}
#[test]
fn the_json_form_is_a_doc_record() {
if !pdftotext_installed() {
return;
}
let path = temp("json");
write_pdf(&path, PROSE, 20);
let out = run(&["--json", path.to_str().expect("utf-8 path")]);
let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
assert_eq!(doc["v"], 1);
assert_eq!(doc["kind"], "pdf");
assert_eq!(doc["declared_ext"], "pdf");
assert_eq!(doc["pages"], 20);
assert_eq!(doc["text_layer"], "healthy");
assert_eq!(doc["needs_ocr"], false);
assert!(
doc["notes"]
.as_array()
.is_some_and(|notes| !notes.is_empty())
);
}
#[test]
fn quiet_says_nothing_at_all() {
if !pdftotext_installed() {
return;
}
let path = temp("quiet");
write_pdf(&path, PROSE, 20);
let out = run(&["--quiet", path.to_str().expect("utf-8 path")]);
assert!(out.stdout.is_empty(), "stdout was not empty");
assert!(out.stderr.is_empty(), "stderr was not empty");
assert_eq!(out.status.code(), Some(1));
}
#[test]
fn a_file_that_is_not_there_is_an_io_error() {
let out = run(&["/definitely/not/here.pdf"]);
assert_eq!(out.status.code(), Some(74));
assert!(out.stdout.is_empty(), "a diagnostic reached stdout");
assert!(!out.stderr.is_empty());
}
#[test]
fn a_file_that_is_not_a_pdf_cannot_be_judged() {
if !pdftotext_installed() {
return;
}
let path = temp("not-a-pdf");
std::fs::write(&path, b"this is not a PDF at all\n").expect("a writable path");
let out = run(&[path.to_str().expect("utf-8 path")]);
assert_eq!(out.status.code(), Some(2), "could not tell must exit 2");
}