pdftriage: text layer verdicts, including the reflected-font recovery

crates/pdftriage/src/reflect.rs

@@ -0,0 +1,187 @@
+//! Recovering a text layer that is not damaged, only permuted.
+
+use crate::score::{ascii_text_score, word_shape};
+
+/// Codes are stored reflected about this point.
+const ABOUT: u32 = 0x120;
+/// The band the reflection applies to.
+const LOW: u32 = 0xa1;
+const HIGH: u32 = 0xff;
+
+/// Below this, the text as extracted does not read as ordinary text at all,
+/// which is the precondition for trying the decode.
+const LOOKS_DEAD: f64 = 0.35;
+/// The decode has to improve the score by at least this much to be believed.
+const MUST_IMPROVE: f64 = 0.30;
+/// Landing in the ASCII range is not the same as landing in a language.
+/// Measured on the two damaged books in the library this came from: tidy-case
+/// 0.999 for the recovered text, 0.224 for a wrong-key decode that also
+/// reflects into ASCII but into nonsense — `zgW$c_`, `iXYc$f\KO`.
+const MIN_TIDY_CASE: f64 = 0.90;
+const MIN_VOWELLED: f64 = 0.95;
+/// Words needed before word shape means anything.
+const MIN_WORDS: usize = 40;
+
+/// Glyph slots outside the reflected band.
+///
+/// These are the ligatures, the quotes and dashes, and the accented letters.
+/// Each was read off decoded context — `Univer<>ity` is an s, `Scienti<>c` an
+/// fi, `Wolfenb<>ttel` a u-umlaut — rather than guessed. A slot with no
+/// determined reading is left exactly as it stands: a rare glyph that still
+/// looks odd is fixable later, and a wrong guess is not.
+const SLOTS: &[(char, &str)] = &[
+    ('-', "s"),
+    ('A', "fl"),
+    ('B', "fi"),
+    ('H', "ij"),
+    ('K', "\u{2019}"),
+    ('L', "\u{201c}"),
+    ('M', "\u{201d}"),
+    ('N', "\u{2018}"),
+    ('O', "\u{2014}"),
+    ('P', "\u{2013}"),
+    ('U', "\u{c0}"),
+    ('a', "\u{f8}"),
+    ('w', "\u{a9}"),
+    ('\u{152}', "\u{ee}"),
+    ('\u{160}', "\u{f1}"),
+    ('\u{178}', "\u{c5}"),
+    ('\u{17d}', "\u{ed}"),
+    ('\u{2c6}', "\u{f2}"),
+    ('\u{2dc}', "\u{e0}"),
+    ('\u{2013}', "\u{e4}"),
+    ('\u{2014}', "\u{e2}"),
+    ('\u{2018}', "\u{e8}"),
+    ('\u{2019}', "\u{e9}"),
+    ('\u{201c}', "\u{e7}"),
+    ('\u{201e}', "\u{fa}"),
+    ('\u{2020}', "\u{f6}"),
+    ('\u{2021}', "\u{f4}"),
+    ('\u{2022}', "\u{fc}"),
+    ('\u{2030}', "\u{f3}"),
+    ('\u{2039}', "\u{ef}"),
+    ('\u{203a}', "\u{d6}"),
+    ('\u{2122}', "\u{e1}"),
+];
+
+/// Reverse a reflected-ASCII font layer, or `None` if that is not what this
+/// is.
+///
+/// Some PDFs store every printable ASCII code reflected about a fixed point,
+/// so that the text extracts as Latin-1 punctuation soup: `ײ¬®±¼«½¬·±²` is
+/// "Introduction". The mapping is exact, so the text comes back in full —
+/// which is strictly better than an OCR pass that could only approximate it.
+///
+/// Deliberately self-validating rather than gated on which program produced
+/// the file. The transform is applied only when the text as extracted does
+/// not read as ordinary text at all *and* reflecting it makes it read as
+/// ordinary text. A healthy book fails the first test, so it can never be
+/// mangled by this; a Greek or Cyrillic book fails the second, because its
+/// code points lie outside the band and the reflection changes nothing.
+pub fn undo(text: &str) -> Option<String> {
+    let before = ascii_text_score(text);
+    if before >= LOOKS_DEAD {
+        return None;
+    }
+
+    let mut decoded = String::with_capacity(text.len());
+    for c in text.chars() {
+        let code = c as u32;
+        if (LOW..=HIGH).contains(&code) {
+            decoded.push(char::from_u32(ABOUT - code).expect("the band reflects into ASCII"));
+        } else {
+            match slot(c) {
+                Some(reading) => decoded.push_str(reading),
+                None => decoded.push(c),
+            }
+        }
+    }
+
+    if ascii_text_score(&decoded) - before < MUST_IMPROVE {
+        return None;
+    }
+    // Declining here leaves the file for a re-OCR, which is the right answer
+    // when the key is not this one.
+    let (vowelled, tidy_case) = word_shape(&decoded, MIN_WORDS)?;
+    if tidy_case < MIN_TIDY_CASE || vowelled < MIN_VOWELLED {
+        return None;
+    }
+    Some(decoded)
+}
+
+fn slot(c: char) -> Option<&'static str> {
+    SLOTS
+        .binary_search_by_key(&c, |(glyph, _)| *glyph)
+        .ok()
+        .map(|at| SLOTS[at].1)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Encode text the way such a PDF stores it, so a test can be written
+    /// without shipping one of the books this was measured on.
+    fn reflect(text: &str) -> String {
+        text.chars()
+            .map(|c| {
+                let code = c as u32;
+                let reflected = ABOUT - code;
+                if (LOW..=HIGH).contains(&reflected) {
+                    char::from_u32(reflected).expect("in range")
+                } else {
+                    c
+                }
+            })
+            .collect()
+    }
+
+    fn prose() -> String {
+        "The second estate held its land in exchange for service and the service was \
+         military That much is agreed and what follows is not The court rolls do not \
+         say how often the obligation was tested because the clerk who recorded it had \
+         no vocabulary for the difference between a rebellion and a bad harvest"
+            .to_string()
+    }
+
+    #[test]
+    fn a_reflected_layer_comes_back_whole() {
+        let stored = reflect(&prose());
+        assert!(
+            ascii_text_score(&stored) < 0.1,
+            "the fixture is not reflected"
+        );
+        assert_eq!(undo(&stored).as_deref(), Some(prose().as_str()));
+    }
+
+    #[test]
+    fn healthy_text_is_never_touched() {
+        assert_eq!(undo(&prose()), None);
+    }
+
+    #[test]
+    fn a_greek_layer_is_left_alone() {
+        let greek = "Ἀθῆναι καὶ Μόσχα καὶ τὰ λοιπά ".repeat(20);
+        assert_eq!(undo(&greek), None);
+    }
+
+    #[test]
+    fn nonsense_that_reflects_into_ascii_is_refused() {
+        // A layer scrambled with some other key reflects into ASCII too, but
+        // into words with no vowels and no case pattern.
+        let nonsense: String = (0..400)
+            .map(|i| char::from_u32(0xa1 + (i * 7) % 0x5f).expect("in band"))
+            .collect();
+        assert_eq!(undo(&nonsense), None);
+    }
+
+    #[test]
+    fn too_little_text_to_judge_is_refused() {
+        assert_eq!(undo(&reflect("Introduction")), None);
+    }
+
+    #[test]
+    fn the_slot_table_is_sorted_so_it_can_be_searched() {
+        assert!(SLOTS.windows(2).all(|pair| pair[0].0 < pair[1].0));
+    }
+}