pdfgutter: never rewrite a page tree it cannot account for

crates/pdfgutter/README.md

@@ -12,8 +12,9 @@ lines interleave, the reading order comes out scrambled, and what lands in the
 index looks like prose and is not.
 
 `pdfgutter` says whether a PDF is that, and splits it when it is. The split
-changes the page boxes and nothing else — no re-rendering, no loss, and a
-250-sheet book becomes a 500-page book at almost no cost in size.
+changes the page boxes and nothing else — nothing is re-rendered, the text and
+the images are the ones that were there, and a 250-sheet book becomes a
+500-page book at almost no cost in size.
 
 ## When you would use it
 
@@ -30,7 +31,8 @@ changes the page boxes and nothing else — no re-rendering, no loss, and a
 cargo install --git https://git.pumpin.online/funkpower/catena pdfgutter
 ```
 
-Detection needs `pdftoppm`, from poppler. Splitting needs nothing.
+Needs `pdftoppm`, from poppler. Nothing is split without first being looked
+at, so it is needed either way.
 
 ## Example
 
@@ -91,8 +93,9 @@ done
 | `--min-ratio <RATIO>` | How wide a sheet must be to hold two pages. Default `1.15` |
 | `--gutter-fraction <FRACTION>` | What share of wide sheets must show a gutter. Default `0.6` |
 
-`--into` writes whatever the verdict was: a document that is not two-up is
-copied through unchanged, so a loop over a directory does not have to branch.
+`--into` writes whatever the verdict was, so a loop over a directory does not
+have to branch. A document with no sheet wide enough to cut is not altered at
+all — every page comes through as it was.
 
 ## Exit codes
 
@@ -100,7 +103,7 @@ copied through unchanged, so a loop over a directory does not have to branch.
 |---|---|
 | `0` | Two pages to a sheet |
 | `1` | One page to a sheet |
-| `2` | It could not be judged — encrypted, or no pages |
+| `2` | It could not be judged — encrypted, no pages, or a page tree that does not add up |
 | `64` | The command line was wrong |
 | `69` | `pdftoppm` is not installed |
 | `74` | The file could not be read or written |
@@ -135,11 +138,24 @@ the answer for a whole book.
 Each wide sheet becomes two pages pointing at the same content, differing only
 in how much of it they show. A sheet the file asks to be turned on its side is
 cut across rather than down, and the halves come out in the order a reader
-would read them.
+would read them. An annotation is kept on the half it sits on rather than
+copied onto both.
 
 Splitting is safe to repeat: the halves are portrait, so a second pass finds
 nothing to do.
 
+Splitting replaces the page tree, which means a page this could not find in
+that tree would be a page dropped from the result. So the number of pages
+found is checked against the number the file says it has, and a document where
+those disagree is refused rather than written short.
+
+### What it measures
+
+The crop box, where a page has one — that is the rectangle a reader shows and
+the one that gets rendered. A wide sheet cropped down to a single page is
+therefore not a spread, which is what stops a second pass cutting a page in
+half. Pages with no crop box, which is most of them, are their media box.
+
 ## As a library
 
 ```rust

crates/pdfgutter/src/lib.rs

@@ -142,6 +142,12 @@ pub fn median(values: &[f64]) -> f64 {
     if values.is_empty() {
         return 0.0;
     }
+    // A run with a number that is not one has no middle. Answering anyway
+    // would put a sheet's shape on the wrong side of a threshold on the
+    // strength of a value that means nothing.
+    if values.iter().any(|value| value.is_nan()) {
+        return f64::NAN;
+    }
     let mut sorted = values.to_vec();
     sorted.sort_by(f64::total_cmp);
     let middle = sorted.len() / 2;
@@ -156,8 +162,20 @@ pub fn median(values: &[f64]) -> f64 {
 /// `gutters` holds one answer per sampled page that was wide enough to look
 /// at, which is why it is usually shorter than `ratios`.
 pub fn is_two_up(ratios: &[f64], gutters: &[bool]) -> bool {
-    let median_ratio = median(ratios);
-    median_ratio > MIN_RATIO && gutter_fraction(gutters) >= MIN_GUTTER_FRACTION
+    two_up_above(ratios, gutters, MIN_RATIO, MIN_GUTTER_FRACTION)
+}
+
+/// The same question against thresholds of your own.
+///
+/// The one place the answer is decided, so that a command offering its own
+/// thresholds cannot come to decide it a second and slightly different way.
+pub fn two_up_above(
+    ratios: &[f64],
+    gutters: &[bool],
+    min_ratio: f64,
+    min_gutter_fraction: f64,
+) -> bool {
+    median(ratios) > min_ratio && gutter_fraction(gutters) >= min_gutter_fraction
 }
 
 /// What share of the pages looked at showed a gutter.
@@ -281,6 +299,25 @@ mod tests {
         assert_eq!(median(&[]), 0.0);
     }
 
+    #[test]
+    fn a_run_with_no_number_in_it_has_no_middle() {
+        assert!(median(&[1.0, f64::NAN, 3.0]).is_nan());
+        // And a shape that is not a number is not a wide sheet.
+        assert!(!is_two_up(&[f64::NAN, 1.4], &[true; 4]));
+    }
+
+    #[test]
+    fn thresholds_of_your_own_decide_it_the_same_way() {
+        let ratios = [1.4; 8];
+        let gutters = [true, true, true, false, false, false, false, false];
+        assert_eq!(
+            is_two_up(&ratios, &gutters),
+            two_up_above(&ratios, &gutters, MIN_RATIO, MIN_GUTTER_FRACTION)
+        );
+        // Three in eight clears a threshold of three in ten.
+        assert!(two_up_above(&ratios, &gutters, MIN_RATIO, 0.3));
+    }
+
     #[test]
     fn a_document_needs_both_the_shape_and_the_gutter() {
         let wide = [1.4; 8];

crates/pdfgutter/src/main.rs

@@ -144,7 +144,7 @@ fn measure(args: &Args, pdf: &pdfgutter::pdf::Pdf) -> Result<Verdict, ExitCode>
     Ok(Verdict {
         v: corpus_core::V,
         path: args.path.display().to_string(),
-        two_up: median_ratio > args.min_ratio && gutter_fraction >= args.gutter_fraction,
+        two_up: pdfgutter::two_up_above(&ratios, &gutters, args.min_ratio, args.gutter_fraction),
         pages: pdf.len(),
         median_ratio,
         gutter_fraction,

crates/pdfgutter/src/pdf.rs

@@ -7,7 +7,7 @@
 use std::fmt;
 use std::path::Path;
 
-use lopdf::{Document, Object, ObjectId};
+use lopdf::{Dictionary, Document, Object, ObjectId};
 
 /// The four page attributes a page may inherit from the tree above it.
 const INHERITED: [&[u8]; 4] = [b"Resources", b"MediaBox", b"CropBox", b"Rotate"];
@@ -33,6 +33,14 @@ pub enum Error {
     Encrypted,
     /// It has no pages.
     Empty,
+    /// The page tree does not add up: the file says one number of pages and
+    /// walking it finds another.
+    Miscounted {
+        /// What the file says.
+        declared: usize,
+        /// What walking the tree found.
+        found: usize,
+    },
     /// It could not be written.
     Unwritable(String),
 }
@@ -43,6 +51,10 @@ impl fmt::Display for Error {
             Error::Unreadable(why) => write!(f, "{why}"),
             Error::Encrypted => write!(f, "it is encrypted"),
             Error::Empty => write!(f, "it has no pages"),
+            Error::Miscounted { declared, found } => write!(
+                f,
+                "it says it has {declared} pages and walking it finds {found}"
+            ),
             Error::Unwritable(why) => write!(f, "{why}"),
         }
     }
@@ -55,16 +67,40 @@ impl std::error::Error for Error {}
 pub struct Geometry {
     /// The media box, left, bottom, right, top, with the corners in order.
     pub media: [f64; 4],
+    /// The crop box, where the page names one. This is what a reader shows
+    /// and what a renderer draws, so it is what everything here measures.
+    pub crop: Option<[f64; 4]>,
     /// A quarter turn clockwise per unit, as the file asks for it.
     pub rotate: i64,
 }
 
 impl Geometry {
-    /// Width and height as displayed, which is the media box turned by
-    /// whatever the page's rotation asks for.
+    /// The rectangle a reader actually shows.
+    ///
+    /// A crop box outside the media box is meaningless, so it is clipped to
+    /// it, and one that clips away the whole page is ignored.
+    pub fn visible(self) -> [f64; 4] {
+        let Some(crop) = self.crop else {
+            return self.media;
+        };
+        let clipped = [
+            crop[0].max(self.media[0]),
+            crop[1].max(self.media[1]),
+            crop[2].min(self.media[2]),
+            crop[3].min(self.media[3]),
+        ];
+        match clipped[2] > clipped[0] && clipped[3] > clipped[1] {
+            true => clipped,
+            false => self.media,
+        }
+    }
+
+    /// Width and height as displayed, which is the visible rectangle turned
+    /// by whatever the page's rotation asks for.
     pub fn displayed(self) -> (f64, f64) {
-        let width = self.media[2] - self.media[0];
-        let height = self.media[3] - self.media[1];
+        let visible = self.visible();
+        let width = visible[2] - visible[0];
+        let height = visible[3] - visible[1];
         match self.quarter_turns() % 2 {
             0 => (width, height),
             _ => (height, width),
@@ -89,7 +125,7 @@ impl Geometry {
     /// Which way to cut depends on the rotation: a sheet turned on its side
     /// has its two pages one above the other in the file, however they look.
     pub fn halves(self) -> [[f64; 4]; 2] {
-        let [left, bottom, right, top] = self.media;
+        let [left, bottom, right, top] = self.visible();
         let middle_x = (left + right) / 2.0;
         let middle_y = (bottom + top) / 2.0;
 
@@ -115,16 +151,30 @@ pub struct Pdf {
 
 impl Pdf {
     /// Open a file.
+    ///
+    /// Refuses a document whose page tree does not add up. Walking a tree and
+    /// finding fewer pages than the file says it has means there is a page
+    /// here this does not understand, and the one thing that must not then
+    /// happen is rewriting the tree around it — a page left out of the new
+    /// tree is a page deleted.
     pub fn open(path: &Path) -> Result<Self, Error> {
         let document = Document::load(path).map_err(|e| Error::Unreadable(e.to_string()))?;
         if document.is_encrypted() {
             return Err(Error::Encrypted);
         }
         let pages: Vec<ObjectId> = document.get_pages().into_values().collect();
-        match pages.is_empty() {
-            true => Err(Error::Empty),
-            false => Ok(Pdf { document, pages }),
+        if pages.is_empty() {
+            return Err(Error::Empty);
         }
+        if let Some(declared) = declared_pages(&document) {
+            if declared != pages.len() {
+                return Err(Error::Miscounted {
+                    declared,
+                    found: pages.len(),
+                });
+            }
+        }
+        Ok(Pdf { document, pages })
     }
 
     /// How many pages it has.
@@ -144,12 +194,19 @@ impl Pdf {
             .inherited(id, b"MediaBox")
             .and_then(|object| rectangle(&self.document, object))
             .unwrap_or(DEFAULT_BOX);
+        let crop = self
+            .inherited(id, b"CropBox")
+            .and_then(|object| rectangle(&self.document, object));
         let rotate = self
             .inherited(id, b"Rotate")
             .and_then(|object| self.document.dereference(object).ok())
             .and_then(|(_, object)| object.as_i64().ok())
             .unwrap_or(0);
-        Geometry { media, rotate }
+        Geometry {
+            media,
+            crop,
+            rotate,
+        }
     }
 
     /// An attribute of a page, from the page itself or from the tree above it.
@@ -169,35 +226,46 @@ impl Pdf {
     /// rest alone.
     ///
     /// Returns how many pages the document now has.
+    /// Cut every sheet wide enough to hold two pages into two, and leave the
+    /// rest alone.
+    ///
+    /// Returns how many pages the document now has. A document with nothing
+    /// to cut is not touched at all, so writing it back out is a copy.
     pub fn split(&mut self, min_ratio: f64) -> usize {
+        let wide: Vec<bool> = (0..self.pages.len())
+            .map(|index| self.geometry(index).ratio() > min_ratio)
+            .collect();
+        if !wide.iter().any(|wide| *wide) {
+            return self.pages.len();
+        }
+
         // Whatever a page would have inherited has to be written onto it
         // first, because the tree it would have inherited from is about to be
         // replaced by a flat one.
         self.settle_inherited();
 
         let mut order: Vec<ObjectId> = Vec::with_capacity(self.pages.len());
-        for index in 0..self.pages.len() {
+        for (index, wide) in wide.iter().enumerate() {
             let id = self.pages[index];
-            let geometry = self.geometry(index);
-            if geometry.ratio() <= min_ratio {
+            if !wide {
                 order.push(id);
                 continue;
             }
 
-            let [first, second] = geometry.halves();
-            let twin = match self.document.get_dictionary(id) {
-                Ok(dictionary) => dictionary.clone(),
-                Err(_) => {
-                    order.push(id);
-                    continue;
-                }
+            // Both halves are new objects and the original is left as it was,
+            // so a page the tree happens to list twice is cut twice rather
+            // than cut once and then measured after the fact.
+            let Ok(original) = self.document.get_dictionary(id).cloned() else {
+                order.push(id);
+                continue;
             };
-            let twin = self.document.add_object(Object::Dictionary(twin));
-
-            self.set_box(id, first);
-            self.set_box(twin, second);
-            order.push(id);
-            order.push(twin);
+            let geometry = self.geometry(index);
+            for half in geometry.halves() {
+                let mut page = original.clone();
+                narrow(&mut page, half);
+                keep_annotations_within(&self.document, &mut page, half);
+                order.push(self.document.add_object(Object::Dictionary(page)));
+            }
         }
 
         self.reparent(&order);
@@ -229,24 +297,6 @@ impl Pdf {
         }
     }
 
-    /// Point a page at half the sheet.
-    fn set_box(&mut self, page: ObjectId, rectangle: [f64; 4]) {
-        let Ok(dictionary) = self.document.get_dictionary_mut(page) else {
-            return;
-        };
-        let value: Vec<Object> = rectangle
-            .iter()
-            .map(|edge| Object::Real(*edge as f32))
-            .collect();
-        dictionary.set("MediaBox", Object::Array(value.clone()));
-        dictionary.set("CropBox", Object::Array(value));
-        // These describe a region inside the old page, and half of them now
-        // lie outside it entirely.
-        for key in NARROWING {
-            dictionary.remove(key);
-        }
-    }
-
     /// Replace the page tree with one flat node holding the pages in order.
     fn reparent(&mut self, order: &[ObjectId]) {
         let Ok(root) = self
@@ -272,6 +322,79 @@ impl Pdf {
     }
 }
 
+/// Point a page dictionary at half the sheet.
+fn narrow(page: &mut Dictionary, rectangle: [f64; 4]) {
+    let value: Vec<Object> = rectangle
+        .iter()
+        .map(|edge| Object::Real(*edge as f32))
+        .collect();
+    page.set("MediaBox", Object::Array(value.clone()));
+    page.set("CropBox", Object::Array(value));
+    // These describe a region inside the old page, and half of them now lie
+    // outside it entirely.
+    for key in NARROWING {
+        page.remove(key);
+    }
+}
+
+/// Drop the annotations that belong to the other half.
+///
+/// A link sitting on the left page would otherwise be attached to the right
+/// one as well, where a reader would show it over whatever text is now in
+/// that position.
+fn keep_annotations_within(document: &Document, page: &mut Dictionary, half: [f64; 4]) {
+    let Ok(annotations) = page.get(b"Annots").and_then(|object| {
+        document
+            .dereference(object)
+            .map(|(_, object)| object)
+            .and_then(Object::as_array)
+    }) else {
+        return;
+    };
+
+    let kept: Vec<Object> = annotations
+        .iter()
+        .filter(|annotation| {
+            document
+                .dereference(annotation)
+                .ok()
+                .and_then(|(_, object)| object.as_dict().ok())
+                .and_then(|dictionary| dictionary.get(b"Rect").ok())
+                .and_then(|object| rectangle(document, object))
+                .is_none_or(|rect| centre_within(rect, half))
+        })
+        .cloned()
+        .collect();
+
+    match kept.is_empty() {
+        true => {
+            page.remove(b"Annots");
+        }
+        false => page.set("Annots", Object::Array(kept)),
+    }
+}
+
+/// Whether one rectangle's middle sits inside another.
+///
+/// The middle rather than any overlap, because an annotation straddling the
+/// gutter has to land on one page and not both.
+fn centre_within(rect: [f64; 4], half: [f64; 4]) -> bool {
+    let (x, y) = ((rect[0] + rect[2]) / 2.0, (rect[1] + rect[3]) / 2.0);
+    (half[0]..=half[2]).contains(&x) && (half[1]..=half[3]).contains(&y)
+}
+
+/// How many pages the file says it has.
+fn declared_pages(document: &Document) -> Option<usize> {
+    let root = document
+        .catalog()
+        .and_then(|catalog| catalog.get(b"Pages"))
+        .and_then(Object::as_reference)
+        .ok()?;
+    let count = document.get_dictionary(root).ok()?.get(b"Count").ok()?;
+    let (_, count) = document.dereference(count).ok()?;
+    usize::try_from(count.as_i64().ok()?).ok()
+}
+
 /// Four numbers, in the order a PDF rectangle gives them, with the corners
 /// put back in order.
 ///
@@ -307,10 +430,64 @@ mod tests {
     fn sheet(rotate: i64) -> Geometry {
         Geometry {
             media: [0.0, 0.0, 1000.0, 700.0],
+            crop: None,
             rotate,
         }
     }
 
+    fn cropped(crop: [f64; 4]) -> Geometry {
+        Geometry {
+            media: [0.0, 0.0, 1000.0, 700.0],
+            crop: Some(crop),
+            rotate: 0,
+        }
+    }
+
+    #[test]
+    fn the_visible_page_is_the_crop_box_where_there_is_one() {
+        assert_eq!(
+            cropped([200.0, 0.0, 1000.0, 700.0]).visible(),
+            [200.0, 0.0, 1000.0, 700.0]
+        );
+        assert_eq!(sheet(0).visible(), [0.0, 0.0, 1000.0, 700.0]);
+        // And the cut goes down the middle of what is shown, not of the sheet.
+        assert_eq!(
+            cropped([200.0, 0.0, 1000.0, 700.0]).halves()[0],
+            [200.0, 0.0, 600.0, 700.0]
+        );
+    }
+
+    #[test]
+    fn a_crop_box_outside_the_sheet_is_clipped_to_it() {
+        assert_eq!(
+            cropped([-50.0, -50.0, 2000.0, 900.0]).visible(),
+            [0.0, 0.0, 1000.0, 700.0]
+        );
+    }
+
+    #[test]
+    fn a_crop_box_that_leaves_nothing_is_ignored_rather_than_obeyed() {
+        // A box outside the sheet entirely would otherwise give a page with
+        // no area, which no reader shows and nothing can measure.
+        assert_eq!(
+            cropped([2000.0, 0.0, 3000.0, 700.0]).visible(),
+            [0.0, 0.0, 1000.0, 700.0]
+        );
+        assert_eq!(
+            cropped([10.0, 10.0, 10.0, 10.0]).visible(),
+            [0.0, 0.0, 1000.0, 700.0]
+        );
+    }
+
+    #[test]
+    fn a_page_cropped_down_to_one_page_is_not_a_spread() {
+        // The shape that matters: a wide sheet whose crop box shows only one
+        // of the two pages on it. Measuring the sheet would split it again.
+        let one_page = cropped([500.0, 0.0, 1000.0, 700.0]);
+        assert_eq!(one_page.displayed(), (500.0, 700.0));
+        assert!(one_page.ratio() < crate::MIN_RATIO);
+    }
+
     #[test]
     fn an_upright_sheet_is_cut_left_then_right() {
         assert_eq!(

crates/pdfgutter/tests/split.rs

@@ -236,6 +236,123 @@ fn splitting_twice_does_not_split_twice() {
     assert_eq!(split.len(), 6);
 }
 
+/// A PDF assembled from object bodies, for the shapes the ordinary writer
+/// above cannot make.
+fn write_objects(path: &Path, objects: &[(usize, &str)], root: usize) {
+    let mut out: Vec<u8> = b"%PDF-1.4\n".to_vec();
+    let top = objects.iter().map(|(n, _)| *n).max().unwrap_or(0) + 1;
+    let mut offsets = vec![0usize; top];
+    for (number, body) in objects {
+        offsets[*number] = out.len();
+        out.extend_from_slice(format!("{number} 0 obj\n{body}\nendobj\n").as_bytes());
+    }
+    let start = out.len();
+    out.extend_from_slice(format!("xref\n0 {top}\n0000000000 65535 f \n").as_bytes());
+    for offset in offsets.iter().take(top).skip(1) {
+        out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
+    }
+    out.extend_from_slice(
+        format!("trailer\n<< /Size {top} /Root {root} 0 R >>\nstartxref\n{start}\n%%EOF\n")
+            .as_bytes(),
+    );
+    std::fs::write(path, out).expect("a writable file");
+}
+
+#[test]
+fn a_page_tree_that_does_not_add_up_is_refused_rather_than_rewritten() {
+    // The inner node has no `/Type`, so a strict walk finds one page where
+    // the file says three. Rewriting the tree around pages it could not see
+    // would delete them, and the document would look like it had always been
+    // one page long.
+    let dir = scratch("miscounted");
+    let path = dir.join("in.pdf");
+    write_objects(
+        &path,
+        &[
+            (1, "<< /Type /Catalog /Pages 2 0 R >>"),
+            (2, "<< /Type /Pages /Count 3 /Kids [4 0 R 3 0 R] >>"),
+            (3, "<< /Count 2 /Kids [6 0 R 8 0 R] /Parent 2 0 R >>"),
+            (4, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 480 720] >>"),
+            (6, "<< /Type /Page /Parent 3 0 R /MediaBox [0 0 480 720] >>"),
+            (8, "<< /Type /Page /Parent 3 0 R /MediaBox [0 0 480 720] >>"),
+        ],
+        1,
+    );
+
+    match Pdf::open(&path) {
+        Err(pdfgutter::pdf::Error::Miscounted { declared, found }) => {
+            assert_eq!((declared, found), (3, 1));
+        }
+        other => panic!("expected a refusal, got {:?}", other.map(|pdf| pdf.len())),
+    }
+}
+
+#[test]
+fn a_page_the_tree_lists_twice_is_cut_twice() {
+    // Cutting the original in place would leave the second listing measuring
+    // a page that had already been halved, and it would come through whole.
+    let dir = scratch("duplicated");
+    let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
+    write_objects(
+        &source,
+        &[
+            (1, "<< /Type /Catalog /Pages 2 0 R >>"),
+            (2, "<< /Type /Pages /Count 2 /Kids [4 0 R 4 0 R] >>"),
+            (
+                4,
+                "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1008 720] >>",
+            ),
+        ],
+        1,
+    );
+
+    let mut pdf = Pdf::open(&source).expect("a readable PDF");
+    assert_eq!(pdf.len(), 2);
+    assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 4);
+    pdf.save(&destination).expect("a written PDF");
+
+    let split = Pdf::open(&destination).expect("a readable PDF");
+    assert_eq!(split.len(), 4);
+    let widths: Vec<f64> = (0..4).map(|i| split.geometry(i).displayed().0).collect();
+    assert_eq!(widths, [504.0, 504.0, 504.0, 504.0]);
+}
+
+#[test]
+fn a_document_with_nothing_to_cut_keeps_every_page() {
+    let dir = scratch("nothing-to-cut");
+    let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
+    write(&source, &[tall(), tall(), tall(), tall(), tall()]);
+
+    let mut pdf = Pdf::open(&source).expect("a readable PDF");
+    assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 5);
+    pdf.save(&destination).expect("a written PDF");
+    assert_eq!(Pdf::open(&destination).expect("a readable PDF").len(), 5);
+}
+
+#[test]
+fn a_page_cropped_to_one_of_its_two_halves_is_not_cut_again() {
+    // What running this over its own output would meet if the crop box were
+    // ignored: a sheet whose box is still wide, showing one page.
+    let dir = scratch("cropped");
+    let path = dir.join("in.pdf");
+    write_objects(
+        &path,
+        &[
+            (1, "<< /Type /Catalog /Pages 2 0 R >>"),
+            (2, "<< /Type /Pages /Count 1 /Kids [4 0 R] >>"),
+            (
+                4,
+                "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1008 720] /CropBox [504 0 1008 720] >>",
+            ),
+        ],
+        1,
+    );
+
+    let mut pdf = Pdf::open(&path).expect("a readable PDF");
+    assert_eq!(pdf.geometry(0).displayed(), (504.0, 720.0));
+    assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 1);
+}
+
 #[test]
 fn what_is_not_a_pdf_is_refused_rather_than_guessed_at() {
     let dir = scratch("refused");

parity/KNOWN.md

@@ -217,13 +217,62 @@ pixel, and a parity test that demanded it would be a test of pdfium rather
 than of this.
 
 What is compared instead is everything that decides: the aspect ratio, which
-pages get sampled, whether a given ink profile has a gutter, and the verdict
-a set of ratios and gutters adds up to. Both sides are handed the same
-profiles and have to agree exactly.
+pages get sampled, the reduction from pixels to a column profile, whether a
+given profile has a gutter, and the verdict a set of ratios and gutters adds
+up to. Both sides are handed the same pixels and the same profiles, and have
+to agree exactly.
 
-**Measured:** 72 cases, including profiles either side of the ink threshold by
-one ten-thousandth and profiles too narrow to hold the three bands the test
-looks at.
+The two renderers do have to be pointed at the same rectangle, which is what
+`-cropbox` is for — see the next note.
+
+**Measured:** 94 cases, including profiles either side of the ink threshold by
+one ten-thousandth, profiles too narrow to hold the three bands the test looks
+at, and greymaps whose heights do not divide evenly so that both sides have to
+round a fraction the same way.
+
+## pdfgutter — the page is the crop box, consistently
+
+**Python:** takes the aspect ratio from the media box and the ink profile
+from `page.render()`, which draws the **crop box**. On a page that carries
+both — most scanned books do — those are two different rectangles, so the
+shape is measured on one and the ink on the other. The bands the gutter is
+looked for in then fall somewhere other than the middle of what was
+rendered, and a real gutter is missed.
+
+**Rust:** the crop box is what a reader shows, so it is what everything
+measures: the ratio, the render (`pdftoppm -cropbox`), and where the cut
+goes. A page with no crop box is its media box, which is what the spec says
+and what most files are.
+
+**Why this is acceptable:** the original is not internally consistent, and on
+a wide sheet cropped to show only one of its two pages it gets the answer
+wrong in the destructive direction — the sheet still measures wide, so it is
+split again, down the middle of a page rather than between two. Measuring one
+rectangle throughout is the only version of this that is right.
+
+**Measured:** a wide sheet cropped to one page is no longer a split
+candidate, and a cropped spread's gutter lands where it is looked for. The
+arithmetic that both sides share is compared exactly, as above.
+
+## pdfgutter — a page tree that does not add up is refused
+
+**Python:** pdfium enumerates the pages and the question never comes up.
+
+**Rust:** the page tree is walked directly, and a walk is stricter than a
+renderer: a node missing its `/Type`, a `/Type` written as a reference, a kid
+that is a dictionary rather than a reference to one — each of those is a page
+a strict walk does not see and a reader does. So the number of pages found is
+checked against the number the file says it has, and a document where they
+disagree is refused with exit 2.
+
+**Why this is acceptable:** splitting replaces the page tree, and a page left
+out of the new tree is a page deleted. A file this does not fully understand
+is one it must not rewrite. Refusing costs a document that could perhaps have
+been handled; not refusing costs pages, silently, in the copy you keep.
+
+**Measured:** five shapes that a strict walk under-counts, each refused rather
+than written short. A document with nothing wide enough to cut is not
+rewritten at all, so every page comes through however the tree is built.
 
 ## pdfgutter — the thresholds are arguments, not environment variables
 

parity/golden/pdfgutter/cases.ndjson

@@ -70,3 +70,25 @@
 {"gutter_fraction": 0.6666666666666666, "kind": "verdict", "median_ratio": 1.4, "name": "two-of-three", "two_up": true}
 {"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.1501, "name": "just-over-the-ratio", "two_up": true}
 {"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.15, "name": "exactly-the-ratio", "two_up": false}
+{"kind": "greymap", "name": "black-and-white-columns", "profile": [1.0, 0.0, 1.0, 0.0]}
+{"kind": "greymap", "name": "either-side-of-the-threshold", "profile": [1.0, 1.0, 0.0]}
+{"kind": "greymap", "name": "the-threshold-itself-everywhere", "profile": [0.0, 0.0, 0.0, 0.0, 0.0]}
+{"kind": "greymap", "name": "one-below-the-threshold-everywhere", "profile": [1.0, 1.0, 1.0, 1.0, 1.0]}
+{"kind": "greymap", "name": "a-page-with-a-gutter", "profile": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.35, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}
+{"kind": "greymap", "name": "a-single-row", "profile": [1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0]}
+{"kind": "greymap", "name": "a-single-column", "profile": [0.5555555555555556]}
+{"kind": "greymap", "name": "awkward-7x3", "profile": [1.0, 1.0, 1.0, 1.0, 0.6666666666666666, 0.0, 0.0]}
+{"kind": "greymap", "name": "awkward-11x7", "profile": [1.0, 1.0, 1.0, 0.7142857142857143, 0.2857142857142857, 0.0, 0.42857142857142855, 1.0, 1.0, 1.0, 0.7142857142857143]}
+{"kind": "greymap", "name": "awkward-13x13", "profile": [1.0, 0.9230769230769231, 0.6153846153846154, 0.38461538461538464, 0.38461538461538464, 0.46153846153846156, 0.6923076923076923, 1.0, 0.8461538461538461, 0.6153846153846154, 0.38461538461538464, 0.3076923076923077, 0.46153846153846156]}
+{"kind": "greymap", "name": "awkward-3x101", "profile": [0.6534653465346535, 0.6534653465346535, 0.6534653465346535]}
+{"kind": "greymap", "name": "awkward-101x3", "profile": [1.0, 1.0, 1.0, 1.0, 0.6666666666666666, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.3333333333333333, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.3333333333333333, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.3333333333333333, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.3333333333333333, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.3333333333333333, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.3333333333333333, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.3333333333333333, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.6666666666666666, 1.0, 1.0, 1.0, 0.6666666666666666, 0.0, 0.0, 0.6666666666666666, 1.0, 1.0, 1.0, 0.6666666666666666, 0.0, 0.0, 0.6666666666666666, 1.0, 1.0, 1.0, 0.6666666666666666, 0.0, 0.0, 0.6666666666666666, 1.0, 1.0, 1.0, 0.6666666666666666, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.3333333333333333, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]}
+{"kind": "greymap", "name": "random-grey-0", "profile": [0.5, 0.5]}
+{"kind": "greymap", "name": "random-grey-1", "profile": [0.625, 0.640625, 0.6875, 0.609375, 0.65625]}
+{"kind": "greymap", "name": "random-grey-2", "profile": [0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 1.0, 0.0, 0.6666666666666666, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 0.0, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 0.0, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 1.0, 0.6666666666666666, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 1.0, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 1.0, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 1.0, 0.6666666666666666, 1.0, 1.0, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 0.3333333333333333, 0.0, 0.6666666666666666, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666]}
+{"kind": "greymap", "name": "random-grey-3", "profile": [0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 1.0, 1.0, 0.3333333333333333, 1.0, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 1.0, 0.6666666666666666, 0.6666666666666666, 1.0, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 1.0, 0.6666666666666666, 0.0, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 1.0, 1.0, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 0.3333333333333333, 1.0, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.6666666666666666, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.6666666666666666, 0.6666666666666666, 1.0, 0.3333333333333333, 0.3333333333333333, 1.0, 0.6666666666666666, 0.6666666666666666, 0.6666666666666666, 1.0, 1.0, 0.6666666666666666, 0.6666666666666666]}
+{"kind": "greymap", "name": "random-grey-4", "profile": [0.0, 0.5]}
+{"kind": "greymap", "name": "random-grey-5", "profile": [0.4, 1.0, 0.6, 0.8, 0.4, 0.8, 0.4, 0.6]}
+{"kind": "greymap", "name": "random-grey-6", "profile": [0.59375, 0.546875, 0.625, 0.5, 0.703125]}
+{"kind": "greymap", "name": "random-grey-7", "profile": [0.671875, 0.671875, 0.609375, 0.625, 0.671875]}
+{"kind": "greymap", "name": "random-grey-8", "profile": [0.8, 0.8, 0.8, 0.8, 0.4, 0.6, 0.6, 0.8]}
+{"kind": "greymap", "name": "random-grey-9", "profile": [0.671875, 0.671875, 0.640625, 0.484375, 0.703125]}

parity/oracle/pdfgutter.py

@@ -21,12 +21,19 @@ import numpy as np
 _MIN_RATIO = 1.15
 _MIN_GUTTER_FRAC = 0.6
 _TEXT_COL_INK = 0.02
+_INK_THRESHOLD = 160
 
 
 def _ratio(width: float, height: float) -> float:
     return width / height if height else 0.0
 
 
+def _col_ink_profile(pixels: list[int], width: int, height: int) -> list[float]:
+    """The reduction the renderer's output went through, minus the render."""
+    arr = np.asarray(pixels, dtype=np.uint8).reshape(height, width)
+    return [float(x) for x in (arr < _INK_THRESHOLD).mean(axis=0)]
+
+
 def _has_gutter(ink: np.ndarray) -> bool:
     w = len(ink)
     center = ink[int(0.46 * w):int(0.54 * w)]
@@ -67,6 +74,10 @@ def answer(case: dict) -> dict:
         return {"gutter": _has_gutter(np.asarray(case["ink"], dtype=float))}
     if kind == "sample":
         return {"indices": _sample_indices(case["pages"])}
+    if kind == "greymap":
+        return {
+            "profile": _col_ink_profile(case["pixels"], case["width"], case["height"])
+        }
     if kind == "verdict":
         return _verdict(case["ratios"], case["gutters"])
     raise SystemExit(f"no such case: {kind}")

parity/tests/pdfgutter.rs

@@ -76,6 +76,28 @@ fn every_answer_matches_python() {
                     note("indices", format!("{ours:?}"), &theirs["indices"]);
                 }
             }
+            "greymap" => {
+                let width = case["width"].as_u64().expect("a width") as usize;
+                let height = case["height"].as_u64().expect("a height") as usize;
+                let pixels: Vec<u8> = case["pixels"]
+                    .as_array()
+                    .expect("an array")
+                    .iter()
+                    .map(|value| value.as_u64().expect("a number") as u8)
+                    .collect();
+
+                // The renderer is not the thing under test; the reduction it
+                // feeds is. Both sides are handed the same pixels.
+                let mut greymap = format!("P5\n{width} {height}\n255\n").into_bytes();
+                greymap.extend_from_slice(&pixels);
+                let ours = pdfgutter::ink::profile(&greymap, pdfgutter::INK_THRESHOLD)
+                    .expect("a readable greymap");
+
+                let theirs_profile = floats(&theirs["profile"]);
+                if ours != theirs_profile {
+                    note("profile", format!("{ours:?}"), &theirs["profile"]);
+                }
+            }
             "verdict" => {
                 let ratios = floats(&case["ratios"]);
                 let gutters: Vec<bool> = case["gutters"]
@@ -124,7 +146,7 @@ fn every_answer_matches_python() {
 #[test]
 fn every_kind_of_case_is_present() {
     let cases = cases();
-    for kind in ["ratio", "gutter", "sample", "verdict"] {
+    for kind in ["ratio", "gutter", "sample", "verdict", "greymap"] {
         assert!(
             cases.iter().any(|case| case["kind"] == kind),
             "no {kind} case"