a missing word, a hidden dependency, and a second pass that changed things

crates/mdscrub/Cargo.toml

@@ -11,6 +11,7 @@ repository.workspace = true
 catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
+fancy-regex.workspace = true
 regex.workspace = true
 textkit.workspace = true
 

crates/mdscrub/src/heading.rs

@@ -8,8 +8,9 @@
 
 use std::sync::LazyLock;
 
+use fancy_regex::Regex as Boundaried;
 use regex::Regex;
-use textkit::pattern::SPACE;
+use textkit::pattern::{SPACE, WORD_END, WORD_START};
 
 fn pattern(source: &str) -> Regex {
     Regex::new(source).expect("a valid pattern")
@@ -36,10 +37,11 @@ static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(&format!(r"^(#{{1,6}}
 
 /// A page anchor: `<span id="…page42">`, `<a id="page-42">`, and the rest of
 /// the shapes an EPUB uses to say where a printed page began.
-pub static PAGE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
-    pattern(&format!(
-        r#"(?i)<(?:span|a|div)\b[^>]*\bid{SPACE}*={SPACE}*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#
+pub static PAGE_ANCHOR: LazyLock<Boundaried> = LazyLock::new(|| {
+    Boundaried::new(&format!(
+        r#"(?i)<(?:span|a|div){WORD_END}[^>]*{WORD_START}id{SPACE}*={SPACE}*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#
     ))
+    .expect("a valid pattern")
 });
 
 /// Clean one heading's text.
@@ -71,6 +73,7 @@ pub fn clean_all(markdown: &str) -> String {
         let raw = &found[2];
         let pages: Vec<u32> = PAGE_ANCHOR
             .captures_iter(raw)
+            .filter_map(Result::ok)
             .map(|anchor| textkit::decimal(&anchor[1]))
             .collect();
         let cleaned = clean(&PAGE_ANCHOR.replace_all(raw, ""));
@@ -86,10 +89,15 @@ pub fn clean_all(markdown: &str) -> String {
 }
 
 /// Turn the page anchors left in the body into page markers.
+///
+/// The number is carried through exactly as the document wrote it, padding
+/// and all: `page007` becomes page 007. A heading's anchor is read as a
+/// number instead, because a heading's page is a fact about the document
+/// rather than a piece of its text.
 pub fn anchors_to_markers(markdown: &str) -> String {
     PAGE_ANCHOR
-        .replace_all(markdown, |found: &regex::Captures| {
-            format!("\n\n<!-- Page {} -->\n\n", textkit::decimal(&found[1]))
+        .replace_all(markdown, |found: &fancy_regex::Captures| {
+            format!("\n\n<!-- Page {} -->\n\n", &found[1])
         })
         .into_owned()
 }
@@ -141,6 +149,12 @@ mod tests {
         assert!(cleaned.contains("<!-- Page 19 -->"));
     }
 
+    #[test]
+    fn a_padded_page_number_in_the_body_keeps_its_padding() {
+        let cleaned = anchors_to_markers("text <span id=\"page007\"></span> more");
+        assert!(cleaned.contains("<!-- Page 007 -->"), "{cleaned}");
+    }
+
     #[test]
     fn a_line_that_is_not_a_heading_is_untouched() {
         assert_eq!(

crates/mdscrub/src/lib.rs

@@ -20,8 +20,7 @@ pub mod unescape;
 ///
 /// Safe to run twice: the second run changes nothing.
 pub fn scrub(markdown: &str) -> String {
-    let text = pages::normalize(markdown);
-    let text = heading::clean_all(&text);
+    let text = heading::clean_all(markdown);
     let text = heading::anchors_to_markers(&text);
     let text = html::strip_tags(&text);
     let text = html::scrub_control(&text);
@@ -62,10 +61,27 @@ mod tests {
     }
 
     #[test]
-    fn form_feeds_become_pages_rather_than_disappearing() {
-        // The control-character pass would delete a form feed, so page
-        // normalisation has to happen first or the boundary is lost.
-        let cleaned = scrub("one\u{c}two");
-        assert!(cleaned.contains("<!-- Page 2 -->"), "{cleaned:?}");
+    fn page_normalisation_is_not_part_of_cleaning() {
+        // A form feed in ordinary markdown is a control character, not a page
+        // boundary. It is one only to a program that knows it just read a PDF.
+        assert_eq!(scrub("one\u{c}two"), "onetwo\n");
+    }
+
+    #[test]
+    fn pagination_first_then_cleaning_is_still_idempotent() {
+        let raw = "one\u{c}<span>two</span>";
+        let once = scrub(&pages::normalize(raw));
+        assert!(once.contains("<!-- Page 2 -->"), "{once:?}");
+        assert_eq!(scrub(&pages::normalize(&once)), once);
+    }
+
+    #[test]
+    fn nothing_manufactures_a_page_boundary_on_a_second_run() {
+        // Stripping a tag can leave a separator flush against a newline. If
+        // pagination ran after that, a second pass would find a boundary the
+        // first pass did not.
+        let raw = "{2}------------------------------<span></span>\nafter\n";
+        let once = scrub(raw);
+        assert_eq!(scrub(&once), once, "{once:?}");
     }
 }

crates/mdscrub/src/main.rs

@@ -10,6 +10,10 @@ use clap::Parser;
 ///
 /// Reads markdown on stdin, or from a path, and writes it to stdout. Safe to
 /// run twice.
+///
+/// Page boundaries are left alone unless `--pages` asks for them, because
+/// only the program that just read a PDF knows that a form feed in the text
+/// is a page break rather than a stray control character.
 #[derive(Parser)]
 #[command(
     version,
@@ -22,6 +26,12 @@ Example:
 struct Args {
     /// Markdown file to read. Reads stdin when absent.
     path: Option<PathBuf>,
+
+    /// First turn page separators and form feeds into page markers. For text
+    /// straight out of a PDF or DjVu extractor, which marks its page
+    /// boundaries that way.
+    #[arg(long)]
+    pages: bool,
 }
 
 fn main() -> ExitCode {
@@ -38,5 +48,9 @@ fn main() -> ExitCode {
         }
     };
 
+    let markdown = match args.pages {
+        true => mdscrub::pages::normalize(&markdown),
+        false => markdown,
+    };
     catena_cli::text("mdscrub", &mdscrub::scrub(&markdown))
 }