ported patterns mean in rust what they mean in python

Cargo.lock

@@ -194,6 +194,7 @@ dependencies = [
  "clap",
  "corpus-core",
  "cuekit",
+ "fancy-regex 0.16.2",
  "regex",
  "schemars",
  "serde",
@@ -474,6 +475,7 @@ dependencies = [
 name = "textkit"
 version = "0.1.0"
 dependencies = [
+ "regex",
  "tiktoken-rs",
 ]
 

crates/chunkmd/src/footnote.rs

@@ -8,7 +8,7 @@
 use std::sync::LazyLock;
 
 use fancy_regex::Regex as FancyRegex;
-use regex::Regex;
+use textkit::pattern::{SPACE, WORD_END, WORD_START};
 
 /// A private-use codepoint marking the start of a note's text.
 ///
@@ -40,47 +40,50 @@ const MIN_CITED_NOTES: usize = 2;
 /// more spaces — several notes can share one physical line — so that a
 /// superscript *reference* to a note, mid-sentence, does not qualify.
 static NOTE_MARKER: LazyLock<FancyRegex> = LazyLock::new(|| {
-    FancyRegex::new(
+    FancyRegex::new(&format!(
         r"(?xm)
-    (?: ^[ \t]{0,8} (?:-[ \t]+)?      # line start, optional '- ' list bullet
+    (?: ^[ \t]{{0,8}} (?:-[ \t]+)?      # line start, optional '- ' list bullet
-      | (?<=\s\s)                     # or after a 2+ space run
+      | (?<={SPACE}{SPACE})            # or after a 2+ space run
       | (?<=\ -\ )                    # or after ' - ' (inline list separator)
     )
     (?:
-        \d{1,3} \\?[.)] [ \t]+ \S                 # plain '12. ' or escaped '12\. '
+        \d{{1,3}} \\?[.)] [ \t]+ \S               # plain '12. ' or escaped '12\. '
-      | \[ \d{1,3} \] [ \t]* \S                   # bracket-numbered '[12] Author'
+      | \[ \d{{1,3}} \] [ \t]* \S                 # bracket-numbered '[12] Author'
-      | \d{1,3} [ \t]+ \S                         # bare '12 ' from a page bottom
+      | \d{{1,3}} [ \t]+ \S                       # bare '12 ' from a page bottom
-      | [¹²³⁰-⁹]+ \s* \S                          # unicode superscript marker
+      | [¹²³⁰-⁹]+ {SPACE}* \S                     # unicode superscript marker
-      | (?:fn|ftn|note) \d{1,4} [.)] [ \t]+ \S    # id-style 'fn39. '
+      | (?:fn|ftn|note) \d{{1,4}} [.)] [ \t]+ \S  # id-style 'fn39. '
-      | (?:<span[^>]*>\s*)*                       # outer span wrap
+      | (?:<span[^>]*>{SPACE}*)*                  # outer span wrap
-        <a\b[^>]*> \s* (?:<span[^>]*>\s*)* (?:<sup>\s*)?
+        <a{WORD_END}[^>]*> {SPACE}* (?:<span[^>]*>{SPACE}*)* (?:<sup>{SPACE}*)?
-            \d{1,3} \s* [.)]?
+            \d{{1,3}} {SPACE}* [.)]?
-        \s* (?:</sup>\s*)? (?:</span>\s*)* </a>
+        {SPACE}* (?:</sup>{SPACE}*)? (?:</span>{SPACE}*)* </a>
-        (?:\s*</span>)* \s* [.)]?                 # anchor forms
+        (?:{SPACE}*</span>)* {SPACE}* [.)]?        # anchor forms
-      | \[ \s* (?:<span[^>]*>)? \s* \d{1,3}
+      | \[ {SPACE}* (?:<span[^>]*>)? {SPACE}* \d{{1,3}}
-        \s* (?:</span>)? \s* \] \( \#[^)]* \)     # [42](#...)
+        {SPACE}* (?:</span>)? {SPACE}* \] \( \#[^)]* \)  # [42](#...)
-      | <sup> \s* \d{1,3} \s* </sup>              # bare sup note
+      | <sup> {SPACE}* \d{{1,3}} {SPACE}* </sup>   # bare sup note
-    )",
+    )"
-    )
+    ))
     .expect("a valid pattern")
 });
 
 /// What makes a note bibliographic rather than discursive.
-static CITE_SIGNAL: LazyLock<Regex> = LazyLock::new(|| {
+static CITE_SIGNAL: LazyLock<FancyRegex> = LazyLock::new(|| {
-    Regex::new(
+    FancyRegex::new(&format!(
         r"(?ix)
-    \((?:[^()]*\b)?(?:1[5-9]|20)\d{2}\)   # parenthesised year: (Oxford, 1998)
+    \((?:[^()]*{WORD_END})?(?:1[5-9]|20)\d{{2}}\)   # parenthesised year: (Oxford, 1998)
-  | \bpp?\.\s*\d                          # p. 12 / pp. 12-34
+  | {WORD_START}pp?\.{SPACE}*\d                      # p. 12 / pp. 12-34
-  | \bvol\.\s*\d                          # vol. 2
+  | {WORD_START}vol\.{SPACE}*\d                      # vol. 2
-  | \bIbid\b | \bop\.\s*cit\b | \bloc\.\s*cit\b
+  | {WORD_START}Ibid{WORD_END}
-  | \bet\s+al\b
+  | {WORD_START}op\.{SPACE}*cit{WORD_END}
-  | \b(?:University|Clarendon|Academic)\s+Press\b | \bPress,\s
+  | {WORD_START}loc\.{SPACE}*cit{WORD_END}
-  | \beds?\.\s | \btrans\.\s
+  | {WORD_START}et{SPACE}+al{WORD_END}
-  | \*[^*\n]{3,80}\*\s*,?\s*\d            # short form: *Title*, 183
+  | {WORD_START}(?:University|Clarendon|Academic){SPACE}+Press{WORD_END}
-  | <span[^>]*italic[^>]*>                # HTML-italic title survives conversion
+  | {WORD_START}Press,{SPACE}
-  | ,\s+\d{1,4}\s*[.;]                    # trailing page: '..., 183.'
+  | {WORD_START}eds?\.{SPACE} | {WORD_START}trans\.{SPACE}
-  | https?://                             # web citations (URLs, DOIs)",
+  | \*[^*\n]{{3,80}}\*{SPACE}*,?{SPACE}*\d         # short form: *Title*, 183
-    )
+  | <span[^>]*italic[^>]*>                          # HTML-italic title survives conversion
+  | ,{SPACE}+\d{{1,4}}{SPACE}*[.;]                   # trailing page: '..., 183.'
+  | https?://                                       # web citations (URLs, DOIs)"
+    ))
     .expect("a valid pattern")
 });
 
@@ -107,7 +110,7 @@ pub fn looks_like_citations(text: &str) -> bool {
     for (at, start) in starts.iter().enumerate() {
         let end = starts.get(at + 1).copied().unwrap_or(text.len());
         let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect();
-        if CITE_SIGNAL.is_match(&entry) {
+        if CITE_SIGNAL.is_match(&entry).unwrap_or(false) {
             cited += 1;
             if cited >= MIN_CITED_NOTES {
                 return true;

crates/chunkmd/src/lib.rs

@@ -28,23 +28,24 @@ pub use pack::Budgets;
 pub fn chunk(markdown: &str, budgets: &Budgets) -> Vec<Chunk> {
     let pages = pages::split_by_page(markdown);
     let chapters = pages::chapters(markdown);
-    let by_page = pages::page_chapters(&chapters);
     let by_offset = pages::ChapterIndex::new(&chapters);
 
     let mut chunks: Vec<Chunk> = Vec::new();
     for page in pages {
         for packed in pack::page(page.text, budgets) {
-            for piece in pack::enforce_hard_max(&packed, budgets.hard_max) {
+            for (piece, tokens) in pack::enforce_hard_max(&packed, budgets.hard_max) {
-                let tokens = textkit::count(&piece);
                 if tokens < budgets.min_tokens {
                     continue;
                 }
                 // A page-based chapter when the document has page markers;
                 // the byte offset otherwise, which is what carries EPUBs
-                // that have headings but no page anchors.
+                // that have headings but no page anchors. Page zero counts as
+                // no page at all, which is what the Python's truth test makes
+                // of it.
                 let chapter = page
                     .number
-                    .and_then(|number| by_page.get(&number).copied())
+                    .filter(|number| *number != 0)
+                    .and_then(|number| pages::chapter_of_page(&chapters, number))
                     .or_else(|| by_offset.at(page.offset));
 
                 let idx = chunks.len() as u32;

crates/chunkmd/src/main.rs

@@ -18,7 +18,6 @@ use corpus_core::{exit, ndjson};
 #[command(
     version,
     about,
-    long_about = None,
     after_help = "\
 Example:
 
@@ -63,9 +62,9 @@ fn main() -> ExitCode {
 
     let markdown = match read(args.path.as_deref()) {
         Ok(markdown) => markdown,
-        Err(e) => {
+        Err((code, why)) => {
-            eprintln!("chunkmd: {e}");
+            eprintln!("chunkmd: {why}");
-            return ExitCode::from(exit::IO);
+            return ExitCode::from(code);
         }
     };
 
@@ -118,13 +117,21 @@ fn sane(budgets: &Budgets) -> Result<(), String> {
     Ok(())
 }
 
-fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
+/// Read the input, distinguishing a file that cannot be read from one that
-    match path {
+/// is not text: the first is an I/O failure and the second is bad input, and
-        Some(path) => std::fs::read_to_string(path),
+/// a caller routing on the exit code needs to tell them apart.
-        None => {
+fn read(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
-            let mut markdown = String::new();
+    let bytes = match path {
-            std::io::stdin().read_to_string(&mut markdown)?;
+        Some(path) => {
-            Ok(markdown)
+            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
         }
-    }
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin()
+                .read_to_end(&mut bytes)
+                .map(|_| bytes)
+                .map_err(|e| (exit::IO, e.to_string()))
+        }
+    }?;
+    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
 }

crates/chunkmd/src/pack.rs

@@ -100,12 +100,25 @@ fn carry_tail(buf: &[String], overlap: usize) -> Vec<String> {
     out
 }
 
-/// Split anything still over the hard maximum, on token boundaries.
+/// Split anything still over the hard maximum, on token boundaries, and hand
+/// back each piece with its token count.
 ///
 /// Defence in depth. The structural passes above should have handled it; this
-/// is what guarantees the promise regardless.
+/// is what guarantees the promise regardless. The counts come back with the
-pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<String> {
+/// pieces because they have just been computed, and counting them again is
+/// the second-largest cost in the whole chunker.
+pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<(String, usize)> {
+    let tokens = textkit::count(text);
+    if tokens <= hard_max {
+        return vec![(text.to_string(), tokens)];
+    }
     textkit::split_at_token_limit(text, hard_max)
+        .into_iter()
+        .map(|piece| {
+            let tokens = textkit::count(&piece);
+            (piece, tokens)
+        })
+        .collect()
 }
 
 #[cfg(test)]
@@ -179,8 +192,9 @@ mod tests {
     #[test]
     fn nothing_leaves_the_packer_over_the_hard_maximum() {
         let long = "word ".repeat(2000);
-        for piece in enforce_hard_max(&long, 100) {
+        for (piece, tokens) in enforce_hard_max(&long, 100) {
-            assert!(textkit::count(&piece) <= 100);
+            assert!(tokens <= 100);
+            assert_eq!(tokens, textkit::count(&piece));
         }
     }
 }

crates/chunkmd/src/pages.rs

@@ -4,16 +4,18 @@
 //! and chapter boundaries as headings. Both are needed on every chunk: a
 //! passage a reader cannot locate in the book is a passage they cannot check.
 
-use std::collections::HashMap;
 use std::sync::LazyLock;
 
 use regex::Regex;
+use textkit::pattern::SPACE;
 
-static PAGE: LazyLock<Regex> =
+static PAGE: LazyLock<Regex> = LazyLock::new(|| {
-    LazyLock::new(|| Regex::new(r"<!--\s*Page\s+(\d+)\s*-->").expect("a valid pattern"));
+    Regex::new(&format!(r"<!--{SPACE}*Page{SPACE}+(\d+){SPACE}*-->")).expect("a valid pattern")
+});
 
-static HEADING: LazyLock<Regex> =
+static HEADING: LazyLock<Regex> = LazyLock::new(|| {
-    LazyLock::new(|| Regex::new(r"(?m)^(#{1,3})\s+(.+)$").expect("a valid pattern"));
+    Regex::new(&format!(r"(?m)^(#{{1,3}}){SPACE}+(.+)$")).expect("a valid pattern")
+});
 
 /// One page's worth of text, and where it started in the source.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -166,31 +168,33 @@ impl<'a> ChapterIndex<'a> {
     }
 }
 
-/// Map each page number to the chapter covering it.
+/// The chapter covering a page, if any.
 ///
 /// Where chapters overlap — which happens when two headings land on one page
 /// — the later one wins, because that is the one a reader scrolling down
 /// would be under.
-pub fn page_chapters(chapters: &[Chapter]) -> HashMap<u32, &str> {
+///
-    let mut out = HashMap::new();
+/// Scanned rather than indexed into a map of every page. A document has tens
-    for chapter in chapters {
+/// of chapters and a scan of them costs nothing, where a map costs one entry
-        let Some(start) = chapter.start_page else {
+/// per page in every chapter's span: a single mistyped marker reading
-            continue;
+/// `<!-- Page 20231115 -->` would ask for a gigabyte of them.
-        };
+pub fn chapter_of_page(chapters: &[Chapter], page: u32) -> Option<&str> {
-        // A zero end page means the same thing as no end page: this chapter
+    chapters
-        // is one page long as far as anyone can tell.
+        .iter()
-        let end = match chapter.end_page {
+        .filter(|chapter| {
-            Some(0) | None => start,
+            let Some(start) = chapter.start_page else {
-            Some(end) => end,
+                return false;
-        };
+            };
-        if end < start {
+            // A zero end page means the same thing as no end page: this
-            continue;
+            // chapter is one page long as far as anyone can tell.
-        }
+            let end = match chapter.end_page {
-        for page in start..=end {
+                Some(0) | None => start,
-            out.insert(page, chapter.title.as_str());
+                Some(end) => end,
-        }
+            };
-    }
+            (start..=end.max(start)).contains(&page) && end >= start
-    out
+        })
+        .next_back()
+        .map(|chapter| chapter.title.as_str())
 }
 
 fn trim(text: &str) -> &str {
@@ -264,10 +268,10 @@ mod tests {
         assert_eq!(found[0].start_page, Some(1));
         assert_eq!(found[0].end_page, Some(4));
         assert_eq!(found[1].start_page, Some(4));
-        let by_page = page_chapters(&found);
+        assert_eq!(chapter_of_page(&found, 1), Some("One"));
-        assert_eq!(by_page.get(&1), Some(&"One"));
+        assert_eq!(chapter_of_page(&found, 3), Some("One"));
-        assert_eq!(by_page.get(&3), Some(&"One"));
+        assert_eq!(chapter_of_page(&found, 4), Some("Two"));
-        assert_eq!(by_page.get(&4), Some(&"Two"));
+        assert_eq!(chapter_of_page(&found, 9), None);
     }
 
     #[test]

crates/chunkmd/src/role.rs

@@ -14,28 +14,46 @@
 use std::sync::LazyLock;
 
 use corpus_core::Role;
+use fancy_regex::Regex as Boundaried;
 use regex::Regex;
+use textkit::pattern::{SPACE, WORD, WORD_END, WORD_START};
 
 fn pattern(source: &str) -> Regex {
     Regex::new(source).expect("a valid pattern")
 }
 
+/// A pattern that needs Python's word boundary, which is a lookaround here
+/// because Rust's own `\b` counts combining marks as word characters.
+fn bounded(source: &str) -> Boundaried {
+    Boundaried::new(source).expect("a valid pattern")
+}
+
 /// Chapter titles that give the section away, tried in this order.
 static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
     vec![
         (
             Role::Bibliography,
-            pattern(
+            pattern(&format!(
-                r"(?i)(bibliograph|works\s+cited|literature\s+cited|further\s+reading|^\s*references\s*$|^\s*(end)?notes\s*$)",
+                r"(?i)(bibliograph|works{SPACE}+cited|literature{SPACE}+cited|further{SPACE}+reading|^{SPACE}*references{SPACE}*$|^{SPACE}*(end)?notes{SPACE}*$)"
-            ),
+            )),
+        ),
+        (
+            Role::Index,
+            pattern(&format!(
+                r"(?i)^{SPACE}*index({SPACE}+of{SPACE}+{WORD}+)?{SPACE}*$"
+            )),
+        ),
+        (
+            Role::Toc,
+            pattern(&format!(
+                r"(?i)^{SPACE}*(table{SPACE}+of{SPACE}+)?contents{SPACE}*$"
+            )),
         ),
-        (Role::Index, pattern(r"(?i)^\s*index(\s+of\s+\w+)?\s*$")),
-        (Role::Toc, pattern(r"(?i)^\s*(table\s+of\s+)?contents\s*$")),
         (
             Role::Frontmatter,
-            pattern(
+            pattern(&format!(
-                r"(?i)(copyright|title\s*page|colophon|^\s*dedication\s*$|^\s*epigraph\s*$|about\s+the\s+(author|publisher)|acknowledg)",
+                r"(?i)(copyright|title{SPACE}*page|colophon|^{SPACE}*dedication{SPACE}*$|^{SPACE}*epigraph{SPACE}*$|about{SPACE}+the{SPACE}+(author|publisher)|acknowledg)"
-            ),
+            )),
         ),
     ]
 });
@@ -43,29 +61,40 @@ static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
 /// A standalone bibliography heading inside the chunk's own text —
 /// "Bibliography", "## Works Cited", "Selected Bibliography".
 static INTEXT_BIBLIO_HEADING: LazyLock<Regex> = LazyLock::new(|| {
-    pattern(
+    pattern(&format!(
-        r"(?im)^\s*#{0,6}\s*(select(?:ed)?\s+)?(bibliography|works\s+cited|references|sources)\s*$",
+        r"(?im)^{SPACE}*#{{0,6}}{SPACE}*(select(?:ed)?{SPACE}+)?(bibliography|works{SPACE}+cited|references|sources){SPACE}*$"
-    )
+    ))
 });
 
 /// One bibliography entry: "Lastname, First…", or the dash run that stands
 /// for "the same author again".
-static BIBLIO_ENTRY: LazyLock<Regex> =
+static BIBLIO_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
-    LazyLock::new(|| pattern(r"^(?:[A-Z][\w'’\-]+,\s+\S|[—–-]{2,3}\s*[.,])"));
+    pattern(&format!(
+        r"^(?:[A-Z][\p{{L}}\p{{N}}_'’\-]+,{SPACE}+\S|[—–-]{{2,3}}{SPACE}*[.,])"
+    ))
+});
 
 /// The continuation of a wrapped entry: "New York: Springer, 2013."
-static BIBLIO_CONTINUATION: LazyLock<Regex> =
+static BIBLIO_CONTINUATION: LazyLock<Boundaried> = LazyLock::new(|| {
-    LazyLock::new(|| pattern(r"\b(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.)\b"));
+    bounded(&format!(
+        r"{WORD_START}(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.){WORD_END}"
+    ))
+});
 
-static YEAR: LazyLock<Regex> = LazyLock::new(|| pattern(r"\b(1[5-9]\d{2}|20\d{2})\b"));
+static YEAR: LazyLock<Boundaried> =
+    LazyLock::new(|| bounded(&format!(r"{WORD_START}(1[5-9]\d{{2}}|20\d{{2}}){WORD_END}")));
 
 /// An index line: a short lead, then a list of page numbers.
-static INDEX_LINE: LazyLock<Regex> =
+static INDEX_LINE: LazyLock<Regex> = LazyLock::new(|| {
-    LazyLock::new(|| pattern(r"^.{1,70}?,\s+[ivxlc\d]{1,4}(?:[,–-]\s*[ivxlc\d]{1,4})*\s*$"));
+    pattern(&format!(
+        r"^.{{1,70}}?,{SPACE}+[ivxlc\d]{{1,4}}(?:[,–-]{SPACE}*[ivxlc\d]{{1,4}})*{SPACE}*$"
+    ))
+});
 
 /// A contents line: dot leaders to a page number, or a link to an internal
 /// anchor, which is what a converted EPUB's table of contents becomes.
-static TOC_LINE: LazyLock<Regex> = LazyLock::new(|| pattern(r"(\.{3,}\s*\d{1,4}\s*$|\]\(#)"));
+static TOC_LINE: LazyLock<Regex> =
+    LazyLock::new(|| pattern(&format!(r"(\.{{3,}}{SPACE}*\d{{1,4}}{SPACE}*$|\]\(#)")));
 
 /// A wrapped entry long enough that its publisher name is probably a
 /// coincidence rather than a citation, in characters.
@@ -139,8 +168,8 @@ pub fn classify(text: &str, chapter: Option<&str>) -> Role {
 }
 
 fn looks_like_entry(paragraph: &str) -> bool {
-    let year = YEAR.is_match(paragraph);
+    let year = YEAR.is_match(paragraph).unwrap_or(false);
-    let publisher = BIBLIO_CONTINUATION.is_match(paragraph);
+    let publisher = BIBLIO_CONTINUATION.is_match(paragraph).unwrap_or(false);
     if BIBLIO_ENTRY.is_match(paragraph) && (year || publisher) {
         return true;
     }

crates/cuenorm/src/main.rs

@@ -20,7 +20,6 @@ use corpus_core::{exit, ndjson};
 #[command(
     version,
     about,
-    long_about = None,
     after_help = "\
 Example:
 

crates/cueseg/Cargo.toml

@@ -11,6 +11,7 @@ repository.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true
+fancy-regex.workspace = true
 regex.workspace = true
 schemars.workspace = true
 serde.workspace = true

crates/cueseg/src/main.rs

@@ -21,7 +21,6 @@ use cuekit::Cue;
 #[command(
     version,
     about,
-    long_about = None,
     after_help = "\
 Example:
 

crates/cueseg/src/title.rs

@@ -3,17 +3,22 @@
 use std::sync::LazyLock;
 
 use regex::Regex;
+use textkit::pattern::{SPACE, WORD_END, WORD_START};
 
 /// Hesitations. A chapter called "Um Yeah So I Think" is worse than no title.
-static FILLER: LazyLock<Regex> =
+static FILLER: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
-    LazyLock::new(|| Regex::new(r"(?i)\b(um+|uh+|ah+|er+)\b").expect("a valid pattern"));
+    fancy_regex::Regex::new(&format!(r"(?i){WORD_START}(um+|uh+|ah+|er+){WORD_END}"))
+        .expect("a valid pattern")
+});
 
 /// The most common discourse marker in recorded speech.
-static YOU_KNOW: LazyLock<Regex> =
+static YOU_KNOW: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
-    LazyLock::new(|| Regex::new(r"(?i)\byou know\b").expect("a valid pattern"));
+    fancy_regex::Regex::new(&format!(r"(?i){WORD_START}you know{WORD_END}"))
+        .expect("a valid pattern")
+});
 
 static SPACE_RUN: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+    LazyLock::new(|| Regex::new(&format!(r"{SPACE}+")).expect("a valid pattern"));
 
 /// Anything that does not belong in a title.
 static NOT_TITLE: LazyLock<Regex> =

crates/mdscrub/src/heading.rs

@@ -9,6 +9,7 @@
 use std::sync::LazyLock;
 
 use regex::Regex;
+use textkit::pattern::SPACE;
 
 fn pattern(source: &str) -> Regex {
     Regex::new(source).expect("a valid pattern")
@@ -21,23 +22,24 @@ static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"
 static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
 
 /// A trailing `{.attr}` block.
-static ATTRIBUTE_TAIL: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s*\{[^}]*\}\s*$"));
+static ATTRIBUTE_TAIL: LazyLock<Regex> =
+    LazyLock::new(|| pattern(&format!(r"{SPACE}*\{{[^}}]*\}}{SPACE}*$")));
 
 /// Any inline tag. Broader than the one used on body text, because a heading
 /// has no code fences to protect.
 static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"<[^>]+>"));
 
-static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s+"));
+static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(&format!(r"{SPACE}+")));
 
 /// A heading line, and the text after its hashes.
-static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(r"^(#{1,6})\s+(.*)$"));
+static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(&format!(r"^(#{{1,6}}){SPACE}+(.*)$")));
 
 /// A page anchor: `<span id="…page42">`, `<a id="page-42">`, and the rest of
 /// the shapes an EPUB uses to say where a printed page began.
 pub static PAGE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
-    pattern(
+    pattern(&format!(
-        r#"(?i)<(?:span|a|div)\b[^>]*\bid\s*=\s*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#,
+        r#"(?i)<(?:span|a|div)\b[^>]*\bid{SPACE}*={SPACE}*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#
-    )
+    ))
 });
 
 /// Clean one heading's text.

crates/mdscrub/src/main.rs

@@ -16,7 +16,6 @@ use corpus_core::exit;
 #[command(
     version,
     about,
-    long_about = None,
     after_help = "\
 Example:
 
@@ -32,9 +31,9 @@ fn main() -> ExitCode {
 
     let markdown = match read(args.path.as_deref()) {
         Ok(markdown) => markdown,
-        Err(e) => {
+        Err((code, why)) => {
-            eprintln!("mdscrub: {e}");
+            eprintln!("mdscrub: {why}");
-            return ExitCode::from(exit::IO);
+            return ExitCode::from(code);
         }
     };
 
@@ -50,13 +49,21 @@ fn main() -> ExitCode {
     }
 }
 
-fn read(path: Option<&Path>) -> std::io::Result<String> {
+/// Read the input, distinguishing a file that cannot be read from one that
-    match path {
+/// is not text: the first is an I/O failure and the second is bad input, and
-        Some(path) => std::fs::read_to_string(path),
+/// a caller routing on the exit code needs to tell them apart.
-        None => {
+fn read(path: Option<&Path>) -> Result<String, (u8, String)> {
-            let mut markdown = String::new();
+    let bytes = match path {
-            std::io::stdin().read_to_string(&mut markdown)?;
+        Some(path) => {
-            Ok(markdown)
+            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
         }
-    }
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin()
+                .read_to_end(&mut bytes)
+                .map(|_| bytes)
+                .map_err(|e| (exit::IO, e.to_string()))
+        }
+    }?;
+    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
 }

crates/pdftriage/src/main.rs

@@ -27,7 +27,6 @@ const SPAN: u32 = 5;
 #[command(
     version,
     about,
-    long_about = None,
     after_help = "\
 Exit codes: 0 the layer needs OCR, 1 it does not, 2 could not tell,
 69 pdftotext is not installed, 74 the file could not be read.

crates/textkit/Cargo.toml

@@ -10,5 +10,8 @@ repository.workspace = true
 [dependencies]
 tiktoken-rs.workspace = true
 
+[dev-dependencies]
+regex.workspace = true
+
 [lints]
 workspace = true

crates/textkit/src/lib.rs

@@ -11,6 +11,7 @@
 //! matches what is already in an index.
 
 pub mod number;
+pub mod pattern;
 pub mod segment;
 pub mod tokens;
 

crates/textkit/src/pattern.rs

@@ -0,0 +1,59 @@
+//! Character classes that mean in Rust what they mean in Python.
+//!
+//! The two regex engines disagree about `\s`, `\w` and therefore `\b`, and
+//! every disagreement is a heuristic that fires on one side and not the
+//! other. The differences are small and entirely real:
+//!
+//! - Python's `\s` also matches `U+001C`–`U+001F`, the four ASCII
+//!   information separators.
+//! - Python's `\w` is the letters, the numbers and the underscore. Rust's
+//!   adds every combining mark, every connector punctuation mark and the
+//!   join controls, and leaves out the letter-like and other numbers — so
+//!   `Gödel` written in decomposed form is one word to Rust and two to
+//!   Python, and `²` is a word character to Python and not to Rust.
+//! - `\b` is defined in terms of `\w`, so it inherits all of that.
+//!
+//! Spell a ported pattern with these and it means the same thing on both
+//! sides. The boundary forms are lookarounds, so a pattern using them needs
+//! an engine that has them.
+
+/// Python's `\s`.
+pub const SPACE: &str = r"[\s\x{1c}-\x{1f}]";
+
+/// Python's `\w`.
+pub const WORD: &str = r"[\p{L}\p{N}_]";
+
+/// Python's `\b`, before a word.
+pub const WORD_START: &str = r"(?<![\p{L}\p{N}_])";
+
+/// Python's `\b`, after a word.
+pub const WORD_END: &str = r"(?![\p{L}\p{N}_])";
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_space_class_covers_what_python_trims() {
+        let space = regex::Regex::new(SPACE).expect("a valid pattern");
+        for c in [' ', '\t', '\n', '\u{a0}', '\u{1c}', '\u{1f}', '\u{2028}'] {
+            assert!(space.is_match(&c.to_string()), "{c:?}");
+            assert!(crate::is_space(c), "{c:?}");
+        }
+        for c in ['a', '-', '\u{200b}'] {
+            assert!(!space.is_match(&c.to_string()), "{c:?}");
+        }
+    }
+
+    #[test]
+    fn the_word_class_is_letters_numbers_and_underscore() {
+        let word = regex::Regex::new(&format!("^{WORD}$")).expect("a valid pattern");
+        for c in ['a', 'Z', '0', '_', 'é', 'Ω', '²', 'Ⅷ'] {
+            assert!(word.is_match(&c.to_string()), "{c:?}");
+        }
+        // The combining marks Rust's own `\w` would accept.
+        for c in ['\u{301}', '\u{363}', '-', ' '] {
+            assert!(!word.is_match(&c.to_string()), "{c:?}");
+        }
+    }
+}

parity/KNOWN.md

@@ -58,26 +58,62 @@ emitting chunks with no page number, which is the honest answer.
 only reachable from the PDF conversion path. Covered by
 `mdscrub::pages::tests::text_with_no_page_boundaries_is_left_exactly_as_it_was`.
 
-## chunkmd, role and footnote patterns — `\s` excludes four control characters
+## Every ported pattern — the two engines disagree about `\d` and `(?i)`
 
-**Python:** `\s` in a `str` pattern matches the Unicode `White_Space`
+**Python:** `re` compiles against the Unicode version of the interpreter;
-property *and* `U+001C`–`U+001F`, the four ASCII information separators.
+`(?i)` folds `ı` and `İ` onto `i`.
 
-**Rust:** `regex` and `fancy-regex` match the `White_Space` property only, so
+**Rust:** `regex` compiles against the Unicode version it was built with, and
-those four characters are not whitespace to a ported pattern.
+`(?i)` uses simple case folding, which leaves the dotless and dotted `i`
+alone.
 
-**Why this is acceptable:** every one of those codepoints is deleted by
+**Why this is acceptable:** `\s`, `\w` and `\b` were the differences that
-`mdscrub` before a chunker sees it — they fall inside the control-character
+mattered and they are closed — every ported pattern now spells Python's
-class that a text column rejects. For the pattern to behave differently, a
+classes explicitly, from `textkit::pattern`. What is left is a `\d` that
-document would have to reach `chunkmd` without having been cleaned, and carry
+disagrees only on decimal digits added between the two Unicode versions
-an information separator in exactly the position where a pattern expects
+(Garay, added in Unicode 16, is the current example), and a case fold that
-whitespace.
+disagrees only for Turkish dotless `i`. A chapter heading reading
+`Bıblıography` is classified as body text here and as a bibliography there;
+a page marker written in Garay digits reads here and does not there.
 
-Note that this does **not** apply to the sentence and paragraph splitters in
+Both will drift again at every dependency bump, which is the real reason this
-`textkit`, which match Python's definition exactly — see
+entry exists: the golden files pin one pair of Unicode versions, and a change
-`textkit::segment::is_space`. Those decide chunk boundaries, so the four
+to either shows up as a parity failure rather than as a silent difference.
-characters are handled there rather than argued about.
 
-**Measured:** zero differences over the parity corpus; the sample in
+**Measured:** zero differences over the parity corpus. Found by brute-forcing
-`parity/src/sample.rs` includes `U+001C` deliberately and the token counts
+every code point below `U+11000` through both engines: `\d` differs on ten,
-agree over all ten thousand lines.
+`(?i)` on three.
+
+## chunkmd — a page number too large for the field saturates
+
+**Python:** integers are unbounded, so `<!-- Page 99999999999 -->` records
+page 99,999,999,999.
+
+**Rust:** the page field is a 32-bit number and saturates at 4,294,967,295.
+
+**Why this is acceptable:** the number is a page number. A document with four
+billion pages does not exist, and the alternative — a 64-bit field — would
+make every chunk record wider to carry a value nothing can produce. The
+saturation is deliberate rather than a wrap, so the wrong answer is at least
+obviously wrong.
+
+**Measured:** no document in any corpus this was run against has a page
+marker above five digits.
+
+## chunkmd — a hard maximum of zero yields nothing rather than raising
+
+**Python:** `_enforce_hard_max` with `hard_max=0` raises `ValueError:
+range() arg 3 must not be zero`.
+
+**Rust:** `chunkmd::chunk` returns no chunks, since no piece could satisfy
+the limit.
+
+**Why this is acceptable:** the command-line tool rejects it before reading
+any input, so this is reachable only by a library caller, and a filter that
+returns nothing for a budget that permits nothing is the more useful answer
+of the two. Every other degenerate budget — overlap at or above target, a
+hard maximum below the target, a minimum of zero — agrees with the Python
+exactly.
+
+**Measured:** checked against the oracle for all of those combinations; only
+the zero hard maximum differs.