From 339f4026105c3bdcf850bffafcf3c12c58d20109 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 12:26:41 -0400 Subject: [PATCH] ported patterns mean in rust what they mean in python --- Cargo.lock | 2 + crates/chunkmd/src/footnote.rs | 71 +++++++++++++++++---------------- crates/chunkmd/src/lib.rs | 11 +++--- crates/chunkmd/src/main.rs | 31 +++++++++------ crates/chunkmd/src/pack.rs | 24 +++++++++--- crates/chunkmd/src/pages.rs | 64 ++++++++++++++++-------------- crates/chunkmd/src/role.rs | 71 +++++++++++++++++++++++---------- crates/cuenorm/src/main.rs | 1 - crates/cueseg/Cargo.toml | 1 + crates/cueseg/src/main.rs | 1 - crates/cueseg/src/title.rs | 15 ++++--- crates/mdscrub/src/heading.rs | 14 ++++--- crates/mdscrub/src/main.rs | 31 +++++++++------ crates/pdftriage/src/main.rs | 1 - crates/textkit/Cargo.toml | 3 ++ crates/textkit/src/lib.rs | 1 + crates/textkit/src/pattern.rs | 59 ++++++++++++++++++++++++++++ parity/KNOWN.md | 72 +++++++++++++++++++++++++--------- 18 files changed, 322 insertions(+), 151 deletions(-) create mode 100644 crates/textkit/src/pattern.rs diff --git a/Cargo.lock b/Cargo.lock index a1b6726..939b9a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,6 +194,7 @@ dependencies = [ "clap", "corpus-core", "cuekit", + "fancy-regex 0.16.2", "regex", "schemars", "serde", @@ -474,6 +475,7 @@ dependencies = [ name = "textkit" version = "0.1.0" dependencies = [ + "regex", "tiktoken-rs", ] diff --git a/crates/chunkmd/src/footnote.rs b/crates/chunkmd/src/footnote.rs index 44242fd..2b9ca7d 100644 --- a/crates/chunkmd/src/footnote.rs +++ b/crates/chunkmd/src/footnote.rs @@ -8,7 +8,7 @@ use std::sync::LazyLock; use fancy_regex::Regex as FancyRegex; -use regex::Regex; +use textkit::pattern::{SPACE, WORD_END, WORD_START}; /// A private-use codepoint marking the start of a note's text. /// @@ -40,47 +40,50 @@ const MIN_CITED_NOTES: usize = 2; /// more spaces — several notes can share one physical line — so that a /// superscript *reference* to a note, mid-sentence, does not qualify. static NOTE_MARKER: LazyLock = LazyLock::new(|| { - FancyRegex::new( + FancyRegex::new(&format!( r"(?xm) - (?: ^[ \t]{0,8} (?:-[ \t]+)? # line start, optional '- ' list bullet - | (?<=\s\s) # or after a 2+ space run + (?: ^[ \t]{{0,8}} (?:-[ \t]+)? # line start, optional '- ' list bullet + | (?<={SPACE}{SPACE}) # or after a 2+ space run | (?<=\ -\ ) # or after ' - ' (inline list separator) ) (?: - \d{1,3} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. ' - | \[ \d{1,3} \] [ \t]* \S # bracket-numbered '[12] Author' - | \d{1,3} [ \t]+ \S # bare '12 ' from a page bottom - | [¹²³⁰-⁹]+ \s* \S # unicode superscript marker - | (?:fn|ftn|note) \d{1,4} [.)] [ \t]+ \S # id-style 'fn39. ' - | (?:]*>\s*)* # outer span wrap - ]*> \s* (?:]*>\s*)* (?:\s*)? - \d{1,3} \s* [.)]? - \s* (?:\s*)? (?:\s*)* - (?:\s*)* \s* [.)]? # anchor forms - | \[ \s* (?:]*>)? \s* \d{1,3} - \s* (?:)? \s* \] \( \#[^)]* \) # [42](#...) - | \s* \d{1,3} \s* # bare sup note - )", - ) + \d{{1,3}} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. ' + | \[ \d{{1,3}} \] [ \t]* \S # bracket-numbered '[12] Author' + | \d{{1,3}} [ \t]+ \S # bare '12 ' from a page bottom + | [¹²³⁰-⁹]+ {SPACE}* \S # unicode superscript marker + | (?:fn|ftn|note) \d{{1,4}} [.)] [ \t]+ \S # id-style 'fn39. ' + | (?:]*>{SPACE}*)* # outer span wrap + ]*> {SPACE}* (?:]*>{SPACE}*)* (?:{SPACE}*)? + \d{{1,3}} {SPACE}* [.)]? + {SPACE}* (?:{SPACE}*)? (?:{SPACE}*)* + (?:{SPACE}*)* {SPACE}* [.)]? # anchor forms + | \[ {SPACE}* (?:]*>)? {SPACE}* \d{{1,3}} + {SPACE}* (?:)? {SPACE}* \] \( \#[^)]* \) # [42](#...) + | {SPACE}* \d{{1,3}} {SPACE}* # bare sup note + )" + )) .expect("a valid pattern") }); /// What makes a note bibliographic rather than discursive. -static CITE_SIGNAL: LazyLock = LazyLock::new(|| { - Regex::new( +static CITE_SIGNAL: LazyLock = LazyLock::new(|| { + FancyRegex::new(&format!( r"(?ix) - \((?:[^()]*\b)?(?:1[5-9]|20)\d{2}\) # parenthesised year: (Oxford, 1998) - | \bpp?\.\s*\d # p. 12 / pp. 12-34 - | \bvol\.\s*\d # vol. 2 - | \bIbid\b | \bop\.\s*cit\b | \bloc\.\s*cit\b - | \bet\s+al\b - | \b(?:University|Clarendon|Academic)\s+Press\b | \bPress,\s - | \beds?\.\s | \btrans\.\s - | \*[^*\n]{3,80}\*\s*,?\s*\d # short form: *Title*, 183 - | ]*italic[^>]*> # HTML-italic title survives conversion - | ,\s+\d{1,4}\s*[.;] # trailing page: '..., 183.' - | https?:// # web citations (URLs, DOIs)", - ) + \((?:[^()]*{WORD_END})?(?:1[5-9]|20)\d{{2}}\) # parenthesised year: (Oxford, 1998) + | {WORD_START}pp?\.{SPACE}*\d # p. 12 / pp. 12-34 + | {WORD_START}vol\.{SPACE}*\d # vol. 2 + | {WORD_START}Ibid{WORD_END} + | {WORD_START}op\.{SPACE}*cit{WORD_END} + | {WORD_START}loc\.{SPACE}*cit{WORD_END} + | {WORD_START}et{SPACE}+al{WORD_END} + | {WORD_START}(?:University|Clarendon|Academic){SPACE}+Press{WORD_END} + | {WORD_START}Press,{SPACE} + | {WORD_START}eds?\.{SPACE} | {WORD_START}trans\.{SPACE} + | \*[^*\n]{{3,80}}\*{SPACE}*,?{SPACE}*\d # short form: *Title*, 183 + | ]*italic[^>]*> # HTML-italic title survives conversion + | ,{SPACE}+\d{{1,4}}{SPACE}*[.;] # trailing page: '..., 183.' + | https?:// # web citations (URLs, DOIs)" + )) .expect("a valid pattern") }); @@ -107,7 +110,7 @@ pub fn looks_like_citations(text: &str) -> bool { for (at, start) in starts.iter().enumerate() { let end = starts.get(at + 1).copied().unwrap_or(text.len()); let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect(); - if CITE_SIGNAL.is_match(&entry) { + if CITE_SIGNAL.is_match(&entry).unwrap_or(false) { cited += 1; if cited >= MIN_CITED_NOTES { return true; diff --git a/crates/chunkmd/src/lib.rs b/crates/chunkmd/src/lib.rs index 6a1f879..8b1403d 100644 --- a/crates/chunkmd/src/lib.rs +++ b/crates/chunkmd/src/lib.rs @@ -28,23 +28,24 @@ pub use pack::Budgets; pub fn chunk(markdown: &str, budgets: &Budgets) -> Vec { let pages = pages::split_by_page(markdown); let chapters = pages::chapters(markdown); - let by_page = pages::page_chapters(&chapters); let by_offset = pages::ChapterIndex::new(&chapters); let mut chunks: Vec = Vec::new(); for page in pages { for packed in pack::page(page.text, budgets) { - for piece in pack::enforce_hard_max(&packed, budgets.hard_max) { - let tokens = textkit::count(&piece); + for (piece, tokens) in pack::enforce_hard_max(&packed, budgets.hard_max) { if tokens < budgets.min_tokens { continue; } // A page-based chapter when the document has page markers; // the byte offset otherwise, which is what carries EPUBs - // that have headings but no page anchors. + // that have headings but no page anchors. Page zero counts as + // no page at all, which is what the Python's truth test makes + // of it. let chapter = page .number - .and_then(|number| by_page.get(&number).copied()) + .filter(|number| *number != 0) + .and_then(|number| pages::chapter_of_page(&chapters, number)) .or_else(|| by_offset.at(page.offset)); let idx = chunks.len() as u32; diff --git a/crates/chunkmd/src/main.rs b/crates/chunkmd/src/main.rs index 7424ced..6b9e17d 100644 --- a/crates/chunkmd/src/main.rs +++ b/crates/chunkmd/src/main.rs @@ -18,7 +18,6 @@ use corpus_core::{exit, ndjson}; #[command( version, about, - long_about = None, after_help = "\ Example: @@ -63,9 +62,9 @@ fn main() -> ExitCode { let markdown = match read(args.path.as_deref()) { Ok(markdown) => markdown, - Err(e) => { - eprintln!("chunkmd: {e}"); - return ExitCode::from(exit::IO); + Err((code, why)) => { + eprintln!("chunkmd: {why}"); + return ExitCode::from(code); } }; @@ -118,13 +117,21 @@ fn sane(budgets: &Budgets) -> Result<(), String> { Ok(()) } -fn read(path: Option<&std::path::Path>) -> std::io::Result { - match path { - Some(path) => std::fs::read_to_string(path), - None => { - let mut markdown = String::new(); - std::io::stdin().read_to_string(&mut markdown)?; - Ok(markdown) +/// Read the input, distinguishing a file that cannot be read from one that +/// is not text: the first is an I/O failure and the second is bad input, and +/// a caller routing on the exit code needs to tell them apart. +fn read(path: Option<&std::path::Path>) -> Result { + let bytes = match path { + Some(path) => { + std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display()))) } - } + None => { + let mut bytes = Vec::new(); + std::io::stdin() + .read_to_end(&mut bytes) + .map(|_| bytes) + .map_err(|e| (exit::IO, e.to_string())) + } + }?; + String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}"))) } diff --git a/crates/chunkmd/src/pack.rs b/crates/chunkmd/src/pack.rs index 1e2ad9a..3a44918 100644 --- a/crates/chunkmd/src/pack.rs +++ b/crates/chunkmd/src/pack.rs @@ -100,12 +100,25 @@ fn carry_tail(buf: &[String], overlap: usize) -> Vec { out } -/// Split anything still over the hard maximum, on token boundaries. +/// Split anything still over the hard maximum, on token boundaries, and hand +/// back each piece with its token count. /// /// Defence in depth. The structural passes above should have handled it; this -/// is what guarantees the promise regardless. -pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec { +/// is what guarantees the promise regardless. The counts come back with the +/// pieces because they have just been computed, and counting them again is +/// the second-largest cost in the whole chunker. +pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<(String, usize)> { + let tokens = textkit::count(text); + if tokens <= hard_max { + return vec![(text.to_string(), tokens)]; + } textkit::split_at_token_limit(text, hard_max) + .into_iter() + .map(|piece| { + let tokens = textkit::count(&piece); + (piece, tokens) + }) + .collect() } #[cfg(test)] @@ -179,8 +192,9 @@ mod tests { #[test] fn nothing_leaves_the_packer_over_the_hard_maximum() { let long = "word ".repeat(2000); - for piece in enforce_hard_max(&long, 100) { - assert!(textkit::count(&piece) <= 100); + for (piece, tokens) in enforce_hard_max(&long, 100) { + assert!(tokens <= 100); + assert_eq!(tokens, textkit::count(&piece)); } } } diff --git a/crates/chunkmd/src/pages.rs b/crates/chunkmd/src/pages.rs index 6e90f0c..1c83899 100644 --- a/crates/chunkmd/src/pages.rs +++ b/crates/chunkmd/src/pages.rs @@ -4,16 +4,18 @@ //! and chapter boundaries as headings. Both are needed on every chunk: a //! passage a reader cannot locate in the book is a passage they cannot check. -use std::collections::HashMap; use std::sync::LazyLock; use regex::Regex; +use textkit::pattern::SPACE; -static PAGE: LazyLock = - LazyLock::new(|| Regex::new(r"").expect("a valid pattern")); +static PAGE: LazyLock = LazyLock::new(|| { + Regex::new(&format!(r"")).expect("a valid pattern") +}); -static HEADING: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^(#{1,3})\s+(.+)$").expect("a valid pattern")); +static HEADING: LazyLock = LazyLock::new(|| { + Regex::new(&format!(r"(?m)^(#{{1,3}}){SPACE}+(.+)$")).expect("a valid pattern") +}); /// One page's worth of text, and where it started in the source. #[derive(Debug, Clone, PartialEq, Eq)] @@ -166,31 +168,33 @@ impl<'a> ChapterIndex<'a> { } } -/// Map each page number to the chapter covering it. +/// The chapter covering a page, if any. /// /// Where chapters overlap — which happens when two headings land on one page /// — the later one wins, because that is the one a reader scrolling down /// would be under. -pub fn page_chapters(chapters: &[Chapter]) -> HashMap { - let mut out = HashMap::new(); - for chapter in chapters { - let Some(start) = chapter.start_page else { - continue; - }; - // A zero end page means the same thing as no end page: this chapter - // is one page long as far as anyone can tell. - let end = match chapter.end_page { - Some(0) | None => start, - Some(end) => end, - }; - if end < start { - continue; - } - for page in start..=end { - out.insert(page, chapter.title.as_str()); - } - } - out +/// +/// Scanned rather than indexed into a map of every page. A document has tens +/// of chapters and a scan of them costs nothing, where a map costs one entry +/// per page in every chapter's span: a single mistyped marker reading +/// `` would ask for a gigabyte of them. +pub fn chapter_of_page(chapters: &[Chapter], page: u32) -> Option<&str> { + chapters + .iter() + .filter(|chapter| { + let Some(start) = chapter.start_page else { + return false; + }; + // A zero end page means the same thing as no end page: this + // chapter is one page long as far as anyone can tell. + let end = match chapter.end_page { + Some(0) | None => start, + Some(end) => end, + }; + (start..=end.max(start)).contains(&page) && end >= start + }) + .next_back() + .map(|chapter| chapter.title.as_str()) } fn trim(text: &str) -> &str { @@ -264,10 +268,10 @@ mod tests { assert_eq!(found[0].start_page, Some(1)); assert_eq!(found[0].end_page, Some(4)); assert_eq!(found[1].start_page, Some(4)); - let by_page = page_chapters(&found); - assert_eq!(by_page.get(&1), Some(&"One")); - assert_eq!(by_page.get(&3), Some(&"One")); - assert_eq!(by_page.get(&4), Some(&"Two")); + assert_eq!(chapter_of_page(&found, 1), Some("One")); + assert_eq!(chapter_of_page(&found, 3), Some("One")); + assert_eq!(chapter_of_page(&found, 4), Some("Two")); + assert_eq!(chapter_of_page(&found, 9), None); } #[test] diff --git a/crates/chunkmd/src/role.rs b/crates/chunkmd/src/role.rs index 4833d25..408009b 100644 --- a/crates/chunkmd/src/role.rs +++ b/crates/chunkmd/src/role.rs @@ -14,28 +14,46 @@ use std::sync::LazyLock; use corpus_core::Role; +use fancy_regex::Regex as Boundaried; use regex::Regex; +use textkit::pattern::{SPACE, WORD, WORD_END, WORD_START}; fn pattern(source: &str) -> Regex { Regex::new(source).expect("a valid pattern") } +/// A pattern that needs Python's word boundary, which is a lookaround here +/// because Rust's own `\b` counts combining marks as word characters. +fn bounded(source: &str) -> Boundaried { + Boundaried::new(source).expect("a valid pattern") +} + /// Chapter titles that give the section away, tried in this order. static BY_TITLE: LazyLock> = LazyLock::new(|| { vec![ ( Role::Bibliography, - pattern( - r"(?i)(bibliograph|works\s+cited|literature\s+cited|further\s+reading|^\s*references\s*$|^\s*(end)?notes\s*$)", - ), + pattern(&format!( + r"(?i)(bibliograph|works{SPACE}+cited|literature{SPACE}+cited|further{SPACE}+reading|^{SPACE}*references{SPACE}*$|^{SPACE}*(end)?notes{SPACE}*$)" + )), + ), + ( + Role::Index, + pattern(&format!( + r"(?i)^{SPACE}*index({SPACE}+of{SPACE}+{WORD}+)?{SPACE}*$" + )), + ), + ( + Role::Toc, + pattern(&format!( + r"(?i)^{SPACE}*(table{SPACE}+of{SPACE}+)?contents{SPACE}*$" + )), ), - (Role::Index, pattern(r"(?i)^\s*index(\s+of\s+\w+)?\s*$")), - (Role::Toc, pattern(r"(?i)^\s*(table\s+of\s+)?contents\s*$")), ( Role::Frontmatter, - pattern( - r"(?i)(copyright|title\s*page|colophon|^\s*dedication\s*$|^\s*epigraph\s*$|about\s+the\s+(author|publisher)|acknowledg)", - ), + pattern(&format!( + r"(?i)(copyright|title{SPACE}*page|colophon|^{SPACE}*dedication{SPACE}*$|^{SPACE}*epigraph{SPACE}*$|about{SPACE}+the{SPACE}+(author|publisher)|acknowledg)" + )), ), ] }); @@ -43,29 +61,40 @@ static BY_TITLE: LazyLock> = LazyLock::new(|| { /// A standalone bibliography heading inside the chunk's own text — /// "Bibliography", "## Works Cited", "Selected Bibliography". static INTEXT_BIBLIO_HEADING: LazyLock = LazyLock::new(|| { - pattern( - r"(?im)^\s*#{0,6}\s*(select(?:ed)?\s+)?(bibliography|works\s+cited|references|sources)\s*$", - ) + pattern(&format!( + r"(?im)^{SPACE}*#{{0,6}}{SPACE}*(select(?:ed)?{SPACE}+)?(bibliography|works{SPACE}+cited|references|sources){SPACE}*$" + )) }); /// One bibliography entry: "Lastname, First…", or the dash run that stands /// for "the same author again". -static BIBLIO_ENTRY: LazyLock = - LazyLock::new(|| pattern(r"^(?:[A-Z][\w'’\-]+,\s+\S|[—–-]{2,3}\s*[.,])")); +static BIBLIO_ENTRY: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"^(?:[A-Z][\p{{L}}\p{{N}}_'’\-]+,{SPACE}+\S|[—–-]{{2,3}}{SPACE}*[.,])" + )) +}); /// The continuation of a wrapped entry: "New York: Springer, 2013." -static BIBLIO_CONTINUATION: LazyLock = - LazyLock::new(|| pattern(r"\b(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.)\b")); +static BIBLIO_CONTINUATION: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"{WORD_START}(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.){WORD_END}" + )) +}); -static YEAR: LazyLock = LazyLock::new(|| pattern(r"\b(1[5-9]\d{2}|20\d{2})\b")); +static YEAR: LazyLock = + LazyLock::new(|| bounded(&format!(r"{WORD_START}(1[5-9]\d{{2}}|20\d{{2}}){WORD_END}"))); /// An index line: a short lead, then a list of page numbers. -static INDEX_LINE: LazyLock = - LazyLock::new(|| pattern(r"^.{1,70}?,\s+[ivxlc\d]{1,4}(?:[,–-]\s*[ivxlc\d]{1,4})*\s*$")); +static INDEX_LINE: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"^.{{1,70}}?,{SPACE}+[ivxlc\d]{{1,4}}(?:[,–-]{SPACE}*[ivxlc\d]{{1,4}})*{SPACE}*$" + )) +}); /// A contents line: dot leaders to a page number, or a link to an internal /// anchor, which is what a converted EPUB's table of contents becomes. -static TOC_LINE: LazyLock = LazyLock::new(|| pattern(r"(\.{3,}\s*\d{1,4}\s*$|\]\(#)")); +static TOC_LINE: LazyLock = + LazyLock::new(|| pattern(&format!(r"(\.{{3,}}{SPACE}*\d{{1,4}}{SPACE}*$|\]\(#)"))); /// A wrapped entry long enough that its publisher name is probably a /// coincidence rather than a citation, in characters. @@ -139,8 +168,8 @@ pub fn classify(text: &str, chapter: Option<&str>) -> Role { } fn looks_like_entry(paragraph: &str) -> bool { - let year = YEAR.is_match(paragraph); - let publisher = BIBLIO_CONTINUATION.is_match(paragraph); + let year = YEAR.is_match(paragraph).unwrap_or(false); + let publisher = BIBLIO_CONTINUATION.is_match(paragraph).unwrap_or(false); if BIBLIO_ENTRY.is_match(paragraph) && (year || publisher) { return true; } diff --git a/crates/cuenorm/src/main.rs b/crates/cuenorm/src/main.rs index a067f5d..51eadc0 100644 --- a/crates/cuenorm/src/main.rs +++ b/crates/cuenorm/src/main.rs @@ -20,7 +20,6 @@ use corpus_core::{exit, ndjson}; #[command( version, about, - long_about = None, after_help = "\ Example: diff --git a/crates/cueseg/Cargo.toml b/crates/cueseg/Cargo.toml index 2089613..6f6e210 100644 --- a/crates/cueseg/Cargo.toml +++ b/crates/cueseg/Cargo.toml @@ -11,6 +11,7 @@ repository.workspace = true clap.workspace = true corpus-core.workspace = true cuekit.workspace = true +fancy-regex.workspace = true regex.workspace = true schemars.workspace = true serde.workspace = true diff --git a/crates/cueseg/src/main.rs b/crates/cueseg/src/main.rs index 8532bfa..13a276c 100644 --- a/crates/cueseg/src/main.rs +++ b/crates/cueseg/src/main.rs @@ -21,7 +21,6 @@ use cuekit::Cue; #[command( version, about, - long_about = None, after_help = "\ Example: diff --git a/crates/cueseg/src/title.rs b/crates/cueseg/src/title.rs index 3a58a1a..4f2bc6b 100644 --- a/crates/cueseg/src/title.rs +++ b/crates/cueseg/src/title.rs @@ -3,17 +3,22 @@ use std::sync::LazyLock; use regex::Regex; +use textkit::pattern::{SPACE, WORD_END, WORD_START}; /// Hesitations. A chapter called "Um Yeah So I Think" is worse than no title. -static FILLER: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\b(um+|uh+|ah+|er+)\b").expect("a valid pattern")); +static FILLER: LazyLock = LazyLock::new(|| { + fancy_regex::Regex::new(&format!(r"(?i){WORD_START}(um+|uh+|ah+|er+){WORD_END}")) + .expect("a valid pattern") +}); /// The most common discourse marker in recorded speech. -static YOU_KNOW: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\byou know\b").expect("a valid pattern")); +static YOU_KNOW: LazyLock = LazyLock::new(|| { + fancy_regex::Regex::new(&format!(r"(?i){WORD_START}you know{WORD_END}")) + .expect("a valid pattern") +}); static SPACE_RUN: LazyLock = - LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern")); + LazyLock::new(|| Regex::new(&format!(r"{SPACE}+")).expect("a valid pattern")); /// Anything that does not belong in a title. static NOT_TITLE: LazyLock = diff --git a/crates/mdscrub/src/heading.rs b/crates/mdscrub/src/heading.rs index d9af8fc..484fc8c 100644 --- a/crates/mdscrub/src/heading.rs +++ b/crates/mdscrub/src/heading.rs @@ -9,6 +9,7 @@ use std::sync::LazyLock; use regex::Regex; +use textkit::pattern::SPACE; fn pattern(source: &str) -> Regex { Regex::new(source).expect("a valid pattern") @@ -21,23 +22,24 @@ static LINK: LazyLock = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)" static BRACKETED: LazyLock = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}")); /// A trailing `{.attr}` block. -static ATTRIBUTE_TAIL: LazyLock = LazyLock::new(|| pattern(r"\s*\{[^}]*\}\s*$")); +static ATTRIBUTE_TAIL: LazyLock = + LazyLock::new(|| pattern(&format!(r"{SPACE}*\{{[^}}]*\}}{SPACE}*$"))); /// Any inline tag. Broader than the one used on body text, because a heading /// has no code fences to protect. static TAG: LazyLock = LazyLock::new(|| pattern(r"<[^>]+>")); -static SPACE_RUN: LazyLock = LazyLock::new(|| pattern(r"\s+")); +static SPACE_RUN: LazyLock = LazyLock::new(|| pattern(&format!(r"{SPACE}+"))); /// A heading line, and the text after its hashes. -static HEADING: LazyLock = LazyLock::new(|| pattern(r"^(#{1,6})\s+(.*)$")); +static HEADING: LazyLock = LazyLock::new(|| pattern(&format!(r"^(#{{1,6}}){SPACE}+(.*)$"))); /// A page anchor: ``, ``, and the rest of /// the shapes an EPUB uses to say where a printed page began. pub static PAGE_ANCHOR: LazyLock = LazyLock::new(|| { - pattern( - r#"(?i)<(?:span|a|div)\b[^>]*\bid\s*=\s*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:)?"#, - ) + pattern(&format!( + r#"(?i)<(?:span|a|div)\b[^>]*\bid{SPACE}*={SPACE}*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:)?"# + )) }); /// Clean one heading's text. diff --git a/crates/mdscrub/src/main.rs b/crates/mdscrub/src/main.rs index 5630d89..70959eb 100644 --- a/crates/mdscrub/src/main.rs +++ b/crates/mdscrub/src/main.rs @@ -16,7 +16,6 @@ use corpus_core::exit; #[command( version, about, - long_about = None, after_help = "\ Example: @@ -32,9 +31,9 @@ fn main() -> ExitCode { let markdown = match read(args.path.as_deref()) { Ok(markdown) => markdown, - Err(e) => { - eprintln!("mdscrub: {e}"); - return ExitCode::from(exit::IO); + Err((code, why)) => { + eprintln!("mdscrub: {why}"); + return ExitCode::from(code); } }; @@ -50,13 +49,21 @@ fn main() -> ExitCode { } } -fn read(path: Option<&Path>) -> std::io::Result { - match path { - Some(path) => std::fs::read_to_string(path), - None => { - let mut markdown = String::new(); - std::io::stdin().read_to_string(&mut markdown)?; - Ok(markdown) +/// Read the input, distinguishing a file that cannot be read from one that +/// is not text: the first is an I/O failure and the second is bad input, and +/// a caller routing on the exit code needs to tell them apart. +fn read(path: Option<&Path>) -> Result { + let bytes = match path { + Some(path) => { + std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display()))) } - } + None => { + let mut bytes = Vec::new(); + std::io::stdin() + .read_to_end(&mut bytes) + .map(|_| bytes) + .map_err(|e| (exit::IO, e.to_string())) + } + }?; + String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}"))) } diff --git a/crates/pdftriage/src/main.rs b/crates/pdftriage/src/main.rs index 2544af8..8540ef2 100644 --- a/crates/pdftriage/src/main.rs +++ b/crates/pdftriage/src/main.rs @@ -27,7 +27,6 @@ const SPAN: u32 = 5; #[command( version, about, - long_about = None, after_help = "\ Exit codes: 0 the layer needs OCR, 1 it does not, 2 could not tell, 69 pdftotext is not installed, 74 the file could not be read. diff --git a/crates/textkit/Cargo.toml b/crates/textkit/Cargo.toml index ebbbab5..30634d2 100644 --- a/crates/textkit/Cargo.toml +++ b/crates/textkit/Cargo.toml @@ -10,5 +10,8 @@ repository.workspace = true [dependencies] tiktoken-rs.workspace = true +[dev-dependencies] +regex.workspace = true + [lints] workspace = true diff --git a/crates/textkit/src/lib.rs b/crates/textkit/src/lib.rs index 21cffcf..0d28cab 100644 --- a/crates/textkit/src/lib.rs +++ b/crates/textkit/src/lib.rs @@ -11,6 +11,7 @@ //! matches what is already in an index. pub mod number; +pub mod pattern; pub mod segment; pub mod tokens; diff --git a/crates/textkit/src/pattern.rs b/crates/textkit/src/pattern.rs new file mode 100644 index 0000000..40cbebf --- /dev/null +++ b/crates/textkit/src/pattern.rs @@ -0,0 +1,59 @@ +//! Character classes that mean in Rust what they mean in Python. +//! +//! The two regex engines disagree about `\s`, `\w` and therefore `\b`, and +//! every disagreement is a heuristic that fires on one side and not the +//! other. The differences are small and entirely real: +//! +//! - Python's `\s` also matches `U+001C`–`U+001F`, the four ASCII +//! information separators. +//! - Python's `\w` is the letters, the numbers and the underscore. Rust's +//! adds every combining mark, every connector punctuation mark and the +//! join controls, and leaves out the letter-like and other numbers — so +//! `Gödel` written in decomposed form is one word to Rust and two to +//! Python, and `²` is a word character to Python and not to Rust. +//! - `\b` is defined in terms of `\w`, so it inherits all of that. +//! +//! Spell a ported pattern with these and it means the same thing on both +//! sides. The boundary forms are lookarounds, so a pattern using them needs +//! an engine that has them. + +/// Python's `\s`. +pub const SPACE: &str = r"[\s\x{1c}-\x{1f}]"; + +/// Python's `\w`. +pub const WORD: &str = r"[\p{L}\p{N}_]"; + +/// Python's `\b`, before a word. +pub const WORD_START: &str = r"(?` records +page 99,999,999,999. + +**Rust:** the page field is a 32-bit number and saturates at 4,294,967,295. + +**Why this is acceptable:** the number is a page number. A document with four +billion pages does not exist, and the alternative — a 64-bit field — would +make every chunk record wider to carry a value nothing can produce. The +saturation is deliberate rather than a wrap, so the wrong answer is at least +obviously wrong. + +**Measured:** no document in any corpus this was run against has a page +marker above five digits. + +## chunkmd — a hard maximum of zero yields nothing rather than raising + +**Python:** `_enforce_hard_max` with `hard_max=0` raises `ValueError: +range() arg 3 must not be zero`. + +**Rust:** `chunkmd::chunk` returns no chunks, since no piece could satisfy +the limit. + +**Why this is acceptable:** the command-line tool rejects it before reading +any input, so this is reachable only by a library caller, and a filter that +returns nothing for a budget that permits nothing is the more useful answer +of the two. Every other degenerate budget — overlap at or above target, a +hard maximum below the target, a minimum of zero — agrees with the Python +exactly. + +**Measured:** checked against the oracle for all of those combinations; only +the zero hard maximum differs.