mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
ported patterns mean in rust what they mean in python
This commit is contained in:
funkpower
Generated
+2
@@ -194,6 +194,7 @@ dependencies = [
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"cuekit",
|
||||
"fancy-regex 0.16.2",
|
||||
"regex",
|
||||
"schemars",
|
||||
"serde",
|
||||
@@ -474,6 +475,7 @@ dependencies = [
|
||||
name = "textkit"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"tiktoken-rs",
|
||||
]
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use fancy_regex::Regex as FancyRegex;
|
||||
use regex::Regex;
|
||||
use textkit::pattern::{SPACE, WORD_END, WORD_START};
|
||||
|
||||
/// A private-use codepoint marking the start of a note's text.
|
||||
///
|
||||
@@ -40,47 +40,50 @@ const MIN_CITED_NOTES: usize = 2;
|
||||
/// more spaces — several notes can share one physical line — so that a
|
||||
/// superscript *reference* to a note, mid-sentence, does not qualify.
|
||||
static NOTE_MARKER: LazyLock<FancyRegex> = LazyLock::new(|| {
|
||||
FancyRegex::new(
|
||||
FancyRegex::new(&format!(
|
||||
r"(?xm)
|
||||
(?: ^[ \t]{0,8} (?:-[ \t]+)? # line start, optional '- ' list bullet
|
||||
| (?<=\s\s) # or after a 2+ space run
|
||||
(?: ^[ \t]{{0,8}} (?:-[ \t]+)? # line start, optional '- ' list bullet
|
||||
| (?<={SPACE}{SPACE}) # or after a 2+ space run
|
||||
| (?<=\ -\ ) # or after ' - ' (inline list separator)
|
||||
)
|
||||
(?:
|
||||
\d{1,3} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. '
|
||||
| \[ \d{1,3} \] [ \t]* \S # bracket-numbered '[12] Author'
|
||||
| \d{1,3} [ \t]+ \S # bare '12 ' from a page bottom
|
||||
| [¹²³⁰-⁹]+ \s* \S # unicode superscript marker
|
||||
| (?:fn|ftn|note) \d{1,4} [.)] [ \t]+ \S # id-style 'fn39. '
|
||||
| (?:<span[^>]*>\s*)* # outer span wrap
|
||||
<a\b[^>]*> \s* (?:<span[^>]*>\s*)* (?:<sup>\s*)?
|
||||
\d{1,3} \s* [.)]?
|
||||
\s* (?:</sup>\s*)? (?:</span>\s*)* </a>
|
||||
(?:\s*</span>)* \s* [.)]? # anchor forms
|
||||
| \[ \s* (?:<span[^>]*>)? \s* \d{1,3}
|
||||
\s* (?:</span>)? \s* \] \( \#[^)]* \) # [42](#...)
|
||||
| <sup> \s* \d{1,3} \s* </sup> # bare sup note
|
||||
)",
|
||||
)
|
||||
\d{{1,3}} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. '
|
||||
| \[ \d{{1,3}} \] [ \t]* \S # bracket-numbered '[12] Author'
|
||||
| \d{{1,3}} [ \t]+ \S # bare '12 ' from a page bottom
|
||||
| [¹²³⁰-⁹]+ {SPACE}* \S # unicode superscript marker
|
||||
| (?:fn|ftn|note) \d{{1,4}} [.)] [ \t]+ \S # id-style 'fn39. '
|
||||
| (?:<span[^>]*>{SPACE}*)* # outer span wrap
|
||||
<a{WORD_END}[^>]*> {SPACE}* (?:<span[^>]*>{SPACE}*)* (?:<sup>{SPACE}*)?
|
||||
\d{{1,3}} {SPACE}* [.)]?
|
||||
{SPACE}* (?:</sup>{SPACE}*)? (?:</span>{SPACE}*)* </a>
|
||||
(?:{SPACE}*</span>)* {SPACE}* [.)]? # anchor forms
|
||||
| \[ {SPACE}* (?:<span[^>]*>)? {SPACE}* \d{{1,3}}
|
||||
{SPACE}* (?:</span>)? {SPACE}* \] \( \#[^)]* \) # [42](#...)
|
||||
| <sup> {SPACE}* \d{{1,3}} {SPACE}* </sup> # bare sup note
|
||||
)"
|
||||
))
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// What makes a note bibliographic rather than discursive.
|
||||
static CITE_SIGNAL: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
static CITE_SIGNAL: LazyLock<FancyRegex> = LazyLock::new(|| {
|
||||
FancyRegex::new(&format!(
|
||||
r"(?ix)
|
||||
\((?:[^()]*\b)?(?:1[5-9]|20)\d{2}\) # parenthesised year: (Oxford, 1998)
|
||||
| \bpp?\.\s*\d # p. 12 / pp. 12-34
|
||||
| \bvol\.\s*\d # vol. 2
|
||||
| \bIbid\b | \bop\.\s*cit\b | \bloc\.\s*cit\b
|
||||
| \bet\s+al\b
|
||||
| \b(?:University|Clarendon|Academic)\s+Press\b | \bPress,\s
|
||||
| \beds?\.\s | \btrans\.\s
|
||||
| \*[^*\n]{3,80}\*\s*,?\s*\d # short form: *Title*, 183
|
||||
\((?:[^()]*{WORD_END})?(?:1[5-9]|20)\d{{2}}\) # parenthesised year: (Oxford, 1998)
|
||||
| {WORD_START}pp?\.{SPACE}*\d # p. 12 / pp. 12-34
|
||||
| {WORD_START}vol\.{SPACE}*\d # vol. 2
|
||||
| {WORD_START}Ibid{WORD_END}
|
||||
| {WORD_START}op\.{SPACE}*cit{WORD_END}
|
||||
| {WORD_START}loc\.{SPACE}*cit{WORD_END}
|
||||
| {WORD_START}et{SPACE}+al{WORD_END}
|
||||
| {WORD_START}(?:University|Clarendon|Academic){SPACE}+Press{WORD_END}
|
||||
| {WORD_START}Press,{SPACE}
|
||||
| {WORD_START}eds?\.{SPACE} | {WORD_START}trans\.{SPACE}
|
||||
| \*[^*\n]{{3,80}}\*{SPACE}*,?{SPACE}*\d # short form: *Title*, 183
|
||||
| <span[^>]*italic[^>]*> # HTML-italic title survives conversion
|
||||
| ,\s+\d{1,4}\s*[.;] # trailing page: '..., 183.'
|
||||
| https?:// # web citations (URLs, DOIs)",
|
||||
)
|
||||
| ,{SPACE}+\d{{1,4}}{SPACE}*[.;] # trailing page: '..., 183.'
|
||||
| https?:// # web citations (URLs, DOIs)"
|
||||
))
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
@@ -107,7 +110,7 @@ pub fn looks_like_citations(text: &str) -> bool {
|
||||
for (at, start) in starts.iter().enumerate() {
|
||||
let end = starts.get(at + 1).copied().unwrap_or(text.len());
|
||||
let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect();
|
||||
if CITE_SIGNAL.is_match(&entry) {
|
||||
if CITE_SIGNAL.is_match(&entry).unwrap_or(false) {
|
||||
cited += 1;
|
||||
if cited >= MIN_CITED_NOTES {
|
||||
return true;
|
||||
|
||||
@@ -28,23 +28,24 @@ pub use pack::Budgets;
|
||||
pub fn chunk(markdown: &str, budgets: &Budgets) -> Vec<Chunk> {
|
||||
let pages = pages::split_by_page(markdown);
|
||||
let chapters = pages::chapters(markdown);
|
||||
let by_page = pages::page_chapters(&chapters);
|
||||
let by_offset = pages::ChapterIndex::new(&chapters);
|
||||
|
||||
let mut chunks: Vec<Chunk> = Vec::new();
|
||||
for page in pages {
|
||||
for packed in pack::page(page.text, budgets) {
|
||||
for piece in pack::enforce_hard_max(&packed, budgets.hard_max) {
|
||||
let tokens = textkit::count(&piece);
|
||||
for (piece, tokens) in pack::enforce_hard_max(&packed, budgets.hard_max) {
|
||||
if tokens < budgets.min_tokens {
|
||||
continue;
|
||||
}
|
||||
// A page-based chapter when the document has page markers;
|
||||
// the byte offset otherwise, which is what carries EPUBs
|
||||
// that have headings but no page anchors.
|
||||
// that have headings but no page anchors. Page zero counts as
|
||||
// no page at all, which is what the Python's truth test makes
|
||||
// of it.
|
||||
let chapter = page
|
||||
.number
|
||||
.and_then(|number| by_page.get(&number).copied())
|
||||
.filter(|number| *number != 0)
|
||||
.and_then(|number| pages::chapter_of_page(&chapters, number))
|
||||
.or_else(|| by_offset.at(page.offset));
|
||||
|
||||
let idx = chunks.len() as u32;
|
||||
|
||||
+18
-11
@@ -18,7 +18,6 @@ use corpus_core::{exit, ndjson};
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Example:
|
||||
|
||||
@@ -63,9 +62,9 @@ fn main() -> ExitCode {
|
||||
|
||||
let markdown = match read(args.path.as_deref()) {
|
||||
Ok(markdown) => markdown,
|
||||
Err(e) => {
|
||||
eprintln!("chunkmd: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
Err((code, why)) => {
|
||||
eprintln!("chunkmd: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -118,13 +117,21 @@ fn sane(budgets: &Budgets) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
|
||||
match path {
|
||||
Some(path) => std::fs::read_to_string(path),
|
||||
/// Read the input, distinguishing a file that cannot be read from one that
|
||||
/// is not text: the first is an I/O failure and the second is bad input, and
|
||||
/// a caller routing on the exit code needs to tell them apart.
|
||||
fn read(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
|
||||
let bytes = match path {
|
||||
Some(path) => {
|
||||
std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
|
||||
}
|
||||
None => {
|
||||
let mut markdown = String::new();
|
||||
std::io::stdin().read_to_string(&mut markdown)?;
|
||||
Ok(markdown)
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
std::io::stdin()
|
||||
.read_to_end(&mut bytes)
|
||||
.map(|_| bytes)
|
||||
.map_err(|e| (exit::IO, e.to_string()))
|
||||
}
|
||||
}?;
|
||||
String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
|
||||
}
|
||||
|
||||
@@ -100,12 +100,25 @@ fn carry_tail(buf: &[String], overlap: usize) -> Vec<String> {
|
||||
out
|
||||
}
|
||||
|
||||
/// Split anything still over the hard maximum, on token boundaries.
|
||||
/// Split anything still over the hard maximum, on token boundaries, and hand
|
||||
/// back each piece with its token count.
|
||||
///
|
||||
/// Defence in depth. The structural passes above should have handled it; this
|
||||
/// is what guarantees the promise regardless.
|
||||
pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<String> {
|
||||
/// is what guarantees the promise regardless. The counts come back with the
|
||||
/// pieces because they have just been computed, and counting them again is
|
||||
/// the second-largest cost in the whole chunker.
|
||||
pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<(String, usize)> {
|
||||
let tokens = textkit::count(text);
|
||||
if tokens <= hard_max {
|
||||
return vec![(text.to_string(), tokens)];
|
||||
}
|
||||
textkit::split_at_token_limit(text, hard_max)
|
||||
.into_iter()
|
||||
.map(|piece| {
|
||||
let tokens = textkit::count(&piece);
|
||||
(piece, tokens)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -179,8 +192,9 @@ mod tests {
|
||||
#[test]
|
||||
fn nothing_leaves_the_packer_over_the_hard_maximum() {
|
||||
let long = "word ".repeat(2000);
|
||||
for piece in enforce_hard_max(&long, 100) {
|
||||
assert!(textkit::count(&piece) <= 100);
|
||||
for (piece, tokens) in enforce_hard_max(&long, 100) {
|
||||
assert!(tokens <= 100);
|
||||
assert_eq!(tokens, textkit::count(&piece));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+28
-24
@@ -4,16 +4,18 @@
|
||||
//! and chapter boundaries as headings. Both are needed on every chunk: a
|
||||
//! passage a reader cannot locate in the book is a passage they cannot check.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use textkit::pattern::SPACE;
|
||||
|
||||
static PAGE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"<!--\s*Page\s+(\d+)\s*-->").expect("a valid pattern"));
|
||||
static PAGE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(&format!(r"<!--{SPACE}*Page{SPACE}+(\d+){SPACE}*-->")).expect("a valid pattern")
|
||||
});
|
||||
|
||||
static HEADING: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?m)^(#{1,3})\s+(.+)$").expect("a valid pattern"));
|
||||
static HEADING: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(&format!(r"(?m)^(#{{1,3}}){SPACE}+(.+)$")).expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// One page's worth of text, and where it started in the source.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -166,31 +168,33 @@ impl<'a> ChapterIndex<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Map each page number to the chapter covering it.
|
||||
/// The chapter covering a page, if any.
|
||||
///
|
||||
/// Where chapters overlap — which happens when two headings land on one page
|
||||
/// — the later one wins, because that is the one a reader scrolling down
|
||||
/// would be under.
|
||||
pub fn page_chapters(chapters: &[Chapter]) -> HashMap<u32, &str> {
|
||||
let mut out = HashMap::new();
|
||||
for chapter in chapters {
|
||||
///
|
||||
/// Scanned rather than indexed into a map of every page. A document has tens
|
||||
/// of chapters and a scan of them costs nothing, where a map costs one entry
|
||||
/// per page in every chapter's span: a single mistyped marker reading
|
||||
/// `<!-- Page 20231115 -->` would ask for a gigabyte of them.
|
||||
pub fn chapter_of_page(chapters: &[Chapter], page: u32) -> Option<&str> {
|
||||
chapters
|
||||
.iter()
|
||||
.filter(|chapter| {
|
||||
let Some(start) = chapter.start_page else {
|
||||
continue;
|
||||
return false;
|
||||
};
|
||||
// A zero end page means the same thing as no end page: this chapter
|
||||
// is one page long as far as anyone can tell.
|
||||
// A zero end page means the same thing as no end page: this
|
||||
// chapter is one page long as far as anyone can tell.
|
||||
let end = match chapter.end_page {
|
||||
Some(0) | None => start,
|
||||
Some(end) => end,
|
||||
};
|
||||
if end < start {
|
||||
continue;
|
||||
}
|
||||
for page in start..=end {
|
||||
out.insert(page, chapter.title.as_str());
|
||||
}
|
||||
}
|
||||
out
|
||||
(start..=end.max(start)).contains(&page) && end >= start
|
||||
})
|
||||
.next_back()
|
||||
.map(|chapter| chapter.title.as_str())
|
||||
}
|
||||
|
||||
fn trim(text: &str) -> &str {
|
||||
@@ -264,10 +268,10 @@ mod tests {
|
||||
assert_eq!(found[0].start_page, Some(1));
|
||||
assert_eq!(found[0].end_page, Some(4));
|
||||
assert_eq!(found[1].start_page, Some(4));
|
||||
let by_page = page_chapters(&found);
|
||||
assert_eq!(by_page.get(&1), Some(&"One"));
|
||||
assert_eq!(by_page.get(&3), Some(&"One"));
|
||||
assert_eq!(by_page.get(&4), Some(&"Two"));
|
||||
assert_eq!(chapter_of_page(&found, 1), Some("One"));
|
||||
assert_eq!(chapter_of_page(&found, 3), Some("One"));
|
||||
assert_eq!(chapter_of_page(&found, 4), Some("Two"));
|
||||
assert_eq!(chapter_of_page(&found, 9), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+49
-20
@@ -14,28 +14,46 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use corpus_core::Role;
|
||||
use fancy_regex::Regex as Boundaried;
|
||||
use regex::Regex;
|
||||
use textkit::pattern::{SPACE, WORD, WORD_END, WORD_START};
|
||||
|
||||
fn pattern(source: &str) -> Regex {
|
||||
Regex::new(source).expect("a valid pattern")
|
||||
}
|
||||
|
||||
/// A pattern that needs Python's word boundary, which is a lookaround here
|
||||
/// because Rust's own `\b` counts combining marks as word characters.
|
||||
fn bounded(source: &str) -> Boundaried {
|
||||
Boundaried::new(source).expect("a valid pattern")
|
||||
}
|
||||
|
||||
/// Chapter titles that give the section away, tried in this order.
|
||||
static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
|
||||
vec![
|
||||
(
|
||||
Role::Bibliography,
|
||||
pattern(
|
||||
r"(?i)(bibliograph|works\s+cited|literature\s+cited|further\s+reading|^\s*references\s*$|^\s*(end)?notes\s*$)",
|
||||
pattern(&format!(
|
||||
r"(?i)(bibliograph|works{SPACE}+cited|literature{SPACE}+cited|further{SPACE}+reading|^{SPACE}*references{SPACE}*$|^{SPACE}*(end)?notes{SPACE}*$)"
|
||||
)),
|
||||
),
|
||||
(
|
||||
Role::Index,
|
||||
pattern(&format!(
|
||||
r"(?i)^{SPACE}*index({SPACE}+of{SPACE}+{WORD}+)?{SPACE}*$"
|
||||
)),
|
||||
),
|
||||
(
|
||||
Role::Toc,
|
||||
pattern(&format!(
|
||||
r"(?i)^{SPACE}*(table{SPACE}+of{SPACE}+)?contents{SPACE}*$"
|
||||
)),
|
||||
),
|
||||
(Role::Index, pattern(r"(?i)^\s*index(\s+of\s+\w+)?\s*$")),
|
||||
(Role::Toc, pattern(r"(?i)^\s*(table\s+of\s+)?contents\s*$")),
|
||||
(
|
||||
Role::Frontmatter,
|
||||
pattern(
|
||||
r"(?i)(copyright|title\s*page|colophon|^\s*dedication\s*$|^\s*epigraph\s*$|about\s+the\s+(author|publisher)|acknowledg)",
|
||||
),
|
||||
pattern(&format!(
|
||||
r"(?i)(copyright|title{SPACE}*page|colophon|^{SPACE}*dedication{SPACE}*$|^{SPACE}*epigraph{SPACE}*$|about{SPACE}+the{SPACE}+(author|publisher)|acknowledg)"
|
||||
)),
|
||||
),
|
||||
]
|
||||
});
|
||||
@@ -43,29 +61,40 @@ static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
|
||||
/// A standalone bibliography heading inside the chunk's own text —
|
||||
/// "Bibliography", "## Works Cited", "Selected Bibliography".
|
||||
static INTEXT_BIBLIO_HEADING: LazyLock<Regex> = LazyLock::new(|| {
|
||||
pattern(
|
||||
r"(?im)^\s*#{0,6}\s*(select(?:ed)?\s+)?(bibliography|works\s+cited|references|sources)\s*$",
|
||||
)
|
||||
pattern(&format!(
|
||||
r"(?im)^{SPACE}*#{{0,6}}{SPACE}*(select(?:ed)?{SPACE}+)?(bibliography|works{SPACE}+cited|references|sources){SPACE}*$"
|
||||
))
|
||||
});
|
||||
|
||||
/// One bibliography entry: "Lastname, First…", or the dash run that stands
|
||||
/// for "the same author again".
|
||||
static BIBLIO_ENTRY: LazyLock<Regex> =
|
||||
LazyLock::new(|| pattern(r"^(?:[A-Z][\w'’\-]+,\s+\S|[—–-]{2,3}\s*[.,])"));
|
||||
static BIBLIO_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
pattern(&format!(
|
||||
r"^(?:[A-Z][\p{{L}}\p{{N}}_'’\-]+,{SPACE}+\S|[—–-]{{2,3}}{SPACE}*[.,])"
|
||||
))
|
||||
});
|
||||
|
||||
/// The continuation of a wrapped entry: "New York: Springer, 2013."
|
||||
static BIBLIO_CONTINUATION: LazyLock<Regex> =
|
||||
LazyLock::new(|| pattern(r"\b(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.)\b"));
|
||||
static BIBLIO_CONTINUATION: LazyLock<Boundaried> = LazyLock::new(|| {
|
||||
bounded(&format!(
|
||||
r"{WORD_START}(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.){WORD_END}"
|
||||
))
|
||||
});
|
||||
|
||||
static YEAR: LazyLock<Regex> = LazyLock::new(|| pattern(r"\b(1[5-9]\d{2}|20\d{2})\b"));
|
||||
static YEAR: LazyLock<Boundaried> =
|
||||
LazyLock::new(|| bounded(&format!(r"{WORD_START}(1[5-9]\d{{2}}|20\d{{2}}){WORD_END}")));
|
||||
|
||||
/// An index line: a short lead, then a list of page numbers.
|
||||
static INDEX_LINE: LazyLock<Regex> =
|
||||
LazyLock::new(|| pattern(r"^.{1,70}?,\s+[ivxlc\d]{1,4}(?:[,–-]\s*[ivxlc\d]{1,4})*\s*$"));
|
||||
static INDEX_LINE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
pattern(&format!(
|
||||
r"^.{{1,70}}?,{SPACE}+[ivxlc\d]{{1,4}}(?:[,–-]{SPACE}*[ivxlc\d]{{1,4}})*{SPACE}*$"
|
||||
))
|
||||
});
|
||||
|
||||
/// A contents line: dot leaders to a page number, or a link to an internal
|
||||
/// anchor, which is what a converted EPUB's table of contents becomes.
|
||||
static TOC_LINE: LazyLock<Regex> = LazyLock::new(|| pattern(r"(\.{3,}\s*\d{1,4}\s*$|\]\(#)"));
|
||||
static TOC_LINE: LazyLock<Regex> =
|
||||
LazyLock::new(|| pattern(&format!(r"(\.{{3,}}{SPACE}*\d{{1,4}}{SPACE}*$|\]\(#)")));
|
||||
|
||||
/// A wrapped entry long enough that its publisher name is probably a
|
||||
/// coincidence rather than a citation, in characters.
|
||||
@@ -139,8 +168,8 @@ pub fn classify(text: &str, chapter: Option<&str>) -> Role {
|
||||
}
|
||||
|
||||
fn looks_like_entry(paragraph: &str) -> bool {
|
||||
let year = YEAR.is_match(paragraph);
|
||||
let publisher = BIBLIO_CONTINUATION.is_match(paragraph);
|
||||
let year = YEAR.is_match(paragraph).unwrap_or(false);
|
||||
let publisher = BIBLIO_CONTINUATION.is_match(paragraph).unwrap_or(false);
|
||||
if BIBLIO_ENTRY.is_match(paragraph) && (year || publisher) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ use corpus_core::{exit, ndjson};
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Example:
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ repository.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
cuekit.workspace = true
|
||||
fancy-regex.workspace = true
|
||||
regex.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
@@ -21,7 +21,6 @@ use cuekit::Cue;
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Example:
|
||||
|
||||
|
||||
@@ -3,17 +3,22 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use textkit::pattern::{SPACE, WORD_END, WORD_START};
|
||||
|
||||
/// Hesitations. A chapter called "Um Yeah So I Think" is worse than no title.
|
||||
static FILLER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(um+|uh+|ah+|er+)\b").expect("a valid pattern"));
|
||||
static FILLER: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
|
||||
fancy_regex::Regex::new(&format!(r"(?i){WORD_START}(um+|uh+|ah+|er+){WORD_END}"))
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// The most common discourse marker in recorded speech.
|
||||
static YOU_KNOW: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\byou know\b").expect("a valid pattern"));
|
||||
static YOU_KNOW: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
|
||||
fancy_regex::Regex::new(&format!(r"(?i){WORD_START}you know{WORD_END}"))
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
static SPACE_RUN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
|
||||
LazyLock::new(|| Regex::new(&format!(r"{SPACE}+")).expect("a valid pattern"));
|
||||
|
||||
/// Anything that does not belong in a title.
|
||||
static NOT_TITLE: LazyLock<Regex> =
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use textkit::pattern::SPACE;
|
||||
|
||||
fn pattern(source: &str) -> Regex {
|
||||
Regex::new(source).expect("a valid pattern")
|
||||
@@ -21,23 +22,24 @@ static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"
|
||||
static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
|
||||
|
||||
/// A trailing `{.attr}` block.
|
||||
static ATTRIBUTE_TAIL: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s*\{[^}]*\}\s*$"));
|
||||
static ATTRIBUTE_TAIL: LazyLock<Regex> =
|
||||
LazyLock::new(|| pattern(&format!(r"{SPACE}*\{{[^}}]*\}}{SPACE}*$")));
|
||||
|
||||
/// Any inline tag. Broader than the one used on body text, because a heading
|
||||
/// has no code fences to protect.
|
||||
static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"<[^>]+>"));
|
||||
|
||||
static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s+"));
|
||||
static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(&format!(r"{SPACE}+")));
|
||||
|
||||
/// A heading line, and the text after its hashes.
|
||||
static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(r"^(#{1,6})\s+(.*)$"));
|
||||
static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(&format!(r"^(#{{1,6}}){SPACE}+(.*)$")));
|
||||
|
||||
/// A page anchor: `<span id="…page42">`, `<a id="page-42">`, and the rest of
|
||||
/// the shapes an EPUB uses to say where a printed page began.
|
||||
pub static PAGE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
|
||||
pattern(
|
||||
r#"(?i)<(?:span|a|div)\b[^>]*\bid\s*=\s*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#,
|
||||
)
|
||||
pattern(&format!(
|
||||
r#"(?i)<(?:span|a|div)\b[^>]*\bid{SPACE}*={SPACE}*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#
|
||||
))
|
||||
});
|
||||
|
||||
/// Clean one heading's text.
|
||||
|
||||
+18
-11
@@ -16,7 +16,6 @@ use corpus_core::exit;
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Example:
|
||||
|
||||
@@ -32,9 +31,9 @@ fn main() -> ExitCode {
|
||||
|
||||
let markdown = match read(args.path.as_deref()) {
|
||||
Ok(markdown) => markdown,
|
||||
Err(e) => {
|
||||
eprintln!("mdscrub: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
Err((code, why)) => {
|
||||
eprintln!("mdscrub: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -50,13 +49,21 @@ fn main() -> ExitCode {
|
||||
}
|
||||
}
|
||||
|
||||
fn read(path: Option<&Path>) -> std::io::Result<String> {
|
||||
match path {
|
||||
Some(path) => std::fs::read_to_string(path),
|
||||
/// Read the input, distinguishing a file that cannot be read from one that
|
||||
/// is not text: the first is an I/O failure and the second is bad input, and
|
||||
/// a caller routing on the exit code needs to tell them apart.
|
||||
fn read(path: Option<&Path>) -> Result<String, (u8, String)> {
|
||||
let bytes = match path {
|
||||
Some(path) => {
|
||||
std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
|
||||
}
|
||||
None => {
|
||||
let mut markdown = String::new();
|
||||
std::io::stdin().read_to_string(&mut markdown)?;
|
||||
Ok(markdown)
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
std::io::stdin()
|
||||
.read_to_end(&mut bytes)
|
||||
.map(|_| bytes)
|
||||
.map_err(|e| (exit::IO, e.to_string()))
|
||||
}
|
||||
}?;
|
||||
String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
|
||||
}
|
||||
|
||||
@@ -27,7 +27,6 @@ const SPAN: u32 = 5;
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Exit codes: 0 the layer needs OCR, 1 it does not, 2 could not tell,
|
||||
69 pdftotext is not installed, 74 the file could not be read.
|
||||
|
||||
@@ -10,5 +10,8 @@ repository.workspace = true
|
||||
[dependencies]
|
||||
tiktoken-rs.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
regex.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
//! matches what is already in an index.
|
||||
|
||||
pub mod number;
|
||||
pub mod pattern;
|
||||
pub mod segment;
|
||||
pub mod tokens;
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Character classes that mean in Rust what they mean in Python.
|
||||
//!
|
||||
//! The two regex engines disagree about `\s`, `\w` and therefore `\b`, and
|
||||
//! every disagreement is a heuristic that fires on one side and not the
|
||||
//! other. The differences are small and entirely real:
|
||||
//!
|
||||
//! - Python's `\s` also matches `U+001C`–`U+001F`, the four ASCII
|
||||
//! information separators.
|
||||
//! - Python's `\w` is the letters, the numbers and the underscore. Rust's
|
||||
//! adds every combining mark, every connector punctuation mark and the
|
||||
//! join controls, and leaves out the letter-like and other numbers — so
|
||||
//! `Gödel` written in decomposed form is one word to Rust and two to
|
||||
//! Python, and `²` is a word character to Python and not to Rust.
|
||||
//! - `\b` is defined in terms of `\w`, so it inherits all of that.
|
||||
//!
|
||||
//! Spell a ported pattern with these and it means the same thing on both
|
||||
//! sides. The boundary forms are lookarounds, so a pattern using them needs
|
||||
//! an engine that has them.
|
||||
|
||||
/// Python's `\s`.
|
||||
pub const SPACE: &str = r"[\s\x{1c}-\x{1f}]";
|
||||
|
||||
/// Python's `\w`.
|
||||
pub const WORD: &str = r"[\p{L}\p{N}_]";
|
||||
|
||||
/// Python's `\b`, before a word.
|
||||
pub const WORD_START: &str = r"(?<![\p{L}\p{N}_])";
|
||||
|
||||
/// Python's `\b`, after a word.
|
||||
pub const WORD_END: &str = r"(?![\p{L}\p{N}_])";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_space_class_covers_what_python_trims() {
|
||||
let space = regex::Regex::new(SPACE).expect("a valid pattern");
|
||||
for c in [' ', '\t', '\n', '\u{a0}', '\u{1c}', '\u{1f}', '\u{2028}'] {
|
||||
assert!(space.is_match(&c.to_string()), "{c:?}");
|
||||
assert!(crate::is_space(c), "{c:?}");
|
||||
}
|
||||
for c in ['a', '-', '\u{200b}'] {
|
||||
assert!(!space.is_match(&c.to_string()), "{c:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_word_class_is_letters_numbers_and_underscore() {
|
||||
let word = regex::Regex::new(&format!("^{WORD}$")).expect("a valid pattern");
|
||||
for c in ['a', 'Z', '0', '_', 'é', 'Ω', '²', 'Ⅷ'] {
|
||||
assert!(word.is_match(&c.to_string()), "{c:?}");
|
||||
}
|
||||
// The combining marks Rust's own `\w` would accept.
|
||||
for c in ['\u{301}', '\u{363}', '-', ' '] {
|
||||
assert!(!word.is_match(&c.to_string()), "{c:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
+54
-18
@@ -58,26 +58,62 @@ emitting chunks with no page number, which is the honest answer.
|
||||
only reachable from the PDF conversion path. Covered by
|
||||
`mdscrub::pages::tests::text_with_no_page_boundaries_is_left_exactly_as_it_was`.
|
||||
|
||||
## chunkmd, role and footnote patterns — `\s` excludes four control characters
|
||||
## Every ported pattern — the two engines disagree about `\d` and `(?i)`
|
||||
|
||||
**Python:** `\s` in a `str` pattern matches the Unicode `White_Space`
|
||||
property *and* `U+001C`–`U+001F`, the four ASCII information separators.
|
||||
**Python:** `re` compiles against the Unicode version of the interpreter;
|
||||
`(?i)` folds `ı` and `İ` onto `i`.
|
||||
|
||||
**Rust:** `regex` and `fancy-regex` match the `White_Space` property only, so
|
||||
those four characters are not whitespace to a ported pattern.
|
||||
**Rust:** `regex` compiles against the Unicode version it was built with, and
|
||||
`(?i)` uses simple case folding, which leaves the dotless and dotted `i`
|
||||
alone.
|
||||
|
||||
**Why this is acceptable:** every one of those codepoints is deleted by
|
||||
`mdscrub` before a chunker sees it — they fall inside the control-character
|
||||
class that a text column rejects. For the pattern to behave differently, a
|
||||
document would have to reach `chunkmd` without having been cleaned, and carry
|
||||
an information separator in exactly the position where a pattern expects
|
||||
whitespace.
|
||||
**Why this is acceptable:** `\s`, `\w` and `\b` were the differences that
|
||||
mattered and they are closed — every ported pattern now spells Python's
|
||||
classes explicitly, from `textkit::pattern`. What is left is a `\d` that
|
||||
disagrees only on decimal digits added between the two Unicode versions
|
||||
(Garay, added in Unicode 16, is the current example), and a case fold that
|
||||
disagrees only for Turkish dotless `i`. A chapter heading reading
|
||||
`Bıblıography` is classified as body text here and as a bibliography there;
|
||||
a page marker written in Garay digits reads here and does not there.
|
||||
|
||||
Note that this does **not** apply to the sentence and paragraph splitters in
|
||||
`textkit`, which match Python's definition exactly — see
|
||||
`textkit::segment::is_space`. Those decide chunk boundaries, so the four
|
||||
characters are handled there rather than argued about.
|
||||
Both will drift again at every dependency bump, which is the real reason this
|
||||
entry exists: the golden files pin one pair of Unicode versions, and a change
|
||||
to either shows up as a parity failure rather than as a silent difference.
|
||||
|
||||
**Measured:** zero differences over the parity corpus; the sample in
|
||||
`parity/src/sample.rs` includes `U+001C` deliberately and the token counts
|
||||
agree over all ten thousand lines.
|
||||
**Measured:** zero differences over the parity corpus. Found by brute-forcing
|
||||
every code point below `U+11000` through both engines: `\d` differs on ten,
|
||||
`(?i)` on three.
|
||||
|
||||
## chunkmd — a page number too large for the field saturates
|
||||
|
||||
**Python:** integers are unbounded, so `<!-- Page 99999999999 -->` records
|
||||
page 99,999,999,999.
|
||||
|
||||
**Rust:** the page field is a 32-bit number and saturates at 4,294,967,295.
|
||||
|
||||
**Why this is acceptable:** the number is a page number. A document with four
|
||||
billion pages does not exist, and the alternative — a 64-bit field — would
|
||||
make every chunk record wider to carry a value nothing can produce. The
|
||||
saturation is deliberate rather than a wrap, so the wrong answer is at least
|
||||
obviously wrong.
|
||||
|
||||
**Measured:** no document in any corpus this was run against has a page
|
||||
marker above five digits.
|
||||
|
||||
## chunkmd — a hard maximum of zero yields nothing rather than raising
|
||||
|
||||
**Python:** `_enforce_hard_max` with `hard_max=0` raises `ValueError:
|
||||
range() arg 3 must not be zero`.
|
||||
|
||||
**Rust:** `chunkmd::chunk` returns no chunks, since no piece could satisfy
|
||||
the limit.
|
||||
|
||||
**Why this is acceptable:** the command-line tool rejects it before reading
|
||||
any input, so this is reachable only by a library caller, and a filter that
|
||||
returns nothing for a budget that permits nothing is the more useful answer
|
||||
of the two. Every other degenerate budget — overlap at or above target, a
|
||||
hard maximum below the target, a minimum of zero — agrees with the Python
|
||||
exactly.
|
||||
|
||||
**Measured:** checked against the oracle for all of those combinations; only
|
||||
the zero hard maximum differs.
|
||||
|
||||
Reference in New Issue
Block a user