funkpower/catena
1
0
Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-10-23 03:07:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

ported patterns mean in rust what they mean in python

Browse Source
This commit is contained in:
funkpower
2026-09-19 12:26:41 -04:00
parent 17a03b0087
commit 339f402610
18 changed files with 322 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chunkmd/src/footnote.rs
+37 -34
View File
@@ -8,7 +8,7 @@
use std::sync::LazyLock;
use fancy_regex::Regex as FancyRegex;
use regex::Regex;
use textkit::pattern::{SPACE, WORD_END, WORD_START};
/// A private-use codepoint marking the start of a note's text.
///
@@ -40,47 +40,50 @@ const MIN_CITED_NOTES: usize = 2;
/// more spaces — several notes can share one physical line — so that a
/// superscript *reference* to a note, mid-sentence, does not qualify.
static NOTE_MARKER: LazyLock<FancyRegex> = LazyLock::new(|| {
FancyRegex::new(
FancyRegex::new(&format!(
r"(?xm)
(?: ^[ \t]{0,8} (?:-[ \t]+)? # line start, optional '- ' list bullet
| (?<=\s\s) # or after a 2+ space run
(?: ^[ \t]{{0,8}} (?:-[ \t]+)? # line start, optional '- ' list bullet
| (?<={SPACE}{SPACE}) # or after a 2+ space run
| (?<=\ -\ ) # or after ' - ' (inline list separator)
)
(?:
\d{1,3} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. '
| \[ \d{1,3} \] [ \t]* \S # bracket-numbered '[12] Author'
| \d{1,3} [ \t]+ \S # bare '12 ' from a page bottom
| [¹²³⁰-⁹]+ \s* \S # unicode superscript marker
| (?:fn|ftn|note) \d{1,4} [.)] [ \t]+ \S # id-style 'fn39. '
| (?:<span[^>]*>\s*)* # outer span wrap
<a\b[^>]*> \s* (?:<span[^>]*>\s*)* (?:<sup>\s*)?
\d{1,3} \s* [.)]?
\s* (?:</sup>\s*)? (?:</span>\s*)* </a>
(?:\s*</span>)* \s* [.)]? # anchor forms
| \[ \s* (?:<span[^>]*>)? \s* \d{1,3}
\s* (?:</span>)? \s* \] \( \#[^)]* \) # [42](#...)
| <sup> \s* \d{1,3} \s* </sup> # bare sup note
)",
)
\d{{1,3}} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. '
| \[ \d{{1,3}} \] [ \t]* \S # bracket-numbered '[12] Author'
| \d{{1,3}} [ \t]+ \S # bare '12 ' from a page bottom
| [¹²³⁰-⁹]+ {SPACE}* \S # unicode superscript marker
| (?:fn|ftn|note) \d{{1,4}} [.)] [ \t]+ \S # id-style 'fn39. '
| (?:<span[^>]*>{SPACE}*)* # outer span wrap
<a{WORD_END}[^>]*> {SPACE}* (?:<span[^>]*>{SPACE}*)* (?:<sup>{SPACE}*)?
\d{{1,3}} {SPACE}* [.)]?
{SPACE}* (?:</sup>{SPACE}*)? (?:</span>{SPACE}*)* </a>
(?:{SPACE}*</span>)* {SPACE}* [.)]? # anchor forms
| \[ {SPACE}* (?:<span[^>]*>)? {SPACE}* \d{{1,3}}
{SPACE}* (?:</span>)? {SPACE}* \] \( \#[^)]* \) # [42](#...)
| <sup> {SPACE}* \d{{1,3}} {SPACE}* </sup> # bare sup note
)"
))
.expect("a valid pattern")
});
/// What makes a note bibliographic rather than discursive.
static CITE_SIGNAL: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(
static CITE_SIGNAL: LazyLock<FancyRegex> = LazyLock::new(|| {
FancyRegex::new(&format!(
r"(?ix)
\((?:[^()]*\b)?(?:1[5-9]|20)\d{2}\) # parenthesised year: (Oxford, 1998)
| \bpp?\.\s*\d # p. 12 / pp. 12-34
| \bvol\.\s*\d # vol. 2
| \bIbid\b | \bop\.\s*cit\b | \bloc\.\s*cit\b
| \bet\s+al\b
| \b(?:University|Clarendon|Academic)\s+Press\b | \bPress,\s
| \beds?\.\s | \btrans\.\s
| \*[^*\n]{3,80}\*\s*,?\s*\d # short form: *Title*, 183
| <span[^>]*italic[^>]*> # HTML-italic title survives conversion
| ,\s+\d{1,4}\s*[.;] # trailing page: '..., 183.'
| https?:// # web citations (URLs, DOIs)",
)
\((?:[^()]*{WORD_END})?(?:1[5-9]|20)\d{{2}}\) # parenthesised year: (Oxford, 1998)
| {WORD_START}pp?\.{SPACE}*\d # p. 12 / pp. 12-34
| {WORD_START}vol\.{SPACE}*\d # vol. 2
| {WORD_START}Ibid{WORD_END}
| {WORD_START}op\.{SPACE}*cit{WORD_END}
| {WORD_START}loc\.{SPACE}*cit{WORD_END}
| {WORD_START}et{SPACE}+al{WORD_END}
| {WORD_START}(?:University|Clarendon|Academic){SPACE}+Press{WORD_END}
| {WORD_START}Press,{SPACE}
| {WORD_START}eds?\.{SPACE} | {WORD_START}trans\.{SPACE}
| \*[^*\n]{{3,80}}\*{SPACE}*,?{SPACE}*\d # short form: *Title*, 183
| <span[^>]*italic[^>]*> # HTML-italic title survives conversion
| ,{SPACE}+\d{{1,4}}{SPACE}*[.;] # trailing page: '..., 183.'
| https?:// # web citations (URLs, DOIs)"
))
.expect("a valid pattern")
});
@@ -107,7 +110,7 @@ pub fn looks_like_citations(text: &str) -> bool {
for (at, start) in starts.iter().enumerate() {
let end = starts.get(at + 1).copied().unwrap_or(text.len());
let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect();
if CITE_SIGNAL.is_match(&entry) {
if CITE_SIGNAL.is_match(&entry).unwrap_or(false) {
cited += 1;
if cited >= MIN_CITED_NOTES {
return true;
crates/chunkmd/src/lib.rs
+6 -5
View File
@@ -28,23 +28,24 @@ pub use pack::Budgets;
pub fn chunk(markdown: &str, budgets: &Budgets) -> Vec<Chunk> {
let pages = pages::split_by_page(markdown);
let chapters = pages::chapters(markdown);
let by_page = pages::page_chapters(&chapters);
let by_offset = pages::ChapterIndex::new(&chapters);
let mut chunks: Vec<Chunk> = Vec::new();
for page in pages {
for packed in pack::page(page.text, budgets) {
for piece in pack::enforce_hard_max(&packed, budgets.hard_max) {
let tokens = textkit::count(&piece);
for (piece, tokens) in pack::enforce_hard_max(&packed, budgets.hard_max) {
if tokens < budgets.min_tokens {
continue;
}
// A page-based chapter when the document has page markers;
// the byte offset otherwise, which is what carries EPUBs
// that have headings but no page anchors.
// that have headings but no page anchors. Page zero counts as
// no page at all, which is what the Python's truth test makes
// of it.
let chapter = page
.number
.and_then(|number| by_page.get(&number).copied())
.filter(|number| *number != 0)
.and_then(|number| pages::chapter_of_page(&chapters, number))
.or_else(|| by_offset.at(page.offset));
let idx = chunks.len() as u32;
crates/chunkmd/src/main.rs
+19 -12
View File
@@ -18,7 +18,6 @@ use corpus_core::{exit, ndjson};
#[command(
version,
about,
long_about = None,
after_help = "\
Example:
@@ -63,9 +62,9 @@ fn main() -> ExitCode {
let markdown = match read(args.path.as_deref()) {
Ok(markdown) => markdown,
Err(e) => {
eprintln!("chunkmd: {e}");
return ExitCode::from(exit::IO);
Err((code, why)) => {
eprintln!("chunkmd: {why}");
return ExitCode::from(code);
}
};
@@ -118,13 +117,21 @@ fn sane(budgets: &Budgets) -> Result<(), String> {
Ok(())
}
fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
match path {
Some(path) => std::fs::read_to_string(path),
None => {
let mut markdown = String::new();
std::io::stdin().read_to_string(&mut markdown)?;
Ok(markdown)
/// Read the input, distinguishing a file that cannot be read from one that
/// is not text: the first is an I/O failure and the second is bad input, and
/// a caller routing on the exit code needs to tell them apart.
fn read(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
let bytes = match path {
Some(path) => {
std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
}
}
None => {
let mut bytes = Vec::new();
std::io::stdin()
.read_to_end(&mut bytes)
.map(|_| bytes)
.map_err(|e| (exit::IO, e.to_string()))
}
}?;
String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
}
crates/chunkmd/src/pack.rs
+19 -5
View File
@@ -100,12 +100,25 @@ fn carry_tail(buf: &[String], overlap: usize) -> Vec<String> {
out
}
/// Split anything still over the hard maximum, on token boundaries.
/// Split anything still over the hard maximum, on token boundaries, and hand
/// back each piece with its token count.
///
/// Defence in depth. The structural passes above should have handled it; this
/// is what guarantees the promise regardless.
pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<String> {
/// is what guarantees the promise regardless. The counts come back with the
/// pieces because they have just been computed, and counting them again is
/// the second-largest cost in the whole chunker.
pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<(String, usize)> {
let tokens = textkit::count(text);
if tokens <= hard_max {
return vec![(text.to_string(), tokens)];
}
textkit::split_at_token_limit(text, hard_max)
.into_iter()
.map(|piece| {
let tokens = textkit::count(&piece);
(piece, tokens)
})
.collect()
}
#[cfg(test)]
@@ -179,8 +192,9 @@ mod tests {
#[test]
fn nothing_leaves_the_packer_over_the_hard_maximum() {
let long = "word ".repeat(2000);
for piece in enforce_hard_max(&long, 100) {
assert!(textkit::count(&piece) <= 100);
for (piece, tokens) in enforce_hard_max(&long, 100) {
assert!(tokens <= 100);
assert_eq!(tokens, textkit::count(&piece));
}
}
}
crates/chunkmd/src/pages.rs
+34 -30
View File
@@ -4,16 +4,18 @@
//! and chapter boundaries as headings. Both are needed on every chunk: a
//! passage a reader cannot locate in the book is a passage they cannot check.
use std::collections::HashMap;
use std::sync::LazyLock;
use regex::Regex;
use textkit::pattern::SPACE;
static PAGE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"<!--\s*Page\s+(\d+)\s*-->").expect("a valid pattern"));
static PAGE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(&format!(r"<!--{SPACE}*Page{SPACE}+(\d+){SPACE}*-->")).expect("a valid pattern")
});
static HEADING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?m)^(#{1,3})\s+(.+)$").expect("a valid pattern"));
static HEADING: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(&format!(r"(?m)^(#{{1,3}}){SPACE}+(.+)$")).expect("a valid pattern")
});
/// One page's worth of text, and where it started in the source.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -166,31 +168,33 @@ impl<'a> ChapterIndex<'a> {
}
}
/// Map each page number to the chapter covering it.
/// The chapter covering a page, if any.
///
/// Where chapters overlap — which happens when two headings land on one page
/// — the later one wins, because that is the one a reader scrolling down
/// would be under.
pub fn page_chapters(chapters: &[Chapter]) -> HashMap<u32, &str> {
let mut out = HashMap::new();
for chapter in chapters {
let Some(start) = chapter.start_page else {
continue;
};
// A zero end page means the same thing as no end page: this chapter
// is one page long as far as anyone can tell.
let end = match chapter.end_page {
Some(0) | None => start,
Some(end) => end,
};
if end < start {
continue;
}
for page in start..=end {
out.insert(page, chapter.title.as_str());
}
}
out
///
/// Scanned rather than indexed into a map of every page. A document has tens
/// of chapters and a scan of them costs nothing, where a map costs one entry
/// per page in every chapter's span: a single mistyped marker reading
/// `<!-- Page 20231115 -->` would ask for a gigabyte of them.
pub fn chapter_of_page(chapters: &[Chapter], page: u32) -> Option<&str> {
chapters
.iter()
.filter(|chapter| {
let Some(start) = chapter.start_page else {
return false;
};
// A zero end page means the same thing as no end page: this
// chapter is one page long as far as anyone can tell.
let end = match chapter.end_page {
Some(0) | None => start,
Some(end) => end,
};
(start..=end.max(start)).contains(&page) && end >= start
})
.next_back()
.map(|chapter| chapter.title.as_str())
}
fn trim(text: &str) -> &str {
@@ -264,10 +268,10 @@ mod tests {
assert_eq!(found[0].start_page, Some(1));
assert_eq!(found[0].end_page, Some(4));
assert_eq!(found[1].start_page, Some(4));
let by_page = page_chapters(&found);
assert_eq!(by_page.get(&1), Some(&"One"));
assert_eq!(by_page.get(&3), Some(&"One"));
assert_eq!(by_page.get(&4), Some(&"Two"));
assert_eq!(chapter_of_page(&found, 1), Some("One"));
assert_eq!(chapter_of_page(&found, 3), Some("One"));
assert_eq!(chapter_of_page(&found, 4), Some("Two"));
assert_eq!(chapter_of_page(&found, 9), None);
}
#[test]
crates/chunkmd/src/role.rs
+50 -21
View File
@@ -14,28 +14,46 @@
use std::sync::LazyLock;
use corpus_core::Role;
use fancy_regex::Regex as Boundaried;
use regex::Regex;
use textkit::pattern::{SPACE, WORD, WORD_END, WORD_START};
fn pattern(source: &str) -> Regex {
Regex::new(source).expect("a valid pattern")
}
/// A pattern that needs Python's word boundary, which is a lookaround here
/// because Rust's own `\b` counts combining marks as word characters.
fn bounded(source: &str) -> Boundaried {
Boundaried::new(source).expect("a valid pattern")
}
/// Chapter titles that give the section away, tried in this order.
static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
vec![
(
Role::Bibliography,
pattern(
r"(?i)(bibliograph|works\s+cited|literature\s+cited|further\s+reading|^\s*references\s*$|^\s*(end)?notes\s*$)",
),
pattern(&format!(
r"(?i)(bibliograph|works{SPACE}+cited|literature{SPACE}+cited|further{SPACE}+reading|^{SPACE}*references{SPACE}*$|^{SPACE}*(end)?notes{SPACE}*$)"
)),
),
(
Role::Index,
pattern(&format!(
r"(?i)^{SPACE}*index({SPACE}+of{SPACE}+{WORD}+)?{SPACE}*$"
)),
),
(
Role::Toc,
pattern(&format!(
r"(?i)^{SPACE}*(table{SPACE}+of{SPACE}+)?contents{SPACE}*$"
)),
),
(Role::Index, pattern(r"(?i)^\s*index(\s+of\s+\w+)?\s*$")),
(Role::Toc, pattern(r"(?i)^\s*(table\s+of\s+)?contents\s*$")),
(
Role::Frontmatter,
pattern(
r"(?i)(copyright|title\s*page|colophon|^\s*dedication\s*$|^\s*epigraph\s*$|about\s+the\s+(author|publisher)|acknowledg)",
),
pattern(&format!(
r"(?i)(copyright|title{SPACE}*page|colophon|^{SPACE}*dedication{SPACE}*$|^{SPACE}*epigraph{SPACE}*$|about{SPACE}+the{SPACE}+(author|publisher)|acknowledg)"
)),
),
]
});
@@ -43,29 +61,40 @@ static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
/// A standalone bibliography heading inside the chunk's own text —
/// "Bibliography", "## Works Cited", "Selected Bibliography".
static INTEXT_BIBLIO_HEADING: LazyLock<Regex> = LazyLock::new(|| {
pattern(
r"(?im)^\s*#{0,6}\s*(select(?:ed)?\s+)?(bibliography|works\s+cited|references|sources)\s*$",
)
pattern(&format!(
r"(?im)^{SPACE}*#{{0,6}}{SPACE}*(select(?:ed)?{SPACE}+)?(bibliography|works{SPACE}+cited|references|sources){SPACE}*$"
))
});
/// One bibliography entry: "Lastname, First…", or the dash run that stands
/// for "the same author again".
static BIBLIO_ENTRY: LazyLock<Regex> =
LazyLock::new(|| pattern(r"^(?:[A-Z][\w'’\-]+,\s+\S|[—–-]{2,3}\s*[.,])"));
static BIBLIO_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
pattern(&format!(
r"^(?:[A-Z][\p{{L}}\p{{N}}_'’\-]+,{SPACE}+\S|[—–-]{{2,3}}{SPACE}*[.,])"
))
});
/// The continuation of a wrapped entry: "New York: Springer, 2013."
static BIBLIO_CONTINUATION: LazyLock<Regex> =
LazyLock::new(|| pattern(r"\b(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.)\b"));
static BIBLIO_CONTINUATION: LazyLock<Boundaried> = LazyLock::new(|| {
bounded(&format!(
r"{WORD_START}(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.){WORD_END}"
))
});
static YEAR: LazyLock<Regex> = LazyLock::new(|| pattern(r"\b(1[5-9]\d{2}|20\d{2})\b"));
static YEAR: LazyLock<Boundaried> =
LazyLock::new(|| bounded(&format!(r"{WORD_START}(1[5-9]\d{{2}}|20\d{{2}}){WORD_END}")));
/// An index line: a short lead, then a list of page numbers.
static INDEX_LINE: LazyLock<Regex> =
LazyLock::new(|| pattern(r"^.{1,70}?,\s+[ivxlc\d]{1,4}(?:[,–-]\s*[ivxlc\d]{1,4})*\s*$"));
static INDEX_LINE: LazyLock<Regex> = LazyLock::new(|| {
pattern(&format!(
r"^.{{1,70}}?,{SPACE}+[ivxlc\d]{{1,4}}(?:[,–-]{SPACE}*[ivxlc\d]{{1,4}})*{SPACE}*$"
))
});
/// A contents line: dot leaders to a page number, or a link to an internal
/// anchor, which is what a converted EPUB's table of contents becomes.
static TOC_LINE: LazyLock<Regex> = LazyLock::new(|| pattern(r"(\.{3,}\s*\d{1,4}\s*$|\]\(#)"));
static TOC_LINE: LazyLock<Regex> =
LazyLock::new(|| pattern(&format!(r"(\.{{3,}}{SPACE}*\d{{1,4}}{SPACE}*$|\]\(#)")));
/// A wrapped entry long enough that its publisher name is probably a
/// coincidence rather than a citation, in characters.
@@ -139,8 +168,8 @@ pub fn classify(text: &str, chapter: Option<&str>) -> Role {
}
fn looks_like_entry(paragraph: &str) -> bool {
let year = YEAR.is_match(paragraph);
let publisher = BIBLIO_CONTINUATION.is_match(paragraph);
let year = YEAR.is_match(paragraph).unwrap_or(false);
let publisher = BIBLIO_CONTINUATION.is_match(paragraph).unwrap_or(false);
if BIBLIO_ENTRY.is_match(paragraph) && (year || publisher) {
return true;
}