From bd0e1ed521a5f5f098012081beea5f28c684e270 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sun, 20 Sep 2026 18:37:13 -0400 Subject: [PATCH] chunkmd: a bibliography converted into a bulleted list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An entry opening was a surname at the start of a line, and a converted bibliography is often a bulleted list — so the bullet sat exactly where the surname was looked for and not one entry was seen: • Skorupski, J. 1995. 'Agent-Neutrality, Consequentialism, Utilitarianism', Utilitas, 7: 49-54. The whole page went unflagged, no chunk of it was ever selected, and books whose only bibliography arrived this way came back with no citations at all. A bullet may now stand where the surname would. Measured against the corpus: of 300 sampled body chunks that open lines on a bullet and an inverted name, 0 were flagged before and 223 after. Across 3,000 randomly sampled unflagged body chunks, the change fires on none — it buys the bulleted list and nothing else. --- crates/chunkmd/README.md | 6 ++++++ crates/chunkmd/src/footnote.rs | 25 ++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/chunkmd/README.md b/crates/chunkmd/README.md index 2c043a4..1176b54 100644 --- a/crates/chunkmd/README.md +++ b/crates/chunkmd/README.md @@ -104,6 +104,12 @@ A chunk whose text is a page of numbered notes is flagged `footnote_citations`, because footnotes are citations in disguise — they read as body text, rank as body text, and answer a query with a publisher and a page number. +A run of entries with nothing numbering them counts too, because a reference +list often arrives with its markup gone: three or more lines that open on an +inverted name and each carry a year, a page range or a publisher. The opening +may be a bullet — a converted bibliography is frequently a bulleted list, and +the bullet sits exactly where the surname would. + ## The roles | Role | What it is | diff --git a/crates/chunkmd/src/footnote.rs b/crates/chunkmd/src/footnote.rs index 726e5c9..bfe887f 100644 --- a/crates/chunkmd/src/footnote.rs +++ b/crates/chunkmd/src/footnote.rs @@ -124,9 +124,13 @@ pub fn cites_something(entry: &str) -> bool { /// or a given name. Deliberately narrow, because it is being counted across /// a page of body prose and a loose version would match dialogue. static ENTRY_OPENING: LazyLock = LazyLock::new(|| { - FancyRegex::new( - r"(?m)^[ \t]{0,12}\p{Lu}[\p{L}'\u{2019}\-]{1,25},[ \t]+(?:\p{Lu}\.|\p{Lu}[\p{L}]{1,20}[,.]?[ \t])", - ) + // A converted bibliography is often a bulleted list, and the bullet sits + // where the surname would: `• Skorupski, J. 1995. Ethical Explorations.` + FancyRegex::new(concat!( + r"(?m)^[ \t]{0,12}", + r"(?:[\u{2022}\u{00B7}\u{25AA}\u{25CF}*+\u{2013}\u{2014}-][ \t]+)?", + r"\p{Lu}[\p{L}'\u{2019}\-]{1,25},[ \t]+(?:\p{Lu}\.|\p{Lu}[\p{L}]{1,20}[,.]?[ \t])", + )) .expect("a valid pattern") }); @@ -258,6 +262,21 @@ pub fn strip_sentinel(text: &str) -> String { #[cfg(test)] mod tests { + #[test] + fn a_bibliography_converted_into_a_bulleted_list_is_still_one() { + // The bullet sits where the surname would, so every entry opening + // was missed and the whole page went unflagged — which meant the + // reader never saw a word of it. + let page = "\u{2022} Skorupski, J. 1995. 'Agent-Neutrality, Consequentialism, \ + Utilitarianism', Utilitas, 7: 49-54.\n\ + \u{2022} Slote, M. 1984. 'Satisficing Consequentialism', Proceedings \ + of the Aristotelian Society, 58: 139-63.\n\ + \u{2022} Smart, J. J. C. 1973. 'An Outline of a System of Utilitarian \ + Ethics'. Cambridge: Cambridge University Press.\n"; + assert!(looks_like_reference_run(page)); + assert!(looks_like_citations(page)); + } + #[test] fn a_reading_list_with_nothing_marking_it_is_citation_material() { // The end of a chapter in a reader: no heading, no numbers, just the