From c92852378f39d326b0b2114efc53eff86532f9ae Mon Sep 17 00:00:00 2001 From: funkpower Date: Sun, 20 Sep 2026 11:47:10 -0400 Subject: [PATCH] citefind: an author-date note carries none of the punctuation chunkmd looks for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Altmann, M. 1951, A Study of Behaviour. Sociometry 14, 351-354.` names an author, a year, a journal and a page range, and has no bracketed year, no `p.`, no `ed.` and no `Press,` in it — so the test chunkmd uses to spot a page of notes says it cites nothing, and it was being dropped before it was read. citefind adds its own test alongside, for the same reason it added its own marker: chunkmd is deciding whether a chunk is a page of notes and can afford to miss one entry in thirty, and this is deciding whether a reference exists. Coverage of the model's citations rises to 84% on bibliographies. The verse pattern now covers the forms a real page of them uses: a chapter and verse with no book named, a note number glued into the middle of the run, a chapter range, and a semicolon inside a verse list. --- crates/citefind/src/parse.rs | 30 ++++++++++++++++++++++++++---- crates/citefind/src/split.rs | 27 ++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/crates/citefind/src/parse.rs b/crates/citefind/src/parse.rs index 744077b..aaa3a1e 100644 --- a/crates/citefind/src/parse.rs +++ b/crates/citefind/src/parse.rs @@ -135,11 +135,22 @@ static NOT_A_TITLE: LazyLock = LazyLock::new(|| { /// a theological book largely are. static VERSE: LazyLock = LazyLock::new(|| { Regex::new( - r"(?i)^(?:cf\.?|see|and)?\s*(?:[1-4]\s+)?\p{L}[\p{L}.]{2,14}\.?\s+\d{1,3}(?::\d{1,3}(?:[-–]\d{1,3})?)?\.?$", + r"(?ix) + ^(?:(?:also|cf|see|and|vv?)\.?\s+)* # 'Cf.', 'also cf.', 'see' + (?:[1-4]\s+)? # 1 Peter, 2 Corinthians + (?:\p{L}[\p{L}.]{2,14}\.?\s+)? # the book, where it is named + \d{1,3}(?:[:.]\d{1,3})? # chapter, and verse + (?:[-–]\d{1,3})? # through to another + \.?$", ) .expect("a valid pattern") }); +/// A note number sitting inside a run of verses, which is where conversion +/// leaves it when several notes share one line. +static EMBEDDED_MARKER: LazyLock = + LazyLock::new(|| Regex::new(r"\s\d{1,3}\.\s").expect("a valid pattern")); + /// A page reference or a bracketed imprint: what a citation has and a remark /// about the argument does not. static CITATION_ANCHOR: LazyLock = LazyLock::new(|| { @@ -457,12 +468,20 @@ fn strip_markup(text: &str) -> String { /// and still not a work: `Cf. Deuteronomy 18:15-19; Isaiah 53; Acts 1:6`. fn is_scripture(text: &str) -> bool { let text = text.trim_start_matches(['.', ' ']); + let text = EMBEDDED_MARKER.replace_all(text, ";"); let parts: Vec<&str> = text - .split(';') + .split([';', ',']) .map(str::trim) .filter(|part| !part.is_empty()) .collect(); - !parts.is_empty() && parts.iter().all(|part| VERSE.is_match(part)) + if parts.is_empty() || !parts.iter().all(|part| VERSE.is_match(part)) { + return false; + } + // A run of bare numbers is not scripture just because it could follow + // from some book named elsewhere. One of them has to name the book. + parts + .iter() + .any(|part| part.chars().any(char::is_alphabetic)) } /// Every identifier the entry carries, normalised. @@ -1108,9 +1127,12 @@ mod tests { fn chapter_and_verse_is_not_a_work_in_a_bibliography() { for verse in [ "Matthew 4:17", - ". Cf. Deuteronomy 18:15-19; Isaiah 53; Acts 1:6", + ". Cf. Deuteronomy 18:15-19; Isaiah 53; Acts 1:6; 2:16-36; 1 Peter 2:6-8", "1 Peter 2:6-8", "Matthew 13:43; cf Exodus 34:29-35", + "Cf. Matthew 3:7-9; Acts 1:8; 11:1-18; 15:16-17; Galatians 3:14;28", + "Colossians 1:18,24; Romans 12:5; 1 Corinthians 12:12-13. 19. 2 Corinthians 11:2", + "Galatians 6:15-16; also cf. John. 1:11-13; Romans 2:28-29; James 1:1 13. Romans 9-11", ] { assert!(one(verse, None).is_none(), "{verse}"); } diff --git a/crates/citefind/src/split.rs b/crates/citefind/src/split.rs index 72b952f..a564f29 100644 --- a/crates/citefind/src/split.rs +++ b/crates/citefind/src/split.rs @@ -74,6 +74,19 @@ static FILE_ANCHOR: LazyLock = LazyLock::new(|| { .expect("a valid pattern") }); +/// What citefind counts as citing something on top of what `chunkmd` does: +/// a year of publication with a number after it, which is every author-date +/// entry ever set — `Altmann, M. 1951, A Study of Behaviour. Sociometry 14, +/// 351-354.` names an author, a year, a journal and a page range, and +/// carries none of the punctuation `chunkmd` looks for. +/// +/// Wider than `chunkmd`'s for the same reason the marker pattern is: it is +/// deciding whether a chunk is a page of notes, where one missed entry in +/// thirty changes nothing, and this is deciding whether a reference exists. +/// What gets through and is not a reference is refused when it is read. +static LOCAL_SIGNAL: LazyLock = + LazyLock::new(|| Regex::new(r"(?s)\b(?:1[4-9]|20)\d{2}\b.*?\d").expect("a valid pattern")); + /// A bullet or a blockquote marker, which conversion puts in front of a note /// often enough to matter. A leading `*` is left alone: it opens an italic /// title as often as it opens a list. @@ -175,7 +188,7 @@ fn numbered_notes(text: &str) -> Vec { for (at, start) in starts.iter().enumerate() { let end = starts.get(at + 1).copied().unwrap_or(text.len()); let raw = &text[*start..end]; - if !chunkmd::footnote::cites_something(raw) { + if !chunkmd::footnote::cites_something(raw) && !LOCAL_SIGNAL.is_match(raw) { continue; } out.push(strip_marker(raw)); @@ -366,6 +379,18 @@ mod tests { ); } + #[test] + fn an_author_date_note_carries_none_of_the_punctuation_chunkmd_looks_for() { + // Author, year, journal, volume, pages — and not one bracketed year, + // `p.`, `ed.` or `Press,` among them. + let page = "\ +1. Altmann, M. 1951, A Study of Behaviour in a Horse-Mule Group. Sociometry 14, 351-354. +2. Huxley, J. S. 1957: Evolution in Action. London."; + let found = entries(page); + assert_eq!(found.len(), 2, "{found:#?}"); + assert!(found[0].text.starts_with("Altmann"), "{:?}", found[0]); + } + #[test] fn blank_lines_cut_a_bibliography() { let block = "Smith, John. The Title. Oxford: Clarendon, 1998.\n\nDoe, Jane. Another.\nCambridge, 2001.";