From f3e70b9bf7a8a0baf3376fca3b715f443f075b35 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sun, 20 Sep 2026 11:10:42 -0400 Subject: [PATCH] citefind: a note that cites three works is three references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured against the citations an LLM had already extracted from 40 books of the same corpus — 19,340 of them over 3,424 chunks — and improved where the comparison showed a real gap rather than a difference of task. On pages of notes, where most citations live: year agreement with the model 44% -> 56%, title 18% -> 23%, author 16% -> 24%. What changed: - A note listing several works, semicolon-separated, becomes one record per work. The note number is kept on all of them. - A full stop is not a separator. It parts the author from the title in every bibliography entry there is. - A page of body text with notes at its foot no longer reads the body as the first note. - A run carrying a page reference or a semicolon is not reported as a title. - Six sentences with no imprint, no page and no identifier is a remark about the argument, not a reference. - A translator standing where the journal goes is a credit, not a container, and `(London, 1973)` is a place, not a publisher. The model is not ground truth and the comparison is not a score: on several sampled disagreements it named the journal where the article was wanted, or the collected works instead of the essay. --- crates/citefind/README.md | 20 ++++- crates/citefind/src/parse.rs | 120 ++++++++++++++++++++++++++-- crates/citefind/src/split.rs | 75 +++++++++++++++-- crates/citefind/tests/references.rs | 41 ++++++++++ 4 files changed, 243 insertions(+), 13 deletions(-) diff --git a/crates/citefind/README.md b/crates/citefind/README.md index 9c55e37..b049e30 100644 --- a/crates/citefind/README.md +++ b/crates/citefind/README.md @@ -145,7 +145,17 @@ two references together. A note that cites nothing is dropped. A page of notes mixes bibliographic notes with discursive ones, and a remark about the argument is not a -reference. +reference — and one that runs on for six sentences with no imprint, no page +reference and no identifier is a remark whatever else it mentions. + +A note that cites **several works** becomes several records, cut at the +semicolons that separate them, each keeping the note's number. A full stop +separates works in a note too, and is not used: it also separates the author +from the title in every bibliography entry ever written. + +Where the first note number sits a long way into the block, what precedes it +is the body of the page rather than a note that lost its number, and it is +left out. **A bibliography** numbers nothing, so the signal is layout: a blank line between entries, a hanging indent under each, or the surname-comma opening @@ -163,7 +173,13 @@ entirely — there is nothing in it to ask about. **Report a title it did not find.** An entry held together by commas alone, with no names to anchor it, yields no title rather than its own text echoed -back. +back. Nor does a run carrying a page reference or a semicolon count as one: +that is two entries run together, or a reference to a work rather than the +name of one. + +**Name a place of publication as the publisher.** `(London, 1973)` says where +a book was published and not by whom, so `container` stays empty. The place +is still in `text`. A rule of dashes where the author should be is the exception, and a deliberate one: `———. *The Second Book*.` means the author of the entry above, diff --git a/crates/citefind/src/parse.rs b/crates/citefind/src/parse.rs index cc89e73..26af070 100644 --- a/crates/citefind/src/parse.rs +++ b/crates/citefind/src/parse.rs @@ -108,6 +108,33 @@ static ROLE: LazyLock = LazyLock::new(|| { .expect("a valid pattern") }); +/// What a title never is: an opening that is a page or a volume number, or a +/// run that carries a page reference or a semicolon — both of which mean two +/// entries were run together, or that this was never an entry at all. +static NOT_A_TITLE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^[\d\p{Pd}]|\bpp?\.\s*\d|;|\bvol\.\s*\d|\bibid\b").expect("a valid pattern") +}); + +/// A page reference or a bracketed imprint: what a citation has and a remark +/// about the argument does not. +static CITATION_ANCHOR: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bpp?\.\s*\d|\bvol\.\s*\d|\bno\.\s*\d").expect("a valid pattern") +}); + +/// How many sentences a reference runs to before it is prose. +/// +/// A full entry in the heaviest style — names, title, an edited volume, an +/// imprint, pages — is five. A discursive endnote runs on past that, and the +/// three it is checked against are the signals a citation carries anyway. +const MAX_SENTENCES: usize = 5; + +/// A credit standing where the container would be: `trans. E. B. Ashton`, +/// `edited by Mary Douglas`. A role is not a journal and not a publisher. +static ROLE_CREDIT: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(?:trans(?:lated)?|ed(?:ited)?|eds|comp(?:iled)?|rev(?:ised)?|intro(?:duction)?|with)\b\.?(?:\s+by\b)?") + .expect("a valid pattern") +}); + /// A run of text that is nothing but an identifier. static IDENTIFIER_ONLY: LazyLock = LazyLock::new(|| { Regex::new(r"(?i)^(?:arxiv:|doi:|https?://|isbn|issn|\b10\.\d{4})\S*\.?$") @@ -176,6 +203,16 @@ pub fn one(text: &str, marker: Option<&str>) -> Option { if !CITATION_SIGNAL.is_match(text) { return None; } + // A note that runs on for six sentences with no imprint, no page + // reference and no identifier is a remark about the argument. The LLM + // path this replaces was told to skip those; a parser has to decide. + if sentences(text).len() > MAX_SENTENCES + && !IMPRINT.is_match(text) + && !CITATION_ANCHOR.is_match(text) + && ids(text).is_empty() + { + return None; + } out.ids = ids(text); out.year = year(text); @@ -238,6 +275,12 @@ pub fn one(text: &str, marker: Option<&str>) -> Option { out.title = found_title; out.container = found_container; } + // A title the reading refused comes back empty rather than absent, + // because the refusal happens where the trimming does. An empty title is + // an absent one, and `unsettled` should say so. + if out.title.as_deref().is_some_and(str::is_empty) { + out.title = None; + } out.kind = kind(text, &out); Some(out) @@ -467,8 +510,14 @@ fn looks_like_a_year(text: &str) -> bool { /// Strip the punctuation a style leaves inside its own title delimiters. fn tidy_title(raw: &str) -> String { let text = raw.split_whitespace().collect::>().join(" "); - text.trim_matches(|c: char| ",.;:".contains(c) || c.is_whitespace()) - .to_string() + let text = text.trim_matches(|c: char| ",.;:".contains(c) || c.is_whitespace()); + // A run that opens on a page number, or carries one, is two entries run + // together or a reference to a work rather than the name of one. Better + // to have no title and say so than to report that as one. + match NOT_A_TITLE.is_match(text) { + true => String::new(), + false => text.to_string(), + } } /// Who wrote it, from the run of text before the title. @@ -575,7 +624,7 @@ fn is_name(text: &str) -> bool { } /// What the work appeared in: a journal, or a book's publisher. -fn container(_whole: &str, after: &str) -> Option { +fn container(whole: &str, after: &str) -> Option { // A book names its publisher in an imprint after the title, whether // bracketed or not. Only after the title: a title with a colon in it — // and half of them have one — would otherwise read as an imprint. @@ -584,14 +633,42 @@ fn container(_whole: &str, after: &str) -> Option { return Some(publisher); } } - // A journal sits between the title and the volume. + // A journal sits between the title and the volume, but a credit can sit + // in front of it — `*Negative Dialectics*, trans. E. B. Ashton` — and a + // translator is neither a journal nor a publisher. let tail = after.trim_start_matches(|c: char| ",.;:\"”'’ ".contains(c)); + let tail = match ROLE_CREDIT.find(tail) { + Some(found) => match tail[found.end()..].find([',', ';', '(']) { + Some(at) => tail[found.end() + at + 1..].trim_start(), + None => "", + }, + None => tail, + }; let end = VOLUME .find(tail) .map(|found| found.start()) .or_else(|| tail.find(['(', '.', ';'])) .unwrap_or(tail.len()); - tidy_container(&tail[..end]) + let found = tidy_container(&tail[..end])?; + // `(London, 1973)` names a place and a year and no publisher. Reporting + // the place as the publisher would be a plausible wrong answer, and the + // place is still there in `text` for anyone who wants it. + match is_place_of_publication(whole, &found) { + true => None, + false => Some(found), + } +} + +/// Whether a candidate is the place a book was published rather than its +/// publisher: it sits inside brackets with the year and nothing else. +fn is_place_of_publication(whole: &str, found: &str) -> bool { + let needle = format!("({found}, "); + let Some(at) = whole.find(&needle) else { + return false; + }; + let rest = &whole[at + needle.len()..]; + let year: String = rest.chars().take_while(char::is_ascii_digit).collect(); + year.len() == 4 && rest[year.len()..].starts_with(')') } /// Reduce a run of text to the name of the thing a work appeared in. @@ -1006,6 +1083,39 @@ mod tests { assert_eq!(found.pages.as_deref(), Some("102")); } + #[test] + fn a_translator_is_not_the_journal_it_stands_in_front_of() { + let found = read( + "Theodor W. Adorno, *Negative Dialectics*, trans. E. B. Ashton (London, 1973), 365.", + ); + assert_eq!(found.title.as_deref(), Some("Negative Dialectics")); + // `(London, 1973)` names a place and a year and no publisher at all. + assert_eq!(found.container, None); + assert_eq!(found.pages.as_deref(), Some("365")); + } + + #[test] + fn a_remark_about_the_argument_is_not_a_reference() { + let found = one( + "Attention is a limited resource. Many psychologists have remarked \ + on this. It follows that what a person attends to is what they \ + come to know. The point is developed at length below. I am \ + grateful to the seminar for pressing me on it.", + None, + ); + assert!(found.is_none(), "{found:#?}"); + } + + #[test] + fn a_page_reference_is_not_a_title() { + // Two notes run together, or a note that names no work at all. + let found = read( + "See Hefele-Leclerq (1909), 72-7, though Diekamp (1899) should have concluded the debate.", + ); + assert_eq!(found.title, None, "{found:#?}"); + assert!(found.missing().contains(&"title")); + } + #[test] fn sentences_do_not_stop_at_an_initial_or_an_abbreviation() { assert_eq!( diff --git a/crates/citefind/src/split.rs b/crates/citefind/src/split.rs index 92bd795..60d1594 100644 --- a/crates/citefind/src/split.rs +++ b/crates/citefind/src/split.rs @@ -25,6 +25,13 @@ pub struct Entry { pub marker: Option, } +/// How far into a block the first note number may sit before what precedes +/// it is read as something other than a note. +/// +/// About the length of one full reference. Past that it is the body of the +/// page, sitting above the notes at its foot. +const HEAD_LIMIT: usize = 400; + /// At least this many numbered notes before the numbering is believed. /// /// Two: one is a stray list item, and a real page of notes has more. @@ -80,10 +87,64 @@ static ID_MARKER: LazyLock = /// Cut a block into entries. pub fn entries(text: &str) -> Vec { let numbered = numbered_notes(text); - if numbered.len() >= MIN_NOTES { - return numbered; + let cut = match numbered.len() >= MIN_NOTES { + true => numbered, + false => unnumbered(text), + }; + cut.into_iter().flat_map(divide).collect() +} + +/// The least an entry can be and still be one, on either side of a split. +const MIN_WORK: usize = 25; + +/// Where another citation begins inside one entry: a semicolon, and then a +/// name with a comma after it. +/// +/// A semicolon and nothing else. A full stop separates works in a note too, +/// but it also separates the author from the title in every bibliography +/// entry ever written — `Kahneman, Daniel. Thinking, Fast and Slow.` would +/// come apart at the one place it must not. +static ANOTHER_WORK: LazyLock = LazyLock::new(|| { + Regex::new( + r#";\s+(?:(?i:see(?:\s+also)?|cf\.|and)\s+)?[\p{Lu}][\p{L}.'’\-]{1,24}(?:\s+[\p{Lu}][\p{L}.'’\-]{1,24}){0,3},\s+[\p{Lu}\*_"“‘]"#, + ) + .expect("a valid pattern") +}); + +/// Cut one entry into the works it cites. +/// +/// A note commonly cites two or three in a row, separated by semicolons — +/// `Adorno, *Negative Dialectics* (London, 1973), 365; Alexander, "On the +/// Social Construction of Moral Universals", 196`. They are distinct works +/// and each deserves its own record. The note number is kept on all of them, +/// because all of them are that note. +fn divide(entry: Entry) -> Vec { + let mut out = Vec::new(); + let mut rest = entry.text.as_str(); + while let Some(found) = ANOTHER_WORK.find(rest) { + // The separator belongs to neither side, and both sides have to be + // long enough to be a citation rather than an abbreviation. + let (before, after) = ( + &rest[..found.start()], + rest[found.start() + 1..].trim_start(), + ); + if before.chars().count() < MIN_WORK || after.chars().count() < MIN_WORK { + break; + } + out.push(before.trim().to_string()); + rest = after; } - unnumbered(text) + if out.is_empty() { + return vec![entry]; + } + out.push(rest.trim().to_string()); + out.into_iter() + .filter(|text| !text.is_empty()) + .map(|text| Entry { + text, + marker: entry.marker.clone(), + }) + .collect() } /// Entries cut at their note numbers. @@ -102,9 +163,11 @@ fn numbered_notes(text: &str) -> Vec { return Vec::new(); } // Whatever is in front of the first marker is the first note, when the - // marker it carried was one neither pattern knows. Dropping it would - // lose a reference silently, which is the one thing a cut must not do. - if starts[0] > 0 { + // marker it carried was one neither pattern knows — but only when it is + // about the length of a reference. A page of body text with its notes + // at the foot has a marker a long way in, and everything above it is the + // page, not a note that lost its number. + if (1..=HEAD_LIMIT).contains(&starts[0]) { starts.insert(0, 0); } diff --git a/crates/citefind/tests/references.rs b/crates/citefind/tests/references.rs index f26d32a..78d2ae1 100644 --- a/crates/citefind/tests/references.rs +++ b/crates/citefind/tests/references.rs @@ -171,6 +171,47 @@ fn a_reply_that_is_not_json_is_refused_rather_than_half_applied() { assert_eq!(found, before); } +#[test] +fn a_note_that_cites_two_works_gives_two_records() { + let found = read( + "12. Theodor W. Adorno, *Negative Dialectics*, trans. E. B. Ashton (London, 1973), 365; \ + Jeffrey C. Alexander, \"On the Social Construction of Moral Universals,\" *Cultural Trauma* 5 (2002), 196.\n\ + 13. Smith, John. Thinking, Fast and Slow. New York: Farrar, Straus and Giroux, 2011.", + ); + assert_eq!(found.len(), 3, "{found:#?}"); + // Both halves of note 12 are note 12. + assert_eq!( + found + .iter() + .map(|one| one.marker.as_deref()) + .collect::>(), + [Some("12"), Some("12"), Some("13")] + ); + assert_eq!(found[0].title.as_deref(), Some("Negative Dialectics")); + assert_eq!( + found[1].title.as_deref(), + Some("On the Social Construction of Moral Universals") + ); + // The full stop between an author and a title is not a separator, or + // every bibliography entry ever written would come apart at it. + assert_eq!(found[2].title.as_deref(), Some("Thinking, Fast and Slow")); +} + +#[test] +fn the_body_of_a_page_is_not_the_first_of_the_notes_at_its_foot() { + let page = format!( + "{}\n\n1. John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154.\n\ + 2. Johanna Bockman, *Markets in the Name of Socialism* (Stanford, CA: Stanford University Press, 2011), 40.", + "The obligation ran both ways, and contemporaries said so plainly whenever they were asked. ".repeat(8) + ); + let found = read(&page); + assert_eq!(found.len(), 2, "{found:#?}"); + assert!( + found.iter().all(|one| !one.text.contains("obligation")), + "{found:#?}" + ); +} + #[test] fn nothing_citation_shaped_gives_nothing() { assert!(read("Bibliography\n\nSee below.\n\n233\n").is_empty());