From 2a2ecdc018d3096dc1ad0e9f4bdec7280fe48bc2 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sun, 20 Sep 2026 11:31:24 -0400 Subject: [PATCH] citefind: every record has every key, and a title the cut landed inside is refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tried against three of the largest books in the corpus — 2,700 chunks, 44,000 citations the model had already extracted — and two things showed up that the smaller sample had not. A chunk that begins part-way through an entry leaves a closing quotation mark with no opening one, and the title read off it starts mid-word and runs to the end of the line. A title carrying a stray quote, an escape, or an editor's credit is now no title, and `unsettled` says so. The optional fields are written as null rather than left out. A key that comes and goes is a key every consumer has to guard, and the documented Python use raised KeyError on the first reference without a page. --- crates/citefind/README.md | 8 ++++-- crates/citefind/src/lib.rs | 9 +++---- crates/citefind/src/parse.rs | 51 +++++++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/crates/citefind/README.md b/crates/citefind/README.md index b049e30..c3ba874 100644 --- a/crates/citefind/README.md +++ b/crates/citefind/README.md @@ -39,8 +39,8 @@ cargo install --git https://git.pumpin.online/funkpower/catena citefind ``` $ citefind bibliography.md -{"v":1,"idx":0,"text":"Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998.","authors":["Smith, John"],"year":1998,"title":"The Making of the Second Estate","container":"Clarendon Press","ids":[],"kind":"book","continuation":false,"unsettled":[]} -{"v":1,"idx":1,"text":"Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.","authors":["Doe, Jane","Peter Brown"],"year":2001,"title":"Obligation and Its Limits","container":"Journal of Medieval Studies","pages":"45-67","ids":[],"kind":"article","continuation":false,"unsettled":[]} +{"v":1,"idx":0,"text":"Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998.","marker":null,"authors":["Smith, John"],"year":1998,"title":"The Making of the Second Estate","container":"Clarendon Press","pages":null,"ids":[],"kind":"book","continuation":false,"unsettled":[]} +{"v":1,"idx":1,"text":"Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.","marker":null,"authors":["Doe, Jane","Peter Brown"],"year":2001,"title":"Obligation and Its Limits","container":"Journal of Medieval Studies","pages":"45-67","ids":[],"kind":"article","continuation":false,"unsettled":[]} ``` What a book cites, by year: @@ -129,6 +129,10 @@ the mangled few, and they are cheaper to ask about together. | `continuation` | Whether it only points back at an earlier entry | | `unsettled` | The fields it could not read off the entry | +Every field is written on every record, `null` where there is nothing. A key +that comes and goes is a key every consumer has to guard, and the point of +`unsettled` is that an absence be legible. + `text` is kept whatever else was read, because every other field is an interpretation of it and somebody will want to check. diff --git a/crates/citefind/src/lib.rs b/crates/citefind/src/lib.rs index fa35a40..8b0bfc3 100644 --- a/crates/citefind/src/lib.rs +++ b/crates/citefind/src/lib.rs @@ -53,6 +53,10 @@ impl Kind { } /// One reference, as found and as far as it could be read. +/// +/// Every field is written on every record, `null` where there is nothing. +/// A key that comes and goes is a key every consumer has to guard, and the +/// whole point of `unsettled` is that an absence be legible. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Reference { /// Record version. @@ -66,21 +70,16 @@ pub struct Reference { /// is an interpretation of this one and somebody will want to check it. pub text: String, /// The note number this entry carried, where it carried one. - #[serde(skip_serializing_if = "Option::is_none")] pub marker: Option, /// Who wrote it, in the order the entry gave them. pub authors: Vec, /// The year of publication. - #[serde(skip_serializing_if = "Option::is_none")] pub year: Option, /// The title of the work cited — the article, not the journal it is in. - #[serde(skip_serializing_if = "Option::is_none")] pub title: Option, /// What it appeared in: a journal, or the publisher of a book. - #[serde(skip_serializing_if = "Option::is_none")] pub container: Option, /// The pages, as written. - #[serde(skip_serializing_if = "Option::is_none")] pub pages: Option, /// Any identifiers the entry carried, normalised. pub ids: Vec, diff --git a/crates/citefind/src/parse.rs b/crates/citefind/src/parse.rs index 26af070..f124075 100644 --- a/crates/citefind/src/parse.rs +++ b/crates/citefind/src/parse.rs @@ -112,7 +112,20 @@ static ROLE: LazyLock = LazyLock::new(|| { /// run that carries a page reference or a semicolon — both of which mean two /// entries were run together, or that this was never an entry at all. static NOT_A_TITLE: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)^[\d\p{Pd}]|\bpp?\.\s*\d|;|\bvol\.\s*\d|\bibid\b").expect("a valid pattern") + Regex::new( + r#"(?ix) + ^[\d\p{Pd}] # opens on a page or a volume number + | \bpp?\.\s*\d # carries a page reference + | ; # two entries run together + | \bvol\.\s*\d + | \bibid\b + | [”“"] # a quotation mark the capture did not + # take, so the cut landed inside a title + | \\ # an escape conversion left behind + | ,\s+(?:eds?|trans)\.\s+\p{Lu} # ran past the title into a credit + "#, + ) + .expect("a valid pattern") }); /// A page reference or a bracketed imprint: what a citation has and a remark @@ -1116,6 +1129,42 @@ mod tests { assert!(found.missing().contains(&"title")); } + #[test] + fn a_title_the_cut_landed_inside_of_is_not_reported() { + // The chunk began part-way through this entry, so the opening of the + // title — and its opening quotation mark — are on the page before. + let found = read( + "Mediterraneans.\" In Rethinking the Mediterranean, ed. William V. \ + Harris, 64-93. Oxford: Oxford University Press, 2005.", + ); + assert_eq!(found.title, None, "{found:#?}"); + assert_eq!(found.container.as_deref(), Some("Oxford University Press")); + } + + #[test] + fn a_title_that_ran_on_into_the_publisher_is_not_reported() { + let found = read( + "Goodwin, F. K. and Jamison, K. R. (1990). Manicdepressive illness \ + Oxford University Press, New York.\\", + ); + assert_eq!(found.authors, ["Goodwin, F. K.", "Jamison, K. R."]); + assert_eq!(found.year, Some(1990)); + assert_eq!(found.title, None, "{found:#?}"); + } + + #[test] + fn an_edition_in_a_title_is_not_an_editors_credit() { + // `, 2nd ed. (` is part of the title; `, ed. William Harris` is not. + let found = read( + "Carter Lindberg, The European Reformations, 2nd ed. (West Sussex, UK: Blackwell, 2010), 161.", + ); + assert_eq!( + found.title.as_deref(), + Some("The European Reformations, 2nd ed") + ); + assert_eq!(found.container.as_deref(), Some("Blackwell")); + } + #[test] fn sentences_do_not_stop_at_an_initial_or_an_abbreviation() { assert_eq!(