citefind: every record has every key, and a title the cut landed inside is refused

Tried against three of the largest books in the corpus — 2,700 chunks, 44,000
citations the model had already extracted — and two things showed up that the
smaller sample had not.

A chunk that begins part-way through an entry leaves a closing quotation mark
with no opening one, and the title read off it starts mid-word and runs to the
end of the line. A title carrying a stray quote, an escape, or an editor's
credit is now no title, and `unsettled` says so.

The optional fields are written as null rather than left out. A key that comes
and goes is a key every consumer has to guard, and the documented Python use
raised KeyError on the first reference without a page.

crates/citefind/README.md

@@ -39,8 +39,8 @@ cargo install --git https://git.pumpin.online/funkpower/catena citefind
 
 ```
 $ citefind bibliography.md
-{"v":1,"idx":0,"text":"Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998.","authors":["Smith, John"],"year":1998,"title":"The Making of the Second Estate","container":"Clarendon Press","ids":[],"kind":"book","continuation":false,"unsettled":[]}
-{"v":1,"idx":1,"text":"Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.","authors":["Doe, Jane","Peter Brown"],"year":2001,"title":"Obligation and Its Limits","container":"Journal of Medieval Studies","pages":"45-67","ids":[],"kind":"article","continuation":false,"unsettled":[]}
+{"v":1,"idx":0,"text":"Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998.","marker":null,"authors":["Smith, John"],"year":1998,"title":"The Making of the Second Estate","container":"Clarendon Press","pages":null,"ids":[],"kind":"book","continuation":false,"unsettled":[]}
+{"v":1,"idx":1,"text":"Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.","marker":null,"authors":["Doe, Jane","Peter Brown"],"year":2001,"title":"Obligation and Its Limits","container":"Journal of Medieval Studies","pages":"45-67","ids":[],"kind":"article","continuation":false,"unsettled":[]}
 ```
 
 What a book cites, by year:
@@ -129,6 +129,10 @@ the mangled few, and they are cheaper to ask about together.
 | `continuation` | Whether it only points back at an earlier entry |
 | `unsettled` | The fields it could not read off the entry |
 
+Every field is written on every record, `null` where there is nothing. A key
+that comes and goes is a key every consumer has to guard, and the point of
+`unsettled` is that an absence be legible.
+
 `text` is kept whatever else was read, because every other field is an
 interpretation of it and somebody will want to check.
 

crates/citefind/src/lib.rs

@@ -53,6 +53,10 @@ impl Kind {
 }
 
 /// One reference, as found and as far as it could be read.
+///
+/// Every field is written on every record, `null` where there is nothing.
+/// A key that comes and goes is a key every consumer has to guard, and the
+/// whole point of `unsettled` is that an absence be legible.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct Reference {
     /// Record version.
@@ -66,21 +70,16 @@ pub struct Reference {
     /// is an interpretation of this one and somebody will want to check it.
     pub text: String,
     /// The note number this entry carried, where it carried one.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub marker: Option<String>,
     /// Who wrote it, in the order the entry gave them.
     pub authors: Vec<String>,
     /// The year of publication.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub year: Option<u32>,
     /// The title of the work cited — the article, not the journal it is in.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub title: Option<String>,
     /// What it appeared in: a journal, or the publisher of a book.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub container: Option<String>,
     /// The pages, as written.
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub pages: Option<String>,
     /// Any identifiers the entry carried, normalised.
     pub ids: Vec<Id>,

crates/citefind/src/parse.rs

@@ -112,7 +112,20 @@ static ROLE: LazyLock<Regex> = LazyLock::new(|| {
 /// run that carries a page reference or a semicolon — both of which mean two
 /// entries were run together, or that this was never an entry at all.
 static NOT_A_TITLE: LazyLock<Regex> = LazyLock::new(|| {
-    Regex::new(r"(?i)^[\d\p{Pd}]|\bpp?\.\s*\d|;|\bvol\.\s*\d|\bibid\b").expect("a valid pattern")
+    Regex::new(
+        r#"(?ix)
+      ^[\d\p{Pd}]                      # opens on a page or a volume number
+    | \bpp?\.\s*\d                     # carries a page reference
+    | ;                                # two entries run together
+    | \bvol\.\s*\d
+    | \bibid\b
+    | [”“"]                            # a quotation mark the capture did not
+                                       # take, so the cut landed inside a title
+    | \\                               # an escape conversion left behind
+    | ,\s+(?:eds?|trans)\.\s+\p{Lu}     # ran past the title into a credit
+    "#,
+    )
+    .expect("a valid pattern")
 });
 
 /// A page reference or a bracketed imprint: what a citation has and a remark
@@ -1116,6 +1129,42 @@ mod tests {
         assert!(found.missing().contains(&"title"));
     }
 
+    #[test]
+    fn a_title_the_cut_landed_inside_of_is_not_reported() {
+        // The chunk began part-way through this entry, so the opening of the
+        // title — and its opening quotation mark — are on the page before.
+        let found = read(
+            "Mediterraneans.\" In Rethinking the Mediterranean, ed. William V. \
+             Harris, 64-93. Oxford: Oxford University Press, 2005.",
+        );
+        assert_eq!(found.title, None, "{found:#?}");
+        assert_eq!(found.container.as_deref(), Some("Oxford University Press"));
+    }
+
+    #[test]
+    fn a_title_that_ran_on_into_the_publisher_is_not_reported() {
+        let found = read(
+            "Goodwin, F. K. and Jamison, K. R. (1990). Manicdepressive illness \
+             Oxford University Press, New York.\\",
+        );
+        assert_eq!(found.authors, ["Goodwin, F. K.", "Jamison, K. R."]);
+        assert_eq!(found.year, Some(1990));
+        assert_eq!(found.title, None, "{found:#?}");
+    }
+
+    #[test]
+    fn an_edition_in_a_title_is_not_an_editors_credit() {
+        // `, 2nd ed. (` is part of the title; `, ed. William Harris` is not.
+        let found = read(
+            "Carter Lindberg, The European Reformations, 2nd ed. (West Sussex, UK: Blackwell, 2010), 161.",
+        );
+        assert_eq!(
+            found.title.as_deref(),
+            Some("The European Reformations, 2nd ed")
+        );
+        assert_eq!(found.container.as_deref(), Some("Blackwell"));
+    }
+
     #[test]
     fn sentences_do_not_stop_at_an_initial_or_an_abbreviation() {
         assert_eq!(