citefind: the references in a bibliography, and what could not be read off them

Cuts a reference list into entries — at the note numbers where there are any,
at the layout where there are not — and reads each as far as its punctuation
allows. Fields that would not parse are named in `unsettled` rather than
guessed at, and `--prompt` asks a model about exactly those and no others.

Over 202 real reference sections from the corpus: 8,566 references, 95% with
a title, 82% with a year, 46% settling every field with no model involved.
A title is refused where the entry was never divided, so no record reports
its own text back as the work it cites.

`ibid.` is flagged and left alone — it points at a work, and the entry does
not say which. A rule of dashes is filled in, because the style defines it as
the author above.

chunkmd gains `note_starts` and `cites_something`: recognising a page of
notes and splitting one are the same knowledge, and two copies of that
pattern would drift.

crates/bindings/src/capi.rs

@@ -417,6 +417,82 @@ pub unsafe extern "C" fn catena_identify(
     give_json(&crate::identify(name, bytes))
 }
 
+/// Read the references out of a bibliography or a page of notes. Returns a
+/// JSON array, one object per reference.
+///
+/// # Safety
+///
+/// `text` must be NUL-terminated UTF-8.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_find_citations(text: *const c_char) -> *mut c_char {
+    clear_error();
+    let Some(text) = (unsafe { borrow(text, "text") }) else {
+        return ptr::null_mut();
+    };
+    give_json(&crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+/// Takes the JSON array `catena_find_citations` returned. Returns the prompt
+/// as a JSON string, or NULL when every reference settled — an answer, not an
+/// error: `catena_last_error` is NULL in that case.
+///
+/// # Safety
+///
+/// `references` must be NUL-terminated UTF-8 holding a JSON array.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_citation_prompt(references: *const c_char) -> *mut c_char {
+    clear_error();
+    let Some(references) = (unsafe { borrow(references, "references") }) else {
+        return ptr::null_mut();
+    };
+    let references: Vec<citefind::Reference> = match serde_json::from_str(references) {
+        Ok(references) => references,
+        Err(why) => {
+            set_error(why.to_string());
+            return ptr::null_mut();
+        }
+    };
+    match crate::citation_prompt(&references) {
+        Some(prompt) => give_json(&prompt),
+        None => ptr::null_mut(),
+    }
+}
+
+/// Fill in what did not parse from a model's reply. Returns the JSON array
+/// with the answer applied, or NULL and an error when the reply is not one
+/// JSON object.
+///
+/// # Safety
+///
+/// `references` and `reply` must be NUL-terminated UTF-8.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_merge_citation_reply(
+    references: *const c_char,
+    reply: *const c_char,
+) -> *mut c_char {
+    clear_error();
+    let (Some(references), Some(reply)) = (unsafe { borrow(references, "references") }, unsafe {
+        borrow(reply, "reply")
+    }) else {
+        return ptr::null_mut();
+    };
+    let references: Vec<citefind::Reference> = match serde_json::from_str(references) {
+        Ok(references) => references,
+        Err(why) => {
+            set_error(why.to_string());
+            return ptr::null_mut();
+        }
+    };
+    match crate::merge_citation_reply(&references, reply) {
+        Ok(merged) => give_json(&merged),
+        Err(why) => {
+            set_error(why);
+            ptr::null_mut()
+        }
+    }
+}
+
 /// The series a title belongs to. Returns a JSON proposal, or NULL when the
 /// title says nothing about a series — which is most titles, and not an
 /// error: `catena_last_error` is NULL in that case.

crates/bindings/src/lib.rs

@@ -215,6 +215,31 @@ pub fn parse_identifier(text: &str) -> Option<idnorm::Identifier> {
     idnorm::parse(text)
 }
 
+/// Read the references out of a bibliography or a page of notes.
+///
+/// Each carries `unsettled`: the fields that could not be read off the entry,
+/// named so a caller can ask somebody else about exactly those and no more.
+pub fn find_citations(text: &str) -> Vec<citefind::Reference> {
+    citefind::read(text)
+}
+
+/// Compose the question to put to a model about the references that did not
+/// parse. `None` when every one of them settled.
+pub fn citation_prompt(references: &[citefind::Reference]) -> Option<String> {
+    citefind::prompt::compose(references)
+}
+
+/// Fill in what did not parse from a model's reply, and return the references
+/// with the answer applied. `Err` when the reply is not one JSON object.
+pub fn merge_citation_reply(
+    references: &[citefind::Reference],
+    reply: &str,
+) -> Result<Vec<citefind::Reference>, String> {
+    let mut references = references.to_vec();
+    citefind::prompt::merge(&mut references, reply)?;
+    Ok(references)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 pub fn series_of(title: &str, author: Option<&str>) -> Option<seriesof::Proposal> {
     seriesof::series_of(title, author)
@@ -236,6 +261,25 @@ mod tests {
         assert_eq!(count_tokens("a passage"), textkit::count("a passage"));
     }
 
+    #[test]
+    fn the_citation_surface_agrees_with_the_library_underneath() {
+        const REFS: &str = "Smith, John. *A Title*. Oxford: Clarendon Press, 1998.\n\nMarron, Kevin Ritual Abuse, Seal Books, 1988";
+        let found = find_citations(REFS);
+        assert_eq!(found, citefind::read(REFS));
+
+        // The clean entry settles and the comma-only one does not, so there
+        // is a question to ask and it is about the second alone.
+        let prompt = citation_prompt(&found).expect("something unsettled");
+        assert!(!prompt.contains("Clarendon"), "{prompt}");
+        assert!(prompt.contains("Ritual Abuse"), "{prompt}");
+
+        let merged = merge_citation_reply(&found, r#"{"1": {"authors": ["Marron, Kevin"]}}"#)
+            .expect("a reply");
+        assert_eq!(merged[1].authors, ["Marron, Kevin"]);
+        assert_eq!(merged[0], found[0]);
+        assert!(merge_citation_reply(&found, "not json").is_err());
+    }
+
     #[test]
     fn the_default_budgets_are_the_tools_defaults() {
         let ours = ChunkBudgets::default();

crates/bindings/src/python.rs

@@ -220,6 +220,34 @@ fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult<Bound<'py
     to_py(py, &crate::identify(name, data))
 }
 
+/// Read the references out of a bibliography or a page of notes.
+#[pyfunction]
+fn find_citations<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
+    list_to_py(py, &crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+///
+/// `None` when every one of them settled, because there is nothing to ask.
+#[pyfunction]
+fn citation_prompt(references: &Bound<'_, PyAny>) -> PyResult<Option<String>> {
+    let references: Vec<citefind::Reference> = from_py_list(references)?;
+    Ok(crate::citation_prompt(&references))
+}
+
+/// Fill in what did not parse from a model's reply.
+#[pyfunction]
+fn merge_citation_reply<'py>(
+    py: Python<'py>,
+    references: &Bound<'py, PyAny>,
+    reply: &str,
+) -> PyResult<Bound<'py, PyList>> {
+    let references: Vec<citefind::Reference> = from_py_list(references)?;
+    let merged = crate::merge_citation_reply(&references, reply)
+        .map_err(pyo3::exceptions::PyValueError::new_err)?;
+    list_to_py(py, &merged)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 ///
 /// `None` when the title says nothing about a series, which is most titles.
@@ -261,6 +289,9 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(split_collection, module)?)?;
     module.add_function(wrap_pyfunction!(fuse_hits, module)?)?;
     module.add_function(wrap_pyfunction!(identify, module)?)?;
+    module.add_function(wrap_pyfunction!(find_citations, module)?)?;
+    module.add_function(wrap_pyfunction!(citation_prompt, module)?)?;
+    module.add_function(wrap_pyfunction!(merge_citation_reply, module)?)?;
     module.add_function(wrap_pyfunction!(series_of, module)?)?;
     module.add_function(wrap_pyfunction!(parse_identifier, module)?)?;
     Ok(())

crates/bindings/src/wasm.rs

@@ -97,6 +97,31 @@ export interface Identifier {
   url?: string;
 }
 
+/** One reference out of a bibliography or a page of notes. */
+export interface Reference {
+  v: number;
+  idx: number;
+  /** The entry as it appeared, whitespace collapsed. */
+  text: string;
+  /** The note number it carried, where it carried one. */
+  marker?: string;
+  /** The names, in the order and the form the entry gave them. */
+  authors: string[];
+  year?: number;
+  /** The work cited — for an article the article, not the journal. */
+  title?: string;
+  /** The journal it appeared in, or the publisher of a book. */
+  container?: string;
+  pages?: string;
+  ids: { kind: string; id: string }[];
+  /** "book", "article", "chapter", "web" or "other". */
+  kind: string;
+  /** Whether it only points back at an earlier entry. */
+  continuation: boolean;
+  /** The fields that could not be read off the entry. */
+  unsettled: string[];
+}
+
 /** What a title says about its series. */
 export interface SeriesProposal {
   v: number;
@@ -280,6 +305,32 @@ pub fn identify(name: &str, data: &[u8]) -> Result<JsValue, JsValue> {
     to_js(&crate::identify(name, data))
 }
 
+/// Read the references out of a bibliography or a page of notes.
+#[wasm_bindgen(js_name = findCitations, skip_typescript)]
+pub fn find_citations(text: &str) -> Result<JsValue, JsValue> {
+    to_js(&crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+///
+/// `undefined` when every one of them settled.
+#[wasm_bindgen(js_name = citationPrompt, skip_typescript)]
+pub fn citation_prompt(references: JsValue) -> Result<JsValue, JsValue> {
+    let references: Vec<citefind::Reference> = from_js(references, "references")?;
+    match crate::citation_prompt(&references) {
+        Some(prompt) => to_js(&prompt),
+        None => Ok(JsValue::UNDEFINED),
+    }
+}
+
+/// Fill in what did not parse from a model's reply.
+#[wasm_bindgen(js_name = mergeCitationReply, skip_typescript)]
+pub fn merge_citation_reply(references: JsValue, reply: &str) -> Result<JsValue, JsValue> {
+    let references: Vec<citefind::Reference> = from_js(references, "references")?;
+    let merged = crate::merge_citation_reply(&references, reply).map_err(JsValue::from)?;
+    to_js(&merged)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 ///
 /// `undefined` when the title says nothing about a series.
@@ -314,6 +365,9 @@ export function cueWindows(cues: Cue[]): string[];
 export function splitCollection(markdown: string): Section[];
 export function fuseHits(hits: Hit[], k?: number, weights?: Record<string, number>, limit?: number): Hit[];
 export function identify(name: string, data: Uint8Array): Doc;
+export function findCitations(text: string): Reference[];
+export function citationPrompt(references: Reference[]): string | undefined;
+export function mergeCitationReply(references: Reference[], reply: string): Reference[];
 export function seriesOf(title: string, author?: string): SeriesProposal | undefined;
 export function parseIdentifier(text: string): Identifier | undefined;
 "#;