citefind: the references in a bibliography, and what could not be read off them

Cuts a reference list into entries — at the note numbers where there are any,
at the layout where there are not — and reads each as far as its punctuation
allows. Fields that would not parse are named in `unsettled` rather than
guessed at, and `--prompt` asks a model about exactly those and no others.

Over 202 real reference sections from the corpus: 8,566 references, 95% with
a title, 82% with a year, 46% settling every field with no model involved.
A title is refused where the entry was never divided, so no record reports
its own text back as the work it cites.

`ibid.` is flagged and left alone — it points at a work, and the entry does
not say which. A rule of dashes is filled in, because the style defines it as
the author above.

chunkmd gains `note_starts` and `cites_something`: recognising a page of
notes and splitting one are the same knowledge, and two copies of that
pattern would drift.

crates/bindings/Cargo.toml

@@ -30,6 +30,7 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 booksniff.workspace = true
 chunkmd.workspace = true
+citefind.workspace = true
 corpus-core.workspace = true
 idnorm.workspace = true
 cuekit.workspace = true

crates/bindings/README.md

@@ -78,6 +78,9 @@ except ValueError as why:
 | `identify` | [`booksniff`](../booksniff/README.md) |
 | `series_of` | [`seriesof`](../seriesof/README.md) |
 | `parse_identifier` | [`idnorm`](../idnorm/README.md) |
+| `find_citations` | [`citefind`](../citefind/README.md) |
+| `citation_prompt` | the question `citefind --prompt` writes |
+| `merge_citation_reply` | what `citefind --merge` does with the answer |
 
 The tools that are not here are the ones a binding would gain nothing over a
 subprocess for: they read files, shell out to another program, or talk to a

crates/bindings/include/catena.h

@@ -88,6 +88,31 @@ char *catena_fuse_hits(const char *hits,
  * record. */
 char *catena_identify(const char *name, const uint8_t *data, size_t len);
 
+/* The series a title belongs to. Returns a JSON proposal, or NULL when the
+ * title says nothing about a series — an answer, not an error. `author` may
+ * be NULL. */
+char *catena_series_of(const char *title, const char *author);
+
+/* Read an identifier — a DOI, ISSN, ISBN or arXiv id — out of whatever it
+ * arrived wrapped in. Returns a JSON record, or NULL when the string is not
+ * one — an answer, not an error. */
+char *catena_parse_identifier(const char *text);
+
+/* Read the references out of a bibliography or a page of notes. Returns a
+ * JSON array, one object per reference. */
+char *catena_find_citations(const char *text);
+
+/* The question to put to a model about the references that did not parse.
+ * `references` is the JSON array catena_find_citations returned. Returns the
+ * prompt as a JSON string, or NULL when every reference settled — an answer,
+ * not an error. */
+char *catena_citation_prompt(const char *references);
+
+/* Fill in what did not parse from a model's reply. Returns the JSON array
+ * with the answer applied, or NULL and an error when the reply is not one
+ * JSON object. */
+char *catena_merge_citation_reply(const char *references, const char *reply);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif

crates/bindings/npm/typecheck.ts

@@ -4,7 +4,9 @@
 import {
   chunkMarkdown, scrub, countTokens, parseSubtitles, packCues,
   splitCollection, fuseHits, identify, seriesOf, parseIdentifier, segmentCues, cueWindows,
+  findCitations, citationPrompt, mergeCitationReply,
   type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, type Identifier, type Segment,
+  type Reference,
 } from "./dist/web/catena.js";
 
 const chunks: Chunk[] = chunkMarkdown("# One\n\nbody\n", 512, 64, 50, 768);
@@ -45,6 +47,14 @@ const proposal: SeriesProposal | undefined = seriesOf("The Philokalia Vol 3");
 const volume: number | undefined = proposal?.volume;
 seriesOf("Aquinas (Companion)", "Cambridge");
 
+const references: Reference[] = findCitations("Smith, John. *A Title*. Oxford: Clarendon Press, 1998.");
+const unsettled: string[] = references[0].unsettled;
+const ask: string | undefined = citationPrompt(references);
+const filled: Reference[] = mergeCitationReply(references, "{}");
+void unsettled;
+void ask;
+void filled;
+
 const ident: Identifier | undefined = parseIdentifier("10.1000/182");
 const checksum: boolean | null | undefined = ident?.checksum;
 

crates/bindings/python/catena/__init__.py

@@ -14,9 +14,12 @@ from ._catena import (
     __version__,
     cue_windows,
     chunk_markdown,
+    citation_prompt,
     count_tokens,
+    find_citations,
     fuse_hits,
     identify,
+    merge_citation_reply,
     pack_cues,
     parse_identifier,
     parse_subtitles,
@@ -30,9 +33,12 @@ __all__ = [
     "__version__",
     "cue_windows",
     "chunk_markdown",
+    "citation_prompt",
     "count_tokens",
+    "find_citations",
     "fuse_hits",
     "identify",
+    "merge_citation_reply",
     "pack_cues",
     "parse_identifier",
     "parse_subtitles",

crates/bindings/python/catena/__init__.pyi

@@ -37,6 +37,11 @@ def fuse_hits(
     limit: int | None = ...,
 ) -> list[dict[str, Any]]: ...
 def identify(name: str, data: bytes) -> dict[str, Any]: ...
+def find_citations(text: str) -> list[dict[str, Any]]: ...
+def citation_prompt(references: Iterable[Mapping[str, Any]]) -> str | None: ...
+def merge_citation_reply(
+    references: Iterable[Mapping[str, Any]], reply: str
+) -> list[dict[str, Any]]: ...
 def parse_identifier(text: str) -> dict[str, Any] | None: ...
 def series_of(
     title: str, author: str | None = ...

crates/bindings/src/capi.rs

@@ -417,6 +417,82 @@ pub unsafe extern "C" fn catena_identify(
     give_json(&crate::identify(name, bytes))
 }
 
+/// Read the references out of a bibliography or a page of notes. Returns a
+/// JSON array, one object per reference.
+///
+/// # Safety
+///
+/// `text` must be NUL-terminated UTF-8.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_find_citations(text: *const c_char) -> *mut c_char {
+    clear_error();
+    let Some(text) = (unsafe { borrow(text, "text") }) else {
+        return ptr::null_mut();
+    };
+    give_json(&crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+/// Takes the JSON array `catena_find_citations` returned. Returns the prompt
+/// as a JSON string, or NULL when every reference settled — an answer, not an
+/// error: `catena_last_error` is NULL in that case.
+///
+/// # Safety
+///
+/// `references` must be NUL-terminated UTF-8 holding a JSON array.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_citation_prompt(references: *const c_char) -> *mut c_char {
+    clear_error();
+    let Some(references) = (unsafe { borrow(references, "references") }) else {
+        return ptr::null_mut();
+    };
+    let references: Vec<citefind::Reference> = match serde_json::from_str(references) {
+        Ok(references) => references,
+        Err(why) => {
+            set_error(why.to_string());
+            return ptr::null_mut();
+        }
+    };
+    match crate::citation_prompt(&references) {
+        Some(prompt) => give_json(&prompt),
+        None => ptr::null_mut(),
+    }
+}
+
+/// Fill in what did not parse from a model's reply. Returns the JSON array
+/// with the answer applied, or NULL and an error when the reply is not one
+/// JSON object.
+///
+/// # Safety
+///
+/// `references` and `reply` must be NUL-terminated UTF-8.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_merge_citation_reply(
+    references: *const c_char,
+    reply: *const c_char,
+) -> *mut c_char {
+    clear_error();
+    let (Some(references), Some(reply)) = (unsafe { borrow(references, "references") }, unsafe {
+        borrow(reply, "reply")
+    }) else {
+        return ptr::null_mut();
+    };
+    let references: Vec<citefind::Reference> = match serde_json::from_str(references) {
+        Ok(references) => references,
+        Err(why) => {
+            set_error(why.to_string());
+            return ptr::null_mut();
+        }
+    };
+    match crate::merge_citation_reply(&references, reply) {
+        Ok(merged) => give_json(&merged),
+        Err(why) => {
+            set_error(why);
+            ptr::null_mut()
+        }
+    }
+}
+
 /// The series a title belongs to. Returns a JSON proposal, or NULL when the
 /// title says nothing about a series — which is most titles, and not an
 /// error: `catena_last_error` is NULL in that case.

crates/bindings/src/lib.rs

@@ -215,6 +215,31 @@ pub fn parse_identifier(text: &str) -> Option<idnorm::Identifier> {
     idnorm::parse(text)
 }
 
+/// Read the references out of a bibliography or a page of notes.
+///
+/// Each carries `unsettled`: the fields that could not be read off the entry,
+/// named so a caller can ask somebody else about exactly those and no more.
+pub fn find_citations(text: &str) -> Vec<citefind::Reference> {
+    citefind::read(text)
+}
+
+/// Compose the question to put to a model about the references that did not
+/// parse. `None` when every one of them settled.
+pub fn citation_prompt(references: &[citefind::Reference]) -> Option<String> {
+    citefind::prompt::compose(references)
+}
+
+/// Fill in what did not parse from a model's reply, and return the references
+/// with the answer applied. `Err` when the reply is not one JSON object.
+pub fn merge_citation_reply(
+    references: &[citefind::Reference],
+    reply: &str,
+) -> Result<Vec<citefind::Reference>, String> {
+    let mut references = references.to_vec();
+    citefind::prompt::merge(&mut references, reply)?;
+    Ok(references)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 pub fn series_of(title: &str, author: Option<&str>) -> Option<seriesof::Proposal> {
     seriesof::series_of(title, author)
@@ -236,6 +261,25 @@ mod tests {
         assert_eq!(count_tokens("a passage"), textkit::count("a passage"));
     }
 
+    #[test]
+    fn the_citation_surface_agrees_with_the_library_underneath() {
+        const REFS: &str = "Smith, John. *A Title*. Oxford: Clarendon Press, 1998.\n\nMarron, Kevin Ritual Abuse, Seal Books, 1988";
+        let found = find_citations(REFS);
+        assert_eq!(found, citefind::read(REFS));
+
+        // The clean entry settles and the comma-only one does not, so there
+        // is a question to ask and it is about the second alone.
+        let prompt = citation_prompt(&found).expect("something unsettled");
+        assert!(!prompt.contains("Clarendon"), "{prompt}");
+        assert!(prompt.contains("Ritual Abuse"), "{prompt}");
+
+        let merged = merge_citation_reply(&found, r#"{"1": {"authors": ["Marron, Kevin"]}}"#)
+            .expect("a reply");
+        assert_eq!(merged[1].authors, ["Marron, Kevin"]);
+        assert_eq!(merged[0], found[0]);
+        assert!(merge_citation_reply(&found, "not json").is_err());
+    }
+
     #[test]
     fn the_default_budgets_are_the_tools_defaults() {
         let ours = ChunkBudgets::default();

crates/bindings/src/python.rs

@@ -220,6 +220,34 @@ fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult<Bound<'py
     to_py(py, &crate::identify(name, data))
 }
 
+/// Read the references out of a bibliography or a page of notes.
+#[pyfunction]
+fn find_citations<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
+    list_to_py(py, &crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+///
+/// `None` when every one of them settled, because there is nothing to ask.
+#[pyfunction]
+fn citation_prompt(references: &Bound<'_, PyAny>) -> PyResult<Option<String>> {
+    let references: Vec<citefind::Reference> = from_py_list(references)?;
+    Ok(crate::citation_prompt(&references))
+}
+
+/// Fill in what did not parse from a model's reply.
+#[pyfunction]
+fn merge_citation_reply<'py>(
+    py: Python<'py>,
+    references: &Bound<'py, PyAny>,
+    reply: &str,
+) -> PyResult<Bound<'py, PyList>> {
+    let references: Vec<citefind::Reference> = from_py_list(references)?;
+    let merged = crate::merge_citation_reply(&references, reply)
+        .map_err(pyo3::exceptions::PyValueError::new_err)?;
+    list_to_py(py, &merged)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 ///
 /// `None` when the title says nothing about a series, which is most titles.
@@ -261,6 +289,9 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(split_collection, module)?)?;
     module.add_function(wrap_pyfunction!(fuse_hits, module)?)?;
     module.add_function(wrap_pyfunction!(identify, module)?)?;
+    module.add_function(wrap_pyfunction!(find_citations, module)?)?;
+    module.add_function(wrap_pyfunction!(citation_prompt, module)?)?;
+    module.add_function(wrap_pyfunction!(merge_citation_reply, module)?)?;
     module.add_function(wrap_pyfunction!(series_of, module)?)?;
     module.add_function(wrap_pyfunction!(parse_identifier, module)?)?;
     Ok(())

crates/bindings/src/wasm.rs

@@ -97,6 +97,31 @@ export interface Identifier {
   url?: string;
 }
 
+/** One reference out of a bibliography or a page of notes. */
+export interface Reference {
+  v: number;
+  idx: number;
+  /** The entry as it appeared, whitespace collapsed. */
+  text: string;
+  /** The note number it carried, where it carried one. */
+  marker?: string;
+  /** The names, in the order and the form the entry gave them. */
+  authors: string[];
+  year?: number;
+  /** The work cited — for an article the article, not the journal. */
+  title?: string;
+  /** The journal it appeared in, or the publisher of a book. */
+  container?: string;
+  pages?: string;
+  ids: { kind: string; id: string }[];
+  /** "book", "article", "chapter", "web" or "other". */
+  kind: string;
+  /** Whether it only points back at an earlier entry. */
+  continuation: boolean;
+  /** The fields that could not be read off the entry. */
+  unsettled: string[];
+}
+
 /** What a title says about its series. */
 export interface SeriesProposal {
   v: number;
@@ -280,6 +305,32 @@ pub fn identify(name: &str, data: &[u8]) -> Result<JsValue, JsValue> {
     to_js(&crate::identify(name, data))
 }
 
+/// Read the references out of a bibliography or a page of notes.
+#[wasm_bindgen(js_name = findCitations, skip_typescript)]
+pub fn find_citations(text: &str) -> Result<JsValue, JsValue> {
+    to_js(&crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+///
+/// `undefined` when every one of them settled.
+#[wasm_bindgen(js_name = citationPrompt, skip_typescript)]
+pub fn citation_prompt(references: JsValue) -> Result<JsValue, JsValue> {
+    let references: Vec<citefind::Reference> = from_js(references, "references")?;
+    match crate::citation_prompt(&references) {
+        Some(prompt) => to_js(&prompt),
+        None => Ok(JsValue::UNDEFINED),
+    }
+}
+
+/// Fill in what did not parse from a model's reply.
+#[wasm_bindgen(js_name = mergeCitationReply, skip_typescript)]
+pub fn merge_citation_reply(references: JsValue, reply: &str) -> Result<JsValue, JsValue> {
+    let references: Vec<citefind::Reference> = from_js(references, "references")?;
+    let merged = crate::merge_citation_reply(&references, reply).map_err(JsValue::from)?;
+    to_js(&merged)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 ///
 /// `undefined` when the title says nothing about a series.
@@ -314,6 +365,9 @@ export function cueWindows(cues: Cue[]): string[];
 export function splitCollection(markdown: string): Section[];
 export function fuseHits(hits: Hit[], k?: number, weights?: Record<string, number>, limit?: number): Hit[];
 export function identify(name: string, data: Uint8Array): Doc;
+export function findCitations(text: string): Reference[];
+export function citationPrompt(references: Reference[]): string | undefined;
+export function mergeCitationReply(references: Reference[], reply: string): Reference[];
 export function seriesOf(title: string, author?: string): SeriesProposal | undefined;
 export function parseIdentifier(text: string): Identifier | undefined;
 "#;