citefind: the references in a bibliography, and what could not be read off them

Cuts a reference list into entries — at the note numbers where there are any,
at the layout where there are not — and reads each as far as its punctuation
allows. Fields that would not parse are named in `unsettled` rather than
guessed at, and `--prompt` asks a model about exactly those and no others.

Over 202 real reference sections from the corpus: 8,566 references, 95% with
a title, 82% with a year, 46% settling every field with no model involved.
A title is refused where the entry was never divided, so no record reports
its own text back as the work it cites.

`ibid.` is flagged and left alone — it points at a work, and the entry does
not say which. A rule of dashes is filled in, because the style defines it as
the author above.

chunkmd gains `note_starts` and `cites_something`: recognising a page of
notes and splitting one are the same knowledge, and two copies of that
pattern would drift.

Cargo.lock

@@ -254,6 +254,7 @@ version = "0.1.0"
 dependencies = [
  "booksniff",
  "chunkmd",
+ "citefind",
  "corpus-core",
  "cuekit",
  "cuepack",
@@ -328,6 +329,21 @@ dependencies = [
  "inout",
 ]
 
+[[package]]
+name = "citefind"
+version = "0.1.0"
+dependencies = [
+ "catena-cli",
+ "chunkmd",
+ "clap",
+ "corpus-core",
+ "idnorm",
+ "regex",
+ "schemars",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "clap"
 version = "4.6.7"

Cargo.toml

@@ -33,6 +33,7 @@ bookmeta = { path = "crates/bookmeta", version = "0.1.0" }
 langid = { path = "crates/langid", version = "0.1.0" }
 tocfind = { path = "crates/tocfind", version = "0.1.0" }
 booksniff = { path = "crates/booksniff", version = "0.1.0" }
+citefind = { path = "crates/citefind", version = "0.1.0" }
 emb = { path = "crates/emb", version = "0.1.0" }
 
 clap = { version = "4", features = ["derive"] }

README.md

@@ -34,6 +34,7 @@ Each tool installs by name, so you can take one without taking the rest.
 | [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is |
 | [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked |
 | [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier |
+| [`citefind`](crates/citefind/README.md) | The references in a bibliography or a page of notes |
 | [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out |
 | [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it |
 | [`chunkmd`](crates/chunkmd/README.md) | Markdown into passages that carry their page, chapter and role |

crates/bindings/Cargo.toml

@@ -30,6 +30,7 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 booksniff.workspace = true
 chunkmd.workspace = true
+citefind.workspace = true
 corpus-core.workspace = true
 idnorm.workspace = true
 cuekit.workspace = true

crates/bindings/README.md

@@ -78,6 +78,9 @@ except ValueError as why:
 | `identify` | [`booksniff`](../booksniff/README.md) |
 | `series_of` | [`seriesof`](../seriesof/README.md) |
 | `parse_identifier` | [`idnorm`](../idnorm/README.md) |
+| `find_citations` | [`citefind`](../citefind/README.md) |
+| `citation_prompt` | the question `citefind --prompt` writes |
+| `merge_citation_reply` | what `citefind --merge` does with the answer |
 
 The tools that are not here are the ones a binding would gain nothing over a
 subprocess for: they read files, shell out to another program, or talk to a

crates/bindings/include/catena.h

@@ -88,6 +88,31 @@ char *catena_fuse_hits(const char *hits,
  * record. */
 char *catena_identify(const char *name, const uint8_t *data, size_t len);
 
+/* The series a title belongs to. Returns a JSON proposal, or NULL when the
+ * title says nothing about a series — an answer, not an error. `author` may
+ * be NULL. */
+char *catena_series_of(const char *title, const char *author);
+
+/* Read an identifier — a DOI, ISSN, ISBN or arXiv id — out of whatever it
+ * arrived wrapped in. Returns a JSON record, or NULL when the string is not
+ * one — an answer, not an error. */
+char *catena_parse_identifier(const char *text);
+
+/* Read the references out of a bibliography or a page of notes. Returns a
+ * JSON array, one object per reference. */
+char *catena_find_citations(const char *text);
+
+/* The question to put to a model about the references that did not parse.
+ * `references` is the JSON array catena_find_citations returned. Returns the
+ * prompt as a JSON string, or NULL when every reference settled — an answer,
+ * not an error. */
+char *catena_citation_prompt(const char *references);
+
+/* Fill in what did not parse from a model's reply. Returns the JSON array
+ * with the answer applied, or NULL and an error when the reply is not one
+ * JSON object. */
+char *catena_merge_citation_reply(const char *references, const char *reply);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif

crates/bindings/npm/typecheck.ts

@@ -4,7 +4,9 @@
 import {
   chunkMarkdown, scrub, countTokens, parseSubtitles, packCues,
   splitCollection, fuseHits, identify, seriesOf, parseIdentifier, segmentCues, cueWindows,
+  findCitations, citationPrompt, mergeCitationReply,
   type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, type Identifier, type Segment,
+  type Reference,
 } from "./dist/web/catena.js";
 
 const chunks: Chunk[] = chunkMarkdown("# One\n\nbody\n", 512, 64, 50, 768);
@@ -45,6 +47,14 @@ const proposal: SeriesProposal | undefined = seriesOf("The Philokalia Vol 3");
 const volume: number | undefined = proposal?.volume;
 seriesOf("Aquinas (Companion)", "Cambridge");
 
+const references: Reference[] = findCitations("Smith, John. *A Title*. Oxford: Clarendon Press, 1998.");
+const unsettled: string[] = references[0].unsettled;
+const ask: string | undefined = citationPrompt(references);
+const filled: Reference[] = mergeCitationReply(references, "{}");
+void unsettled;
+void ask;
+void filled;
+
 const ident: Identifier | undefined = parseIdentifier("10.1000/182");
 const checksum: boolean | null | undefined = ident?.checksum;
 

crates/bindings/python/catena/__init__.py

@@ -14,9 +14,12 @@ from ._catena import (
     __version__,
     cue_windows,
     chunk_markdown,
+    citation_prompt,
     count_tokens,
+    find_citations,
     fuse_hits,
     identify,
+    merge_citation_reply,
     pack_cues,
     parse_identifier,
     parse_subtitles,
@@ -30,9 +33,12 @@ __all__ = [
     "__version__",
     "cue_windows",
     "chunk_markdown",
+    "citation_prompt",
     "count_tokens",
+    "find_citations",
     "fuse_hits",
     "identify",
+    "merge_citation_reply",
     "pack_cues",
     "parse_identifier",
     "parse_subtitles",

crates/bindings/python/catena/__init__.pyi

@@ -37,6 +37,11 @@ def fuse_hits(
     limit: int | None = ...,
 ) -> list[dict[str, Any]]: ...
 def identify(name: str, data: bytes) -> dict[str, Any]: ...
+def find_citations(text: str) -> list[dict[str, Any]]: ...
+def citation_prompt(references: Iterable[Mapping[str, Any]]) -> str | None: ...
+def merge_citation_reply(
+    references: Iterable[Mapping[str, Any]], reply: str
+) -> list[dict[str, Any]]: ...
 def parse_identifier(text: str) -> dict[str, Any] | None: ...
 def series_of(
     title: str, author: str | None = ...

crates/bindings/src/capi.rs

@@ -417,6 +417,82 @@ pub unsafe extern "C" fn catena_identify(
     give_json(&crate::identify(name, bytes))
 }
 
+/// Read the references out of a bibliography or a page of notes. Returns a
+/// JSON array, one object per reference.
+///
+/// # Safety
+///
+/// `text` must be NUL-terminated UTF-8.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_find_citations(text: *const c_char) -> *mut c_char {
+    clear_error();
+    let Some(text) = (unsafe { borrow(text, "text") }) else {
+        return ptr::null_mut();
+    };
+    give_json(&crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+/// Takes the JSON array `catena_find_citations` returned. Returns the prompt
+/// as a JSON string, or NULL when every reference settled — an answer, not an
+/// error: `catena_last_error` is NULL in that case.
+///
+/// # Safety
+///
+/// `references` must be NUL-terminated UTF-8 holding a JSON array.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_citation_prompt(references: *const c_char) -> *mut c_char {
+    clear_error();
+    let Some(references) = (unsafe { borrow(references, "references") }) else {
+        return ptr::null_mut();
+    };
+    let references: Vec<citefind::Reference> = match serde_json::from_str(references) {
+        Ok(references) => references,
+        Err(why) => {
+            set_error(why.to_string());
+            return ptr::null_mut();
+        }
+    };
+    match crate::citation_prompt(&references) {
+        Some(prompt) => give_json(&prompt),
+        None => ptr::null_mut(),
+    }
+}
+
+/// Fill in what did not parse from a model's reply. Returns the JSON array
+/// with the answer applied, or NULL and an error when the reply is not one
+/// JSON object.
+///
+/// # Safety
+///
+/// `references` and `reply` must be NUL-terminated UTF-8.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn catena_merge_citation_reply(
+    references: *const c_char,
+    reply: *const c_char,
+) -> *mut c_char {
+    clear_error();
+    let (Some(references), Some(reply)) = (unsafe { borrow(references, "references") }, unsafe {
+        borrow(reply, "reply")
+    }) else {
+        return ptr::null_mut();
+    };
+    let references: Vec<citefind::Reference> = match serde_json::from_str(references) {
+        Ok(references) => references,
+        Err(why) => {
+            set_error(why.to_string());
+            return ptr::null_mut();
+        }
+    };
+    match crate::merge_citation_reply(&references, reply) {
+        Ok(merged) => give_json(&merged),
+        Err(why) => {
+            set_error(why);
+            ptr::null_mut()
+        }
+    }
+}
+
 /// The series a title belongs to. Returns a JSON proposal, or NULL when the
 /// title says nothing about a series — which is most titles, and not an
 /// error: `catena_last_error` is NULL in that case.

crates/bindings/src/lib.rs

@@ -215,6 +215,31 @@ pub fn parse_identifier(text: &str) -> Option<idnorm::Identifier> {
     idnorm::parse(text)
 }
 
+/// Read the references out of a bibliography or a page of notes.
+///
+/// Each carries `unsettled`: the fields that could not be read off the entry,
+/// named so a caller can ask somebody else about exactly those and no more.
+pub fn find_citations(text: &str) -> Vec<citefind::Reference> {
+    citefind::read(text)
+}
+
+/// Compose the question to put to a model about the references that did not
+/// parse. `None` when every one of them settled.
+pub fn citation_prompt(references: &[citefind::Reference]) -> Option<String> {
+    citefind::prompt::compose(references)
+}
+
+/// Fill in what did not parse from a model's reply, and return the references
+/// with the answer applied. `Err` when the reply is not one JSON object.
+pub fn merge_citation_reply(
+    references: &[citefind::Reference],
+    reply: &str,
+) -> Result<Vec<citefind::Reference>, String> {
+    let mut references = references.to_vec();
+    citefind::prompt::merge(&mut references, reply)?;
+    Ok(references)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 pub fn series_of(title: &str, author: Option<&str>) -> Option<seriesof::Proposal> {
     seriesof::series_of(title, author)
@@ -236,6 +261,25 @@ mod tests {
         assert_eq!(count_tokens("a passage"), textkit::count("a passage"));
     }
 
+    #[test]
+    fn the_citation_surface_agrees_with_the_library_underneath() {
+        const REFS: &str = "Smith, John. *A Title*. Oxford: Clarendon Press, 1998.\n\nMarron, Kevin Ritual Abuse, Seal Books, 1988";
+        let found = find_citations(REFS);
+        assert_eq!(found, citefind::read(REFS));
+
+        // The clean entry settles and the comma-only one does not, so there
+        // is a question to ask and it is about the second alone.
+        let prompt = citation_prompt(&found).expect("something unsettled");
+        assert!(!prompt.contains("Clarendon"), "{prompt}");
+        assert!(prompt.contains("Ritual Abuse"), "{prompt}");
+
+        let merged = merge_citation_reply(&found, r#"{"1": {"authors": ["Marron, Kevin"]}}"#)
+            .expect("a reply");
+        assert_eq!(merged[1].authors, ["Marron, Kevin"]);
+        assert_eq!(merged[0], found[0]);
+        assert!(merge_citation_reply(&found, "not json").is_err());
+    }
+
     #[test]
     fn the_default_budgets_are_the_tools_defaults() {
         let ours = ChunkBudgets::default();

crates/bindings/src/python.rs

@@ -220,6 +220,34 @@ fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult<Bound<'py
     to_py(py, &crate::identify(name, data))
 }
 
+/// Read the references out of a bibliography or a page of notes.
+#[pyfunction]
+fn find_citations<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
+    list_to_py(py, &crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+///
+/// `None` when every one of them settled, because there is nothing to ask.
+#[pyfunction]
+fn citation_prompt(references: &Bound<'_, PyAny>) -> PyResult<Option<String>> {
+    let references: Vec<citefind::Reference> = from_py_list(references)?;
+    Ok(crate::citation_prompt(&references))
+}
+
+/// Fill in what did not parse from a model's reply.
+#[pyfunction]
+fn merge_citation_reply<'py>(
+    py: Python<'py>,
+    references: &Bound<'py, PyAny>,
+    reply: &str,
+) -> PyResult<Bound<'py, PyList>> {
+    let references: Vec<citefind::Reference> = from_py_list(references)?;
+    let merged = crate::merge_citation_reply(&references, reply)
+        .map_err(pyo3::exceptions::PyValueError::new_err)?;
+    list_to_py(py, &merged)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 ///
 /// `None` when the title says nothing about a series, which is most titles.
@@ -261,6 +289,9 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(split_collection, module)?)?;
     module.add_function(wrap_pyfunction!(fuse_hits, module)?)?;
     module.add_function(wrap_pyfunction!(identify, module)?)?;
+    module.add_function(wrap_pyfunction!(find_citations, module)?)?;
+    module.add_function(wrap_pyfunction!(citation_prompt, module)?)?;
+    module.add_function(wrap_pyfunction!(merge_citation_reply, module)?)?;
     module.add_function(wrap_pyfunction!(series_of, module)?)?;
     module.add_function(wrap_pyfunction!(parse_identifier, module)?)?;
     Ok(())

crates/bindings/src/wasm.rs

@@ -97,6 +97,31 @@ export interface Identifier {
   url?: string;
 }
 
+/** One reference out of a bibliography or a page of notes. */
+export interface Reference {
+  v: number;
+  idx: number;
+  /** The entry as it appeared, whitespace collapsed. */
+  text: string;
+  /** The note number it carried, where it carried one. */
+  marker?: string;
+  /** The names, in the order and the form the entry gave them. */
+  authors: string[];
+  year?: number;
+  /** The work cited — for an article the article, not the journal. */
+  title?: string;
+  /** The journal it appeared in, or the publisher of a book. */
+  container?: string;
+  pages?: string;
+  ids: { kind: string; id: string }[];
+  /** "book", "article", "chapter", "web" or "other". */
+  kind: string;
+  /** Whether it only points back at an earlier entry. */
+  continuation: boolean;
+  /** The fields that could not be read off the entry. */
+  unsettled: string[];
+}
+
 /** What a title says about its series. */
 export interface SeriesProposal {
   v: number;
@@ -280,6 +305,32 @@ pub fn identify(name: &str, data: &[u8]) -> Result<JsValue, JsValue> {
     to_js(&crate::identify(name, data))
 }
 
+/// Read the references out of a bibliography or a page of notes.
+#[wasm_bindgen(js_name = findCitations, skip_typescript)]
+pub fn find_citations(text: &str) -> Result<JsValue, JsValue> {
+    to_js(&crate::find_citations(text))
+}
+
+/// The question to put to a model about the references that did not parse.
+///
+/// `undefined` when every one of them settled.
+#[wasm_bindgen(js_name = citationPrompt, skip_typescript)]
+pub fn citation_prompt(references: JsValue) -> Result<JsValue, JsValue> {
+    let references: Vec<citefind::Reference> = from_js(references, "references")?;
+    match crate::citation_prompt(&references) {
+        Some(prompt) => to_js(&prompt),
+        None => Ok(JsValue::UNDEFINED),
+    }
+}
+
+/// Fill in what did not parse from a model's reply.
+#[wasm_bindgen(js_name = mergeCitationReply, skip_typescript)]
+pub fn merge_citation_reply(references: JsValue, reply: &str) -> Result<JsValue, JsValue> {
+    let references: Vec<citefind::Reference> = from_js(references, "references")?;
+    let merged = crate::merge_citation_reply(&references, reply).map_err(JsValue::from)?;
+    to_js(&merged)
+}
+
 /// The series a title belongs to, and which volume of it this is.
 ///
 /// `undefined` when the title says nothing about a series.
@@ -314,6 +365,9 @@ export function cueWindows(cues: Cue[]): string[];
 export function splitCollection(markdown: string): Section[];
 export function fuseHits(hits: Hit[], k?: number, weights?: Record<string, number>, limit?: number): Hit[];
 export function identify(name: string, data: Uint8Array): Doc;
+export function findCitations(text: string): Reference[];
+export function citationPrompt(references: Reference[]): string | undefined;
+export function mergeCitationReply(references: Reference[], reply: string): Reference[];
 export function seriesOf(title: string, author?: string): SeriesProposal | undefined;
 export function parseIdentifier(text: string): Identifier | undefined;
 "#;

crates/chunkmd/src/footnote.rs

@@ -94,6 +94,28 @@ static CITE_SIGNAL: LazyLock<FancyRegex> = LazyLock::new(|| {
     .expect("a valid pattern")
 });
 
+/// Where each numbered note begins, in reading order.
+///
+/// Public because splitting a page of notes into the notes on it is the same
+/// knowledge as recognising one, and two copies of the pattern above would
+/// drift apart.
+pub fn note_starts(text: &str) -> Vec<usize> {
+    NOTE_MARKER
+        .find_iter(text)
+        .filter_map(Result::ok)
+        .map(|found| found.start())
+        .collect()
+}
+
+/// Whether an entry reads as a citation rather than as a remark.
+///
+/// Also public: a note that cites nothing is a note a bibliography has no
+/// use for, and the caller deciding that should ask the same question this
+/// module asks.
+pub fn cites_something(entry: &str) -> bool {
+    CITE_SIGNAL.is_match(entry).unwrap_or(false)
+}
+
 /// Whether a chunk carries footnote or endnote citation material.
 ///
 /// The structural signal first, when the source had one. Otherwise each note
@@ -104,11 +126,7 @@ pub fn looks_like_citations(text: &str) -> bool {
         return true;
     }
 
-    let starts: Vec<usize> = NOTE_MARKER
-        .find_iter(text)
-        .filter_map(Result::ok)
-        .map(|found| found.start())
-        .collect();
+    let starts = note_starts(text);
     if starts.len() < MIN_CITED_NOTES {
         return false;
     }
@@ -117,7 +135,7 @@ pub fn looks_like_citations(text: &str) -> bool {
     for (at, start) in starts.iter().enumerate() {
         let end = starts.get(at + 1).copied().unwrap_or(text.len());
         let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect();
-        if CITE_SIGNAL.is_match(&entry).unwrap_or(false) {
+        if cites_something(&entry) {
             cited += 1;
             if cited >= MIN_CITED_NOTES {
                 return true;

crates/citefind/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "citefind"
+description = "The references in a bibliography or a page of notes, one record each."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+catena-cli.workspace = true
+chunkmd.workspace = true
+clap.workspace = true
+corpus-core.workspace = true
+idnorm.workspace = true
+regex.workspace = true
+schemars.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+
+[lints]
+workspace = true

crates/citefind/README.md

@@ -0,0 +1,199 @@
+# citefind
+
+The references in a bibliography or a page of notes.
+
+Part of [catena](../../README.md).
+
+## What it does
+
+A reference list is the one place in a book where the text is already
+structured. Every entry names a work, and most name it in one of a handful of
+shapes that have not changed in a century. What is missing is not information
+but punctuation: the entries run together, and nothing in the file says where
+one stops.
+
+So this does two jobs. It cuts the block into entries — at the note numbers
+where there are any, at the layout where there are not. Then it reads each
+entry as far as its shape allows, and **names the fields it could not settle**
+rather than guessing at them.
+
+That last part is the point. A made-up year and a real one are
+indistinguishable three stages downstream; a missing one is not.
+
+## When you would use it
+
+- **Building a citation graph** — which works a book leans on, and how often.
+- **Finding the identifiers** in a reference list, to hand to
+  [`idnorm`](../idnorm/README.md) and [`bibfetch`](../bibfetch/README.md).
+- **Filling a catalogue** from the back of a book rather than from a register.
+- **Costing a model run**: ask one about the entries that did not parse, and
+  nothing else.
+
+## Install
+
+```sh
+cargo install --git https://git.pumpin.online/funkpower/catena citefind
+```
+
+## Example
+
+```
+$ citefind bibliography.md
+{"v":1,"idx":0,"text":"Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998.","authors":["Smith, John"],"year":1998,"title":"The Making of the Second Estate","container":"Clarendon Press","ids":[],"kind":"book","continuation":false,"unsettled":[]}
+{"v":1,"idx":1,"text":"Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.","authors":["Doe, Jane","Peter Brown"],"year":2001,"title":"Obligation and Its Limits","container":"Journal of Medieval Studies","pages":"45-67","ids":[],"kind":"article","continuation":false,"unsettled":[]}
+```
+
+What a book cites, by year:
+
+```sh
+citefind bibliography.md | jq -r 'select(.year) | .year' | sort | uniq -c
+```
+
+The identifiers, checked and resolved:
+
+```sh
+citefind refs.md | jq -r '.ids[] | select(.kind == "doi") | .id' | idnorm | bibfetch
+```
+
+The works it leans on hardest, from a page of endnotes:
+
+```sh
+citefind notes.md | jq -r 'select(.title) | .title' | sort | uniq -c | sort -rn | head
+```
+
+## Asking a model about the rest
+
+`--prompt` writes a question about the entries that did not parse, and only
+those. Nothing is sent from here: the prompt goes to stdout and what happens
+to it is yours to decide.
+
+```
+$ citefind --prompt refs.md
+Below are entries from the bibliography or notes of a book. Each was read as
+far as its punctuation allowed, and the fields that could not be read off it
+are listed with it.
+
+--- 0
+text: Marron, Kevin Ritual Abuse, Seal Books, 1988
+year: 1988
+wanted: authors, title, container
+...
+```
+
+```sh
+citefind --prompt refs.md | ollama run qwen3:8b > reply.json
+citefind --merge reply.json refs.md
+```
+
+A reply fills in only the fields that were asked about. A model that
+volunteers a better title for an entry whose title was read off the page is
+overruled — the page is evidence and the model is a guess — and a reply that
+is not one JSON object is refused rather than half-applied.
+
+One prompt covers every unsettled entry rather than one prompt each. A
+reference list is long and mostly ordinary; the entries worth asking about are
+the mangled few, and they are cheaper to ask about together.
+
+## Options
+
+| Option | Effect |
+|---|---|
+| `--prompt` | Write the question about what did not parse, instead of the records |
+| `--merge FILE` | Fill in what did not parse from a model's reply |
+| `--quiet`, `-q` | Say nothing; the exit code is the answer |
+| `--man` | Write this page as a manual page, in roff |
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| `0` | It found references |
+| `1` | It found none |
+| `64` | The command line was wrong |
+| `65` | The input was not readable as text, or a reply was not JSON |
+| `74` | The file could not be read |
+
+## The record
+
+| Field | What it holds |
+|---|---|
+| `text` | The entry as it appeared, whitespace collapsed |
+| `marker` | The note number it carried, where it carried one |
+| `authors` | The names, in the order and the form the entry gave them |
+| `year` | The year of publication |
+| `title` | The work cited — for an article the article, not the journal |
+| `container` | The journal it appeared in, or the publisher of a book |
+| `pages` | The pages, as written |
+| `ids` | Any DOI, ISBN, ISSN, arXiv id or URL, normalised |
+| `kind` | `book`, `article`, `chapter`, `web` or `other` |
+| `continuation` | Whether it only points back at an earlier entry |
+| `unsettled` | The fields it could not read off the entry |
+
+`text` is kept whatever else was read, because every other field is an
+interpretation of it and somebody will want to check.
+
+## How it cuts
+
+**A page of notes** is cut at its note numbers, in any of the shapes
+conversion leaves them: `12.`, `[12]`, `fn39.`, a superscript, or a
+back-reference link. The pattern is the one
+[`chunkmd`](../chunkmd/README.md) uses to recognise a page of notes, so the
+two cannot drift apart, with one form added — a link pointing at another file
+rather than at a fragment. `chunkmd` is deciding whether a chunk is a page of
+notes and can afford to miss one marker in thirty; here a missed marker runs
+two references together.
+
+A note that cites nothing is dropped. A page of notes mixes bibliographic
+notes with discursive ones, and a remark about the argument is not a
+reference.
+
+**A bibliography** numbers nothing, so the signal is layout: a blank line
+between entries, a hanging indent under each, or the surname-comma opening
+that nearly every author-first entry begins with. Whichever the block carries
+is the one used. When it carries none of them the fallback is one entry per
+line, which is right for a list that was never wrapped and visibly wrong for
+one that was.
+
+## What it will not do
+
+**Resolve `ibid.`** A continuation refers to whatever was cited before it,
+which is a fact about the surrounding page and not about the entry. It is
+flagged `continuation` and left alone, and it is left out of `--prompt`
+entirely — there is nothing in it to ask about.
+
+**Report a title it did not find.** An entry held together by commas alone,
+with no names to anchor it, yields no title rather than its own text echoed
+back.
+
+A rule of dashes where the author should be is the exception, and a
+deliberate one: `———. *The Second Book*.` means the author of the entry above,
+the style says so, and the entry it refers to is the one immediately before.
+That is read and filled in.
+
+## As a library
+
+```rust
+for one in citefind::read(&text) {
+    println!("{} ({}) {:?}", one.authors.join("; "), one.year.unwrap_or(0), one.title);
+}
+```
+
+`citefind::split::entries` cuts a block without reading the entries.
+`citefind::parse::one` reads an entry already cut. `citefind::prompt::compose`
+and `merge` are the two halves of the model-facing path, and
+`prompt::PROMPT_VERSION` says which wording produced a given answer.
+
+From Python, JavaScript or C, through
+[`bindings`](../bindings/README.md):
+
+```python
+for one in catena.find_citations(text):
+    print(one["year"], one["title"], one["unsettled"])
+```
+
+## See also
+
+- [`idnorm`](../idnorm/README.md) — the identifiers it finds, checked.
+- [`bibfetch`](../bibfetch/README.md) — what a register says about them.
+- [`chunkmd`](../chunkmd/README.md) — which flags the chunks worth feeding it.
+- [`bibclean`](../bibclean/README.md) — the same audit for a catalogue record.

crates/citefind/src/lib.rs

@@ -0,0 +1,214 @@
+//! The references inside a bibliography or a page of notes.
+//!
+//! A reference list is the one place in a book where the text is already
+//! structured — every entry names a work, and most of them name it in one of
+//! a handful of shapes that have not changed in a century. What is missing is
+//! not information but punctuation: the entries run together, and nothing in
+//! the file says where one stops.
+//!
+//! So this does two jobs. It cuts the block into entries, which is mechanical
+//! and gets it right or visibly wrong. Then it reads each entry as far as the
+//! shape allows, and says which fields it could not settle rather than
+//! guessing at them — because a made-up year is indistinguishable downstream
+//! from a real one, and an absence is not.
+//!
+//! What it will not do is resolve `ibid.`. A continuation refers to whatever
+//! was cited before it, which is a fact about the surrounding page and not
+//! about the entry; the entry is flagged as a continuation and left alone.
+
+pub mod parse;
+pub mod prompt;
+pub mod split;
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+/// What sort of work a reference points at.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum Kind {
+    /// A whole book, with a publisher.
+    Book,
+    /// An article in a journal.
+    Article,
+    /// A piece within a larger edited work.
+    Chapter,
+    /// Something whose address is a URL.
+    Web,
+    /// Recognisably a reference, not recognisably any of the above.
+    Other,
+}
+
+impl Kind {
+    /// What it is called.
+    pub fn name(self) -> &'static str {
+        match self {
+            Kind::Book => "book",
+            Kind::Article => "article",
+            Kind::Chapter => "chapter",
+            Kind::Web => "web",
+            Kind::Other => "other",
+        }
+    }
+}
+
+/// One reference, as found and as far as it could be read.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Reference {
+    /// Record version.
+    #[serde(default = "version")]
+    pub v: u32,
+    /// Position in the block, from zero.
+    pub idx: u32,
+    /// The entry exactly as it appeared, with its own whitespace collapsed.
+    ///
+    /// Kept whatever else was read, because every other field on this record
+    /// is an interpretation of this one and somebody will want to check it.
+    pub text: String,
+    /// The note number this entry carried, where it carried one.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub marker: Option<String>,
+    /// Who wrote it, in the order the entry gave them.
+    pub authors: Vec<String>,
+    /// The year of publication.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub year: Option<u32>,
+    /// The title of the work cited — the article, not the journal it is in.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+    /// What it appeared in: a journal, or the publisher of a book.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub container: Option<String>,
+    /// The pages, as written.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pages: Option<String>,
+    /// Any identifiers the entry carried, normalised.
+    pub ids: Vec<Id>,
+    /// What sort of work it is.
+    pub kind: Kind,
+    /// Whether this entry only points back at an earlier one: `ibid.`,
+    /// `op. cit.`, and the rest. What it points at is not read off the entry,
+    /// because the entry does not say.
+    pub continuation: bool,
+    /// The fields this entry did not settle, named so a caller can ask
+    /// somebody else about exactly those and no more.
+    pub unsettled: Vec<String>,
+}
+
+/// An identifier found in a reference, normalised to the form it stores in.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Id {
+    /// Which kind it is: `doi`, `isbn`, `issn`, `arxiv`, or `url`.
+    pub kind: String,
+    /// The identifier itself.
+    pub id: String,
+}
+
+fn version() -> u32 {
+    corpus_core::V
+}
+
+impl corpus_core::Record for Reference {
+    fn version(&self) -> u32 {
+        self.v
+    }
+}
+
+/// The fields a complete reference has, in the order they are reported.
+pub const FIELDS: [&str; 4] = ["authors", "year", "title", "container"];
+
+impl Reference {
+    /// An empty reference carrying only the text it was cut from.
+    pub fn of(text: &str) -> Self {
+        Reference {
+            v: corpus_core::V,
+            idx: 0,
+            text: text.to_string(),
+            marker: None,
+            authors: Vec::new(),
+            year: None,
+            title: None,
+            container: None,
+            pages: None,
+            ids: Vec::new(),
+            kind: Kind::Other,
+            continuation: false,
+            unsettled: Vec::new(),
+        }
+    }
+
+    /// Which of the four fields worth having are still empty.
+    ///
+    /// A continuation is excluded: it has nothing to settle, and asking about
+    /// one invites the guess this refuses to make.
+    pub fn missing(&self) -> Vec<&'static str> {
+        if self.continuation {
+            return Vec::new();
+        }
+        FIELDS
+            .into_iter()
+            .filter(|field| match *field {
+                "authors" => self.authors.is_empty(),
+                "year" => self.year.is_none(),
+                "title" => self.title.is_none(),
+                _ => self.container.is_none(),
+            })
+            .collect()
+    }
+}
+
+/// Read every reference in a block of text.
+///
+/// The block is cut into entries and each is read on its own. Entries that
+/// carry nothing citation-shaped at all are dropped: a bibliography page has
+/// a heading on it, and a heading is not a reference.
+pub fn read(text: &str) -> Vec<Reference> {
+    let mut out: Vec<Reference> = split::entries(text)
+        .into_iter()
+        .filter_map(|entry| parse::one(&entry.text, entry.marker.as_deref()))
+        .collect();
+
+    let mut previous: Vec<String> = Vec::new();
+    for (at, reference) in out.iter_mut().enumerate() {
+        reference.idx = at as u32;
+        // A rule of dashes where the author should be means the author of
+        // the entry above. That is not a guess: the style defines it, and
+        // the entry it refers to is the one immediately before. `ibid.` is
+        // refused because it points at a work, which may be pages back.
+        if reference.authors.is_empty() && parse::repeats_author(&reference.text) {
+            reference.authors = previous.clone();
+        }
+        if !reference.authors.is_empty() {
+            previous = reference.authors.clone();
+        }
+        reference.unsettled = reference.missing().iter().map(|s| s.to_string()).collect();
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_reference_reports_what_it_does_not_have() {
+        let mut one = Reference::of("something");
+        assert_eq!(one.missing(), FIELDS);
+        one.year = Some(1998);
+        one.title = Some("A Title".into());
+        assert_eq!(one.missing(), ["authors", "container"]);
+    }
+
+    #[test]
+    fn a_continuation_has_nothing_to_ask_about() {
+        let mut one = Reference::of("Ibid., 183.");
+        one.continuation = true;
+        assert!(one.missing().is_empty());
+    }
+
+    #[test]
+    fn a_block_of_nothing_yields_nothing() {
+        assert!(read("").is_empty());
+        assert!(read("Bibliography\n\n").is_empty());
+    }
+}

crates/citefind/src/main.rs

@@ -0,0 +1,110 @@
+//! The references in a bibliography or a page of notes.
+
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::exit;
+
+/// Read a reference list, one record per reference.
+///
+/// The input is the bibliography, works-cited list, or page of numbered
+/// notes as text. It is cut into entries — at the note numbers where there
+/// are any, at the layout where there are not — and each entry is read as far
+/// as its punctuation allows.
+///
+/// Fields that could not be read off the entry are left empty and named in
+/// `unsettled`, because a wrong year and a real one are indistinguishable
+/// downstream and a missing one is not. `--prompt` writes a question about
+/// exactly those, for a model of your choosing; nothing is sent from here.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Exit codes: 0 it found references, 1 it found none, 65 the input was not
+readable as text, 74 the file could not be read.
+
+Example:
+
+    citefind bibliography.md | jq -r '[.year, .title] | @tsv'
+    mdsplit book.md | citefind --prompt > ask.txt
+    citefind --prompt refs.md | ollama run qwen3:8b > reply.json
+    citefind --merge reply.json refs.md"
+)]
+struct Args {
+    /// The reference list to read. Standard input when not given.
+    path: Option<PathBuf>,
+
+    /// Say nothing; the exit code is the answer.
+    #[arg(long, short)]
+    quiet: bool,
+
+    /// Write the question to put to a model about the entries that did not
+    /// parse, instead of the records. Nothing is sent: it goes to stdout.
+    #[arg(long)]
+    prompt: bool,
+
+    /// Fill in what would not parse from a model's reply, written earlier by
+    /// `--prompt`, and write the completed records.
+    #[arg(long, value_name = "FILE")]
+    merge: Option<PathBuf>,
+}
+
+fn main() -> ExitCode {
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
+    if args.prompt && args.merge.is_some() {
+        eprintln!("citefind: --prompt composes a question and --merge answers one");
+        return ExitCode::from(exit::USAGE);
+    }
+
+    let text = match catena_cli::read_text(args.path.as_deref()) {
+        Ok(text) => text,
+        Err((code, why)) => {
+            eprintln!("citefind: {why}");
+            return ExitCode::from(code);
+        }
+    };
+
+    let mut references = citefind::read(&text);
+    if references.is_empty() {
+        if !args.quiet {
+            eprintln!("citefind: nothing in this looks like a reference");
+        }
+        return ExitCode::from(exit::NO);
+    }
+
+    if let Some(path) = args.merge.as_deref() {
+        let reply = match std::fs::read_to_string(path) {
+            Ok(reply) => reply,
+            Err(e) => {
+                eprintln!("citefind: {}: {e}", path.display());
+                return ExitCode::from(exit::IO);
+            }
+        };
+        match citefind::prompt::merge(&mut references, &reply) {
+            Err(why) => {
+                eprintln!("citefind: {why}");
+                return ExitCode::from(exit::DATA);
+            }
+            Ok(0) => eprintln!("citefind: the reply filled nothing in"),
+            Ok(filled) => eprintln!("citefind: {filled} entries filled in from the reply"),
+        }
+    }
+
+    if args.quiet {
+        return ExitCode::from(exit::OK);
+    }
+    if args.prompt {
+        return match citefind::prompt::compose(&references) {
+            // Nothing unsettled is not a failure, and a prompt nobody needs
+            // would waste a model's time.
+            None => ExitCode::from(exit::NO),
+            Some(prompt) => catena_cli::text("citefind", &prompt),
+        };
+    }
+    catena_cli::stream("citefind", &references)
+}

crates/citefind/src/prompt.rs

@@ -0,0 +1,194 @@
+//! A question to put to a model, for the references that would not parse.
+//!
+//! Nothing here sends anything. The prompt is composed and printed, and what
+//! happens to it is the caller's business — their model, their machine, their
+//! GPU. The tool stays what every tool here is: the same output for the same
+//! input, on every machine, with no network and nothing to install.
+//!
+//! One prompt covers every unsettled entry rather than one prompt each. A
+//! reference list is long and mostly ordinary; the entries worth asking about
+//! are the mangled few, and they are cheaper to ask about together.
+
+use crate::Reference;
+
+/// The version of the prompt below.
+///
+/// Bumped when the wording changes in a way that could change an answer, so a
+/// caller keeping results can tell which prompt produced them.
+pub const PROMPT_VERSION: u32 = 1;
+
+/// Compose one question covering every reference that did not fully parse.
+///
+/// `None` when every reference settled, because there is nothing to ask and
+/// a call to a model would be waste.
+pub fn compose(references: &[Reference]) -> Option<String> {
+    let wanted: Vec<&Reference> = references
+        .iter()
+        .filter(|one| !one.missing().is_empty())
+        .collect();
+    if wanted.is_empty() {
+        return None;
+    }
+
+    let mut out = String::new();
+    out.push_str(
+        "Below are entries from the bibliography or notes of a book. Each was \
+         read as far as its punctuation allowed, and the fields that could not \
+         be read off it are listed with it.\n\n",
+    );
+    for one in &wanted {
+        out.push_str(&format!("--- {}\n", one.idx));
+        out.push_str(&format!("text: {}\n", one.text));
+        let known = known(one);
+        if !known.is_empty() {
+            out.push_str(&known);
+        }
+        out.push_str(&format!("wanted: {}\n\n", one.missing().join(", ")));
+    }
+
+    out.push_str(
+        "Reply with one JSON object and nothing else. Its keys are the entry \
+         numbers above, as strings; each value is an object using only these \
+         keys, and only the ones listed as wanted for that entry: authors, \
+         year, title, container. `authors` is a list of names as the entry \
+         writes them. `year` is a four-digit number. `title` is the work \
+         cited — for an article the article, not the journal. `container` is \
+         the journal an article appeared in, or the publisher of a book.\n\n\
+         Use null for anything the entry does not settle. A guess is worse \
+         than an absence, because nobody downstream can tell them apart. Do \
+         not resolve `ibid.` or `op. cit.` — an entry that only points at an \
+         earlier one has nothing of its own to give.\n",
+    );
+    Some(out)
+}
+
+/// What was read off the entry, as the prompt shows it.
+fn known(one: &Reference) -> String {
+    let mut out = String::new();
+    if !one.authors.is_empty() {
+        out.push_str(&format!("authors: {}\n", one.authors.join("; ")));
+    }
+    if let Some(year) = one.year {
+        out.push_str(&format!("year: {year}\n"));
+    }
+    if let Some(title) = &one.title {
+        out.push_str(&format!("title: {title}\n"));
+    }
+    if let Some(container) = &one.container {
+        out.push_str(&format!("container: {container}\n"));
+    }
+    out
+}
+
+/// Take a model's reply and fill in the fields that were missing.
+///
+/// Only the missing ones, and only on entries that asked: a model that
+/// volunteers a better title for a reference whose title was read off the
+/// page is overruled, because the page is evidence and the model is a guess.
+/// A reply that is not one JSON object is refused rather than half-applied.
+pub fn merge(references: &mut [Reference], reply: &str) -> Result<usize, String> {
+    let json = object_in(reply).ok_or("no JSON object in the reply")?;
+    let value: serde_json::Value =
+        serde_json::from_str(json).map_err(|e| format!("not readable as JSON: {e}"))?;
+    let object = value.as_object().ok_or("the reply is not an object")?;
+
+    let mut filled = 0;
+    for one in references.iter_mut() {
+        let missing = one.missing();
+        if missing.is_empty() {
+            continue;
+        }
+        let Some(answer) = object.get(&one.idx.to_string()) else {
+            continue;
+        };
+        let mut touched = false;
+
+        if missing.contains(&"authors") {
+            let authors: Vec<String> = answer
+                .get("authors")
+                .and_then(serde_json::Value::as_array)
+                .map(|list| {
+                    list.iter()
+                        .filter_map(serde_json::Value::as_str)
+                        .map(|name| name.trim().to_string())
+                        .filter(|name| !name.is_empty())
+                        .collect()
+                })
+                .unwrap_or_default();
+            if !authors.is_empty() {
+                one.authors = authors;
+                touched = true;
+            }
+        }
+        if missing.contains(&"year") {
+            let year = answer
+                .get("year")
+                .and_then(|found| match found {
+                    serde_json::Value::Number(number) => number.as_u64(),
+                    serde_json::Value::String(text) => text.trim().parse().ok(),
+                    _ => None,
+                })
+                .filter(|year| (1000..=2100).contains(year));
+            if let Some(year) = year {
+                one.year = Some(year as u32);
+                touched = true;
+            }
+        }
+        for (key, slot) in [("title", &mut one.title), ("container", &mut one.container)] {
+            if !missing.contains(&key) {
+                continue;
+            }
+            let found = answer
+                .get(key)
+                .and_then(serde_json::Value::as_str)
+                .map(str::trim)
+                .filter(|text| !text.is_empty());
+            if let Some(found) = found {
+                *slot = Some(found.to_string());
+                touched = true;
+            }
+        }
+
+        if touched {
+            filled += 1;
+        }
+        one.unsettled = one.missing().iter().map(|s| s.to_string()).collect();
+    }
+    Ok(filled)
+}
+
+/// The first balanced `{…}` in a reply.
+///
+/// Models wrap JSON in prose and in code fences however they were asked not
+/// to, and refusing a good answer over a fence helps nobody.
+fn object_in(reply: &str) -> Option<&str> {
+    let start = reply.find('{')?;
+    let bytes = reply.as_bytes();
+    let mut depth = 0usize;
+    let mut in_string = false;
+    let mut escaped = false;
+    for at in start..bytes.len() {
+        let byte = bytes[at];
+        if in_string {
+            match (escaped, byte) {
+                (true, _) => escaped = false,
+                (false, b'\\') => escaped = true,
+                (false, b'"') => in_string = false,
+                _ => {}
+            }
+            continue;
+        }
+        match byte {
+            b'"' => in_string = true,
+            b'{' => depth += 1,
+            b'}' => {
+                depth -= 1;
+                if depth == 0 {
+                    return Some(&reply[start..=at]);
+                }
+            }
+            _ => {}
+        }
+    }
+    None
+}

crates/citefind/src/split.rs

@@ -0,0 +1,383 @@
+//! Cutting a block of references into the references in it.
+//!
+//! Two shapes arrive. A page of notes numbers its entries, and the numbering
+//! says exactly where each one starts — that case is decided, not guessed at,
+//! and the pattern that recognises it is the one `chunkmd` already uses to
+//! recognise a page of notes, so the two cannot drift apart.
+//!
+//! A bibliography numbers nothing. There the signal is layout: a blank line
+//! between entries, a hanging indent under each, or the surname-comma opening
+//! that nearly every author-first entry begins with. Whichever of those the
+//! block actually carries is the one used, and when it carries none of them
+//! the fallback is one entry per line, which is right for a list that was
+//! never wrapped and visibly wrong for one that was.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// One entry, cut out of the block.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Entry {
+    /// The entry's text, with its marker removed and whitespace collapsed.
+    pub text: String,
+    /// The note number it carried, where it carried one.
+    pub marker: Option<String>,
+}
+
+/// At least this many numbered notes before the numbering is believed.
+///
+/// Two: one is a stray list item, and a real page of notes has more.
+const MIN_NOTES: usize = 2;
+
+/// The opening of an author-first bibliography entry: a surname, a comma,
+/// and something after it. Deliberately narrow — it is being used to decide
+/// where an entry *starts*, and a loose version would cut entries in half at
+/// every "Oxford, 1998".
+static SURNAME_FIRST: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r"^[\p{Lu}][\p{L}'’\-]{1,30}(?:\s+[\p{Lu}][\p{L}'’\-]{1,30}){0,2},\s+[\p{Lu}]")
+        .expect("a valid pattern")
+});
+
+/// A leading note number, once the tags around it are gone.
+///
+/// Three digits at most, which is what `chunkmd` counts as a note number.
+/// Four would swallow the year off the front of an author-date entry.
+static LEADING_NUMBER: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^\[?(\d{1,3})\]?\\?[.)]?\s+").expect("a valid pattern"));
+
+/// An HTML tag, which conversion leaves wrapped around note numbers.
+static TAG: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"</?[a-zA-Z][^>]*>").expect("a valid pattern"));
+
+/// A markdown link left where a back-reference anchor was. The href points
+/// at a fragment in some EPUBs and at another file in most: `[42](#note-42)`
+/// and `[4.](part0011.html#ke_00001)` are the same marker.
+static ANCHOR_LINK: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r"^\[\s*(\d{1,4})\s*\\?[.)]?\s*\]\([^)\s]*\)\s*[.)]?\s*").expect("a valid pattern")
+});
+
+/// A marker `chunkmd` does not count, because it does not need to: a
+/// back-reference whose link points at another file rather than at a
+/// fragment. `chunkmd` is deciding whether a chunk is a page of notes, and
+/// one missed marker among thirty does not change that answer. Here a missed
+/// marker runs two references together, so these are added to its.
+static FILE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r"(?m)^[ \t]{0,8}(?:[>•]+[ \t]*|-[ \t]+)?\[\s*\d{1,3}\s*\\?[.)]?\s*\]\([^)\s]*\)")
+        .expect("a valid pattern")
+});
+
+/// A bullet or a blockquote marker, which conversion puts in front of a note
+/// often enough to matter. A leading `*` is left alone: it opens an italic
+/// title as often as it opens a list.
+static BULLET: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^(?:[>•]+\s*|-\s+)+").expect("a valid pattern"));
+
+/// An id-style marker: `fn39.`, `note12)`.
+static ID_MARKER: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^(?:fn|ftn|note)(\d{1,4})[.)]\s+").expect("a valid pattern"));
+
+/// Cut a block into entries.
+pub fn entries(text: &str) -> Vec<Entry> {
+    let numbered = numbered_notes(text);
+    if numbered.len() >= MIN_NOTES {
+        return numbered;
+    }
+    unnumbered(text)
+}
+
+/// Entries cut at their note numbers.
+///
+/// An entry runs from its marker to the next one. Only entries that cite
+/// something are kept: a page of notes mixes bibliographic notes with
+/// discursive ones, and a remark about the argument is not a reference.
+fn numbered_notes(text: &str) -> Vec<Entry> {
+    let mut starts = chunkmd::footnote::note_starts(text);
+    starts.extend(FILE_ANCHOR.find_iter(text).map(|found| found.start()));
+    starts.sort_unstable();
+    starts.dedup();
+    // Decided before the head is added, so that one stray `12.` in a
+    // bibliography cannot turn the whole of it into two entries.
+    if starts.len() < MIN_NOTES {
+        return Vec::new();
+    }
+    // Whatever is in front of the first marker is the first note, when the
+    // marker it carried was one neither pattern knows. Dropping it would
+    // lose a reference silently, which is the one thing a cut must not do.
+    if starts[0] > 0 {
+        starts.insert(0, 0);
+    }
+
+    let mut out = Vec::new();
+    for (at, start) in starts.iter().enumerate() {
+        let end = starts.get(at + 1).copied().unwrap_or(text.len());
+        let raw = &text[*start..end];
+        if !chunkmd::footnote::cites_something(raw) {
+            continue;
+        }
+        out.push(strip_marker(raw));
+    }
+    out
+}
+
+/// Entries cut at whatever the layout offers.
+fn unnumbered(text: &str) -> Vec<Entry> {
+    let blocks: Vec<&str> = text
+        .split("\n\n")
+        .map(str::trim)
+        .filter(|block| !block.is_empty())
+        .collect();
+    if blocks.len() >= 2 {
+        return blocks.iter().map(|block| strip_marker(block)).collect();
+    }
+
+    let lines: Vec<&str> = text
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .collect();
+    if lines.len() < 2 {
+        return lines.iter().map(|line| strip_marker(line)).collect();
+    }
+
+    // A surname-comma opening on most lines means the list is one entry per
+    // line and any line without it is a wrap of the line above. Without that,
+    // fall back to the hanging indent, and without that to the lines
+    // themselves.
+    let openers = lines
+        .iter()
+        .filter(|line| SURNAME_FIRST.is_match(line.trim_start()))
+        .count();
+    let by_surname = openers >= 2 && openers * 2 >= lines.len();
+
+    let mut out: Vec<String> = Vec::new();
+    for line in &lines {
+        let indented = line.starts_with(' ') || line.starts_with('\t');
+        let starts_entry = match by_surname {
+            true => SURNAME_FIRST.is_match(line.trim_start()),
+            false => !indented,
+        };
+        match (starts_entry, out.last_mut()) {
+            (false, Some(previous)) => {
+                previous.push(' ');
+                previous.push_str(line.trim());
+            }
+            _ => out.push(line.trim().to_string()),
+        }
+    }
+    out.iter().map(|entry| strip_marker(entry)).collect()
+}
+
+/// Take the note number off the front of an entry, and tidy what is left.
+///
+/// The number arrives wrapped in whatever the converter left behind, so the
+/// patterns are matched against the opening with its tags removed — and the
+/// cut is then made in the original, so that an italic span marking a title
+/// further along survives.
+fn strip_marker(raw: &str) -> Entry {
+    let text = tidy(raw);
+    let text = BULLET.replace(&text, "").to_string();
+    let (bare, from) = visible(&text, 160);
+
+    for pattern in [&*ANCHOR_LINK, &*ID_MARKER, &*LEADING_NUMBER] {
+        let Some(found) = pattern.captures(&bare) else {
+            continue;
+        };
+        let whole = found.get(0).expect("the whole match");
+        let consumed = bare[..whole.end()].chars().count();
+        let cut = from[consumed];
+        return Entry {
+            text: text[cut..].trim_start().to_string(),
+            marker: Some(found[1].to_string()),
+        };
+    }
+
+    if let Some((marker, consumed)) = superscript(&bare) {
+        return Entry {
+            text: text[from[consumed]..].trim_start().to_string(),
+            marker: Some(marker),
+        };
+    }
+
+    Entry { text, marker: None }
+}
+
+/// The opening of an entry with its tags dropped, and where each character
+/// that was kept came from in the original.
+///
+/// The last element is where the kept text ends, so that a marker running to
+/// the end of the opening still has somewhere to cut.
+fn visible(text: &str, limit: usize) -> (String, Vec<usize>) {
+    let mut out = String::new();
+    let mut from = Vec::new();
+    let mut at = 0;
+    while at < text.len() && from.len() < limit {
+        let rest = &text[at..];
+        if rest.starts_with('<') {
+            if let Some(close) = rest.find('>') {
+                if TAG.is_match(&rest[..=close]) {
+                    at += close + 1;
+                    continue;
+                }
+            }
+        }
+        let ch = rest.chars().next().expect("a character");
+        if !(out.is_empty() && ch.is_whitespace()) {
+            out.push(ch);
+            from.push(at);
+        }
+        at += ch.len_utf8();
+    }
+    from.push(at);
+    (out, from)
+}
+
+/// A leading run of superscript digits, as the number it spells and how many
+/// characters it took.
+fn superscript(text: &str) -> Option<(String, usize)> {
+    const SUPER: [char; 10] = ['⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹'];
+    let mut digits = String::new();
+    let mut taken = 0;
+    for ch in text.chars() {
+        match SUPER.iter().position(|s| *s == ch) {
+            Some(value) => {
+                digits.push(char::from_digit(value as u32, 10)?);
+                taken += 1;
+            }
+            None => break,
+        }
+    }
+    if digits.is_empty() {
+        return None;
+    }
+    // Step over the space the marker is separated from its entry by.
+    let after: String = text
+        .chars()
+        .skip(taken)
+        .take_while(|c| c.is_whitespace())
+        .collect();
+    Some((digits, taken + after.chars().count()))
+}
+
+/// Collapse the whitespace a converted entry is wrapped with.
+fn tidy(text: &str) -> String {
+    text.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn texts(entries: &[Entry]) -> Vec<&str> {
+        entries.iter().map(|entry| entry.text.as_str()).collect()
+    }
+
+    #[test]
+    fn a_page_of_numbered_notes_is_cut_at_its_numbers() {
+        let page = "\
+1. John Smith, *The Title* (Oxford: Clarendon Press, 1998), 183.
+2. Jane Doe, *Another Book* (Cambridge: University Press, 2001), p. 12.";
+        let found = entries(page);
+        assert_eq!(found.len(), 2, "{found:#?}");
+        assert_eq!(found[0].marker.as_deref(), Some("1"));
+        assert_eq!(found[1].marker.as_deref(), Some("2"));
+        assert!(found[0].text.starts_with("John Smith"), "{:?}", found[0]);
+    }
+
+    #[test]
+    fn a_note_that_cites_nothing_is_not_a_reference() {
+        let page = "\
+1. John Smith, *The Title* (Oxford: Clarendon Press, 1998), 183.
+2. I am indebted to the seminar for this point.
+3. Jane Doe, *Another Book* (Cambridge: University Press, 2001), p. 12.";
+        let found = entries(page);
+        assert_eq!(
+            found
+                .iter()
+                .filter_map(|e| e.marker.clone())
+                .collect::<Vec<_>>(),
+            ["1", "3"]
+        );
+    }
+
+    #[test]
+    fn blank_lines_cut_a_bibliography() {
+        let block = "Smith, John. The Title. Oxford: Clarendon, 1998.\n\nDoe, Jane. Another.\nCambridge, 2001.";
+        let found = entries(block);
+        assert_eq!(found.len(), 2);
+        assert_eq!(found[1].text, "Doe, Jane. Another. Cambridge, 2001.");
+    }
+
+    #[test]
+    fn a_surname_opening_cuts_a_list_that_was_wrapped() {
+        let block = "\
+Smith, John. The Title of the Book That Ran On.
+Oxford: Clarendon Press, 1998.
+Doe, Jane. Another Work Entirely. Cambridge, 2001.";
+        let found = entries(block);
+        assert_eq!(
+            texts(&found),
+            [
+                "Smith, John. The Title of the Book That Ran On. Oxford: Clarendon Press, 1998.",
+                "Doe, Jane. Another Work Entirely. Cambridge, 2001.",
+            ]
+        );
+    }
+
+    #[test]
+    fn a_hanging_indent_cuts_a_list_with_no_surnames_to_go_on() {
+        let block = "\
+The Oxford Companion to Everything. Oxford, 1998.
+    Second line of the same entry.
+Another Anonymous Work. Cambridge, 2001.";
+        let found = entries(block);
+        assert_eq!(found.len(), 2, "{found:#?}");
+        assert!(found[0].text.ends_with("same entry."), "{:?}", found[0]);
+    }
+
+    #[test]
+    fn a_marker_wrapped_in_tags_is_still_a_marker() {
+        let entry = strip_marker("<a href=\"#ref12\"><sup>12</sup></a>. Smith, *Title*, p. 4.");
+        assert_eq!(entry.marker.as_deref(), Some("12"));
+        assert_eq!(entry.text, "Smith, *Title*, p. 4.");
+    }
+
+    #[test]
+    fn an_anchor_link_marker_keeps_the_entry_and_not_the_link() {
+        let entry = strip_marker("[42](#note-42). Smith, *Title*, p. 4.");
+        assert_eq!(entry.marker.as_deref(), Some("42"));
+        assert_eq!(entry.text, "Smith, *Title*, p. 4.");
+    }
+
+    #[test]
+    fn an_id_style_marker_is_read_as_its_number() {
+        let entry = strip_marker("fn39. Smith, *Title*, p. 4.");
+        assert_eq!(entry.marker.as_deref(), Some("39"));
+        assert_eq!(entry.text, "Smith, *Title*, p. 4.");
+    }
+
+    #[test]
+    fn a_superscript_marker_is_read_as_its_number() {
+        let entry = strip_marker("¹² Smith, *Title*, p. 4.");
+        assert_eq!(entry.marker.as_deref(), Some("12"));
+        assert_eq!(entry.text, "Smith, *Title*, p. 4.");
+    }
+
+    #[test]
+    fn an_entry_with_no_marker_keeps_all_of_itself() {
+        let entry = strip_marker("Smith, John. The Title. Oxford, 1998.");
+        assert_eq!(entry.marker, None);
+        assert_eq!(entry.text, "Smith, John. The Title. Oxford, 1998.");
+    }
+
+    #[test]
+    fn a_year_in_a_place_of_publication_does_not_start_an_entry() {
+        // "Oxford, 1998." would match a loose surname-comma rule.
+        let block = "\
+Smith, John. A Title.
+Oxford, 1998.
+Doe, Jane. Another.
+Cambridge, 2001.";
+        let found = entries(block);
+        assert_eq!(found.len(), 2, "{found:#?}");
+    }
+}

crates/citefind/tests/references.rs

@@ -0,0 +1,177 @@
+//! Reading a block of references end to end.
+
+use citefind::{Kind, read};
+
+/// A page of endnotes as conversion leaves it: numbers, anchors, and a
+/// discursive note in among the citations.
+const NOTES: &str = "\
+[1.](part0011.html#ke_00001_00003) John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154.
+2. I am grateful to the seminar at Leiden for pressing me on this.
+[3](text0007.xhtml#en3). Johanna Bockman, *Markets in the Name of Socialism* (Stanford, CA: Stanford University Press, 2011), 40-42.
+4. Ibid., p. 44.
+";
+
+#[test]
+fn a_page_of_notes_keeps_its_numbering_and_drops_its_asides() {
+    let found = read(NOTES);
+    assert_eq!(found.len(), 3, "{found:#?}");
+    assert_eq!(
+        found
+            .iter()
+            .map(|one| one.marker.as_deref())
+            .collect::<Vec<_>>(),
+        [Some("1"), Some("3"), Some("4")]
+    );
+    // Numbered from zero in the order they appeared, whatever the markers say.
+    assert_eq!(
+        found.iter().map(|one| one.idx).collect::<Vec<_>>(),
+        [0, 1, 2]
+    );
+    assert_eq!(found[0].authors, ["John MacGowan"]);
+    assert_eq!(found[0].container.as_deref(), Some("Curzon"));
+    assert_eq!(found[1].year, Some(2011));
+    assert_eq!(found[1].kind, Kind::Book);
+    assert!(found[2].continuation);
+}
+
+#[test]
+fn a_rule_of_dashes_means_the_author_above() {
+    let block = "\
+Smith, John. *The First Book*. Oxford: Clarendon Press, 1998.
+
+———. *The Second Book*. Oxford: Clarendon Press, 2001.
+
+——— *The Third Book*. Oxford: Clarendon Press, 2004.
+
+Doe, Jane. *Something Else*. Cambridge: University Press, 2005.
+
+———. *And Another*. Cambridge: University Press, 2007.";
+    let found = read(block);
+    assert_eq!(found.len(), 5, "{found:#?}");
+    assert_eq!(
+        found
+            .iter()
+            .map(|one| one.authors.clone())
+            .collect::<Vec<_>>(),
+        [
+            vec!["Smith, John"],
+            vec!["Smith, John"],
+            vec!["Smith, John"],
+            vec!["Doe, Jane"],
+            vec!["Doe, Jane"],
+        ]
+    );
+    assert_eq!(found[2].title.as_deref(), Some("The Third Book"));
+}
+
+#[test]
+fn an_ibid_does_not_inherit_an_author_the_way_a_dash_does() {
+    let block = "\
+Smith, John. *The First Book*. Oxford: Clarendon Press, 1998.
+
+Ibid., p. 44.";
+    let found = read(block);
+    assert_eq!(found.len(), 2);
+    assert!(found[1].authors.is_empty(), "{:#?}", found[1]);
+    assert!(found[1].continuation);
+}
+
+#[test]
+fn every_record_says_what_it_could_not_settle() {
+    let found = read(
+        "Marron, Kevin Ritual Abuse, Seal Books, 1988\n\nAnother, Line. Of nothing much, 1990.",
+    );
+    for one in &found {
+        assert_eq!(
+            one.unsettled,
+            one.missing()
+                .iter()
+                .map(|s| s.to_string())
+                .collect::<Vec<_>>()
+        );
+    }
+}
+
+#[test]
+fn the_same_block_reads_the_same_way_every_time() {
+    assert_eq!(read(NOTES), read(NOTES));
+}
+
+/// A block mixing one entry that reads cleanly, one that gives up nothing
+/// but a title, and a continuation.
+const MIXED: &str = "\
+1. John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154.
+2. Marron, Kevin Ritual Abuse, Seal Books, 1988.
+3. Ibid., p. 44.
+";
+
+#[test]
+fn a_prompt_asks_about_the_entries_that_did_not_settle_and_no_others() {
+    let found = read(MIXED);
+    assert_eq!(found.len(), 3, "{found:#?}");
+    let prompt = citefind::prompt::compose(&found).expect("something unsettled");
+
+    // The entry that read cleanly is not in the question, and neither is the
+    // continuation — it has nothing of its own to give.
+    assert!(!prompt.contains("MacGowan"), "{prompt}");
+    assert!(!prompt.contains("Ibid"), "{prompt}");
+    assert!(prompt.contains("Ritual Abuse"), "{prompt}");
+    assert!(prompt.contains("wanted:"), "{prompt}");
+}
+
+#[test]
+fn a_prompt_is_only_composed_where_there_is_something_to_ask() {
+    let settled = read(
+        "1. John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154.\n2. Johanna Bockman, *Markets in the Name of Socialism* (Stanford, CA: Stanford University Press, 2011), 40.",
+    );
+    assert!(
+        settled.iter().all(|one| one.unsettled.is_empty()),
+        "{settled:#?}"
+    );
+    assert_eq!(citefind::prompt::compose(&settled), None);
+}
+
+#[test]
+fn a_reply_about_an_entry_that_settled_does_not_overrule_the_page() {
+    let mut found = read(MIXED);
+    let before = found[0].clone();
+    let filled = citefind::prompt::merge(
+        &mut found,
+        r#"{"0": {"title": "Invented", "container": "Invented"}, "2": {"title": "Invented"}}"#,
+    )
+    .expect("a reply");
+    assert_eq!(filled, 0);
+    assert_eq!(found[0], before);
+    assert_eq!(found[2].title, None);
+}
+
+#[test]
+fn a_reply_fills_in_only_what_was_asked_about() {
+    let mut found = read("Marron, Kevin Ritual Abuse, Seal Books, 1988");
+    assert_eq!(found.len(), 1);
+    assert!(found[0].unsettled.contains(&"authors".to_string()));
+
+    let filled = citefind::prompt::merge(
+        &mut found,
+        "Sure!\n```json\n{\"0\": {\"authors\": [\"Marron, Kevin\"], \"year\": 1066}}\n```",
+    )
+    .expect("a reply");
+    assert_eq!(filled, 1);
+    assert_eq!(found[0].authors, ["Marron, Kevin"]);
+    // The year was read off the page, so the model's does not replace it.
+    assert_eq!(found[0].year, Some(1988));
+    assert!(!found[0].unsettled.contains(&"authors".to_string()));
+}
+
+#[test]
+fn a_reply_that_is_not_json_is_refused_rather_than_half_applied() {
+    let mut found = read(NOTES);
+    let before = found.clone();
+    assert!(citefind::prompt::merge(&mut found, "I could not tell.").is_err());
+    assert_eq!(found, before);
+}
+
+#[test]
+fn nothing_citation_shaped_gives_nothing() {
+    assert!(read("Bibliography\n\nSee below.\n\n233\n").is_empty());
+}