diff --git a/Cargo.lock b/Cargo.lock index f791068..b9fbcda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,6 +254,7 @@ version = "0.1.0" dependencies = [ "booksniff", "chunkmd", + "citefind", "corpus-core", "cuekit", "cuepack", @@ -328,6 +329,21 @@ dependencies = [ "inout", ] +[[package]] +name = "citefind" +version = "0.1.0" +dependencies = [ + "catena-cli", + "chunkmd", + "clap", + "corpus-core", + "idnorm", + "regex", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "clap" version = "4.6.7" diff --git a/Cargo.toml b/Cargo.toml index b986cb7..e9e9b37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ bookmeta = { path = "crates/bookmeta", version = "0.1.0" } langid = { path = "crates/langid", version = "0.1.0" } tocfind = { path = "crates/tocfind", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } +citefind = { path = "crates/citefind", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } clap = { version = "4", features = ["derive"] } diff --git a/README.md b/README.md index 94e651c..7175e69 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Each tool installs by name, so you can take one without taking the rest. | [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is | | [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked | | [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier | +| [`citefind`](crates/citefind/README.md) | The references in a bibliography or a page of notes | | [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out | | [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it | | [`chunkmd`](crates/chunkmd/README.md) | Markdown into passages that carry their page, chapter and role | diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index 802f9db..4ceafac 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -30,6 +30,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] booksniff.workspace = true chunkmd.workspace = true +citefind.workspace = true corpus-core.workspace = true idnorm.workspace = true cuekit.workspace = true diff --git a/crates/bindings/README.md b/crates/bindings/README.md index f5fbb27..8907f33 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -78,6 +78,9 @@ except ValueError as why: | `identify` | [`booksniff`](../booksniff/README.md) | | `series_of` | [`seriesof`](../seriesof/README.md) | | `parse_identifier` | [`idnorm`](../idnorm/README.md) | +| `find_citations` | [`citefind`](../citefind/README.md) | +| `citation_prompt` | the question `citefind --prompt` writes | +| `merge_citation_reply` | what `citefind --merge` does with the answer | The tools that are not here are the ones a binding would gain nothing over a subprocess for: they read files, shell out to another program, or talk to a diff --git a/crates/bindings/include/catena.h b/crates/bindings/include/catena.h index baab238..2a1f18d 100644 --- a/crates/bindings/include/catena.h +++ b/crates/bindings/include/catena.h @@ -88,6 +88,31 @@ char *catena_fuse_hits(const char *hits, * record. */ char *catena_identify(const char *name, const uint8_t *data, size_t len); +/* The series a title belongs to. Returns a JSON proposal, or NULL when the + * title says nothing about a series — an answer, not an error. `author` may + * be NULL. */ +char *catena_series_of(const char *title, const char *author); + +/* Read an identifier — a DOI, ISSN, ISBN or arXiv id — out of whatever it + * arrived wrapped in. Returns a JSON record, or NULL when the string is not + * one — an answer, not an error. */ +char *catena_parse_identifier(const char *text); + +/* Read the references out of a bibliography or a page of notes. Returns a + * JSON array, one object per reference. */ +char *catena_find_citations(const char *text); + +/* The question to put to a model about the references that did not parse. + * `references` is the JSON array catena_find_citations returned. Returns the + * prompt as a JSON string, or NULL when every reference settled — an answer, + * not an error. */ +char *catena_citation_prompt(const char *references); + +/* Fill in what did not parse from a model's reply. Returns the JSON array + * with the answer applied, or NULL and an error when the reply is not one + * JSON object. */ +char *catena_merge_citation_reply(const char *references, const char *reply); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/crates/bindings/npm/typecheck.ts b/crates/bindings/npm/typecheck.ts index 85669c7..cff81a5 100644 --- a/crates/bindings/npm/typecheck.ts +++ b/crates/bindings/npm/typecheck.ts @@ -4,7 +4,9 @@ import { chunkMarkdown, scrub, countTokens, parseSubtitles, packCues, splitCollection, fuseHits, identify, seriesOf, parseIdentifier, segmentCues, cueWindows, + findCitations, citationPrompt, mergeCitationReply, type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, type Identifier, type Segment, + type Reference, } from "./dist/web/catena.js"; const chunks: Chunk[] = chunkMarkdown("# One\n\nbody\n", 512, 64, 50, 768); @@ -45,6 +47,14 @@ const proposal: SeriesProposal | undefined = seriesOf("The Philokalia Vol 3"); const volume: number | undefined = proposal?.volume; seriesOf("Aquinas (Companion)", "Cambridge"); +const references: Reference[] = findCitations("Smith, John. *A Title*. Oxford: Clarendon Press, 1998."); +const unsettled: string[] = references[0].unsettled; +const ask: string | undefined = citationPrompt(references); +const filled: Reference[] = mergeCitationReply(references, "{}"); +void unsettled; +void ask; +void filled; + const ident: Identifier | undefined = parseIdentifier("10.1000/182"); const checksum: boolean | null | undefined = ident?.checksum; diff --git a/crates/bindings/python/catena/__init__.py b/crates/bindings/python/catena/__init__.py index 73685f9..9199aba 100644 --- a/crates/bindings/python/catena/__init__.py +++ b/crates/bindings/python/catena/__init__.py @@ -14,9 +14,12 @@ from ._catena import ( __version__, cue_windows, chunk_markdown, + citation_prompt, count_tokens, + find_citations, fuse_hits, identify, + merge_citation_reply, pack_cues, parse_identifier, parse_subtitles, @@ -30,9 +33,12 @@ __all__ = [ "__version__", "cue_windows", "chunk_markdown", + "citation_prompt", "count_tokens", + "find_citations", "fuse_hits", "identify", + "merge_citation_reply", "pack_cues", "parse_identifier", "parse_subtitles", diff --git a/crates/bindings/python/catena/__init__.pyi b/crates/bindings/python/catena/__init__.pyi index 39a1180..6e850aa 100644 --- a/crates/bindings/python/catena/__init__.pyi +++ b/crates/bindings/python/catena/__init__.pyi @@ -37,6 +37,11 @@ def fuse_hits( limit: int | None = ..., ) -> list[dict[str, Any]]: ... def identify(name: str, data: bytes) -> dict[str, Any]: ... +def find_citations(text: str) -> list[dict[str, Any]]: ... +def citation_prompt(references: Iterable[Mapping[str, Any]]) -> str | None: ... +def merge_citation_reply( + references: Iterable[Mapping[str, Any]], reply: str +) -> list[dict[str, Any]]: ... def parse_identifier(text: str) -> dict[str, Any] | None: ... def series_of( title: str, author: str | None = ... diff --git a/crates/bindings/src/capi.rs b/crates/bindings/src/capi.rs index 956c327..cd30b11 100644 --- a/crates/bindings/src/capi.rs +++ b/crates/bindings/src/capi.rs @@ -417,6 +417,82 @@ pub unsafe extern "C" fn catena_identify( give_json(&crate::identify(name, bytes)) } +/// Read the references out of a bibliography or a page of notes. Returns a +/// JSON array, one object per reference. +/// +/// # Safety +/// +/// `text` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_find_citations(text: *const c_char) -> *mut c_char { + clear_error(); + let Some(text) = (unsafe { borrow(text, "text") }) else { + return ptr::null_mut(); + }; + give_json(&crate::find_citations(text)) +} + +/// The question to put to a model about the references that did not parse. +/// Takes the JSON array `catena_find_citations` returned. Returns the prompt +/// as a JSON string, or NULL when every reference settled — an answer, not an +/// error: `catena_last_error` is NULL in that case. +/// +/// # Safety +/// +/// `references` must be NUL-terminated UTF-8 holding a JSON array. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_citation_prompt(references: *const c_char) -> *mut c_char { + clear_error(); + let Some(references) = (unsafe { borrow(references, "references") }) else { + return ptr::null_mut(); + }; + let references: Vec = match serde_json::from_str(references) { + Ok(references) => references, + Err(why) => { + set_error(why.to_string()); + return ptr::null_mut(); + } + }; + match crate::citation_prompt(&references) { + Some(prompt) => give_json(&prompt), + None => ptr::null_mut(), + } +} + +/// Fill in what did not parse from a model's reply. Returns the JSON array +/// with the answer applied, or NULL and an error when the reply is not one +/// JSON object. +/// +/// # Safety +/// +/// `references` and `reply` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_merge_citation_reply( + references: *const c_char, + reply: *const c_char, +) -> *mut c_char { + clear_error(); + let (Some(references), Some(reply)) = (unsafe { borrow(references, "references") }, unsafe { + borrow(reply, "reply") + }) else { + return ptr::null_mut(); + }; + let references: Vec = match serde_json::from_str(references) { + Ok(references) => references, + Err(why) => { + set_error(why.to_string()); + return ptr::null_mut(); + } + }; + match crate::merge_citation_reply(&references, reply) { + Ok(merged) => give_json(&merged), + Err(why) => { + set_error(why); + ptr::null_mut() + } + } +} + /// The series a title belongs to. Returns a JSON proposal, or NULL when the /// title says nothing about a series — which is most titles, and not an /// error: `catena_last_error` is NULL in that case. diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 0d154b9..758c302 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -215,6 +215,31 @@ pub fn parse_identifier(text: &str) -> Option { idnorm::parse(text) } +/// Read the references out of a bibliography or a page of notes. +/// +/// Each carries `unsettled`: the fields that could not be read off the entry, +/// named so a caller can ask somebody else about exactly those and no more. +pub fn find_citations(text: &str) -> Vec { + citefind::read(text) +} + +/// Compose the question to put to a model about the references that did not +/// parse. `None` when every one of them settled. +pub fn citation_prompt(references: &[citefind::Reference]) -> Option { + citefind::prompt::compose(references) +} + +/// Fill in what did not parse from a model's reply, and return the references +/// with the answer applied. `Err` when the reply is not one JSON object. +pub fn merge_citation_reply( + references: &[citefind::Reference], + reply: &str, +) -> Result, String> { + let mut references = references.to_vec(); + citefind::prompt::merge(&mut references, reply)?; + Ok(references) +} + /// The series a title belongs to, and which volume of it this is. pub fn series_of(title: &str, author: Option<&str>) -> Option { seriesof::series_of(title, author) @@ -236,6 +261,25 @@ mod tests { assert_eq!(count_tokens("a passage"), textkit::count("a passage")); } + #[test] + fn the_citation_surface_agrees_with_the_library_underneath() { + const REFS: &str = "Smith, John. *A Title*. Oxford: Clarendon Press, 1998.\n\nMarron, Kevin Ritual Abuse, Seal Books, 1988"; + let found = find_citations(REFS); + assert_eq!(found, citefind::read(REFS)); + + // The clean entry settles and the comma-only one does not, so there + // is a question to ask and it is about the second alone. + let prompt = citation_prompt(&found).expect("something unsettled"); + assert!(!prompt.contains("Clarendon"), "{prompt}"); + assert!(prompt.contains("Ritual Abuse"), "{prompt}"); + + let merged = merge_citation_reply(&found, r#"{"1": {"authors": ["Marron, Kevin"]}}"#) + .expect("a reply"); + assert_eq!(merged[1].authors, ["Marron, Kevin"]); + assert_eq!(merged[0], found[0]); + assert!(merge_citation_reply(&found, "not json").is_err()); + } + #[test] fn the_default_budgets_are_the_tools_defaults() { let ours = ChunkBudgets::default(); diff --git a/crates/bindings/src/python.rs b/crates/bindings/src/python.rs index 87e5c78..a2c2d11 100644 --- a/crates/bindings/src/python.rs +++ b/crates/bindings/src/python.rs @@ -220,6 +220,34 @@ fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult(py: Python<'py>, text: &str) -> PyResult> { + list_to_py(py, &crate::find_citations(text)) +} + +/// The question to put to a model about the references that did not parse. +/// +/// `None` when every one of them settled, because there is nothing to ask. +#[pyfunction] +fn citation_prompt(references: &Bound<'_, PyAny>) -> PyResult> { + let references: Vec = from_py_list(references)?; + Ok(crate::citation_prompt(&references)) +} + +/// Fill in what did not parse from a model's reply. +#[pyfunction] +fn merge_citation_reply<'py>( + py: Python<'py>, + references: &Bound<'py, PyAny>, + reply: &str, +) -> PyResult> { + let references: Vec = from_py_list(references)?; + let merged = crate::merge_citation_reply(&references, reply) + .map_err(pyo3::exceptions::PyValueError::new_err)?; + list_to_py(py, &merged) +} + /// The series a title belongs to, and which volume of it this is. /// /// `None` when the title says nothing about a series, which is most titles. @@ -261,6 +289,9 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(split_collection, module)?)?; module.add_function(wrap_pyfunction!(fuse_hits, module)?)?; module.add_function(wrap_pyfunction!(identify, module)?)?; + module.add_function(wrap_pyfunction!(find_citations, module)?)?; + module.add_function(wrap_pyfunction!(citation_prompt, module)?)?; + module.add_function(wrap_pyfunction!(merge_citation_reply, module)?)?; module.add_function(wrap_pyfunction!(series_of, module)?)?; module.add_function(wrap_pyfunction!(parse_identifier, module)?)?; Ok(()) diff --git a/crates/bindings/src/wasm.rs b/crates/bindings/src/wasm.rs index 3603137..8c50a43 100644 --- a/crates/bindings/src/wasm.rs +++ b/crates/bindings/src/wasm.rs @@ -97,6 +97,31 @@ export interface Identifier { url?: string; } +/** One reference out of a bibliography or a page of notes. */ +export interface Reference { + v: number; + idx: number; + /** The entry as it appeared, whitespace collapsed. */ + text: string; + /** The note number it carried, where it carried one. */ + marker?: string; + /** The names, in the order and the form the entry gave them. */ + authors: string[]; + year?: number; + /** The work cited — for an article the article, not the journal. */ + title?: string; + /** The journal it appeared in, or the publisher of a book. */ + container?: string; + pages?: string; + ids: { kind: string; id: string }[]; + /** "book", "article", "chapter", "web" or "other". */ + kind: string; + /** Whether it only points back at an earlier entry. */ + continuation: boolean; + /** The fields that could not be read off the entry. */ + unsettled: string[]; +} + /** What a title says about its series. */ export interface SeriesProposal { v: number; @@ -280,6 +305,32 @@ pub fn identify(name: &str, data: &[u8]) -> Result { to_js(&crate::identify(name, data)) } +/// Read the references out of a bibliography or a page of notes. +#[wasm_bindgen(js_name = findCitations, skip_typescript)] +pub fn find_citations(text: &str) -> Result { + to_js(&crate::find_citations(text)) +} + +/// The question to put to a model about the references that did not parse. +/// +/// `undefined` when every one of them settled. +#[wasm_bindgen(js_name = citationPrompt, skip_typescript)] +pub fn citation_prompt(references: JsValue) -> Result { + let references: Vec = from_js(references, "references")?; + match crate::citation_prompt(&references) { + Some(prompt) => to_js(&prompt), + None => Ok(JsValue::UNDEFINED), + } +} + +/// Fill in what did not parse from a model's reply. +#[wasm_bindgen(js_name = mergeCitationReply, skip_typescript)] +pub fn merge_citation_reply(references: JsValue, reply: &str) -> Result { + let references: Vec = from_js(references, "references")?; + let merged = crate::merge_citation_reply(&references, reply).map_err(JsValue::from)?; + to_js(&merged) +} + /// The series a title belongs to, and which volume of it this is. /// /// `undefined` when the title says nothing about a series. @@ -314,6 +365,9 @@ export function cueWindows(cues: Cue[]): string[]; export function splitCollection(markdown: string): Section[]; export function fuseHits(hits: Hit[], k?: number, weights?: Record, limit?: number): Hit[]; export function identify(name: string, data: Uint8Array): Doc; +export function findCitations(text: string): Reference[]; +export function citationPrompt(references: Reference[]): string | undefined; +export function mergeCitationReply(references: Reference[], reply: string): Reference[]; export function seriesOf(title: string, author?: string): SeriesProposal | undefined; export function parseIdentifier(text: string): Identifier | undefined; "#; diff --git a/crates/chunkmd/src/footnote.rs b/crates/chunkmd/src/footnote.rs index 88cd953..5ffd6b5 100644 --- a/crates/chunkmd/src/footnote.rs +++ b/crates/chunkmd/src/footnote.rs @@ -94,6 +94,28 @@ static CITE_SIGNAL: LazyLock = LazyLock::new(|| { .expect("a valid pattern") }); +/// Where each numbered note begins, in reading order. +/// +/// Public because splitting a page of notes into the notes on it is the same +/// knowledge as recognising one, and two copies of the pattern above would +/// drift apart. +pub fn note_starts(text: &str) -> Vec { + NOTE_MARKER + .find_iter(text) + .filter_map(Result::ok) + .map(|found| found.start()) + .collect() +} + +/// Whether an entry reads as a citation rather than as a remark. +/// +/// Also public: a note that cites nothing is a note a bibliography has no +/// use for, and the caller deciding that should ask the same question this +/// module asks. +pub fn cites_something(entry: &str) -> bool { + CITE_SIGNAL.is_match(entry).unwrap_or(false) +} + /// Whether a chunk carries footnote or endnote citation material. /// /// The structural signal first, when the source had one. Otherwise each note @@ -104,11 +126,7 @@ pub fn looks_like_citations(text: &str) -> bool { return true; } - let starts: Vec = NOTE_MARKER - .find_iter(text) - .filter_map(Result::ok) - .map(|found| found.start()) - .collect(); + let starts = note_starts(text); if starts.len() < MIN_CITED_NOTES { return false; } @@ -117,7 +135,7 @@ pub fn looks_like_citations(text: &str) -> bool { for (at, start) in starts.iter().enumerate() { let end = starts.get(at + 1).copied().unwrap_or(text.len()); let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect(); - if CITE_SIGNAL.is_match(&entry).unwrap_or(false) { + if cites_something(&entry) { cited += 1; if cited >= MIN_CITED_NOTES { return true; diff --git a/crates/citefind/Cargo.toml b/crates/citefind/Cargo.toml new file mode 100644 index 0000000..872c898 --- /dev/null +++ b/crates/citefind/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "citefind" +description = "The references in a bibliography or a page of notes, one record each." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +chunkmd.workspace = true +clap.workspace = true +corpus-core.workspace = true +idnorm.workspace = true +regex.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/citefind/README.md b/crates/citefind/README.md new file mode 100644 index 0000000..9c55e37 --- /dev/null +++ b/crates/citefind/README.md @@ -0,0 +1,199 @@ +# citefind + +The references in a bibliography or a page of notes. + +Part of [catena](../../README.md). + +## What it does + +A reference list is the one place in a book where the text is already +structured. Every entry names a work, and most name it in one of a handful of +shapes that have not changed in a century. What is missing is not information +but punctuation: the entries run together, and nothing in the file says where +one stops. + +So this does two jobs. It cuts the block into entries — at the note numbers +where there are any, at the layout where there are not. Then it reads each +entry as far as its shape allows, and **names the fields it could not settle** +rather than guessing at them. + +That last part is the point. A made-up year and a real one are +indistinguishable three stages downstream; a missing one is not. + +## When you would use it + +- **Building a citation graph** — which works a book leans on, and how often. +- **Finding the identifiers** in a reference list, to hand to + [`idnorm`](../idnorm/README.md) and [`bibfetch`](../bibfetch/README.md). +- **Filling a catalogue** from the back of a book rather than from a register. +- **Costing a model run**: ask one about the entries that did not parse, and + nothing else. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena citefind +``` + +## Example + +``` +$ citefind bibliography.md +{"v":1,"idx":0,"text":"Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998.","authors":["Smith, John"],"year":1998,"title":"The Making of the Second Estate","container":"Clarendon Press","ids":[],"kind":"book","continuation":false,"unsettled":[]} +{"v":1,"idx":1,"text":"Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.","authors":["Doe, Jane","Peter Brown"],"year":2001,"title":"Obligation and Its Limits","container":"Journal of Medieval Studies","pages":"45-67","ids":[],"kind":"article","continuation":false,"unsettled":[]} +``` + +What a book cites, by year: + +```sh +citefind bibliography.md | jq -r 'select(.year) | .year' | sort | uniq -c +``` + +The identifiers, checked and resolved: + +```sh +citefind refs.md | jq -r '.ids[] | select(.kind == "doi") | .id' | idnorm | bibfetch +``` + +The works it leans on hardest, from a page of endnotes: + +```sh +citefind notes.md | jq -r 'select(.title) | .title' | sort | uniq -c | sort -rn | head +``` + +## Asking a model about the rest + +`--prompt` writes a question about the entries that did not parse, and only +those. Nothing is sent from here: the prompt goes to stdout and what happens +to it is yours to decide. + +``` +$ citefind --prompt refs.md +Below are entries from the bibliography or notes of a book. Each was read as +far as its punctuation allowed, and the fields that could not be read off it +are listed with it. + +--- 0 +text: Marron, Kevin Ritual Abuse, Seal Books, 1988 +year: 1988 +wanted: authors, title, container +... +``` + +```sh +citefind --prompt refs.md | ollama run qwen3:8b > reply.json +citefind --merge reply.json refs.md +``` + +A reply fills in only the fields that were asked about. A model that +volunteers a better title for an entry whose title was read off the page is +overruled — the page is evidence and the model is a guess — and a reply that +is not one JSON object is refused rather than half-applied. + +One prompt covers every unsettled entry rather than one prompt each. A +reference list is long and mostly ordinary; the entries worth asking about are +the mangled few, and they are cheaper to ask about together. + +## Options + +| Option | Effect | +|---|---| +| `--prompt` | Write the question about what did not parse, instead of the records | +| `--merge FILE` | Fill in what did not parse from a model's reply | +| `--quiet`, `-q` | Say nothing; the exit code is the answer | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | It found references | +| `1` | It found none | +| `64` | The command line was wrong | +| `65` | The input was not readable as text, or a reply was not JSON | +| `74` | The file could not be read | + +## The record + +| Field | What it holds | +|---|---| +| `text` | The entry as it appeared, whitespace collapsed | +| `marker` | The note number it carried, where it carried one | +| `authors` | The names, in the order and the form the entry gave them | +| `year` | The year of publication | +| `title` | The work cited — for an article the article, not the journal | +| `container` | The journal it appeared in, or the publisher of a book | +| `pages` | The pages, as written | +| `ids` | Any DOI, ISBN, ISSN, arXiv id or URL, normalised | +| `kind` | `book`, `article`, `chapter`, `web` or `other` | +| `continuation` | Whether it only points back at an earlier entry | +| `unsettled` | The fields it could not read off the entry | + +`text` is kept whatever else was read, because every other field is an +interpretation of it and somebody will want to check. + +## How it cuts + +**A page of notes** is cut at its note numbers, in any of the shapes +conversion leaves them: `12.`, `[12]`, `fn39.`, a superscript, or a +back-reference link. The pattern is the one +[`chunkmd`](../chunkmd/README.md) uses to recognise a page of notes, so the +two cannot drift apart, with one form added — a link pointing at another file +rather than at a fragment. `chunkmd` is deciding whether a chunk is a page of +notes and can afford to miss one marker in thirty; here a missed marker runs +two references together. + +A note that cites nothing is dropped. A page of notes mixes bibliographic +notes with discursive ones, and a remark about the argument is not a +reference. + +**A bibliography** numbers nothing, so the signal is layout: a blank line +between entries, a hanging indent under each, or the surname-comma opening +that nearly every author-first entry begins with. Whichever the block carries +is the one used. When it carries none of them the fallback is one entry per +line, which is right for a list that was never wrapped and visibly wrong for +one that was. + +## What it will not do + +**Resolve `ibid.`** A continuation refers to whatever was cited before it, +which is a fact about the surrounding page and not about the entry. It is +flagged `continuation` and left alone, and it is left out of `--prompt` +entirely — there is nothing in it to ask about. + +**Report a title it did not find.** An entry held together by commas alone, +with no names to anchor it, yields no title rather than its own text echoed +back. + +A rule of dashes where the author should be is the exception, and a +deliberate one: `———. *The Second Book*.` means the author of the entry above, +the style says so, and the entry it refers to is the one immediately before. +That is read and filled in. + +## As a library + +```rust +for one in citefind::read(&text) { + println!("{} ({}) {:?}", one.authors.join("; "), one.year.unwrap_or(0), one.title); +} +``` + +`citefind::split::entries` cuts a block without reading the entries. +`citefind::parse::one` reads an entry already cut. `citefind::prompt::compose` +and `merge` are the two halves of the model-facing path, and +`prompt::PROMPT_VERSION` says which wording produced a given answer. + +From Python, JavaScript or C, through +[`bindings`](../bindings/README.md): + +```python +for one in catena.find_citations(text): + print(one["year"], one["title"], one["unsettled"]) +``` + +## See also + +- [`idnorm`](../idnorm/README.md) — the identifiers it finds, checked. +- [`bibfetch`](../bibfetch/README.md) — what a register says about them. +- [`chunkmd`](../chunkmd/README.md) — which flags the chunks worth feeding it. +- [`bibclean`](../bibclean/README.md) — the same audit for a catalogue record. diff --git a/crates/citefind/src/lib.rs b/crates/citefind/src/lib.rs new file mode 100644 index 0000000..fa35a40 --- /dev/null +++ b/crates/citefind/src/lib.rs @@ -0,0 +1,214 @@ +//! The references inside a bibliography or a page of notes. +//! +//! A reference list is the one place in a book where the text is already +//! structured — every entry names a work, and most of them name it in one of +//! a handful of shapes that have not changed in a century. What is missing is +//! not information but punctuation: the entries run together, and nothing in +//! the file says where one stops. +//! +//! So this does two jobs. It cuts the block into entries, which is mechanical +//! and gets it right or visibly wrong. Then it reads each entry as far as the +//! shape allows, and says which fields it could not settle rather than +//! guessing at them — because a made-up year is indistinguishable downstream +//! from a real one, and an absence is not. +//! +//! What it will not do is resolve `ibid.`. A continuation refers to whatever +//! was cited before it, which is a fact about the surrounding page and not +//! about the entry; the entry is flagged as a continuation and left alone. + +pub mod parse; +pub mod prompt; +pub mod split; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// What sort of work a reference points at. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + /// A whole book, with a publisher. + Book, + /// An article in a journal. + Article, + /// A piece within a larger edited work. + Chapter, + /// Something whose address is a URL. + Web, + /// Recognisably a reference, not recognisably any of the above. + Other, +} + +impl Kind { + /// What it is called. + pub fn name(self) -> &'static str { + match self { + Kind::Book => "book", + Kind::Article => "article", + Kind::Chapter => "chapter", + Kind::Web => "web", + Kind::Other => "other", + } + } +} + +/// One reference, as found and as far as it could be read. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Reference { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// Position in the block, from zero. + pub idx: u32, + /// The entry exactly as it appeared, with its own whitespace collapsed. + /// + /// Kept whatever else was read, because every other field on this record + /// is an interpretation of this one and somebody will want to check it. + pub text: String, + /// The note number this entry carried, where it carried one. + #[serde(skip_serializing_if = "Option::is_none")] + pub marker: Option, + /// Who wrote it, in the order the entry gave them. + pub authors: Vec, + /// The year of publication. + #[serde(skip_serializing_if = "Option::is_none")] + pub year: Option, + /// The title of the work cited — the article, not the journal it is in. + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + /// What it appeared in: a journal, or the publisher of a book. + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, + /// The pages, as written. + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option, + /// Any identifiers the entry carried, normalised. + pub ids: Vec, + /// What sort of work it is. + pub kind: Kind, + /// Whether this entry only points back at an earlier one: `ibid.`, + /// `op. cit.`, and the rest. What it points at is not read off the entry, + /// because the entry does not say. + pub continuation: bool, + /// The fields this entry did not settle, named so a caller can ask + /// somebody else about exactly those and no more. + pub unsettled: Vec, +} + +/// An identifier found in a reference, normalised to the form it stores in. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Id { + /// Which kind it is: `doi`, `isbn`, `issn`, `arxiv`, or `url`. + pub kind: String, + /// The identifier itself. + pub id: String, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Reference { + fn version(&self) -> u32 { + self.v + } +} + +/// The fields a complete reference has, in the order they are reported. +pub const FIELDS: [&str; 4] = ["authors", "year", "title", "container"]; + +impl Reference { + /// An empty reference carrying only the text it was cut from. + pub fn of(text: &str) -> Self { + Reference { + v: corpus_core::V, + idx: 0, + text: text.to_string(), + marker: None, + authors: Vec::new(), + year: None, + title: None, + container: None, + pages: None, + ids: Vec::new(), + kind: Kind::Other, + continuation: false, + unsettled: Vec::new(), + } + } + + /// Which of the four fields worth having are still empty. + /// + /// A continuation is excluded: it has nothing to settle, and asking about + /// one invites the guess this refuses to make. + pub fn missing(&self) -> Vec<&'static str> { + if self.continuation { + return Vec::new(); + } + FIELDS + .into_iter() + .filter(|field| match *field { + "authors" => self.authors.is_empty(), + "year" => self.year.is_none(), + "title" => self.title.is_none(), + _ => self.container.is_none(), + }) + .collect() + } +} + +/// Read every reference in a block of text. +/// +/// The block is cut into entries and each is read on its own. Entries that +/// carry nothing citation-shaped at all are dropped: a bibliography page has +/// a heading on it, and a heading is not a reference. +pub fn read(text: &str) -> Vec { + let mut out: Vec = split::entries(text) + .into_iter() + .filter_map(|entry| parse::one(&entry.text, entry.marker.as_deref())) + .collect(); + + let mut previous: Vec = Vec::new(); + for (at, reference) in out.iter_mut().enumerate() { + reference.idx = at as u32; + // A rule of dashes where the author should be means the author of + // the entry above. That is not a guess: the style defines it, and + // the entry it refers to is the one immediately before. `ibid.` is + // refused because it points at a work, which may be pages back. + if reference.authors.is_empty() && parse::repeats_author(&reference.text) { + reference.authors = previous.clone(); + } + if !reference.authors.is_empty() { + previous = reference.authors.clone(); + } + reference.unsettled = reference.missing().iter().map(|s| s.to_string()).collect(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_reference_reports_what_it_does_not_have() { + let mut one = Reference::of("something"); + assert_eq!(one.missing(), FIELDS); + one.year = Some(1998); + one.title = Some("A Title".into()); + assert_eq!(one.missing(), ["authors", "container"]); + } + + #[test] + fn a_continuation_has_nothing_to_ask_about() { + let mut one = Reference::of("Ibid., 183."); + one.continuation = true; + assert!(one.missing().is_empty()); + } + + #[test] + fn a_block_of_nothing_yields_nothing() { + assert!(read("").is_empty()); + assert!(read("Bibliography\n\n").is_empty()); + } +} diff --git a/crates/citefind/src/main.rs b/crates/citefind/src/main.rs new file mode 100644 index 0000000..0e521af --- /dev/null +++ b/crates/citefind/src/main.rs @@ -0,0 +1,110 @@ +//! The references in a bibliography or a page of notes. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::exit; + +/// Read a reference list, one record per reference. +/// +/// The input is the bibliography, works-cited list, or page of numbered +/// notes as text. It is cut into entries — at the note numbers where there +/// are any, at the layout where there are not — and each entry is read as far +/// as its punctuation allows. +/// +/// Fields that could not be read off the entry are left empty and named in +/// `unsettled`, because a wrong year and a real one are indistinguishable +/// downstream and a missing one is not. `--prompt` writes a question about +/// exactly those, for a model of your choosing; nothing is sent from here. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 it found references, 1 it found none, 65 the input was not +readable as text, 74 the file could not be read. + +Example: + + citefind bibliography.md | jq -r '[.year, .title] | @tsv' + mdsplit book.md | citefind --prompt > ask.txt + citefind --prompt refs.md | ollama run qwen3:8b > reply.json + citefind --merge reply.json refs.md" +)] +struct Args { + /// The reference list to read. Standard input when not given. + path: Option, + + /// Say nothing; the exit code is the answer. + #[arg(long, short)] + quiet: bool, + + /// Write the question to put to a model about the entries that did not + /// parse, instead of the records. Nothing is sent: it goes to stdout. + #[arg(long)] + prompt: bool, + + /// Fill in what would not parse from a model's reply, written earlier by + /// `--prompt`, and write the completed records. + #[arg(long, value_name = "FILE")] + merge: Option, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + if args.prompt && args.merge.is_some() { + eprintln!("citefind: --prompt composes a question and --merge answers one"); + return ExitCode::from(exit::USAGE); + } + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("citefind: {why}"); + return ExitCode::from(code); + } + }; + + let mut references = citefind::read(&text); + if references.is_empty() { + if !args.quiet { + eprintln!("citefind: nothing in this looks like a reference"); + } + return ExitCode::from(exit::NO); + } + + if let Some(path) = args.merge.as_deref() { + let reply = match std::fs::read_to_string(path) { + Ok(reply) => reply, + Err(e) => { + eprintln!("citefind: {}: {e}", path.display()); + return ExitCode::from(exit::IO); + } + }; + match citefind::prompt::merge(&mut references, &reply) { + Err(why) => { + eprintln!("citefind: {why}"); + return ExitCode::from(exit::DATA); + } + Ok(0) => eprintln!("citefind: the reply filled nothing in"), + Ok(filled) => eprintln!("citefind: {filled} entries filled in from the reply"), + } + } + + if args.quiet { + return ExitCode::from(exit::OK); + } + if args.prompt { + return match citefind::prompt::compose(&references) { + // Nothing unsettled is not a failure, and a prompt nobody needs + // would waste a model's time. + None => ExitCode::from(exit::NO), + Some(prompt) => catena_cli::text("citefind", &prompt), + }; + } + catena_cli::stream("citefind", &references) +} diff --git a/crates/citefind/src/parse.rs b/crates/citefind/src/parse.rs new file mode 100644 index 0000000..cc89e73 --- /dev/null +++ b/crates/citefind/src/parse.rs @@ -0,0 +1,1016 @@ +//! Reading one reference as far as its shape allows. +//! +//! Reference styles differ in punctuation and agree on order: who, when, +//! what, where. So the reading goes for the parts that announce themselves — +//! an identifier, a year in brackets, a title in italics or quotation marks — +//! and uses where those landed to divide up what is left. A field that does +//! not announce itself is left empty and named in `unsettled`, which is the +//! whole point: a wrong year and a real one look identical three stages +//! downstream, and a missing one does not. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::{Id, Kind, Reference}; + +/// Shorter than this and it is a page number or a running head. +const MIN_ENTRY: usize = 12; + +/// A reference that only points back at an earlier one. +static CONTINUATION: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^[\s\p{Pi}\p{Pf}*_]*(?:ibid|idem|op\.\s*cit|loc\.\s*cit|op\.cit|loc\.cit)\b") + .expect("a valid pattern") +}); + +static DOI: LazyLock = + LazyLock::new(|| Regex::new(r"\b10\.\d{4,9}/[^\s,;)\]>]+").expect("a valid pattern")); +static ARXIV: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\barxiv:\s*([A-Za-z\-.]*/?\d{4,7}(?:\.\d{4,5})?(?:v\d+)?)") + .expect("a valid pattern") +}); +static ISBN: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bisbn(?:-?1[03])?:?\s*((?:97[89][\s\-]?)?(?:\d[\s\-]?){9}[\dXx])") + .expect("a valid pattern") +}); +static ISSN: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bissn:?\s*(\d{4}[\s\-]?\d{3}[\dXx])").expect("a valid pattern") +}); +static URL: LazyLock = + LazyLock::new(|| Regex::new(r"https?://[^\s,;>\]]+").expect("a valid pattern")); + +/// A year inside brackets, whether alone or at the end of an imprint. +static PAREN_YEAR: LazyLock = LazyLock::new(|| { + Regex::new(r"\((?:[^()]*?[^\d])?((?:1[4-9]|20)\d{2})[a-z]?\)").expect("a valid pattern") +}); +/// A year after a comma, which is where an imprint puts it. +static COMMA_YEAR: LazyLock = + LazyLock::new(|| Regex::new(r"[,(]\s*((?:1[4-9]|20)\d{2})[a-z]?\b").expect("a valid pattern")); +/// A year anywhere, which is the last resort. +static ANY_YEAR: LazyLock = + LazyLock::new(|| Regex::new(r"\b((?:1[4-9]|20)\d{2})\b").expect("a valid pattern")); + +/// Pages announced as pages. +static PP: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bpp?\.?\s*(\d{1,5}(?:\s*[-–—]+\s*\d{1,5})?)\b").expect("a valid pattern") +}); +/// The page range a journal citation ends with: `12 (1998): 45-67`. +static COLON_PAGES: LazyLock = LazyLock::new(|| { + Regex::new(r":\s*(\d{1,5}(?:\s*[-–—]+\s*\d{1,5})?)\s*\.?\s*$").expect("a valid pattern") +}); +/// A page range after a comma or colon, which is where author-date styles +/// put one: `Nature, 570(7761), 321-325`. +static RANGE_PAGES: LazyLock = LazyLock::new(|| { + Regex::new(r"[,:]\s*(\d{1,5}\s*[-–—]+\s*\d{1,5})(?:\b|$)").expect("a valid pattern") +}); +/// The bare page a note ends with: `…, 183.` +static TRAILING_PAGE: LazyLock = + LazyLock::new(|| Regex::new(r",\s*(\d{1,5})\s*\.?\s*$").expect("a valid pattern")); + +/// A title in markdown emphasis, or in the tags a converter left instead. +static EMPHASISED: LazyLock = LazyLock::new(|| { + Regex::new(r"(?s)\*([^*\n]{3,300})\*|_([^_\n]{3,300})_|<(?:em|i|cite)\b[^>]*>(.{3,300}?)") + .expect("a valid pattern") +}); +/// A title in quotation marks, straight or curly. +static QUOTED: LazyLock = LazyLock::new(|| { + Regex::new(r#""([^"\n]{3,300})"|“([^”\n]{3,300})”|«([^»\n]{3,300})»|‘([^’\n]{3,300})’"#) + .expect("a valid pattern") +}); + +/// The imprint of a book: a place, a colon, a publisher. +static IMPRINT: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^|[(.,;]\s*)[\p{Lu}][^:;()\d]{1,40}:\s*([^;()]{2,80})") + .expect("a valid pattern") +}); + +/// Words that name a publisher rather than a journal. +static PUBLISHER_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:press|publisher|publishing|books|verlag|editions?|university|imprint|&\s*co\b|ltd\b|inc\b)") + .expect("a valid pattern") +}); + +/// The volume-and-number a journal citation carries. +static VOLUME: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:vol\.?\s*)?\d{1,4}\s*(?:,\s*)?(?:\(\s*\d+\s*\)|no\.\s*\d+|\(\s*(?:1[4-9]|20)\d{2}\s*\))") + .expect("a valid pattern") +}); + +/// What marks a piece inside a larger edited work. +static WITHIN: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)(?:^|[,.]\s*)in\s|\bed(?:s|ited by)?\.?\s").expect("a valid pattern") +}); + +/// The role a compiler of a book is credited with, at the end of the run of +/// names it follows. +static ROLE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i),?\s+(?:ed|eds|edited\s+by|trans|translated\s+by|comp|comps|rev)\.?$") + .expect("a valid pattern") +}); + +/// A run of text that is nothing but an identifier. +static IDENTIFIER_ONLY: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(?:arxiv:|doi:|https?://|isbn|issn|\b10\.\d{4})\S*\.?$") + .expect("a valid pattern") +}); + +/// Any HTML tag, which conversion scatters through a reference list. +static MARKUP_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"]*>").expect("a valid pattern")); + +/// The Chicago note: a name, the title, the imprint in brackets, the page. +/// Nothing in it is marked up, so the commas and the brackets are all there +/// is to divide it by. +static NOTE_FORM: LazyLock = LazyLock::new(|| { + Regex::new(r"^([^,()]{2,60}),\s+([^()]{4,300}?)\s*\(([^()]*(?:1[4-9]|20)\d{2}[a-z]?)\)") + .expect("a valid pattern") +}); + +/// What a note says before it gets to the citation. +static LEAD_IN: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(?:see|cf\.|compare|quoted|cited|as\s+quoted|reprinted)(?:\s*,?\s*(?:also|for\s+(?:example|instance)|e\.g\.|in))*\s*,?\s+") + .expect("a valid pattern") +}); + +/// The lowercase words a name may contain. Anything else lowercase in the +/// middle of a supposed name means it is a sentence. +const PARTICLES: [&str; 20] = [ + "van", "von", "de", "del", "della", "der", "den", "di", "da", "du", "la", "le", "of", "y", + "bin", "ibn", "al", "ter", "v", "zu", +]; + +/// Anything that says this is a reference at all. +static CITATION_SIGNAL: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:1[4-9]|20)\d{2}\b|\bpp?\.\s*\d|https?://|\b10\.\d{4}|\bed\.|\btrans\.|\bvol\.|\bno\.|:\s*\d|\bpress\b|\buniversity\b|\bjournal\b") + .expect("a valid pattern") +}); + +/// A rule of dashes standing in for the author of the entry above, which is +/// how every style avoids printing the same name forty times running. +static REPEATED_AUTHOR: LazyLock = + LazyLock::new(|| Regex::new(r"^\s*(?:[—–]+|-{3,}|_{3,})").expect("a valid pattern")); + +/// Whether an entry defers to the one above it for its author. +pub fn repeats_author(text: &str) -> bool { + REPEATED_AUTHOR.is_match(text) +} + +/// Read one entry. +/// +/// `None` when it is not a reference at all — a section heading, a page +/// number, a stray line of running text that the cut picked up. +pub fn one(text: &str, marker: Option<&str>) -> Option { + let text = text.trim(); + if text.chars().count() < MIN_ENTRY { + return None; + } + + let mut out = Reference::of(text); + out.marker = marker.map(str::to_string); + + if CONTINUATION.is_match(text) { + out.continuation = true; + out.pages = pages(text); + return Some(out); + } + if !CITATION_SIGNAL.is_match(text) { + return None; + } + + out.ids = ids(text); + out.year = year(text); + out.pages = pages(text); + + let (quoted, quoted_at) = span("ED, text); + let (emphasised, emphasised_at) = span(&EMPHASISED, text); + let dated = author_date(text); + + // An author-date opening settles the authors and the year whatever the + // rest of the entry marks up, so it is asked first. A bibliography full + // of German titles has quotation marks inside its titles, and taking + // those for the title is how a parser that asked the other way round + // gets it wrong. + if let Some(year_at) = dated { + out.authors = authors(&text[..year_at.0]); + let tail = text[year_at.1..].trim_start_matches(|c: char| ".,:‘“\" ".contains(c)); + let tail_at = text.len() - tail.len(); + // Quotation marks count as the title only where they open the tail. + // Further in they are a quotation inside a title, not the title. + let opens_tail = quoted_at.is_some_and(|at| at.0 + 2 >= tail_at && at.0 <= tail_at + 2); + let after_year = |at: (usize, usize)| at.0 >= year_at.1; + + if opens_tail { + out.title = quoted; + let at = quoted_at.expect("a span"); + out.container = match (emphasised, emphasised_at) { + (Some(book), Some(book_at)) if book_at.0 >= at.1 => tidy_container(&book), + _ => container(text, &text[at.1..]), + }; + } else { + let parts = sentences(tail); + out.title = parts.first().map(|first| tidy_title(&strip_markup(first))); + out.container = match (emphasised, emphasised_at) { + (Some(journal), Some(at)) if after_year(at) => tidy_container(&journal), + _ => parts.get(1).and_then(|second| tidy_container(second)), + }; + } + } else if let (Some(title), Some(at)) = (quoted, quoted_at) { + out.title = Some(title); + out.authors = authors(&text[..at.0]); + // A piece inside an edited book appeared in that book, and the book + // is the italic title after this one. + let within = WITHIN.is_match(&text[at.1..]); + out.container = match (within, emphasised, emphasised_at) { + (true, Some(book), Some(book_at)) if book_at.0 >= at.1 => Some(book), + _ => container(text, &text[at.1..]), + }; + } else if let (Some(title), Some(at)) = (emphasised, emphasised_at) { + out.title = Some(title); + out.authors = authors(&text[..at.0]); + out.container = container(text, &text[at.1..]); + } else if let Some(found) = note_form(text).or_else(|| comma_form(text)) { + out.authors = found.0; + out.title = found.1; + out.container = found.2; + } else { + let (found_authors, found_title, found_container) = by_sentence(text); + out.authors = found_authors; + out.title = found_title; + out.container = found_container; + } + out.kind = kind(text, &out); + + Some(out) +} + +/// The first match of a title pattern: what it held, and where it sat. +fn span(pattern: &Regex, text: &str) -> (Option, Option<(usize, usize)>) { + let Some(found) = pattern.captures(text) else { + return (None, None); + }; + let whole = found.get(0).expect("the whole match"); + let Some(inner) = (1..=4).find_map(|at| found.get(at)) else { + return (None, None); + }; + let cleaned = tidy_title(&strip_markup(inner.as_str())); + match cleaned.is_empty() { + true => (None, None), + false => (Some(cleaned), Some((whole.start(), whole.end()))), + } +} + +/// An author-date opening with no brackets round the year: `Altmann, M. +/// 1951, A Study of Behaviour`. Common in the sciences and in older German +/// and Russian bibliographies. +static NAME_YEAR: LazyLock = LazyLock::new(|| { + Regex::new(r"^(.{2,80}?)[,.]?\s+((?:1[4-9]|20)\d{2})[a-z]?\s*[,.:]\s+") + .expect("a valid pattern") +}); + +/// Where an author-date entry puts its year, when it is one. +/// +/// The bracketed year has to come early and what precedes it has to read as +/// names — otherwise `(Oxford, 1998)` at the end of a note would look like +/// the opening of an author-date entry with the whole note as its authors. +fn author_date(text: &str) -> Option<(usize, usize)> { + let span = match PAREN_YEAR.find(text) { + Some(found) => (found.start(), found.end()), + None => { + let found = NAME_YEAR.captures(text)?; + let year = found.get(2).expect("the year"); + let whole = found.get(0).expect("the whole match"); + (year.start(), whole.end()) + } + }; + let head = &text[..span.0]; + if head.chars().count() > 120 || !reads_as_names(head) { + return None; + } + // Something has to follow the year, or there is no title to find. + match text[span.1..].trim().chars().count() >= 4 { + true => Some(span), + false => None, + } +} + +/// Read a note that marked none of its parts but bracketed its imprint. +/// +/// `Name, Title (Place: Publisher, Year), page.` — the commonest shape in +/// any book with endnotes, and the one that offers a parser nothing but its +/// punctuation. The guard is that the part standing where the title goes has +/// to be one sentence: a contributor's biography opens the same way and runs +/// on for three. +fn note_form(text: &str) -> Option<(Vec, Option, Option)> { + let text = LEAD_IN.replace(text.trim(), ""); + let found = NOTE_FORM.captures(&text)?; + let names = authors(&found[1]); + if names.is_empty() { + return None; + } + let middle = found[2].trim(); + if sentences(middle).len() > 1 { + return None; + } + let title = tidy_title(&strip_markup(middle)); + if title.chars().count() < 4 { + return None; + } + Some((names, Some(title), tidy_container(&found[3]))) +} + +/// Read an entry whose only punctuation is commas. +/// +/// `Miller, David, On Nationality, Oxford: Clarendon Press, 1995` — the style +/// gives nothing away except that the imprint has a colon in it, so the title +/// is taken to be whatever sits between the names and the imprint. Without an +/// imprint there is no second edge and nothing to be read here. +fn comma_form(text: &str) -> Option<(Vec, Option, Option)> { + let text = LEAD_IN.replace(text.trim(), ""); + if sentences(&text).len() > 1 { + return None; + } + let parts: Vec<&str> = text.split(',').collect(); + if parts.len() < 4 { + return None; + } + // The imprint is whichever comma-part carries the colon. Found by + // counting commas rather than by where a pattern matched: a pattern for + // `Place: Publisher` starts at the comma before the place, which is + // exactly where the title ends, and the two cannot both have it. + let at = parts.iter().position(|part| part.contains(':'))?; + + // `Miller, David` is the usual opening, and `Procopius` on its own is the + // other one. Whichever reads as a name, the title starts after it. + let pair = format!("{},{}", parts[0], parts[1]); + let (names, used) = match reads_as_names(&pair) { + true => (authors(&pair), 2), + false if reads_as_names(parts[0]) => (authors(parts[0]), 1), + false => return None, + }; + if names.is_empty() || at <= used { + return None; + } + + let title = tidy_title(&strip_markup(&parts[used..at].join(","))); + if title.chars().count() < 4 { + return None; + } + Some((names, Some(title), tidy_container(parts[at]))) +} + +/// Whether a run of text is names and nothing else. +/// +/// Stricter than asking whether it *contains* a name. `Doe, Jane, and Peter +/// Brown. "Obligation and Its Limits." Journal of Medieval Studies 12, no. 3` +/// contains one, and reading it as the authors of an author-date entry would +/// take the journal's issue number for a year. +fn reads_as_names(head: &str) -> bool { + let head = head.trim_end_matches(|c: char| ",.;: ".contains(c)); + if head.contains(['"', '“', '”', '*', '_', '(', ')']) { + return false; + } + let head = ROLE.replace(head, ""); + let parts = split_parts(&head); + !parts.is_empty() && parts.iter().all(|part| is_name(part)) +} + +/// Drop the emphasis and the tags a converter leaves round a title. +fn strip_markup(text: &str) -> String { + let without_tags = MARKUP_TAG.replace_all(text, ""); + without_tags.replace(['*', '_'], "").trim().to_string() +} + +/// Every identifier the entry carries, normalised. +/// +/// Order is fixed rather than found: two entries carrying the same +/// identifiers produce the same record whichever way round they wrote them. +fn ids(text: &str) -> Vec { + let mut out = Vec::new(); + let mut push = |kind: &str, raw: &str| { + let normalised = match kind { + "url" => Some(raw.trim_end_matches(['.', ',', ';', ')']).to_string()), + _ => idnorm::parse(raw).map(|found| found.id), + }; + if let Some(id) = normalised { + let one = Id { + kind: kind.to_string(), + id, + }; + if !out.contains(&one) { + out.push(one); + } + } + }; + + for found in DOI.find_iter(text) { + push("doi", found.as_str().trim_end_matches(['.', ',', ';'])); + } + for found in ARXIV.captures_iter(text) { + push("arxiv", &found[1]); + } + for found in ISBN.captures_iter(text) { + push("isbn", &found[1]); + } + for found in ISSN.captures_iter(text) { + push("issn", &found[1]); + } + for found in URL.find_iter(text) { + // A DOI written as a URL is already recorded as the DOI it is. + if !found.as_str().contains("doi.org") && !found.as_str().contains("arxiv.org") { + push("url", found.as_str()); + } + } + out +} + +/// The year of publication. +/// +/// A bracketed year first, because that is where a style puts the one it +/// means; then a year after a comma, which is where an imprint ends; then +/// any year at all, which may well be part of a title. +fn year(text: &str) -> Option { + for pattern in [&*PAREN_YEAR, &*COMMA_YEAR, &*ANY_YEAR] { + if let Some(found) = pattern.captures(text) { + return found[1].parse().ok(); + } + } + None +} + +/// The pages, as the entry wrote them. +/// +/// The last two patterns end at a number, and so does an imprint — so a +/// value that reads as a year is refused rather than reported as a page. A +/// book published in 1998 is not a book cited at page 1998. +fn pages(text: &str) -> Option { + for pattern in [&*PP, &*COLON_PAGES, &*RANGE_PAGES, &*TRAILING_PAGE] { + let Some(found) = pattern.captures(text) else { + continue; + }; + let value: String = found[1].split_whitespace().collect::>().join(""); + if value.split(['-', '–', '—']).all(looks_like_a_year) { + continue; + } + return Some(value); + } + None +} + +/// Whether a number is in the range a year of publication falls in. +fn looks_like_a_year(text: &str) -> bool { + match text.parse::() { + Ok(number) => (1400..=2099).contains(&number), + Err(_) => false, + } +} + +/// Strip the punctuation a style leaves inside its own title delimiters. +fn tidy_title(raw: &str) -> String { + let text = raw.split_whitespace().collect::>().join(" "); + text.trim_matches(|c: char| ",.;:".contains(c) || c.is_whitespace()) + .to_string() +} + +/// Who wrote it, from the run of text before the title. +fn authors(before: &str) -> Vec { + let mut head = LEAD_IN.replace(before.trim(), "").trim().to_string(); + let mut head = head.as_mut_str() as &str; + // An author-date entry puts the year between the authors and the title. + if let Some(at) = head.rfind('(') { + if head[at..].contains(|c: char| c.is_ascii_digit()) { + head = head[..at].trim(); + } + } + let head = head.trim_end_matches(|c: char| ",.;:".contains(c) || c.is_whitespace()); + // `Eberhardt, Christian, ed.` is an editor, and an editor is a person. + // The role is dropped rather than the name with it. + split_authors(&ROLE.replace(head, "")) +} + +/// Cut a run of names at the separators a reference list uses. +fn split_authors(head: &str) -> Vec { + split_parts(head) + .into_iter() + .filter(|part| is_name(part)) + .collect() +} + +/// The separators a list of authors is joined by. +/// +/// A bare comma is not among them: `Smith, John` is one author and +/// `Smith, John, and Jane Doe` is two, so only the words can divide them. +fn split_parts(head: &str) -> Vec { + let head = head.trim(); + if head.is_empty() || head.chars().count() > 200 { + return Vec::new(); + } + let mut parts: Vec = vec![head.to_string()]; + for separator in [", and ", " and ", "; ", " & ", ", & "] { + if head.contains(separator) { + parts = head + .split(separator) + .map(|part| part.trim().to_string()) + .collect(); + break; + } + } + + parts + .into_iter() + .map(|part| { + // A trailing "et al." means the entry dropped the rest itself, + // and there is nothing here to recover. + restore_initial( + part.trim_end_matches(|c: char| ",.;".contains(c) || c.is_whitespace()) + .trim_end_matches("et al") + .trim_end_matches(|c: char| ",.;".contains(c) || c.is_whitespace()), + ) + }) + .filter(|part| !part.is_empty()) + .collect() +} + +/// Put back the full stop that belongs to a closing initial. +/// +/// Trimming trailing punctuation is right for `Smith, John,` and wrong for +/// `Peirce, C. S.`, where the stop is part of the initial and not the +/// separator that followed it. +fn restore_initial(name: &str) -> String { + let last = name + .rsplit(|c: char| c.is_whitespace()) + .next() + .unwrap_or(""); + match last.chars().count() == 1 && last.chars().all(char::is_uppercase) { + true => format!("{name}."), + false => name.to_string(), + } +} + +/// Whether a run of words reads as somebody's name. +fn is_name(text: &str) -> bool { + let words: Vec<&str> = text.split_whitespace().collect(); + if words.is_empty() || words.len() > 6 { + return false; + } + if text.chars().any(|c| c.is_ascii_digit()) { + return false; + } + // An editor's or translator's credit is a role, not a name. + if text.to_lowercase().contains("ed.") || text.to_lowercase().contains("trans.") { + return false; + } + // A lowercase word in the middle of a name is a particle or a sentence, + // and there are only so many particles. + let sentence = words.iter().any(|word| { + let core = word.trim_matches(|c: char| !c.is_alphanumeric()); + core.chars().next().is_some_and(char::is_lowercase) + && !PARTICLES.contains(&core.to_lowercase().as_str()) + }); + if sentence { + return false; + } + words + .first() + .is_some_and(|word| word.chars().next().is_some_and(char::is_uppercase)) +} + +/// What the work appeared in: a journal, or a book's publisher. +fn container(_whole: &str, after: &str) -> Option { + // A book names its publisher in an imprint after the title, whether + // bracketed or not. Only after the title: a title with a colon in it — + // and half of them have one — would otherwise read as an imprint. + if let Some(found) = IMPRINT.captures(after) { + if let Some(publisher) = tidy_container(found[1].trim()) { + return Some(publisher); + } + } + // A journal sits between the title and the volume. + let tail = after.trim_start_matches(|c: char| ",.;:\"”'’ ".contains(c)); + let end = VOLUME + .find(tail) + .map(|found| found.start()) + .or_else(|| tail.find(['(', '.', ';'])) + .unwrap_or(tail.len()); + tidy_container(&tail[..end]) +} + +/// Reduce a run of text to the name of the thing a work appeared in. +/// +/// What arrives is a whole trailing segment — the journal with its volume, +/// issue and pages after it, or the imprint with its year. All of those are +/// recorded in their own fields already, so what is wanted here is only the +/// name, and everything that is somewhere else on the record is cut away. +fn tidy_container(raw: &str) -> Option { + let text = strip_markup(raw); + // An identifier is not a container. A preprint has no journal, and + // saying `arXiv:1706.03762` where a journal goes invents one. + if IDENTIFIER_ONLY.is_match(text.trim()) { + return None; + } + let mut text = text.as_str(); + for pattern in [&*VOLUME, &*RANGE_PAGES, &*COMMA_YEAR, &*PP] { + if let Some(found) = pattern.find(text) { + text = &text[..found.start()]; + } + } + let text = tidy_title(text); + let text = text.trim_end_matches(|c: char| ",.;: ".contains(c)); + // `Oxford: Clarendon Press` names a place and then a publisher, and it + // is the publisher a catalogue is keyed on. The place stays in `text`. + let text = match text.split_once(": ") { + Some((place, publisher)) + if place.chars().count() <= 40 + && !place.chars().any(|c| c.is_ascii_digit()) + && publisher.chars().count() >= 3 => + { + publisher + } + _ => text, + }; + // `Sociometry 14` is a journal and a volume, and the volume is a field + // this record does not keep. What it is not is part of the name. + let text = match text.rsplit_once(' ') { + Some((name, last)) + if name.chars().count() >= 3 + && last.chars().all(|c| c.is_ascii_digit()) + && !last.is_empty() => + { + name.trim_end_matches(',') + } + _ => text, + }; + match text.chars().count() >= 3 && text.chars().any(char::is_alphabetic) { + true => Some(text.to_string()), + false => None, + } +} + +/// Divide an entry that marked none of its parts, on its sentence stops. +/// +/// The order is the one every style agrees on: who, then what, then where. +/// It is weaker than reading a marked-up title and is only reached when the +/// entry offered nothing to read. +fn by_sentence(text: &str) -> (Vec, Option, Option) { + let head = match PAREN_YEAR.find(text) { + // Author-date: the authors are what comes before the bracketed year. + Some(found) => &text[..found.start()], + None => text, + }; + let segments: Vec<&str> = sentences(text); + let found_authors = match PAREN_YEAR.find(text) { + Some(_) => authors(head), + None => segments + .first() + .map(|first| authors(first)) + .unwrap_or_default(), + }; + + // One segment and no names in it means the entry was never divided: the + // commas that hold it together say nothing about where the title starts. + // Reporting the whole entry as the title would be a wrong answer dressed + // as a settled one, which is worse than no answer at all. + if segments.len() < 2 && found_authors.is_empty() { + return (found_authors, None, None); + } + + // The title is the first segment that is not the authors and is not an + // imprint. An imprint has a colon in it, or names a publisher. + let mut rest = segments.iter().skip(match found_authors.is_empty() { + true => 0, + false => 1, + }); + let title = rest + .by_ref() + .find(|segment| { + let segment = segment.trim(); + segment.chars().count() >= 4 && !PAREN_YEAR.is_match(segment) + }) + .map(|segment| tidy_title(segment)); + let container = rest + .find(|segment| { + let segment = segment.trim(); + segment.chars().count() >= 3 && segment.chars().any(char::is_alphabetic) + }) + .and_then(|segment| tidy_container(segment)); + (found_authors, title, container) +} + +/// Cut an entry at its sentence stops, which is where styles put their parts. +/// +/// A stop inside an initial — `J. R. R.` — or after a known abbreviation is +/// not a stop, which is why this is not a split on '.'. +fn sentences(text: &str) -> Vec<&str> { + let bytes = text.as_bytes(); + let mut out = Vec::new(); + let mut start = 0; + let mut at = 0; + while at < bytes.len() { + if bytes[at] == b'.' { + let before = text[start..at].trim_end(); + let last = before + .rsplit(|c: char| c.is_whitespace()) + .next() + .unwrap_or(""); + let initial = last.chars().count() <= 1; + let abbreviated = matches!( + last.to_lowercase().as_str(), + "ed" | "eds" + | "trans" + | "vol" + | "no" + | "pp" + | "p" + | "rev" + | "st" + | "mr" + | "mrs" + | "dr" + | "jr" + | "sr" + | "cf" + | "esp" + | "repr" + ); + let followed_by_space = bytes.get(at + 1).is_some_and(|c| *c == b' '); + if !initial && !abbreviated && followed_by_space { + out.push(text[start..at].trim()); + start = at + 1; + } + } + at += 1; + } + if start < text.len() { + let tail = text[start..].trim(); + if !tail.is_empty() { + out.push(tail); + } + } + out.into_iter().filter(|part| !part.is_empty()).collect() +} + +/// What sort of work the entry points at. +fn kind(text: &str, found: &Reference) -> Kind { + if found.ids.iter().any(|id| id.kind == "arxiv") { + return Kind::Article; + } + let quoted = QUOTED.is_match(text); + let within = WITHIN.is_match(text); + if quoted && within { + return Kind::Chapter; + } + if found + .ids + .iter() + .any(|id| id.kind == "doi" || id.kind == "issn") + { + return Kind::Article; + } + if quoted && VOLUME.is_match(text) { + return Kind::Article; + } + // An imprint — a place, a colon, a publisher — is what a book has and a + // journal article does not. Asked of the entry rather than of the + // container, because the container has had the place cut off it by then. + let publisher = IMPRINT.is_match(text) + || found + .container + .as_deref() + .is_some_and(|container| PUBLISHER_WORD.is_match(container)); + if publisher { + return match within { + true => Kind::Chapter, + false => Kind::Book, + }; + } + if found.ids.iter().any(|id| id.kind == "isbn") { + return Kind::Book; + } + if VOLUME.is_match(text) && found.container.is_some() { + return Kind::Article; + } + // A container that is not a publisher, with a run of pages inside it, is + // a journal however sparely the style wrote the volume out. + let ranged = found + .pages + .as_deref() + .is_some_and(|pages| pages.contains(['-', '–', '—'])); + if ranged && found.container.is_some() { + return Kind::Article; + } + if found.ids.iter().any(|id| id.kind == "url") { + return Kind::Web; + } + Kind::Other +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(text: &str) -> Reference { + one(text, None).expect("a reference") + } + + #[test] + fn a_book_in_a_bibliography() { + let found = + read("Smith, John. *The Making of the Second Estate*. Oxford: Clarendon Press, 1998."); + assert_eq!(found.authors, ["Smith, John"]); + assert_eq!(found.year, Some(1998)); + assert_eq!( + found.title.as_deref(), + Some("The Making of the Second Estate") + ); + assert_eq!(found.container.as_deref(), Some("Clarendon Press")); + assert_eq!(found.kind, Kind::Book); + // The year of an imprint is not the page the work was cited at. + assert_eq!(found.pages, None); + } + + #[test] + fn an_article_with_its_journal_in_italics() { + let found = read( + "Doe, Jane, and Peter Brown. \"Obligation and Its Limits.\" \ + *Journal of Medieval Studies* 12, no. 3 (2001): 45-67.", + ); + assert_eq!(found.authors, ["Doe, Jane", "Peter Brown"]); + assert_eq!(found.title.as_deref(), Some("Obligation and Its Limits")); + assert_eq!( + found.container.as_deref(), + Some("Journal of Medieval Studies") + ); + assert_eq!(found.pages.as_deref(), Some("45-67")); + assert_eq!(found.kind, Kind::Article); + } + + #[test] + fn an_author_date_entry_puts_the_journal_in_the_italics_not_the_title() { + let found = read( + "Peirce, C. S. (1878). How to make our ideas clear. \ + *Popular Science Monthly*, 12, 286-302.", + ); + assert_eq!(found.authors, ["Peirce, C. S."]); + assert_eq!(found.title.as_deref(), Some("How to make our ideas clear")); + assert_eq!(found.container.as_deref(), Some("Popular Science Monthly")); + assert_eq!(found.pages.as_deref(), Some("286-302")); + } + + #[test] + fn an_author_date_entry_need_not_bracket_its_year() { + let found = read( + "Altmann, M. 1951, A Study of Behaviour in a Horse-Mule Group. Sociometry 14, 351-354.", + ); + assert_eq!(found.authors, ["Altmann, M."]); + assert_eq!(found.year, Some(1951)); + assert_eq!( + found.title.as_deref(), + Some("A Study of Behaviour in a Horse-Mule Group") + ); + // The volume belongs to the journal and is not part of its name. + assert_eq!(found.container.as_deref(), Some("Sociometry")); + } + + #[test] + fn a_chapter_appeared_in_the_book_and_not_in_its_publisher() { + let found = read( + "Brown, Peter. \"Sorcery, Demons and the Rise of Christianity.\" \ + In *Witchcraft Confessions and Accusations*, edited by Mary Douglas, \ + 17-45. London: Tavistock, 1970.", + ); + assert_eq!(found.kind, Kind::Chapter); + assert_eq!( + found.title.as_deref(), + Some("Sorcery, Demons and the Rise of Christianity") + ); + assert_eq!( + found.container.as_deref(), + Some("Witchcraft Confessions and Accusations") + ); + } + + #[test] + fn a_note_that_marks_nothing_up_is_still_read() { + let found = read("John MacGowan, The Imperial History of China (1897), p. 154."); + assert_eq!(found.authors, ["John MacGowan"]); + assert_eq!( + found.title.as_deref(), + Some("The Imperial History of China") + ); + assert_eq!(found.year, Some(1897)); + assert_eq!(found.pages.as_deref(), Some("154")); + } + + #[test] + fn what_a_note_says_before_the_citation_is_not_the_author() { + for lead in ["Quoted in ", "See ", "See, for example, ", "Cf. "] { + let found = read(&format!( + "{lead}Eugene F. Hemrick, One Nation Under God (Huntington, IN: Our Sunday Visitor, 2001), 72-74." + )); + assert_eq!(found.authors, ["Eugene F. Hemrick"], "{lead}"); + assert_eq!( + found.container.as_deref(), + Some("Our Sunday Visitor"), + "{lead}" + ); + } + } + + #[test] + fn a_contributors_biography_is_not_a_citation() { + // Opens exactly as a Chicago note does and then runs on for three + // sentences, which a citation does not. + let found = read( + "Thomas L. Sheridan, SJ, is Professor Emeritus of Theology at St \ + Peter's College. He is the author of Newman on Justification (1967).", + ); + assert!(found.authors.is_empty(), "{found:#?}"); + } + + #[test] + fn an_editor_is_a_person_and_not_a_disqualification() { + let found = read( + "Eberhardt, Christian, ed. *Ritual and Metaphor: Sacrifice in the Bible*. Atlanta: Society of Biblical Literature, 2011.", + ); + assert_eq!(found.authors, ["Eberhardt, Christian"]); + assert_eq!( + found.title.as_deref(), + Some("Ritual and Metaphor: Sacrifice in the Bible") + ); + } + + #[test] + fn an_entry_with_nothing_but_commas_is_read_between_the_names_and_the_imprint() { + let found = read( + "Schopflin, George, Nations, Identity, Power, New York: New York University Press, 2000", + ); + assert_eq!(found.authors, ["Schopflin, George"]); + assert_eq!(found.title.as_deref(), Some("Nations, Identity, Power")); + assert_eq!( + found.container.as_deref(), + Some("New York University Press") + ); + assert_eq!(found.year, Some(2000)); + } + + #[test] + fn a_continuation_is_flagged_and_not_resolved() { + let found = read("Ibid., pp. 23-24, 39."); + assert!(found.continuation); + assert!(found.title.is_none()); + assert!(found.authors.is_empty()); + assert_eq!(found.pages.as_deref(), Some("23-24")); + // Nothing to ask a model about either: the entry does not say. + assert!(found.missing().is_empty()); + } + + #[test] + fn identifiers_are_normalised_and_the_doi_is_not_repeated_as_a_url() { + let found = read( + "Chen, L. (2019). A study of things. Nature, 570(7761), 321-325. \ + https://doi.org/10.1038/s41586-019-1234-5", + ); + assert_eq!(found.ids.len(), 1); + assert_eq!(found.ids[0].kind, "doi"); + assert_eq!(found.ids[0].id, "10.1038/s41586-019-1234-5"); + assert_eq!(found.kind, Kind::Article); + } + + #[test] + fn a_preprint_has_an_identifier_and_no_journal() { + let found = + read("Vaswani, A., et al. (2017). Attention is all you need. arXiv:1706.03762."); + assert_eq!(found.ids[0].kind, "arxiv"); + assert_eq!(found.kind, Kind::Article); + // Naming arXiv as the journal would invent one. + assert_eq!(found.container, None); + } + + #[test] + fn a_heading_or_a_page_number_is_not_a_reference() { + for junk in ["Bibliography", "233", "Notes to pages 12-14", "see below"] { + assert!(one(junk, None).is_none(), "{junk}"); + } + } + + #[test] + fn the_same_entry_reads_the_same_way_every_time() { + let entry = "Smith, John. *A Title*. Oxford: Clarendon Press, 1998."; + assert_eq!(one(entry, None), one(entry, None)); + } + + #[test] + fn a_year_range_inside_a_title_is_not_a_page_range() { + let found = read( + "A. J. P. Taylor, The Struggle for Mastery in Europe 1848-1918 (Oxford: Oxford University Press, 1954), p. 102.", + ); + assert_eq!(found.pages.as_deref(), Some("102")); + } + + #[test] + fn sentences_do_not_stop_at_an_initial_or_an_abbreviation() { + assert_eq!( + sentences("Tolkien, J. R. R. The Hobbit, rev. ed. London, 1951."), + ["Tolkien, J. R. R. The Hobbit, rev. ed. London, 1951."] + ); + } +} diff --git a/crates/citefind/src/prompt.rs b/crates/citefind/src/prompt.rs new file mode 100644 index 0000000..e29854f --- /dev/null +++ b/crates/citefind/src/prompt.rs @@ -0,0 +1,194 @@ +//! A question to put to a model, for the references that would not parse. +//! +//! Nothing here sends anything. The prompt is composed and printed, and what +//! happens to it is the caller's business — their model, their machine, their +//! GPU. The tool stays what every tool here is: the same output for the same +//! input, on every machine, with no network and nothing to install. +//! +//! One prompt covers every unsettled entry rather than one prompt each. A +//! reference list is long and mostly ordinary; the entries worth asking about +//! are the mangled few, and they are cheaper to ask about together. + +use crate::Reference; + +/// The version of the prompt below. +/// +/// Bumped when the wording changes in a way that could change an answer, so a +/// caller keeping results can tell which prompt produced them. +pub const PROMPT_VERSION: u32 = 1; + +/// Compose one question covering every reference that did not fully parse. +/// +/// `None` when every reference settled, because there is nothing to ask and +/// a call to a model would be waste. +pub fn compose(references: &[Reference]) -> Option { + let wanted: Vec<&Reference> = references + .iter() + .filter(|one| !one.missing().is_empty()) + .collect(); + if wanted.is_empty() { + return None; + } + + let mut out = String::new(); + out.push_str( + "Below are entries from the bibliography or notes of a book. Each was \ + read as far as its punctuation allowed, and the fields that could not \ + be read off it are listed with it.\n\n", + ); + for one in &wanted { + out.push_str(&format!("--- {}\n", one.idx)); + out.push_str(&format!("text: {}\n", one.text)); + let known = known(one); + if !known.is_empty() { + out.push_str(&known); + } + out.push_str(&format!("wanted: {}\n\n", one.missing().join(", "))); + } + + out.push_str( + "Reply with one JSON object and nothing else. Its keys are the entry \ + numbers above, as strings; each value is an object using only these \ + keys, and only the ones listed as wanted for that entry: authors, \ + year, title, container. `authors` is a list of names as the entry \ + writes them. `year` is a four-digit number. `title` is the work \ + cited — for an article the article, not the journal. `container` is \ + the journal an article appeared in, or the publisher of a book.\n\n\ + Use null for anything the entry does not settle. A guess is worse \ + than an absence, because nobody downstream can tell them apart. Do \ + not resolve `ibid.` or `op. cit.` — an entry that only points at an \ + earlier one has nothing of its own to give.\n", + ); + Some(out) +} + +/// What was read off the entry, as the prompt shows it. +fn known(one: &Reference) -> String { + let mut out = String::new(); + if !one.authors.is_empty() { + out.push_str(&format!("authors: {}\n", one.authors.join("; "))); + } + if let Some(year) = one.year { + out.push_str(&format!("year: {year}\n")); + } + if let Some(title) = &one.title { + out.push_str(&format!("title: {title}\n")); + } + if let Some(container) = &one.container { + out.push_str(&format!("container: {container}\n")); + } + out +} + +/// Take a model's reply and fill in the fields that were missing. +/// +/// Only the missing ones, and only on entries that asked: a model that +/// volunteers a better title for a reference whose title was read off the +/// page is overruled, because the page is evidence and the model is a guess. +/// A reply that is not one JSON object is refused rather than half-applied. +pub fn merge(references: &mut [Reference], reply: &str) -> Result { + let json = object_in(reply).ok_or("no JSON object in the reply")?; + let value: serde_json::Value = + serde_json::from_str(json).map_err(|e| format!("not readable as JSON: {e}"))?; + let object = value.as_object().ok_or("the reply is not an object")?; + + let mut filled = 0; + for one in references.iter_mut() { + let missing = one.missing(); + if missing.is_empty() { + continue; + } + let Some(answer) = object.get(&one.idx.to_string()) else { + continue; + }; + let mut touched = false; + + if missing.contains(&"authors") { + let authors: Vec = answer + .get("authors") + .and_then(serde_json::Value::as_array) + .map(|list| { + list.iter() + .filter_map(serde_json::Value::as_str) + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect() + }) + .unwrap_or_default(); + if !authors.is_empty() { + one.authors = authors; + touched = true; + } + } + if missing.contains(&"year") { + let year = answer + .get("year") + .and_then(|found| match found { + serde_json::Value::Number(number) => number.as_u64(), + serde_json::Value::String(text) => text.trim().parse().ok(), + _ => None, + }) + .filter(|year| (1000..=2100).contains(year)); + if let Some(year) = year { + one.year = Some(year as u32); + touched = true; + } + } + for (key, slot) in [("title", &mut one.title), ("container", &mut one.container)] { + if !missing.contains(&key) { + continue; + } + let found = answer + .get(key) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|text| !text.is_empty()); + if let Some(found) = found { + *slot = Some(found.to_string()); + touched = true; + } + } + + if touched { + filled += 1; + } + one.unsettled = one.missing().iter().map(|s| s.to_string()).collect(); + } + Ok(filled) +} + +/// The first balanced `{…}` in a reply. +/// +/// Models wrap JSON in prose and in code fences however they were asked not +/// to, and refusing a good answer over a fence helps nobody. +fn object_in(reply: &str) -> Option<&str> { + let start = reply.find('{')?; + let bytes = reply.as_bytes(); + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + for at in start..bytes.len() { + let byte = bytes[at]; + if in_string { + match (escaped, byte) { + (true, _) => escaped = false, + (false, b'\\') => escaped = true, + (false, b'"') => in_string = false, + _ => {} + } + continue; + } + match byte { + b'"' => in_string = true, + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return Some(&reply[start..=at]); + } + } + _ => {} + } + } + None +} diff --git a/crates/citefind/src/split.rs b/crates/citefind/src/split.rs new file mode 100644 index 0000000..92bd795 --- /dev/null +++ b/crates/citefind/src/split.rs @@ -0,0 +1,383 @@ +//! Cutting a block of references into the references in it. +//! +//! Two shapes arrive. A page of notes numbers its entries, and the numbering +//! says exactly where each one starts — that case is decided, not guessed at, +//! and the pattern that recognises it is the one `chunkmd` already uses to +//! recognise a page of notes, so the two cannot drift apart. +//! +//! A bibliography numbers nothing. There the signal is layout: a blank line +//! between entries, a hanging indent under each, or the surname-comma opening +//! that nearly every author-first entry begins with. Whichever of those the +//! block actually carries is the one used, and when it carries none of them +//! the fallback is one entry per line, which is right for a list that was +//! never wrapped and visibly wrong for one that was. + +use std::sync::LazyLock; + +use regex::Regex; + +/// One entry, cut out of the block. +#[derive(Debug, Clone, PartialEq)] +pub struct Entry { + /// The entry's text, with its marker removed and whitespace collapsed. + pub text: String, + /// The note number it carried, where it carried one. + pub marker: Option, +} + +/// At least this many numbered notes before the numbering is believed. +/// +/// Two: one is a stray list item, and a real page of notes has more. +const MIN_NOTES: usize = 2; + +/// The opening of an author-first bibliography entry: a surname, a comma, +/// and something after it. Deliberately narrow — it is being used to decide +/// where an entry *starts*, and a loose version would cut entries in half at +/// every "Oxford, 1998". +static SURNAME_FIRST: LazyLock = LazyLock::new(|| { + Regex::new(r"^[\p{Lu}][\p{L}'’\-]{1,30}(?:\s+[\p{Lu}][\p{L}'’\-]{1,30}){0,2},\s+[\p{Lu}]") + .expect("a valid pattern") +}); + +/// A leading note number, once the tags around it are gone. +/// +/// Three digits at most, which is what `chunkmd` counts as a note number. +/// Four would swallow the year off the front of an author-date entry. +static LEADING_NUMBER: LazyLock = + LazyLock::new(|| Regex::new(r"^\[?(\d{1,3})\]?\\?[.)]?\s+").expect("a valid pattern")); + +/// An HTML tag, which conversion leaves wrapped around note numbers. +static TAG: LazyLock = + LazyLock::new(|| Regex::new(r"]*>").expect("a valid pattern")); + +/// A markdown link left where a back-reference anchor was. The href points +/// at a fragment in some EPUBs and at another file in most: `[42](#note-42)` +/// and `[4.](part0011.html#ke_00001)` are the same marker. +static ANCHOR_LINK: LazyLock = LazyLock::new(|| { + Regex::new(r"^\[\s*(\d{1,4})\s*\\?[.)]?\s*\]\([^)\s]*\)\s*[.)]?\s*").expect("a valid pattern") +}); + +/// A marker `chunkmd` does not count, because it does not need to: a +/// back-reference whose link points at another file rather than at a +/// fragment. `chunkmd` is deciding whether a chunk is a page of notes, and +/// one missed marker among thirty does not change that answer. Here a missed +/// marker runs two references together, so these are added to its. +static FILE_ANCHOR: LazyLock = LazyLock::new(|| { + Regex::new(r"(?m)^[ \t]{0,8}(?:[>•]+[ \t]*|-[ \t]+)?\[\s*\d{1,3}\s*\\?[.)]?\s*\]\([^)\s]*\)") + .expect("a valid pattern") +}); + +/// A bullet or a blockquote marker, which conversion puts in front of a note +/// often enough to matter. A leading `*` is left alone: it opens an italic +/// title as often as it opens a list. +static BULLET: LazyLock = + LazyLock::new(|| Regex::new(r"^(?:[>•]+\s*|-\s+)+").expect("a valid pattern")); + +/// An id-style marker: `fn39.`, `note12)`. +static ID_MARKER: LazyLock = + LazyLock::new(|| Regex::new(r"^(?:fn|ftn|note)(\d{1,4})[.)]\s+").expect("a valid pattern")); + +/// Cut a block into entries. +pub fn entries(text: &str) -> Vec { + let numbered = numbered_notes(text); + if numbered.len() >= MIN_NOTES { + return numbered; + } + unnumbered(text) +} + +/// Entries cut at their note numbers. +/// +/// An entry runs from its marker to the next one. Only entries that cite +/// something are kept: a page of notes mixes bibliographic notes with +/// discursive ones, and a remark about the argument is not a reference. +fn numbered_notes(text: &str) -> Vec { + let mut starts = chunkmd::footnote::note_starts(text); + starts.extend(FILE_ANCHOR.find_iter(text).map(|found| found.start())); + starts.sort_unstable(); + starts.dedup(); + // Decided before the head is added, so that one stray `12.` in a + // bibliography cannot turn the whole of it into two entries. + if starts.len() < MIN_NOTES { + return Vec::new(); + } + // Whatever is in front of the first marker is the first note, when the + // marker it carried was one neither pattern knows. Dropping it would + // lose a reference silently, which is the one thing a cut must not do. + if starts[0] > 0 { + starts.insert(0, 0); + } + + let mut out = Vec::new(); + for (at, start) in starts.iter().enumerate() { + let end = starts.get(at + 1).copied().unwrap_or(text.len()); + let raw = &text[*start..end]; + if !chunkmd::footnote::cites_something(raw) { + continue; + } + out.push(strip_marker(raw)); + } + out +} + +/// Entries cut at whatever the layout offers. +fn unnumbered(text: &str) -> Vec { + let blocks: Vec<&str> = text + .split("\n\n") + .map(str::trim) + .filter(|block| !block.is_empty()) + .collect(); + if blocks.len() >= 2 { + return blocks.iter().map(|block| strip_marker(block)).collect(); + } + + let lines: Vec<&str> = text + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + if lines.len() < 2 { + return lines.iter().map(|line| strip_marker(line)).collect(); + } + + // A surname-comma opening on most lines means the list is one entry per + // line and any line without it is a wrap of the line above. Without that, + // fall back to the hanging indent, and without that to the lines + // themselves. + let openers = lines + .iter() + .filter(|line| SURNAME_FIRST.is_match(line.trim_start())) + .count(); + let by_surname = openers >= 2 && openers * 2 >= lines.len(); + + let mut out: Vec = Vec::new(); + for line in &lines { + let indented = line.starts_with(' ') || line.starts_with('\t'); + let starts_entry = match by_surname { + true => SURNAME_FIRST.is_match(line.trim_start()), + false => !indented, + }; + match (starts_entry, out.last_mut()) { + (false, Some(previous)) => { + previous.push(' '); + previous.push_str(line.trim()); + } + _ => out.push(line.trim().to_string()), + } + } + out.iter().map(|entry| strip_marker(entry)).collect() +} + +/// Take the note number off the front of an entry, and tidy what is left. +/// +/// The number arrives wrapped in whatever the converter left behind, so the +/// patterns are matched against the opening with its tags removed — and the +/// cut is then made in the original, so that an italic span marking a title +/// further along survives. +fn strip_marker(raw: &str) -> Entry { + let text = tidy(raw); + let text = BULLET.replace(&text, "").to_string(); + let (bare, from) = visible(&text, 160); + + for pattern in [&*ANCHOR_LINK, &*ID_MARKER, &*LEADING_NUMBER] { + let Some(found) = pattern.captures(&bare) else { + continue; + }; + let whole = found.get(0).expect("the whole match"); + let consumed = bare[..whole.end()].chars().count(); + let cut = from[consumed]; + return Entry { + text: text[cut..].trim_start().to_string(), + marker: Some(found[1].to_string()), + }; + } + + if let Some((marker, consumed)) = superscript(&bare) { + return Entry { + text: text[from[consumed]..].trim_start().to_string(), + marker: Some(marker), + }; + } + + Entry { text, marker: None } +} + +/// The opening of an entry with its tags dropped, and where each character +/// that was kept came from in the original. +/// +/// The last element is where the kept text ends, so that a marker running to +/// the end of the opening still has somewhere to cut. +fn visible(text: &str, limit: usize) -> (String, Vec) { + let mut out = String::new(); + let mut from = Vec::new(); + let mut at = 0; + while at < text.len() && from.len() < limit { + let rest = &text[at..]; + if rest.starts_with('<') { + if let Some(close) = rest.find('>') { + if TAG.is_match(&rest[..=close]) { + at += close + 1; + continue; + } + } + } + let ch = rest.chars().next().expect("a character"); + if !(out.is_empty() && ch.is_whitespace()) { + out.push(ch); + from.push(at); + } + at += ch.len_utf8(); + } + from.push(at); + (out, from) +} + +/// A leading run of superscript digits, as the number it spells and how many +/// characters it took. +fn superscript(text: &str) -> Option<(String, usize)> { + const SUPER: [char; 10] = ['⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹']; + let mut digits = String::new(); + let mut taken = 0; + for ch in text.chars() { + match SUPER.iter().position(|s| *s == ch) { + Some(value) => { + digits.push(char::from_digit(value as u32, 10)?); + taken += 1; + } + None => break, + } + } + if digits.is_empty() { + return None; + } + // Step over the space the marker is separated from its entry by. + let after: String = text + .chars() + .skip(taken) + .take_while(|c| c.is_whitespace()) + .collect(); + Some((digits, taken + after.chars().count())) +} + +/// Collapse the whitespace a converted entry is wrapped with. +fn tidy(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn texts(entries: &[Entry]) -> Vec<&str> { + entries.iter().map(|entry| entry.text.as_str()).collect() + } + + #[test] + fn a_page_of_numbered_notes_is_cut_at_its_numbers() { + let page = "\ +1. John Smith, *The Title* (Oxford: Clarendon Press, 1998), 183. +2. Jane Doe, *Another Book* (Cambridge: University Press, 2001), p. 12."; + let found = entries(page); + assert_eq!(found.len(), 2, "{found:#?}"); + assert_eq!(found[0].marker.as_deref(), Some("1")); + assert_eq!(found[1].marker.as_deref(), Some("2")); + assert!(found[0].text.starts_with("John Smith"), "{:?}", found[0]); + } + + #[test] + fn a_note_that_cites_nothing_is_not_a_reference() { + let page = "\ +1. John Smith, *The Title* (Oxford: Clarendon Press, 1998), 183. +2. I am indebted to the seminar for this point. +3. Jane Doe, *Another Book* (Cambridge: University Press, 2001), p. 12."; + let found = entries(page); + assert_eq!( + found + .iter() + .filter_map(|e| e.marker.clone()) + .collect::>(), + ["1", "3"] + ); + } + + #[test] + fn blank_lines_cut_a_bibliography() { + let block = "Smith, John. The Title. Oxford: Clarendon, 1998.\n\nDoe, Jane. Another.\nCambridge, 2001."; + let found = entries(block); + assert_eq!(found.len(), 2); + assert_eq!(found[1].text, "Doe, Jane. Another. Cambridge, 2001."); + } + + #[test] + fn a_surname_opening_cuts_a_list_that_was_wrapped() { + let block = "\ +Smith, John. The Title of the Book That Ran On. +Oxford: Clarendon Press, 1998. +Doe, Jane. Another Work Entirely. Cambridge, 2001."; + let found = entries(block); + assert_eq!( + texts(&found), + [ + "Smith, John. The Title of the Book That Ran On. Oxford: Clarendon Press, 1998.", + "Doe, Jane. Another Work Entirely. Cambridge, 2001.", + ] + ); + } + + #[test] + fn a_hanging_indent_cuts_a_list_with_no_surnames_to_go_on() { + let block = "\ +The Oxford Companion to Everything. Oxford, 1998. + Second line of the same entry. +Another Anonymous Work. Cambridge, 2001."; + let found = entries(block); + assert_eq!(found.len(), 2, "{found:#?}"); + assert!(found[0].text.ends_with("same entry."), "{:?}", found[0]); + } + + #[test] + fn a_marker_wrapped_in_tags_is_still_a_marker() { + let entry = strip_marker("12. Smith, *Title*, p. 4."); + assert_eq!(entry.marker.as_deref(), Some("12")); + assert_eq!(entry.text, "Smith, *Title*, p. 4."); + } + + #[test] + fn an_anchor_link_marker_keeps_the_entry_and_not_the_link() { + let entry = strip_marker("[42](#note-42). Smith, *Title*, p. 4."); + assert_eq!(entry.marker.as_deref(), Some("42")); + assert_eq!(entry.text, "Smith, *Title*, p. 4."); + } + + #[test] + fn an_id_style_marker_is_read_as_its_number() { + let entry = strip_marker("fn39. Smith, *Title*, p. 4."); + assert_eq!(entry.marker.as_deref(), Some("39")); + assert_eq!(entry.text, "Smith, *Title*, p. 4."); + } + + #[test] + fn a_superscript_marker_is_read_as_its_number() { + let entry = strip_marker("¹² Smith, *Title*, p. 4."); + assert_eq!(entry.marker.as_deref(), Some("12")); + assert_eq!(entry.text, "Smith, *Title*, p. 4."); + } + + #[test] + fn an_entry_with_no_marker_keeps_all_of_itself() { + let entry = strip_marker("Smith, John. The Title. Oxford, 1998."); + assert_eq!(entry.marker, None); + assert_eq!(entry.text, "Smith, John. The Title. Oxford, 1998."); + } + + #[test] + fn a_year_in_a_place_of_publication_does_not_start_an_entry() { + // "Oxford, 1998." would match a loose surname-comma rule. + let block = "\ +Smith, John. A Title. +Oxford, 1998. +Doe, Jane. Another. +Cambridge, 2001."; + let found = entries(block); + assert_eq!(found.len(), 2, "{found:#?}"); + } +} diff --git a/crates/citefind/tests/references.rs b/crates/citefind/tests/references.rs new file mode 100644 index 0000000..f26d32a --- /dev/null +++ b/crates/citefind/tests/references.rs @@ -0,0 +1,177 @@ +//! Reading a block of references end to end. + +use citefind::{Kind, read}; + +/// A page of endnotes as conversion leaves it: numbers, anchors, and a +/// discursive note in among the citations. +const NOTES: &str = "\ +[1.](part0011.html#ke_00001_00003) John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154. +2. I am grateful to the seminar at Leiden for pressing me on this. +[3](text0007.xhtml#en3). Johanna Bockman, *Markets in the Name of Socialism* (Stanford, CA: Stanford University Press, 2011), 40-42. +4. Ibid., p. 44. +"; + +#[test] +fn a_page_of_notes_keeps_its_numbering_and_drops_its_asides() { + let found = read(NOTES); + assert_eq!(found.len(), 3, "{found:#?}"); + assert_eq!( + found + .iter() + .map(|one| one.marker.as_deref()) + .collect::>(), + [Some("1"), Some("3"), Some("4")] + ); + // Numbered from zero in the order they appeared, whatever the markers say. + assert_eq!( + found.iter().map(|one| one.idx).collect::>(), + [0, 1, 2] + ); + assert_eq!(found[0].authors, ["John MacGowan"]); + assert_eq!(found[0].container.as_deref(), Some("Curzon")); + assert_eq!(found[1].year, Some(2011)); + assert_eq!(found[1].kind, Kind::Book); + assert!(found[2].continuation); +} + +#[test] +fn a_rule_of_dashes_means_the_author_above() { + let block = "\ +Smith, John. *The First Book*. Oxford: Clarendon Press, 1998. + +———. *The Second Book*. Oxford: Clarendon Press, 2001. + +——— *The Third Book*. Oxford: Clarendon Press, 2004. + +Doe, Jane. *Something Else*. Cambridge: University Press, 2005. + +———. *And Another*. Cambridge: University Press, 2007."; + let found = read(block); + assert_eq!(found.len(), 5, "{found:#?}"); + assert_eq!( + found + .iter() + .map(|one| one.authors.clone()) + .collect::>(), + [ + vec!["Smith, John"], + vec!["Smith, John"], + vec!["Smith, John"], + vec!["Doe, Jane"], + vec!["Doe, Jane"], + ] + ); + assert_eq!(found[2].title.as_deref(), Some("The Third Book")); +} + +#[test] +fn an_ibid_does_not_inherit_an_author_the_way_a_dash_does() { + let block = "\ +Smith, John. *The First Book*. Oxford: Clarendon Press, 1998. + +Ibid., p. 44."; + let found = read(block); + assert_eq!(found.len(), 2); + assert!(found[1].authors.is_empty(), "{:#?}", found[1]); + assert!(found[1].continuation); +} + +#[test] +fn every_record_says_what_it_could_not_settle() { + let found = read( + "Marron, Kevin Ritual Abuse, Seal Books, 1988\n\nAnother, Line. Of nothing much, 1990.", + ); + for one in &found { + assert_eq!( + one.unsettled, + one.missing() + .iter() + .map(|s| s.to_string()) + .collect::>() + ); + } +} + +#[test] +fn the_same_block_reads_the_same_way_every_time() { + assert_eq!(read(NOTES), read(NOTES)); +} + +/// A block mixing one entry that reads cleanly, one that gives up nothing +/// but a title, and a continuation. +const MIXED: &str = "\ +1. John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154. +2. Marron, Kevin Ritual Abuse, Seal Books, 1988. +3. Ibid., p. 44. +"; + +#[test] +fn a_prompt_asks_about_the_entries_that_did_not_settle_and_no_others() { + let found = read(MIXED); + assert_eq!(found.len(), 3, "{found:#?}"); + let prompt = citefind::prompt::compose(&found).expect("something unsettled"); + + // The entry that read cleanly is not in the question, and neither is the + // continuation — it has nothing of its own to give. + assert!(!prompt.contains("MacGowan"), "{prompt}"); + assert!(!prompt.contains("Ibid"), "{prompt}"); + assert!(prompt.contains("Ritual Abuse"), "{prompt}"); + assert!(prompt.contains("wanted:"), "{prompt}"); +} + +#[test] +fn a_prompt_is_only_composed_where_there_is_something_to_ask() { + let settled = read( + "1. John MacGowan, *The Imperial History of China* (London: Curzon, 1897), p. 154.\n2. Johanna Bockman, *Markets in the Name of Socialism* (Stanford, CA: Stanford University Press, 2011), 40.", + ); + assert!( + settled.iter().all(|one| one.unsettled.is_empty()), + "{settled:#?}" + ); + assert_eq!(citefind::prompt::compose(&settled), None); +} + +#[test] +fn a_reply_about_an_entry_that_settled_does_not_overrule_the_page() { + let mut found = read(MIXED); + let before = found[0].clone(); + let filled = citefind::prompt::merge( + &mut found, + r#"{"0": {"title": "Invented", "container": "Invented"}, "2": {"title": "Invented"}}"#, + ) + .expect("a reply"); + assert_eq!(filled, 0); + assert_eq!(found[0], before); + assert_eq!(found[2].title, None); +} + +#[test] +fn a_reply_fills_in_only_what_was_asked_about() { + let mut found = read("Marron, Kevin Ritual Abuse, Seal Books, 1988"); + assert_eq!(found.len(), 1); + assert!(found[0].unsettled.contains(&"authors".to_string())); + + let filled = citefind::prompt::merge( + &mut found, + "Sure!\n```json\n{\"0\": {\"authors\": [\"Marron, Kevin\"], \"year\": 1066}}\n```", + ) + .expect("a reply"); + assert_eq!(filled, 1); + assert_eq!(found[0].authors, ["Marron, Kevin"]); + // The year was read off the page, so the model's does not replace it. + assert_eq!(found[0].year, Some(1988)); + assert!(!found[0].unsettled.contains(&"authors".to_string())); +} + +#[test] +fn a_reply_that_is_not_json_is_refused_rather_than_half_applied() { + let mut found = read(NOTES); + let before = found.clone(); + assert!(citefind::prompt::merge(&mut found, "I could not tell.").is_err()); + assert_eq!(found, before); +} + +#[test] +fn nothing_citation_shaped_gives_nothing() { + assert!(read("Bibliography\n\nSee below.\n\n233\n").is_empty()); +}