From d551adc14c75e5fd59f8d10f88ad36a0f07f263d Mon Sep 17 00:00:00 2001 From: funkpower Date: Tue, 22 Sep 2026 08:39:27 -0400 Subject: [PATCH] =?UTF-8?q?0.3.0=20=E2=80=94=20ask=20every=20register=20an?= =?UTF-8?q?d=20keep=20all=20of=20the=20answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain asked three registers and returned whichever answered first with a title and an author, discarding the rest. The registers are complementary rather than redundant: OpenLibrary's edition record carries the publisher, the pagination and the language and names its authors by key, while its search index carries the authors and not the publisher. One book, two half-answers, and the whole of it only if they are put together. They are merged now, first answer winning each field, so the order of the chain is the order of trust. The record says which registers spoke. It also carries what was being thrown away: subjects, language and a page count, from whichever register has them. The index will not return those unless asked for by name, which is how a language it held went unread. Subjects are capped at twelve — the index lists every heading every edition of a work has ever been given, which on a well-known book is hundreds. An edition record keeps its subtitle in a separate field, so a title came back without it. --- Cargo.lock | 52 ++++++------ Cargo.toml | 54 ++++++------ crates/bibfetch/src/ask.rs | 17 ++-- crates/bibfetch/src/lib.rs | 150 ++++++++++++++++++++++++++++++++++ crates/bibfetch/src/source.rs | 63 +++++++++++++- 5 files changed, 275 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c22c3de..1e20184 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "avplan" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bibclean" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "bibfetch" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -196,7 +196,7 @@ dependencies = [ [[package]] name = "bookmeta" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -210,7 +210,7 @@ dependencies = [ [[package]] name = "booksniff" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "catena" -version = "0.2.3" +version = "0.3.0" dependencies = [ "bibclean", "bibfetch", @@ -277,7 +277,7 @@ dependencies = [ [[package]] name = "catena-cli" -version = "0.2.3" +version = "0.3.0" dependencies = [ "clap", "clap_mangen", @@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chunkmd" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -334,7 +334,7 @@ dependencies = [ [[package]] name = "citefind" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "chunkmd", @@ -399,7 +399,7 @@ dependencies = [ [[package]] name = "colophon" -version = "0.2.3" +version = "0.3.0" dependencies = [ "bibclean", "catena-cli", @@ -448,7 +448,7 @@ dependencies = [ [[package]] name = "corpus-core" -version = "0.2.3" +version = "0.3.0" dependencies = [ "schemars", "serde", @@ -492,7 +492,7 @@ dependencies = [ [[package]] name = "cuekit" -version = "0.2.3" +version = "0.3.0" dependencies = [ "corpus-core", "regex", @@ -504,7 +504,7 @@ dependencies = [ [[package]] name = "cuenorm" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -514,7 +514,7 @@ dependencies = [ [[package]] name = "cuepack" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -525,7 +525,7 @@ dependencies = [ [[package]] name = "cueseg" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -612,7 +612,7 @@ dependencies = [ [[package]] name = "emb" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -876,7 +876,7 @@ dependencies = [ [[package]] name = "idnorm" -version = "0.2.3" +version = "0.3.0" dependencies = [ "bibclean", "catena-cli", @@ -942,7 +942,7 @@ dependencies = [ [[package]] name = "langid" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "mdscrub" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1034,7 +1034,7 @@ dependencies = [ [[package]] name = "mdsplit" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "pdfgutter" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1126,7 +1126,7 @@ dependencies = [ [[package]] name = "pdftriage" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" [[package]] name = "rrf" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "seriesof" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "textkit" -version = "0.2.3" +version = "0.3.0" dependencies = [ "regex", "tiktoken-rs", @@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tocfind" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", diff --git a/Cargo.toml b/Cargo.toml index 94f11b6..e53e0a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,39 +3,39 @@ resolver = "3" members = ["crates/*"] [workspace.package] -version = "0.2.3" +version = "0.3.0" edition = "2024" rust-version = "1.85" license = "MPL-2.0" repository = "https://git.pumpin.online/funkpower/catena" [workspace.dependencies] -catena-cli = { path = "crates/catena-cli", version = "0.2.3" } -corpus-core = { path = "crates/corpus-core", version = "0.2.3" } -cuekit = { path = "crates/cuekit", version = "0.2.3" } -textkit = { path = "crates/textkit", version = "0.2.3" } -chunkmd = { path = "crates/chunkmd", version = "0.2.3" } -mdscrub = { path = "crates/mdscrub", version = "0.2.3" } -mdsplit = { path = "crates/mdsplit", version = "0.2.3" } -pdftriage = { path = "crates/pdftriage", version = "0.2.3" } -cuenorm = { path = "crates/cuenorm", version = "0.2.3" } -cueseg = { path = "crates/cueseg", version = "0.2.3" } -rrf = { path = "crates/rrf", version = "0.2.3" } -cuepack = { path = "crates/cuepack", version = "0.2.3" } -bibclean = { path = "crates/bibclean", version = "0.2.3" } -avplan = { path = "crates/avplan", version = "0.2.3" } -pdfgutter = { path = "crates/pdfgutter", version = "0.2.3" } -catena = { path = "crates/bindings", version = "0.2.3" } -seriesof = { path = "crates/seriesof", version = "0.2.3" } -idnorm = { path = "crates/idnorm", version = "0.2.3" } -bibfetch = { path = "crates/bibfetch", version = "0.2.3" } -bookmeta = { path = "crates/bookmeta", version = "0.2.3" } -langid = { path = "crates/langid", version = "0.2.3" } -tocfind = { path = "crates/tocfind", version = "0.2.3" } -booksniff = { path = "crates/booksniff", version = "0.2.3" } -citefind = { path = "crates/citefind", version = "0.2.3" } -colophon = { path = "crates/colophon", version = "0.2.3" } -emb = { path = "crates/emb", version = "0.2.3" } +catena-cli = { path = "crates/catena-cli", version = "0.3.0" } +corpus-core = { path = "crates/corpus-core", version = "0.3.0" } +cuekit = { path = "crates/cuekit", version = "0.3.0" } +textkit = { path = "crates/textkit", version = "0.3.0" } +chunkmd = { path = "crates/chunkmd", version = "0.3.0" } +mdscrub = { path = "crates/mdscrub", version = "0.3.0" } +mdsplit = { path = "crates/mdsplit", version = "0.3.0" } +pdftriage = { path = "crates/pdftriage", version = "0.3.0" } +cuenorm = { path = "crates/cuenorm", version = "0.3.0" } +cueseg = { path = "crates/cueseg", version = "0.3.0" } +rrf = { path = "crates/rrf", version = "0.3.0" } +cuepack = { path = "crates/cuepack", version = "0.3.0" } +bibclean = { path = "crates/bibclean", version = "0.3.0" } +avplan = { path = "crates/avplan", version = "0.3.0" } +pdfgutter = { path = "crates/pdfgutter", version = "0.3.0" } +catena = { path = "crates/bindings", version = "0.3.0" } +seriesof = { path = "crates/seriesof", version = "0.3.0" } +idnorm = { path = "crates/idnorm", version = "0.3.0" } +bibfetch = { path = "crates/bibfetch", version = "0.3.0" } +bookmeta = { path = "crates/bookmeta", version = "0.3.0" } +langid = { path = "crates/langid", version = "0.3.0" } +tocfind = { path = "crates/tocfind", version = "0.3.0" } +booksniff = { path = "crates/booksniff", version = "0.3.0" } +citefind = { path = "crates/citefind", version = "0.3.0" } +colophon = { path = "crates/colophon", version = "0.3.0" } +emb = { path = "crates/emb", version = "0.3.0" } clap = { version = "4", features = ["derive"] } clap_mangen = "0.3" diff --git a/crates/bibfetch/src/ask.rs b/crates/bibfetch/src/ask.rs index c08de32..bececa2 100644 --- a/crates/bibfetch/src/ask.rs +++ b/crates/bibfetch/src/ask.rs @@ -63,13 +63,18 @@ pub fn lookup( for source in chain { match ask(agent, *source, id, kind, attempts) { Ok(record) => { - let complete = record.found && record.title.is_some() && !record.authors.is_empty(); - let better = best.as_ref().is_none_or(|kept| record.found && !kept.found); - if complete { - return Ok(Some(record)); + match &mut best { + // Merged rather than replaced. The registers answer about + // different parts of one book: OpenLibrary's edition record + // carries the publisher, the pagination and the language + // and names its authors by key, while its search index + // carries the authors and not the publisher. Keeping + // whichever arrived first throws the other half away. + Some(kept) => kept.fill_from(record), + None => best = Some(record), } - if better { - best = Some(record); + if best.as_ref().is_some_and(Record::complete) { + break; } } Err(why) => last_error = Some(why), diff --git a/crates/bibfetch/src/lib.rs b/crates/bibfetch/src/lib.rs index 6b686a8..cd47802 100644 --- a/crates/bibfetch/src/lib.rs +++ b/crates/bibfetch/src/lib.rs @@ -59,6 +59,21 @@ pub struct Record { /// The abstract, where the register carries one. #[serde(skip_serializing_if = "Option::is_none")] pub summary: Option, + /// The subject headings the register files it under, as it words them. + /// + /// A register states these for far more books than print a cataloguing + /// block, which makes them the second source a shelf otherwise lacks. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub subjects: Vec, + /// The language it is written in, as the register gives it: a two- or + /// three-letter code, or a name. Not normalised here — what a register + /// said is the record, and deciding that "eng" and "English" are one + /// thing belongs to whoever is filing them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub language: Option, + /// How many pages the register says it has. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pages: Option, } fn version() -> u32 { @@ -112,6 +127,70 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record { year: None, container: None, summary: None, + subjects: Vec::new(), + language: None, + pages: None, + } +} + +impl Record { + /// Whether this record says everything a catalogue wants. + /// + /// Used to decide when to stop asking. Title and authors are the minimum + /// that makes a record usable; the rest is worth another register only + /// while one is left to ask. + pub fn complete(&self) -> bool { + self.found + && self.title.is_some() + && !self.authors.is_empty() + && self.container.is_some() + && self.language.is_some() + } + + /// Take from another record whatever this one does not have. + /// + /// The registers are complementary rather than redundant, and asking them + /// in turn and keeping one answer throws away the difference. OpenLibrary's + /// edition record gives the publisher, the pagination and the language but + /// names its authors by key; its search index gives the authors but not the + /// publisher. One book, two half-answers, and the whole of it only if they + /// are put together. + /// + /// Nothing already answered is overwritten, so the order of the chain is + /// the order of trust. + pub fn fill_from(&mut self, other: Record) { + if !other.found { + return; + } + self.found = true; + if self.title.is_none() { + self.title = other.title; + } + if self.authors.is_empty() { + self.authors = other.authors; + } + if self.year.is_none() { + self.year = other.year; + } + if self.container.is_none() { + self.container = other.container; + } + if self.summary.is_none() { + self.summary = other.summary; + } + if self.subjects.is_empty() { + self.subjects = other.subjects; + } + if self.language.is_none() { + self.language = other.language; + } + if self.pages.is_none() { + self.pages = other.pages; + } + // Which register answered becomes which registers did. + if !self.source.split('+').any(|s| s == other.source) { + self.source = format!("{}+{}", self.source, other.source); + } } } @@ -135,3 +214,74 @@ mod tests { assert!(json.contains("\"found\":false"), "{json}"); } } + +#[cfg(test)] +mod merge_tests { + use super::*; + + fn rec(source: &str) -> Record { + let mut r = missing("9780199205592", "isbn", Source::OpenLibrary); + r.source = source.to_string(); + r.found = true; + r + } + + #[test] + fn two_half_answers_make_one_record() { + // Exactly what the two OpenLibrary endpoints give for one book: the + // edition record has the publisher and the language and names its + // authors by key; the index has the authors and not the publisher. + let mut edition = rec("openlibrary"); + edition.title = Some("The First World War: A Very Short Introduction".into()); + edition.container = Some("Oxford University Press".into()); + edition.language = Some("eng".into()); + + let mut index = rec("openlibrary"); + index.title = Some("The First World War".into()); + index.authors = vec!["Michael Howard".into()]; + index.year = Some(2007); + + edition.fill_from(index); + assert_eq!(edition.authors, ["Michael Howard"]); + assert_eq!(edition.year, Some(2007)); + // The fuller title arrived first and is not displaced by the shorter. + assert_eq!( + edition.title.as_deref(), + Some("The First World War: A Very Short Introduction") + ); + } + + #[test] + fn a_register_that_has_nothing_contributes_nothing() { + let mut kept = rec("openlibrary"); + kept.title = Some("A Book".into()); + let absent = missing("9780199205592", "isbn", Source::GoogleBooks); + kept.fill_from(absent); + assert_eq!(kept.title.as_deref(), Some("A Book")); + assert_eq!(kept.source, "openlibrary"); + } + + #[test] + fn the_record_names_every_register_that_answered() { + let mut a = rec("openlibrary"); + a.title = Some("A Book".into()); + let mut b = rec("googlebooks"); + b.found = true; + b.pages = Some(134); + a.fill_from(b); + assert_eq!(a.source, "openlibrary+googlebooks"); + assert_eq!(a.pages, Some(134)); + } + + #[test] + fn complete_means_a_catalogue_has_what_it_needs() { + let mut r = rec("openlibrary"); + assert!(!r.complete()); + r.title = Some("A Book".into()); + r.authors = vec!["Someone".into()]; + assert!(!r.complete(), "still missing the publisher and language"); + r.container = Some("A Press".into()); + r.language = Some("eng".into()); + assert!(r.complete()); + } +} diff --git a/crates/bibfetch/src/source.rs b/crates/bibfetch/src/source.rs index eb62ffc..ca1ab80 100644 --- a/crates/bibfetch/src/source.rs +++ b/crates/bibfetch/src/source.rs @@ -75,9 +75,12 @@ impl Source { Source::OpenLibrary => { format!("https://openlibrary.org/isbn/{}.json", encode(id)) } + // `fields` or the index answers with a handful of defaults and + // keeps the rest, which is how a language it holds went unread. Source::OpenLibrarySearch => format!( "https://openlibrary.org/search.json?q=isbn:{}\ -&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1", +&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\ +number_of_pages_median&limit=1", encode(id) ), Source::GoogleBooks => format!( @@ -222,9 +225,24 @@ struct OpenLibraryBook { #[serde(default)] title: Option, #[serde(default)] + subtitle: Option, + #[serde(default)] publish_date: Option, #[serde(default)] publishers: Vec, + #[serde(default)] + subjects: Vec, + /// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one. + #[serde(default)] + languages: Vec, + #[serde(default)] + number_of_pages: Option, +} + +#[derive(serde::Deserialize)] +struct OpenLibraryKey { + #[serde(default)] + key: String, } fn open_library(id: &str, kind: &str, body: &str) -> Result { @@ -232,10 +250,23 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result { serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; let mut record = blank(id, kind, Source::OpenLibrary); - record.title = book.title; + // An edition record keeps the subtitle in its own field, and a catalogue + // wants the whole title. Google Books is already joined the same way. + record.title = match (book.title, book.subtitle) { + (Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")), + (title, _) => title, + }; record.container = book.publishers.into_iter().next(); // The date is free text — "1999", "March 1999", "1999-03-01". record.year = book.publish_date.as_deref().and_then(year_in); + record.subjects = book.subjects; + // "/languages/eng" names the language by the last segment of a key. + record.language = book + .languages + .into_iter() + .find_map(|l| l.key.rsplit('/').next().map(str::to_string)) + .filter(|l| !l.is_empty()); + record.pages = book.number_of_pages; Ok(record) } @@ -257,8 +288,21 @@ struct SearchDoc { first_publish_year: Option, #[serde(default)] publisher: Vec, + #[serde(default)] + language: Vec, + #[serde(default)] + subject: Vec, + #[serde(rename = "number_of_pages_median", default)] + pages: Option, } +/// How many of a work's subject headings are worth carrying. +/// +/// The index lists every heading every edition has ever been given, which on a +/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed +/// from a handful, and the first are the broadest. +const SUBJECT_LIMIT: usize = 12; + /// A search that matched nothing answers with no documents rather than a /// status, so an empty reply is the absence and not a failure. fn open_library_search(id: &str, kind: &str, body: &str) -> Result { @@ -275,6 +319,12 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result, #[serde(default)] description: Option, + #[serde(default)] + categories: Vec, + #[serde(rename = "language", default)] + language: Option, + #[serde(rename = "pageCount", default)] + page_count: Option, } /// A search that matched nothing answers with no items rather than a status, @@ -324,6 +380,9 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result { record.year = volume.published_date.as_deref().and_then(year_in); record.container = volume.publisher; record.summary = volume.description.map(|text| collapse(&text)); + record.subjects = volume.categories; + record.language = volume.language; + record.pages = volume.page_count; Ok(record) }