diff --git a/Cargo.lock b/Cargo.lock index c22c3de..1e20184 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "avplan" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bibclean" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "bibfetch" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -196,7 +196,7 @@ dependencies = [ [[package]] name = "bookmeta" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -210,7 +210,7 @@ dependencies = [ [[package]] name = "booksniff" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "catena" -version = "0.2.3" +version = "0.3.0" dependencies = [ "bibclean", "bibfetch", @@ -277,7 +277,7 @@ dependencies = [ [[package]] name = "catena-cli" -version = "0.2.3" +version = "0.3.0" dependencies = [ "clap", "clap_mangen", @@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chunkmd" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -334,7 +334,7 @@ dependencies = [ [[package]] name = "citefind" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "chunkmd", @@ -399,7 +399,7 @@ dependencies = [ [[package]] name = "colophon" -version = "0.2.3" +version = "0.3.0" dependencies = [ "bibclean", "catena-cli", @@ -448,7 +448,7 @@ dependencies = [ [[package]] name = "corpus-core" -version = "0.2.3" +version = "0.3.0" dependencies = [ "schemars", "serde", @@ -492,7 +492,7 @@ dependencies = [ [[package]] name = "cuekit" -version = "0.2.3" +version = "0.3.0" dependencies = [ "corpus-core", "regex", @@ -504,7 +504,7 @@ dependencies = [ [[package]] name = "cuenorm" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -514,7 +514,7 @@ dependencies = [ [[package]] name = "cuepack" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -525,7 +525,7 @@ dependencies = [ [[package]] name = "cueseg" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -612,7 +612,7 @@ dependencies = [ [[package]] name = "emb" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -876,7 +876,7 @@ dependencies = [ [[package]] name = "idnorm" -version = "0.2.3" +version = "0.3.0" dependencies = [ "bibclean", "catena-cli", @@ -942,7 +942,7 @@ dependencies = [ [[package]] name = "langid" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "mdscrub" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1034,7 +1034,7 @@ dependencies = [ [[package]] name = "mdsplit" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "pdfgutter" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1126,7 +1126,7 @@ dependencies = [ [[package]] name = "pdftriage" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" [[package]] name = "rrf" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "seriesof" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", @@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "textkit" -version = "0.2.3" +version = "0.3.0" dependencies = [ "regex", "tiktoken-rs", @@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tocfind" -version = "0.2.3" +version = "0.3.0" dependencies = [ "catena-cli", "clap", diff --git a/Cargo.toml b/Cargo.toml index 94f11b6..e53e0a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,39 +3,39 @@ resolver = "3" members = ["crates/*"] [workspace.package] -version = "0.2.3" +version = "0.3.0" edition = "2024" rust-version = "1.85" license = "MPL-2.0" repository = "https://git.pumpin.online/funkpower/catena" [workspace.dependencies] -catena-cli = { path = "crates/catena-cli", version = "0.2.3" } -corpus-core = { path = "crates/corpus-core", version = "0.2.3" } -cuekit = { path = "crates/cuekit", version = "0.2.3" } -textkit = { path = "crates/textkit", version = "0.2.3" } -chunkmd = { path = "crates/chunkmd", version = "0.2.3" } -mdscrub = { path = "crates/mdscrub", version = "0.2.3" } -mdsplit = { path = "crates/mdsplit", version = "0.2.3" } -pdftriage = { path = "crates/pdftriage", version = "0.2.3" } -cuenorm = { path = "crates/cuenorm", version = "0.2.3" } -cueseg = { path = "crates/cueseg", version = "0.2.3" } -rrf = { path = "crates/rrf", version = "0.2.3" } -cuepack = { path = "crates/cuepack", version = "0.2.3" } -bibclean = { path = "crates/bibclean", version = "0.2.3" } -avplan = { path = "crates/avplan", version = "0.2.3" } -pdfgutter = { path = "crates/pdfgutter", version = "0.2.3" } -catena = { path = "crates/bindings", version = "0.2.3" } -seriesof = { path = "crates/seriesof", version = "0.2.3" } -idnorm = { path = "crates/idnorm", version = "0.2.3" } -bibfetch = { path = "crates/bibfetch", version = "0.2.3" } -bookmeta = { path = "crates/bookmeta", version = "0.2.3" } -langid = { path = "crates/langid", version = "0.2.3" } -tocfind = { path = "crates/tocfind", version = "0.2.3" } -booksniff = { path = "crates/booksniff", version = "0.2.3" } -citefind = { path = "crates/citefind", version = "0.2.3" } -colophon = { path = "crates/colophon", version = "0.2.3" } -emb = { path = "crates/emb", version = "0.2.3" } +catena-cli = { path = "crates/catena-cli", version = "0.3.0" } +corpus-core = { path = "crates/corpus-core", version = "0.3.0" } +cuekit = { path = "crates/cuekit", version = "0.3.0" } +textkit = { path = "crates/textkit", version = "0.3.0" } +chunkmd = { path = "crates/chunkmd", version = "0.3.0" } +mdscrub = { path = "crates/mdscrub", version = "0.3.0" } +mdsplit = { path = "crates/mdsplit", version = "0.3.0" } +pdftriage = { path = "crates/pdftriage", version = "0.3.0" } +cuenorm = { path = "crates/cuenorm", version = "0.3.0" } +cueseg = { path = "crates/cueseg", version = "0.3.0" } +rrf = { path = "crates/rrf", version = "0.3.0" } +cuepack = { path = "crates/cuepack", version = "0.3.0" } +bibclean = { path = "crates/bibclean", version = "0.3.0" } +avplan = { path = "crates/avplan", version = "0.3.0" } +pdfgutter = { path = "crates/pdfgutter", version = "0.3.0" } +catena = { path = "crates/bindings", version = "0.3.0" } +seriesof = { path = "crates/seriesof", version = "0.3.0" } +idnorm = { path = "crates/idnorm", version = "0.3.0" } +bibfetch = { path = "crates/bibfetch", version = "0.3.0" } +bookmeta = { path = "crates/bookmeta", version = "0.3.0" } +langid = { path = "crates/langid", version = "0.3.0" } +tocfind = { path = "crates/tocfind", version = "0.3.0" } +booksniff = { path = "crates/booksniff", version = "0.3.0" } +citefind = { path = "crates/citefind", version = "0.3.0" } +colophon = { path = "crates/colophon", version = "0.3.0" } +emb = { path = "crates/emb", version = "0.3.0" } clap = { version = "4", features = ["derive"] } clap_mangen = "0.3" diff --git a/crates/bibfetch/src/ask.rs b/crates/bibfetch/src/ask.rs index c08de32..bececa2 100644 --- a/crates/bibfetch/src/ask.rs +++ b/crates/bibfetch/src/ask.rs @@ -63,13 +63,18 @@ pub fn lookup( for source in chain { match ask(agent, *source, id, kind, attempts) { Ok(record) => { - let complete = record.found && record.title.is_some() && !record.authors.is_empty(); - let better = best.as_ref().is_none_or(|kept| record.found && !kept.found); - if complete { - return Ok(Some(record)); + match &mut best { + // Merged rather than replaced. The registers answer about + // different parts of one book: OpenLibrary's edition record + // carries the publisher, the pagination and the language + // and names its authors by key, while its search index + // carries the authors and not the publisher. Keeping + // whichever arrived first throws the other half away. + Some(kept) => kept.fill_from(record), + None => best = Some(record), } - if better { - best = Some(record); + if best.as_ref().is_some_and(Record::complete) { + break; } } Err(why) => last_error = Some(why), diff --git a/crates/bibfetch/src/lib.rs b/crates/bibfetch/src/lib.rs index 6b686a8..cd47802 100644 --- a/crates/bibfetch/src/lib.rs +++ b/crates/bibfetch/src/lib.rs @@ -59,6 +59,21 @@ pub struct Record { /// The abstract, where the register carries one. #[serde(skip_serializing_if = "Option::is_none")] pub summary: Option, + /// The subject headings the register files it under, as it words them. + /// + /// A register states these for far more books than print a cataloguing + /// block, which makes them the second source a shelf otherwise lacks. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub subjects: Vec, + /// The language it is written in, as the register gives it: a two- or + /// three-letter code, or a name. Not normalised here — what a register + /// said is the record, and deciding that "eng" and "English" are one + /// thing belongs to whoever is filing them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub language: Option, + /// How many pages the register says it has. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pages: Option, } fn version() -> u32 { @@ -112,6 +127,70 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record { year: None, container: None, summary: None, + subjects: Vec::new(), + language: None, + pages: None, + } +} + +impl Record { + /// Whether this record says everything a catalogue wants. + /// + /// Used to decide when to stop asking. Title and authors are the minimum + /// that makes a record usable; the rest is worth another register only + /// while one is left to ask. + pub fn complete(&self) -> bool { + self.found + && self.title.is_some() + && !self.authors.is_empty() + && self.container.is_some() + && self.language.is_some() + } + + /// Take from another record whatever this one does not have. + /// + /// The registers are complementary rather than redundant, and asking them + /// in turn and keeping one answer throws away the difference. OpenLibrary's + /// edition record gives the publisher, the pagination and the language but + /// names its authors by key; its search index gives the authors but not the + /// publisher. One book, two half-answers, and the whole of it only if they + /// are put together. + /// + /// Nothing already answered is overwritten, so the order of the chain is + /// the order of trust. + pub fn fill_from(&mut self, other: Record) { + if !other.found { + return; + } + self.found = true; + if self.title.is_none() { + self.title = other.title; + } + if self.authors.is_empty() { + self.authors = other.authors; + } + if self.year.is_none() { + self.year = other.year; + } + if self.container.is_none() { + self.container = other.container; + } + if self.summary.is_none() { + self.summary = other.summary; + } + if self.subjects.is_empty() { + self.subjects = other.subjects; + } + if self.language.is_none() { + self.language = other.language; + } + if self.pages.is_none() { + self.pages = other.pages; + } + // Which register answered becomes which registers did. + if !self.source.split('+').any(|s| s == other.source) { + self.source = format!("{}+{}", self.source, other.source); + } } } @@ -135,3 +214,74 @@ mod tests { assert!(json.contains("\"found\":false"), "{json}"); } } + +#[cfg(test)] +mod merge_tests { + use super::*; + + fn rec(source: &str) -> Record { + let mut r = missing("9780199205592", "isbn", Source::OpenLibrary); + r.source = source.to_string(); + r.found = true; + r + } + + #[test] + fn two_half_answers_make_one_record() { + // Exactly what the two OpenLibrary endpoints give for one book: the + // edition record has the publisher and the language and names its + // authors by key; the index has the authors and not the publisher. + let mut edition = rec("openlibrary"); + edition.title = Some("The First World War: A Very Short Introduction".into()); + edition.container = Some("Oxford University Press".into()); + edition.language = Some("eng".into()); + + let mut index = rec("openlibrary"); + index.title = Some("The First World War".into()); + index.authors = vec!["Michael Howard".into()]; + index.year = Some(2007); + + edition.fill_from(index); + assert_eq!(edition.authors, ["Michael Howard"]); + assert_eq!(edition.year, Some(2007)); + // The fuller title arrived first and is not displaced by the shorter. + assert_eq!( + edition.title.as_deref(), + Some("The First World War: A Very Short Introduction") + ); + } + + #[test] + fn a_register_that_has_nothing_contributes_nothing() { + let mut kept = rec("openlibrary"); + kept.title = Some("A Book".into()); + let absent = missing("9780199205592", "isbn", Source::GoogleBooks); + kept.fill_from(absent); + assert_eq!(kept.title.as_deref(), Some("A Book")); + assert_eq!(kept.source, "openlibrary"); + } + + #[test] + fn the_record_names_every_register_that_answered() { + let mut a = rec("openlibrary"); + a.title = Some("A Book".into()); + let mut b = rec("googlebooks"); + b.found = true; + b.pages = Some(134); + a.fill_from(b); + assert_eq!(a.source, "openlibrary+googlebooks"); + assert_eq!(a.pages, Some(134)); + } + + #[test] + fn complete_means_a_catalogue_has_what_it_needs() { + let mut r = rec("openlibrary"); + assert!(!r.complete()); + r.title = Some("A Book".into()); + r.authors = vec!["Someone".into()]; + assert!(!r.complete(), "still missing the publisher and language"); + r.container = Some("A Press".into()); + r.language = Some("eng".into()); + assert!(r.complete()); + } +} diff --git a/crates/bibfetch/src/source.rs b/crates/bibfetch/src/source.rs index eb62ffc..ca1ab80 100644 --- a/crates/bibfetch/src/source.rs +++ b/crates/bibfetch/src/source.rs @@ -75,9 +75,12 @@ impl Source { Source::OpenLibrary => { format!("https://openlibrary.org/isbn/{}.json", encode(id)) } + // `fields` or the index answers with a handful of defaults and + // keeps the rest, which is how a language it holds went unread. Source::OpenLibrarySearch => format!( "https://openlibrary.org/search.json?q=isbn:{}\ -&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1", +&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\ +number_of_pages_median&limit=1", encode(id) ), Source::GoogleBooks => format!( @@ -222,9 +225,24 @@ struct OpenLibraryBook { #[serde(default)] title: Option, #[serde(default)] + subtitle: Option, + #[serde(default)] publish_date: Option, #[serde(default)] publishers: Vec, + #[serde(default)] + subjects: Vec, + /// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one. + #[serde(default)] + languages: Vec, + #[serde(default)] + number_of_pages: Option, +} + +#[derive(serde::Deserialize)] +struct OpenLibraryKey { + #[serde(default)] + key: String, } fn open_library(id: &str, kind: &str, body: &str) -> Result { @@ -232,10 +250,23 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result { serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; let mut record = blank(id, kind, Source::OpenLibrary); - record.title = book.title; + // An edition record keeps the subtitle in its own field, and a catalogue + // wants the whole title. Google Books is already joined the same way. + record.title = match (book.title, book.subtitle) { + (Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")), + (title, _) => title, + }; record.container = book.publishers.into_iter().next(); // The date is free text — "1999", "March 1999", "1999-03-01". record.year = book.publish_date.as_deref().and_then(year_in); + record.subjects = book.subjects; + // "/languages/eng" names the language by the last segment of a key. + record.language = book + .languages + .into_iter() + .find_map(|l| l.key.rsplit('/').next().map(str::to_string)) + .filter(|l| !l.is_empty()); + record.pages = book.number_of_pages; Ok(record) } @@ -257,8 +288,21 @@ struct SearchDoc { first_publish_year: Option, #[serde(default)] publisher: Vec, + #[serde(default)] + language: Vec, + #[serde(default)] + subject: Vec, + #[serde(rename = "number_of_pages_median", default)] + pages: Option, } +/// How many of a work's subject headings are worth carrying. +/// +/// The index lists every heading every edition has ever been given, which on a +/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed +/// from a handful, and the first are the broadest. +const SUBJECT_LIMIT: usize = 12; + /// A search that matched nothing answers with no documents rather than a /// status, so an empty reply is the absence and not a failure. fn open_library_search(id: &str, kind: &str, body: &str) -> Result { @@ -275,6 +319,12 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result, #[serde(default)] description: Option, + #[serde(default)] + categories: Vec, + #[serde(rename = "language", default)] + language: Option, + #[serde(rename = "pageCount", default)] + page_count: Option, } /// A search that matched nothing answers with no items rather than a status, @@ -324,6 +380,9 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result { record.year = volume.published_date.as_deref().and_then(year_in); record.container = volume.publisher; record.summary = volume.description.map(|text| collapse(&text)); + record.subjects = volume.categories; + record.language = volume.language; + record.pages = volume.page_count; Ok(record) }