0.3.1, properly — the version, and the test that names the first register

The commit that reordered the chain did not carry the version bump. The
bump was chained after check.sh with &&, the commit after a semicolon, so
a failing check skipped the bump and let the commit through. The tag then
pointed at a tree still calling itself 0.3.0, which the release workflow
asserts against and would have refused.

check.sh was failing because a test names which register is asked first
about an ISBN, which is exactly what changed. It now names the index, for
the reason the reorder happened.

Cargo.lock

@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "avplan"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
 
 [[package]]
 name = "bibclean"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -143,7 +143,7 @@ dependencies = [
 
 [[package]]
 name = "bibfetch"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -196,7 +196,7 @@ dependencies = [
 
 [[package]]
 name = "bookmeta"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -210,7 +210,7 @@ dependencies = [
 
 [[package]]
 name = "booksniff"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "catena"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "bibclean",
  "bibfetch",
@@ -277,7 +277,7 @@ dependencies = [
 
 [[package]]
 name = "catena-cli"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "clap",
  "clap_mangen",
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
 
 [[package]]
 name = "chunkmd"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -334,7 +334,7 @@ dependencies = [
 
 [[package]]
 name = "citefind"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "chunkmd",
@@ -399,7 +399,7 @@ dependencies = [
 
 [[package]]
 name = "colophon"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "bibclean",
  "catena-cli",
@@ -448,7 +448,7 @@ dependencies = [
 
 [[package]]
 name = "corpus-core"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "schemars",
  "serde",
@@ -492,7 +492,7 @@ dependencies = [
 
 [[package]]
 name = "cuekit"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "corpus-core",
  "regex",
@@ -504,7 +504,7 @@ dependencies = [
 
 [[package]]
 name = "cuenorm"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -514,7 +514,7 @@ dependencies = [
 
 [[package]]
 name = "cuepack"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -525,7 +525,7 @@ dependencies = [
 
 [[package]]
 name = "cueseg"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -612,7 +612,7 @@ dependencies = [
 
 [[package]]
 name = "emb"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -876,7 +876,7 @@ dependencies = [
 
 [[package]]
 name = "idnorm"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "bibclean",
  "catena-cli",
@@ -942,7 +942,7 @@ dependencies = [
 
 [[package]]
 name = "langid"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "mdscrub"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1034,7 +1034,7 @@ dependencies = [
 
 [[package]]
 name = "mdsplit"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
 [[package]]
 name = "pdfgutter"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1126,7 +1126,7 @@ dependencies = [
 
 [[package]]
 name = "pdftriage"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
 
 [[package]]
 name = "rrf"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1517,7 +1517,7 @@ dependencies = [
 
 [[package]]
 name = "seriesof"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "textkit"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "regex",
  "tiktoken-rs",
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
 
 [[package]]
 name = "tocfind"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "catena-cli",
  "clap",

Cargo.toml

@@ -3,39 +3,39 @@ resolver = "3"
 members = ["crates/*"]
 
 [workspace.package]
-version = "0.2.3"
+version = "0.3.1"
 edition = "2024"
 rust-version = "1.85"
 license = "MPL-2.0"
 repository = "https://git.pumpin.online/funkpower/catena"
 
 [workspace.dependencies]
-catena-cli = { path = "crates/catena-cli", version = "0.2.3" }
+catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
-corpus-core = { path = "crates/corpus-core", version = "0.2.3" }
+corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
-cuekit = { path = "crates/cuekit", version = "0.2.3" }
+cuekit = { path = "crates/cuekit", version = "0.3.1" }
-textkit = { path = "crates/textkit", version = "0.2.3" }
+textkit = { path = "crates/textkit", version = "0.3.1" }
-chunkmd = { path = "crates/chunkmd", version = "0.2.3" }
+chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
-mdscrub = { path = "crates/mdscrub", version = "0.2.3" }
+mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
-mdsplit = { path = "crates/mdsplit", version = "0.2.3" }
+mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
-pdftriage = { path = "crates/pdftriage", version = "0.2.3" }
+pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
-cuenorm = { path = "crates/cuenorm", version = "0.2.3" }
+cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
-cueseg = { path = "crates/cueseg", version = "0.2.3" }
+cueseg = { path = "crates/cueseg", version = "0.3.1" }
-rrf = { path = "crates/rrf", version = "0.2.3" }
+rrf = { path = "crates/rrf", version = "0.3.1" }
-cuepack = { path = "crates/cuepack", version = "0.2.3" }
+cuepack = { path = "crates/cuepack", version = "0.3.1" }
-bibclean = { path = "crates/bibclean", version = "0.2.3" }
+bibclean = { path = "crates/bibclean", version = "0.3.1" }
-avplan = { path = "crates/avplan", version = "0.2.3" }
+avplan = { path = "crates/avplan", version = "0.3.1" }
-pdfgutter = { path = "crates/pdfgutter", version = "0.2.3" }
+pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
-catena = { path = "crates/bindings", version = "0.2.3" }
+catena = { path = "crates/bindings", version = "0.3.1" }
-seriesof = { path = "crates/seriesof", version = "0.2.3" }
+seriesof = { path = "crates/seriesof", version = "0.3.1" }
-idnorm = { path = "crates/idnorm", version = "0.2.3" }
+idnorm = { path = "crates/idnorm", version = "0.3.1" }
-bibfetch = { path = "crates/bibfetch", version = "0.2.3" }
+bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
-bookmeta = { path = "crates/bookmeta", version = "0.2.3" }
+bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
-langid = { path = "crates/langid", version = "0.2.3" }
+langid = { path = "crates/langid", version = "0.3.1" }
-tocfind = { path = "crates/tocfind", version = "0.2.3" }
+tocfind = { path = "crates/tocfind", version = "0.3.1" }
-booksniff = { path = "crates/booksniff", version = "0.2.3" }
+booksniff = { path = "crates/booksniff", version = "0.3.1" }
-citefind = { path = "crates/citefind", version = "0.2.3" }
+citefind = { path = "crates/citefind", version = "0.3.1" }
-colophon = { path = "crates/colophon", version = "0.2.3" }
+colophon = { path = "crates/colophon", version = "0.3.1" }
-emb = { path = "crates/emb", version = "0.2.3" }
+emb = { path = "crates/emb", version = "0.3.1" }
 
 clap = { version = "4", features = ["derive"] }
 clap_mangen = "0.3"

crates/bibfetch/src/ask.rs

@@ -63,13 +63,18 @@ pub fn lookup(
     for source in chain {
         match ask(agent, *source, id, kind, attempts) {
             Ok(record) => {
-                let complete = record.found && record.title.is_some() && !record.authors.is_empty();
+                match &mut best {
-                let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
+                    // Merged rather than replaced. The registers answer about
-                if complete {
+                    // different parts of one book: OpenLibrary's edition record
-                    return Ok(Some(record));
+                    // carries the publisher, the pagination and the language
+                    // and names its authors by key, while its search index
+                    // carries the authors and not the publisher. Keeping
+                    // whichever arrived first throws the other half away.
+                    Some(kept) => kept.fill_from(record),
+                    None => best = Some(record),
                 }
-                if better {
+                if best.as_ref().is_some_and(Record::complete) {
-                    best = Some(record);
+                    break;
                 }
             }
             Err(why) => last_error = Some(why),

crates/bibfetch/src/lib.rs

@@ -59,6 +59,21 @@ pub struct Record {
     /// The abstract, where the register carries one.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub summary: Option<String>,
+    /// The subject headings the register files it under, as it words them.
+    ///
+    /// A register states these for far more books than print a cataloguing
+    /// block, which makes them the second source a shelf otherwise lacks.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub subjects: Vec<String>,
+    /// The language it is written in, as the register gives it: a two- or
+    /// three-letter code, or a name. Not normalised here — what a register
+    /// said is the record, and deciding that "eng" and "English" are one
+    /// thing belongs to whoever is filing them.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub language: Option<String>,
+    /// How many pages the register says it has.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pages: Option<u32>,
 }
 
 fn version() -> u32 {
@@ -112,6 +127,70 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record {
         year: None,
         container: None,
         summary: None,
+        subjects: Vec::new(),
+        language: None,
+        pages: None,
+    }
+}
+
+impl Record {
+    /// Whether this record says everything a catalogue wants.
+    ///
+    /// Used to decide when to stop asking. Title and authors are the minimum
+    /// that makes a record usable; the rest is worth another register only
+    /// while one is left to ask.
+    pub fn complete(&self) -> bool {
+        self.found
+            && self.title.is_some()
+            && !self.authors.is_empty()
+            && self.container.is_some()
+            && self.language.is_some()
+    }
+
+    /// Take from another record whatever this one does not have.
+    ///
+    /// The registers are complementary rather than redundant, and asking them
+    /// in turn and keeping one answer throws away the difference. OpenLibrary's
+    /// edition record gives the publisher, the pagination and the language but
+    /// names its authors by key; its search index gives the authors but not the
+    /// publisher. One book, two half-answers, and the whole of it only if they
+    /// are put together.
+    ///
+    /// Nothing already answered is overwritten, so the order of the chain is
+    /// the order of trust.
+    pub fn fill_from(&mut self, other: Record) {
+        if !other.found {
+            return;
+        }
+        self.found = true;
+        if self.title.is_none() {
+            self.title = other.title;
+        }
+        if self.authors.is_empty() {
+            self.authors = other.authors;
+        }
+        if self.year.is_none() {
+            self.year = other.year;
+        }
+        if self.container.is_none() {
+            self.container = other.container;
+        }
+        if self.summary.is_none() {
+            self.summary = other.summary;
+        }
+        if self.subjects.is_empty() {
+            self.subjects = other.subjects;
+        }
+        if self.language.is_none() {
+            self.language = other.language;
+        }
+        if self.pages.is_none() {
+            self.pages = other.pages;
+        }
+        // Which register answered becomes which registers did.
+        if !self.source.split('+').any(|s| s == other.source) {
+            self.source = format!("{}+{}", self.source, other.source);
+        }
     }
 }
 
@@ -135,3 +214,74 @@ mod tests {
         assert!(json.contains("\"found\":false"), "{json}");
     }
 }
+
+#[cfg(test)]
+mod merge_tests {
+    use super::*;
+
+    fn rec(source: &str) -> Record {
+        let mut r = missing("9780199205592", "isbn", Source::OpenLibrary);
+        r.source = source.to_string();
+        r.found = true;
+        r
+    }
+
+    #[test]
+    fn two_half_answers_make_one_record() {
+        // Exactly what the two OpenLibrary endpoints give for one book: the
+        // edition record has the publisher and the language and names its
+        // authors by key; the index has the authors and not the publisher.
+        let mut edition = rec("openlibrary");
+        edition.title = Some("The First World War: A Very Short Introduction".into());
+        edition.container = Some("Oxford University Press".into());
+        edition.language = Some("eng".into());
+
+        let mut index = rec("openlibrary");
+        index.title = Some("The First World War".into());
+        index.authors = vec!["Michael Howard".into()];
+        index.year = Some(2007);
+
+        edition.fill_from(index);
+        assert_eq!(edition.authors, ["Michael Howard"]);
+        assert_eq!(edition.year, Some(2007));
+        // The fuller title arrived first and is not displaced by the shorter.
+        assert_eq!(
+            edition.title.as_deref(),
+            Some("The First World War: A Very Short Introduction")
+        );
+    }
+
+    #[test]
+    fn a_register_that_has_nothing_contributes_nothing() {
+        let mut kept = rec("openlibrary");
+        kept.title = Some("A Book".into());
+        let absent = missing("9780199205592", "isbn", Source::GoogleBooks);
+        kept.fill_from(absent);
+        assert_eq!(kept.title.as_deref(), Some("A Book"));
+        assert_eq!(kept.source, "openlibrary");
+    }
+
+    #[test]
+    fn the_record_names_every_register_that_answered() {
+        let mut a = rec("openlibrary");
+        a.title = Some("A Book".into());
+        let mut b = rec("googlebooks");
+        b.found = true;
+        b.pages = Some(134);
+        a.fill_from(b);
+        assert_eq!(a.source, "openlibrary+googlebooks");
+        assert_eq!(a.pages, Some(134));
+    }
+
+    #[test]
+    fn complete_means_a_catalogue_has_what_it_needs() {
+        let mut r = rec("openlibrary");
+        assert!(!r.complete());
+        r.title = Some("A Book".into());
+        r.authors = vec!["Someone".into()];
+        assert!(!r.complete(), "still missing the publisher and language");
+        r.container = Some("A Press".into());
+        r.language = Some("eng".into());
+        assert!(r.complete());
+    }
+}

crates/bibfetch/src/source.rs

@@ -55,9 +55,17 @@ impl Source {
         match kind {
             "doi" => &[Source::Crossref],
             "arxiv" => &[Source::Arxiv],
+            // The index first. Asked for the fields by name it answers with
+            // the title, the authors, the publisher, the language, a page
+            // count and the subjects in one request, which is the whole of
+            // what a catalogue wants; the edition record is then needed only
+            // for a subtitle or a publisher the index did not give. Asking it
+            // second cost three requests per book where one usually does, and
+            // at three books in flight that is the difference between two
+            // days and two hours.
             "isbn" => &[
-                Source::OpenLibrary,
                 Source::OpenLibrarySearch,
+                Source::OpenLibrary,
                 Source::GoogleBooks,
             ],
             _ => &[],
@@ -75,9 +83,12 @@ impl Source {
             Source::OpenLibrary => {
                 format!("https://openlibrary.org/isbn/{}.json", encode(id))
             }
+            // `fields` or the index answers with a handful of defaults and
+            // keeps the rest, which is how a language it holds went unread.
             Source::OpenLibrarySearch => format!(
                 "https://openlibrary.org/search.json?q=isbn:{}\
-&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1",
+&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\
+number_of_pages_median&limit=1",
                 encode(id)
             ),
             Source::GoogleBooks => format!(
@@ -222,9 +233,24 @@ struct OpenLibraryBook {
     #[serde(default)]
     title: Option<String>,
     #[serde(default)]
+    subtitle: Option<String>,
+    #[serde(default)]
     publish_date: Option<String>,
     #[serde(default)]
     publishers: Vec<String>,
+    #[serde(default)]
+    subjects: Vec<String>,
+    /// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one.
+    #[serde(default)]
+    languages: Vec<OpenLibraryKey>,
+    #[serde(default)]
+    number_of_pages: Option<u32>,
+}
+
+#[derive(serde::Deserialize)]
+struct OpenLibraryKey {
+    #[serde(default)]
+    key: String,
 }
 
 fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
@@ -232,10 +258,23 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
         serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
 
     let mut record = blank(id, kind, Source::OpenLibrary);
-    record.title = book.title;
+    // An edition record keeps the subtitle in its own field, and a catalogue
+    // wants the whole title. Google Books is already joined the same way.
+    record.title = match (book.title, book.subtitle) {
+        (Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
+        (title, _) => title,
+    };
     record.container = book.publishers.into_iter().next();
     // The date is free text — "1999", "March 1999", "1999-03-01".
     record.year = book.publish_date.as_deref().and_then(year_in);
+    record.subjects = book.subjects;
+    // "/languages/eng" names the language by the last segment of a key.
+    record.language = book
+        .languages
+        .into_iter()
+        .find_map(|l| l.key.rsplit('/').next().map(str::to_string))
+        .filter(|l| !l.is_empty());
+    record.pages = book.number_of_pages;
     Ok(record)
 }
 
@@ -257,8 +296,21 @@ struct SearchDoc {
     first_publish_year: Option<i64>,
     #[serde(default)]
     publisher: Vec<String>,
+    #[serde(default)]
+    language: Vec<String>,
+    #[serde(default)]
+    subject: Vec<String>,
+    #[serde(rename = "number_of_pages_median", default)]
+    pages: Option<u32>,
 }
 
+/// How many of a work's subject headings are worth carrying.
+///
+/// The index lists every heading every edition has ever been given, which on a
+/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
+/// from a handful, and the first are the broadest.
+const SUBJECT_LIMIT: usize = 12;
+
 /// A search that matched nothing answers with no documents rather than a
 /// status, so an empty reply is the absence and not a failure.
 fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
@@ -275,6 +327,12 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
     record.authors = doc.author_name;
     record.year = doc.first_publish_year;
     record.container = doc.publisher.into_iter().next();
+    record.language = doc.language.into_iter().next();
+    // The index lists every subject any edition of the work carries, which
+    // runs to hundreds on a well-known book. A shelf needs a handful.
+    record.subjects = doc.subject;
+    record.subjects.truncate(SUBJECT_LIMIT);
+    record.pages = doc.pages;
     Ok(record)
 }
 
@@ -304,6 +362,12 @@ struct GoogleVolume {
     publisher: Option<String>,
     #[serde(default)]
     description: Option<String>,
+    #[serde(default)]
+    categories: Vec<String>,
+    #[serde(rename = "language", default)]
+    language: Option<String>,
+    #[serde(rename = "pageCount", default)]
+    page_count: Option<u32>,
 }
 
 /// A search that matched nothing answers with no items rather than a status,
@@ -324,6 +388,9 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
     record.year = volume.published_date.as_deref().and_then(year_in);
     record.container = volume.publisher;
     record.summary = volume.description.map(|text| collapse(&text));
+    record.subjects = volume.categories;
+    record.language = volume.language;
+    record.pages = volume.page_count;
     Ok(record)
 }
 
@@ -379,7 +446,10 @@ mod tests {
     fn each_kind_goes_to_the_body_that_assigns_it() {
         assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
         assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
-        assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
+        // The index, not the edition record: it answers with the whole of
+        // what a catalogue wants in one request, where the edition record
+        // takes up to twenty-eight seconds and names its authors by key.
+        assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrarySearch));
         // No register answers about an ISSN without a key.
         assert_eq!(Source::for_kind("issn"), None);
     }