diff --git a/crates/bibfetch/README.md b/crates/bibfetch/README.md index a4065f3..2dcad41 100644 --- a/crates/bibfetch/README.md +++ b/crates/bibfetch/README.md @@ -84,7 +84,20 @@ is a failure — nobody replied, and the identifier may be perfectly good. |---|---|---| | `doi` | Crossref | It assigns most scholarly DOIs | | `arxiv` | the arXiv | Its own register of preprints | -| `isbn` | OpenLibrary | Books, without a key | +| `isbn` | OpenLibrary's edition record, then its index, then Google Books | Books, without a key | + +One register is enough for a DOI or a preprint: the body that assigns the +identifier is the only one that has it. A book is different. An ISBN is +assigned by a publisher and recorded by whoever gets round to it, so an +answer that names the book but not who wrote it is common — OpenLibrary's +edition record names its authors by key rather than by name, and its index +names them. So the registers are asked in turn and the first complete answer +wins. An incomplete one is kept in case nothing better comes, and a register +being down is a failure only when every register is. + +Google Books is asked last because it answers anonymous callers only until a +daily quota runs out, after which it replies `429` to everyone sharing the +address. An **ISSN** has no register that answers without a key, so it is not looked up on its own. A journal's ISSN comes back on the works published in it instead. diff --git a/crates/bibfetch/src/main.rs b/crates/bibfetch/src/main.rs index 7cc610c..535e0c2 100644 --- a/crates/bibfetch/src/main.rs +++ b/crates/bibfetch/src/main.rs @@ -86,20 +86,47 @@ fn main() -> ExitCode { } }; - let Some(source) = Source::for_kind(&kind) else { + let chain = Source::chain_for_kind(&kind); + if chain.is_empty() { eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers"); missing += 1; continue; - }; + } - match ask(&agent, source, &id, &kind, args.attempts) { - Ok(record) => { + // Ask the registers in turn and stop at the first complete answer. + // An incomplete one is kept in case nothing better comes: a title + // with no authors is still more than nothing, and a register being + // down is only a failure when every register is. + let mut best: Option = None; + let mut last_error = None; + for source in chain { + match ask(&agent, *source, &id, &kind, args.attempts) { + Ok(record) => { + let complete = + record.found && record.title.is_some() && !record.authors.is_empty(); + let better = best.as_ref().is_none_or(|kept| record.found && !kept.found); + if complete { + best = Some(record); + break; + } + if better { + best = Some(record); + } + } + Err(why) => last_error = Some(why), + } + } + + match best { + Some(record) => { if !record.found { missing += 1; } records.push(record); } - Err(why) => { + None => { + let why = last_error + .map_or_else(|| "no register answered".to_string(), |e| e.to_string()); eprintln!("bibfetch: {id}: {why}"); return ExitCode::from(exit::UNAVAILABLE); } diff --git a/crates/bibfetch/src/source.rs b/crates/bibfetch/src/source.rs index 7abd058..eb62ffc 100644 --- a/crates/bibfetch/src/source.rs +++ b/crates/bibfetch/src/source.rs @@ -17,6 +17,13 @@ pub enum Source { Arxiv, /// OpenLibrary, for books by ISBN. OpenLibrary, + /// OpenLibrary's index, rather than its edition record. Asked when the + /// edition record has the book but not who wrote it, which is often: an + /// edition names its authors by key, and the index names them. + OpenLibrarySearch, + /// Google Books, for books by ISBN. Asked last, because it answers + /// anonymous callers only until a daily quota runs out. + GoogleBooks, } impl Source { @@ -26,16 +33,34 @@ impl Source { Source::Crossref => "crossref", Source::Arxiv => "arxiv", Source::OpenLibrary => "openlibrary", + Source::OpenLibrarySearch => "openlibrary", + Source::GoogleBooks => "googlebooks", } } /// Which register answers about a kind of identifier. pub fn for_kind(kind: &str) -> Option { + Self::chain_for_kind(kind).first().copied() + } + + /// Every register that answers about a kind, in the order to ask them. + /// + /// One register is enough for a DOI or a preprint: the body that assigns + /// the identifier is the only one that has it. A book is different. An + /// ISBN is assigned by a publisher and recorded by whoever gets round to + /// it, so coverage is patchy in different places for different books, + /// and an answer missing the authors is common enough to be worth asking + /// a second register about. + pub fn chain_for_kind(kind: &str) -> &'static [Self] { match kind { - "doi" => Some(Source::Crossref), - "arxiv" => Some(Source::Arxiv), - "isbn" => Some(Source::OpenLibrary), - _ => None, + "doi" => &[Source::Crossref], + "arxiv" => &[Source::Arxiv], + "isbn" => &[ + Source::OpenLibrary, + Source::OpenLibrarySearch, + Source::GoogleBooks, + ], + _ => &[], } } @@ -50,6 +75,15 @@ impl Source { Source::OpenLibrary => { format!("https://openlibrary.org/isbn/{}.json", encode(id)) } + Source::OpenLibrarySearch => format!( + "https://openlibrary.org/search.json?q=isbn:{}\ +&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1", + encode(id) + ), + Source::GoogleBooks => format!( + "https://www.googleapis.com/books/v1/volumes?q=isbn:{}", + encode(id) + ), } } @@ -59,6 +93,8 @@ impl Source { Source::Crossref => crossref(id, kind, body), Source::Arxiv => arxiv(id, kind, body), Source::OpenLibrary => open_library(id, kind, body), + Source::OpenLibrarySearch => open_library_search(id, kind, body), + Source::GoogleBooks => google_books(id, kind, body), } } } @@ -203,6 +239,94 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result { Ok(record) } +#[derive(Deserialize)] +struct SearchReply { + #[serde(default)] + docs: Vec, +} + +#[derive(Deserialize)] +struct SearchDoc { + #[serde(default)] + title: Option, + #[serde(default)] + subtitle: Option, + #[serde(default)] + author_name: Vec, + #[serde(default)] + first_publish_year: Option, + #[serde(default)] + publisher: Vec, +} + +/// A search that matched nothing answers with no documents rather than a +/// status, so an empty reply is the absence and not a failure. +fn open_library_search(id: &str, kind: &str, body: &str) -> Result { + let reply: SearchReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; + let Some(doc) = reply.docs.into_iter().next() else { + return Ok(crate::missing(id, kind, Source::OpenLibrarySearch)); + }; + + let mut record = blank(id, kind, Source::OpenLibrarySearch); + record.title = match (doc.title, doc.subtitle) { + (Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")), + (title, _) => title, + }; + record.authors = doc.author_name; + record.year = doc.first_publish_year; + record.container = doc.publisher.into_iter().next(); + Ok(record) +} + +#[derive(Deserialize)] +struct GoogleReply { + #[serde(default)] + items: Vec, +} + +#[derive(Deserialize)] +struct GoogleItem { + #[serde(rename = "volumeInfo", default)] + volume_info: GoogleVolume, +} + +#[derive(Deserialize, Default)] +struct GoogleVolume { + #[serde(default)] + title: Option, + #[serde(default)] + subtitle: Option, + #[serde(default)] + authors: Vec, + #[serde(rename = "publishedDate", default)] + published_date: Option, + #[serde(default)] + publisher: Option, + #[serde(default)] + description: Option, +} + +/// A search that matched nothing answers with no items rather than a status, +/// so an empty reply is the absence and not a failure. +fn google_books(id: &str, kind: &str, body: &str) -> Result { + let reply: GoogleReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; + let Some(item) = reply.items.into_iter().next() else { + return Ok(crate::missing(id, kind, Source::GoogleBooks)); + }; + let volume = item.volume_info; + + let mut record = blank(id, kind, Source::GoogleBooks); + record.title = match (volume.title, volume.subtitle) { + (Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")), + (title, _) => title, + }; + record.authors = volume.authors; + record.year = volume.published_date.as_deref().and_then(year_in); + record.container = volume.publisher; + record.summary = volume.description.map(|text| collapse(&text)); + Ok(record) +} + /// The first four-digit year in a string. fn year_in(text: &str) -> Option { let bytes: Vec = text.chars().collect(); diff --git a/crates/colophon/src/imprint.rs b/crates/colophon/src/imprint.rs index e6817cc..e402978 100644 --- a/crates/colophon/src/imprint.rs +++ b/crates/colophon/src/imprint.rs @@ -26,7 +26,15 @@ static PUBLISHED_BY: LazyLock = LazyLock::new(|| { .expect("a valid pattern") }); static PRESS_LINE: LazyLock = LazyLock::new(|| { - Regex::new(r"(?im)^\s*(?P[A-Z][^\n]{2,58}?(?:University Press|Press|Publishers|Publishing|Books|Verlag))\s*[,.]?\s*$") + Regex::new(r"(?im)^\s*(?P[A-Z][^\n]{2,48}?(?:University Press|Press|Publishers|Publishing|Books|Verlag))\s*[,.]?\s*$") + .expect("a valid pattern") +}); +/// A sentence that merely mentions a press is not the press. +/// +/// `Oxford is a registered trade mark of Oxford University Press` ends in a +/// publisher's name and is a legal notice. +static NOT_AN_IMPRINT: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)( is | are | was |registered|trade mark|trademark|imprint of|division of|©|\(c\)|@|;|\d{4})") .expect("a valid pattern") }); static FIRST_PUBLISHED: LazyLock = LazyLock::new(|| { @@ -84,9 +92,10 @@ pub fn read(text: &str) -> Imprint { } out.publisher = PUBLISHED_BY - .captures(text) - .or_else(|| PRESS_LINE.captures(text)) - .map(|caps| tidy(&caps["who"])); + .captures_iter(text) + .chain(PRESS_LINE.captures_iter(text)) + .map(|caps| tidy(&caps["who"])) + .find(|who| !NOT_AN_IMPRINT.is_match(who) && who.split_whitespace().count() <= 8); out.year = FIRST_PUBLISHED .captures(text) .or_else(|| COPYRIGHT_YEAR.captures(text)) @@ -97,8 +106,8 @@ pub fn read(text: &str) -> Imprint { .find(|line| crate::title::PLACE.is_match(line) && line.split_whitespace().count() <= 4) .and_then(|line| { crate::title::PLACE - .find(line) - .map(|m| tidy(m.as_str().trim_end_matches(|c: char| !c.is_alphabetic()))) + .captures(line) + .map(|caps| tidy(&caps["city"])) }); out } diff --git a/crates/colophon/src/title.rs b/crates/colophon/src/title.rs index dc74dbb..f903b87 100644 --- a/crates/colophon/src/title.rs +++ b/crates/colophon/src/title.rs @@ -61,7 +61,7 @@ copyright|all rights reserved|printed in)\b", /// A city a publisher puts on a title page, alone on its line. pub static PLACE: LazyLock = LazyLock::new(|| { Regex::new( - r"(?i)^(oxford|cambridge|london|new york|new haven|chicago|boston|princeton|\ + r"(?i)^(?Poxford|cambridge|london|new york|new haven|chicago|boston|princeton|\ berlin|paris|munich|rome|madrid|grand rapids|edinburgh|dublin|toronto|sydney|\ melbourne|minneapolis|philadelphia|san francisco|los angeles|leiden|amsterdam|\ copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*",