//! The registers, and how to read what each one says. //! //! Each is the body that assigns the identifiers it answers about, so each is //! the one place the answer is not a guess. The shapes they reply in have //! nothing in common, which is most of what this module is. use serde::Deserialize; use crate::{Error, Record}; /// A register that can be asked about an identifier. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Source { /// Crossref, which assigns most scholarly DOIs. Crossref, /// The arXiv's own register of preprints. Arxiv, /// OpenLibrary, for books by ISBN. OpenLibrary, } impl Source { /// What it is called. pub fn name(self) -> &'static str { match self { Source::Crossref => "crossref", Source::Arxiv => "arxiv", Source::OpenLibrary => "openlibrary", } } /// Which register answers about a kind of identifier. pub fn for_kind(kind: &str) -> Option { match kind { "doi" => Some(Source::Crossref), "arxiv" => Some(Source::Arxiv), "isbn" => Some(Source::OpenLibrary), _ => None, } } /// Where to ask about one identifier. pub fn url(self, id: &str) -> String { match self { Source::Crossref => format!("https://api.crossref.org/works/{}", encode(id)), Source::Arxiv => format!( "https://export.arxiv.org/api/query?id_list={}&max_results=1", encode(id) ), Source::OpenLibrary => { format!("https://openlibrary.org/isbn/{}.json", encode(id)) } } } /// Read what it said into a record. pub fn read(self, id: &str, kind: &str, body: &str) -> Result { match self { Source::Crossref => crossref(id, kind, body), Source::Arxiv => arxiv(id, kind, body), Source::OpenLibrary => open_library(id, kind, body), } } } /// Percent-encode what a path or a query would otherwise read as structure. /// /// A DOI suffix may hold a slash, a question mark or a hash, and every one of /// those means something else in a URL. fn encode(text: &str) -> String { let mut out = String::with_capacity(text.len()); for byte in text.bytes() { match byte { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { out.push(byte as char) } byte => out.push_str(&format!("%{byte:02X}")), } } out } /// A record with the fields nothing found fills in. fn blank(id: &str, kind: &str, source: Source) -> Record { let mut record = crate::missing(id, kind, source); record.found = true; record } #[derive(Deserialize)] struct CrossrefReply { message: CrossrefWork, } #[derive(Deserialize)] struct CrossrefWork { #[serde(default)] title: Vec, #[serde(default)] author: Vec, #[serde(default, rename = "container-title")] container_title: Vec, #[serde(default)] issued: Option, #[serde(default, rename = "abstract")] summary: Option, } #[derive(Deserialize)] struct CrossrefAuthor { #[serde(default)] given: Option, #[serde(default)] family: Option, #[serde(default)] name: Option, } #[derive(Deserialize)] struct CrossrefDate { #[serde(default, rename = "date-parts")] date_parts: Vec>, } fn crossref(id: &str, kind: &str, body: &str) -> Result { let reply: CrossrefReply = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; let work = reply.message; let mut record = blank(id, kind, Source::Crossref); record.title = work.title.into_iter().next(); record.container = work.container_title.into_iter().next(); record.year = work .issued .and_then(|issued| issued.date_parts.into_iter().next()) .and_then(|parts| parts.into_iter().next()); record.authors = work .author .into_iter() .filter_map(|author| match (author.given, author.family, author.name) { (Some(given), Some(family), _) => Some(format!("{given} {family}")), (None, Some(family), _) => Some(family), (_, _, Some(name)) => Some(name), _ => None, }) .collect(); // Crossref's abstracts arrive as JATS markup. record.summary = work.summary.map(|summary| strip_tags(&summary)); Ok(record) } /// The arXiv answers in Atom, which is XML. Reading it properly would mean a /// parser; what is wanted is four fields, and they are each one tag. fn arxiv(id: &str, kind: &str, body: &str) -> Result { // An empty feed means no such preprint, which the API signals by omitting // the entry rather than by a status code. let Some(entry) = between(body, "", "") else { return Ok(crate::missing(id, kind, Source::Arxiv)); }; let mut record = blank(id, kind, Source::Arxiv); record.title = between(entry, "", "").map(collapse); record.summary = between(entry, "", "").map(collapse); record.container = Some("arXiv".to_string()); record.year = between(entry, "", "") .and_then(|published| published.get(..4)) .and_then(|year| year.parse().ok()); let mut authors = Vec::new(); let mut rest = entry; while let Some(author) = between(rest, "", "") { if let Some(name) = between(author, "", "") { authors.push(collapse(name)); } let Some(at) = rest.find("") else { break; }; rest = &rest[at + "".len()..]; } record.authors = authors; Ok(record) } #[derive(Deserialize)] struct OpenLibraryBook { #[serde(default)] title: Option, #[serde(default)] publish_date: Option, #[serde(default)] publishers: Vec, } fn open_library(id: &str, kind: &str, body: &str) -> Result { let book: OpenLibraryBook = serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; let mut record = blank(id, kind, Source::OpenLibrary); record.title = book.title; record.container = book.publishers.into_iter().next(); // The date is free text — "1999", "March 1999", "1999-03-01". record.year = book.publish_date.as_deref().and_then(year_in); Ok(record) } /// The first four-digit year in a string. fn year_in(text: &str) -> Option { let bytes: Vec = text.chars().collect(); for window in bytes.windows(4) { if window.iter().all(char::is_ascii_digit) { let year: i64 = window.iter().collect::().parse().ok()?; if (1000..=2999).contains(&year) { return Some(year); } } } None } /// What sits between two markers, the first time they appear in order. fn between<'a>(text: &'a str, open: &str, close: &str) -> Option<&'a str> { let start = text.find(open)? + open.len(); let end = text[start..].find(close)? + start; Some(&text[start..end]) } /// Runs of whitespace down to one space, trimmed. fn collapse(text: &str) -> String { text.split_whitespace().collect::>().join(" ") } /// Text with its tags removed. /// /// Crossref hands back JATS — `` and friends — which is markup for a /// typesetter rather than anything a reader wants. fn strip_tags(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut inside = false; for c in text.chars() { match c { '<' => inside = true, '>' => inside = false, c if !inside => out.push(c), _ => {} } } collapse(&out) } #[cfg(test)] mod tests { use super::*; #[test] fn each_kind_goes_to_the_body_that_assigns_it() { assert_eq!(Source::for_kind("doi"), Some(Source::Crossref)); assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv)); assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary)); // No register answers about an ISSN without a key. assert_eq!(Source::for_kind("issn"), None); } #[test] fn a_doi_suffix_cannot_change_the_shape_of_the_url() { let url = Source::Crossref.url("10.1000/a?b#c/d"); assert!(url.ends_with("10.1000%2Fa%3Fb%23c%2Fd"), "{url}"); assert_eq!(url.matches('?').count(), 0); } #[test] fn crossref_is_read_into_the_fields_that_matter() { let body = r#"{"message":{"title":["On Computable Numbers"], "author":[{"given":"Alan","family":"Turing"},{"family":"Church"},{"name":"A Committee"}], "container-title":["Proceedings"], "issued":{"date-parts":[[1936,11,12]]}, "abstract":"A paragraph."}}"#; let record = crossref("10.1", "doi", body).expect("a record"); assert!(record.found); assert_eq!(record.title.as_deref(), Some("On Computable Numbers")); assert_eq!(record.authors, ["Alan Turing", "Church", "A Committee"]); assert_eq!(record.year, Some(1936)); assert_eq!(record.container.as_deref(), Some("Proceedings")); assert_eq!(record.summary.as_deref(), Some("A paragraph.")); } #[test] fn a_crossref_reply_missing_everything_is_still_a_record() { let record = crossref("10.1", "doi", r#"{"message":{}}"#).expect("a record"); assert!(record.found); assert_eq!(record.title, None); assert!(record.authors.is_empty()); } #[test] fn an_arxiv_feed_with_no_entry_means_no_such_preprint() { let record = arxiv("2401.01234", "arxiv", "").expect("a record"); assert!(!record.found); } #[test] fn an_arxiv_entry_is_read_out_of_its_atom() { let body = "A\n Title\ Some words. \ 2024-01-02T00:00:00Z\ Ada Lovelace\ Charles Babbage"; let record = arxiv("2401.01234", "arxiv", body).expect("a record"); assert!(record.found); assert_eq!(record.title.as_deref(), Some("A Title")); assert_eq!(record.summary.as_deref(), Some("Some words.")); assert_eq!(record.year, Some(2024)); assert_eq!(record.authors, ["Ada Lovelace", "Charles Babbage"]); } #[test] fn openlibrary_gives_up_a_year_from_whatever_it_wrote() { for (date, year) in [ ("1999", Some(1999)), ("March 1999", Some(1999)), ("1999-03-01", Some(1999)), ("n.d.", None), ] { let body = format!(r#"{{"title":"A Book","publish_date":"{date}"}}"#); let record = open_library("1", "isbn", &body).expect("a record"); assert_eq!(record.year, year, "{date}"); } } #[test] fn an_answer_that_is_not_what_it_should_be_says_so() { assert!(matches!( crossref("10.1", "doi", "not json"), Err(Error::Shape(_)) )); } #[test] fn markup_is_not_part_of_an_abstract() { assert_eq!(strip_tags("One two"), "One two"); assert_eq!(strip_tags("plain"), "plain"); } }