idnorm and bibfetch: an identifier read, and a register asked

crates/bibfetch/src/lib.rs

@@ -0,0 +1,136 @@
+//! What a catalogue says about an identifier.
+//!
+//! [`idnorm`] can tell you a DOI is well formed. It cannot tell you the paper
+//! exists, because that is not a fact about the string — it is a fact about
+//! the world, and the only way to learn it is to ask somebody who keeps a
+//! register.
+//!
+//! So this asks. It is the only thing it does, which is the condition under
+//! which a filter here is allowed to touch the network at all.
+//!
+//! Three registers, one per kind of identifier, each the body that assigns
+//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a
+//! book. There is no register for an ISSN that answers without a key, so an
+//! ISSN is looked up as the journal of a work rather than on its own.
+
+pub mod source;
+
+use std::time::Duration;
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+pub use source::Source;
+
+/// How long to wait on a register before giving up.
+pub const TIMEOUT: Duration = Duration::from_secs(20);
+
+/// How many times to ask before believing the answer.
+pub const ATTEMPTS: u32 = 3;
+
+/// What a register said about one identifier.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Record {
+    /// Record version.
+    #[serde(default = "version")]
+    pub v: u32,
+    /// The identifier asked about, normalised.
+    pub id: String,
+    /// Which kind it is.
+    pub kind: String,
+    /// Whether the register has it. This is the answer the string alone
+    /// cannot give.
+    pub found: bool,
+    /// Which register answered.
+    pub source: String,
+    /// The title, where there is one.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+    /// The authors, in the order the register lists them.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub authors: Vec<String>,
+    /// The year of publication.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub year: Option<i64>,
+    /// The journal, publisher or archive it appeared under.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub container: Option<String>,
+    /// The abstract, where the register carries one.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub summary: Option<String>,
+}
+
+fn version() -> u32 {
+    corpus_core::V
+}
+
+impl corpus_core::Record for Record {
+    fn version(&self) -> u32 {
+        self.v
+    }
+}
+
+/// What can go wrong asking.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Error {
+    /// The register could not be reached, or refused.
+    Unreachable(String),
+    /// It answered with something this could not read.
+    Shape(String),
+    /// Nothing here knows where to ask about this kind.
+    NoRegister(String),
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Error::Unreachable(why) => write!(f, "{why}"),
+            Error::Shape(why) => write!(f, "the answer could not be read: {why}"),
+            Error::NoRegister(kind) => {
+                write!(f, "no register here answers about {kind} identifiers")
+            }
+        }
+    }
+}
+
+impl std::error::Error for Error {}
+
+/// A record for an identifier the register does not have.
+///
+/// Absence is an answer, and a useful one: it is what tells a catalogue that
+/// a well-formed DOI it has been carrying for three years was mistyped.
+pub fn missing(id: &str, kind: &str, source: Source) -> Record {
+    Record {
+        v: corpus_core::V,
+        id: id.to_string(),
+        kind: kind.to_string(),
+        found: false,
+        source: source.name().to_string(),
+        title: None,
+        authors: Vec::new(),
+        year: None,
+        container: None,
+        summary: None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn an_identifier_nobody_has_is_still_an_answer() {
+        let record = missing("10.1000/nope", "doi", Source::Crossref);
+        assert!(!record.found);
+        assert_eq!(record.source, "crossref");
+        assert_eq!(record.title, None);
+    }
+
+    #[test]
+    fn a_record_that_found_nothing_serialises_without_empty_fields() {
+        let json = serde_json::to_string(&missing("x", "doi", Source::Crossref)).expect("json");
+        assert!(!json.contains("title"), "{json}");
+        assert!(!json.contains("authors"), "{json}");
+        assert!(json.contains("\"found\":false"), "{json}");
+    }
+}

crates/bibfetch/src/main.rs

@@ -0,0 +1,193 @@
+//! What a catalogue says about an identifier.
+
+use std::path::PathBuf;
+use std::process::ExitCode;
+use std::time::Duration;
+
+use bibfetch::{Error, Source};
+use clap::Parser;
+use corpus_core::exit;
+
+/// Ask the body that assigns an identifier what it knows about one.
+///
+/// `idnorm` can say a DOI is well formed. Only a register can say the paper
+/// exists — that is a fact about the world rather than about the string, and
+/// it is what catches a well-formed DOI a catalogue has been carrying wrongly
+/// for three years.
+///
+/// Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a book. This
+/// is the only tool here besides `emb` that touches the network, and it is the
+/// whole of what it does.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Exit codes: 0 every identifier was found, 1 at least one was not,
+65 the input was not identifiers, 69 a register could not be reached,
+74 the file could not be read.
+
+Example:
+
+    idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title'
+    idnorm --kind doi refs.txt | bibfetch --polite you@example.org"
+)]
+struct Args {
+    /// Identifier records to look up, as `idnorm` writes them. Reads stdin
+    /// when absent.
+    path: Option<PathBuf>,
+
+    /// An address to put in the request, which the registers ask for and
+    /// answer faster when they get.
+    #[arg(long, value_name = "EMAIL")]
+    polite: Option<String>,
+
+    /// How long to wait on a register, in seconds.
+    #[arg(long, default_value_t = bibfetch::TIMEOUT.as_secs(), value_name = "SECONDS")]
+    timeout: u64,
+
+    /// How many times to ask before believing a failure.
+    #[arg(long, default_value_t = bibfetch::ATTEMPTS, value_name = "COUNT")]
+    attempts: u32,
+}
+
+fn main() -> ExitCode {
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
+
+    let text = match catena_cli::read_text(args.path.as_deref()) {
+        Ok(text) => text,
+        Err((code, why)) => {
+            eprintln!("bibfetch: {why}");
+            return ExitCode::from(code);
+        }
+    };
+
+    let agent: ureq::Agent = ureq::Agent::config_builder()
+        .timeout_global(Some(Duration::from_secs(args.timeout)))
+        .user_agent(user_agent(args.polite.as_deref()))
+        .build()
+        .into();
+
+    let mut records = Vec::new();
+    let mut missing = 0usize;
+
+    for (number, line) in text.lines().enumerate() {
+        if line.trim().is_empty() {
+            continue;
+        }
+        let (id, kind) = match read_identifier(line) {
+            Ok(pair) => pair,
+            Err(why) => {
+                eprintln!("bibfetch: line {}: {why}", number + 1);
+                return ExitCode::from(exit::DATA);
+            }
+        };
+
+        let Some(source) = Source::for_kind(&kind) else {
+            eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers");
+            missing += 1;
+            continue;
+        };
+
+        match ask(&agent, source, &id, &kind, args.attempts) {
+            Ok(record) => {
+                if !record.found {
+                    missing += 1;
+                }
+                records.push(record);
+            }
+            Err(why) => {
+                eprintln!("bibfetch: {id}: {why}");
+                return ExitCode::from(exit::UNAVAILABLE);
+            }
+        }
+    }
+
+    let wrote = catena_cli::stream("bibfetch", &records);
+    if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
+        return wrote;
+    }
+    ExitCode::from(match missing {
+        0 => exit::OK,
+        _ => exit::NO,
+    })
+}
+
+/// Ask a register, retrying a failure that might not be one.
+///
+/// Waits a second before the second attempt and two before the third: long
+/// enough for a register that is rate-limiting, short enough that one which
+/// is simply down does not hold up a pipeline.
+fn ask(
+    agent: &ureq::Agent,
+    source: Source,
+    id: &str,
+    kind: &str,
+    attempts: u32,
+) -> Result<bibfetch::Record, Error> {
+    let mut last = Error::Unreachable("no attempt was made".into());
+
+    for attempt in 0..attempts.max(1) {
+        match fetch(agent, source, id, kind) {
+            Ok(record) => return Ok(record),
+            // A shape this cannot read will not read any better next time.
+            Err(e @ Error::Shape(_)) => return Err(e),
+            Err(e) => last = e,
+        }
+        std::thread::sleep(Duration::from_secs(1 << attempt));
+    }
+    Err(last)
+}
+
+fn fetch(
+    agent: &ureq::Agent,
+    source: Source,
+    id: &str,
+    kind: &str,
+) -> Result<bibfetch::Record, Error> {
+    let response = agent.get(source.url(id)).call();
+    let mut response = match response {
+        Ok(response) => response,
+        // A register that does not have it says so with a status, and that is
+        // an answer rather than a failure.
+        Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)),
+        Err(e) => return Err(Error::Unreachable(e.to_string())),
+    };
+    let body = response
+        .body_mut()
+        .read_to_string()
+        .map_err(|e| Error::Unreachable(e.to_string()))?;
+    source.read(id, kind, &body)
+}
+
+/// Who is asking.
+///
+/// The registers ask for an address and answer faster when they get one; a
+/// tool that does not say who it is gets the slow pool.
+fn user_agent(polite: Option<&str>) -> String {
+    let version = env!("CARGO_PKG_VERSION");
+    match polite {
+        Some(address) => format!("bibfetch/{version} (mailto:{address})"),
+        None => format!("bibfetch/{version}"),
+    }
+}
+
+/// The identifier and its kind out of one record.
+fn read_identifier(line: &str) -> Result<(String, String), String> {
+    let value: serde_json::Value =
+        serde_json::from_str(line).map_err(|e| format!("not a record: {e}"))?;
+    let id = value
+        .get("id")
+        .and_then(serde_json::Value::as_str)
+        .ok_or("no id")?
+        .to_string();
+    let kind = value
+        .get("kind")
+        .and_then(serde_json::Value::as_str)
+        .ok_or("no kind")?
+        .to_string();
+    Ok((id, kind))
+}

crates/bibfetch/src/source.rs

@@ -0,0 +1,342 @@
+//! The registers, and how to read what each one says.
+//!
+//! Each is the body that assigns the identifiers it answers about, so each is
+//! the one place the answer is not a guess. The shapes they reply in have
+//! nothing in common, which is most of what this module is.
+
+use serde::Deserialize;
+
+use crate::{Error, Record};
+
+/// A register that can be asked about an identifier.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Source {
+    /// Crossref, which assigns most scholarly DOIs.
+    Crossref,
+    /// The arXiv's own register of preprints.
+    Arxiv,
+    /// OpenLibrary, for books by ISBN.
+    OpenLibrary,
+}
+
+impl Source {
+    /// What it is called.
+    pub fn name(self) -> &'static str {
+        match self {
+            Source::Crossref => "crossref",
+            Source::Arxiv => "arxiv",
+            Source::OpenLibrary => "openlibrary",
+        }
+    }
+
+    /// Which register answers about a kind of identifier.
+    pub fn for_kind(kind: &str) -> Option<Self> {
+        match kind {
+            "doi" => Some(Source::Crossref),
+            "arxiv" => Some(Source::Arxiv),
+            "isbn" => Some(Source::OpenLibrary),
+            _ => None,
+        }
+    }
+
+    /// Where to ask about one identifier.
+    pub fn url(self, id: &str) -> String {
+        match self {
+            Source::Crossref => format!("https://api.crossref.org/works/{}", encode(id)),
+            Source::Arxiv => format!(
+                "https://export.arxiv.org/api/query?id_list={}&max_results=1",
+                encode(id)
+            ),
+            Source::OpenLibrary => {
+                format!("https://openlibrary.org/isbn/{}.json", encode(id))
+            }
+        }
+    }
+
+    /// Read what it said into a record.
+    pub fn read(self, id: &str, kind: &str, body: &str) -> Result<Record, Error> {
+        match self {
+            Source::Crossref => crossref(id, kind, body),
+            Source::Arxiv => arxiv(id, kind, body),
+            Source::OpenLibrary => open_library(id, kind, body),
+        }
+    }
+}
+
+/// Percent-encode what a path or a query would otherwise read as structure.
+///
+/// A DOI suffix may hold a slash, a question mark or a hash, and every one of
+/// those means something else in a URL.
+fn encode(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    for byte in text.bytes() {
+        match byte {
+            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => {
+                out.push(byte as char)
+            }
+            byte => out.push_str(&format!("%{byte:02X}")),
+        }
+    }
+    out
+}
+
+/// A record with the fields nothing found fills in.
+fn blank(id: &str, kind: &str, source: Source) -> Record {
+    let mut record = crate::missing(id, kind, source);
+    record.found = true;
+    record
+}
+
+#[derive(Deserialize)]
+struct CrossrefReply {
+    message: CrossrefWork,
+}
+
+#[derive(Deserialize)]
+struct CrossrefWork {
+    #[serde(default)]
+    title: Vec<String>,
+    #[serde(default)]
+    author: Vec<CrossrefAuthor>,
+    #[serde(default, rename = "container-title")]
+    container_title: Vec<String>,
+    #[serde(default)]
+    issued: Option<CrossrefDate>,
+    #[serde(default, rename = "abstract")]
+    summary: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct CrossrefAuthor {
+    #[serde(default)]
+    given: Option<String>,
+    #[serde(default)]
+    family: Option<String>,
+    #[serde(default)]
+    name: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct CrossrefDate {
+    #[serde(default, rename = "date-parts")]
+    date_parts: Vec<Vec<i64>>,
+}
+
+fn crossref(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
+    let reply: CrossrefReply =
+        serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
+    let work = reply.message;
+
+    let mut record = blank(id, kind, Source::Crossref);
+    record.title = work.title.into_iter().next();
+    record.container = work.container_title.into_iter().next();
+    record.year = work
+        .issued
+        .and_then(|issued| issued.date_parts.into_iter().next())
+        .and_then(|parts| parts.into_iter().next());
+    record.authors = work
+        .author
+        .into_iter()
+        .filter_map(|author| match (author.given, author.family, author.name) {
+            (Some(given), Some(family), _) => Some(format!("{given} {family}")),
+            (None, Some(family), _) => Some(family),
+            (_, _, Some(name)) => Some(name),
+            _ => None,
+        })
+        .collect();
+    // Crossref's abstracts arrive as JATS markup.
+    record.summary = work.summary.map(|summary| strip_tags(&summary));
+    Ok(record)
+}
+
+/// The arXiv answers in Atom, which is XML. Reading it properly would mean a
+/// parser; what is wanted is four fields, and they are each one tag.
+fn arxiv(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
+    // An empty feed means no such preprint, which the API signals by omitting
+    // the entry rather than by a status code.
+    let Some(entry) = between(body, "<entry>", "</entry>") else {
+        return Ok(crate::missing(id, kind, Source::Arxiv));
+    };
+
+    let mut record = blank(id, kind, Source::Arxiv);
+    record.title = between(entry, "<title>", "</title>").map(collapse);
+    record.summary = between(entry, "<summary>", "</summary>").map(collapse);
+    record.container = Some("arXiv".to_string());
+    record.year = between(entry, "<published>", "</published>")
+        .and_then(|published| published.get(..4))
+        .and_then(|year| year.parse().ok());
+
+    let mut authors = Vec::new();
+    let mut rest = entry;
+    while let Some(author) = between(rest, "<author>", "</author>") {
+        if let Some(name) = between(author, "<name>", "</name>") {
+            authors.push(collapse(name));
+        }
+        let Some(at) = rest.find("</author>") else {
+            break;
+        };
+        rest = &rest[at + "</author>".len()..];
+    }
+    record.authors = authors;
+    Ok(record)
+}
+
+#[derive(Deserialize)]
+struct OpenLibraryBook {
+    #[serde(default)]
+    title: Option<String>,
+    #[serde(default)]
+    publish_date: Option<String>,
+    #[serde(default)]
+    publishers: Vec<String>,
+}
+
+fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
+    let book: OpenLibraryBook =
+        serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
+
+    let mut record = blank(id, kind, Source::OpenLibrary);
+    record.title = book.title;
+    record.container = book.publishers.into_iter().next();
+    // The date is free text — "1999", "March 1999", "1999-03-01".
+    record.year = book.publish_date.as_deref().and_then(year_in);
+    Ok(record)
+}
+
+/// The first four-digit year in a string.
+fn year_in(text: &str) -> Option<i64> {
+    let bytes: Vec<char> = text.chars().collect();
+    for window in bytes.windows(4) {
+        if window.iter().all(char::is_ascii_digit) {
+            let year: i64 = window.iter().collect::<String>().parse().ok()?;
+            if (1000..=2999).contains(&year) {
+                return Some(year);
+            }
+        }
+    }
+    None
+}
+
+/// What sits between two markers, the first time they appear in order.
+fn between<'a>(text: &'a str, open: &str, close: &str) -> Option<&'a str> {
+    let start = text.find(open)? + open.len();
+    let end = text[start..].find(close)? + start;
+    Some(&text[start..end])
+}
+
+/// Runs of whitespace down to one space, trimmed.
+fn collapse(text: &str) -> String {
+    text.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+/// Text with its tags removed.
+///
+/// Crossref hands back JATS — `<jats:p>` and friends — which is markup for a
+/// typesetter rather than anything a reader wants.
+fn strip_tags(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    let mut inside = false;
+    for c in text.chars() {
+        match c {
+            '<' => inside = true,
+            '>' => inside = false,
+            c if !inside => out.push(c),
+            _ => {}
+        }
+    }
+    collapse(&out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn each_kind_goes_to_the_body_that_assigns_it() {
+        assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
+        assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
+        assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
+        // No register answers about an ISSN without a key.
+        assert_eq!(Source::for_kind("issn"), None);
+    }
+
+    #[test]
+    fn a_doi_suffix_cannot_change_the_shape_of_the_url() {
+        let url = Source::Crossref.url("10.1000/a?b#c/d");
+        assert!(url.ends_with("10.1000%2Fa%3Fb%23c%2Fd"), "{url}");
+        assert_eq!(url.matches('?').count(), 0);
+    }
+
+    #[test]
+    fn crossref_is_read_into_the_fields_that_matter() {
+        let body = r#"{"message":{"title":["On Computable Numbers"],
+            "author":[{"given":"Alan","family":"Turing"},{"family":"Church"},{"name":"A Committee"}],
+            "container-title":["Proceedings"],
+            "issued":{"date-parts":[[1936,11,12]]},
+            "abstract":"<jats:p>A paragraph.</jats:p>"}}"#;
+        let record = crossref("10.1", "doi", body).expect("a record");
+        assert!(record.found);
+        assert_eq!(record.title.as_deref(), Some("On Computable Numbers"));
+        assert_eq!(record.authors, ["Alan Turing", "Church", "A Committee"]);
+        assert_eq!(record.year, Some(1936));
+        assert_eq!(record.container.as_deref(), Some("Proceedings"));
+        assert_eq!(record.summary.as_deref(), Some("A paragraph."));
+    }
+
+    #[test]
+    fn a_crossref_reply_missing_everything_is_still_a_record() {
+        let record = crossref("10.1", "doi", r#"{"message":{}}"#).expect("a record");
+        assert!(record.found);
+        assert_eq!(record.title, None);
+        assert!(record.authors.is_empty());
+    }
+
+    #[test]
+    fn an_arxiv_feed_with_no_entry_means_no_such_preprint() {
+        let record = arxiv("2401.01234", "arxiv", "<feed></feed>").expect("a record");
+        assert!(!record.found);
+    }
+
+    #[test]
+    fn an_arxiv_entry_is_read_out_of_its_atom() {
+        let body = "<feed><entry><title>A\n  Title</title>\
+            <summary>  Some words.  </summary>\
+            <published>2024-01-02T00:00:00Z</published>\
+            <author><name>Ada Lovelace</name></author>\
+            <author><name>Charles Babbage</name></author></entry></feed>";
+        let record = arxiv("2401.01234", "arxiv", body).expect("a record");
+        assert!(record.found);
+        assert_eq!(record.title.as_deref(), Some("A Title"));
+        assert_eq!(record.summary.as_deref(), Some("Some words."));
+        assert_eq!(record.year, Some(2024));
+        assert_eq!(record.authors, ["Ada Lovelace", "Charles Babbage"]);
+    }
+
+    #[test]
+    fn openlibrary_gives_up_a_year_from_whatever_it_wrote() {
+        for (date, year) in [
+            ("1999", Some(1999)),
+            ("March 1999", Some(1999)),
+            ("1999-03-01", Some(1999)),
+            ("n.d.", None),
+        ] {
+            let body = format!(r#"{{"title":"A Book","publish_date":"{date}"}}"#);
+            let record = open_library("1", "isbn", &body).expect("a record");
+            assert_eq!(record.year, year, "{date}");
+        }
+    }
+
+    #[test]
+    fn an_answer_that_is_not_what_it_should_be_says_so() {
+        assert!(matches!(
+            crossref("10.1", "doi", "not json"),
+            Err(Error::Shape(_))
+        ));
+    }
+
+    #[test]
+    fn markup_is_not_part_of_an_abstract() {
+        assert_eq!(strip_tags("<jats:p>One <i>two</i></jats:p>"), "One two");
+        assert_eq!(strip_tags("plain"), "plain");
+    }
+}