//! What a catalogue says about an identifier. use std::path::PathBuf; use std::process::ExitCode; use std::time::Duration; use bibfetch::{Error, Source}; use clap::Parser; use corpus_core::exit; /// Ask the body that assigns an identifier what it knows about one. /// /// `idnorm` can say a DOI is well formed. Only a register can say the paper /// exists — that is a fact about the world rather than about the string, and /// it is what catches a well-formed DOI a catalogue has been carrying wrongly /// for three years. /// /// Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a book. This /// is the only tool here besides `emb` that touches the network, and it is the /// whole of what it does. #[derive(Parser)] #[command( version, about, after_help = "\ Exit codes: 0 every identifier was found, 1 at least one was not, 65 the input was not identifiers, 69 a register could not be reached, 74 the file could not be read. Example: idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title' idnorm --kind doi refs.txt | bibfetch --polite you@example.org" )] struct Args { /// Identifier records to look up, as `idnorm` writes them. Reads stdin /// when absent. path: Option, /// An address to put in the request, which the registers ask for and /// answer faster when they get. #[arg(long, value_name = "EMAIL")] polite: Option, /// How long to wait on a register, in seconds. #[arg(long, default_value_t = bibfetch::TIMEOUT.as_secs(), value_name = "SECONDS")] timeout: u64, /// How many times to ask before believing a failure. #[arg(long, default_value_t = bibfetch::ATTEMPTS, value_name = "COUNT")] attempts: u32, } fn main() -> ExitCode { let args: Args = match catena_cli::parse() { Ok(args) => args, Err(code) => return code, }; let text = match catena_cli::read_text(args.path.as_deref()) { Ok(text) => text, Err((code, why)) => { eprintln!("bibfetch: {why}"); return ExitCode::from(code); } }; let agent: ureq::Agent = ureq::Agent::config_builder() .timeout_global(Some(Duration::from_secs(args.timeout))) .user_agent(user_agent(args.polite.as_deref())) .build() .into(); let mut records = Vec::new(); let mut missing = 0usize; for (number, line) in text.lines().enumerate() { if line.trim().is_empty() { continue; } let (id, kind) = match read_identifier(line) { Ok(pair) => pair, Err(why) => { eprintln!("bibfetch: line {}: {why}", number + 1); return ExitCode::from(exit::DATA); } }; let Some(source) = Source::for_kind(&kind) else { eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers"); missing += 1; continue; }; match ask(&agent, source, &id, &kind, args.attempts) { Ok(record) => { if !record.found { missing += 1; } records.push(record); } Err(why) => { eprintln!("bibfetch: {id}: {why}"); return ExitCode::from(exit::UNAVAILABLE); } } } let wrote = catena_cli::stream("bibfetch", &records); if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { return wrote; } ExitCode::from(match missing { 0 => exit::OK, _ => exit::NO, }) } /// Ask a register, retrying a failure that might not be one. /// /// Waits a second before the second attempt and two before the third: long /// enough for a register that is rate-limiting, short enough that one which /// is simply down does not hold up a pipeline. fn ask( agent: &ureq::Agent, source: Source, id: &str, kind: &str, attempts: u32, ) -> Result { let mut last = Error::Unreachable("no attempt was made".into()); for attempt in 0..attempts.max(1) { match fetch(agent, source, id, kind) { Ok(record) => return Ok(record), // A shape this cannot read will not read any better next time. Err(e @ Error::Shape(_)) => return Err(e), Err(e) => last = e, } std::thread::sleep(Duration::from_secs(1 << attempt)); } Err(last) } fn fetch( agent: &ureq::Agent, source: Source, id: &str, kind: &str, ) -> Result { let response = agent.get(source.url(id)).call(); let mut response = match response { Ok(response) => response, // A register that does not have it says so with a status, and that is // an answer rather than a failure. Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)), Err(e) => return Err(Error::Unreachable(e.to_string())), }; let body = response .body_mut() .read_to_string() .map_err(|e| Error::Unreachable(e.to_string()))?; source.read(id, kind, &body) } /// Who is asking. /// /// The registers ask for an address and answer faster when they get one; a /// tool that does not say who it is gets the slow pool. fn user_agent(polite: Option<&str>) -> String { let version = env!("CARGO_PKG_VERSION"); match polite { Some(address) => format!("bibfetch/{version} (mailto:{address})"), None => format!("bibfetch/{version}"), } } /// The identifier and its kind out of one record. fn read_identifier(line: &str) -> Result<(String, String), String> { let value: serde_json::Value = serde_json::from_str(line).map_err(|e| format!("not a record: {e}"))?; let id = value .get("id") .and_then(serde_json::Value::as_str) .ok_or("no id")? .to_string(); let kind = value .get("kind") .and_then(serde_json::Value::as_str) .ok_or("no kind")? .to_string(); Ok((id, kind)) }