funkpower/catena
1
0
Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-10-23 11:17:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

bindings: what a book says about itself, and what the register says back

Browse Source 
book-pipeline reaches for colophon and bibfetch on every book it repairs.
Shelling out would mean a process per book per tool across five thousand
books, and parsing back the JSON it is about to parse anyway.

bibfetch's fetch loop lived in main.rs, so binding it meant either
lifting it or writing a second one. A second implementation of 'which
register answers about this kind of identifier, and in what order' would
drift from the first, and the copy nobody runs from a terminal would
drift silently. It is now bibfetch::ask, and the command line calls it
too — verified by running the CLI before and after and diffing.

fetch_identifier releases the GIL around the request. It is the only
function here that opens a socket, and holding the interpreter for the
length of an HTTP call would stop every other thread in it.

bibfetch is optional and only the python feature turns it on: its HTTP
client does not build for wasm32, and an unconditional dependency would
have taken the JavaScript package down with it.
This commit is contained in:
funkpower
2026-09-21 10:20:24 -04:00
parent 6199fce251
commit 18cc217c0d
7 changed files with 268 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bibfetch/src/main.rs
+14 -96
View File
@@ -4,7 +4,6 @@ use std::path::PathBuf;
use std::process::ExitCode;
use std::time::Duration;
use bibfetch::{Error, Source};
use clap::Parser;
use corpus_core::exit;
@@ -65,11 +64,12 @@ fn main() -> ExitCode {
}
};
let agent: ureq::Agent = ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(args.timeout)))
.user_agent(user_agent(args.polite.as_deref()))
.build()
.into();
let agent = bibfetch::ask::Ask {
polite: args.polite.clone(),
timeout: Duration::from_secs(args.timeout),
attempts: args.attempts,
}
.agent();
let mut records = Vec::new();
let mut missing = 0usize;
@@ -86,47 +86,18 @@ fn main() -> ExitCode {
}
};
let chain = Source::chain_for_kind(&kind);
if chain.is_empty() {
eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers");
missing += 1;
continue;
}
// Ask the registers in turn and stop at the first complete answer.
// An incomplete one is kept in case nothing better comes: a title
// with no authors is still more than nothing, and a register being
// down is only a failure when every register is.
let mut best: Option<bibfetch::Record> = None;
let mut last_error = None;
for source in chain {
match ask(&agent, *source, &id, &kind, args.attempts) {
Ok(record) => {
let complete =
record.found && record.title.is_some() && !record.authors.is_empty();
let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
if complete {
best = Some(record);
break;
}
if better {
best = Some(record);
}
}
Err(why) => last_error = Some(why),
match bibfetch::ask::lookup(&agent, &id, &kind, args.attempts) {
Ok(None) => {
eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers");
missing += 1;
}
}
match best {
Some(record) => {
Ok(Some(record)) => {
if !record.found {
missing += 1;
}
records.push(record);
}
None => {
let why = last_error
.map_or_else(|| "no register answered".to_string(), |e| e.to_string());
Err(why) => {
eprintln!("bibfetch: {id}: {why}");
return ExitCode::from(exit::UNAVAILABLE);
}
@@ -143,64 +114,11 @@ fn main() -> ExitCode {
})
}
/// Ask a register, retrying a failure that might not be one.
///
/// Waits a second before the second attempt and two before the third: long
/// enough for a register that is rate-limiting, short enough that one which
/// is simply down does not hold up a pipeline.
fn ask(
agent: &ureq::Agent,
source: Source,
id: &str,
kind: &str,
attempts: u32,
) -> Result<bibfetch::Record, Error> {
let mut last = Error::Unreachable("no attempt was made".into());
for attempt in 0..attempts.max(1) {
match fetch(agent, source, id, kind) {
Ok(record) => return Ok(record),
// A shape this cannot read will not read any better next time.
Err(e @ Error::Shape(_)) => return Err(e),
Err(e) => last = e,
}
std::thread::sleep(Duration::from_secs(1 << attempt));
}
Err(last)
}
fn fetch(
agent: &ureq::Agent,
source: Source,
id: &str,
kind: &str,
) -> Result<bibfetch::Record, Error> {
let response = agent.get(source.url(id)).call();
let mut response = match response {
Ok(response) => response,
// A register that does not have it says so with a status, and that is
// an answer rather than a failure.
Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)),
Err(e) => return Err(Error::Unreachable(e.to_string())),
};
let body = response
.body_mut()
.read_to_string()
.map_err(|e| Error::Unreachable(e.to_string()))?;
source.read(id, kind, &body)
}
/// Who is asking.
///
/// The registers ask for an address and answer faster when they get one; a
/// tool that does not say who it is gets the slow pool.
fn user_agent(polite: Option<&str>) -> String {
let version = env!("CARGO_PKG_VERSION");
match polite {
Some(address) => format!("bibfetch/{version} (mailto:{address})"),
None => format!("bibfetch/{version}"),
}
}
/// The identifier and its kind out of one record.
fn read_identifier(line: &str) -> Result<(String, String), String> {