diff --git a/crates/bibfetch/src/ask.rs b/crates/bibfetch/src/ask.rs new file mode 100644 index 0000000..1eb9756 --- /dev/null +++ b/crates/bibfetch/src/ask.rs @@ -0,0 +1,137 @@ +//! Asking the registers, which is the part that touches the network. +//! +//! Lifted out of the command line so that the Python bindings ask in exactly +//! the same order, with the same retries and the same backoff. Two +//! implementations of "which register answers about this identifier" would +//! drift, and the one nobody runs from a terminal would drift silently. + +use std::time::Duration; + +use crate::{Error, Record, Source}; + +/// How to ask: who to say you are, how long to wait, how often to retry. +#[derive(Debug, Clone)] +pub struct Ask { + /// An address the registers ask for, and answer faster when they get. + pub polite: Option, + /// How long to wait on any one register. + pub timeout: Duration, + /// How many times to try a register that is unreachable. + pub attempts: u32, +} + +impl Default for Ask { + fn default() -> Self { + Self { polite: None, timeout: crate::TIMEOUT, attempts: crate::ATTEMPTS } + } +} + +impl Ask { + /// An agent configured to ask this way. + pub fn agent(&self) -> ureq::Agent { + ureq::Agent::config_builder() + .timeout_global(Some(self.timeout)) + .user_agent(user_agent(self.polite.as_deref())) + .build() + .into() + } +} + +/// Ask every register that answers about this kind of identifier, in turn, +/// and stop at the first complete answer. +/// +/// An incomplete answer is kept in case nothing better comes: a title with no +/// authors is still more than nothing, and a register being down is only a +/// failure when every register is. `Ok(None)` means no register here answers +/// about that kind at all. +pub fn lookup( + agent: &ureq::Agent, + id: &str, + kind: &str, + attempts: u32, +) -> Result, Error> { + let chain = Source::chain_for_kind(kind); + if chain.is_empty() { + return Ok(None); + } + let mut best: Option = None; + let mut last_error = None; + for source in chain { + match ask(agent, *source, id, kind, attempts) { + Ok(record) => { + let complete = record.found && record.title.is_some() && !record.authors.is_empty(); + let better = best.as_ref().is_none_or(|kept| record.found && !kept.found); + if complete { + return Ok(Some(record)); + } + if better { + best = Some(record); + } + } + Err(why) => last_error = Some(why), + } + } + match best { + Some(record) => Ok(Some(record)), + None => Err(last_error.unwrap_or_else(|| Error::Unreachable("no register answered".into()))), + } +} + +/// Ask a register, retrying a failure that might not be one. +/// +/// Waits a second before the second attempt and two before the third: long +/// enough for a register that is rate-limiting, short enough that one which +/// is simply down does not hold up a pipeline. +fn ask( + agent: &ureq::Agent, + source: Source, + id: &str, + kind: &str, + attempts: u32, +) -> Result { + let mut last = Error::Unreachable("no attempt was made".into()); + + for attempt in 0..attempts.max(1) { + match fetch(agent, source, id, kind) { + Ok(record) => return Ok(record), + // A shape this cannot read will not read any better next time. + Err(e @ Error::Shape(_)) => return Err(e), + Err(e) => last = e, + } + std::thread::sleep(Duration::from_secs(1 << attempt)); + } + Err(last) +} + +fn fetch( + agent: &ureq::Agent, + source: Source, + id: &str, + kind: &str, +) -> Result { + let response = agent.get(source.url(id)).call(); + let mut response = match response { + Ok(response) => response, + // A register that does not have it says so with a status, and that is + // an answer rather than a failure. + Err(ureq::Error::StatusCode(404)) => return Ok(crate::missing(id, kind, source)), + Err(e) => return Err(Error::Unreachable(e.to_string())), + }; + let body = response + .body_mut() + .read_to_string() + .map_err(|e| Error::Unreachable(e.to_string()))?; + source.read(id, kind, &body) +} + +/// Who is asking. +/// +/// The registers ask for an address and answer faster when they get one; a +/// tool that does not say who it is gets the slow pool. +fn user_agent(polite: Option<&str>) -> String { + let version = env!("CARGO_PKG_VERSION"); + match polite { + Some(address) => format!("bibfetch/{version} (mailto:{address})"), + None => format!("bibfetch/{version}"), + } +} diff --git a/crates/bibfetch/src/lib.rs b/crates/bibfetch/src/lib.rs index 347fe8f..6b686a8 100644 --- a/crates/bibfetch/src/lib.rs +++ b/crates/bibfetch/src/lib.rs @@ -13,6 +13,7 @@ //! book. There is no register for an ISSN that answers without a key, so an //! ISSN is looked up as the journal of a work rather than on its own. +pub mod ask; pub mod source; use std::time::Duration; diff --git a/crates/bibfetch/src/main.rs b/crates/bibfetch/src/main.rs index 535e0c2..32906a1 100644 --- a/crates/bibfetch/src/main.rs +++ b/crates/bibfetch/src/main.rs @@ -4,7 +4,6 @@ use std::path::PathBuf; use std::process::ExitCode; use std::time::Duration; -use bibfetch::{Error, Source}; use clap::Parser; use corpus_core::exit; @@ -65,11 +64,12 @@ fn main() -> ExitCode { } }; - let agent: ureq::Agent = ureq::Agent::config_builder() - .timeout_global(Some(Duration::from_secs(args.timeout))) - .user_agent(user_agent(args.polite.as_deref())) - .build() - .into(); + let agent = bibfetch::ask::Ask { + polite: args.polite.clone(), + timeout: Duration::from_secs(args.timeout), + attempts: args.attempts, + } + .agent(); let mut records = Vec::new(); let mut missing = 0usize; @@ -86,47 +86,18 @@ fn main() -> ExitCode { } }; - let chain = Source::chain_for_kind(&kind); - if chain.is_empty() { - eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers"); - missing += 1; - continue; - } - - // Ask the registers in turn and stop at the first complete answer. - // An incomplete one is kept in case nothing better comes: a title - // with no authors is still more than nothing, and a register being - // down is only a failure when every register is. - let mut best: Option = None; - let mut last_error = None; - for source in chain { - match ask(&agent, *source, &id, &kind, args.attempts) { - Ok(record) => { - let complete = - record.found && record.title.is_some() && !record.authors.is_empty(); - let better = best.as_ref().is_none_or(|kept| record.found && !kept.found); - if complete { - best = Some(record); - break; - } - if better { - best = Some(record); - } - } - Err(why) => last_error = Some(why), + match bibfetch::ask::lookup(&agent, &id, &kind, args.attempts) { + Ok(None) => { + eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers"); + missing += 1; } - } - - match best { - Some(record) => { + Ok(Some(record)) => { if !record.found { missing += 1; } records.push(record); } - None => { - let why = last_error - .map_or_else(|| "no register answered".to_string(), |e| e.to_string()); + Err(why) => { eprintln!("bibfetch: {id}: {why}"); return ExitCode::from(exit::UNAVAILABLE); } @@ -143,64 +114,11 @@ fn main() -> ExitCode { }) } -/// Ask a register, retrying a failure that might not be one. -/// -/// Waits a second before the second attempt and two before the third: long -/// enough for a register that is rate-limiting, short enough that one which -/// is simply down does not hold up a pipeline. -fn ask( - agent: &ureq::Agent, - source: Source, - id: &str, - kind: &str, - attempts: u32, -) -> Result { - let mut last = Error::Unreachable("no attempt was made".into()); - for attempt in 0..attempts.max(1) { - match fetch(agent, source, id, kind) { - Ok(record) => return Ok(record), - // A shape this cannot read will not read any better next time. - Err(e @ Error::Shape(_)) => return Err(e), - Err(e) => last = e, - } - std::thread::sleep(Duration::from_secs(1 << attempt)); - } - Err(last) -} -fn fetch( - agent: &ureq::Agent, - source: Source, - id: &str, - kind: &str, -) -> Result { - let response = agent.get(source.url(id)).call(); - let mut response = match response { - Ok(response) => response, - // A register that does not have it says so with a status, and that is - // an answer rather than a failure. - Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)), - Err(e) => return Err(Error::Unreachable(e.to_string())), - }; - let body = response - .body_mut() - .read_to_string() - .map_err(|e| Error::Unreachable(e.to_string()))?; - source.read(id, kind, &body) -} -/// Who is asking. -/// -/// The registers ask for an address and answer faster when they get one; a -/// tool that does not say who it is gets the slow pool. -fn user_agent(polite: Option<&str>) -> String { - let version = env!("CARGO_PKG_VERSION"); - match polite { - Some(address) => format!("bibfetch/{version} (mailto:{address})"), - None => format!("bibfetch/{version}"), - } -} + + /// The identifier and its kind out of one record. fn read_identifier(line: &str) -> Result<(String, String), String> { diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index 0a22591..ae0f7b4 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -30,6 +30,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] booksniff.workspace = true bibclean.workspace = true +colophon.workspace = true chunkmd.workspace = true citefind.workspace = true corpus-core.workspace = true @@ -47,6 +48,11 @@ textkit.workspace = true pyo3 = { workspace = true, optional = true } +# Only the Python build gets this. bibfetch is the one tool here that opens a +# socket, and its HTTP client does not build for wasm32 — so a dependency that +# is unconditional would take the JavaScript package down with it. +bibfetch = { workspace = true, optional = true } + # The JavaScript build. Only ever compiled for a wasm32 target; on any other # the module is compiled out and these sit unused. wasm-bindgen = { workspace = true, optional = true } @@ -56,7 +62,7 @@ serde-wasm-bindgen = { workspace = true, optional = true } # Only maturin turns this on. pyo3's extension-module feature leaves the # Python symbols to be resolved by the interpreter at load time, so an # ordinary `cargo build` would fail to link if it were on by default. -python = ["dep:pyo3"] +python = ["dep:pyo3", "dep:bibfetch"] # The C ABI. Off by default so an ordinary build does not also export a # hundred symbols nobody asked for. capi = [] diff --git a/crates/bindings/python/catena/__init__.py b/crates/bindings/python/catena/__init__.py index c1cb82e..aa2c3da 100644 --- a/crates/bindings/python/catena/__init__.py +++ b/crates/bindings/python/catena/__init__.py @@ -20,6 +20,7 @@ from ._catena import ( compose_subjects, count_tokens, cue_windows, + fetch_identifier, find_citations, fuse_hits, identify, @@ -33,12 +34,14 @@ from ._catena import ( pack_cues, parse_identifier, parse_subtitles, + read_colophon, record_problems, record_severity, same_work, scrub, segment_cues, series_of, + shelve, split_collection, suggest_author, title_case, @@ -56,6 +59,7 @@ __all__ = [ "compose_subjects", "count_tokens", "cue_windows", + "fetch_identifier", "find_citations", "fuse_hits", "identify", @@ -69,12 +73,14 @@ __all__ = [ "pack_cues", "parse_identifier", "parse_subtitles", + "read_colophon", "record_problems", "record_severity", "same_work", "scrub", "segment_cues", "series_of", + "shelve", "split_collection", "suggest_author", "title_case", diff --git a/crates/bindings/python/catena/__init__.pyi b/crates/bindings/python/catena/__init__.pyi index e003c5b..4881d97 100644 --- a/crates/bindings/python/catena/__init__.pyi +++ b/crates/bindings/python/catena/__init__.pyi @@ -75,3 +75,30 @@ def valid_isbn(text: str) -> bool: ... def isbn_from_text(text: str) -> str | None: ... def isbn_from_filename(name: str) -> str | None: ... def same_work(ours: str, theirs: str) -> bool: ... + +def read_colophon( + text: str, pages: int = ..., shelf: str | None = ... +) -> dict[str, Any]: + """What a book prints about itself in its opening pages. + + `shelf` is a closed list of headings to file the printed subjects onto, + one per line, optionally `Heading = term, term`. + """ + ... + +def shelve(shelf: str, subjects: list[str]) -> tuple[list[str], list[str]]: + """File subject headings onto a shelf. Returns (shelved, unshelved).""" + ... + +def fetch_identifier( + id: str, + kind: str = ..., + polite: str | None = ..., + timeout: int = ..., + attempts: int = ..., +) -> dict[str, Any] | None: + """Ask the register that assigns an identifier what it knows about one. + + Blocks on the network. `None` when no register answers about that kind. + """ + ... diff --git a/crates/bindings/src/python.rs b/crates/bindings/src/python.rs index 75bf2f3..69a47f1 100644 --- a/crates/bindings/src/python.rs +++ b/crates/bindings/src/python.rs @@ -431,5 +431,81 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(merge_citation_reply, module)?)?; module.add_function(wrap_pyfunction!(series_of, module)?)?; module.add_function(wrap_pyfunction!(parse_identifier, module)?)?; + module.add_function(wrap_pyfunction!(read_colophon, module)?)?; + module.add_function(wrap_pyfunction!(shelve, module)?)?; + module.add_function(wrap_pyfunction!(fetch_identifier, module)?)?; Ok(()) } + +// ───────────────────── what a book says about itself ────────────────────── + +/// Read what a book prints about itself in its opening pages. +/// +/// `shelf` is the closed list of headings to file the printed subjects onto, +/// in the format `colophon --shelf` takes: one heading per line, optionally +/// `Heading = term, term`. Without it the subjects come back as the book +/// printed them and nothing is filed. +#[pyfunction] +#[pyo3(signature = (text, pages = colophon::PAGES, shelf = None))] +fn read_colophon<'py>( + py: Python<'py>, + text: &str, + pages: usize, + shelf: Option<&str>, +) -> PyResult> { + if pages == 0 { + return Err(PyValueError::new_err("pages = 0 would read nothing")); + } + let shelf = shelf.map(colophon::shelf::Shelf::parse); + to_py(py, &colophon::read_onto(text, pages, shelf.as_ref())) +} + +/// File subject headings onto a shelf without reading a book. +/// +/// For the headings a catalogue already holds: the same matching the printed +/// ones get, so a record's existing subjects and its book's agree about what +/// counts as a heading. Returns `(shelved, unshelved)`. +#[pyfunction] +fn shelve(shelf: &str, subjects: Vec) -> (Vec, Vec) { + let filed = colophon::shelf::Shelf::parse(shelf).file(&subjects); + (filed.on, filed.unfiled) +} + +// ──────────────────────────── the registers ─────────────────────────────── + +/// Ask the body that assigns an identifier what it knows about one. +/// +/// Crossref for a DOI, the arXiv for a preprint, OpenLibrary and Google Books +/// for an ISBN, asked in turn until one gives a complete answer. This is the +/// only function here that touches the network, and it blocks: call it from a +/// thread, or with `asyncio.to_thread`. +/// +/// Returns the record, or `None` when no register here answers about that kind +/// of identifier. Raises when every register that should answer was +/// unreachable, because that is a different thing from the book not existing. +#[pyfunction] +#[pyo3(signature = (id, kind = "isbn", polite = None, timeout = 20, attempts = 3))] +fn fetch_identifier<'py>( + py: Python<'py>, + id: &str, + kind: &str, + polite: Option, + timeout: u64, + attempts: u32, +) -> PyResult>> { + let ask = bibfetch::ask::Ask { + polite, + timeout: std::time::Duration::from_secs(timeout.max(1)), + attempts: attempts.max(1), + }; + let agent = ask.agent(); + // The GIL is not needed while waiting on a socket, and holding it there + // would stop every other thread in the interpreter for the whole request. + let found = py + .detach(|| bibfetch::ask::lookup(&agent, id, kind, ask.attempts)) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + match found { + Some(record) => Ok(Some(to_py(py, &record)?)), + None => Ok(None), + } +}