bindings: what a book says about itself, and what the register says back

book-pipeline reaches for colophon and bibfetch on every book it repairs.
Shelling out would mean a process per book per tool across five thousand
books, and parsing back the JSON it is about to parse anyway.

bibfetch's fetch loop lived in main.rs, so binding it meant either
lifting it or writing a second one. A second implementation of 'which
register answers about this kind of identifier, and in what order' would
drift from the first, and the copy nobody runs from a terminal would
drift silently. It is now bibfetch::ask, and the command line calls it
too — verified by running the CLI before and after and diffing.

fetch_identifier releases the GIL around the request. It is the only
function here that opens a socket, and holding the interpreter for the
length of an HTTP call would stop every other thread in it.

bibfetch is optional and only the python feature turns it on: its HTTP
client does not build for wasm32, and an unconditional dependency would
have taken the JavaScript package down with it.

crates/bibfetch/src/ask.rs

@@ -0,0 +1,137 @@
+//! Asking the registers, which is the part that touches the network.
+//!
+//! Lifted out of the command line so that the Python bindings ask in exactly
+//! the same order, with the same retries and the same backoff. Two
+//! implementations of "which register answers about this identifier" would
+//! drift, and the one nobody runs from a terminal would drift silently.
+
+use std::time::Duration;
+
+use crate::{Error, Record, Source};
+
+/// How to ask: who to say you are, how long to wait, how often to retry.
+#[derive(Debug, Clone)]
+pub struct Ask {
+    /// An address the registers ask for, and answer faster when they get.
+    pub polite: Option<String>,
+    /// How long to wait on any one register.
+    pub timeout: Duration,
+    /// How many times to try a register that is unreachable.
+    pub attempts: u32,
+}
+
+impl Default for Ask {
+    fn default() -> Self {
+        Self { polite: None, timeout: crate::TIMEOUT, attempts: crate::ATTEMPTS }
+    }
+}
+
+impl Ask {
+    /// An agent configured to ask this way.
+    pub fn agent(&self) -> ureq::Agent {
+        ureq::Agent::config_builder()
+            .timeout_global(Some(self.timeout))
+            .user_agent(user_agent(self.polite.as_deref()))
+            .build()
+            .into()
+    }
+}
+
+/// Ask every register that answers about this kind of identifier, in turn,
+/// and stop at the first complete answer.
+///
+/// An incomplete answer is kept in case nothing better comes: a title with no
+/// authors is still more than nothing, and a register being down is only a
+/// failure when every register is. `Ok(None)` means no register here answers
+/// about that kind at all.
+pub fn lookup(
+    agent: &ureq::Agent,
+    id: &str,
+    kind: &str,
+    attempts: u32,
+) -> Result<Option<Record>, Error> {
+    let chain = Source::chain_for_kind(kind);
+    if chain.is_empty() {
+        return Ok(None);
+    }
+    let mut best: Option<Record> = None;
+    let mut last_error = None;
+    for source in chain {
+        match ask(agent, *source, id, kind, attempts) {
+            Ok(record) => {
+                let complete = record.found && record.title.is_some() && !record.authors.is_empty();
+                let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
+                if complete {
+                    return Ok(Some(record));
+                }
+                if better {
+                    best = Some(record);
+                }
+            }
+            Err(why) => last_error = Some(why),
+        }
+    }
+    match best {
+        Some(record) => Ok(Some(record)),
+        None => Err(last_error.unwrap_or_else(|| Error::Unreachable("no register answered".into()))),
+    }
+}
+
+/// Ask a register, retrying a failure that might not be one.
+///
+/// Waits a second before the second attempt and two before the third: long
+/// enough for a register that is rate-limiting, short enough that one which
+/// is simply down does not hold up a pipeline.
+fn ask(
+    agent: &ureq::Agent,
+    source: Source,
+    id: &str,
+    kind: &str,
+    attempts: u32,
+) -> Result<crate::Record, Error> {
+    let mut last = Error::Unreachable("no attempt was made".into());
+
+    for attempt in 0..attempts.max(1) {
+        match fetch(agent, source, id, kind) {
+            Ok(record) => return Ok(record),
+            // A shape this cannot read will not read any better next time.
+            Err(e @ Error::Shape(_)) => return Err(e),
+            Err(e) => last = e,
+        }
+        std::thread::sleep(Duration::from_secs(1 << attempt));
+    }
+    Err(last)
+}
+
+fn fetch(
+    agent: &ureq::Agent,
+    source: Source,
+    id: &str,
+    kind: &str,
+) -> Result<crate::Record, Error> {
+    let response = agent.get(source.url(id)).call();
+    let mut response = match response {
+        Ok(response) => response,
+        // A register that does not have it says so with a status, and that is
+        // an answer rather than a failure.
+        Err(ureq::Error::StatusCode(404)) => return Ok(crate::missing(id, kind, source)),
+        Err(e) => return Err(Error::Unreachable(e.to_string())),
+    };
+    let body = response
+        .body_mut()
+        .read_to_string()
+        .map_err(|e| Error::Unreachable(e.to_string()))?;
+    source.read(id, kind, &body)
+}
+
+/// Who is asking.
+///
+/// The registers ask for an address and answer faster when they get one; a
+/// tool that does not say who it is gets the slow pool.
+fn user_agent(polite: Option<&str>) -> String {
+    let version = env!("CARGO_PKG_VERSION");
+    match polite {
+        Some(address) => format!("bibfetch/{version} (mailto:{address})"),
+        None => format!("bibfetch/{version}"),
+    }
+}

crates/bibfetch/src/lib.rs

@@ -13,6 +13,7 @@
 //! book. There is no register for an ISSN that answers without a key, so an
 //! ISSN is looked up as the journal of a work rather than on its own.
 
+pub mod ask;
 pub mod source;
 
 use std::time::Duration;

crates/bibfetch/src/main.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 use std::process::ExitCode;
 use std::time::Duration;
 
-use bibfetch::{Error, Source};
 use clap::Parser;
 use corpus_core::exit;
 
@@ -65,11 +64,12 @@ fn main() -> ExitCode {
         }
     };
 
-    let agent: ureq::Agent = ureq::Agent::config_builder()
-        .timeout_global(Some(Duration::from_secs(args.timeout)))
-        .user_agent(user_agent(args.polite.as_deref()))
-        .build()
-        .into();
+    let agent = bibfetch::ask::Ask {
+        polite: args.polite.clone(),
+        timeout: Duration::from_secs(args.timeout),
+        attempts: args.attempts,
+    }
+    .agent();
 
     let mut records = Vec::new();
     let mut missing = 0usize;
@@ -86,47 +86,18 @@ fn main() -> ExitCode {
             }
         };
 
-        let chain = Source::chain_for_kind(&kind);
-        if chain.is_empty() {
-            eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers");
-            missing += 1;
-            continue;
-        }
-
-        // Ask the registers in turn and stop at the first complete answer.
-        // An incomplete one is kept in case nothing better comes: a title
-        // with no authors is still more than nothing, and a register being
-        // down is only a failure when every register is.
-        let mut best: Option<bibfetch::Record> = None;
-        let mut last_error = None;
-        for source in chain {
-            match ask(&agent, *source, &id, &kind, args.attempts) {
-                Ok(record) => {
-                    let complete =
-                        record.found && record.title.is_some() && !record.authors.is_empty();
-                    let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
-                    if complete {
-                        best = Some(record);
-                        break;
-                    }
-                    if better {
-                        best = Some(record);
-                    }
-                }
-                Err(why) => last_error = Some(why),
+        match bibfetch::ask::lookup(&agent, &id, &kind, args.attempts) {
+            Ok(None) => {
+                eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers");
+                missing += 1;
             }
-        }
-
-        match best {
-            Some(record) => {
+            Ok(Some(record)) => {
                 if !record.found {
                     missing += 1;
                 }
                 records.push(record);
             }
-            None => {
-                let why = last_error
-                    .map_or_else(|| "no register answered".to_string(), |e| e.to_string());
+            Err(why) => {
                 eprintln!("bibfetch: {id}: {why}");
                 return ExitCode::from(exit::UNAVAILABLE);
             }
@@ -143,64 +114,11 @@ fn main() -> ExitCode {
     })
 }
 
-/// Ask a register, retrying a failure that might not be one.
-///
-/// Waits a second before the second attempt and two before the third: long
-/// enough for a register that is rate-limiting, short enough that one which
-/// is simply down does not hold up a pipeline.
-fn ask(
-    agent: &ureq::Agent,
-    source: Source,
-    id: &str,
-    kind: &str,
-    attempts: u32,
-) -> Result<bibfetch::Record, Error> {
-    let mut last = Error::Unreachable("no attempt was made".into());
 
-    for attempt in 0..attempts.max(1) {
-        match fetch(agent, source, id, kind) {
-            Ok(record) => return Ok(record),
-            // A shape this cannot read will not read any better next time.
-            Err(e @ Error::Shape(_)) => return Err(e),
-            Err(e) => last = e,
-        }
-        std::thread::sleep(Duration::from_secs(1 << attempt));
-    }
-    Err(last)
-}
 
-fn fetch(
-    agent: &ureq::Agent,
-    source: Source,
-    id: &str,
-    kind: &str,
-) -> Result<bibfetch::Record, Error> {
-    let response = agent.get(source.url(id)).call();
-    let mut response = match response {
-        Ok(response) => response,
-        // A register that does not have it says so with a status, and that is
-        // an answer rather than a failure.
-        Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)),
-        Err(e) => return Err(Error::Unreachable(e.to_string())),
-    };
-    let body = response
-        .body_mut()
-        .read_to_string()
-        .map_err(|e| Error::Unreachable(e.to_string()))?;
-    source.read(id, kind, &body)
-}
 
-/// Who is asking.
-///
-/// The registers ask for an address and answer faster when they get one; a
-/// tool that does not say who it is gets the slow pool.
-fn user_agent(polite: Option<&str>) -> String {
-    let version = env!("CARGO_PKG_VERSION");
-    match polite {
-        Some(address) => format!("bibfetch/{version} (mailto:{address})"),
-        None => format!("bibfetch/{version}"),
-    }
-}
+
+
 
 /// The identifier and its kind out of one record.
 fn read_identifier(line: &str) -> Result<(String, String), String> {

crates/bindings/Cargo.toml

@@ -30,6 +30,7 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 booksniff.workspace = true
 bibclean.workspace = true
+colophon.workspace = true
 chunkmd.workspace = true
 citefind.workspace = true
 corpus-core.workspace = true
@@ -47,6 +48,11 @@ textkit.workspace = true
 
 pyo3 = { workspace = true, optional = true }
 
+# Only the Python build gets this. bibfetch is the one tool here that opens a
+# socket, and its HTTP client does not build for wasm32 — so a dependency that
+# is unconditional would take the JavaScript package down with it.
+bibfetch = { workspace = true, optional = true }
+
 # The JavaScript build. Only ever compiled for a wasm32 target; on any other
 # the module is compiled out and these sit unused.
 wasm-bindgen = { workspace = true, optional = true }
@@ -56,7 +62,7 @@ serde-wasm-bindgen = { workspace = true, optional = true }
 # Only maturin turns this on. pyo3's extension-module feature leaves the
 # Python symbols to be resolved by the interpreter at load time, so an
 # ordinary `cargo build` would fail to link if it were on by default.
-python = ["dep:pyo3"]
+python = ["dep:pyo3", "dep:bibfetch"]
 # The C ABI. Off by default so an ordinary build does not also export a
 # hundred symbols nobody asked for.
 capi = []

crates/bindings/python/catena/__init__.py

@@ -20,6 +20,7 @@ from ._catena import (
     compose_subjects,
     count_tokens,
     cue_windows,
+    fetch_identifier,
     find_citations,
     fuse_hits,
     identify,
@@ -33,12 +34,14 @@ from ._catena import (
     pack_cues,
     parse_identifier,
     parse_subtitles,
+    read_colophon,
     record_problems,
     record_severity,
     same_work,
     scrub,
     segment_cues,
     series_of,
+    shelve,
     split_collection,
     suggest_author,
     title_case,
@@ -56,6 +59,7 @@ __all__ = [
     "compose_subjects",
     "count_tokens",
     "cue_windows",
+    "fetch_identifier",
     "find_citations",
     "fuse_hits",
     "identify",
@@ -69,12 +73,14 @@ __all__ = [
     "pack_cues",
     "parse_identifier",
     "parse_subtitles",
+    "read_colophon",
     "record_problems",
     "record_severity",
     "same_work",
     "scrub",
     "segment_cues",
     "series_of",
+    "shelve",
     "split_collection",
     "suggest_author",
     "title_case",

crates/bindings/python/catena/__init__.pyi

@@ -75,3 +75,30 @@ def valid_isbn(text: str) -> bool: ...
 def isbn_from_text(text: str) -> str | None: ...
 def isbn_from_filename(name: str) -> str | None: ...
 def same_work(ours: str, theirs: str) -> bool: ...
+
+def read_colophon(
+    text: str, pages: int = ..., shelf: str | None = ...
+) -> dict[str, Any]:
+    """What a book prints about itself in its opening pages.
+
+    `shelf` is a closed list of headings to file the printed subjects onto,
+    one per line, optionally `Heading = term, term`.
+    """
+    ...
+
+def shelve(shelf: str, subjects: list[str]) -> tuple[list[str], list[str]]:
+    """File subject headings onto a shelf. Returns (shelved, unshelved)."""
+    ...
+
+def fetch_identifier(
+    id: str,
+    kind: str = ...,
+    polite: str | None = ...,
+    timeout: int = ...,
+    attempts: int = ...,
+) -> dict[str, Any] | None:
+    """Ask the register that assigns an identifier what it knows about one.
+
+    Blocks on the network. `None` when no register answers about that kind.
+    """
+    ...

crates/bindings/src/python.rs

@@ -431,5 +431,81 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(merge_citation_reply, module)?)?;
     module.add_function(wrap_pyfunction!(series_of, module)?)?;
     module.add_function(wrap_pyfunction!(parse_identifier, module)?)?;
+    module.add_function(wrap_pyfunction!(read_colophon, module)?)?;
+    module.add_function(wrap_pyfunction!(shelve, module)?)?;
+    module.add_function(wrap_pyfunction!(fetch_identifier, module)?)?;
     Ok(())
 }
+
+// ───────────────────── what a book says about itself ──────────────────────
+
+/// Read what a book prints about itself in its opening pages.
+///
+/// `shelf` is the closed list of headings to file the printed subjects onto,
+/// in the format `colophon --shelf` takes: one heading per line, optionally
+/// `Heading = term, term`. Without it the subjects come back as the book
+/// printed them and nothing is filed.
+#[pyfunction]
+#[pyo3(signature = (text, pages = colophon::PAGES, shelf = None))]
+fn read_colophon<'py>(
+    py: Python<'py>,
+    text: &str,
+    pages: usize,
+    shelf: Option<&str>,
+) -> PyResult<Bound<'py, PyAny>> {
+    if pages == 0 {
+        return Err(PyValueError::new_err("pages = 0 would read nothing"));
+    }
+    let shelf = shelf.map(colophon::shelf::Shelf::parse);
+    to_py(py, &colophon::read_onto(text, pages, shelf.as_ref()))
+}
+
+/// File subject headings onto a shelf without reading a book.
+///
+/// For the headings a catalogue already holds: the same matching the printed
+/// ones get, so a record's existing subjects and its book's agree about what
+/// counts as a heading. Returns `(shelved, unshelved)`.
+#[pyfunction]
+fn shelve(shelf: &str, subjects: Vec<String>) -> (Vec<String>, Vec<String>) {
+    let filed = colophon::shelf::Shelf::parse(shelf).file(&subjects);
+    (filed.on, filed.unfiled)
+}
+
+// ──────────────────────────── the registers ───────────────────────────────
+
+/// Ask the body that assigns an identifier what it knows about one.
+///
+/// Crossref for a DOI, the arXiv for a preprint, OpenLibrary and Google Books
+/// for an ISBN, asked in turn until one gives a complete answer. This is the
+/// only function here that touches the network, and it blocks: call it from a
+/// thread, or with `asyncio.to_thread`.
+///
+/// Returns the record, or `None` when no register here answers about that kind
+/// of identifier. Raises when every register that should answer was
+/// unreachable, because that is a different thing from the book not existing.
+#[pyfunction]
+#[pyo3(signature = (id, kind = "isbn", polite = None, timeout = 20, attempts = 3))]
+fn fetch_identifier<'py>(
+    py: Python<'py>,
+    id: &str,
+    kind: &str,
+    polite: Option<String>,
+    timeout: u64,
+    attempts: u32,
+) -> PyResult<Option<Bound<'py, PyAny>>> {
+    let ask = bibfetch::ask::Ask {
+        polite,
+        timeout: std::time::Duration::from_secs(timeout.max(1)),
+        attempts: attempts.max(1),
+    };
+    let agent = ask.agent();
+    // The GIL is not needed while waiting on a socket, and holding it there
+    // would stop every other thread in the interpreter for the whole request.
+    let found = py
+        .detach(|| bibfetch::ask::lookup(&agent, id, kind, ask.attempts))
+        .map_err(|e| PyValueError::new_err(e.to_string()))?;
+    match found {
+        Some(record) => Ok(Some(to_py(py, &record)?)),
+        None => Ok(None),
+    }
+}