From d47881ef3b4cf8287f0ec72921990f95ffb0fc4e Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 19:19:54 -0400 Subject: [PATCH] idnorm and bibfetch: an identifier read, and a register asked --- Cargo.lock | 29 ++ Cargo.toml | 2 + README.md | 2 + crates/bibclean/src/abstracts.rs | 66 +++- crates/bibfetch/Cargo.toml | 21 ++ crates/bibfetch/README.md | 110 +++++++ crates/bibfetch/src/lib.rs | 136 ++++++++ crates/bibfetch/src/main.rs | 193 ++++++++++++ crates/bibfetch/src/source.rs | 342 +++++++++++++++++++++ crates/bindings/Cargo.toml | 1 + crates/bindings/README.md | 1 + crates/bindings/npm/README.md | 1 + crates/bindings/npm/typecheck.ts | 9 +- crates/bindings/python/catena/__init__.py | 2 + crates/bindings/python/catena/__init__.pyi | 1 + crates/bindings/src/capi.rs | 29 ++ crates/bindings/src/lib.rs | 13 + crates/bindings/src/python.rs | 12 + crates/bindings/src/wasm.rs | 27 ++ crates/idnorm/Cargo.toml | 21 ++ crates/idnorm/README.md | 126 ++++++++ crates/idnorm/src/arxiv.rs | 122 ++++++++ crates/idnorm/src/doi.rs | 129 ++++++++ crates/idnorm/src/issn.rs | 129 ++++++++ crates/idnorm/src/lib.rs | 242 +++++++++++++++ crates/idnorm/src/main.rs | 112 +++++++ docs/contract.md | 3 +- docs/recipes.md | 42 +++ parity/tests/abstracts.rs | 6 +- 29 files changed, 1920 insertions(+), 9 deletions(-) create mode 100644 crates/bibfetch/Cargo.toml create mode 100644 crates/bibfetch/README.md create mode 100644 crates/bibfetch/src/lib.rs create mode 100644 crates/bibfetch/src/main.rs create mode 100644 crates/bibfetch/src/source.rs create mode 100644 crates/idnorm/Cargo.toml create mode 100644 crates/idnorm/README.md create mode 100644 crates/idnorm/src/arxiv.rs create mode 100644 crates/idnorm/src/doi.rs create mode 100644 crates/idnorm/src/issn.rs create mode 100644 crates/idnorm/src/lib.rs create mode 100644 crates/idnorm/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 176742f..d0921d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,6 +132,20 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "bibfetch" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "idnorm", + "schemars", + "serde", + "serde_json", + "ureq", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -220,6 +234,7 @@ dependencies = [ "corpus-core", "cuekit", "cuepack", + "idnorm", "mdscrub", "mdsplit", "pyo3", @@ -785,6 +800,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "idnorm" +version = "0.1.0" +dependencies = [ + "bibclean", + "catena-cli", + "clap", + "corpus-core", + "fancy-regex 0.16.2", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "indexmap" version = "2.14.2" diff --git a/Cargo.toml b/Cargo.toml index 665a27c..cae7b78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,8 @@ avplan = { path = "crates/avplan", version = "0.1.0" } pdfgutter = { path = "crates/pdfgutter", version = "0.1.0" } catena = { path = "crates/bindings", version = "0.1.0" } seriesof = { path = "crates/seriesof", version = "0.1.0" } +idnorm = { path = "crates/idnorm", version = "0.1.0" } +bibfetch = { path = "crates/bibfetch", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } diff --git a/README.md b/README.md index 52d164b..0e86e85 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ Each tool installs by name, so you can take one without taking the rest. | [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change | | [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary | | [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is | +| [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked | +| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier | | [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out | | [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it | | [`chunkmd`](crates/chunkmd/README.md) | Markdown into passages that carry their page, chapter and role | diff --git a/crates/bibclean/src/abstracts.rs b/crates/bibclean/src/abstracts.rs index 29f2f62..448cfe7 100644 --- a/crates/bibclean/src/abstracts.rs +++ b/crates/bibclean/src/abstracts.rs @@ -9,10 +9,56 @@ //! out is the original with its punctuation and spacing intact, because those //! travelled attached to the words. -use std::collections::HashMap; - /// An abstract as its catalogue publishes it: word to the positions it holds. -pub type Inverted = HashMap>; +/// +/// A list of pairs rather than a map, so that the order the catalogue listed +/// the words in survives. It matters exactly once — when two words claim the +/// same position — and a map would settle that tie by hash order, which is a +/// different answer on every run. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Inverted(pub Vec<(String, Vec)>); + +impl Inverted { + /// The pairs, in the order they were written. + pub fn pairs(&self) -> &[(String, Vec)] { + &self.0 + } +} + +impl FromIterator<(String, Vec)> for Inverted { + fn from_iter)>>(pairs: T) -> Self { + Inverted(pairs.into_iter().collect()) + } +} + +impl<'de> serde::Deserialize<'de> for Inverted { + /// Read the object entry by entry rather than through a map, which is + /// what keeps the document's own order. + fn deserialize>(deserializer: D) -> Result { + struct InOrder; + + impl<'de> serde::de::Visitor<'de> for InOrder { + type Value = Inverted; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an object of words to the positions they hold") + } + + fn visit_map>( + self, + mut map: M, + ) -> Result { + let mut pairs = Vec::with_capacity(map.size_hint().unwrap_or(0)); + while let Some((word, at)) = map.next_entry::>()? { + pairs.push((word, at)); + } + Ok(Inverted(pairs)) + } + } + + deserializer.deserialize_map(InOrder) + } +} /// Put an inverted abstract back in order. /// @@ -24,6 +70,7 @@ pub type Inverted = HashMap>; /// not the exact offsets. pub fn from_inverted(index: &Inverted) -> Option { let mut positions: Vec<(i64, &str)> = index + .pairs() .iter() .flat_map(|(word, at)| at.iter().map(move |at| (*at, word.as_str()))) .collect(); @@ -54,6 +101,17 @@ mod tests { .collect() } + #[test] + fn two_words_at_one_position_keep_the_order_they_were_listed_in() { + // The only place the catalogue's own ordering matters, and the only + // reason this is a list of pairs rather than a map. + let index = inverted(&[("alpha", &[1]), ("beta", &[1])]); + assert_eq!(from_inverted(&index).as_deref(), Some("alpha beta")); + + let other_way = inverted(&[("beta", &[1]), ("alpha", &[1])]); + assert_eq!(from_inverted(&other_way).as_deref(), Some("beta alpha")); + } + #[test] fn words_come_back_in_the_order_their_positions_give() { let index = inverted(&[("obligation", &[1]), ("The", &[0]), ("ran", &[2])]); @@ -77,7 +135,7 @@ mod tests { #[test] fn an_index_with_nothing_in_it_is_no_abstract() { - assert_eq!(from_inverted(&Inverted::new()), None); + assert_eq!(from_inverted(&Inverted::default()), None); assert_eq!(from_inverted(&inverted(&[("word", &[])])), None); } diff --git a/crates/bibfetch/Cargo.toml b/crates/bibfetch/Cargo.toml new file mode 100644 index 0000000..3559385 --- /dev/null +++ b/crates/bibfetch/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "bibfetch" +description = "What a catalogue says about an identifier, asked of the body that assigns it." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +idnorm.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +ureq.workspace = true + +[lints] +workspace = true diff --git a/crates/bibfetch/README.md b/crates/bibfetch/README.md new file mode 100644 index 0000000..a4065f3 --- /dev/null +++ b/crates/bibfetch/README.md @@ -0,0 +1,110 @@ +# bibfetch + +What a catalogue says about an identifier, asked of the body that assigns it. + +Part of [catena](../../README.md). + +## What it does + +[`idnorm`](../idnorm/README.md) can tell you a DOI is well formed. It cannot +tell you the paper exists — that is not a fact about the string, it is a fact +about the world, and the only way to learn it is to ask somebody who keeps a +register. + +So this asks, and that is the only thing it does. + +## When you would use it + +- **Auditing a bibliography.** A well-formed DOI that no register has is a + mistyped one, and a catalogue can carry it for years without noticing. +- **Filling in a reference** you have only the identifier for. +- **Before citing something**, to check the year and the authors are what you + think they are. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena bibfetch +``` + +## Example + +``` +$ printf '10.1093/mind/LIX.236.433\narXiv:2401.01234\n978-0-306-40615-7\n' | idnorm | bibfetch --polite you@example.org +{"v":1,"id":"10.1093/mind/lix.236.433","kind":"doi","found":true,"source":"crossref","title":"I.—COMPUTING MACHINERY AND INTELLIGENCE","authors":["A. M. TURING"],"year":1950,"container":"Mind"} +{"v":1,"id":"2401.01234","kind":"arxiv","found":true,"source":"arxiv","title":"Mixture cure semiparametric additive hazard models under partly interval censoring -- a penalized likelihood approach","authors":["Jinqing Li","Jun Ma"],"year":2024,"container":"arXiv","summary":"Survival analysis can sometimes involve individuals who will not experience…"} +{"v":1,"id":"9780306406157","kind":"isbn","found":true,"source":"openlibrary","title":"Error-correction coding for digital communications","year":1981,"container":"Plenum Press"} +``` + +The ones nobody has: + +``` +$ printf '10.9999/definitely-not-a-real-doi\n' | idnorm | bibfetch +{"v":1,"id":"10.9999/definitely-not-a-real-doi","kind":"doi","found":false,"source":"crossref"} +$ echo $? +1 +``` + +Absence is the useful answer here. It is what tells a catalogue that a DOI it +has been carrying for three years was mistyped on the way in. + +Auditing a whole bibliography: + +```sh +idnorm --quiet refs.txt | bibfetch --polite you@example.org \ + | jq -r 'select(.found | not) | "missing: \(.id)"' +``` + +## Options + +| Option | Effect | +|---|---| +| `--polite ` | An address to put in the request. The registers ask for one and answer faster when they get it | +| `--timeout ` | How long to wait on a register. Default `20` | +| `--attempts ` | How many times to ask before believing a failure. Default `3` | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Every identifier was found | +| `1` | At least one was not | +| `64` | The command line was wrong | +| `65` | The input was not identifier records | +| `69` | A register could not be reached | +| `74` | The file could not be read | + +Exit `1` is an answer — the register replied and does not have it. Exit `69` +is a failure — nobody replied, and the identifier may be perfectly good. + +## The registers + +| Kind | Asked of | Why that one | +|---|---|---| +| `doi` | Crossref | It assigns most scholarly DOIs | +| `arxiv` | the arXiv | Its own register of preprints | +| `isbn` | OpenLibrary | Books, without a key | + +An **ISSN** has no register that answers without a key, so it is not looked up +on its own. A journal's ISSN comes back on the works published in it instead. + +A failure is retried, waiting a second and then two: long enough for a +register that is rate-limiting, short enough that one which is simply down +does not hold up a pipeline. An answer this cannot read is not retried, +because it will not read any better next time. + +## Reaching the network + +This and [`emb`](../emb/README.md) are the only two tools here that do. Both +say so in `--help`, and for both it is the whole of what they do rather than +something that happens on the way to something else — which is the condition +under which [the contract](../../docs/contract.md) allows it at all. + +Nothing is cached, nothing is stored, and nothing is sent but the identifier +and the address you pass to `--polite`. + +## See also + +- [`idnorm`](../idnorm/README.md) — reading the identifier in the first place. +- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record. diff --git a/crates/bibfetch/src/lib.rs b/crates/bibfetch/src/lib.rs new file mode 100644 index 0000000..347fe8f --- /dev/null +++ b/crates/bibfetch/src/lib.rs @@ -0,0 +1,136 @@ +//! What a catalogue says about an identifier. +//! +//! [`idnorm`] can tell you a DOI is well formed. It cannot tell you the paper +//! exists, because that is not a fact about the string — it is a fact about +//! the world, and the only way to learn it is to ask somebody who keeps a +//! register. +//! +//! So this asks. It is the only thing it does, which is the condition under +//! which a filter here is allowed to touch the network at all. +//! +//! Three registers, one per kind of identifier, each the body that assigns +//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a +//! book. There is no register for an ISSN that answers without a key, so an +//! ISSN is looked up as the journal of a work rather than on its own. + +pub mod source; + +use std::time::Duration; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub use source::Source; + +/// How long to wait on a register before giving up. +pub const TIMEOUT: Duration = Duration::from_secs(20); + +/// How many times to ask before believing the answer. +pub const ATTEMPTS: u32 = 3; + +/// What a register said about one identifier. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Record { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// The identifier asked about, normalised. + pub id: String, + /// Which kind it is. + pub kind: String, + /// Whether the register has it. This is the answer the string alone + /// cannot give. + pub found: bool, + /// Which register answered. + pub source: String, + /// The title, where there is one. + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + /// The authors, in the order the register lists them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub authors: Vec, + /// The year of publication. + #[serde(skip_serializing_if = "Option::is_none")] + pub year: Option, + /// The journal, publisher or archive it appeared under. + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, + /// The abstract, where the register carries one. + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Record { + fn version(&self) -> u32 { + self.v + } +} + +/// What can go wrong asking. +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + /// The register could not be reached, or refused. + Unreachable(String), + /// It answered with something this could not read. + Shape(String), + /// Nothing here knows where to ask about this kind. + NoRegister(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Unreachable(why) => write!(f, "{why}"), + Error::Shape(why) => write!(f, "the answer could not be read: {why}"), + Error::NoRegister(kind) => { + write!(f, "no register here answers about {kind} identifiers") + } + } + } +} + +impl std::error::Error for Error {} + +/// A record for an identifier the register does not have. +/// +/// Absence is an answer, and a useful one: it is what tells a catalogue that +/// a well-formed DOI it has been carrying for three years was mistyped. +pub fn missing(id: &str, kind: &str, source: Source) -> Record { + Record { + v: corpus_core::V, + id: id.to_string(), + kind: kind.to_string(), + found: false, + source: source.name().to_string(), + title: None, + authors: Vec::new(), + year: None, + container: None, + summary: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_identifier_nobody_has_is_still_an_answer() { + let record = missing("10.1000/nope", "doi", Source::Crossref); + assert!(!record.found); + assert_eq!(record.source, "crossref"); + assert_eq!(record.title, None); + } + + #[test] + fn a_record_that_found_nothing_serialises_without_empty_fields() { + let json = serde_json::to_string(&missing("x", "doi", Source::Crossref)).expect("json"); + assert!(!json.contains("title"), "{json}"); + assert!(!json.contains("authors"), "{json}"); + assert!(json.contains("\"found\":false"), "{json}"); + } +} diff --git a/crates/bibfetch/src/main.rs b/crates/bibfetch/src/main.rs new file mode 100644 index 0000000..7cc610c --- /dev/null +++ b/crates/bibfetch/src/main.rs @@ -0,0 +1,193 @@ +//! What a catalogue says about an identifier. + +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Duration; + +use bibfetch::{Error, Source}; +use clap::Parser; +use corpus_core::exit; + +/// Ask the body that assigns an identifier what it knows about one. +/// +/// `idnorm` can say a DOI is well formed. Only a register can say the paper +/// exists — that is a fact about the world rather than about the string, and +/// it is what catches a well-formed DOI a catalogue has been carrying wrongly +/// for three years. +/// +/// Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a book. This +/// is the only tool here besides `emb` that touches the network, and it is the +/// whole of what it does. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 every identifier was found, 1 at least one was not, +65 the input was not identifiers, 69 a register could not be reached, +74 the file could not be read. + +Example: + + idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title' + idnorm --kind doi refs.txt | bibfetch --polite you@example.org" +)] +struct Args { + /// Identifier records to look up, as `idnorm` writes them. Reads stdin + /// when absent. + path: Option, + + /// An address to put in the request, which the registers ask for and + /// answer faster when they get. + #[arg(long, value_name = "EMAIL")] + polite: Option, + + /// How long to wait on a register, in seconds. + #[arg(long, default_value_t = bibfetch::TIMEOUT.as_secs(), value_name = "SECONDS")] + timeout: u64, + + /// How many times to ask before believing a failure. + #[arg(long, default_value_t = bibfetch::ATTEMPTS, value_name = "COUNT")] + attempts: u32, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("bibfetch: {why}"); + return ExitCode::from(code); + } + }; + + let agent: ureq::Agent = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(args.timeout))) + .user_agent(user_agent(args.polite.as_deref())) + .build() + .into(); + + let mut records = Vec::new(); + let mut missing = 0usize; + + for (number, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let (id, kind) = match read_identifier(line) { + Ok(pair) => pair, + Err(why) => { + eprintln!("bibfetch: line {}: {why}", number + 1); + return ExitCode::from(exit::DATA); + } + }; + + let Some(source) = Source::for_kind(&kind) else { + eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers"); + missing += 1; + continue; + }; + + match ask(&agent, source, &id, &kind, args.attempts) { + Ok(record) => { + if !record.found { + missing += 1; + } + records.push(record); + } + Err(why) => { + eprintln!("bibfetch: {id}: {why}"); + return ExitCode::from(exit::UNAVAILABLE); + } + } + } + + let wrote = catena_cli::stream("bibfetch", &records); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + ExitCode::from(match missing { + 0 => exit::OK, + _ => exit::NO, + }) +} + +/// Ask a register, retrying a failure that might not be one. +/// +/// Waits a second before the second attempt and two before the third: long +/// enough for a register that is rate-limiting, short enough that one which +/// is simply down does not hold up a pipeline. +fn ask( + agent: &ureq::Agent, + source: Source, + id: &str, + kind: &str, + attempts: u32, +) -> Result { + let mut last = Error::Unreachable("no attempt was made".into()); + + for attempt in 0..attempts.max(1) { + match fetch(agent, source, id, kind) { + Ok(record) => return Ok(record), + // A shape this cannot read will not read any better next time. + Err(e @ Error::Shape(_)) => return Err(e), + Err(e) => last = e, + } + std::thread::sleep(Duration::from_secs(1 << attempt)); + } + Err(last) +} + +fn fetch( + agent: &ureq::Agent, + source: Source, + id: &str, + kind: &str, +) -> Result { + let response = agent.get(source.url(id)).call(); + let mut response = match response { + Ok(response) => response, + // A register that does not have it says so with a status, and that is + // an answer rather than a failure. + Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)), + Err(e) => return Err(Error::Unreachable(e.to_string())), + }; + let body = response + .body_mut() + .read_to_string() + .map_err(|e| Error::Unreachable(e.to_string()))?; + source.read(id, kind, &body) +} + +/// Who is asking. +/// +/// The registers ask for an address and answer faster when they get one; a +/// tool that does not say who it is gets the slow pool. +fn user_agent(polite: Option<&str>) -> String { + let version = env!("CARGO_PKG_VERSION"); + match polite { + Some(address) => format!("bibfetch/{version} (mailto:{address})"), + None => format!("bibfetch/{version}"), + } +} + +/// The identifier and its kind out of one record. +fn read_identifier(line: &str) -> Result<(String, String), String> { + let value: serde_json::Value = + serde_json::from_str(line).map_err(|e| format!("not a record: {e}"))?; + let id = value + .get("id") + .and_then(serde_json::Value::as_str) + .ok_or("no id")? + .to_string(); + let kind = value + .get("kind") + .and_then(serde_json::Value::as_str) + .ok_or("no kind")? + .to_string(); + Ok((id, kind)) +} diff --git a/crates/bibfetch/src/source.rs b/crates/bibfetch/src/source.rs new file mode 100644 index 0000000..7abd058 --- /dev/null +++ b/crates/bibfetch/src/source.rs @@ -0,0 +1,342 @@ +//! The registers, and how to read what each one says. +//! +//! Each is the body that assigns the identifiers it answers about, so each is +//! the one place the answer is not a guess. The shapes they reply in have +//! nothing in common, which is most of what this module is. + +use serde::Deserialize; + +use crate::{Error, Record}; + +/// A register that can be asked about an identifier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Source { + /// Crossref, which assigns most scholarly DOIs. + Crossref, + /// The arXiv's own register of preprints. + Arxiv, + /// OpenLibrary, for books by ISBN. + OpenLibrary, +} + +impl Source { + /// What it is called. + pub fn name(self) -> &'static str { + match self { + Source::Crossref => "crossref", + Source::Arxiv => "arxiv", + Source::OpenLibrary => "openlibrary", + } + } + + /// Which register answers about a kind of identifier. + pub fn for_kind(kind: &str) -> Option { + match kind { + "doi" => Some(Source::Crossref), + "arxiv" => Some(Source::Arxiv), + "isbn" => Some(Source::OpenLibrary), + _ => None, + } + } + + /// Where to ask about one identifier. + pub fn url(self, id: &str) -> String { + match self { + Source::Crossref => format!("https://api.crossref.org/works/{}", encode(id)), + Source::Arxiv => format!( + "https://export.arxiv.org/api/query?id_list={}&max_results=1", + encode(id) + ), + Source::OpenLibrary => { + format!("https://openlibrary.org/isbn/{}.json", encode(id)) + } + } + } + + /// Read what it said into a record. + pub fn read(self, id: &str, kind: &str, body: &str) -> Result { + match self { + Source::Crossref => crossref(id, kind, body), + Source::Arxiv => arxiv(id, kind, body), + Source::OpenLibrary => open_library(id, kind, body), + } + } +} + +/// Percent-encode what a path or a query would otherwise read as structure. +/// +/// A DOI suffix may hold a slash, a question mark or a hash, and every one of +/// those means something else in a URL. +fn encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + out.push(byte as char) + } + byte => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +/// A record with the fields nothing found fills in. +fn blank(id: &str, kind: &str, source: Source) -> Record { + let mut record = crate::missing(id, kind, source); + record.found = true; + record +} + +#[derive(Deserialize)] +struct CrossrefReply { + message: CrossrefWork, +} + +#[derive(Deserialize)] +struct CrossrefWork { + #[serde(default)] + title: Vec, + #[serde(default)] + author: Vec, + #[serde(default, rename = "container-title")] + container_title: Vec, + #[serde(default)] + issued: Option, + #[serde(default, rename = "abstract")] + summary: Option, +} + +#[derive(Deserialize)] +struct CrossrefAuthor { + #[serde(default)] + given: Option, + #[serde(default)] + family: Option, + #[serde(default)] + name: Option, +} + +#[derive(Deserialize)] +struct CrossrefDate { + #[serde(default, rename = "date-parts")] + date_parts: Vec>, +} + +fn crossref(id: &str, kind: &str, body: &str) -> Result { + let reply: CrossrefReply = + serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; + let work = reply.message; + + let mut record = blank(id, kind, Source::Crossref); + record.title = work.title.into_iter().next(); + record.container = work.container_title.into_iter().next(); + record.year = work + .issued + .and_then(|issued| issued.date_parts.into_iter().next()) + .and_then(|parts| parts.into_iter().next()); + record.authors = work + .author + .into_iter() + .filter_map(|author| match (author.given, author.family, author.name) { + (Some(given), Some(family), _) => Some(format!("{given} {family}")), + (None, Some(family), _) => Some(family), + (_, _, Some(name)) => Some(name), + _ => None, + }) + .collect(); + // Crossref's abstracts arrive as JATS markup. + record.summary = work.summary.map(|summary| strip_tags(&summary)); + Ok(record) +} + +/// The arXiv answers in Atom, which is XML. Reading it properly would mean a +/// parser; what is wanted is four fields, and they are each one tag. +fn arxiv(id: &str, kind: &str, body: &str) -> Result { + // An empty feed means no such preprint, which the API signals by omitting + // the entry rather than by a status code. + let Some(entry) = between(body, "", "") else { + return Ok(crate::missing(id, kind, Source::Arxiv)); + }; + + let mut record = blank(id, kind, Source::Arxiv); + record.title = between(entry, "", "").map(collapse); + record.summary = between(entry, "", "").map(collapse); + record.container = Some("arXiv".to_string()); + record.year = between(entry, "", "") + .and_then(|published| published.get(..4)) + .and_then(|year| year.parse().ok()); + + let mut authors = Vec::new(); + let mut rest = entry; + while let Some(author) = between(rest, "", "") { + if let Some(name) = between(author, "", "") { + authors.push(collapse(name)); + } + let Some(at) = rest.find("") else { + break; + }; + rest = &rest[at + "".len()..]; + } + record.authors = authors; + Ok(record) +} + +#[derive(Deserialize)] +struct OpenLibraryBook { + #[serde(default)] + title: Option, + #[serde(default)] + publish_date: Option, + #[serde(default)] + publishers: Vec, +} + +fn open_library(id: &str, kind: &str, body: &str) -> Result { + let book: OpenLibraryBook = + serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?; + + let mut record = blank(id, kind, Source::OpenLibrary); + record.title = book.title; + record.container = book.publishers.into_iter().next(); + // The date is free text — "1999", "March 1999", "1999-03-01". + record.year = book.publish_date.as_deref().and_then(year_in); + Ok(record) +} + +/// The first four-digit year in a string. +fn year_in(text: &str) -> Option { + let bytes: Vec = text.chars().collect(); + for window in bytes.windows(4) { + if window.iter().all(char::is_ascii_digit) { + let year: i64 = window.iter().collect::().parse().ok()?; + if (1000..=2999).contains(&year) { + return Some(year); + } + } + } + None +} + +/// What sits between two markers, the first time they appear in order. +fn between<'a>(text: &'a str, open: &str, close: &str) -> Option<&'a str> { + let start = text.find(open)? + open.len(); + let end = text[start..].find(close)? + start; + Some(&text[start..end]) +} + +/// Runs of whitespace down to one space, trimmed. +fn collapse(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +/// Text with its tags removed. +/// +/// Crossref hands back JATS — `` and friends — which is markup for a +/// typesetter rather than anything a reader wants. +fn strip_tags(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut inside = false; + for c in text.chars() { + match c { + '<' => inside = true, + '>' => inside = false, + c if !inside => out.push(c), + _ => {} + } + } + collapse(&out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_kind_goes_to_the_body_that_assigns_it() { + assert_eq!(Source::for_kind("doi"), Some(Source::Crossref)); + assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv)); + assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary)); + // No register answers about an ISSN without a key. + assert_eq!(Source::for_kind("issn"), None); + } + + #[test] + fn a_doi_suffix_cannot_change_the_shape_of_the_url() { + let url = Source::Crossref.url("10.1000/a?b#c/d"); + assert!(url.ends_with("10.1000%2Fa%3Fb%23c%2Fd"), "{url}"); + assert_eq!(url.matches('?').count(), 0); + } + + #[test] + fn crossref_is_read_into_the_fields_that_matter() { + let body = r#"{"message":{"title":["On Computable Numbers"], + "author":[{"given":"Alan","family":"Turing"},{"family":"Church"},{"name":"A Committee"}], + "container-title":["Proceedings"], + "issued":{"date-parts":[[1936,11,12]]}, + "abstract":"A paragraph."}}"#; + let record = crossref("10.1", "doi", body).expect("a record"); + assert!(record.found); + assert_eq!(record.title.as_deref(), Some("On Computable Numbers")); + assert_eq!(record.authors, ["Alan Turing", "Church", "A Committee"]); + assert_eq!(record.year, Some(1936)); + assert_eq!(record.container.as_deref(), Some("Proceedings")); + assert_eq!(record.summary.as_deref(), Some("A paragraph.")); + } + + #[test] + fn a_crossref_reply_missing_everything_is_still_a_record() { + let record = crossref("10.1", "doi", r#"{"message":{}}"#).expect("a record"); + assert!(record.found); + assert_eq!(record.title, None); + assert!(record.authors.is_empty()); + } + + #[test] + fn an_arxiv_feed_with_no_entry_means_no_such_preprint() { + let record = arxiv("2401.01234", "arxiv", "").expect("a record"); + assert!(!record.found); + } + + #[test] + fn an_arxiv_entry_is_read_out_of_its_atom() { + let body = "A\n Title\ + Some words. \ + 2024-01-02T00:00:00Z\ + Ada Lovelace\ + Charles Babbage"; + let record = arxiv("2401.01234", "arxiv", body).expect("a record"); + assert!(record.found); + assert_eq!(record.title.as_deref(), Some("A Title")); + assert_eq!(record.summary.as_deref(), Some("Some words.")); + assert_eq!(record.year, Some(2024)); + assert_eq!(record.authors, ["Ada Lovelace", "Charles Babbage"]); + } + + #[test] + fn openlibrary_gives_up_a_year_from_whatever_it_wrote() { + for (date, year) in [ + ("1999", Some(1999)), + ("March 1999", Some(1999)), + ("1999-03-01", Some(1999)), + ("n.d.", None), + ] { + let body = format!(r#"{{"title":"A Book","publish_date":"{date}"}}"#); + let record = open_library("1", "isbn", &body).expect("a record"); + assert_eq!(record.year, year, "{date}"); + } + } + + #[test] + fn an_answer_that_is_not_what_it_should_be_says_so() { + assert!(matches!( + crossref("10.1", "doi", "not json"), + Err(Error::Shape(_)) + )); + } + + #[test] + fn markup_is_not_part_of_an_abstract() { + assert_eq!(strip_tags("One two"), "One two"); + assert_eq!(strip_tags("plain"), "plain"); + } +} diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index a7064dc..487793d 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -31,6 +31,7 @@ crate-type = ["cdylib", "rlib"] booksniff.workspace = true chunkmd.workspace = true corpus-core.workspace = true +idnorm.workspace = true cuekit.workspace = true cuepack.workspace = true mdscrub.workspace = true diff --git a/crates/bindings/README.md b/crates/bindings/README.md index 8589261..657ed02 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -68,6 +68,7 @@ except ValueError as why: | `fuse_hits` | [`rrf`](../rrf/README.md) | | `identify` | [`booksniff`](../booksniff/README.md) | | `series_of` | [`seriesof`](../seriesof/README.md) | +| `parse_identifier` | [`idnorm`](../idnorm/README.md) | The tools that are not here are the ones a binding would gain nothing over a subprocess for: they read files, shell out to another program, or talk to a diff --git a/crates/bindings/npm/README.md b/crates/bindings/npm/README.md index 1ab762c..139feba 100644 --- a/crates/bindings/npm/README.md +++ b/crates/bindings/npm/README.md @@ -49,6 +49,7 @@ try { | `fuseHits` | Several ranked lists into one | | `identify` | What a file is, from its first bytes | | `seriesOf` | The series a book belongs to, and which volume this is | +| `parseIdentifier` | A DOI, ISSN, ISBN or arXiv id, read and checked | The tools that read files or call another program are not here: a browser has neither, and a binding that pretends otherwise fails at run time rather than diff --git a/crates/bindings/npm/typecheck.ts b/crates/bindings/npm/typecheck.ts index 3d3fa16..5cadbea 100644 --- a/crates/bindings/npm/typecheck.ts +++ b/crates/bindings/npm/typecheck.ts @@ -3,8 +3,8 @@ // check, and nothing else would notice. import { chunkMarkdown, scrub, countTokens, parseSubtitles, packCues, - splitCollection, fuseHits, identify, seriesOf, - type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, + splitCollection, fuseHits, identify, seriesOf, parseIdentifier, + type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, type Identifier, } from "./dist/web/catena.js"; const chunks: Chunk[] = chunkMarkdown("# One\n\nbody\n", 512, 64, 50, 768); @@ -39,4 +39,7 @@ const proposal: SeriesProposal | undefined = seriesOf("The Philokalia Vol 3"); const volume: number | undefined = proposal?.volume; seriesOf("Aquinas (Companion)", "Cambridge"); -export const checked = [volume, page, chapter, flags, clean, tokens, start, packed, slug, fused, kind, notes]; +const ident: Identifier | undefined = parseIdentifier("10.1000/182"); +const checksum: boolean | null | undefined = ident?.checksum; + +export const checked = [volume, checksum, page, chapter, flags, clean, tokens, start, packed, slug, fused, kind, notes]; diff --git a/crates/bindings/python/catena/__init__.py b/crates/bindings/python/catena/__init__.py index c72d8cb..6025e83 100644 --- a/crates/bindings/python/catena/__init__.py +++ b/crates/bindings/python/catena/__init__.py @@ -17,6 +17,7 @@ from ._catena import ( fuse_hits, identify, pack_cues, + parse_identifier, parse_subtitles, scrub, series_of, @@ -30,6 +31,7 @@ __all__ = [ "fuse_hits", "identify", "pack_cues", + "parse_identifier", "parse_subtitles", "scrub", "series_of", diff --git a/crates/bindings/python/catena/__init__.pyi b/crates/bindings/python/catena/__init__.pyi index 482c74c..8e462a9 100644 --- a/crates/bindings/python/catena/__init__.pyi +++ b/crates/bindings/python/catena/__init__.pyi @@ -30,6 +30,7 @@ def fuse_hits( limit: int | None = ..., ) -> list[dict[str, Any]]: ... def identify(name: str, data: bytes) -> dict[str, Any]: ... +def parse_identifier(text: str) -> dict[str, Any] | None: ... def series_of( title: str, author: str | None = ... ) -> dict[str, Any] | None: ... diff --git a/crates/bindings/src/capi.rs b/crates/bindings/src/capi.rs index ee8e193..f4391c0 100644 --- a/crates/bindings/src/capi.rs +++ b/crates/bindings/src/capi.rs @@ -385,6 +385,24 @@ pub unsafe extern "C" fn catena_series_of( } } +/// Read an identifier out of whatever it arrived wrapped in. Returns a JSON +/// record, or NULL when the string is not one — an answer, not an error. +/// +/// # Safety +/// +/// `text` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_parse_identifier(text: *const c_char) -> *mut c_char { + clear_error(); + let Some(text) = (unsafe { borrow(text, "text") }) else { + return ptr::null_mut(); + }; + match crate::parse_identifier(text) { + Some(found) => give_json(&found), + None => ptr::null_mut(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -496,6 +514,17 @@ mod tests { assert!(json.contains("\"volume\":3")); } + #[test] + fn an_identifier_is_normalised_across_the_boundary() { + let text = c"https://doi.org/10.1000/182"; + let json = through(|| unsafe { catena_parse_identifier(text.as_ptr()) }).expect("a record"); + assert!(json.contains("\"id\":\"10.1000/182\""), "{json}"); + + let nope = c"hello"; + assert!(through(|| unsafe { catena_parse_identifier(nope.as_ptr()) }).is_none()); + assert_eq!(error(), None, "not an identifier is an answer"); + } + #[test] fn the_abi_version_is_the_one_the_header_declares() { assert_eq!(catena_abi_version(), 1); diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index b299cd5..251d8e6 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -176,6 +176,12 @@ pub fn identify(name: &str, bytes: &[u8]) -> Doc { booksniff::sniff(&bytes, name) } +/// Read an identifier — a DOI, ISSN, ISBN or arXiv id — out of whatever it +/// arrived wrapped in. +pub fn parse_identifier(text: &str) -> Option { + idnorm::parse(text) +} + /// The series a title belongs to, and which volume of it this is. pub fn series_of(title: &str, author: Option<&str>) -> Option { seriesof::series_of(title, author) @@ -237,6 +243,13 @@ mod tests { assert_eq!(channel_named("lexical"), None); } + #[test] + fn an_identifier_comes_back_normalised() { + let found = parse_identifier("https://doi.org/10.1000/182").expect("an identifier"); + assert_eq!(found.id, "10.1000/182"); + assert_eq!(parse_identifier("hello"), None); + } + #[test] fn a_series_comes_back_the_same_as_the_library_reads_it() { let found = series_of("The Philokalia Vol 3", None).expect("a series"); diff --git a/crates/bindings/src/python.rs b/crates/bindings/src/python.rs index 259990b..fd388bf 100644 --- a/crates/bindings/src/python.rs +++ b/crates/bindings/src/python.rs @@ -208,6 +208,17 @@ fn series_of<'py>( } } +/// Read an identifier out of whatever it arrived wrapped in. +/// +/// `None` when the string is not one. +#[pyfunction] +fn parse_identifier<'py>(py: Python<'py>, text: &str) -> PyResult>> { + match crate::parse_identifier(text) { + Some(found) => Ok(Some(to_py(py, &found)?)), + None => Ok(None), + } +} + /// The module Python imports. #[pymodule] fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -221,5 +232,6 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(fuse_hits, module)?)?; module.add_function(wrap_pyfunction!(identify, module)?)?; module.add_function(wrap_pyfunction!(series_of, module)?)?; + module.add_function(wrap_pyfunction!(parse_identifier, module)?)?; Ok(()) } diff --git a/crates/bindings/src/wasm.rs b/crates/bindings/src/wasm.rs index 228064d..80a1235 100644 --- a/crates/bindings/src/wasm.rs +++ b/crates/bindings/src/wasm.rs @@ -70,6 +70,21 @@ export interface Hit { payload: unknown; } +/** A DOI, ISSN, ISBN or arXiv identifier, read and checked. */ +export interface Identifier { + v: number; + /** The text it was read from. */ + input: string; + /** "doi", "issn", "isbn" or "arxiv". */ + kind: string; + /** The identifier, in the form it should be stored and compared in. */ + id: string; + /** Whether the check digit agrees, or null where the kind carries none. */ + checksum: boolean | null; + /** A URL that resolves it. */ + url?: string; +} + /** What a title says about its series. */ export interface SeriesProposal { v: number; @@ -233,6 +248,17 @@ pub fn series_of(title: &str, author: Option) -> Result Result { + match crate::parse_identifier(text) { + Some(found) => to_js(&found), + None => Ok(JsValue::UNDEFINED), + } +} + /// Declarations for the functions whose return type the generator cannot /// infer, so a caller gets `Chunk[]` rather than `any`. #[wasm_bindgen(typescript_custom_section)] @@ -244,4 +270,5 @@ export function splitCollection(markdown: string): Section[]; export function fuseHits(hits: Hit[], k?: number, weights?: Record, limit?: number): Hit[]; export function identify(name: string, data: Uint8Array): Doc; export function seriesOf(title: string, author?: string): SeriesProposal | undefined; +export function parseIdentifier(text: string): Identifier | undefined; "#; diff --git a/crates/idnorm/Cargo.toml b/crates/idnorm/Cargo.toml new file mode 100644 index 0000000..1657058 --- /dev/null +++ b/crates/idnorm/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "idnorm" +description = "The identifiers a bibliography is held together by, read and checked." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +bibclean.workspace = true +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +fancy-regex.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/idnorm/README.md b/crates/idnorm/README.md new file mode 100644 index 0000000..3a623c4 --- /dev/null +++ b/crates/idnorm/README.md @@ -0,0 +1,126 @@ +# idnorm + +The identifiers a bibliography is held together by, read and checked. + +Part of [catena](../../README.md). + +## What it does + +A DOI arrives bare, as a URL, with a `doi:` prefix, with the full stop the +sentence around it left behind, in whatever case the publisher felt like. All +of those are one identifier. A catalogue that stores them as they arrived +cannot tell, so the same paper is three records and none of them join up. + +`idnorm` reads one out of whatever it is wrapped in and writes it in the form +it should be stored and compared in. + +## When you would use it + +- **Before storing a reference**, so that two spellings of one DOI are one row. +- **Auditing a bibliography** for identifiers that cannot be right — an ISBN + and an ISSN carry a check digit, and a mistyped one fails it. +- **Pulling identifiers out of running text**, where the surrounding sentence + is attached to them. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena idnorm +``` + +## Example + +``` +$ printf '10.1000/182\ndoi:10.1093/mind/LIX.236.433\n0028-0836\n978-0-306-40615-7\narXiv:2401.01234\n' | idnorm +{"v":1,"input":"10.1000/182","kind":"doi","id":"10.1000/182","checksum":null,"url":"https://doi.org/10.1000/182"} +{"v":1,"input":"doi:10.1093/mind/LIX.236.433","kind":"doi","id":"10.1093/mind/lix.236.433","checksum":null,"url":"https://doi.org/10.1093/mind/lix.236.433"} +{"v":1,"input":"0028-0836","kind":"issn","id":"0028-0836","checksum":true,"url":"https://portal.issn.org/resource/ISSN/0028-0836"} +{"v":1,"input":"978-0-306-40615-7","kind":"isbn","id":"9780306406157","checksum":true,"url":"https://openlibrary.org/isbn/9780306406157"} +{"v":1,"input":"arXiv:2401.01234","kind":"arxiv","id":"2401.01234","checksum":null,"url":"https://arxiv.org/abs/2401.01234"} +``` + +The ones that cannot be right: + +```sh +idnorm refs.txt | jq -r 'select(.checksum == false) | .input' +``` + +What a line was not: + +``` +$ printf 'hello\n' | idnorm +idnorm: not an identifier: hello +$ echo $? +1 +``` + +## Options + +| Option | Effect | +|---|---| +| `--kind ` | Read every line as this kind rather than whatever it looks like | +| `--quiet` | Say nothing about the lines that are not identifiers | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Every line was an identifier | +| `1` | At least one was not | +| `64` | The command line was wrong | +| `74` | The file could not be read | + +## What it can and cannot tell you + +**It can say a string is well formed.** An ISBN and an ISSN carry a check +digit, weighted by position, which catches one mistyped figure and two +transposed ones. A DOI and an arXiv identifier carry no checksum at all, so +for those `checksum` is `null` rather than `true` — there is nothing to check +against, and saying `true` would claim more than is known. + +**It cannot say the work exists.** That is a fact about the world rather than +about the string, and the only way to learn it is to ask a register. +[`bibfetch`](../bibfetch/README.md) does that: + +```sh +idnorm refs.txt | bibfetch | jq -r 'select(.found | not) | .id' +``` + +Those two together are what "validate against a catalogue" means, and they are +two tools because one of them touches the network and the other must not. + +## The four kinds + +**DOI.** A registrant prefix beginning `10.` and a suffix the registrant +chooses. Stored lowercased, because a DOI is case-insensitive and two +catalogues will disagree about the case of one. Read out of running text, with +the trailing full stop, comma or bracket the sentence left on it removed. + +**ISSN.** Eight digits in two fours, the last a check digit that may be `X` +for ten. Stored hyphenated. + +**ISBN.** Ten or thirteen digits. Stored as digits, without the hyphens, since +where a publisher puts them varies. Only where the line is the identifier and +nothing else — an ISBN has no prefix announcing it, so a sentence with a +thirteen-digit number in it would otherwise read as one. + +**arXiv.** Both forms: `2401.01234` since 2007, and `math.GT/0309136` before +it. A version suffix is kept, because `v2` names a particular revision and +citing without one means the latest. + +## As a library + +```rust +let found = idnorm::parse("https://doi.org/10.1000/182").expect("an identifier"); +assert_eq!(found.kind, idnorm::Kind::Doi); +assert_eq!(found.id, "10.1000/182"); +``` + +`idnorm::doi`, `idnorm::issn` and `idnorm::arxiv` read one kind each, and +`idnorm::issn::valid` checks a serial number's check digit. + +## See also + +- [`bibfetch`](../bibfetch/README.md) — what a register says about one. +- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record. diff --git a/crates/idnorm/src/arxiv.rs b/crates/idnorm/src/arxiv.rs new file mode 100644 index 0000000..12a0961 --- /dev/null +++ b/crates/idnorm/src/arxiv.rs @@ -0,0 +1,122 @@ +//! arXiv preprint identifiers. +//! +//! Two forms, because the scheme changed in 2007. The new one is four digits +//! of year and month, a dot, and a number: `2401.01234`. The old one is an +//! archive name, optionally a subject class, a slash, and seven digits: +//! `math.GT/0309136`. +//! +//! Either may carry a version — `v2` — which is part of the identifier: it +//! names a particular revision, and citing one without it means the latest. + +use std::sync::LazyLock; + +use fancy_regex::Regex; + +/// The post-2007 form: `YYMM.NNNNN`, four or five digits after the dot. +static MODERN: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b(\d{4}\.\d{4,5}(?:v\d+)?)\b").expect("a valid pattern")); + +/// The pre-2007 form: an archive, an optional subject class, then seven +/// digits. +static LEGACY: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b([a-z][a-z-]{1,20}(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?)\b") + .expect("a valid pattern") +}); + +/// Read an arXiv identifier out of a string. +/// +/// The old form keeps the case of its subject class, which is capitalised by +/// convention and is how the archives are written; the rest is lowercased. +pub fn parse(text: &str) -> Option { + let text = text.trim(); + let without = strip_prefix(text); + + if let Some(found) = LEGACY.captures(without).ok().flatten() { + let id = found.get(1)?.as_str(); + let (archive, number) = id.split_once('/')?; + return Some(format!( + "{}/{}", + lowercase_archive(archive), + number.to_lowercase() + )); + } + let found = MODERN.captures(without).ok().flatten()?; + Some(found.get(1)?.as_str().to_lowercase()) +} + +/// The archive lowercased, but its subject class left as it was written. +fn lowercase_archive(archive: &str) -> String { + match archive.split_once('.') { + Some((name, class)) => format!("{}.{}", name.to_lowercase(), class.to_uppercase()), + None => archive.to_lowercase(), + } +} + +/// Whatever introduced it. +fn strip_prefix(text: &str) -> &str { + for prefix in [ + "https://arxiv.org/abs/", + "http://arxiv.org/abs/", + "arxiv:", + "arxiv ", + ] { + if text.len() >= prefix.len() && text[..prefix.len()].eq_ignore_ascii_case(prefix) { + return text[prefix.len()..].trim(); + } + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_modern_form_reads_as_itself() { + assert_eq!(parse("2401.01234").as_deref(), Some("2401.01234")); + // Four digits after the dot before 2015, five after it. + assert_eq!(parse("0704.0001").as_deref(), Some("0704.0001")); + } + + #[test] + fn a_version_is_part_of_the_identifier() { + // It names a revision; without it a citation means the latest. + assert_eq!(parse("2401.01234v2").as_deref(), Some("2401.01234v2")); + assert_eq!( + parse("math.GT/0309136v1").as_deref(), + Some("math.GT/0309136v1") + ); + } + + #[test] + fn the_old_form_keeps_its_subject_class_capitalised() { + assert_eq!(parse("math.GT/0309136").as_deref(), Some("math.GT/0309136")); + assert_eq!(parse("MATH.gt/0309136").as_deref(), Some("math.GT/0309136")); + assert_eq!(parse("hep-th/9901001").as_deref(), Some("hep-th/9901001")); + } + + #[test] + fn every_way_of_introducing_one_gives_the_same_identifier() { + for text in [ + "arXiv:2401.01234", + "arxiv:2401.01234", + "arXiv 2401.01234", + "https://arxiv.org/abs/2401.01234", + ] { + assert_eq!(parse(text).as_deref(), Some("2401.01234"), "{text}"); + } + } + + #[test] + fn what_is_not_an_arxiv_identifier_is_not_one() { + for text in ["", "hello", "24.01234", "2401.012", "2401.012345"] { + assert_eq!(parse(text), None, "{text:?}"); + } + } + + #[test] + fn a_doi_is_not_mistaken_for_one() { + // `10.1000/182` has a slash and digits, but not seven of them. + assert_eq!(parse("10.1000/182"), None); + } +} diff --git a/crates/idnorm/src/doi.rs b/crates/idnorm/src/doi.rs new file mode 100644 index 0000000..db34f03 --- /dev/null +++ b/crates/idnorm/src/doi.rs @@ -0,0 +1,129 @@ +//! Digital object identifiers. +//! +//! A DOI is a registrant prefix beginning `10.` and a suffix the registrant +//! chooses, separated by a slash. The suffix may be almost anything, which is +//! what makes reading one out of running text hard: there is no character +//! that reliably ends it. + +use std::sync::LazyLock; + +use fancy_regex::Regex; + +/// The shape, wherever it appears and however it was introduced. +/// +/// The suffix stops at whitespace or at one of the brackets and quotes that +/// wrap a citation, because those are never part of one in practice and +/// leaving them on makes two spellings of the same identifier. +static DOI: LazyLock = LazyLock::new(|| { + Regex::new(r#"(?i)\b(10\.\d{4,9}/[^\s"'<>,;()\[\]{}]+)"#).expect("a valid pattern") +}); + +/// The prefixes a DOI arrives wearing. +const PREFIXES: &[&str] = &[ + "https://doi.org/", + "http://doi.org/", + "https://dx.doi.org/", + "http://dx.doi.org/", + "doi:", + "doi ", +]; + +/// Read a DOI out of a string, in the form it should be stored in. +/// +/// Lowercased, because a DOI is case-insensitive and two catalogues will +/// disagree about the case of the same one. The trailing punctuation a +/// sentence leaves behind is removed. +pub fn parse(text: &str) -> Option { + let text = text.trim(); + let mut without = text; + for prefix in PREFIXES { + if without.len() >= prefix.len() && without[..prefix.len()].eq_ignore_ascii_case(prefix) { + without = &without[prefix.len()..]; + break; + } + } + + let found = DOI.captures(without).ok().flatten()?; + let id = found.get(1)?.as_str(); + + // A full stop, comma or semicolon at the end belongs to the sentence + // rather than to the identifier. A closing bracket does too, unless the + // suffix opened one. + let id = id.trim_end_matches(['.', ',', ';', ':']); + if id.len() < 6 || !id.contains('/') { + return None; + } + // A suffix of nothing is not a suffix. + if id.split_once('/')?.1.is_empty() { + return None; + } + Some(id.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_plain_form_reads_as_itself() { + assert_eq!(parse("10.1000/182").as_deref(), Some("10.1000/182")); + } + + #[test] + fn every_way_of_introducing_one_gives_the_same_identifier() { + for text in [ + "doi:10.1000/182", + "DOI: 10.1000/182", + "https://doi.org/10.1000/182", + "http://dx.doi.org/10.1000/182", + "HTTPS://DOI.ORG/10.1000/182", + ] { + assert_eq!(parse(text).as_deref(), Some("10.1000/182"), "{text}"); + } + } + + #[test] + fn the_sentence_around_it_is_not_part_of_it() { + assert_eq!( + parse("See 10.1093/mind/LIX.236.433, which argues otherwise.").as_deref(), + Some("10.1093/mind/lix.236.433") + ); + assert_eq!(parse("(10.1000/182)").as_deref(), Some("10.1000/182")); + assert_eq!(parse("\"10.1000/182\"").as_deref(), Some("10.1000/182")); + } + + #[test] + fn case_is_not_part_of_it_either() { + // A DOI is case-insensitive, and two catalogues will disagree. + assert_eq!( + parse("10.1093/MIND/LIX.236.433").as_deref(), + parse("10.1093/mind/lix.236.433").as_deref() + ); + } + + #[test] + fn a_suffix_may_hold_almost_anything() { + assert_eq!( + parse("10.1371/journal.pone.0012345").as_deref(), + Some("10.1371/journal.pone.0012345") + ); + assert_eq!( + parse("10.1016/j.cell.2020.01.001").as_deref(), + Some("10.1016/j.cell.2020.01.001") + ); + } + + #[test] + fn what_is_not_a_doi_is_not_one() { + for text in ["", "10.", "10.1000", "11.1000/182", "10.100/182", "hello"] { + assert_eq!(parse(text), None, "{text:?}"); + } + } + + #[test] + fn a_prefix_with_no_identifier_after_it_is_nothing() { + assert_eq!(parse("doi:"), None); + assert_eq!(parse("https://doi.org/"), None); + assert_eq!(parse("10.1000/"), None); + } +} diff --git a/crates/idnorm/src/issn.rs b/crates/idnorm/src/issn.rs new file mode 100644 index 0000000..b978fce --- /dev/null +++ b/crates/idnorm/src/issn.rs @@ -0,0 +1,129 @@ +//! International standard serial numbers, which identify a journal. +//! +//! Eight digits in two groups of four, the last of which is a check digit +//! that may be `X` for ten. + +use std::sync::LazyLock; + +use fancy_regex::Regex; + +/// Eight characters in two fours, with or without the hyphen, and with +/// whatever introduced it. +static ISSN: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b(\d{4})-?(\d{3}[\dX])\b").expect("a valid pattern")); + +/// Read an ISSN out of a string, hyphenated as it should be stored. +pub fn parse(text: &str) -> Option { + let text = text.trim(); + let without = text + .strip_prefix("ISSN ") + .or_else(|| text.strip_prefix("issn ")) + .or_else(|| text.strip_prefix("ISSN:")) + .or_else(|| text.strip_prefix("issn:")) + .unwrap_or(text) + .trim(); + + let found = ISSN.captures(without).ok().flatten()?; + Some(format!( + "{}-{}", + found.get(1)?.as_str(), + found.get(2)?.as_str().to_uppercase() + )) +} + +/// Whether the check digit agrees with the other seven. +/// +/// Each of the first seven digits is weighted by its position counting down +/// from eight, and the total plus the check digit must be a multiple of +/// eleven. One mistyped digit, or two transposed, fails. +pub fn valid(issn: &str) -> bool { + let digits: Vec = issn + .chars() + .filter(|c| !c.is_whitespace() && *c != '-') + .collect(); + if digits.len() != 8 { + return false; + } + + let mut total = 0u32; + for (index, digit) in digits[..7].iter().enumerate() { + let Some(value) = digit.to_digit(10) else { + return false; + }; + total += value * (8 - index as u32); + } + let check = match digits[7] { + 'X' | 'x' => 10, + digit => match digit.to_digit(10) { + Some(value) => value, + None => return false, + }, + }; + (total + check) % 11 == 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_real_issn_checks_out() { + // Nature. + assert!(valid("0028-0836")); + assert!(valid("00280836")); + // Science. + assert!(valid("0036-8075")); + } + + #[test] + fn one_digit_out_does_not() { + assert!(!valid("0028-0837")); + assert!(!valid("0028-0835")); + } + + #[test] + fn two_digits_swapped_does_not_either() { + // Which is the whole point of a weighted checksum. + assert!(!valid("0082-0836")); + } + + #[test] + fn a_check_digit_of_ten_is_written_x() { + assert!(valid("1050-124X")); + assert!(valid("1050-124x"), "however it is cased"); + } + + #[test] + fn what_is_the_wrong_length_or_shape_is_not_valid() { + assert!(!valid("")); + assert!(!valid("0028-083")); + assert!(!valid("0028-08366")); + assert!(!valid("0028-083Y")); + assert!(!valid("ABCD-EFGH")); + } + + #[test] + fn it_is_read_however_it_was_written() { + for text in [ + "0028-0836", + "00280836", + "ISSN 0028-0836", + "ISSN: 0028-0836", + " 0028-0836 ", + ] { + assert_eq!(parse(text).as_deref(), Some("0028-0836"), "{text}"); + } + } + + #[test] + fn a_lower_case_check_digit_is_stored_upper() { + assert_eq!(parse("1050-124x").as_deref(), Some("1050-124X")); + } + + #[test] + fn what_is_not_an_issn_is_not_one() { + for text in ["", "hello", "123", "0028-08AB"] { + assert_eq!(parse(text), None, "{text:?}"); + } + } +} diff --git a/crates/idnorm/src/lib.rs b/crates/idnorm/src/lib.rs new file mode 100644 index 0000000..55fe9d7 --- /dev/null +++ b/crates/idnorm/src/lib.rs @@ -0,0 +1,242 @@ +//! The identifiers a bibliography is held together by, read and checked. +//! +//! A DOI arrives as a bare string, a URL, a `doi:` prefix, with a trailing +//! full stop the sentence around it left behind, in whatever case the +//! publisher felt like. All of those are the same identifier, and a catalogue +//! that stores them as they arrived cannot tell that. +//! +//! Nothing here asks anybody whether an identifier exists. What it can say is +//! whether a string is *well formed*: an ISBN and an ISSN carry a check digit +//! that catches a mistyped one, and a DOI or an arXiv id has a shape. That is +//! a different question from whether the work is real, and the honest answer +//! to the second needs a catalogue. + +pub mod arxiv; +pub mod doi; +pub mod issn; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// What kind of identifier something is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + /// A digital object identifier: `10.1000/182`. + Doi, + /// An international standard serial number, for a journal: `0028-0836`. + Issn, + /// An international standard book number, for a book. + Isbn, + /// An arXiv preprint identifier: `2401.01234` or `math.GT/0309136`. + Arxiv, +} + +impl Kind { + /// What it is called. + pub fn name(self) -> &'static str { + match self { + Kind::Doi => "doi", + Kind::Issn => "issn", + Kind::Isbn => "isbn", + Kind::Arxiv => "arxiv", + } + } +} + +/// One identifier, as it arrived and as it should be stored. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Identifier { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// The text it was read from. + pub input: String, + /// Which kind it turned out to be. + pub kind: Kind, + /// The identifier, in the form it should be stored and compared in. + pub id: String, + /// Whether the check digit agrees, where the kind carries one. `None` + /// where the kind has nothing to check against. + pub checksum: Option, + /// A URL that resolves it, where the kind has a resolver. + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Identifier { + fn version(&self) -> u32 { + self.v + } +} + +/// Read whatever kind of identifier a string holds. +/// +/// Tried in order of how distinctive each shape is: a DOI announces itself +/// with `10.`, an arXiv id with its own prefix or a dotted archive name, and +/// what is left is a run of digits that is either thirteen or ten long (an +/// ISBN) or eight (an ISSN). +pub fn parse(text: &str) -> Option { + parse_as(text, None) +} + +/// Read a string as one particular kind, or as whatever it looks like. +pub fn parse_as(text: &str, kind: Option) -> Option { + let input = text.trim(); + if input.is_empty() { + return None; + } + + // Order matters: a DOI announces itself with `10.`, an arXiv id with its + // own prefix, and what is left is a run of digits whose length decides + // between a book and a journal. + const EVERY: [Kind; 4] = [Kind::Doi, Kind::Arxiv, Kind::Isbn, Kind::Issn]; + let one = [kind.unwrap_or(Kind::Doi)]; + let kinds: &[Kind] = match kind { + Some(_) => &one, + None => &EVERY, + }; + + for kind in kinds { + let found = match kind { + Kind::Doi => doi::parse(input).map(|id| Identifier { + v: corpus_core::V, + url: Some(format!("https://doi.org/{id}")), + input: input.to_string(), + kind: Kind::Doi, + id, + checksum: None, + }), + Kind::Arxiv => arxiv::parse(input).map(|id| Identifier { + v: corpus_core::V, + url: Some(format!("https://arxiv.org/abs/{id}")), + input: input.to_string(), + kind: Kind::Arxiv, + id, + checksum: None, + }), + Kind::Issn => issn::parse(input).map(|id| Identifier { + v: corpus_core::V, + checksum: Some(issn::valid(&id)), + url: Some(format!("https://portal.issn.org/resource/ISSN/{id}")), + input: input.to_string(), + kind: Kind::Issn, + id, + }), + Kind::Isbn => isbn(input), + }; + if let Some(found) = found { + return Some(found); + } + } + None +} + +/// An ISBN, using the reader the catalogue tools already share. +fn isbn(input: &str) -> Option { + let digits = bibclean::isbn::digits(input); + if !matches!(digits.len(), 10 | 13) { + return None; + } + // Only where the input is the identifier and nothing else. An ISBN has no + // prefix that announces it, so a sentence holding a thirteen-digit number + // would otherwise read as one. + if !input + .chars() + .all(|c| c.is_ascii_digit() || "- xX".contains(c)) + { + return None; + } + Some(Identifier { + v: corpus_core::V, + checksum: Some(bibclean::isbn::valid(&digits)), + url: Some(format!("https://openlibrary.org/isbn/{digits}")), + input: input.to_string(), + kind: Kind::Isbn, + id: digits, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_doi_is_read_however_it_was_written() { + for text in [ + "10.1000/182", + "doi:10.1000/182", + "https://doi.org/10.1000/182", + "http://dx.doi.org/10.1000/182", + " 10.1000/182. ", + ] { + let found = parse(text).unwrap_or_else(|| panic!("{text}")); + assert_eq!(found.kind, Kind::Doi, "{text}"); + assert_eq!(found.id, "10.1000/182", "{text}"); + } + } + + #[test] + fn an_issn_carries_a_check_digit() { + let found = parse("0028-0836").expect("an issn"); + assert_eq!(found.kind, Kind::Issn); + assert_eq!(found.id, "0028-0836"); + assert_eq!(found.checksum, Some(true)); + + let wrong = parse("0028-0837").expect("an issn"); + assert_eq!(wrong.checksum, Some(false), "one digit out is caught"); + } + + #[test] + fn an_isbn_carries_one_too() { + let found = parse("978-0-306-40615-7").expect("an isbn"); + assert_eq!(found.kind, Kind::Isbn); + assert_eq!(found.id, "9780306406157"); + assert_eq!(found.checksum, Some(true)); + } + + #[test] + fn an_arxiv_identifier_is_read_in_both_its_forms() { + assert_eq!(parse("arXiv:2401.01234").expect("an id").id, "2401.01234"); + assert_eq!(parse("2401.01234v2").expect("an id").id, "2401.01234v2"); + assert_eq!( + parse("math.GT/0309136").expect("an id").id, + "math.GT/0309136" + ); + } + + #[test] + fn every_kind_resolves_somewhere() { + for text in [ + "10.1000/182", + "0028-0836", + "978-0-306-40615-7", + "arXiv:2401.01234", + ] { + assert!(parse(text).expect("an id").url.is_some(), "{text}"); + } + } + + #[test] + fn what_is_not_an_identifier_is_not_one() { + for text in ["", " ", "hello", "10.", "1234", "not/an/id"] { + assert_eq!(parse(text), None, "{text:?}"); + } + } + + #[test] + fn a_kind_can_be_insisted_on() { + // Eight digits are an ISSN; asked for a DOI, they are nothing. + assert!(parse_as("00280836", Some(Kind::Issn)).is_some()); + assert_eq!(parse_as("00280836", Some(Kind::Doi)), None); + } + + #[test] + fn a_sentence_that_merely_contains_digits_is_not_an_isbn() { + assert_eq!(parse("published in 9780306406157 by somebody"), None); + } +} diff --git a/crates/idnorm/src/main.rs b/crates/idnorm/src/main.rs new file mode 100644 index 0000000..9279dcc --- /dev/null +++ b/crates/idnorm/src/main.rs @@ -0,0 +1,112 @@ +//! The identifiers a bibliography is held together by, read and checked. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::exit; +use idnorm::Kind; + +/// Read identifiers and say what they are, one record per line. +/// +/// A DOI arrives bare, as a URL, with a `doi:` prefix, with the full stop the +/// sentence around it left behind, in whatever case the publisher felt like. +/// All of those are one identifier, and a catalogue storing them as they +/// arrived cannot tell. +/// +/// Asks nobody whether the thing exists. What it can say is whether the +/// string is well formed — an ISBN and an ISSN carry a check digit — which is +/// a different question, and the honest answer to the other one needs a +/// catalogue. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 every line was an identifier, 1 at least one was not, +74 the file could not be read. + +Example: + + idnorm refs.txt | jq -r 'select(.checksum == false) | .input' + idnorm --kind doi refs.txt | jq -r .url" +)] +struct Args { + /// File of identifiers, one per line. Reads stdin when absent. + path: Option, + + /// Read every line as this kind, rather than whatever it looks like. + #[arg(long, value_enum)] + kind: Option, + + /// Say nothing about the lines that are not identifiers. + #[arg(long)] + quiet: bool, +} + +/// The kinds a line can be insisted on being. +#[derive(Clone, Copy, clap::ValueEnum)] +enum Sort { + /// A digital object identifier. + Doi, + /// A journal's serial number. + Issn, + /// A book's number. + Isbn, + /// An arXiv preprint. + Arxiv, +} + +impl From for Kind { + fn from(sort: Sort) -> Self { + match sort { + Sort::Doi => Kind::Doi, + Sort::Issn => Kind::Issn, + Sort::Isbn => Kind::Isbn, + Sort::Arxiv => Kind::Arxiv, + } + } +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("idnorm: {why}"); + return ExitCode::from(code); + } + }; + + let kind = args.kind.map(Kind::from); + let mut found = Vec::new(); + let mut unread = 0usize; + + for line in text.lines() { + if line.trim().is_empty() { + continue; + } + match idnorm::parse_as(line, kind) { + Some(identifier) => found.push(identifier), + None => { + unread += 1; + if !args.quiet { + eprintln!("idnorm: not an identifier: {line}"); + } + } + } + } + + let wrote = catena_cli::stream("idnorm", &found); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + ExitCode::from(match unread { + 0 => exit::OK, + _ => exit::NO, + }) +} diff --git a/docs/contract.md b/docs/contract.md index 082aa4f..6e009d6 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -61,9 +61,10 @@ same as any other filter. ## Nothing reaches outside the process No network, no database, no configuration file, no state kept between runs. -Four exceptions, each declared in `--help`: +Five exceptions, each declared in `--help`: - [`emb`](../crates/emb/README.md) makes an HTTP call, which is the whole of what it does. +- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier, likewise. - [`pdftriage`](../crates/pdftriage/README.md) runs `pdftotext` and `pdfinfo`. - [`pdfgutter`](../crates/pdfgutter/README.md) runs `pdftoppm` to look at a page. - [`avplan`](../crates/avplan/README.md) runs `ffprobe` to read a file's streams, and diff --git a/docs/recipes.md b/docs/recipes.md index 678bb89..9d1c84c 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -12,6 +12,7 @@ this suite, because a real pipeline has a database at the end of it. - [Passages into pgvector](#passages-into-pgvector) - [Hybrid search, fused](#hybrid-search-fused) - [A catalogue that has drifted](#a-catalogue-that-has-drifted) +- [A bibliography that has been copied between formats](#a-bibliography-that-has-been-copied-between-formats) - [A video library that plays on the thing you carry](#a-video-library-that-plays-on-the-thing-you-carry) - [Notes on composing these](#notes-on-composing-these) @@ -267,6 +268,14 @@ vocabmerge --kind author authors.ndjson > merges.ndjson vocabmerge --kind tag tags.ndjson >> merges.ndjson ``` +Series and volume, which most catalogues have on almost nothing: + +```sh +seriesof catalogue.ndjson \ + | jq -r 'select(.volume) | "UPDATE book SET series = \(.series|@json), series_index = \(.volume) WHERE title = \(.title|@json);"' \ + | psql +``` + Apply only what you believe: ```sh @@ -275,6 +284,39 @@ jq -r 'select(.action == "merge" and .confidence >= 0.9) merges.ndjson | psql ``` +## A bibliography that has been copied between formats + +Identifiers arrive wrapped in whatever the last tool left on them, and a +catalogue that stores them as they came cannot tell two spellings of one DOI +apart. + +```sh +idnorm --quiet refs.txt > ids.ndjson +``` + +The ones that cannot be right — an ISBN or an ISSN whose check digit fails: + +```sh +jq -r 'select(.checksum == false) | .input' ids.ndjson +``` + +The ones that are well formed and still wrong, which needs a register: + +```sh +bibfetch --polite you@example.org ids.ndjson \ + | jq -r 'select(.found | not) | "missing: \(.id)"' +``` + +That second question is a different one, which is why it is a different tool: +`idnorm` never touches the network and `bibfetch` does nothing else. + +Filling in what you only had the identifier for: + +```sh +idnorm --quiet refs.txt | bibfetch \ + | jq -r 'select(.found) | "\(.year)\t\(.authors | join(", "))\t\(.title)"' +``` + ## A video library that plays on the thing you carry ```sh diff --git a/parity/tests/abstracts.rs b/parity/tests/abstracts.rs index 9ca9210..47b4c6f 100644 --- a/parity/tests/abstracts.rs +++ b/parity/tests/abstracts.rs @@ -15,6 +15,10 @@ fn records(text: &str) -> Vec { .collect() } +/// The cases, as text, so the index keeps the order it was written in. +/// +/// Reading through `Value` would sort the keys, and which of two words at one +/// position comes first is exactly what that order decides. fn cases() -> Vec { let text = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson")) .expect("a readable case file"); @@ -33,7 +37,7 @@ fn every_abstract_reads_back_the_same_as_python() { assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order"); let index: Inverted = - serde_json::from_value(case["index"].clone()).expect("an index of positions"); + serde_json::from_str(&case["index"].to_string()).expect("an index of positions"); let ours = abstracts::from_inverted(&index); if ours.as_deref() != theirs["abstract"].as_str() { wrong.push(format!(