mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
idnorm and bibfetch: an identifier read, and a register asked
This commit is contained in:
funkpower
Generated
+29
@@ -132,6 +132,20 @@ dependencies = [
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bibfetch"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"idnorm",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.8.0"
|
||||
@@ -220,6 +234,7 @@ dependencies = [
|
||||
"corpus-core",
|
||||
"cuekit",
|
||||
"cuepack",
|
||||
"idnorm",
|
||||
"mdscrub",
|
||||
"mdsplit",
|
||||
"pyo3",
|
||||
@@ -785,6 +800,20 @@ dependencies = [
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"fancy-regex 0.16.2",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.2"
|
||||
|
||||
@@ -27,6 +27,8 @@ avplan = { path = "crates/avplan", version = "0.1.0" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.1.0" }
|
||||
catena = { path = "crates/bindings", version = "0.1.0" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.1.0" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.1.0" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.1.0" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.1.0" }
|
||||
emb = { path = "crates/emb", version = "0.1.0" }
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ Each tool installs by name, so you can take one without taking the rest.
|
||||
| [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change |
|
||||
| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary |
|
||||
| [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is |
|
||||
| [`idnorm`](crates/idnorm/README.md) | DOIs, ISSNs, ISBNs and arXiv ids, read and checked |
|
||||
| [`bibfetch`](crates/bibfetch/README.md) | What a register says about an identifier |
|
||||
| [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out |
|
||||
| [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it |
|
||||
| [`chunkmd`](crates/chunkmd/README.md) | Markdown into passages that carry their page, chapter and role |
|
||||
|
||||
@@ -9,10 +9,56 @@
|
||||
//! out is the original with its punctuation and spacing intact, because those
|
||||
//! travelled attached to the words.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// An abstract as its catalogue publishes it: word to the positions it holds.
|
||||
pub type Inverted = HashMap<String, Vec<i64>>;
|
||||
///
|
||||
/// A list of pairs rather than a map, so that the order the catalogue listed
|
||||
/// the words in survives. It matters exactly once — when two words claim the
|
||||
/// same position — and a map would settle that tie by hash order, which is a
|
||||
/// different answer on every run.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Inverted(pub Vec<(String, Vec<i64>)>);
|
||||
|
||||
impl Inverted {
|
||||
/// The pairs, in the order they were written.
|
||||
pub fn pairs(&self) -> &[(String, Vec<i64>)] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl FromIterator<(String, Vec<i64>)> for Inverted {
|
||||
fn from_iter<T: IntoIterator<Item = (String, Vec<i64>)>>(pairs: T) -> Self {
|
||||
Inverted(pairs.into_iter().collect())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for Inverted {
|
||||
/// Read the object entry by entry rather than through a map, which is
|
||||
/// what keeps the document's own order.
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct InOrder;
|
||||
|
||||
impl<'de> serde::de::Visitor<'de> for InOrder {
|
||||
type Value = Inverted;
|
||||
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("an object of words to the positions they hold")
|
||||
}
|
||||
|
||||
fn visit_map<M: serde::de::MapAccess<'de>>(
|
||||
self,
|
||||
mut map: M,
|
||||
) -> Result<Inverted, M::Error> {
|
||||
let mut pairs = Vec::with_capacity(map.size_hint().unwrap_or(0));
|
||||
while let Some((word, at)) = map.next_entry::<String, Vec<i64>>()? {
|
||||
pairs.push((word, at));
|
||||
}
|
||||
Ok(Inverted(pairs))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_map(InOrder)
|
||||
}
|
||||
}
|
||||
|
||||
/// Put an inverted abstract back in order.
|
||||
///
|
||||
@@ -24,6 +70,7 @@ pub type Inverted = HashMap<String, Vec<i64>>;
|
||||
/// not the exact offsets.
|
||||
pub fn from_inverted(index: &Inverted) -> Option<String> {
|
||||
let mut positions: Vec<(i64, &str)> = index
|
||||
.pairs()
|
||||
.iter()
|
||||
.flat_map(|(word, at)| at.iter().map(move |at| (*at, word.as_str())))
|
||||
.collect();
|
||||
@@ -54,6 +101,17 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_words_at_one_position_keep_the_order_they_were_listed_in() {
|
||||
// The only place the catalogue's own ordering matters, and the only
|
||||
// reason this is a list of pairs rather than a map.
|
||||
let index = inverted(&[("alpha", &[1]), ("beta", &[1])]);
|
||||
assert_eq!(from_inverted(&index).as_deref(), Some("alpha beta"));
|
||||
|
||||
let other_way = inverted(&[("beta", &[1]), ("alpha", &[1])]);
|
||||
assert_eq!(from_inverted(&other_way).as_deref(), Some("beta alpha"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn words_come_back_in_the_order_their_positions_give() {
|
||||
let index = inverted(&[("obligation", &[1]), ("The", &[0]), ("ran", &[2])]);
|
||||
@@ -77,7 +135,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn an_index_with_nothing_in_it_is_no_abstract() {
|
||||
assert_eq!(from_inverted(&Inverted::new()), None);
|
||||
assert_eq!(from_inverted(&Inverted::default()), None);
|
||||
assert_eq!(from_inverted(&inverted(&[("word", &[])])), None);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "bibfetch"
|
||||
description = "What a catalogue says about an identifier, asked of the body that assigns it."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
idnorm.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
ureq.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,110 @@
|
||||
# bibfetch
|
||||
|
||||
What a catalogue says about an identifier, asked of the body that assigns it.
|
||||
|
||||
Part of [catena](../../README.md).
|
||||
|
||||
## What it does
|
||||
|
||||
[`idnorm`](../idnorm/README.md) can tell you a DOI is well formed. It cannot
|
||||
tell you the paper exists — that is not a fact about the string, it is a fact
|
||||
about the world, and the only way to learn it is to ask somebody who keeps a
|
||||
register.
|
||||
|
||||
So this asks, and that is the only thing it does.
|
||||
|
||||
## When you would use it
|
||||
|
||||
- **Auditing a bibliography.** A well-formed DOI that no register has is a
|
||||
mistyped one, and a catalogue can carry it for years without noticing.
|
||||
- **Filling in a reference** you have only the identifier for.
|
||||
- **Before citing something**, to check the year and the authors are what you
|
||||
think they are.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
cargo install --git https://git.pumpin.online/funkpower/catena bibfetch
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
$ printf '10.1093/mind/LIX.236.433\narXiv:2401.01234\n978-0-306-40615-7\n' | idnorm | bibfetch --polite you@example.org
|
||||
{"v":1,"id":"10.1093/mind/lix.236.433","kind":"doi","found":true,"source":"crossref","title":"I.—COMPUTING MACHINERY AND INTELLIGENCE","authors":["A. M. TURING"],"year":1950,"container":"Mind"}
|
||||
{"v":1,"id":"2401.01234","kind":"arxiv","found":true,"source":"arxiv","title":"Mixture cure semiparametric additive hazard models under partly interval censoring -- a penalized likelihood approach","authors":["Jinqing Li","Jun Ma"],"year":2024,"container":"arXiv","summary":"Survival analysis can sometimes involve individuals who will not experience…"}
|
||||
{"v":1,"id":"9780306406157","kind":"isbn","found":true,"source":"openlibrary","title":"Error-correction coding for digital communications","year":1981,"container":"Plenum Press"}
|
||||
```
|
||||
|
||||
The ones nobody has:
|
||||
|
||||
```
|
||||
$ printf '10.9999/definitely-not-a-real-doi\n' | idnorm | bibfetch
|
||||
{"v":1,"id":"10.9999/definitely-not-a-real-doi","kind":"doi","found":false,"source":"crossref"}
|
||||
$ echo $?
|
||||
1
|
||||
```
|
||||
|
||||
Absence is the useful answer here. It is what tells a catalogue that a DOI it
|
||||
has been carrying for three years was mistyped on the way in.
|
||||
|
||||
Auditing a whole bibliography:
|
||||
|
||||
```sh
|
||||
idnorm --quiet refs.txt | bibfetch --polite you@example.org \
|
||||
| jq -r 'select(.found | not) | "missing: \(.id)"'
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `--polite <EMAIL>` | An address to put in the request. The registers ask for one and answer faster when they get it |
|
||||
| `--timeout <SECONDS>` | How long to wait on a register. Default `20` |
|
||||
| `--attempts <COUNT>` | How many times to ask before believing a failure. Default `3` |
|
||||
| `--man` | Write this page as a manual page, in roff |
|
||||
|
||||
## Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| `0` | Every identifier was found |
|
||||
| `1` | At least one was not |
|
||||
| `64` | The command line was wrong |
|
||||
| `65` | The input was not identifier records |
|
||||
| `69` | A register could not be reached |
|
||||
| `74` | The file could not be read |
|
||||
|
||||
Exit `1` is an answer — the register replied and does not have it. Exit `69`
|
||||
is a failure — nobody replied, and the identifier may be perfectly good.
|
||||
|
||||
## The registers
|
||||
|
||||
| Kind | Asked of | Why that one |
|
||||
|---|---|---|
|
||||
| `doi` | Crossref | It assigns most scholarly DOIs |
|
||||
| `arxiv` | the arXiv | Its own register of preprints |
|
||||
| `isbn` | OpenLibrary | Books, without a key |
|
||||
|
||||
An **ISSN** has no register that answers without a key, so it is not looked up
|
||||
on its own. A journal's ISSN comes back on the works published in it instead.
|
||||
|
||||
A failure is retried, waiting a second and then two: long enough for a
|
||||
register that is rate-limiting, short enough that one which is simply down
|
||||
does not hold up a pipeline. An answer this cannot read is not retried,
|
||||
because it will not read any better next time.
|
||||
|
||||
## Reaching the network
|
||||
|
||||
This and [`emb`](../emb/README.md) are the only two tools here that do. Both
|
||||
say so in `--help`, and for both it is the whole of what they do rather than
|
||||
something that happens on the way to something else — which is the condition
|
||||
under which [the contract](../../docs/contract.md) allows it at all.
|
||||
|
||||
Nothing is cached, nothing is stored, and nothing is sent but the identifier
|
||||
and the address you pass to `--polite`.
|
||||
|
||||
## See also
|
||||
|
||||
- [`idnorm`](../idnorm/README.md) — reading the identifier in the first place.
|
||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record.
|
||||
@@ -0,0 +1,136 @@
|
||||
//! What a catalogue says about an identifier.
|
||||
//!
|
||||
//! [`idnorm`] can tell you a DOI is well formed. It cannot tell you the paper
|
||||
//! exists, because that is not a fact about the string — it is a fact about
|
||||
//! the world, and the only way to learn it is to ask somebody who keeps a
|
||||
//! register.
|
||||
//!
|
||||
//! So this asks. It is the only thing it does, which is the condition under
|
||||
//! which a filter here is allowed to touch the network at all.
|
||||
//!
|
||||
//! Three registers, one per kind of identifier, each the body that assigns
|
||||
//! them: Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a
|
||||
//! book. There is no register for an ISSN that answers without a key, so an
|
||||
//! ISSN is looked up as the journal of a work rather than on its own.
|
||||
|
||||
pub mod source;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use source::Source;
|
||||
|
||||
/// How long to wait on a register before giving up.
|
||||
pub const TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// How many times to ask before believing the answer.
|
||||
pub const ATTEMPTS: u32 = 3;
|
||||
|
||||
/// What a register said about one identifier.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Record {
|
||||
/// Record version.
|
||||
#[serde(default = "version")]
|
||||
pub v: u32,
|
||||
/// The identifier asked about, normalised.
|
||||
pub id: String,
|
||||
/// Which kind it is.
|
||||
pub kind: String,
|
||||
/// Whether the register has it. This is the answer the string alone
|
||||
/// cannot give.
|
||||
pub found: bool,
|
||||
/// Which register answered.
|
||||
pub source: String,
|
||||
/// The title, where there is one.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
/// The authors, in the order the register lists them.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub authors: Vec<String>,
|
||||
/// The year of publication.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub year: Option<i64>,
|
||||
/// The journal, publisher or archive it appeared under.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub container: Option<String>,
|
||||
/// The abstract, where the register carries one.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub summary: Option<String>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
corpus_core::V
|
||||
}
|
||||
|
||||
impl corpus_core::Record for Record {
|
||||
fn version(&self) -> u32 {
|
||||
self.v
|
||||
}
|
||||
}
|
||||
|
||||
/// What can go wrong asking.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Error {
|
||||
/// The register could not be reached, or refused.
|
||||
Unreachable(String),
|
||||
/// It answered with something this could not read.
|
||||
Shape(String),
|
||||
/// Nothing here knows where to ask about this kind.
|
||||
NoRegister(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Error::Unreachable(why) => write!(f, "{why}"),
|
||||
Error::Shape(why) => write!(f, "the answer could not be read: {why}"),
|
||||
Error::NoRegister(kind) => {
|
||||
write!(f, "no register here answers about {kind} identifiers")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
/// A record for an identifier the register does not have.
|
||||
///
|
||||
/// Absence is an answer, and a useful one: it is what tells a catalogue that
|
||||
/// a well-formed DOI it has been carrying for three years was mistyped.
|
||||
pub fn missing(id: &str, kind: &str, source: Source) -> Record {
|
||||
Record {
|
||||
v: corpus_core::V,
|
||||
id: id.to_string(),
|
||||
kind: kind.to_string(),
|
||||
found: false,
|
||||
source: source.name().to_string(),
|
||||
title: None,
|
||||
authors: Vec::new(),
|
||||
year: None,
|
||||
container: None,
|
||||
summary: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_identifier_nobody_has_is_still_an_answer() {
|
||||
let record = missing("10.1000/nope", "doi", Source::Crossref);
|
||||
assert!(!record.found);
|
||||
assert_eq!(record.source, "crossref");
|
||||
assert_eq!(record.title, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_that_found_nothing_serialises_without_empty_fields() {
|
||||
let json = serde_json::to_string(&missing("x", "doi", Source::Crossref)).expect("json");
|
||||
assert!(!json.contains("title"), "{json}");
|
||||
assert!(!json.contains("authors"), "{json}");
|
||||
assert!(json.contains("\"found\":false"), "{json}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
//! What a catalogue says about an identifier.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
use std::time::Duration;
|
||||
|
||||
use bibfetch::{Error, Source};
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
|
||||
/// Ask the body that assigns an identifier what it knows about one.
|
||||
///
|
||||
/// `idnorm` can say a DOI is well formed. Only a register can say the paper
|
||||
/// exists — that is a fact about the world rather than about the string, and
|
||||
/// it is what catches a well-formed DOI a catalogue has been carrying wrongly
|
||||
/// for three years.
|
||||
///
|
||||
/// Crossref for a DOI, the arXiv for a preprint, OpenLibrary for a book. This
|
||||
/// is the only tool here besides `emb` that touches the network, and it is the
|
||||
/// whole of what it does.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 every identifier was found, 1 at least one was not,
|
||||
65 the input was not identifiers, 69 a register could not be reached,
|
||||
74 the file could not be read.
|
||||
|
||||
Example:
|
||||
|
||||
idnorm refs.txt | bibfetch | jq -r 'select(.found) | .title'
|
||||
idnorm --kind doi refs.txt | bibfetch --polite you@example.org"
|
||||
)]
|
||||
struct Args {
|
||||
/// Identifier records to look up, as `idnorm` writes them. Reads stdin
|
||||
/// when absent.
|
||||
path: Option<PathBuf>,
|
||||
|
||||
/// An address to put in the request, which the registers ask for and
|
||||
/// answer faster when they get.
|
||||
#[arg(long, value_name = "EMAIL")]
|
||||
polite: Option<String>,
|
||||
|
||||
/// How long to wait on a register, in seconds.
|
||||
#[arg(long, default_value_t = bibfetch::TIMEOUT.as_secs(), value_name = "SECONDS")]
|
||||
timeout: u64,
|
||||
|
||||
/// How many times to ask before believing a failure.
|
||||
#[arg(long, default_value_t = bibfetch::ATTEMPTS, value_name = "COUNT")]
|
||||
attempts: u32,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
|
||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||
Ok(text) => text,
|
||||
Err((code, why)) => {
|
||||
eprintln!("bibfetch: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.timeout_global(Some(Duration::from_secs(args.timeout)))
|
||||
.user_agent(user_agent(args.polite.as_deref()))
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let mut records = Vec::new();
|
||||
let mut missing = 0usize;
|
||||
|
||||
for (number, line) in text.lines().enumerate() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (id, kind) = match read_identifier(line) {
|
||||
Ok(pair) => pair,
|
||||
Err(why) => {
|
||||
eprintln!("bibfetch: line {}: {why}", number + 1);
|
||||
return ExitCode::from(exit::DATA);
|
||||
}
|
||||
};
|
||||
|
||||
let Some(source) = Source::for_kind(&kind) else {
|
||||
eprintln!("bibfetch: {id}: no register here answers about {kind} identifiers");
|
||||
missing += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
match ask(&agent, source, &id, &kind, args.attempts) {
|
||||
Ok(record) => {
|
||||
if !record.found {
|
||||
missing += 1;
|
||||
}
|
||||
records.push(record);
|
||||
}
|
||||
Err(why) => {
|
||||
eprintln!("bibfetch: {id}: {why}");
|
||||
return ExitCode::from(exit::UNAVAILABLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let wrote = catena_cli::stream("bibfetch", &records);
|
||||
if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
|
||||
return wrote;
|
||||
}
|
||||
ExitCode::from(match missing {
|
||||
0 => exit::OK,
|
||||
_ => exit::NO,
|
||||
})
|
||||
}
|
||||
|
||||
/// Ask a register, retrying a failure that might not be one.
|
||||
///
|
||||
/// Waits a second before the second attempt and two before the third: long
|
||||
/// enough for a register that is rate-limiting, short enough that one which
|
||||
/// is simply down does not hold up a pipeline.
|
||||
fn ask(
|
||||
agent: &ureq::Agent,
|
||||
source: Source,
|
||||
id: &str,
|
||||
kind: &str,
|
||||
attempts: u32,
|
||||
) -> Result<bibfetch::Record, Error> {
|
||||
let mut last = Error::Unreachable("no attempt was made".into());
|
||||
|
||||
for attempt in 0..attempts.max(1) {
|
||||
match fetch(agent, source, id, kind) {
|
||||
Ok(record) => return Ok(record),
|
||||
// A shape this cannot read will not read any better next time.
|
||||
Err(e @ Error::Shape(_)) => return Err(e),
|
||||
Err(e) => last = e,
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1 << attempt));
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
fn fetch(
|
||||
agent: &ureq::Agent,
|
||||
source: Source,
|
||||
id: &str,
|
||||
kind: &str,
|
||||
) -> Result<bibfetch::Record, Error> {
|
||||
let response = agent.get(source.url(id)).call();
|
||||
let mut response = match response {
|
||||
Ok(response) => response,
|
||||
// A register that does not have it says so with a status, and that is
|
||||
// an answer rather than a failure.
|
||||
Err(ureq::Error::StatusCode(404)) => return Ok(bibfetch::missing(id, kind, source)),
|
||||
Err(e) => return Err(Error::Unreachable(e.to_string())),
|
||||
};
|
||||
let body = response
|
||||
.body_mut()
|
||||
.read_to_string()
|
||||
.map_err(|e| Error::Unreachable(e.to_string()))?;
|
||||
source.read(id, kind, &body)
|
||||
}
|
||||
|
||||
/// Who is asking.
|
||||
///
|
||||
/// The registers ask for an address and answer faster when they get one; a
|
||||
/// tool that does not say who it is gets the slow pool.
|
||||
fn user_agent(polite: Option<&str>) -> String {
|
||||
let version = env!("CARGO_PKG_VERSION");
|
||||
match polite {
|
||||
Some(address) => format!("bibfetch/{version} (mailto:{address})"),
|
||||
None => format!("bibfetch/{version}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The identifier and its kind out of one record.
|
||||
fn read_identifier(line: &str) -> Result<(String, String), String> {
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(line).map_err(|e| format!("not a record: {e}"))?;
|
||||
let id = value
|
||||
.get("id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or("no id")?
|
||||
.to_string();
|
||||
let kind = value
|
||||
.get("kind")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or("no kind")?
|
||||
.to_string();
|
||||
Ok((id, kind))
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
//! The registers, and how to read what each one says.
|
||||
//!
|
||||
//! Each is the body that assigns the identifiers it answers about, so each is
|
||||
//! the one place the answer is not a guess. The shapes they reply in have
|
||||
//! nothing in common, which is most of what this module is.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{Error, Record};
|
||||
|
||||
/// A register that can be asked about an identifier.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Source {
|
||||
/// Crossref, which assigns most scholarly DOIs.
|
||||
Crossref,
|
||||
/// The arXiv's own register of preprints.
|
||||
Arxiv,
|
||||
/// OpenLibrary, for books by ISBN.
|
||||
OpenLibrary,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// What it is called.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Source::Crossref => "crossref",
|
||||
Source::Arxiv => "arxiv",
|
||||
Source::OpenLibrary => "openlibrary",
|
||||
}
|
||||
}
|
||||
|
||||
/// Which register answers about a kind of identifier.
|
||||
pub fn for_kind(kind: &str) -> Option<Self> {
|
||||
match kind {
|
||||
"doi" => Some(Source::Crossref),
|
||||
"arxiv" => Some(Source::Arxiv),
|
||||
"isbn" => Some(Source::OpenLibrary),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Where to ask about one identifier.
|
||||
pub fn url(self, id: &str) -> String {
|
||||
match self {
|
||||
Source::Crossref => format!("https://api.crossref.org/works/{}", encode(id)),
|
||||
Source::Arxiv => format!(
|
||||
"https://export.arxiv.org/api/query?id_list={}&max_results=1",
|
||||
encode(id)
|
||||
),
|
||||
Source::OpenLibrary => {
|
||||
format!("https://openlibrary.org/isbn/{}.json", encode(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read what it said into a record.
|
||||
pub fn read(self, id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
match self {
|
||||
Source::Crossref => crossref(id, kind, body),
|
||||
Source::Arxiv => arxiv(id, kind, body),
|
||||
Source::OpenLibrary => open_library(id, kind, body),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Percent-encode what a path or a query would otherwise read as structure.
|
||||
///
|
||||
/// A DOI suffix may hold a slash, a question mark or a hash, and every one of
|
||||
/// those means something else in a URL.
|
||||
fn encode(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
for byte in text.bytes() {
|
||||
match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => {
|
||||
out.push(byte as char)
|
||||
}
|
||||
byte => out.push_str(&format!("%{byte:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A record with the fields nothing found fills in.
|
||||
fn blank(id: &str, kind: &str, source: Source) -> Record {
|
||||
let mut record = crate::missing(id, kind, source);
|
||||
record.found = true;
|
||||
record
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CrossrefReply {
|
||||
message: CrossrefWork,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CrossrefWork {
|
||||
#[serde(default)]
|
||||
title: Vec<String>,
|
||||
#[serde(default)]
|
||||
author: Vec<CrossrefAuthor>,
|
||||
#[serde(default, rename = "container-title")]
|
||||
container_title: Vec<String>,
|
||||
#[serde(default)]
|
||||
issued: Option<CrossrefDate>,
|
||||
#[serde(default, rename = "abstract")]
|
||||
summary: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CrossrefAuthor {
|
||||
#[serde(default)]
|
||||
given: Option<String>,
|
||||
#[serde(default)]
|
||||
family: Option<String>,
|
||||
#[serde(default)]
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CrossrefDate {
|
||||
#[serde(default, rename = "date-parts")]
|
||||
date_parts: Vec<Vec<i64>>,
|
||||
}
|
||||
|
||||
fn crossref(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
let reply: CrossrefReply =
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
let work = reply.message;
|
||||
|
||||
let mut record = blank(id, kind, Source::Crossref);
|
||||
record.title = work.title.into_iter().next();
|
||||
record.container = work.container_title.into_iter().next();
|
||||
record.year = work
|
||||
.issued
|
||||
.and_then(|issued| issued.date_parts.into_iter().next())
|
||||
.and_then(|parts| parts.into_iter().next());
|
||||
record.authors = work
|
||||
.author
|
||||
.into_iter()
|
||||
.filter_map(|author| match (author.given, author.family, author.name) {
|
||||
(Some(given), Some(family), _) => Some(format!("{given} {family}")),
|
||||
(None, Some(family), _) => Some(family),
|
||||
(_, _, Some(name)) => Some(name),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
// Crossref's abstracts arrive as JATS markup.
|
||||
record.summary = work.summary.map(|summary| strip_tags(&summary));
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// The arXiv answers in Atom, which is XML. Reading it properly would mean a
|
||||
/// parser; what is wanted is four fields, and they are each one tag.
|
||||
fn arxiv(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
// An empty feed means no such preprint, which the API signals by omitting
|
||||
// the entry rather than by a status code.
|
||||
let Some(entry) = between(body, "<entry>", "</entry>") else {
|
||||
return Ok(crate::missing(id, kind, Source::Arxiv));
|
||||
};
|
||||
|
||||
let mut record = blank(id, kind, Source::Arxiv);
|
||||
record.title = between(entry, "<title>", "</title>").map(collapse);
|
||||
record.summary = between(entry, "<summary>", "</summary>").map(collapse);
|
||||
record.container = Some("arXiv".to_string());
|
||||
record.year = between(entry, "<published>", "</published>")
|
||||
.and_then(|published| published.get(..4))
|
||||
.and_then(|year| year.parse().ok());
|
||||
|
||||
let mut authors = Vec::new();
|
||||
let mut rest = entry;
|
||||
while let Some(author) = between(rest, "<author>", "</author>") {
|
||||
if let Some(name) = between(author, "<name>", "</name>") {
|
||||
authors.push(collapse(name));
|
||||
}
|
||||
let Some(at) = rest.find("</author>") else {
|
||||
break;
|
||||
};
|
||||
rest = &rest[at + "</author>".len()..];
|
||||
}
|
||||
record.authors = authors;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OpenLibraryBook {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
publish_date: Option<String>,
|
||||
#[serde(default)]
|
||||
publishers: Vec<String>,
|
||||
}
|
||||
|
||||
fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
let book: OpenLibraryBook =
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
|
||||
let mut record = blank(id, kind, Source::OpenLibrary);
|
||||
record.title = book.title;
|
||||
record.container = book.publishers.into_iter().next();
|
||||
// The date is free text — "1999", "March 1999", "1999-03-01".
|
||||
record.year = book.publish_date.as_deref().and_then(year_in);
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// The first four-digit year in a string.
|
||||
fn year_in(text: &str) -> Option<i64> {
|
||||
let bytes: Vec<char> = text.chars().collect();
|
||||
for window in bytes.windows(4) {
|
||||
if window.iter().all(char::is_ascii_digit) {
|
||||
let year: i64 = window.iter().collect::<String>().parse().ok()?;
|
||||
if (1000..=2999).contains(&year) {
|
||||
return Some(year);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// What sits between two markers, the first time they appear in order.
|
||||
fn between<'a>(text: &'a str, open: &str, close: &str) -> Option<&'a str> {
|
||||
let start = text.find(open)? + open.len();
|
||||
let end = text[start..].find(close)? + start;
|
||||
Some(&text[start..end])
|
||||
}
|
||||
|
||||
/// Runs of whitespace down to one space, trimmed.
|
||||
fn collapse(text: &str) -> String {
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// Text with its tags removed.
|
||||
///
|
||||
/// Crossref hands back JATS — `<jats:p>` and friends — which is markup for a
|
||||
/// typesetter rather than anything a reader wants.
|
||||
fn strip_tags(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut inside = false;
|
||||
for c in text.chars() {
|
||||
match c {
|
||||
'<' => inside = true,
|
||||
'>' => inside = false,
|
||||
c if !inside => out.push(c),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
collapse(&out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn each_kind_goes_to_the_body_that_assigns_it() {
|
||||
assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
|
||||
assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
|
||||
// No register answers about an ISSN without a key.
|
||||
assert_eq!(Source::for_kind("issn"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_doi_suffix_cannot_change_the_shape_of_the_url() {
|
||||
let url = Source::Crossref.url("10.1000/a?b#c/d");
|
||||
assert!(url.ends_with("10.1000%2Fa%3Fb%23c%2Fd"), "{url}");
|
||||
assert_eq!(url.matches('?').count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crossref_is_read_into_the_fields_that_matter() {
|
||||
let body = r#"{"message":{"title":["On Computable Numbers"],
|
||||
"author":[{"given":"Alan","family":"Turing"},{"family":"Church"},{"name":"A Committee"}],
|
||||
"container-title":["Proceedings"],
|
||||
"issued":{"date-parts":[[1936,11,12]]},
|
||||
"abstract":"<jats:p>A paragraph.</jats:p>"}}"#;
|
||||
let record = crossref("10.1", "doi", body).expect("a record");
|
||||
assert!(record.found);
|
||||
assert_eq!(record.title.as_deref(), Some("On Computable Numbers"));
|
||||
assert_eq!(record.authors, ["Alan Turing", "Church", "A Committee"]);
|
||||
assert_eq!(record.year, Some(1936));
|
||||
assert_eq!(record.container.as_deref(), Some("Proceedings"));
|
||||
assert_eq!(record.summary.as_deref(), Some("A paragraph."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_crossref_reply_missing_everything_is_still_a_record() {
|
||||
let record = crossref("10.1", "doi", r#"{"message":{}}"#).expect("a record");
|
||||
assert!(record.found);
|
||||
assert_eq!(record.title, None);
|
||||
assert!(record.authors.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_arxiv_feed_with_no_entry_means_no_such_preprint() {
|
||||
let record = arxiv("2401.01234", "arxiv", "<feed></feed>").expect("a record");
|
||||
assert!(!record.found);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_arxiv_entry_is_read_out_of_its_atom() {
|
||||
let body = "<feed><entry><title>A\n Title</title>\
|
||||
<summary> Some words. </summary>\
|
||||
<published>2024-01-02T00:00:00Z</published>\
|
||||
<author><name>Ada Lovelace</name></author>\
|
||||
<author><name>Charles Babbage</name></author></entry></feed>";
|
||||
let record = arxiv("2401.01234", "arxiv", body).expect("a record");
|
||||
assert!(record.found);
|
||||
assert_eq!(record.title.as_deref(), Some("A Title"));
|
||||
assert_eq!(record.summary.as_deref(), Some("Some words."));
|
||||
assert_eq!(record.year, Some(2024));
|
||||
assert_eq!(record.authors, ["Ada Lovelace", "Charles Babbage"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openlibrary_gives_up_a_year_from_whatever_it_wrote() {
|
||||
for (date, year) in [
|
||||
("1999", Some(1999)),
|
||||
("March 1999", Some(1999)),
|
||||
("1999-03-01", Some(1999)),
|
||||
("n.d.", None),
|
||||
] {
|
||||
let body = format!(r#"{{"title":"A Book","publish_date":"{date}"}}"#);
|
||||
let record = open_library("1", "isbn", &body).expect("a record");
|
||||
assert_eq!(record.year, year, "{date}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_that_is_not_what_it_should_be_says_so() {
|
||||
assert!(matches!(
|
||||
crossref("10.1", "doi", "not json"),
|
||||
Err(Error::Shape(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markup_is_not_part_of_an_abstract() {
|
||||
assert_eq!(strip_tags("<jats:p>One <i>two</i></jats:p>"), "One two");
|
||||
assert_eq!(strip_tags("plain"), "plain");
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ crate-type = ["cdylib", "rlib"]
|
||||
booksniff.workspace = true
|
||||
chunkmd.workspace = true
|
||||
corpus-core.workspace = true
|
||||
idnorm.workspace = true
|
||||
cuekit.workspace = true
|
||||
cuepack.workspace = true
|
||||
mdscrub.workspace = true
|
||||
|
||||
@@ -68,6 +68,7 @@ except ValueError as why:
|
||||
| `fuse_hits` | [`rrf`](../rrf/README.md) |
|
||||
| `identify` | [`booksniff`](../booksniff/README.md) |
|
||||
| `series_of` | [`seriesof`](../seriesof/README.md) |
|
||||
| `parse_identifier` | [`idnorm`](../idnorm/README.md) |
|
||||
|
||||
The tools that are not here are the ones a binding would gain nothing over a
|
||||
subprocess for: they read files, shell out to another program, or talk to a
|
||||
|
||||
@@ -49,6 +49,7 @@ try {
|
||||
| `fuseHits` | Several ranked lists into one |
|
||||
| `identify` | What a file is, from its first bytes |
|
||||
| `seriesOf` | The series a book belongs to, and which volume this is |
|
||||
| `parseIdentifier` | A DOI, ISSN, ISBN or arXiv id, read and checked |
|
||||
|
||||
The tools that read files or call another program are not here: a browser has
|
||||
neither, and a binding that pretends otherwise fails at run time rather than
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
// check, and nothing else would notice.
|
||||
import {
|
||||
chunkMarkdown, scrub, countTokens, parseSubtitles, packCues,
|
||||
splitCollection, fuseHits, identify, seriesOf,
|
||||
type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal,
|
||||
splitCollection, fuseHits, identify, seriesOf, parseIdentifier,
|
||||
type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, type Identifier,
|
||||
} from "./dist/web/catena.js";
|
||||
|
||||
const chunks: Chunk[] = chunkMarkdown("# One\n\nbody\n", 512, 64, 50, 768);
|
||||
@@ -39,4 +39,7 @@ const proposal: SeriesProposal | undefined = seriesOf("The Philokalia Vol 3");
|
||||
const volume: number | undefined = proposal?.volume;
|
||||
seriesOf("Aquinas (Companion)", "Cambridge");
|
||||
|
||||
export const checked = [volume, page, chapter, flags, clean, tokens, start, packed, slug, fused, kind, notes];
|
||||
const ident: Identifier | undefined = parseIdentifier("10.1000/182");
|
||||
const checksum: boolean | null | undefined = ident?.checksum;
|
||||
|
||||
export const checked = [volume, checksum, page, chapter, flags, clean, tokens, start, packed, slug, fused, kind, notes];
|
||||
|
||||
@@ -17,6 +17,7 @@ from ._catena import (
|
||||
fuse_hits,
|
||||
identify,
|
||||
pack_cues,
|
||||
parse_identifier,
|
||||
parse_subtitles,
|
||||
scrub,
|
||||
series_of,
|
||||
@@ -30,6 +31,7 @@ __all__ = [
|
||||
"fuse_hits",
|
||||
"identify",
|
||||
"pack_cues",
|
||||
"parse_identifier",
|
||||
"parse_subtitles",
|
||||
"scrub",
|
||||
"series_of",
|
||||
|
||||
@@ -30,6 +30,7 @@ def fuse_hits(
|
||||
limit: int | None = ...,
|
||||
) -> list[dict[str, Any]]: ...
|
||||
def identify(name: str, data: bytes) -> dict[str, Any]: ...
|
||||
def parse_identifier(text: str) -> dict[str, Any] | None: ...
|
||||
def series_of(
|
||||
title: str, author: str | None = ...
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@@ -385,6 +385,24 @@ pub unsafe extern "C" fn catena_series_of(
|
||||
}
|
||||
}
|
||||
|
||||
/// Read an identifier out of whatever it arrived wrapped in. Returns a JSON
|
||||
/// record, or NULL when the string is not one — an answer, not an error.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_parse_identifier(text: *const c_char) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(text) = (unsafe { borrow(text, "text") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
match crate::parse_identifier(text) {
|
||||
Some(found) => give_json(&found),
|
||||
None => ptr::null_mut(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -496,6 +514,17 @@ mod tests {
|
||||
assert!(json.contains("\"volume\":3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_identifier_is_normalised_across_the_boundary() {
|
||||
let text = c"https://doi.org/10.1000/182";
|
||||
let json = through(|| unsafe { catena_parse_identifier(text.as_ptr()) }).expect("a record");
|
||||
assert!(json.contains("\"id\":\"10.1000/182\""), "{json}");
|
||||
|
||||
let nope = c"hello";
|
||||
assert!(through(|| unsafe { catena_parse_identifier(nope.as_ptr()) }).is_none());
|
||||
assert_eq!(error(), None, "not an identifier is an answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_abi_version_is_the_one_the_header_declares() {
|
||||
assert_eq!(catena_abi_version(), 1);
|
||||
|
||||
@@ -176,6 +176,12 @@ pub fn identify(name: &str, bytes: &[u8]) -> Doc {
|
||||
booksniff::sniff(&bytes, name)
|
||||
}
|
||||
|
||||
/// Read an identifier — a DOI, ISSN, ISBN or arXiv id — out of whatever it
|
||||
/// arrived wrapped in.
|
||||
pub fn parse_identifier(text: &str) -> Option<idnorm::Identifier> {
|
||||
idnorm::parse(text)
|
||||
}
|
||||
|
||||
/// The series a title belongs to, and which volume of it this is.
|
||||
pub fn series_of(title: &str, author: Option<&str>) -> Option<seriesof::Proposal> {
|
||||
seriesof::series_of(title, author)
|
||||
@@ -237,6 +243,13 @@ mod tests {
|
||||
assert_eq!(channel_named("lexical"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_identifier_comes_back_normalised() {
|
||||
let found = parse_identifier("https://doi.org/10.1000/182").expect("an identifier");
|
||||
assert_eq!(found.id, "10.1000/182");
|
||||
assert_eq!(parse_identifier("hello"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_series_comes_back_the_same_as_the_library_reads_it() {
|
||||
let found = series_of("The Philokalia Vol 3", None).expect("a series");
|
||||
|
||||
@@ -208,6 +208,17 @@ fn series_of<'py>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Read an identifier out of whatever it arrived wrapped in.
|
||||
///
|
||||
/// `None` when the string is not one.
|
||||
#[pyfunction]
|
||||
fn parse_identifier<'py>(py: Python<'py>, text: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
match crate::parse_identifier(text) {
|
||||
Some(found) => Ok(Some(to_py(py, &found)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The module Python imports.
|
||||
#[pymodule]
|
||||
fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
@@ -221,5 +232,6 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
module.add_function(wrap_pyfunction!(fuse_hits, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(identify, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(series_of, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(parse_identifier, module)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -70,6 +70,21 @@ export interface Hit {
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
/** A DOI, ISSN, ISBN or arXiv identifier, read and checked. */
|
||||
export interface Identifier {
|
||||
v: number;
|
||||
/** The text it was read from. */
|
||||
input: string;
|
||||
/** "doi", "issn", "isbn" or "arxiv". */
|
||||
kind: string;
|
||||
/** The identifier, in the form it should be stored and compared in. */
|
||||
id: string;
|
||||
/** Whether the check digit agrees, or null where the kind carries none. */
|
||||
checksum: boolean | null;
|
||||
/** A URL that resolves it. */
|
||||
url?: string;
|
||||
}
|
||||
|
||||
/** What a title says about its series. */
|
||||
export interface SeriesProposal {
|
||||
v: number;
|
||||
@@ -233,6 +248,17 @@ pub fn series_of(title: &str, author: Option<String>) -> Result<JsValue, JsValue
|
||||
}
|
||||
}
|
||||
|
||||
/// Read an identifier out of whatever it arrived wrapped in.
|
||||
///
|
||||
/// `undefined` when the string is not one.
|
||||
#[wasm_bindgen(js_name = parseIdentifier, skip_typescript)]
|
||||
pub fn parse_identifier(text: &str) -> Result<JsValue, JsValue> {
|
||||
match crate::parse_identifier(text) {
|
||||
Some(found) => to_js(&found),
|
||||
None => Ok(JsValue::UNDEFINED),
|
||||
}
|
||||
}
|
||||
|
||||
/// Declarations for the functions whose return type the generator cannot
|
||||
/// infer, so a caller gets `Chunk[]` rather than `any`.
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
@@ -244,4 +270,5 @@ export function splitCollection(markdown: string): Section[];
|
||||
export function fuseHits(hits: Hit[], k?: number, weights?: Record<string, number>, limit?: number): Hit[];
|
||||
export function identify(name: string, data: Uint8Array): Doc;
|
||||
export function seriesOf(title: string, author?: string): SeriesProposal | undefined;
|
||||
export function parseIdentifier(text: string): Identifier | undefined;
|
||||
"#;
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "idnorm"
|
||||
description = "The identifiers a bibliography is held together by, read and checked."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bibclean.workspace = true
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
fancy-regex.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,126 @@
|
||||
# idnorm
|
||||
|
||||
The identifiers a bibliography is held together by, read and checked.
|
||||
|
||||
Part of [catena](../../README.md).
|
||||
|
||||
## What it does
|
||||
|
||||
A DOI arrives bare, as a URL, with a `doi:` prefix, with the full stop the
|
||||
sentence around it left behind, in whatever case the publisher felt like. All
|
||||
of those are one identifier. A catalogue that stores them as they arrived
|
||||
cannot tell, so the same paper is three records and none of them join up.
|
||||
|
||||
`idnorm` reads one out of whatever it is wrapped in and writes it in the form
|
||||
it should be stored and compared in.
|
||||
|
||||
## When you would use it
|
||||
|
||||
- **Before storing a reference**, so that two spellings of one DOI are one row.
|
||||
- **Auditing a bibliography** for identifiers that cannot be right — an ISBN
|
||||
and an ISSN carry a check digit, and a mistyped one fails it.
|
||||
- **Pulling identifiers out of running text**, where the surrounding sentence
|
||||
is attached to them.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
cargo install --git https://git.pumpin.online/funkpower/catena idnorm
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
$ printf '10.1000/182\ndoi:10.1093/mind/LIX.236.433\n0028-0836\n978-0-306-40615-7\narXiv:2401.01234\n' | idnorm
|
||||
{"v":1,"input":"10.1000/182","kind":"doi","id":"10.1000/182","checksum":null,"url":"https://doi.org/10.1000/182"}
|
||||
{"v":1,"input":"doi:10.1093/mind/LIX.236.433","kind":"doi","id":"10.1093/mind/lix.236.433","checksum":null,"url":"https://doi.org/10.1093/mind/lix.236.433"}
|
||||
{"v":1,"input":"0028-0836","kind":"issn","id":"0028-0836","checksum":true,"url":"https://portal.issn.org/resource/ISSN/0028-0836"}
|
||||
{"v":1,"input":"978-0-306-40615-7","kind":"isbn","id":"9780306406157","checksum":true,"url":"https://openlibrary.org/isbn/9780306406157"}
|
||||
{"v":1,"input":"arXiv:2401.01234","kind":"arxiv","id":"2401.01234","checksum":null,"url":"https://arxiv.org/abs/2401.01234"}
|
||||
```
|
||||
|
||||
The ones that cannot be right:
|
||||
|
||||
```sh
|
||||
idnorm refs.txt | jq -r 'select(.checksum == false) | .input'
|
||||
```
|
||||
|
||||
What a line was not:
|
||||
|
||||
```
|
||||
$ printf 'hello\n' | idnorm
|
||||
idnorm: not an identifier: hello
|
||||
$ echo $?
|
||||
1
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `--kind <doi\|issn\|isbn\|arxiv>` | Read every line as this kind rather than whatever it looks like |
|
||||
| `--quiet` | Say nothing about the lines that are not identifiers |
|
||||
| `--man` | Write this page as a manual page, in roff |
|
||||
|
||||
## Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| `0` | Every line was an identifier |
|
||||
| `1` | At least one was not |
|
||||
| `64` | The command line was wrong |
|
||||
| `74` | The file could not be read |
|
||||
|
||||
## What it can and cannot tell you
|
||||
|
||||
**It can say a string is well formed.** An ISBN and an ISSN carry a check
|
||||
digit, weighted by position, which catches one mistyped figure and two
|
||||
transposed ones. A DOI and an arXiv identifier carry no checksum at all, so
|
||||
for those `checksum` is `null` rather than `true` — there is nothing to check
|
||||
against, and saying `true` would claim more than is known.
|
||||
|
||||
**It cannot say the work exists.** That is a fact about the world rather than
|
||||
about the string, and the only way to learn it is to ask a register.
|
||||
[`bibfetch`](../bibfetch/README.md) does that:
|
||||
|
||||
```sh
|
||||
idnorm refs.txt | bibfetch | jq -r 'select(.found | not) | .id'
|
||||
```
|
||||
|
||||
Those two together are what "validate against a catalogue" means, and they are
|
||||
two tools because one of them touches the network and the other must not.
|
||||
|
||||
## The four kinds
|
||||
|
||||
**DOI.** A registrant prefix beginning `10.` and a suffix the registrant
|
||||
chooses. Stored lowercased, because a DOI is case-insensitive and two
|
||||
catalogues will disagree about the case of one. Read out of running text, with
|
||||
the trailing full stop, comma or bracket the sentence left on it removed.
|
||||
|
||||
**ISSN.** Eight digits in two fours, the last a check digit that may be `X`
|
||||
for ten. Stored hyphenated.
|
||||
|
||||
**ISBN.** Ten or thirteen digits. Stored as digits, without the hyphens, since
|
||||
where a publisher puts them varies. Only where the line is the identifier and
|
||||
nothing else — an ISBN has no prefix announcing it, so a sentence with a
|
||||
thirteen-digit number in it would otherwise read as one.
|
||||
|
||||
**arXiv.** Both forms: `2401.01234` since 2007, and `math.GT/0309136` before
|
||||
it. A version suffix is kept, because `v2` names a particular revision and
|
||||
citing without one means the latest.
|
||||
|
||||
## As a library
|
||||
|
||||
```rust
|
||||
let found = idnorm::parse("https://doi.org/10.1000/182").expect("an identifier");
|
||||
assert_eq!(found.kind, idnorm::Kind::Doi);
|
||||
assert_eq!(found.id, "10.1000/182");
|
||||
```
|
||||
|
||||
`idnorm::doi`, `idnorm::issn` and `idnorm::arxiv` read one kind each, and
|
||||
`idnorm::issn::valid` checks a serial number's check digit.
|
||||
|
||||
## See also
|
||||
|
||||
- [`bibfetch`](../bibfetch/README.md) — what a register says about one.
|
||||
- [`bibclean`](../bibclean/README.md) — the rest of a catalogue record.
|
||||
@@ -0,0 +1,122 @@
|
||||
//! arXiv preprint identifiers.
|
||||
//!
|
||||
//! Two forms, because the scheme changed in 2007. The new one is four digits
|
||||
//! of year and month, a dot, and a number: `2401.01234`. The old one is an
|
||||
//! archive name, optionally a subject class, a slash, and seven digits:
|
||||
//! `math.GT/0309136`.
|
||||
//!
|
||||
//! Either may carry a version — `v2` — which is part of the identifier: it
|
||||
//! names a particular revision, and citing one without it means the latest.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use fancy_regex::Regex;
|
||||
|
||||
/// The post-2007 form: `YYMM.NNNNN`, four or five digits after the dot.
|
||||
static MODERN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(\d{4}\.\d{4,5}(?:v\d+)?)\b").expect("a valid pattern"));
|
||||
|
||||
/// The pre-2007 form: an archive, an optional subject class, then seven
|
||||
/// digits.
|
||||
static LEGACY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\b([a-z][a-z-]{1,20}(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?)\b")
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// Read an arXiv identifier out of a string.
|
||||
///
|
||||
/// The old form keeps the case of its subject class, which is capitalised by
|
||||
/// convention and is how the archives are written; the rest is lowercased.
|
||||
pub fn parse(text: &str) -> Option<String> {
|
||||
let text = text.trim();
|
||||
let without = strip_prefix(text);
|
||||
|
||||
if let Some(found) = LEGACY.captures(without).ok().flatten() {
|
||||
let id = found.get(1)?.as_str();
|
||||
let (archive, number) = id.split_once('/')?;
|
||||
return Some(format!(
|
||||
"{}/{}",
|
||||
lowercase_archive(archive),
|
||||
number.to_lowercase()
|
||||
));
|
||||
}
|
||||
let found = MODERN.captures(without).ok().flatten()?;
|
||||
Some(found.get(1)?.as_str().to_lowercase())
|
||||
}
|
||||
|
||||
/// The archive lowercased, but its subject class left as it was written.
|
||||
fn lowercase_archive(archive: &str) -> String {
|
||||
match archive.split_once('.') {
|
||||
Some((name, class)) => format!("{}.{}", name.to_lowercase(), class.to_uppercase()),
|
||||
None => archive.to_lowercase(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whatever introduced it.
|
||||
fn strip_prefix(text: &str) -> &str {
|
||||
for prefix in [
|
||||
"https://arxiv.org/abs/",
|
||||
"http://arxiv.org/abs/",
|
||||
"arxiv:",
|
||||
"arxiv ",
|
||||
] {
|
||||
if text.len() >= prefix.len() && text[..prefix.len()].eq_ignore_ascii_case(prefix) {
|
||||
return text[prefix.len()..].trim();
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_modern_form_reads_as_itself() {
|
||||
assert_eq!(parse("2401.01234").as_deref(), Some("2401.01234"));
|
||||
// Four digits after the dot before 2015, five after it.
|
||||
assert_eq!(parse("0704.0001").as_deref(), Some("0704.0001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_version_is_part_of_the_identifier() {
|
||||
// It names a revision; without it a citation means the latest.
|
||||
assert_eq!(parse("2401.01234v2").as_deref(), Some("2401.01234v2"));
|
||||
assert_eq!(
|
||||
parse("math.GT/0309136v1").as_deref(),
|
||||
Some("math.GT/0309136v1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_old_form_keeps_its_subject_class_capitalised() {
|
||||
assert_eq!(parse("math.GT/0309136").as_deref(), Some("math.GT/0309136"));
|
||||
assert_eq!(parse("MATH.gt/0309136").as_deref(), Some("math.GT/0309136"));
|
||||
assert_eq!(parse("hep-th/9901001").as_deref(), Some("hep-th/9901001"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_way_of_introducing_one_gives_the_same_identifier() {
|
||||
for text in [
|
||||
"arXiv:2401.01234",
|
||||
"arxiv:2401.01234",
|
||||
"arXiv 2401.01234",
|
||||
"https://arxiv.org/abs/2401.01234",
|
||||
] {
|
||||
assert_eq!(parse(text).as_deref(), Some("2401.01234"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_an_arxiv_identifier_is_not_one() {
|
||||
for text in ["", "hello", "24.01234", "2401.012", "2401.012345"] {
|
||||
assert_eq!(parse(text), None, "{text:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_doi_is_not_mistaken_for_one() {
|
||||
// `10.1000/182` has a slash and digits, but not seven of them.
|
||||
assert_eq!(parse("10.1000/182"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! Digital object identifiers.
|
||||
//!
|
||||
//! A DOI is a registrant prefix beginning `10.` and a suffix the registrant
|
||||
//! chooses, separated by a slash. The suffix may be almost anything, which is
|
||||
//! what makes reading one out of running text hard: there is no character
|
||||
//! that reliably ends it.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use fancy_regex::Regex;
|
||||
|
||||
/// The shape, wherever it appears and however it was introduced.
|
||||
///
|
||||
/// The suffix stops at whitespace or at one of the brackets and quotes that
|
||||
/// wrap a citation, because those are never part of one in practice and
|
||||
/// leaving them on makes two spellings of the same identifier.
|
||||
static DOI: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)\b(10\.\d{4,9}/[^\s"'<>,;()\[\]{}]+)"#).expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// The prefixes a DOI arrives wearing.
|
||||
const PREFIXES: &[&str] = &[
|
||||
"https://doi.org/",
|
||||
"http://doi.org/",
|
||||
"https://dx.doi.org/",
|
||||
"http://dx.doi.org/",
|
||||
"doi:",
|
||||
"doi ",
|
||||
];
|
||||
|
||||
/// Read a DOI out of a string, in the form it should be stored in.
|
||||
///
|
||||
/// Lowercased, because a DOI is case-insensitive and two catalogues will
|
||||
/// disagree about the case of the same one. The trailing punctuation a
|
||||
/// sentence leaves behind is removed.
|
||||
pub fn parse(text: &str) -> Option<String> {
|
||||
let text = text.trim();
|
||||
let mut without = text;
|
||||
for prefix in PREFIXES {
|
||||
if without.len() >= prefix.len() && without[..prefix.len()].eq_ignore_ascii_case(prefix) {
|
||||
without = &without[prefix.len()..];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let found = DOI.captures(without).ok().flatten()?;
|
||||
let id = found.get(1)?.as_str();
|
||||
|
||||
// A full stop, comma or semicolon at the end belongs to the sentence
|
||||
// rather than to the identifier. A closing bracket does too, unless the
|
||||
// suffix opened one.
|
||||
let id = id.trim_end_matches(['.', ',', ';', ':']);
|
||||
if id.len() < 6 || !id.contains('/') {
|
||||
return None;
|
||||
}
|
||||
// A suffix of nothing is not a suffix.
|
||||
if id.split_once('/')?.1.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(id.to_lowercase())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_plain_form_reads_as_itself() {
|
||||
assert_eq!(parse("10.1000/182").as_deref(), Some("10.1000/182"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_way_of_introducing_one_gives_the_same_identifier() {
|
||||
for text in [
|
||||
"doi:10.1000/182",
|
||||
"DOI: 10.1000/182",
|
||||
"https://doi.org/10.1000/182",
|
||||
"http://dx.doi.org/10.1000/182",
|
||||
"HTTPS://DOI.ORG/10.1000/182",
|
||||
] {
|
||||
assert_eq!(parse(text).as_deref(), Some("10.1000/182"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_sentence_around_it_is_not_part_of_it() {
|
||||
assert_eq!(
|
||||
parse("See 10.1093/mind/LIX.236.433, which argues otherwise.").as_deref(),
|
||||
Some("10.1093/mind/lix.236.433")
|
||||
);
|
||||
assert_eq!(parse("(10.1000/182)").as_deref(), Some("10.1000/182"));
|
||||
assert_eq!(parse("\"10.1000/182\"").as_deref(), Some("10.1000/182"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn case_is_not_part_of_it_either() {
|
||||
// A DOI is case-insensitive, and two catalogues will disagree.
|
||||
assert_eq!(
|
||||
parse("10.1093/MIND/LIX.236.433").as_deref(),
|
||||
parse("10.1093/mind/lix.236.433").as_deref()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_suffix_may_hold_almost_anything() {
|
||||
assert_eq!(
|
||||
parse("10.1371/journal.pone.0012345").as_deref(),
|
||||
Some("10.1371/journal.pone.0012345")
|
||||
);
|
||||
assert_eq!(
|
||||
parse("10.1016/j.cell.2020.01.001").as_deref(),
|
||||
Some("10.1016/j.cell.2020.01.001")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_doi_is_not_one() {
|
||||
for text in ["", "10.", "10.1000", "11.1000/182", "10.100/182", "hello"] {
|
||||
assert_eq!(parse(text), None, "{text:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prefix_with_no_identifier_after_it_is_nothing() {
|
||||
assert_eq!(parse("doi:"), None);
|
||||
assert_eq!(parse("https://doi.org/"), None);
|
||||
assert_eq!(parse("10.1000/"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! International standard serial numbers, which identify a journal.
|
||||
//!
|
||||
//! Eight digits in two groups of four, the last of which is a check digit
|
||||
//! that may be `X` for ten.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use fancy_regex::Regex;
|
||||
|
||||
/// Eight characters in two fours, with or without the hyphen, and with
|
||||
/// whatever introduced it.
|
||||
static ISSN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(\d{4})-?(\d{3}[\dX])\b").expect("a valid pattern"));
|
||||
|
||||
/// Read an ISSN out of a string, hyphenated as it should be stored.
|
||||
pub fn parse(text: &str) -> Option<String> {
|
||||
let text = text.trim();
|
||||
let without = text
|
||||
.strip_prefix("ISSN ")
|
||||
.or_else(|| text.strip_prefix("issn "))
|
||||
.or_else(|| text.strip_prefix("ISSN:"))
|
||||
.or_else(|| text.strip_prefix("issn:"))
|
||||
.unwrap_or(text)
|
||||
.trim();
|
||||
|
||||
let found = ISSN.captures(without).ok().flatten()?;
|
||||
Some(format!(
|
||||
"{}-{}",
|
||||
found.get(1)?.as_str(),
|
||||
found.get(2)?.as_str().to_uppercase()
|
||||
))
|
||||
}
|
||||
|
||||
/// Whether the check digit agrees with the other seven.
|
||||
///
|
||||
/// Each of the first seven digits is weighted by its position counting down
|
||||
/// from eight, and the total plus the check digit must be a multiple of
|
||||
/// eleven. One mistyped digit, or two transposed, fails.
|
||||
pub fn valid(issn: &str) -> bool {
|
||||
let digits: Vec<char> = issn
|
||||
.chars()
|
||||
.filter(|c| !c.is_whitespace() && *c != '-')
|
||||
.collect();
|
||||
if digits.len() != 8 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut total = 0u32;
|
||||
for (index, digit) in digits[..7].iter().enumerate() {
|
||||
let Some(value) = digit.to_digit(10) else {
|
||||
return false;
|
||||
};
|
||||
total += value * (8 - index as u32);
|
||||
}
|
||||
let check = match digits[7] {
|
||||
'X' | 'x' => 10,
|
||||
digit => match digit.to_digit(10) {
|
||||
Some(value) => value,
|
||||
None => return false,
|
||||
},
|
||||
};
|
||||
(total + check) % 11 == 0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_real_issn_checks_out() {
|
||||
// Nature.
|
||||
assert!(valid("0028-0836"));
|
||||
assert!(valid("00280836"));
|
||||
// Science.
|
||||
assert!(valid("0036-8075"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_digit_out_does_not() {
|
||||
assert!(!valid("0028-0837"));
|
||||
assert!(!valid("0028-0835"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_digits_swapped_does_not_either() {
|
||||
// Which is the whole point of a weighted checksum.
|
||||
assert!(!valid("0082-0836"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_check_digit_of_ten_is_written_x() {
|
||||
assert!(valid("1050-124X"));
|
||||
assert!(valid("1050-124x"), "however it is cased");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_the_wrong_length_or_shape_is_not_valid() {
|
||||
assert!(!valid(""));
|
||||
assert!(!valid("0028-083"));
|
||||
assert!(!valid("0028-08366"));
|
||||
assert!(!valid("0028-083Y"));
|
||||
assert!(!valid("ABCD-EFGH"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn it_is_read_however_it_was_written() {
|
||||
for text in [
|
||||
"0028-0836",
|
||||
"00280836",
|
||||
"ISSN 0028-0836",
|
||||
"ISSN: 0028-0836",
|
||||
" 0028-0836 ",
|
||||
] {
|
||||
assert_eq!(parse(text).as_deref(), Some("0028-0836"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lower_case_check_digit_is_stored_upper() {
|
||||
assert_eq!(parse("1050-124x").as_deref(), Some("1050-124X"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_an_issn_is_not_one() {
|
||||
for text in ["", "hello", "123", "0028-08AB"] {
|
||||
assert_eq!(parse(text), None, "{text:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
//! The identifiers a bibliography is held together by, read and checked.
|
||||
//!
|
||||
//! A DOI arrives as a bare string, a URL, a `doi:` prefix, with a trailing
|
||||
//! full stop the sentence around it left behind, in whatever case the
|
||||
//! publisher felt like. All of those are the same identifier, and a catalogue
|
||||
//! that stores them as they arrived cannot tell that.
|
||||
//!
|
||||
//! Nothing here asks anybody whether an identifier exists. What it can say is
|
||||
//! whether a string is *well formed*: an ISBN and an ISSN carry a check digit
|
||||
//! that catches a mistyped one, and a DOI or an arXiv id has a shape. That is
|
||||
//! a different question from whether the work is real, and the honest answer
|
||||
//! to the second needs a catalogue.
|
||||
|
||||
pub mod arxiv;
|
||||
pub mod doi;
|
||||
pub mod issn;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What kind of identifier something is.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Kind {
|
||||
/// A digital object identifier: `10.1000/182`.
|
||||
Doi,
|
||||
/// An international standard serial number, for a journal: `0028-0836`.
|
||||
Issn,
|
||||
/// An international standard book number, for a book.
|
||||
Isbn,
|
||||
/// An arXiv preprint identifier: `2401.01234` or `math.GT/0309136`.
|
||||
Arxiv,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
/// What it is called.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Kind::Doi => "doi",
|
||||
Kind::Issn => "issn",
|
||||
Kind::Isbn => "isbn",
|
||||
Kind::Arxiv => "arxiv",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One identifier, as it arrived and as it should be stored.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Identifier {
|
||||
/// Record version.
|
||||
#[serde(default = "version")]
|
||||
pub v: u32,
|
||||
/// The text it was read from.
|
||||
pub input: String,
|
||||
/// Which kind it turned out to be.
|
||||
pub kind: Kind,
|
||||
/// The identifier, in the form it should be stored and compared in.
|
||||
pub id: String,
|
||||
/// Whether the check digit agrees, where the kind carries one. `None`
|
||||
/// where the kind has nothing to check against.
|
||||
pub checksum: Option<bool>,
|
||||
/// A URL that resolves it, where the kind has a resolver.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
corpus_core::V
|
||||
}
|
||||
|
||||
impl corpus_core::Record for Identifier {
|
||||
fn version(&self) -> u32 {
|
||||
self.v
|
||||
}
|
||||
}
|
||||
|
||||
/// Read whatever kind of identifier a string holds.
|
||||
///
|
||||
/// Tried in order of how distinctive each shape is: a DOI announces itself
|
||||
/// with `10.`, an arXiv id with its own prefix or a dotted archive name, and
|
||||
/// what is left is a run of digits that is either thirteen or ten long (an
|
||||
/// ISBN) or eight (an ISSN).
|
||||
pub fn parse(text: &str) -> Option<Identifier> {
|
||||
parse_as(text, None)
|
||||
}
|
||||
|
||||
/// Read a string as one particular kind, or as whatever it looks like.
|
||||
pub fn parse_as(text: &str, kind: Option<Kind>) -> Option<Identifier> {
|
||||
let input = text.trim();
|
||||
if input.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Order matters: a DOI announces itself with `10.`, an arXiv id with its
|
||||
// own prefix, and what is left is a run of digits whose length decides
|
||||
// between a book and a journal.
|
||||
const EVERY: [Kind; 4] = [Kind::Doi, Kind::Arxiv, Kind::Isbn, Kind::Issn];
|
||||
let one = [kind.unwrap_or(Kind::Doi)];
|
||||
let kinds: &[Kind] = match kind {
|
||||
Some(_) => &one,
|
||||
None => &EVERY,
|
||||
};
|
||||
|
||||
for kind in kinds {
|
||||
let found = match kind {
|
||||
Kind::Doi => doi::parse(input).map(|id| Identifier {
|
||||
v: corpus_core::V,
|
||||
url: Some(format!("https://doi.org/{id}")),
|
||||
input: input.to_string(),
|
||||
kind: Kind::Doi,
|
||||
id,
|
||||
checksum: None,
|
||||
}),
|
||||
Kind::Arxiv => arxiv::parse(input).map(|id| Identifier {
|
||||
v: corpus_core::V,
|
||||
url: Some(format!("https://arxiv.org/abs/{id}")),
|
||||
input: input.to_string(),
|
||||
kind: Kind::Arxiv,
|
||||
id,
|
||||
checksum: None,
|
||||
}),
|
||||
Kind::Issn => issn::parse(input).map(|id| Identifier {
|
||||
v: corpus_core::V,
|
||||
checksum: Some(issn::valid(&id)),
|
||||
url: Some(format!("https://portal.issn.org/resource/ISSN/{id}")),
|
||||
input: input.to_string(),
|
||||
kind: Kind::Issn,
|
||||
id,
|
||||
}),
|
||||
Kind::Isbn => isbn(input),
|
||||
};
|
||||
if let Some(found) = found {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// An ISBN, using the reader the catalogue tools already share.
|
||||
fn isbn(input: &str) -> Option<Identifier> {
|
||||
let digits = bibclean::isbn::digits(input);
|
||||
if !matches!(digits.len(), 10 | 13) {
|
||||
return None;
|
||||
}
|
||||
// Only where the input is the identifier and nothing else. An ISBN has no
|
||||
// prefix that announces it, so a sentence holding a thirteen-digit number
|
||||
// would otherwise read as one.
|
||||
if !input
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_digit() || "- xX".contains(c))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(Identifier {
|
||||
v: corpus_core::V,
|
||||
checksum: Some(bibclean::isbn::valid(&digits)),
|
||||
url: Some(format!("https://openlibrary.org/isbn/{digits}")),
|
||||
input: input.to_string(),
|
||||
kind: Kind::Isbn,
|
||||
id: digits,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_doi_is_read_however_it_was_written() {
|
||||
for text in [
|
||||
"10.1000/182",
|
||||
"doi:10.1000/182",
|
||||
"https://doi.org/10.1000/182",
|
||||
"http://dx.doi.org/10.1000/182",
|
||||
" 10.1000/182. ",
|
||||
] {
|
||||
let found = parse(text).unwrap_or_else(|| panic!("{text}"));
|
||||
assert_eq!(found.kind, Kind::Doi, "{text}");
|
||||
assert_eq!(found.id, "10.1000/182", "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_issn_carries_a_check_digit() {
|
||||
let found = parse("0028-0836").expect("an issn");
|
||||
assert_eq!(found.kind, Kind::Issn);
|
||||
assert_eq!(found.id, "0028-0836");
|
||||
assert_eq!(found.checksum, Some(true));
|
||||
|
||||
let wrong = parse("0028-0837").expect("an issn");
|
||||
assert_eq!(wrong.checksum, Some(false), "one digit out is caught");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_isbn_carries_one_too() {
|
||||
let found = parse("978-0-306-40615-7").expect("an isbn");
|
||||
assert_eq!(found.kind, Kind::Isbn);
|
||||
assert_eq!(found.id, "9780306406157");
|
||||
assert_eq!(found.checksum, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_arxiv_identifier_is_read_in_both_its_forms() {
|
||||
assert_eq!(parse("arXiv:2401.01234").expect("an id").id, "2401.01234");
|
||||
assert_eq!(parse("2401.01234v2").expect("an id").id, "2401.01234v2");
|
||||
assert_eq!(
|
||||
parse("math.GT/0309136").expect("an id").id,
|
||||
"math.GT/0309136"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_kind_resolves_somewhere() {
|
||||
for text in [
|
||||
"10.1000/182",
|
||||
"0028-0836",
|
||||
"978-0-306-40615-7",
|
||||
"arXiv:2401.01234",
|
||||
] {
|
||||
assert!(parse(text).expect("an id").url.is_some(), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_an_identifier_is_not_one() {
|
||||
for text in ["", " ", "hello", "10.", "1234", "not/an/id"] {
|
||||
assert_eq!(parse(text), None, "{text:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_kind_can_be_insisted_on() {
|
||||
// Eight digits are an ISSN; asked for a DOI, they are nothing.
|
||||
assert!(parse_as("00280836", Some(Kind::Issn)).is_some());
|
||||
assert_eq!(parse_as("00280836", Some(Kind::Doi)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sentence_that_merely_contains_digits_is_not_an_isbn() {
|
||||
assert_eq!(parse("published in 9780306406157 by somebody"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//! The identifiers a bibliography is held together by, read and checked.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
use idnorm::Kind;
|
||||
|
||||
/// Read identifiers and say what they are, one record per line.
|
||||
///
|
||||
/// A DOI arrives bare, as a URL, with a `doi:` prefix, with the full stop the
|
||||
/// sentence around it left behind, in whatever case the publisher felt like.
|
||||
/// All of those are one identifier, and a catalogue storing them as they
|
||||
/// arrived cannot tell.
|
||||
///
|
||||
/// Asks nobody whether the thing exists. What it can say is whether the
|
||||
/// string is well formed — an ISBN and an ISSN carry a check digit — which is
|
||||
/// a different question, and the honest answer to the other one needs a
|
||||
/// catalogue.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 every line was an identifier, 1 at least one was not,
|
||||
74 the file could not be read.
|
||||
|
||||
Example:
|
||||
|
||||
idnorm refs.txt | jq -r 'select(.checksum == false) | .input'
|
||||
idnorm --kind doi refs.txt | jq -r .url"
|
||||
)]
|
||||
struct Args {
|
||||
/// File of identifiers, one per line. Reads stdin when absent.
|
||||
path: Option<PathBuf>,
|
||||
|
||||
/// Read every line as this kind, rather than whatever it looks like.
|
||||
#[arg(long, value_enum)]
|
||||
kind: Option<Sort>,
|
||||
|
||||
/// Say nothing about the lines that are not identifiers.
|
||||
#[arg(long)]
|
||||
quiet: bool,
|
||||
}
|
||||
|
||||
/// The kinds a line can be insisted on being.
|
||||
#[derive(Clone, Copy, clap::ValueEnum)]
|
||||
enum Sort {
|
||||
/// A digital object identifier.
|
||||
Doi,
|
||||
/// A journal's serial number.
|
||||
Issn,
|
||||
/// A book's number.
|
||||
Isbn,
|
||||
/// An arXiv preprint.
|
||||
Arxiv,
|
||||
}
|
||||
|
||||
impl From<Sort> for Kind {
|
||||
fn from(sort: Sort) -> Self {
|
||||
match sort {
|
||||
Sort::Doi => Kind::Doi,
|
||||
Sort::Issn => Kind::Issn,
|
||||
Sort::Isbn => Kind::Isbn,
|
||||
Sort::Arxiv => Kind::Arxiv,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
|
||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||
Ok(text) => text,
|
||||
Err((code, why)) => {
|
||||
eprintln!("idnorm: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
|
||||
let kind = args.kind.map(Kind::from);
|
||||
let mut found = Vec::new();
|
||||
let mut unread = 0usize;
|
||||
|
||||
for line in text.lines() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
match idnorm::parse_as(line, kind) {
|
||||
Some(identifier) => found.push(identifier),
|
||||
None => {
|
||||
unread += 1;
|
||||
if !args.quiet {
|
||||
eprintln!("idnorm: not an identifier: {line}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let wrote = catena_cli::stream("idnorm", &found);
|
||||
if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
|
||||
return wrote;
|
||||
}
|
||||
ExitCode::from(match unread {
|
||||
0 => exit::OK,
|
||||
_ => exit::NO,
|
||||
})
|
||||
}
|
||||
+2
-1
@@ -61,9 +61,10 @@ same as any other filter.
|
||||
## Nothing reaches outside the process
|
||||
|
||||
No network, no database, no configuration file, no state kept between runs.
|
||||
Four exceptions, each declared in `--help`:
|
||||
Five exceptions, each declared in `--help`:
|
||||
|
||||
- [`emb`](../crates/emb/README.md) makes an HTTP call, which is the whole of what it does.
|
||||
- [`bibfetch`](../crates/bibfetch/README.md) asks a register about an identifier, likewise.
|
||||
- [`pdftriage`](../crates/pdftriage/README.md) runs `pdftotext` and `pdfinfo`.
|
||||
- [`pdfgutter`](../crates/pdfgutter/README.md) runs `pdftoppm` to look at a page.
|
||||
- [`avplan`](../crates/avplan/README.md) runs `ffprobe` to read a file's streams, and
|
||||
|
||||
@@ -12,6 +12,7 @@ this suite, because a real pipeline has a database at the end of it.
|
||||
- [Passages into pgvector](#passages-into-pgvector)
|
||||
- [Hybrid search, fused](#hybrid-search-fused)
|
||||
- [A catalogue that has drifted](#a-catalogue-that-has-drifted)
|
||||
- [A bibliography that has been copied between formats](#a-bibliography-that-has-been-copied-between-formats)
|
||||
- [A video library that plays on the thing you carry](#a-video-library-that-plays-on-the-thing-you-carry)
|
||||
- [Notes on composing these](#notes-on-composing-these)
|
||||
|
||||
@@ -267,6 +268,14 @@ vocabmerge --kind author authors.ndjson > merges.ndjson
|
||||
vocabmerge --kind tag tags.ndjson >> merges.ndjson
|
||||
```
|
||||
|
||||
Series and volume, which most catalogues have on almost nothing:
|
||||
|
||||
```sh
|
||||
seriesof catalogue.ndjson \
|
||||
| jq -r 'select(.volume) | "UPDATE book SET series = \(.series|@json), series_index = \(.volume) WHERE title = \(.title|@json);"' \
|
||||
| psql
|
||||
```
|
||||
|
||||
Apply only what you believe:
|
||||
|
||||
```sh
|
||||
@@ -275,6 +284,39 @@ jq -r 'select(.action == "merge" and .confidence >= 0.9)
|
||||
merges.ndjson | psql
|
||||
```
|
||||
|
||||
## A bibliography that has been copied between formats
|
||||
|
||||
Identifiers arrive wrapped in whatever the last tool left on them, and a
|
||||
catalogue that stores them as they came cannot tell two spellings of one DOI
|
||||
apart.
|
||||
|
||||
```sh
|
||||
idnorm --quiet refs.txt > ids.ndjson
|
||||
```
|
||||
|
||||
The ones that cannot be right — an ISBN or an ISSN whose check digit fails:
|
||||
|
||||
```sh
|
||||
jq -r 'select(.checksum == false) | .input' ids.ndjson
|
||||
```
|
||||
|
||||
The ones that are well formed and still wrong, which needs a register:
|
||||
|
||||
```sh
|
||||
bibfetch --polite you@example.org ids.ndjson \
|
||||
| jq -r 'select(.found | not) | "missing: \(.id)"'
|
||||
```
|
||||
|
||||
That second question is a different one, which is why it is a different tool:
|
||||
`idnorm` never touches the network and `bibfetch` does nothing else.
|
||||
|
||||
Filling in what you only had the identifier for:
|
||||
|
||||
```sh
|
||||
idnorm --quiet refs.txt | bibfetch \
|
||||
| jq -r 'select(.found) | "\(.year)\t\(.authors | join(", "))\t\(.title)"'
|
||||
```
|
||||
|
||||
## A video library that plays on the thing you carry
|
||||
|
||||
```sh
|
||||
|
||||
@@ -15,6 +15,10 @@ fn records(text: &str) -> Vec<Value> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The cases, as text, so the index keeps the order it was written in.
|
||||
///
|
||||
/// Reading through `Value` would sort the keys, and which of two words at one
|
||||
/// position comes first is exactly what that order decides.
|
||||
fn cases() -> Vec<Value> {
|
||||
let text = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson"))
|
||||
.expect("a readable case file");
|
||||
@@ -33,7 +37,7 @@ fn every_abstract_reads_back_the_same_as_python() {
|
||||
assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
|
||||
|
||||
let index: Inverted =
|
||||
serde_json::from_value(case["index"].clone()).expect("an index of positions");
|
||||
serde_json::from_str(&case["index"].to_string()).expect("an index of positions");
|
||||
let ours = abstracts::from_inverted(&index);
|
||||
if ours.as_deref() != theirs["abstract"].as_str() {
|
||||
wrong.push(format!(
|
||||
|
||||
Reference in New Issue
Block a user