mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f32f52293 | ||
|
|
e480f07c32 | ||
|
|
d551adc14c | ||
|
|
90fb977a10 |
Generated
+26
-26
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "avplan"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bibclean"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -143,7 +143,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bibfetch"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -196,7 +196,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bookmeta"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -210,7 +210,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"bibfetch",
|
||||
@@ -277,7 +277,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_mangen",
|
||||
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -334,7 +334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "citefind"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"chunkmd",
|
||||
@@ -399,7 +399,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "colophon"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -448,7 +448,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
@@ -492,7 +492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"corpus-core",
|
||||
"regex",
|
||||
@@ -504,7 +504,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuenorm"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -514,7 +514,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuepack"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -525,7 +525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cueseg"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -612,7 +612,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -876,7 +876,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -942,7 +942,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "langid"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1022,7 +1022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1034,7 +1034,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdsplit"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdfgutter"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1126,7 +1126,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdftriage"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
||||
|
||||
[[package]]
|
||||
name = "rrf"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1517,7 +1517,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "seriesof"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"tiktoken-rs",
|
||||
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "tocfind"
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
|
||||
+27
-27
@@ -3,39 +3,39 @@ resolver = "3"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.2"
|
||||
version = "0.3.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MPL-2.0"
|
||||
repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[workspace.dependencies]
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.2.2" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.2.2" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.2.2" }
|
||||
textkit = { path = "crates/textkit", version = "0.2.2" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.2.2" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.2.2" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.2.2" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.2.2" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.2.2" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.2.2" }
|
||||
rrf = { path = "crates/rrf", version = "0.2.2" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.2.2" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.2.2" }
|
||||
avplan = { path = "crates/avplan", version = "0.2.2" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.2.2" }
|
||||
catena = { path = "crates/bindings", version = "0.2.2" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.2.2" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.2.2" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.2.2" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.2.2" }
|
||||
langid = { path = "crates/langid", version = "0.2.2" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.2.2" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.2.2" }
|
||||
citefind = { path = "crates/citefind", version = "0.2.2" }
|
||||
colophon = { path = "crates/colophon", version = "0.2.2" }
|
||||
emb = { path = "crates/emb", version = "0.2.2" }
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.3.1" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.3.1" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.3.1" }
|
||||
textkit = { path = "crates/textkit", version = "0.3.1" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.3.1" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.3.1" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.3.1" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.3.1" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.3.1" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.3.1" }
|
||||
rrf = { path = "crates/rrf", version = "0.3.1" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.3.1" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.3.1" }
|
||||
avplan = { path = "crates/avplan", version = "0.3.1" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.3.1" }
|
||||
catena = { path = "crates/bindings", version = "0.3.1" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.3.1" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.3.1" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.3.1" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.3.1" }
|
||||
langid = { path = "crates/langid", version = "0.3.1" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.3.1" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.3.1" }
|
||||
citefind = { path = "crates/citefind", version = "0.3.1" }
|
||||
colophon = { path = "crates/colophon", version = "0.3.1" }
|
||||
emb = { path = "crates/emb", version = "0.3.1" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
|
||||
@@ -63,13 +63,18 @@ pub fn lookup(
|
||||
for source in chain {
|
||||
match ask(agent, *source, id, kind, attempts) {
|
||||
Ok(record) => {
|
||||
let complete = record.found && record.title.is_some() && !record.authors.is_empty();
|
||||
let better = best.as_ref().is_none_or(|kept| record.found && !kept.found);
|
||||
if complete {
|
||||
return Ok(Some(record));
|
||||
match &mut best {
|
||||
// Merged rather than replaced. The registers answer about
|
||||
// different parts of one book: OpenLibrary's edition record
|
||||
// carries the publisher, the pagination and the language
|
||||
// and names its authors by key, while its search index
|
||||
// carries the authors and not the publisher. Keeping
|
||||
// whichever arrived first throws the other half away.
|
||||
Some(kept) => kept.fill_from(record),
|
||||
None => best = Some(record),
|
||||
}
|
||||
if better {
|
||||
best = Some(record);
|
||||
if best.as_ref().is_some_and(Record::complete) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(why) => last_error = Some(why),
|
||||
|
||||
@@ -59,6 +59,21 @@ pub struct Record {
|
||||
/// The abstract, where the register carries one.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub summary: Option<String>,
|
||||
/// The subject headings the register files it under, as it words them.
|
||||
///
|
||||
/// A register states these for far more books than print a cataloguing
|
||||
/// block, which makes them the second source a shelf otherwise lacks.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub subjects: Vec<String>,
|
||||
/// The language it is written in, as the register gives it: a two- or
|
||||
/// three-letter code, or a name. Not normalised here — what a register
|
||||
/// said is the record, and deciding that "eng" and "English" are one
|
||||
/// thing belongs to whoever is filing them.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<String>,
|
||||
/// How many pages the register says it has.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub pages: Option<u32>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
@@ -112,6 +127,70 @@ pub fn missing(id: &str, kind: &str, source: Source) -> Record {
|
||||
year: None,
|
||||
container: None,
|
||||
summary: None,
|
||||
subjects: Vec::new(),
|
||||
language: None,
|
||||
pages: None,
|
||||
}
|
||||
}
|
||||
|
||||
impl Record {
|
||||
/// Whether this record says everything a catalogue wants.
|
||||
///
|
||||
/// Used to decide when to stop asking. Title and authors are the minimum
|
||||
/// that makes a record usable; the rest is worth another register only
|
||||
/// while one is left to ask.
|
||||
pub fn complete(&self) -> bool {
|
||||
self.found
|
||||
&& self.title.is_some()
|
||||
&& !self.authors.is_empty()
|
||||
&& self.container.is_some()
|
||||
&& self.language.is_some()
|
||||
}
|
||||
|
||||
/// Take from another record whatever this one does not have.
|
||||
///
|
||||
/// The registers are complementary rather than redundant, and asking them
|
||||
/// in turn and keeping one answer throws away the difference. OpenLibrary's
|
||||
/// edition record gives the publisher, the pagination and the language but
|
||||
/// names its authors by key; its search index gives the authors but not the
|
||||
/// publisher. One book, two half-answers, and the whole of it only if they
|
||||
/// are put together.
|
||||
///
|
||||
/// Nothing already answered is overwritten, so the order of the chain is
|
||||
/// the order of trust.
|
||||
pub fn fill_from(&mut self, other: Record) {
|
||||
if !other.found {
|
||||
return;
|
||||
}
|
||||
self.found = true;
|
||||
if self.title.is_none() {
|
||||
self.title = other.title;
|
||||
}
|
||||
if self.authors.is_empty() {
|
||||
self.authors = other.authors;
|
||||
}
|
||||
if self.year.is_none() {
|
||||
self.year = other.year;
|
||||
}
|
||||
if self.container.is_none() {
|
||||
self.container = other.container;
|
||||
}
|
||||
if self.summary.is_none() {
|
||||
self.summary = other.summary;
|
||||
}
|
||||
if self.subjects.is_empty() {
|
||||
self.subjects = other.subjects;
|
||||
}
|
||||
if self.language.is_none() {
|
||||
self.language = other.language;
|
||||
}
|
||||
if self.pages.is_none() {
|
||||
self.pages = other.pages;
|
||||
}
|
||||
// Which register answered becomes which registers did.
|
||||
if !self.source.split('+').any(|s| s == other.source) {
|
||||
self.source = format!("{}+{}", self.source, other.source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,3 +214,74 @@ mod tests {
|
||||
assert!(json.contains("\"found\":false"), "{json}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod merge_tests {
|
||||
use super::*;
|
||||
|
||||
fn rec(source: &str) -> Record {
|
||||
let mut r = missing("9780199205592", "isbn", Source::OpenLibrary);
|
||||
r.source = source.to_string();
|
||||
r.found = true;
|
||||
r
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_half_answers_make_one_record() {
|
||||
// Exactly what the two OpenLibrary endpoints give for one book: the
|
||||
// edition record has the publisher and the language and names its
|
||||
// authors by key; the index has the authors and not the publisher.
|
||||
let mut edition = rec("openlibrary");
|
||||
edition.title = Some("The First World War: A Very Short Introduction".into());
|
||||
edition.container = Some("Oxford University Press".into());
|
||||
edition.language = Some("eng".into());
|
||||
|
||||
let mut index = rec("openlibrary");
|
||||
index.title = Some("The First World War".into());
|
||||
index.authors = vec!["Michael Howard".into()];
|
||||
index.year = Some(2007);
|
||||
|
||||
edition.fill_from(index);
|
||||
assert_eq!(edition.authors, ["Michael Howard"]);
|
||||
assert_eq!(edition.year, Some(2007));
|
||||
// The fuller title arrived first and is not displaced by the shorter.
|
||||
assert_eq!(
|
||||
edition.title.as_deref(),
|
||||
Some("The First World War: A Very Short Introduction")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_register_that_has_nothing_contributes_nothing() {
|
||||
let mut kept = rec("openlibrary");
|
||||
kept.title = Some("A Book".into());
|
||||
let absent = missing("9780199205592", "isbn", Source::GoogleBooks);
|
||||
kept.fill_from(absent);
|
||||
assert_eq!(kept.title.as_deref(), Some("A Book"));
|
||||
assert_eq!(kept.source, "openlibrary");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_record_names_every_register_that_answered() {
|
||||
let mut a = rec("openlibrary");
|
||||
a.title = Some("A Book".into());
|
||||
let mut b = rec("googlebooks");
|
||||
b.found = true;
|
||||
b.pages = Some(134);
|
||||
a.fill_from(b);
|
||||
assert_eq!(a.source, "openlibrary+googlebooks");
|
||||
assert_eq!(a.pages, Some(134));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_means_a_catalogue_has_what_it_needs() {
|
||||
let mut r = rec("openlibrary");
|
||||
assert!(!r.complete());
|
||||
r.title = Some("A Book".into());
|
||||
r.authors = vec!["Someone".into()];
|
||||
assert!(!r.complete(), "still missing the publisher and language");
|
||||
r.container = Some("A Press".into());
|
||||
r.language = Some("eng".into());
|
||||
assert!(r.complete());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,9 +55,17 @@ impl Source {
|
||||
match kind {
|
||||
"doi" => &[Source::Crossref],
|
||||
"arxiv" => &[Source::Arxiv],
|
||||
// The index first. Asked for the fields by name it answers with
|
||||
// the title, the authors, the publisher, the language, a page
|
||||
// count and the subjects in one request, which is the whole of
|
||||
// what a catalogue wants; the edition record is then needed only
|
||||
// for a subtitle or a publisher the index did not give. Asking it
|
||||
// second cost three requests per book where one usually does, and
|
||||
// at three books in flight that is the difference between two
|
||||
// days and two hours.
|
||||
"isbn" => &[
|
||||
Source::OpenLibrary,
|
||||
Source::OpenLibrarySearch,
|
||||
Source::OpenLibrary,
|
||||
Source::GoogleBooks,
|
||||
],
|
||||
_ => &[],
|
||||
@@ -75,9 +83,12 @@ impl Source {
|
||||
Source::OpenLibrary => {
|
||||
format!("https://openlibrary.org/isbn/{}.json", encode(id))
|
||||
}
|
||||
// `fields` or the index answers with a handful of defaults and
|
||||
// keeps the rest, which is how a language it holds went unread.
|
||||
Source::OpenLibrarySearch => format!(
|
||||
"https://openlibrary.org/search.json?q=isbn:{}\
|
||||
&fields=title,subtitle,author_name,first_publish_year,publisher&limit=1",
|
||||
&fields=title,subtitle,author_name,first_publish_year,publisher,language,subject,\
|
||||
number_of_pages_median&limit=1",
|
||||
encode(id)
|
||||
),
|
||||
Source::GoogleBooks => format!(
|
||||
@@ -222,9 +233,24 @@ struct OpenLibraryBook {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(default)]
|
||||
subtitle: Option<String>,
|
||||
#[serde(default)]
|
||||
publish_date: Option<String>,
|
||||
#[serde(default)]
|
||||
publishers: Vec<String>,
|
||||
#[serde(default)]
|
||||
subjects: Vec<String>,
|
||||
/// `[{"key": "/languages/eng"}]`, which is how OpenLibrary names one.
|
||||
#[serde(default)]
|
||||
languages: Vec<OpenLibraryKey>,
|
||||
#[serde(default)]
|
||||
number_of_pages: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct OpenLibraryKey {
|
||||
#[serde(default)]
|
||||
key: String,
|
||||
}
|
||||
|
||||
fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
@@ -232,10 +258,23 @@ fn open_library(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
serde_json::from_str(body).map_err(|e| Error::Shape(e.to_string()))?;
|
||||
|
||||
let mut record = blank(id, kind, Source::OpenLibrary);
|
||||
record.title = book.title;
|
||||
// An edition record keeps the subtitle in its own field, and a catalogue
|
||||
// wants the whole title. Google Books is already joined the same way.
|
||||
record.title = match (book.title, book.subtitle) {
|
||||
(Some(title), Some(subtitle)) => Some(format!("{title}: {subtitle}")),
|
||||
(title, _) => title,
|
||||
};
|
||||
record.container = book.publishers.into_iter().next();
|
||||
// The date is free text — "1999", "March 1999", "1999-03-01".
|
||||
record.year = book.publish_date.as_deref().and_then(year_in);
|
||||
record.subjects = book.subjects;
|
||||
// "/languages/eng" names the language by the last segment of a key.
|
||||
record.language = book
|
||||
.languages
|
||||
.into_iter()
|
||||
.find_map(|l| l.key.rsplit('/').next().map(str::to_string))
|
||||
.filter(|l| !l.is_empty());
|
||||
record.pages = book.number_of_pages;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -257,8 +296,21 @@ struct SearchDoc {
|
||||
first_publish_year: Option<i64>,
|
||||
#[serde(default)]
|
||||
publisher: Vec<String>,
|
||||
#[serde(default)]
|
||||
language: Vec<String>,
|
||||
#[serde(default)]
|
||||
subject: Vec<String>,
|
||||
#[serde(rename = "number_of_pages_median", default)]
|
||||
pages: Option<u32>,
|
||||
}
|
||||
|
||||
/// How many of a work's subject headings are worth carrying.
|
||||
///
|
||||
/// The index lists every heading every edition has ever been given, which on a
|
||||
/// well-known book is hundreds and on Hamlet is thousands. A shelf is filed
|
||||
/// from a handful, and the first are the broadest.
|
||||
const SUBJECT_LIMIT: usize = 12;
|
||||
|
||||
/// A search that matched nothing answers with no documents rather than a
|
||||
/// status, so an empty reply is the absence and not a failure.
|
||||
fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
@@ -275,6 +327,12 @@ fn open_library_search(id: &str, kind: &str, body: &str) -> Result<Record, Error
|
||||
record.authors = doc.author_name;
|
||||
record.year = doc.first_publish_year;
|
||||
record.container = doc.publisher.into_iter().next();
|
||||
record.language = doc.language.into_iter().next();
|
||||
// The index lists every subject any edition of the work carries, which
|
||||
// runs to hundreds on a well-known book. A shelf needs a handful.
|
||||
record.subjects = doc.subject;
|
||||
record.subjects.truncate(SUBJECT_LIMIT);
|
||||
record.pages = doc.pages;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -304,6 +362,12 @@ struct GoogleVolume {
|
||||
publisher: Option<String>,
|
||||
#[serde(default)]
|
||||
description: Option<String>,
|
||||
#[serde(default)]
|
||||
categories: Vec<String>,
|
||||
#[serde(rename = "language", default)]
|
||||
language: Option<String>,
|
||||
#[serde(rename = "pageCount", default)]
|
||||
page_count: Option<u32>,
|
||||
}
|
||||
|
||||
/// A search that matched nothing answers with no items rather than a status,
|
||||
@@ -324,6 +388,9 @@ fn google_books(id: &str, kind: &str, body: &str) -> Result<Record, Error> {
|
||||
record.year = volume.published_date.as_deref().and_then(year_in);
|
||||
record.container = volume.publisher;
|
||||
record.summary = volume.description.map(|text| collapse(&text));
|
||||
record.subjects = volume.categories;
|
||||
record.language = volume.language;
|
||||
record.pages = volume.page_count;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -379,7 +446,10 @@ mod tests {
|
||||
fn each_kind_goes_to_the_body_that_assigns_it() {
|
||||
assert_eq!(Source::for_kind("doi"), Some(Source::Crossref));
|
||||
assert_eq!(Source::for_kind("arxiv"), Some(Source::Arxiv));
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrary));
|
||||
// The index, not the edition record: it answers with the whole of
|
||||
// what a catalogue wants in one request, where the edition record
|
||||
// takes up to twenty-eight seconds and names its authors by key.
|
||||
assert_eq!(Source::for_kind("isbn"), Some(Source::OpenLibrarySearch));
|
||||
// No register answers about an ISSN without a key.
|
||||
assert_eq!(Source::for_kind("issn"), None);
|
||||
}
|
||||
|
||||
@@ -35,6 +35,29 @@ static LIFESPAN: LazyLock<Regex> = LazyLock::new(|| {
|
||||
static BARE_DATE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)^\d{1,4}\s*(bc|ad|bce|ce)?\.?$").expect("a valid pattern"));
|
||||
/// `Copyright © 2013 by`, `\u{a9} 2013 Richard Swinburne`, `Copyright 1971 by`.
|
||||
/// `1. General Remarks`, `2 James W. McAllister` — a numbered heading.
|
||||
static NUMBERED_HEADING: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*\d+\s*[.)]\s+\S|^\s*\d+\s{2,}\S").expect("a valid pattern"));
|
||||
/// `5222 What Is This`, `827 North Fourth Street` — a number that is not a
|
||||
/// year, leading a line.
|
||||
static LEADING_NUMBER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*(\d{1,5})\s+\S").expect("a valid pattern"));
|
||||
/// `Page xxxvi`, `Chapter I`, `Fig. 3` — a marker, not a name.
|
||||
static MARKER: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)^\s*(page|pages|p\.|pp\.|fig\.|figure|plate|chapter|ch\.|section|part)\s+[\divxlcm]",
|
||||
)
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
/// `Sidebottom, Harry. Ancient warfare` — a bibliography entry, which front
|
||||
/// matter prints in its cataloguing block and in its further reading.
|
||||
static CATALOGUE_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^[A-Z][A-Za-z'\-]+,\s+[A-Z][A-Za-z'.\-]*\.\s+\S").expect("a valid pattern")
|
||||
});
|
||||
/// `To Mackenzie,` — a dedication, set alone on its own page.
|
||||
static DEDICATION: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)^\s*(to|for|in memory of)\s+\S.*,\s*$").expect("a valid pattern")
|
||||
});
|
||||
static COPYRIGHT_NOTICE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
// The year need not follow the symbol directly: "(c) Alfredo Ferrarin
|
||||
// 2004" puts the holder between them, and was read as a title.
|
||||
@@ -79,10 +102,13 @@ copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*",
|
||||
static STRUCTURAL: &[&str] = &[
|
||||
"abbreviations",
|
||||
"about the author",
|
||||
"abstract",
|
||||
"acknowledgements",
|
||||
"acknowledgments",
|
||||
"afterword",
|
||||
"appendix",
|
||||
"author",
|
||||
"authors",
|
||||
"bibliography",
|
||||
"contents",
|
||||
"dedication",
|
||||
@@ -95,6 +121,7 @@ static STRUCTURAL: &[&str] = &[
|
||||
"illustrations",
|
||||
"index",
|
||||
"introduction",
|
||||
"keywords",
|
||||
"list of contributors",
|
||||
"list of illustrations",
|
||||
"maps",
|
||||
@@ -102,7 +129,9 @@ static STRUCTURAL: &[&str] = &[
|
||||
"postscript",
|
||||
"preface",
|
||||
"prologue",
|
||||
"subtitle",
|
||||
"table of contents",
|
||||
"title",
|
||||
"title page",
|
||||
"translator's note",
|
||||
];
|
||||
@@ -182,6 +211,22 @@ fn is_furniture(line: &str) -> bool {
|
||||
}
|
||||
// Sentences a copyright page sets, which are not notices and so not caught
|
||||
// by the notice pattern.
|
||||
if NUMBERED_HEADING.is_match(t) || MARKER.is_match(t) || CATALOGUE_ENTRY.is_match(t) {
|
||||
return true;
|
||||
}
|
||||
// A line opening with a number that is not a year. "5222 What Is This" and
|
||||
// "827 North Fourth Street" are a stray folio and an address; "1984
|
||||
// Revisited" is a book, and the difference is whether the number could be
|
||||
// the year of one.
|
||||
if let Some(c) = LEADING_NUMBER.captures(t) {
|
||||
let n: u32 = c[1].parse().unwrap_or(0);
|
||||
if !(1000..=2099).contains(&n) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if DEDICATION.is_match(t) {
|
||||
return true;
|
||||
}
|
||||
let lower = t.to_lowercase();
|
||||
for phrase in [
|
||||
"moral rights of the author",
|
||||
@@ -496,3 +541,54 @@ mod conversion_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod review_pile_tests {
|
||||
use super::*;
|
||||
|
||||
/// Everything here was proposed as a title by a real run over this
|
||||
/// library, and sat in the review queue waiting for a person to say no.
|
||||
#[test]
|
||||
fn what_the_review_pile_was_full_of() {
|
||||
for line in [
|
||||
"1. General Remarks",
|
||||
"2 James W. McAllister",
|
||||
"5222 What Is This",
|
||||
"827 North Fourth Street",
|
||||
"Page xxxvi",
|
||||
"Page xxxviii",
|
||||
"Chapter I",
|
||||
"title",
|
||||
"Sidebottom, Harry. Ancient warfare",
|
||||
"To Mackenzie,",
|
||||
] {
|
||||
assert!(
|
||||
is_structural(line) || is_furniture(line),
|
||||
"still read as a title: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// And none of it catches a book. The numbers are the delicate part: a
|
||||
/// year can open a title, and a folio cannot.
|
||||
#[test]
|
||||
fn a_book_is_still_a_book() {
|
||||
for line in [
|
||||
"1984 Revisited",
|
||||
"1914: The Year the World Ended",
|
||||
"Chapter and Verse",
|
||||
"Part of the Furniture",
|
||||
"To Kill a Mockingbird",
|
||||
"For Whom the Bell Tolls",
|
||||
"The Title of This Book",
|
||||
"Pages from a Cold Island",
|
||||
"Sections of a Broken Whole",
|
||||
"In Memory of Bread",
|
||||
] {
|
||||
assert!(
|
||||
!(is_structural(line) || is_furniture(line)),
|
||||
"rejected a real title: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user