diff --git a/Cargo.lock b/Cargo.lock index 51a591e..91d796d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,21 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "bibclean" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "fancy-regex 0.16.2", + "regex", + "schemars", + "serde", + "serde_json", + "textkit", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -653,6 +668,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" name = "parity" version = "0.1.0" dependencies = [ + "bibclean", "booksniff", "chunkmd", "corpus-core", diff --git a/Cargo.toml b/Cargo.toml index c38edcb..6f62e59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ cuenorm = { path = "crates/cuenorm", version = "0.1.0" } cueseg = { path = "crates/cueseg", version = "0.1.0" } rrf = { path = "crates/rrf", version = "0.1.0" } cuepack = { path = "crates/cuepack", version = "0.1.0" } +bibclean = { path = "crates/bibclean", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } diff --git a/README.md b/README.md index 2ec695e..43b4b43 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done | | | |---|---| +| `bibclean` | Catalogue records audited and repaired, with a reason for every change | | `booksniff` | What a file actually is, whatever its name claims | | `mdscrub` | Converted markdown in, clean markdown out: residual HTML, control characters, headings, page markers | | `chunkmd` | Markdown into passages that carry their page, chapter and role | diff --git a/crates/bibclean/Cargo.toml b/crates/bibclean/Cargo.toml new file mode 100644 index 0000000..825496d --- /dev/null +++ b/crates/bibclean/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "bibclean" +description = "Bibliographic hygiene over catalogue records, with a reason for every change." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +fancy-regex.workspace = true +regex.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +textkit.workspace = true + +[lints] +workspace = true diff --git a/crates/bibclean/src/fix.rs b/crates/bibclean/src/fix.rs new file mode 100644 index 0000000..28a1f09 --- /dev/null +++ b/crates/bibclean/src/fix.rs @@ -0,0 +1,220 @@ +//! Repairs that can be made without looking anything up. + +use std::sync::LazyLock; + +use regex::Regex; +use textkit::pattern::SPACE; + +/// A format name stuck on the end of a title taken from a filename. +static FORMAT_SUFFIX: LazyLock = LazyLock::new(|| { + Regex::new(&format!( + r"(?i)[_\-{SPACE}]*(txt|pdf|epub|mobi|azw3?|htmlz?|djvu){SPACE}*$" + )) + .expect("a valid pattern") +}); + +/// An underscore standing in for the colon before a subtitle. +static SUBTITLE_UNDERSCORE: LazyLock = + LazyLock::new(|| Regex::new(&format!(r"_{SPACE}+")).expect("a valid pattern")); + +/// Two or more spaces where one belongs. +static SPACE_RUN: LazyLock = + LazyLock::new(|| Regex::new(&format!(r"{SPACE}{{2,}}")).expect("a valid pattern")); + +/// A run of letters, by Python's definition of a letter. +static LETTERS: LazyLock = LazyLock::new(|| Regex::new(r"\p{L}+").expect("a valid pattern")); + +/// Words that stay lowercase inside a title. +const SMALL_WORDS: &[&str] = &[ + "a", "an", "and", "as", "at", "but", "by", "for", "from", "in", "into", "nor", "of", "on", + "or", "over", "per", "the", "to", "upon", "via", "with", +]; + +/// Whether a title is in capitals throughout. +/// +/// A space is required, so a title that is only an acronym is left alone. +pub fn is_shouting(title: &str) -> bool { + let title = title.trim_matches(textkit::is_space); + !title.is_empty() + && title == title.to_uppercase() + && title.contains(' ') + && title + .chars() + .collect::>() + .windows(2) + .any(|pair| pair[0].is_ascii_uppercase() && pair[1].is_ascii_uppercase()) +} + +/// Title-case a shouting title without mangling it. +/// +/// The obvious library function is unusable here: it gives "Don'T" and +/// capitalises every preposition. Small words stay lowercase unless they open +/// or close the title, hyphenated parts are cased on both sides, and the +/// first *letter* is capitalised rather than the first character — an +/// ASCII-only rule skips an accented opening letter and capitalises the one +/// after it, turning "ÊTRE" into "êTre", and a third of a real library is not +/// in English. +pub fn title_case(title: &str) -> String { + let words: Vec<&str> = title.split_whitespace().collect(); + let last = words.len().saturating_sub(1); + + words + .iter() + .enumerate() + .map(|(at, word)| { + let core: String = word + .trim_start_matches(['(', '[', '"', '\'']) + .to_lowercase() + .trim_end_matches(['.', ',', ':', ';', '!', '?', ')', ']', '"', '\'']) + .to_string(); + if SMALL_WORDS.binary_search(&core.as_str()).is_ok() && at > 0 && at < last { + return word.to_lowercase(); + } + let lowered = word.to_lowercase(); + if lowered.contains('-') { + lowered + .split('-') + .map(capitalise) + .collect::>() + .join("-") + } else { + capitalise_first_letters(&lowered) + } + }) + .collect::>() + .join(" ") +} + +/// Undo the underscore mangling a filename-derived title carries. +/// +/// Three habits, in order: a format name on the end, an underscore standing +/// in for the colon before a subtitle, and underscores used as plain spaces. +pub fn clean_underscores(title: &str) -> String { + let title = FORMAT_SUFFIX.replace(title, ""); + let title = title.trim_matches(textkit::is_space); + let title = SUBTITLE_UNDERSCORE.replace_all(title, ": "); + let title = title.replace('_', " "); + SPACE_RUN + .replace_all(&title, " ") + .trim_matches(|c: char| textkit::is_space(c) || c == ':' || c == ',' || c == '-') + .to_string() +} + +/// Capitalise the first letter and lowercase the rest. +fn capitalise(word: &str) -> String { + let lowered = word.to_lowercase(); + capitalise_first_letters(&lowered) +} + +/// Capitalise the first run of letters, leaving everything before it alone. +fn capitalise_first_letters(lowered: &str) -> String { + match LETTERS.find(lowered) { + None => lowered.to_string(), + Some(found) => { + let mut out = String::with_capacity(lowered.len()); + out.push_str(&lowered[..found.start()]); + let mut letters = found.as_str().chars(); + if let Some(first) = letters.next() { + out.extend(first.to_uppercase()); + out.push_str(letters.as_str()); + } + out.push_str(&lowered[found.end()..]); + out + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_small_word_list_is_sorted_so_it_can_be_searched() { + assert!(SMALL_WORDS.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn shouting_needs_more_than_one_word() { + assert!(is_shouting("THE HUMAN CONDITION")); + assert!(!is_shouting("MIT")); + assert!(!is_shouting("The Human Condition")); + assert!(!is_shouting("")); + assert!(!is_shouting("A B")); + } + + #[test] + fn a_shouting_title_comes_back_readable() { + assert_eq!(title_case("THE HUMAN CONDITION"), "The Human Condition"); + assert_eq!( + title_case("A HISTORY OF THE WORLD IN TEN CHAPTERS"), + "A History of the World in Ten Chapters" + ); + } + + #[test] + fn a_small_word_at_either_end_is_capitalised() { + assert_eq!(title_case("OF MICE AND MEN"), "Of Mice and Men"); + assert_eq!(title_case("WHAT IS IT ALL FOR"), "What Is It All For"); + } + + #[test] + fn an_apostrophe_does_not_start_a_new_word() { + assert_eq!(title_case("DON'T LOOK NOW"), "Don't Look Now"); + } + + #[test] + fn a_hyphenated_word_is_cased_on_both_sides() { + assert_eq!(title_case("THE WELL-BUILT WHEEL"), "The Well-Built Wheel"); + } + + #[test] + fn an_accented_first_letter_is_the_one_capitalised() { + // The small-word list is English, so a French preposition is cased + // like any other word. That is what the original does, and a list per + // language is a different tool. + assert_eq!(title_case("ÊTRE ET TEMPS"), "Être Et Temps"); + assert_eq!( + title_case("ÉTUDES SUR LE MOYEN ÂGE"), + "Études Sur Le Moyen Âge" + ); + } + + #[test] + fn a_bracketed_word_keeps_its_bracket() { + assert_eq!(title_case("THE (SECOND) ESTATE"), "The (Second) Estate"); + } + + #[test] + fn a_format_suffix_comes_off() { + assert_eq!(clean_underscores("The Fall_txt"), "The Fall"); + assert_eq!(clean_underscores("The Fall - pdf"), "The Fall"); + // A format name takes no separator to be one, so an extension goes + // too and leaves its dot behind. Faithful to the original; a title + // that is really a filename is being repaired either way. + assert_eq!(clean_underscores("The Fall.epub"), "The Fall."); + } + + #[test] + fn an_underscore_before_a_subtitle_becomes_a_colon() { + assert_eq!( + clean_underscores("The 9/11 Mystery Plane_ And the Vanishing of America"), + "The 9/11 Mystery Plane: And the Vanishing of America" + ); + } + + #[test] + fn underscores_used_as_spaces_become_spaces() { + assert_eq!( + clean_underscores("Board_Advisors_etc"), + "Board Advisors etc" + ); + } + + #[test] + fn a_title_with_nothing_wrong_with_it_is_unchanged() { + assert_eq!( + clean_underscores("The Human Condition"), + "The Human Condition" + ); + } +} diff --git a/crates/bibclean/src/isbn.rs b/crates/bibclean/src/isbn.rs new file mode 100644 index 0000000..6ad3a33 --- /dev/null +++ b/crates/bibclean/src/isbn.rs @@ -0,0 +1,149 @@ +//! Finding and checking ISBNs. + +use std::sync::LazyLock; + +use regex::Regex; +use textkit::pattern::SPACE; + +/// An ISBN announced as one, in running text or a catalogue field. +static ANNOUNCED: LazyLock = LazyLock::new(|| { + Regex::new(&format!( + r"(?i)ISBN[-{SPACE}:]*((?:97[89][{SPACE}-]?)?(?:[0-9][{SPACE}-]?){{9}}[0-9Xx])" + )) + .expect("a valid pattern") +}); + +/// A bare ISBN sitting in a filename. +static BARE: LazyLock = + LazyLock::new(|| Regex::new(r"(97[89][0-9]{10}|[0-9]{9}[0-9Xx])").expect("a valid pattern")); + +/// The extension at the end of a filename. +static EXTENSION: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\.[a-z0-9]+$").expect("a valid pattern")); + +/// Whether this is a real ISBN, by its check digit. +/// +/// The checksum is the point. Without it any thirteen digits in a filename — +/// a scan date, an internal identifier, a phone number — reads as an ISBN, +/// and an ISBN is what a catalogue lookup trusts absolutely. +pub fn valid(text: &str) -> bool { + let digits: String = text + .chars() + .filter(|c| c.is_ascii_digit() || *c == 'X' || *c == 'x') + .collect(); + + match digits.len() { + 13 if digits.chars().all(|c| c.is_ascii_digit()) => { + let sum: u32 = digits + .chars() + .enumerate() + .map(|(at, digit)| { + let weight = if at % 2 == 0 { 1 } else { 3 }; + weight * digit.to_digit(10).expect("a digit") + }) + .sum(); + sum % 10 == 0 + } + 10 if digits[..9].chars().all(|c| c.is_ascii_digit()) => { + let sum: u32 = digits + .chars() + .enumerate() + .map(|(at, c)| { + let value = if c == 'X' || c == 'x' { + 10 + } else { + c.to_digit(10).unwrap_or(0) + }; + (10 - at as u32) * value + }) + .sum(); + sum % 11 == 0 + } + _ => false, + } +} + +/// Just the digits of an ISBN, with the spacing and hyphens taken out. +pub fn digits(text: &str) -> String { + text.chars() + .filter(|c| c.is_ascii_digit() || *c == 'X' || *c == 'x') + .collect() +} + +/// An ISBN announced in a piece of text, if it is the right length. +pub fn from_text(text: &str) -> Option { + let found = ANNOUNCED.captures(text)?; + let digits = digits(&found[1]); + matches!(digits.len(), 10 | 13).then_some(digits) +} + +/// A checksum-valid ISBN embedded in a filename. +/// +/// A file is often named exactly its ISBN and nothing else, which makes a +/// free and certain identifier out of a record that otherwise has none. +pub fn from_filename(name: &str) -> Option { + let stem = EXTENSION.replace(name, ""); + BARE.find_iter(&stem) + .map(|found| found.as_str()) + .find(|candidate| valid(candidate)) + .map(digits) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_real_isbn_checks_out() { + assert!(valid("9780198237907")); + assert!(valid("978-0-19-823790-7")); + assert!(valid("0198237901")); + assert!(valid("080213825X")); + } + + #[test] + fn a_number_that_is_merely_the_right_length_does_not() { + assert!(!valid("9780198237908")); + assert!(!valid("0198237902")); + assert!(!valid("1234567890123")); + assert!(!valid("")); + assert!(!valid("not a number")); + } + + #[test] + fn an_isbn_is_found_where_it_says_it_is() { + assert_eq!( + from_text("ISBN: 978-0-19-823790-7"), + Some("9780198237907".to_string()) + ); + assert_eq!( + from_text("isbn 0198237901 (paperback)"), + Some("0198237901".to_string()) + ); + assert_eq!(from_text("no identifier here"), None); + } + + #[test] + fn a_filename_that_is_an_isbn_gives_it_up() { + assert_eq!( + from_filename("9780198237907.pdf"), + Some("9780198237907".to_string()) + ); + assert_eq!( + from_filename("Aquinas - 9780198237907 (scan).epub"), + Some("9780198237907".to_string()) + ); + } + + #[test] + fn a_filename_of_digits_that_is_not_an_isbn_gives_nothing() { + assert_eq!(from_filename("20231115123456.pdf"), None); + assert_eq!(from_filename("chapter one.pdf"), None); + } + + #[test] + fn the_extension_is_not_searched() { + // Otherwise a name could borrow digits from its own suffix. + assert_eq!(from_filename("book.9780198237907"), None); + } +} diff --git a/crates/bibclean/src/lib.rs b/crates/bibclean/src/lib.rs new file mode 100644 index 0000000..11cd78f --- /dev/null +++ b/crates/bibclean/src/lib.rs @@ -0,0 +1,234 @@ +//! Bibliographic hygiene over catalogue records. +//! +//! A library's metadata rots in recognisable ways: a title that is really a +//! filename, an author field holding a publisher, a scanner's name where the +//! work's name should be, capitals shouted by a converter. None of it stops +//! anything working, and all of it makes a catalogue harder to search than +//! the shelf it describes. +//! +//! Nothing here is rewritten silently. Every change comes back with the +//! reason it was made, so a person can read a run before agreeing to it — +//! which is the difference between a tool that cleans a catalogue and one +//! that damages it in a new way. + +pub mod fix; +pub mod isbn; +pub mod matching; +pub mod reasons; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A catalogue record, as it arrives. +/// +/// Anything this does not know about is carried through untouched, so a +/// record can be piped through and back into whatever it came from. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Book { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// The work's title. + #[serde(default)] + pub title: Option, + /// Its author, as one field, however many there are. + #[serde(default)] + pub author: Option, + /// Its ISBN, if the catalogue has one. + #[serde(default)] + pub isbn: Option, + /// Where the file is, which is often where a missing ISBN is hiding. + #[serde(default)] + pub path: Option, + /// Everything else the record carried. + #[serde(flatten)] + #[schemars(with = "Map")] + pub extra: Map, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Book { + fn version(&self) -> u32 { + self.v + } +} + +/// One change, and why it was made. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Change { + /// Which field changed. + pub field: String, + /// What it was. + pub from: String, + /// What it became. + pub to: String, + /// The fault that prompted it. + pub why: String, +} + +/// A record, cleaned, with everything that was found and done to it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Cleaned { + /// The record, with the changes applied. + #[serde(flatten)] + pub book: Book, + /// What is wrong with it, named. + pub reasons: Vec, + /// How much attention it deserves, worst first. + pub severity: u32, + /// What was changed, and why. + pub changes: Vec, +} + +/// Audit a record and make the repairs that need no lookup. +pub fn clean(book: &Book) -> Cleaned { + let mut cleaned = book.clone(); + let mut changes = Vec::new(); + + let reasons = reasons::problems( + book.title.as_deref(), + book.author.as_deref(), + book.isbn.as_deref(), + ); + + // Underscores first: a title mangled by a filename may also be shouting, + // and title-casing it before the underscores come out would case the + // format suffix as though it were a word. + if let Some(title) = cleaned.title.clone() + && reasons.iter().any(|reason| reason == "underscore_title") + { + let fixed = fix::clean_underscores(&title); + if fixed != title { + changes.push(Change { + field: "title".into(), + from: title, + to: fixed.clone(), + why: "underscore_title".into(), + }); + cleaned.title = Some(fixed); + } + } + + if let Some(title) = cleaned.title.clone() + && fix::is_shouting(&title) + { + let fixed = fix::title_case(&title); + if fixed != title { + changes.push(Change { + field: "title".into(), + from: title, + to: fixed.clone(), + why: "allcaps_title".into(), + }); + cleaned.title = Some(fixed); + } + } + + // A missing ISBN is often sitting in the filename, where it is free and + // certain — the checksum is what makes it certain. + if reasons.iter().any(|reason| reason == "missing_isbn") + && let Some(path) = book.path.as_deref() + && let Some(found) = isbn::from_filename(path) + { + changes.push(Change { + field: "isbn".into(), + from: book.isbn.clone().unwrap_or_default(), + to: found.clone(), + why: "missing_isbn".into(), + }); + cleaned.isbn = Some(found); + } + + Cleaned { + severity: reasons::severity(&reasons), + reasons, + changes, + book: cleaned, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn book(title: &str, author: &str) -> Book { + Book { + v: corpus_core::V, + title: Some(title.into()), + author: Some(author.into()), + isbn: None, + path: None, + extra: Map::new(), + } + } + + #[test] + fn a_good_record_comes_back_unchanged() { + let book = Book { + isbn: Some("9780198237907".into()), + ..book("The Human Condition", "Arendt, Hannah") + }; + let cleaned = clean(&book); + assert_eq!(cleaned.book, book); + assert!(cleaned.reasons.is_empty()); + assert!(cleaned.changes.is_empty()); + assert_eq!(cleaned.severity, 0); + } + + #[test] + fn a_shouting_title_is_cased_and_the_change_is_recorded() { + let cleaned = clean(&book("THE HUMAN CONDITION", "Arendt, Hannah")); + assert_eq!(cleaned.book.title.as_deref(), Some("The Human Condition")); + assert_eq!(cleaned.changes.len(), 1); + assert_eq!(cleaned.changes[0].field, "title"); + assert_eq!(cleaned.changes[0].why, "allcaps_title"); + assert_eq!(cleaned.changes[0].from, "THE HUMAN CONDITION"); + } + + #[test] + fn an_underscored_title_is_cleaned_before_it_is_cased() { + let cleaned = clean(&book("THE FALL_ AND THE RISE_txt", "Arendt, Hannah")); + assert_eq!( + cleaned.book.title.as_deref(), + Some("The Fall: and the Rise") + ); + assert_eq!(cleaned.changes.len(), 2); + } + + #[test] + fn an_isbn_in_the_filename_fills_a_missing_one() { + let mut book = book("The Human Condition", "Arendt, Hannah"); + book.path = Some("/library/9780198237907.pdf".into()); + let cleaned = clean(&book); + assert_eq!(cleaned.book.isbn.as_deref(), Some("9780198237907")); + assert_eq!(cleaned.changes[0].why, "missing_isbn"); + } + + #[test] + fn a_filename_that_is_not_an_isbn_fills_nothing() { + let mut book = book("The Human Condition", "Arendt, Hannah"); + book.path = Some("/library/20231115123456.pdf".into()); + assert_eq!(clean(&book).book.isbn, None); + } + + #[test] + fn fields_it_does_not_know_about_are_carried_through() { + let mut book = book("A Book", "An Author"); + book.extra.insert("shelf".into(), Value::from("upstairs")); + let cleaned = clean(&book); + assert_eq!(cleaned.book.extra["shelf"], "upstairs"); + } + + #[test] + fn the_output_carries_the_record_and_the_audit_on_one_line() { + let cleaned = clean(&book("THE HUMAN CONDITION", "Arendt, Hannah")); + let json: Value = serde_json::to_value(&cleaned).expect("a record serialises"); + assert_eq!(json["title"], "The Human Condition"); + assert!(json["reasons"].as_array().is_some()); + assert!(json["changes"].as_array().is_some()); + } +} diff --git a/crates/bibclean/src/main.rs b/crates/bibclean/src/main.rs new file mode 100644 index 0000000..e106fe1 --- /dev/null +++ b/crates/bibclean/src/main.rs @@ -0,0 +1,77 @@ +//! Bibliographic hygiene over catalogue records. + +use std::path::PathBuf; +use std::process::ExitCode; + +use bibclean::Book; +use clap::Parser; +use corpus_core::{exit, ndjson}; + +/// Audit catalogue records and repair what can be repaired without a lookup. +/// +/// Reads one JSON record per line — `title`, `author`, `isbn` and `path` are +/// read, everything else is carried through untouched — and writes each one +/// back with the faults it has, how much they weigh, and a line for every +/// change made and why. +/// +/// Nothing is rewritten silently. A run can be read before it is agreed to, +/// which is the difference between cleaning a catalogue and damaging it in a +/// new way. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 nothing needed attention, 1 something did, 65 a record could +not be read. + +Example: + + calibredb list --for-machine | jq -c '.[]' | bibclean --flagged" +)] +struct Args { + /// Record stream to read. Reads stdin when absent. + path: Option, + + /// Write only the records that need attention. A missing ISBN alone does + /// not count as needing attention: nearly every book lacks one. + #[arg(long)] + flagged: bool, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("bibclean: {why}"); + return ExitCode::from(code); + } + }; + + let books: Vec = match ndjson::read(text.as_bytes()).collect() { + Ok(books) => books, + Err(e) => { + eprintln!("bibclean: {e}"); + return ExitCode::from(exit::DATA); + } + }; + + let mut flagged = false; + let cleaned: Vec = books + .iter() + .map(bibclean::clean) + .inspect(|cleaned| flagged |= bibclean::reasons::worth_fixing(&cleaned.reasons)) + .filter(|cleaned| !args.flagged || bibclean::reasons::worth_fixing(&cleaned.reasons)) + .collect(); + + let wrote = catena_cli::stream("bibclean", &cleaned); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + ExitCode::from(if flagged { exit::NO } else { exit::OK }) +} diff --git a/crates/bibclean/src/matching.rs b/crates/bibclean/src/matching.rs new file mode 100644 index 0000000..b639a51 --- /dev/null +++ b/crates/bibclean/src/matching.rs @@ -0,0 +1,98 @@ +//! Deciding whether two records name the same work. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; + +/// Words in a title, as the comparison sees them. +static WORDS: LazyLock = LazyLock::new(|| Regex::new(r"[a-z0-9]+").expect("valid")); + +/// How much of the shorter title's distinctive vocabulary must appear in the +/// other before both are believed to name the same book. +const THRESHOLD: f64 = 0.8; + +/// Words that carry no identifying weight. +/// +/// Kept deliberately small: anything dropped here stops being able to tell +/// two books apart. +const STOPWORDS: &[&str] = &[ + "a", "an", "and", "as", "at", "by", "ed", "edition", "for", "from", "her", "his", "in", "its", + "new", "of", "on", "or", "part", "revised", "second", "the", "their", "third", "to", "vol", + "volume", "with", +]; + +/// The words of a title that identify it. +pub fn tokens(title: &str) -> BTreeSet { + WORDS + .find_iter(&title.to_lowercase()) + .map(|found| found.as_str().to_string()) + .filter(|word| word.chars().count() > 2) + .filter(|word| STOPWORDS.binary_search(&word.as_str()).is_err()) + .collect() +} + +/// Whether two titles name the same work. +/// +/// Character similarity is not usable here, and a real library shows exactly +/// why: "The Cambridge Companion to Aquinas" and "The Cambridge Companion to +/// Hume" are eighty-six per cent the same string, because they differ in one +/// short word out of five. Believing that attaches Hume's blurb and Hume's +/// subjects to Aquinas — silently, and to a whole shelf of series titles at +/// once. +/// +/// So the comparison is of vocabulary rather than characters: nearly all of +/// the shorter title's distinctive words have to appear in the longer one. +/// A subtitle and a leading "The" are then free, while a swapped subject is +/// fatal. +pub fn same_work(ours: &str, theirs: &str) -> bool { + let ours = tokens(ours); + let theirs = tokens(theirs); + if ours.is_empty() || theirs.is_empty() { + return false; + } + let (shorter, longer) = if ours.len() <= theirs.len() { + (&ours, &theirs) + } else { + (&theirs, &ours) + }; + shorter.intersection(longer).count() as f64 / shorter.len() as f64 >= THRESHOLD +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_stopword_list_is_sorted_so_it_can_be_searched() { + assert!(STOPWORDS.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn a_subtitle_does_not_make_it_a_different_book() { + assert!(same_work( + "Arguing About War", + "Arguing About War: A Study in Ethics" + )); + assert!(same_work("The Human Condition", "Human Condition")); + } + + #[test] + fn one_swapped_word_makes_it_a_different_book() { + assert!(!same_work( + "The Cambridge Companion to Aquinas", + "The Cambridge Companion to Hume" + )); + } + + #[test] + fn a_title_with_nothing_distinctive_in_it_matches_nothing() { + assert!(!same_work("The Of And", "The Of And")); + assert!(!same_work("", "The Human Condition")); + } + + #[test] + fn case_and_punctuation_do_not_matter() { + assert!(same_work("THE HUMAN CONDITION!", "the human condition")); + } +} diff --git a/crates/bibclean/src/reasons.rs b/crates/bibclean/src/reasons.rs new file mode 100644 index 0000000..e9b82ab --- /dev/null +++ b/crates/bibclean/src/reasons.rs @@ -0,0 +1,515 @@ +//! What is wrong with a record, and how badly. + +use std::sync::LazyLock; + +use fancy_regex::Regex as Boundaried; +use regex::Regex; +use textkit::pattern::{SPACE, WORD_END, WORD_START}; + +fn pattern(source: &str) -> Regex { + Regex::new(source).expect("a valid pattern") +} + +fn bounded(source: &str) -> Boundaried { + Boundaried::new(source).expect("a valid pattern") +} + +/// A bare filename sitting in the title field. +static FILENAME_TITLE: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"(?i)\.(png|jpe?g|pdf|epub|mobi|azw3|azw|txt|wpd|docx?|rtf|tex|qxd|indd|pm\d){SPACE}*$" + )) +}); + +/// A title that names the tool that made the file rather than the work. +/// +/// Authoring and desktop-publishing programs stamp their scratch filename as +/// the PDF title; scanners and converters name themselves. The last of these +/// is a Turkish locale's "full page fax print" — the same driver, localised. +static JUNK_TITLE: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?i)microsoft word|z-?lib|libgen|\(z-lib|annas?-?archive|{WORD_START}ocr{WORD_END}|nota bene|pagemaker|quarkxpress|adobe indesign|acrobat distiller|word ?pro|wordperfect|{WORD_START}dvips{WORD_END}|{WORD_START}ltx{WORD_END}|{WORD_START}pdftex{WORD_END}|djvu document|untitled document|full page fax print|tam sayfa faks" + )) +}); + +/// A path sitting in the title field. Distinct from a filename, which is a +/// bare name ending in an extension. +static PATH_TITLE: LazyLock = + LazyLock::new(|| pattern(r"(^[A-Za-z]:[\\/])|(\\\\[^\\]+\\)|(^/(?:home|users|mnt|tmp|var)/)")); + +/// A title that is a placeholder rather than a name. +/// +/// Anchored and whole-string: a real book called "The Document" or "Test" +/// must not be swept up, so these only match when the entire field is the +/// placeholder. +static PLACEHOLDER_TITLE: LazyLock = LazyLock::new(|| { + pattern( + r"(?i)^\W*(?:some[\s_-]*title|no[\s_-]*title|untitled|unnamed|title|document|documento|test|sample|default|placeholder|tbd|todo|draft|temp|new[\s_-]*document|book\d*|file\d*|scan\d*|doc\d*|n/?a|null|none|x{3,}|unknown)\W*$", + ) +}); + +static PLACEHOLDER_AUTHOR: LazyLock = LazyLock::new(|| { + pattern( + r"(?i)^\W*(?:some[\s_-]*author|no[\s_-]*author|unnamed|author|default|test|sample|placeholder|tbd|todo|n/?a|null|none|x{3,}|administrator|user|owner|admin)\W*$", + ) +}); + +/// An author field that opens with a bare three or four digit number is a +/// publication year or a series index in the wrong column, never a person. +/// Anchored, so a real name containing a number is untouched. +static NUMERIC_AUTHOR: LazyLock = + LazyLock::new(|| pattern(&format!(r"^{SPACE}*\d{{3,4}}{SPACE}*(?:$|[{SPACE}.\-–—])"))); + +/// A possessive apostrophe that a filename or an OCR pass stripped: +/// "Lossky s Argument". +static MISSING_APOSTROPHE: LazyLock = + LazyLock::new(|| bounded(&format!(r"{WORD_START}[A-Za-z]{{2,}} s [A-Z]"))); + +/// Punctuation that a title has and a name does not. +static TITLE_PUNCTUATION: LazyLock = LazyLock::new(|| pattern(r"[!?:]")); + +/// A surname-first comma or a run of initials. +static INITIALS: LazyLock = + LazyLock::new(|| bounded(&format!(r"{WORD_START}[A-Z]\.{SPACE}*[A-Z]?\.?"))); + +/// Function words that essentially never appear in a personal name and are +/// everywhere in titles. +static TITLE_WORDS: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?i){WORD_START}(of|the|and|in|on|for|to|from|with|a|an|about|between|through|introduction|dictionary|handbook|companion|history|study|essays|conception|objections|architecture|philosophy|theory){WORD_END}" + )) +}); + +/// How authors are joined when several share a book. +static AUTHOR_SEPARATOR: LazyLock = + LazyLock::new(|| pattern(&format!(r"{SPACE}+&{SPACE}+|{SPACE}*;{SPACE}*"))); + +/// Publisher and imprint names that landed in the author field. +/// +/// Matched whole — "Oxford University Press" as an author is wrong, but +/// "Oxford, Michael" is a person. Kept lowercase and compared against the +/// normalised field. +const PUBLISHER_AUTHORS: &[&str] = &[ + "baker academic", + "blackwell", + "bloomsbury", + "brill", + "cambridge", + "cambridge university press", + "continuum", + "cornell university press", + "crossway", + "de gruyter", + "eerdmans", + "elsevier", + "fortress press", + "harpercollins", + "harvard university press", + "ignatius press", + "intervarsity press", + "macmillan", + "mit press", + "oxford", + "oxford university press", + "palgrave", + "palgrave macmillan", + "penguin", + "penguin books", + "princeton university press", + "random house", + "routledge", + "sage", + "simon schuster", + "springer", + "state university of new york press", + "suny", + "suny press", + "t t clark", + "university of chicago press", + "wiley blackwell", + "wipf stock", + "yale university press", + "zondervan", +]; + +/// Author fields that say there is no author. +const UNKNOWN_AUTHORS: &[&str] = &["", "anonymous", "n/a", "unknown", "unknown author"]; + +/// Word endings that mark a personal name. +const PERSON_SUFFIXES: &[&str] = &["ed", "eds", "ii", "iii", "iv", "jr", "md", "phd", "sr"]; + +/// What each fault is worth when the worst records are wanted first. +const WEIGHTS: &[(&str, u32)] = &[ + ("allcaps_title", 2), + ("author_last_first", 1), + ("author_role_suffix", 2), + ("edited_volume", 1), + ("filename_title", 3), + ("junk_title", 3), + ("lowercase_title", 2), + ("missing_apostrophe", 2), + ("numeric_author", 4), + ("path_title", 4), + ("placeholder_author", 4), + ("placeholder_title", 4), + ("possible_swap", 2), + ("publisher_as_author", 3), + ("sentence_case_title", 1), + ("title_in_author", 3), + ("underscore_title", 1), + ("unknown_author", 3), +]; + +/// Everything wrong with a record, named. +/// +/// `missing_isbn` is reported but is not on its own a reason to do anything: +/// nearly every book lacks one. +pub fn problems(title: Option<&str>, author: Option<&str>, isbn: Option<&str>) -> Vec { + let title = title.unwrap_or("").trim_matches(textkit::is_space); + let author = author.unwrap_or("").trim_matches(textkit::is_space); + let mut reasons = Vec::new(); + + if UNKNOWN_AUTHORS + .binary_search(&author.to_lowercase().as_str()) + .is_ok() + { + reasons.push("unknown_author"); + } + if PLACEHOLDER_TITLE.is_match(title) { + reasons.push("placeholder_title"); + } + if PLACEHOLDER_AUTHOR.is_match(author) { + reasons.push("placeholder_author"); + } + if PATH_TITLE.is_match(title) { + reasons.push("path_title"); + } + if FILENAME_TITLE.is_match(title) { + reasons.push("filename_title"); + } + if JUNK_TITLE.is_match(title).unwrap_or(false) { + reasons.push("junk_title"); + } + if is_publisher(author) { + reasons.push("publisher_as_author"); + } + if NUMERIC_AUTHOR + .find(author) + .is_some_and(|at| at.start() == 0) + { + reasons.push("numeric_author"); + } + if title.contains('_') { + reasons.push("underscore_title"); + } + if !title.is_empty() && title == title.to_uppercase() && title.contains(' ') { + reasons.push("allcaps_title"); + } + // A filename stub: "heidegger areopagite". Distinct from sentence case, + // which is a much weaker signal. + if !title.is_empty() + && title == title.to_lowercase() + && title.chars().any(|c| c.is_lowercase()) + && (title.contains(' ') || title.chars().count() >= 12) + { + reasons.push("lowercase_title"); + } + if MISSING_APOSTROPHE.is_match(title).unwrap_or(false) { + reasons.push("missing_apostrophe"); + } + // Punctuation in the author field, or a long author against a short + // title. Gated on the author not looking like a person, because several + // authors joined together are many words against what may be a one-word + // title, and that is not a swap. + if !looks_like_person(author) + && (TITLE_PUNCTUATION.is_match(author) + || (author.split_whitespace().count() >= 5 + && (1..=3).contains(&title.split_whitespace().count()))) + { + reasons.push("possible_swap"); + } + if title_in_author(Some(title), Some(author)) { + reasons.push("title_in_author"); + } + if isbn + .unwrap_or("") + .trim_matches(textkit::is_space) + .is_empty() + { + reasons.push("missing_isbn"); + } + + reasons.into_iter().map(str::to_string).collect() +} + +/// How much attention a set of faults deserves. +pub fn severity(reasons: &[String]) -> u32 { + reasons + .iter() + .filter_map(|reason| { + WEIGHTS + .binary_search_by_key(&reason.as_str(), |(name, _)| name) + .ok() + .map(|at| WEIGHTS[at].1) + }) + .sum() +} + +/// Whether anything here needs doing. A missing ISBN alone does not. +pub fn worth_fixing(reasons: &[String]) -> bool { + reasons.iter().any(|reason| reason != "missing_isbn") +} + +/// Whether the title field is a placeholder and the author field is holding +/// the real title. +/// +/// Deliberately conservative: it must be both a placeholder title and an +/// author that does not read as a person, because what this drives — looking +/// the book up by the author text as a title — is wasted work otherwise. +pub fn title_in_author(title: Option<&str>, author: Option<&str>) -> bool { + let title = title.unwrap_or("").trim_matches(textkit::is_space); + let author = author.unwrap_or("").trim_matches(textkit::is_space); + + if author.is_empty() || !(PLACEHOLDER_TITLE.is_match(title) || PATH_TITLE.is_match(title)) { + return false; + } + // A placeholder author is not a stray title. Treating it as one sends a + // junk string to a catalogue as a title query, which is the spurious + // match every other part of this guards against. + if PLACEHOLDER_AUTHOR.is_match(author) { + return false; + } + if is_publisher(author) + || UNKNOWN_AUTHORS + .binary_search(&author.to_lowercase().as_str()) + .is_ok() + { + return false; + } + !looks_like_person(author) +} + +/// Whether a field reads as a human name. +/// +/// Multi-author strings are judged one author at a time. A catalogue joins +/// authors with an ampersand, so two people arrive as one seven-word field — +/// judged whole it fails every test and reads as a title, which is how a +/// naive version of this flagged hundreds of perfectly good records. +pub fn looks_like_person(author: &str) -> bool { + let author = author.trim_matches(textkit::is_space); + if author.is_empty() { + return false; + } + + let parts: Vec<&str> = AUTHOR_SEPARATOR + .split(author) + .map(|part| part.trim_matches(textkit::is_space)) + .filter(|part| !part.is_empty()) + .collect(); + if parts.len() > 1 { + return parts.iter().all(|part| looks_like_person(part)); + } + + let words: Vec<&str> = author.split_whitespace().collect(); + if author.contains(',') && words.len() <= 5 { + return true; // "Surname, Given" + } + if INITIALS.is_match(author).unwrap_or(false) && words.len() <= 5 { + return true; + } + if words.len() > 5 { + return false; + } + if TITLE_WORDS.is_match(author).unwrap_or(false) { + return false; + } + if words + .iter() + .any(|word| PERSON_SUFFIXES.contains(&word.to_lowercase().trim_matches('.'))) + { + return true; + } + // A bare single word is a mononym far more often than a stray title — + // Plato, Homer and Josephus are all over a philosophy library, and a + // one-word "title" recovered from an author field is nearly never right. + (1..=4).contains(&words.len()) +} + +/// A catalogue string reduced to the letters and digits in it. +pub fn normalise(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut space = false; + for c in text.to_lowercase().chars() { + if c.is_ascii_alphanumeric() { + if space && !out.is_empty() { + out.push(' '); + } + space = false; + out.push(c); + } else { + space = true; + } + } + out +} + +fn is_publisher(author: &str) -> bool { + PUBLISHER_AUTHORS + .binary_search(&normalise(author).as_str()) + .is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn problems_of(title: &str, author: &str) -> Vec { + problems(Some(title), Some(author), Some("9780198237907")) + } + + #[test] + fn the_tables_are_sorted_so_they_can_be_searched() { + assert!(PUBLISHER_AUTHORS.windows(2).all(|pair| pair[0] < pair[1])); + assert!(UNKNOWN_AUTHORS.windows(2).all(|pair| pair[0] < pair[1])); + assert!(PERSON_SUFFIXES.windows(2).all(|pair| pair[0] < pair[1])); + assert!(WEIGHTS.windows(2).all(|pair| pair[0].0 < pair[1].0)); + } + + #[test] + fn a_good_record_has_nothing_wrong_with_it() { + assert!(problems_of("The Human Condition", "Arendt, Hannah").is_empty()); + } + + #[test] + fn a_missing_isbn_is_reported_but_is_not_worth_fixing() { + let reasons = problems(Some("The Human Condition"), Some("Arendt, Hannah"), None); + assert_eq!(reasons, ["missing_isbn"]); + assert!(!worth_fixing(&reasons)); + } + + #[test] + fn a_placeholder_title_is_found() { + for title in ["Untitled", "untitled", "no title", "Document", "xxx", "N/A"] { + assert!( + problems_of(title, "Arendt, Hannah").contains(&"placeholder_title".to_string()), + "{title}" + ); + } + } + + #[test] + fn a_real_title_that_resembles_a_placeholder_is_left_alone() { + for title in ["The Document", "A Test of Faith", "Untitled Symphony"] { + assert!( + !problems_of(title, "Arendt, Hannah").contains(&"placeholder_title".to_string()), + "{title}" + ); + } + } + + #[test] + fn a_filename_and_a_path_in_the_title_are_told_apart() { + assert!(problems_of("chapter1.pdf", "x").contains(&"filename_title".to_string())); + assert!(problems_of("A:\\pwecent2005.wpd", "x").contains(&"path_title".to_string())); + assert!(problems_of("/home/matt/book", "x").contains(&"path_title".to_string())); + } + + #[test] + fn a_tool_name_in_the_title_is_junk() { + for title in [ + "Microsoft Word - chapter3", + "Book (z-lib.org)", + "DjVu Document", + "Full Page Fax Print", + "scan OCR output", + ] { + assert!( + problems_of(title, "x").contains(&"junk_title".to_string()), + "{title}" + ); + } + } + + #[test] + fn a_word_that_merely_contains_a_tool_name_is_not_junk() { + assert!(!problems_of("Procrastination", "x").contains(&"junk_title".to_string())); + } + + #[test] + fn a_publisher_in_the_author_field_is_found() { + assert!( + problems_of("A Book", "Oxford University Press") + .contains(&"publisher_as_author".to_string()) + ); + assert!( + !problems_of("A Book", "Oxford, Michael").contains(&"publisher_as_author".to_string()) + ); + } + + #[test] + fn shouting_and_whispering_are_both_noticed() { + assert!(problems_of("THE HUMAN CONDITION", "x").contains(&"allcaps_title".to_string())); + assert!(problems_of("heidegger areopagite", "x").contains(&"lowercase_title".to_string())); + assert!(!problems_of("MIT", "x").contains(&"allcaps_title".to_string())); + } + + #[test] + fn several_authors_are_not_a_swapped_title() { + let reasons = problems_of("Rama", "Arthur C. Clarke & Gentry Lee"); + assert!( + !reasons.contains(&"possible_swap".to_string()), + "{reasons:?}" + ); + } + + #[test] + fn a_title_in_the_author_field_is_found() { + assert!(title_in_author(Some("Untitled"), Some("Arguing About War"))); + assert!(!title_in_author(Some("Untitled"), Some("Arendt, Hannah"))); + assert!(!title_in_author( + Some("The Human Condition"), + Some("Arguing About War") + )); + assert!(!title_in_author(Some("Untitled"), Some("Default"))); + assert!(!title_in_author(Some("Untitled"), Some(""))); + } + + #[test] + fn people_and_titles_are_told_apart() { + for person in [ + "Arendt, Hannah", + "C. S. Lewis", + "Plato", + "Hannah Arendt", + "C. S. Lewis & Madeleine L'Engle", + "Smith Jr", + ] { + assert!(looks_like_person(person), "{person}"); + } + for title in [ + "An Ethnohistorical Dictionary China", + "Kierkegaard's Conception of God", + "The Oxford Handbook of Something Or Other", + ] { + assert!(!looks_like_person(title), "{title}"); + } + } + + #[test] + fn severity_ranks_the_worst_first() { + let bad = problems_of("Untitled", "Arguing About War"); + let mild = problems_of("Some_Book", "Arendt, Hannah"); + assert!(severity(&bad) > severity(&mild), "{bad:?} vs {mild:?}"); + assert_eq!(severity(&[]), 0); + } + + #[test] + fn normalising_keeps_only_what_identifies() { + assert_eq!( + normalise("Oxford University Press!"), + "oxford university press" + ); + assert_eq!(normalise(" T&T Clark "), "t t clark"); + assert_eq!(normalise(""), ""); + } +} diff --git a/parity/Cargo.toml b/parity/Cargo.toml index 6a0cde3..9d00f26 100644 --- a/parity/Cargo.toml +++ b/parity/Cargo.toml @@ -14,6 +14,7 @@ publish = false python = [] [dependencies] +bibclean.workspace = true booksniff.workspace = true chunkmd.workspace = true cuekit.workspace = true diff --git a/parity/cases/bibclean/records.ndjson b/parity/cases/bibclean/records.ndjson new file mode 100644 index 0000000..ab87250 --- /dev/null +++ b/parity/cases/bibclean/records.ndjson @@ -0,0 +1,33 @@ +{"title": "The Human Condition", "author": "Arendt, Hannah", "isbn": "9780226586601"} +{"title": "THE HUMAN CONDITION", "author": "Arendt, Hannah", "isbn": null} +{"title": "Untitled", "author": "Arguing About War", "isbn": null} +{"title": "Untitled", "author": "Default", "isbn": null} +{"title": "chapter1.pdf", "author": "Unknown", "isbn": null} +{"title": "A:\\pwecent2005.wpd", "author": "", "isbn": null} +{"title": "/home/matt/book", "author": "1993", "isbn": null} +{"title": "Microsoft Word - ch3.doc", "author": "Oxford University Press", "isbn": null} +{"title": "Board_Advisors_etc", "author": "C. S. Lewis & Madeleine L'Engle", "isbn": null} +{"title": "The Fall_ And the Rise_txt", "author": "Plato", "isbn": null} +{"title": "heidegger areopagite", "author": "Smith Jr", "isbn": null} +{"title": "Lossky s Argument", "author": "Kierkegaard's Conception of God", "isbn": null} +{"title": "Rama", "author": "Arthur C. Clarke & Gentry Lee", "isbn": null} +{"title": "MIT", "author": "An Ethnohistorical Dictionary China", "isbn": null} +{"title": "ÊTRE ET TEMPS", "author": "Heidegger, Martin", "isbn": null} +{"title": "ÉTUDES SUR LE MOYEN ÂGE", "author": "Gilson, Étienne", "isbn": null} +{"title": "DON'T LOOK NOW", "author": "du Maurier, Daphne", "isbn": null} +{"title": "THE WELL-BUILT WHEEL", "author": "", "isbn": null} +{"title": "The (Second) Estate", "author": "Bloch, Marc", "isbn": null} +{"title": null, "author": null, "isbn": null} +{"title": "", "author": "", "isbn": ""} +{"title": " spaced ", "author": " Arendt, Hannah ", "isbn": "978-0-226-58660-1"} +{"title": "A Book", "author": "anonymous", "isbn": "080213825X"} +{"title": "Scan 001", "author": "administrator", "isbn": "1234567890123"} +{"title": "xxx", "author": "xxxx", "isbn": null} +{"title": "The Document", "author": "Oxford, Michael", "isbn": null} +{"title": "Full Page Fax Print", "author": "user", "isbn": null} +{"title": "Tam Sayfa Faks Yazdırma", "author": "Routledge", "isbn": null} +{"title": "A Book", "author": "Arendt", "isbn": null, "path": "/library/9780198237907.pdf"} +{"title": "A Book", "author": "Arendt", "isbn": null, "path": "/library/20231115123456.pdf"} +{"title": "A Book", "author": "Arendt", "isbn": null, "path": "Aquinas - 080213825X.epub"} +{"title": "Ἀθῆναι καὶ Μόσχα", "author": "Θουκυδίδης", "isbn": null} +{"title": "東京の手紙", "author": "夏目漱石", "isbn": null} diff --git a/parity/golden/bibclean/records.ndjson b/parity/golden/bibclean/records.ndjson new file mode 100644 index 0000000..0164238 --- /dev/null +++ b/parity/golden/bibclean/records.ndjson @@ -0,0 +1,33 @@ +{"clean_underscores": "The Human Condition", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": [], "severity": 0, "title_case": "The Human Condition", "title_in_author": false, "valid_isbn": true} +{"clean_underscores": "THE HUMAN CONDITION", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "The Human Condition", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Untitled", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "arguing about war", "reasons": ["placeholder_title", "title_in_author", "missing_isbn"], "severity": 7, "title_case": "Untitled", "title_in_author": true, "valid_isbn": false} +{"clean_underscores": "Untitled", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "default", "reasons": ["placeholder_title", "placeholder_author", "missing_isbn"], "severity": 8, "title_case": "Untitled", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "chapter1.", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "unknown", "reasons": ["unknown_author", "filename_title", "lowercase_title", "missing_isbn"], "severity": 8, "title_case": "Chapter1.pdf", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "A:\\pwecent2005.wpd", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "path_title", "filename_title", "missing_isbn"], "severity": 10, "title_case": "A:\\pwecent2005.wpd", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "/home/matt/book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "1993", "reasons": ["path_title", "numeric_author", "lowercase_title", "missing_isbn"], "severity": 10, "title_case": "/Home/matt/book", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Microsoft Word - ch3.doc", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "oxford university press", "reasons": ["filename_title", "junk_title", "publisher_as_author", "missing_isbn"], "severity": 9, "title_case": "Microsoft Word - Ch3.doc", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Board Advisors etc", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "c s lewis madeleine l engle", "reasons": ["underscore_title", "missing_isbn"], "severity": 1, "title_case": "Board_advisors_etc", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "The Fall: And the Rise", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "plato", "reasons": ["underscore_title", "missing_isbn"], "severity": 1, "title_case": "The Fall_ and the Rise_txt", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "heidegger areopagite", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "smith jr", "reasons": ["lowercase_title", "missing_isbn"], "severity": 2, "title_case": "Heidegger Areopagite", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Lossky s Argument", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "kierkegaard s conception of god", "reasons": ["missing_apostrophe", "missing_isbn"], "severity": 2, "title_case": "Lossky S Argument", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Rama", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arthur c clarke gentry lee", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Rama", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "MIT", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "an ethnohistorical dictionary china", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Mit", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "ÊTRE ET TEMPS", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "heidegger martin", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Être Et Temps", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "ÉTUDES SUR LE MOYEN ÂGE", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "gilson tienne", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Études Sur Le Moyen Âge", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "DON'T LOOK NOW", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "du maurier daphne", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Don't Look Now", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "THE WELL-BUILT WHEEL", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "allcaps_title", "missing_isbn"], "severity": 5, "title_case": "The Well-Built Wheel", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "The (Second) Estate", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "bloch marc", "reasons": ["missing_isbn"], "severity": 0, "title_case": "The (Second) Estate", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": null, "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "missing_isbn"], "severity": 3, "title_case": null, "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "missing_isbn"], "severity": 3, "title_case": "", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "spaced", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": [], "severity": 0, "title_case": "Spaced", "title_in_author": false, "valid_isbn": true} +{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "anonymous", "reasons": ["unknown_author"], "severity": 3, "title_case": "A Book", "title_in_author": false, "valid_isbn": true} +{"clean_underscores": "Scan 001", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "administrator", "reasons": ["placeholder_author"], "severity": 4, "title_case": "Scan 001", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "xxx", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "xxxx", "reasons": ["placeholder_title", "placeholder_author", "missing_isbn"], "severity": 8, "title_case": "Xxx", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "The Document", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "oxford michael", "reasons": ["missing_isbn"], "severity": 0, "title_case": "The Document", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Full Page Fax Print", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "user", "reasons": ["placeholder_author", "junk_title", "missing_isbn"], "severity": 7, "title_case": "Full Page Fax Print", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Tam Sayfa Faks Yazdırma", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "routledge", "reasons": ["junk_title", "publisher_as_author", "missing_isbn"], "severity": 6, "title_case": "Tam Sayfa Faks Yazdırma", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": "9780198237907", "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": "080213825X", "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "Ἀθῆναι καὶ Μόσχα", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Ἀθῆναι Καὶ Μόσχα", "title_in_author": false, "valid_isbn": false} +{"clean_underscores": "東京の手紙", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "", "reasons": ["missing_isbn"], "severity": 0, "title_case": "東京の手紙", "title_in_author": false, "valid_isbn": false} diff --git a/parity/oracle/bibclean.py b/parity/oracle/bibclean.py new file mode 100644 index 0000000..753a05c --- /dev/null +++ b/parity/oracle/bibclean.py @@ -0,0 +1,48 @@ +"""Audit catalogue records the way the Python pipeline audits them. + +Reads one `{"title", "author", "isbn", "path"}` object per line and writes +what each of the metadata functions makes of it. + + CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \ + python3 oracle/bibclean.py < cases/bibclean/records.ndjson +""" + +import json +import os +import sys + +for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep): + if path: + sys.path.insert(0, path) + +from book_pipeline import metadata # noqa: E402 + + +def main() -> int: + out = sys.stdout + for line in sys.stdin.buffer.read().decode("utf-8").split("\n"): + if not line.strip(): + continue + book = json.loads(line) + title = book.get("title") + author = book.get("author") + isbn = book.get("isbn") + reasons = metadata.problem_reasons(title, author, isbn) + record = { + "reasons": reasons, + "severity": metadata.severity(reasons), + "title_in_author": metadata.title_in_author(title, author), + "looks_like_person": metadata._looks_like_person(author or ""), + "is_shouting": metadata.is_shouting(title), + "title_case": metadata.title_case(title) if title is not None else None, + "clean_underscores": metadata.clean_underscores(title) if title is not None else None, + "valid_isbn": metadata.valid_isbn(isbn), + "isbn_from_filename": metadata.isbn_from_filename(book.get("path")), + "normalise": metadata._norm(author), + } + out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index d972d2e..1adde87 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 10] = [ + let jobs: [Job; 11] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -30,6 +30,7 @@ fn main() -> ExitCode { ("rrf", rrf), ("cuepack", cuepack), ("booksniff", booksniff), + ("bibclean", bibclean), ]; for (name, job) in jobs { @@ -73,6 +74,18 @@ fn unescape() -> Written { Ok(vec![path.display().to_string()]) } +fn bibclean() -> Written { + let dir = parity::golden("bibclean"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/bibclean/records.ndjson"); + let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let judgements = parity::run_oracle("bibclean.py", &cases)?; + let path = dir.join("records.ndjson"); + std::fs::write(&path, judgements).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn booksniff() -> Written { let dir = parity::golden("booksniff"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/bibclean.rs b/parity/tests/bibclean.rs new file mode 100644 index 0000000..5ac925d --- /dev/null +++ b/parity/tests/bibclean.rs @@ -0,0 +1,123 @@ +//! Every judgement about a catalogue record must match the Python's. +//! +//! These decide what a cleanup run touches. A reason that fires here and not +//! there means a record is rewritten that should have been left alone, which +//! is the one failure mode a metadata tool cannot recover from. + +use serde_json::Value; + +fn records(path: &str) -> Vec { + let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file"); + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> { + record.get(name).and_then(Value::as_str) +} + +#[test] +fn every_judgement_matches_python() { + let cases = records("cases/bibclean/records.ndjson"); + let golden = records("golden/bibclean/records.ndjson"); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + let mut wrong: Vec = Vec::new(); + for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() { + let title = field(case, "title"); + let author = field(case, "author"); + let isbn = field(case, "isbn"); + + let mut differs = |what: &str, ours: String, theirs: &Value| { + wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}")); + }; + + let reasons = bibclean::reasons::problems(title, author, isbn); + let their_reasons: Vec = theirs["reasons"] + .as_array() + .expect("a list") + .iter() + .map(|reason| reason.as_str().expect("a name").to_string()) + .collect(); + if reasons != their_reasons { + differs("reasons", format!("{reasons:?}"), &theirs["reasons"]); + } + + let severity = bibclean::reasons::severity(&reasons); + if severity as u64 != theirs["severity"].as_u64().expect("a number") { + differs("severity", severity.to_string(), &theirs["severity"]); + } + + let ours = bibclean::reasons::title_in_author(title, author); + if ours != theirs["title_in_author"].as_bool().expect("a verdict") { + differs( + "title_in_author", + ours.to_string(), + &theirs["title_in_author"], + ); + } + + let ours = bibclean::reasons::looks_like_person(author.unwrap_or("")); + if ours != theirs["looks_like_person"].as_bool().expect("a verdict") { + differs( + "looks_like_person", + ours.to_string(), + &theirs["looks_like_person"], + ); + } + + let ours = bibclean::fix::is_shouting(title.unwrap_or("")); + if ours != theirs["is_shouting"].as_bool().expect("a verdict") { + differs("is_shouting", ours.to_string(), &theirs["is_shouting"]); + } + + if let Some(title) = title { + let ours = bibclean::fix::title_case(title); + if Some(ours.as_str()) != theirs["title_case"].as_str() { + differs("title_case", ours, &theirs["title_case"]); + } + let ours = bibclean::fix::clean_underscores(title); + if Some(ours.as_str()) != theirs["clean_underscores"].as_str() { + differs("clean_underscores", ours, &theirs["clean_underscores"]); + } + } + + let ours = bibclean::isbn::valid(isbn.unwrap_or("")); + if ours != theirs["valid_isbn"].as_bool().expect("a verdict") { + differs("valid_isbn", ours.to_string(), &theirs["valid_isbn"]); + } + + let ours = field(case, "path").and_then(bibclean::isbn::from_filename); + if ours.as_deref() != theirs["isbn_from_filename"].as_str() { + differs( + "isbn_from_filename", + format!("{ours:?}"), + &theirs["isbn_from_filename"], + ); + } + + let ours = bibclean::reasons::normalise(author.unwrap_or("")); + if Some(ours.as_str()) != theirs["normalise"].as_str() { + differs("normalise", ours, &theirs["normalise"]); + } + } + + assert!( + wrong.is_empty(), + "{} judgements differ:\n{}", + wrong.len(), + wrong.join("\n") + ); +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/bibclean/records.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("bibclean.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("bibclean/records.ndjson")); +}