bibclean: record faults, weighed, with the repairs that need no lookup

Cargo.lock

@@ -85,6 +85,21 @@ version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
 
+[[package]]
+name = "bibclean"
+version = "0.1.0"
+dependencies = [
+ "catena-cli",
+ "clap",
+ "corpus-core",
+ "fancy-regex 0.16.2",
+ "regex",
+ "schemars",
+ "serde",
+ "serde_json",
+ "textkit",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -653,6 +668,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 name = "parity"
 version = "0.1.0"
 dependencies = [
+ "bibclean",
  "booksniff",
  "chunkmd",
  "corpus-core",

Cargo.toml

@@ -21,6 +21,7 @@ cuenorm = { path = "crates/cuenorm", version = "0.1.0" }
 cueseg = { path = "crates/cueseg", version = "0.1.0" }
 rrf = { path = "crates/rrf", version = "0.1.0" }
 cuepack = { path = "crates/cuepack", version = "0.1.0" }
+bibclean = { path = "crates/bibclean", version = "0.1.0" }
 booksniff = { path = "crates/booksniff", version = "0.1.0" }
 emb = { path = "crates/emb", version = "0.1.0" }
 

README.md

@@ -53,6 +53,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done
 
 | | |
 |---|---|
+| `bibclean` | Catalogue records audited and repaired, with a reason for every change |
 | `booksniff` | What a file actually is, whatever its name claims |
 | `mdscrub` | Converted markdown in, clean markdown out: residual HTML, control characters, headings, page markers |
 | `chunkmd` | Markdown into passages that carry their page, chapter and role |

crates/bibclean/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "bibclean"
+description = "Bibliographic hygiene over catalogue records, with a reason for every change."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+catena-cli.workspace = true
+clap.workspace = true
+corpus-core.workspace = true
+fancy-regex.workspace = true
+regex.workspace = true
+schemars.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+textkit.workspace = true
+
+[lints]
+workspace = true

crates/bibclean/src/fix.rs

@@ -0,0 +1,220 @@
+//! Repairs that can be made without looking anything up.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+use textkit::pattern::SPACE;
+
+/// A format name stuck on the end of a title taken from a filename.
+static FORMAT_SUFFIX: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(&format!(
+        r"(?i)[_\-{SPACE}]*(txt|pdf|epub|mobi|azw3?|htmlz?|djvu){SPACE}*$"
+    ))
+    .expect("a valid pattern")
+});
+
+/// An underscore standing in for the colon before a subtitle.
+static SUBTITLE_UNDERSCORE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(&format!(r"_{SPACE}+")).expect("a valid pattern"));
+
+/// Two or more spaces where one belongs.
+static SPACE_RUN: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(&format!(r"{SPACE}{{2,}}")).expect("a valid pattern"));
+
+/// A run of letters, by Python's definition of a letter.
+static LETTERS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\p{L}+").expect("a valid pattern"));
+
+/// Words that stay lowercase inside a title.
+const SMALL_WORDS: &[&str] = &[
+    "a", "an", "and", "as", "at", "but", "by", "for", "from", "in", "into", "nor", "of", "on",
+    "or", "over", "per", "the", "to", "upon", "via", "with",
+];
+
+/// Whether a title is in capitals throughout.
+///
+/// A space is required, so a title that is only an acronym is left alone.
+pub fn is_shouting(title: &str) -> bool {
+    let title = title.trim_matches(textkit::is_space);
+    !title.is_empty()
+        && title == title.to_uppercase()
+        && title.contains(' ')
+        && title
+            .chars()
+            .collect::<Vec<_>>()
+            .windows(2)
+            .any(|pair| pair[0].is_ascii_uppercase() && pair[1].is_ascii_uppercase())
+}
+
+/// Title-case a shouting title without mangling it.
+///
+/// The obvious library function is unusable here: it gives "Don'T" and
+/// capitalises every preposition. Small words stay lowercase unless they open
+/// or close the title, hyphenated parts are cased on both sides, and the
+/// first *letter* is capitalised rather than the first character — an
+/// ASCII-only rule skips an accented opening letter and capitalises the one
+/// after it, turning "ÊTRE" into "êTre", and a third of a real library is not
+/// in English.
+pub fn title_case(title: &str) -> String {
+    let words: Vec<&str> = title.split_whitespace().collect();
+    let last = words.len().saturating_sub(1);
+
+    words
+        .iter()
+        .enumerate()
+        .map(|(at, word)| {
+            let core: String = word
+                .trim_start_matches(['(', '[', '"', '\''])
+                .to_lowercase()
+                .trim_end_matches(['.', ',', ':', ';', '!', '?', ')', ']', '"', '\''])
+                .to_string();
+            if SMALL_WORDS.binary_search(&core.as_str()).is_ok() && at > 0 && at < last {
+                return word.to_lowercase();
+            }
+            let lowered = word.to_lowercase();
+            if lowered.contains('-') {
+                lowered
+                    .split('-')
+                    .map(capitalise)
+                    .collect::<Vec<_>>()
+                    .join("-")
+            } else {
+                capitalise_first_letters(&lowered)
+            }
+        })
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
+/// Undo the underscore mangling a filename-derived title carries.
+///
+/// Three habits, in order: a format name on the end, an underscore standing
+/// in for the colon before a subtitle, and underscores used as plain spaces.
+pub fn clean_underscores(title: &str) -> String {
+    let title = FORMAT_SUFFIX.replace(title, "");
+    let title = title.trim_matches(textkit::is_space);
+    let title = SUBTITLE_UNDERSCORE.replace_all(title, ": ");
+    let title = title.replace('_', " ");
+    SPACE_RUN
+        .replace_all(&title, " ")
+        .trim_matches(|c: char| textkit::is_space(c) || c == ':' || c == ',' || c == '-')
+        .to_string()
+}
+
+/// Capitalise the first letter and lowercase the rest.
+fn capitalise(word: &str) -> String {
+    let lowered = word.to_lowercase();
+    capitalise_first_letters(&lowered)
+}
+
+/// Capitalise the first run of letters, leaving everything before it alone.
+fn capitalise_first_letters(lowered: &str) -> String {
+    match LETTERS.find(lowered) {
+        None => lowered.to_string(),
+        Some(found) => {
+            let mut out = String::with_capacity(lowered.len());
+            out.push_str(&lowered[..found.start()]);
+            let mut letters = found.as_str().chars();
+            if let Some(first) = letters.next() {
+                out.extend(first.to_uppercase());
+                out.push_str(letters.as_str());
+            }
+            out.push_str(&lowered[found.end()..]);
+            out
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_small_word_list_is_sorted_so_it_can_be_searched() {
+        assert!(SMALL_WORDS.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
+    #[test]
+    fn shouting_needs_more_than_one_word() {
+        assert!(is_shouting("THE HUMAN CONDITION"));
+        assert!(!is_shouting("MIT"));
+        assert!(!is_shouting("The Human Condition"));
+        assert!(!is_shouting(""));
+        assert!(!is_shouting("A B"));
+    }
+
+    #[test]
+    fn a_shouting_title_comes_back_readable() {
+        assert_eq!(title_case("THE HUMAN CONDITION"), "The Human Condition");
+        assert_eq!(
+            title_case("A HISTORY OF THE WORLD IN TEN CHAPTERS"),
+            "A History of the World in Ten Chapters"
+        );
+    }
+
+    #[test]
+    fn a_small_word_at_either_end_is_capitalised() {
+        assert_eq!(title_case("OF MICE AND MEN"), "Of Mice and Men");
+        assert_eq!(title_case("WHAT IS IT ALL FOR"), "What Is It All For");
+    }
+
+    #[test]
+    fn an_apostrophe_does_not_start_a_new_word() {
+        assert_eq!(title_case("DON'T LOOK NOW"), "Don't Look Now");
+    }
+
+    #[test]
+    fn a_hyphenated_word_is_cased_on_both_sides() {
+        assert_eq!(title_case("THE WELL-BUILT WHEEL"), "The Well-Built Wheel");
+    }
+
+    #[test]
+    fn an_accented_first_letter_is_the_one_capitalised() {
+        // The small-word list is English, so a French preposition is cased
+        // like any other word. That is what the original does, and a list per
+        // language is a different tool.
+        assert_eq!(title_case("ÊTRE ET TEMPS"), "Être Et Temps");
+        assert_eq!(
+            title_case("ÉTUDES SUR LE MOYEN ÂGE"),
+            "Études Sur Le Moyen Âge"
+        );
+    }
+
+    #[test]
+    fn a_bracketed_word_keeps_its_bracket() {
+        assert_eq!(title_case("THE (SECOND) ESTATE"), "The (Second) Estate");
+    }
+
+    #[test]
+    fn a_format_suffix_comes_off() {
+        assert_eq!(clean_underscores("The Fall_txt"), "The Fall");
+        assert_eq!(clean_underscores("The Fall - pdf"), "The Fall");
+        // A format name takes no separator to be one, so an extension goes
+        // too and leaves its dot behind. Faithful to the original; a title
+        // that is really a filename is being repaired either way.
+        assert_eq!(clean_underscores("The Fall.epub"), "The Fall.");
+    }
+
+    #[test]
+    fn an_underscore_before_a_subtitle_becomes_a_colon() {
+        assert_eq!(
+            clean_underscores("The 9/11 Mystery Plane_ And the Vanishing of America"),
+            "The 9/11 Mystery Plane: And the Vanishing of America"
+        );
+    }
+
+    #[test]
+    fn underscores_used_as_spaces_become_spaces() {
+        assert_eq!(
+            clean_underscores("Board_Advisors_etc"),
+            "Board Advisors etc"
+        );
+    }
+
+    #[test]
+    fn a_title_with_nothing_wrong_with_it_is_unchanged() {
+        assert_eq!(
+            clean_underscores("The Human Condition"),
+            "The Human Condition"
+        );
+    }
+}

crates/bibclean/src/isbn.rs

@@ -0,0 +1,149 @@
+//! Finding and checking ISBNs.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+use textkit::pattern::SPACE;
+
+/// An ISBN announced as one, in running text or a catalogue field.
+static ANNOUNCED: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(&format!(
+        r"(?i)ISBN[-{SPACE}:]*((?:97[89][{SPACE}-]?)?(?:[0-9][{SPACE}-]?){{9}}[0-9Xx])"
+    ))
+    .expect("a valid pattern")
+});
+
+/// A bare ISBN sitting in a filename.
+static BARE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(97[89][0-9]{10}|[0-9]{9}[0-9Xx])").expect("a valid pattern"));
+
+/// The extension at the end of a filename.
+static EXTENSION: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)\.[a-z0-9]+$").expect("a valid pattern"));
+
+/// Whether this is a real ISBN, by its check digit.
+///
+/// The checksum is the point. Without it any thirteen digits in a filename —
+/// a scan date, an internal identifier, a phone number — reads as an ISBN,
+/// and an ISBN is what a catalogue lookup trusts absolutely.
+pub fn valid(text: &str) -> bool {
+    let digits: String = text
+        .chars()
+        .filter(|c| c.is_ascii_digit() || *c == 'X' || *c == 'x')
+        .collect();
+
+    match digits.len() {
+        13 if digits.chars().all(|c| c.is_ascii_digit()) => {
+            let sum: u32 = digits
+                .chars()
+                .enumerate()
+                .map(|(at, digit)| {
+                    let weight = if at % 2 == 0 { 1 } else { 3 };
+                    weight * digit.to_digit(10).expect("a digit")
+                })
+                .sum();
+            sum % 10 == 0
+        }
+        10 if digits[..9].chars().all(|c| c.is_ascii_digit()) => {
+            let sum: u32 = digits
+                .chars()
+                .enumerate()
+                .map(|(at, c)| {
+                    let value = if c == 'X' || c == 'x' {
+                        10
+                    } else {
+                        c.to_digit(10).unwrap_or(0)
+                    };
+                    (10 - at as u32) * value
+                })
+                .sum();
+            sum % 11 == 0
+        }
+        _ => false,
+    }
+}
+
+/// Just the digits of an ISBN, with the spacing and hyphens taken out.
+pub fn digits(text: &str) -> String {
+    text.chars()
+        .filter(|c| c.is_ascii_digit() || *c == 'X' || *c == 'x')
+        .collect()
+}
+
+/// An ISBN announced in a piece of text, if it is the right length.
+pub fn from_text(text: &str) -> Option<String> {
+    let found = ANNOUNCED.captures(text)?;
+    let digits = digits(&found[1]);
+    matches!(digits.len(), 10 | 13).then_some(digits)
+}
+
+/// A checksum-valid ISBN embedded in a filename.
+///
+/// A file is often named exactly its ISBN and nothing else, which makes a
+/// free and certain identifier out of a record that otherwise has none.
+pub fn from_filename(name: &str) -> Option<String> {
+    let stem = EXTENSION.replace(name, "");
+    BARE.find_iter(&stem)
+        .map(|found| found.as_str())
+        .find(|candidate| valid(candidate))
+        .map(digits)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_real_isbn_checks_out() {
+        assert!(valid("9780198237907"));
+        assert!(valid("978-0-19-823790-7"));
+        assert!(valid("0198237901"));
+        assert!(valid("080213825X"));
+    }
+
+    #[test]
+    fn a_number_that_is_merely_the_right_length_does_not() {
+        assert!(!valid("9780198237908"));
+        assert!(!valid("0198237902"));
+        assert!(!valid("1234567890123"));
+        assert!(!valid(""));
+        assert!(!valid("not a number"));
+    }
+
+    #[test]
+    fn an_isbn_is_found_where_it_says_it_is() {
+        assert_eq!(
+            from_text("ISBN: 978-0-19-823790-7"),
+            Some("9780198237907".to_string())
+        );
+        assert_eq!(
+            from_text("isbn 0198237901 (paperback)"),
+            Some("0198237901".to_string())
+        );
+        assert_eq!(from_text("no identifier here"), None);
+    }
+
+    #[test]
+    fn a_filename_that_is_an_isbn_gives_it_up() {
+        assert_eq!(
+            from_filename("9780198237907.pdf"),
+            Some("9780198237907".to_string())
+        );
+        assert_eq!(
+            from_filename("Aquinas - 9780198237907 (scan).epub"),
+            Some("9780198237907".to_string())
+        );
+    }
+
+    #[test]
+    fn a_filename_of_digits_that_is_not_an_isbn_gives_nothing() {
+        assert_eq!(from_filename("20231115123456.pdf"), None);
+        assert_eq!(from_filename("chapter one.pdf"), None);
+    }
+
+    #[test]
+    fn the_extension_is_not_searched() {
+        // Otherwise a name could borrow digits from its own suffix.
+        assert_eq!(from_filename("book.9780198237907"), None);
+    }
+}

crates/bibclean/src/lib.rs

@@ -0,0 +1,234 @@
+//! Bibliographic hygiene over catalogue records.
+//!
+//! A library's metadata rots in recognisable ways: a title that is really a
+//! filename, an author field holding a publisher, a scanner's name where the
+//! work's name should be, capitals shouted by a converter. None of it stops
+//! anything working, and all of it makes a catalogue harder to search than
+//! the shelf it describes.
+//!
+//! Nothing here is rewritten silently. Every change comes back with the
+//! reason it was made, so a person can read a run before agreeing to it —
+//! which is the difference between a tool that cleans a catalogue and one
+//! that damages it in a new way.
+
+pub mod fix;
+pub mod isbn;
+pub mod matching;
+pub mod reasons;
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+
+/// A catalogue record, as it arrives.
+///
+/// Anything this does not know about is carried through untouched, so a
+/// record can be piped through and back into whatever it came from.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Book {
+    /// Record version.
+    #[serde(default = "version")]
+    pub v: u32,
+    /// The work's title.
+    #[serde(default)]
+    pub title: Option<String>,
+    /// Its author, as one field, however many there are.
+    #[serde(default)]
+    pub author: Option<String>,
+    /// Its ISBN, if the catalogue has one.
+    #[serde(default)]
+    pub isbn: Option<String>,
+    /// Where the file is, which is often where a missing ISBN is hiding.
+    #[serde(default)]
+    pub path: Option<String>,
+    /// Everything else the record carried.
+    #[serde(flatten)]
+    #[schemars(with = "Map<String, Value>")]
+    pub extra: Map<String, Value>,
+}
+
+fn version() -> u32 {
+    corpus_core::V
+}
+
+impl corpus_core::Record for Book {
+    fn version(&self) -> u32 {
+        self.v
+    }
+}
+
+/// One change, and why it was made.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Change {
+    /// Which field changed.
+    pub field: String,
+    /// What it was.
+    pub from: String,
+    /// What it became.
+    pub to: String,
+    /// The fault that prompted it.
+    pub why: String,
+}
+
+/// A record, cleaned, with everything that was found and done to it.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Cleaned {
+    /// The record, with the changes applied.
+    #[serde(flatten)]
+    pub book: Book,
+    /// What is wrong with it, named.
+    pub reasons: Vec<String>,
+    /// How much attention it deserves, worst first.
+    pub severity: u32,
+    /// What was changed, and why.
+    pub changes: Vec<Change>,
+}
+
+/// Audit a record and make the repairs that need no lookup.
+pub fn clean(book: &Book) -> Cleaned {
+    let mut cleaned = book.clone();
+    let mut changes = Vec::new();
+
+    let reasons = reasons::problems(
+        book.title.as_deref(),
+        book.author.as_deref(),
+        book.isbn.as_deref(),
+    );
+
+    // Underscores first: a title mangled by a filename may also be shouting,
+    // and title-casing it before the underscores come out would case the
+    // format suffix as though it were a word.
+    if let Some(title) = cleaned.title.clone()
+        && reasons.iter().any(|reason| reason == "underscore_title")
+    {
+        let fixed = fix::clean_underscores(&title);
+        if fixed != title {
+            changes.push(Change {
+                field: "title".into(),
+                from: title,
+                to: fixed.clone(),
+                why: "underscore_title".into(),
+            });
+            cleaned.title = Some(fixed);
+        }
+    }
+
+    if let Some(title) = cleaned.title.clone()
+        && fix::is_shouting(&title)
+    {
+        let fixed = fix::title_case(&title);
+        if fixed != title {
+            changes.push(Change {
+                field: "title".into(),
+                from: title,
+                to: fixed.clone(),
+                why: "allcaps_title".into(),
+            });
+            cleaned.title = Some(fixed);
+        }
+    }
+
+    // A missing ISBN is often sitting in the filename, where it is free and
+    // certain — the checksum is what makes it certain.
+    if reasons.iter().any(|reason| reason == "missing_isbn")
+        && let Some(path) = book.path.as_deref()
+        && let Some(found) = isbn::from_filename(path)
+    {
+        changes.push(Change {
+            field: "isbn".into(),
+            from: book.isbn.clone().unwrap_or_default(),
+            to: found.clone(),
+            why: "missing_isbn".into(),
+        });
+        cleaned.isbn = Some(found);
+    }
+
+    Cleaned {
+        severity: reasons::severity(&reasons),
+        reasons,
+        changes,
+        book: cleaned,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn book(title: &str, author: &str) -> Book {
+        Book {
+            v: corpus_core::V,
+            title: Some(title.into()),
+            author: Some(author.into()),
+            isbn: None,
+            path: None,
+            extra: Map::new(),
+        }
+    }
+
+    #[test]
+    fn a_good_record_comes_back_unchanged() {
+        let book = Book {
+            isbn: Some("9780198237907".into()),
+            ..book("The Human Condition", "Arendt, Hannah")
+        };
+        let cleaned = clean(&book);
+        assert_eq!(cleaned.book, book);
+        assert!(cleaned.reasons.is_empty());
+        assert!(cleaned.changes.is_empty());
+        assert_eq!(cleaned.severity, 0);
+    }
+
+    #[test]
+    fn a_shouting_title_is_cased_and_the_change_is_recorded() {
+        let cleaned = clean(&book("THE HUMAN CONDITION", "Arendt, Hannah"));
+        assert_eq!(cleaned.book.title.as_deref(), Some("The Human Condition"));
+        assert_eq!(cleaned.changes.len(), 1);
+        assert_eq!(cleaned.changes[0].field, "title");
+        assert_eq!(cleaned.changes[0].why, "allcaps_title");
+        assert_eq!(cleaned.changes[0].from, "THE HUMAN CONDITION");
+    }
+
+    #[test]
+    fn an_underscored_title_is_cleaned_before_it_is_cased() {
+        let cleaned = clean(&book("THE FALL_ AND THE RISE_txt", "Arendt, Hannah"));
+        assert_eq!(
+            cleaned.book.title.as_deref(),
+            Some("The Fall: and the Rise")
+        );
+        assert_eq!(cleaned.changes.len(), 2);
+    }
+
+    #[test]
+    fn an_isbn_in_the_filename_fills_a_missing_one() {
+        let mut book = book("The Human Condition", "Arendt, Hannah");
+        book.path = Some("/library/9780198237907.pdf".into());
+        let cleaned = clean(&book);
+        assert_eq!(cleaned.book.isbn.as_deref(), Some("9780198237907"));
+        assert_eq!(cleaned.changes[0].why, "missing_isbn");
+    }
+
+    #[test]
+    fn a_filename_that_is_not_an_isbn_fills_nothing() {
+        let mut book = book("The Human Condition", "Arendt, Hannah");
+        book.path = Some("/library/20231115123456.pdf".into());
+        assert_eq!(clean(&book).book.isbn, None);
+    }
+
+    #[test]
+    fn fields_it_does_not_know_about_are_carried_through() {
+        let mut book = book("A Book", "An Author");
+        book.extra.insert("shelf".into(), Value::from("upstairs"));
+        let cleaned = clean(&book);
+        assert_eq!(cleaned.book.extra["shelf"], "upstairs");
+    }
+
+    #[test]
+    fn the_output_carries_the_record_and_the_audit_on_one_line() {
+        let cleaned = clean(&book("THE HUMAN CONDITION", "Arendt, Hannah"));
+        let json: Value = serde_json::to_value(&cleaned).expect("a record serialises");
+        assert_eq!(json["title"], "The Human Condition");
+        assert!(json["reasons"].as_array().is_some());
+        assert!(json["changes"].as_array().is_some());
+    }
+}

crates/bibclean/src/main.rs

@@ -0,0 +1,77 @@
+//! Bibliographic hygiene over catalogue records.
+
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use bibclean::Book;
+use clap::Parser;
+use corpus_core::{exit, ndjson};
+
+/// Audit catalogue records and repair what can be repaired without a lookup.
+///
+/// Reads one JSON record per line — `title`, `author`, `isbn` and `path` are
+/// read, everything else is carried through untouched — and writes each one
+/// back with the faults it has, how much they weigh, and a line for every
+/// change made and why.
+///
+/// Nothing is rewritten silently. A run can be read before it is agreed to,
+/// which is the difference between cleaning a catalogue and damaging it in a
+/// new way.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Exit codes: 0 nothing needed attention, 1 something did, 65 a record could
+not be read.
+
+Example:
+
+    calibredb list --for-machine | jq -c '.[]' | bibclean --flagged"
+)]
+struct Args {
+    /// Record stream to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+
+    /// Write only the records that need attention. A missing ISBN alone does
+    /// not count as needing attention: nearly every book lacks one.
+    #[arg(long)]
+    flagged: bool,
+}
+
+fn main() -> ExitCode {
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
+
+    let text = match catena_cli::read_text(args.path.as_deref()) {
+        Ok(text) => text,
+        Err((code, why)) => {
+            eprintln!("bibclean: {why}");
+            return ExitCode::from(code);
+        }
+    };
+
+    let books: Vec<Book> = match ndjson::read(text.as_bytes()).collect() {
+        Ok(books) => books,
+        Err(e) => {
+            eprintln!("bibclean: {e}");
+            return ExitCode::from(exit::DATA);
+        }
+    };
+
+    let mut flagged = false;
+    let cleaned: Vec<bibclean::Cleaned> = books
+        .iter()
+        .map(bibclean::clean)
+        .inspect(|cleaned| flagged |= bibclean::reasons::worth_fixing(&cleaned.reasons))
+        .filter(|cleaned| !args.flagged || bibclean::reasons::worth_fixing(&cleaned.reasons))
+        .collect();
+
+    let wrote = catena_cli::stream("bibclean", &cleaned);
+    if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
+        return wrote;
+    }
+    ExitCode::from(if flagged { exit::NO } else { exit::OK })
+}

crates/bibclean/src/matching.rs

@@ -0,0 +1,98 @@
+//! Deciding whether two records name the same work.
+
+use std::collections::BTreeSet;
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// Words in a title, as the comparison sees them.
+static WORDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[a-z0-9]+").expect("valid"));
+
+/// How much of the shorter title's distinctive vocabulary must appear in the
+/// other before both are believed to name the same book.
+const THRESHOLD: f64 = 0.8;
+
+/// Words that carry no identifying weight.
+///
+/// Kept deliberately small: anything dropped here stops being able to tell
+/// two books apart.
+const STOPWORDS: &[&str] = &[
+    "a", "an", "and", "as", "at", "by", "ed", "edition", "for", "from", "her", "his", "in", "its",
+    "new", "of", "on", "or", "part", "revised", "second", "the", "their", "third", "to", "vol",
+    "volume", "with",
+];
+
+/// The words of a title that identify it.
+pub fn tokens(title: &str) -> BTreeSet<String> {
+    WORDS
+        .find_iter(&title.to_lowercase())
+        .map(|found| found.as_str().to_string())
+        .filter(|word| word.chars().count() > 2)
+        .filter(|word| STOPWORDS.binary_search(&word.as_str()).is_err())
+        .collect()
+}
+
+/// Whether two titles name the same work.
+///
+/// Character similarity is not usable here, and a real library shows exactly
+/// why: "The Cambridge Companion to Aquinas" and "The Cambridge Companion to
+/// Hume" are eighty-six per cent the same string, because they differ in one
+/// short word out of five. Believing that attaches Hume's blurb and Hume's
+/// subjects to Aquinas — silently, and to a whole shelf of series titles at
+/// once.
+///
+/// So the comparison is of vocabulary rather than characters: nearly all of
+/// the shorter title's distinctive words have to appear in the longer one.
+/// A subtitle and a leading "The" are then free, while a swapped subject is
+/// fatal.
+pub fn same_work(ours: &str, theirs: &str) -> bool {
+    let ours = tokens(ours);
+    let theirs = tokens(theirs);
+    if ours.is_empty() || theirs.is_empty() {
+        return false;
+    }
+    let (shorter, longer) = if ours.len() <= theirs.len() {
+        (&ours, &theirs)
+    } else {
+        (&theirs, &ours)
+    };
+    shorter.intersection(longer).count() as f64 / shorter.len() as f64 >= THRESHOLD
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_stopword_list_is_sorted_so_it_can_be_searched() {
+        assert!(STOPWORDS.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
+    #[test]
+    fn a_subtitle_does_not_make_it_a_different_book() {
+        assert!(same_work(
+            "Arguing About War",
+            "Arguing About War: A Study in Ethics"
+        ));
+        assert!(same_work("The Human Condition", "Human Condition"));
+    }
+
+    #[test]
+    fn one_swapped_word_makes_it_a_different_book() {
+        assert!(!same_work(
+            "The Cambridge Companion to Aquinas",
+            "The Cambridge Companion to Hume"
+        ));
+    }
+
+    #[test]
+    fn a_title_with_nothing_distinctive_in_it_matches_nothing() {
+        assert!(!same_work("The Of And", "The Of And"));
+        assert!(!same_work("", "The Human Condition"));
+    }
+
+    #[test]
+    fn case_and_punctuation_do_not_matter() {
+        assert!(same_work("THE HUMAN CONDITION!", "the human condition"));
+    }
+}

crates/bibclean/src/reasons.rs

@@ -0,0 +1,515 @@
+//! What is wrong with a record, and how badly.
+
+use std::sync::LazyLock;
+
+use fancy_regex::Regex as Boundaried;
+use regex::Regex;
+use textkit::pattern::{SPACE, WORD_END, WORD_START};
+
+fn pattern(source: &str) -> Regex {
+    Regex::new(source).expect("a valid pattern")
+}
+
+fn bounded(source: &str) -> Boundaried {
+    Boundaried::new(source).expect("a valid pattern")
+}
+
+/// A bare filename sitting in the title field.
+static FILENAME_TITLE: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(&format!(
+        r"(?i)\.(png|jpe?g|pdf|epub|mobi|azw3|azw|txt|wpd|docx?|rtf|tex|qxd|indd|pm\d){SPACE}*$"
+    ))
+});
+
+/// A title that names the tool that made the file rather than the work.
+///
+/// Authoring and desktop-publishing programs stamp their scratch filename as
+/// the PDF title; scanners and converters name themselves. The last of these
+/// is a Turkish locale's "full page fax print" — the same driver, localised.
+static JUNK_TITLE: LazyLock<Boundaried> = LazyLock::new(|| {
+    bounded(&format!(
+        r"(?i)microsoft word|z-?lib|libgen|\(z-lib|annas?-?archive|{WORD_START}ocr{WORD_END}|nota bene|pagemaker|quarkxpress|adobe indesign|acrobat distiller|word ?pro|wordperfect|{WORD_START}dvips{WORD_END}|{WORD_START}ltx{WORD_END}|{WORD_START}pdftex{WORD_END}|djvu document|untitled document|full page fax print|tam sayfa faks"
+    ))
+});
+
+/// A path sitting in the title field. Distinct from a filename, which is a
+/// bare name ending in an extension.
+static PATH_TITLE: LazyLock<Regex> =
+    LazyLock::new(|| pattern(r"(^[A-Za-z]:[\\/])|(\\\\[^\\]+\\)|(^/(?:home|users|mnt|tmp|var)/)"));
+
+/// A title that is a placeholder rather than a name.
+///
+/// Anchored and whole-string: a real book called "The Document" or "Test"
+/// must not be swept up, so these only match when the entire field is the
+/// placeholder.
+static PLACEHOLDER_TITLE: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(
+        r"(?i)^\W*(?:some[\s_-]*title|no[\s_-]*title|untitled|unnamed|title|document|documento|test|sample|default|placeholder|tbd|todo|draft|temp|new[\s_-]*document|book\d*|file\d*|scan\d*|doc\d*|n/?a|null|none|x{3,}|unknown)\W*$",
+    )
+});
+
+static PLACEHOLDER_AUTHOR: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(
+        r"(?i)^\W*(?:some[\s_-]*author|no[\s_-]*author|unnamed|author|default|test|sample|placeholder|tbd|todo|n/?a|null|none|x{3,}|administrator|user|owner|admin)\W*$",
+    )
+});
+
+/// An author field that opens with a bare three or four digit number is a
+/// publication year or a series index in the wrong column, never a person.
+/// Anchored, so a real name containing a number is untouched.
+static NUMERIC_AUTHOR: LazyLock<Regex> =
+    LazyLock::new(|| pattern(&format!(r"^{SPACE}*\d{{3,4}}{SPACE}*(?:$|[{SPACE}.\-–—])")));
+
+/// A possessive apostrophe that a filename or an OCR pass stripped:
+/// "Lossky s Argument".
+static MISSING_APOSTROPHE: LazyLock<Boundaried> =
+    LazyLock::new(|| bounded(&format!(r"{WORD_START}[A-Za-z]{{2,}} s [A-Z]")));
+
+/// Punctuation that a title has and a name does not.
+static TITLE_PUNCTUATION: LazyLock<Regex> = LazyLock::new(|| pattern(r"[!?:]"));
+
+/// A surname-first comma or a run of initials.
+static INITIALS: LazyLock<Boundaried> =
+    LazyLock::new(|| bounded(&format!(r"{WORD_START}[A-Z]\.{SPACE}*[A-Z]?\.?")));
+
+/// Function words that essentially never appear in a personal name and are
+/// everywhere in titles.
+static TITLE_WORDS: LazyLock<Boundaried> = LazyLock::new(|| {
+    bounded(&format!(
+        r"(?i){WORD_START}(of|the|and|in|on|for|to|from|with|a|an|about|between|through|introduction|dictionary|handbook|companion|history|study|essays|conception|objections|architecture|philosophy|theory){WORD_END}"
+    ))
+});
+
+/// How authors are joined when several share a book.
+static AUTHOR_SEPARATOR: LazyLock<Regex> =
+    LazyLock::new(|| pattern(&format!(r"{SPACE}+&{SPACE}+|{SPACE}*;{SPACE}*")));
+
+/// Publisher and imprint names that landed in the author field.
+///
+/// Matched whole — "Oxford University Press" as an author is wrong, but
+/// "Oxford, Michael" is a person. Kept lowercase and compared against the
+/// normalised field.
+const PUBLISHER_AUTHORS: &[&str] = &[
+    "baker academic",
+    "blackwell",
+    "bloomsbury",
+    "brill",
+    "cambridge",
+    "cambridge university press",
+    "continuum",
+    "cornell university press",
+    "crossway",
+    "de gruyter",
+    "eerdmans",
+    "elsevier",
+    "fortress press",
+    "harpercollins",
+    "harvard university press",
+    "ignatius press",
+    "intervarsity press",
+    "macmillan",
+    "mit press",
+    "oxford",
+    "oxford university press",
+    "palgrave",
+    "palgrave macmillan",
+    "penguin",
+    "penguin books",
+    "princeton university press",
+    "random house",
+    "routledge",
+    "sage",
+    "simon schuster",
+    "springer",
+    "state university of new york press",
+    "suny",
+    "suny press",
+    "t t clark",
+    "university of chicago press",
+    "wiley blackwell",
+    "wipf stock",
+    "yale university press",
+    "zondervan",
+];
+
+/// Author fields that say there is no author.
+const UNKNOWN_AUTHORS: &[&str] = &["", "anonymous", "n/a", "unknown", "unknown author"];
+
+/// Word endings that mark a personal name.
+const PERSON_SUFFIXES: &[&str] = &["ed", "eds", "ii", "iii", "iv", "jr", "md", "phd", "sr"];
+
+/// What each fault is worth when the worst records are wanted first.
+const WEIGHTS: &[(&str, u32)] = &[
+    ("allcaps_title", 2),
+    ("author_last_first", 1),
+    ("author_role_suffix", 2),
+    ("edited_volume", 1),
+    ("filename_title", 3),
+    ("junk_title", 3),
+    ("lowercase_title", 2),
+    ("missing_apostrophe", 2),
+    ("numeric_author", 4),
+    ("path_title", 4),
+    ("placeholder_author", 4),
+    ("placeholder_title", 4),
+    ("possible_swap", 2),
+    ("publisher_as_author", 3),
+    ("sentence_case_title", 1),
+    ("title_in_author", 3),
+    ("underscore_title", 1),
+    ("unknown_author", 3),
+];
+
+/// Everything wrong with a record, named.
+///
+/// `missing_isbn` is reported but is not on its own a reason to do anything:
+/// nearly every book lacks one.
+pub fn problems(title: Option<&str>, author: Option<&str>, isbn: Option<&str>) -> Vec<String> {
+    let title = title.unwrap_or("").trim_matches(textkit::is_space);
+    let author = author.unwrap_or("").trim_matches(textkit::is_space);
+    let mut reasons = Vec::new();
+
+    if UNKNOWN_AUTHORS
+        .binary_search(&author.to_lowercase().as_str())
+        .is_ok()
+    {
+        reasons.push("unknown_author");
+    }
+    if PLACEHOLDER_TITLE.is_match(title) {
+        reasons.push("placeholder_title");
+    }
+    if PLACEHOLDER_AUTHOR.is_match(author) {
+        reasons.push("placeholder_author");
+    }
+    if PATH_TITLE.is_match(title) {
+        reasons.push("path_title");
+    }
+    if FILENAME_TITLE.is_match(title) {
+        reasons.push("filename_title");
+    }
+    if JUNK_TITLE.is_match(title).unwrap_or(false) {
+        reasons.push("junk_title");
+    }
+    if is_publisher(author) {
+        reasons.push("publisher_as_author");
+    }
+    if NUMERIC_AUTHOR
+        .find(author)
+        .is_some_and(|at| at.start() == 0)
+    {
+        reasons.push("numeric_author");
+    }
+    if title.contains('_') {
+        reasons.push("underscore_title");
+    }
+    if !title.is_empty() && title == title.to_uppercase() && title.contains(' ') {
+        reasons.push("allcaps_title");
+    }
+    // A filename stub: "heidegger areopagite". Distinct from sentence case,
+    // which is a much weaker signal.
+    if !title.is_empty()
+        && title == title.to_lowercase()
+        && title.chars().any(|c| c.is_lowercase())
+        && (title.contains(' ') || title.chars().count() >= 12)
+    {
+        reasons.push("lowercase_title");
+    }
+    if MISSING_APOSTROPHE.is_match(title).unwrap_or(false) {
+        reasons.push("missing_apostrophe");
+    }
+    // Punctuation in the author field, or a long author against a short
+    // title. Gated on the author not looking like a person, because several
+    // authors joined together are many words against what may be a one-word
+    // title, and that is not a swap.
+    if !looks_like_person(author)
+        && (TITLE_PUNCTUATION.is_match(author)
+            || (author.split_whitespace().count() >= 5
+                && (1..=3).contains(&title.split_whitespace().count())))
+    {
+        reasons.push("possible_swap");
+    }
+    if title_in_author(Some(title), Some(author)) {
+        reasons.push("title_in_author");
+    }
+    if isbn
+        .unwrap_or("")
+        .trim_matches(textkit::is_space)
+        .is_empty()
+    {
+        reasons.push("missing_isbn");
+    }
+
+    reasons.into_iter().map(str::to_string).collect()
+}
+
+/// How much attention a set of faults deserves.
+pub fn severity(reasons: &[String]) -> u32 {
+    reasons
+        .iter()
+        .filter_map(|reason| {
+            WEIGHTS
+                .binary_search_by_key(&reason.as_str(), |(name, _)| name)
+                .ok()
+                .map(|at| WEIGHTS[at].1)
+        })
+        .sum()
+}
+
+/// Whether anything here needs doing. A missing ISBN alone does not.
+pub fn worth_fixing(reasons: &[String]) -> bool {
+    reasons.iter().any(|reason| reason != "missing_isbn")
+}
+
+/// Whether the title field is a placeholder and the author field is holding
+/// the real title.
+///
+/// Deliberately conservative: it must be both a placeholder title and an
+/// author that does not read as a person, because what this drives — looking
+/// the book up by the author text as a title — is wasted work otherwise.
+pub fn title_in_author(title: Option<&str>, author: Option<&str>) -> bool {
+    let title = title.unwrap_or("").trim_matches(textkit::is_space);
+    let author = author.unwrap_or("").trim_matches(textkit::is_space);
+
+    if author.is_empty() || !(PLACEHOLDER_TITLE.is_match(title) || PATH_TITLE.is_match(title)) {
+        return false;
+    }
+    // A placeholder author is not a stray title. Treating it as one sends a
+    // junk string to a catalogue as a title query, which is the spurious
+    // match every other part of this guards against.
+    if PLACEHOLDER_AUTHOR.is_match(author) {
+        return false;
+    }
+    if is_publisher(author)
+        || UNKNOWN_AUTHORS
+            .binary_search(&author.to_lowercase().as_str())
+            .is_ok()
+    {
+        return false;
+    }
+    !looks_like_person(author)
+}
+
+/// Whether a field reads as a human name.
+///
+/// Multi-author strings are judged one author at a time. A catalogue joins
+/// authors with an ampersand, so two people arrive as one seven-word field —
+/// judged whole it fails every test and reads as a title, which is how a
+/// naive version of this flagged hundreds of perfectly good records.
+pub fn looks_like_person(author: &str) -> bool {
+    let author = author.trim_matches(textkit::is_space);
+    if author.is_empty() {
+        return false;
+    }
+
+    let parts: Vec<&str> = AUTHOR_SEPARATOR
+        .split(author)
+        .map(|part| part.trim_matches(textkit::is_space))
+        .filter(|part| !part.is_empty())
+        .collect();
+    if parts.len() > 1 {
+        return parts.iter().all(|part| looks_like_person(part));
+    }
+
+    let words: Vec<&str> = author.split_whitespace().collect();
+    if author.contains(',') && words.len() <= 5 {
+        return true; // "Surname, Given"
+    }
+    if INITIALS.is_match(author).unwrap_or(false) && words.len() <= 5 {
+        return true;
+    }
+    if words.len() > 5 {
+        return false;
+    }
+    if TITLE_WORDS.is_match(author).unwrap_or(false) {
+        return false;
+    }
+    if words
+        .iter()
+        .any(|word| PERSON_SUFFIXES.contains(&word.to_lowercase().trim_matches('.')))
+    {
+        return true;
+    }
+    // A bare single word is a mononym far more often than a stray title —
+    // Plato, Homer and Josephus are all over a philosophy library, and a
+    // one-word "title" recovered from an author field is nearly never right.
+    (1..=4).contains(&words.len())
+}
+
+/// A catalogue string reduced to the letters and digits in it.
+pub fn normalise(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    let mut space = false;
+    for c in text.to_lowercase().chars() {
+        if c.is_ascii_alphanumeric() {
+            if space && !out.is_empty() {
+                out.push(' ');
+            }
+            space = false;
+            out.push(c);
+        } else {
+            space = true;
+        }
+    }
+    out
+}
+
+fn is_publisher(author: &str) -> bool {
+    PUBLISHER_AUTHORS
+        .binary_search(&normalise(author).as_str())
+        .is_ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn problems_of(title: &str, author: &str) -> Vec<String> {
+        problems(Some(title), Some(author), Some("9780198237907"))
+    }
+
+    #[test]
+    fn the_tables_are_sorted_so_they_can_be_searched() {
+        assert!(PUBLISHER_AUTHORS.windows(2).all(|pair| pair[0] < pair[1]));
+        assert!(UNKNOWN_AUTHORS.windows(2).all(|pair| pair[0] < pair[1]));
+        assert!(PERSON_SUFFIXES.windows(2).all(|pair| pair[0] < pair[1]));
+        assert!(WEIGHTS.windows(2).all(|pair| pair[0].0 < pair[1].0));
+    }
+
+    #[test]
+    fn a_good_record_has_nothing_wrong_with_it() {
+        assert!(problems_of("The Human Condition", "Arendt, Hannah").is_empty());
+    }
+
+    #[test]
+    fn a_missing_isbn_is_reported_but_is_not_worth_fixing() {
+        let reasons = problems(Some("The Human Condition"), Some("Arendt, Hannah"), None);
+        assert_eq!(reasons, ["missing_isbn"]);
+        assert!(!worth_fixing(&reasons));
+    }
+
+    #[test]
+    fn a_placeholder_title_is_found() {
+        for title in ["Untitled", "untitled", "no title", "Document", "xxx", "N/A"] {
+            assert!(
+                problems_of(title, "Arendt, Hannah").contains(&"placeholder_title".to_string()),
+                "{title}"
+            );
+        }
+    }
+
+    #[test]
+    fn a_real_title_that_resembles_a_placeholder_is_left_alone() {
+        for title in ["The Document", "A Test of Faith", "Untitled Symphony"] {
+            assert!(
+                !problems_of(title, "Arendt, Hannah").contains(&"placeholder_title".to_string()),
+                "{title}"
+            );
+        }
+    }
+
+    #[test]
+    fn a_filename_and_a_path_in_the_title_are_told_apart() {
+        assert!(problems_of("chapter1.pdf", "x").contains(&"filename_title".to_string()));
+        assert!(problems_of("A:\\pwecent2005.wpd", "x").contains(&"path_title".to_string()));
+        assert!(problems_of("/home/matt/book", "x").contains(&"path_title".to_string()));
+    }
+
+    #[test]
+    fn a_tool_name_in_the_title_is_junk() {
+        for title in [
+            "Microsoft Word - chapter3",
+            "Book (z-lib.org)",
+            "DjVu Document",
+            "Full Page Fax Print",
+            "scan OCR output",
+        ] {
+            assert!(
+                problems_of(title, "x").contains(&"junk_title".to_string()),
+                "{title}"
+            );
+        }
+    }
+
+    #[test]
+    fn a_word_that_merely_contains_a_tool_name_is_not_junk() {
+        assert!(!problems_of("Procrastination", "x").contains(&"junk_title".to_string()));
+    }
+
+    #[test]
+    fn a_publisher_in_the_author_field_is_found() {
+        assert!(
+            problems_of("A Book", "Oxford University Press")
+                .contains(&"publisher_as_author".to_string())
+        );
+        assert!(
+            !problems_of("A Book", "Oxford, Michael").contains(&"publisher_as_author".to_string())
+        );
+    }
+
+    #[test]
+    fn shouting_and_whispering_are_both_noticed() {
+        assert!(problems_of("THE HUMAN CONDITION", "x").contains(&"allcaps_title".to_string()));
+        assert!(problems_of("heidegger areopagite", "x").contains(&"lowercase_title".to_string()));
+        assert!(!problems_of("MIT", "x").contains(&"allcaps_title".to_string()));
+    }
+
+    #[test]
+    fn several_authors_are_not_a_swapped_title() {
+        let reasons = problems_of("Rama", "Arthur C. Clarke & Gentry Lee");
+        assert!(
+            !reasons.contains(&"possible_swap".to_string()),
+            "{reasons:?}"
+        );
+    }
+
+    #[test]
+    fn a_title_in_the_author_field_is_found() {
+        assert!(title_in_author(Some("Untitled"), Some("Arguing About War")));
+        assert!(!title_in_author(Some("Untitled"), Some("Arendt, Hannah")));
+        assert!(!title_in_author(
+            Some("The Human Condition"),
+            Some("Arguing About War")
+        ));
+        assert!(!title_in_author(Some("Untitled"), Some("Default")));
+        assert!(!title_in_author(Some("Untitled"), Some("")));
+    }
+
+    #[test]
+    fn people_and_titles_are_told_apart() {
+        for person in [
+            "Arendt, Hannah",
+            "C. S. Lewis",
+            "Plato",
+            "Hannah Arendt",
+            "C. S. Lewis & Madeleine L'Engle",
+            "Smith Jr",
+        ] {
+            assert!(looks_like_person(person), "{person}");
+        }
+        for title in [
+            "An Ethnohistorical Dictionary China",
+            "Kierkegaard's Conception of God",
+            "The Oxford Handbook of Something Or Other",
+        ] {
+            assert!(!looks_like_person(title), "{title}");
+        }
+    }
+
+    #[test]
+    fn severity_ranks_the_worst_first() {
+        let bad = problems_of("Untitled", "Arguing About War");
+        let mild = problems_of("Some_Book", "Arendt, Hannah");
+        assert!(severity(&bad) > severity(&mild), "{bad:?} vs {mild:?}");
+        assert_eq!(severity(&[]), 0);
+    }
+
+    #[test]
+    fn normalising_keeps_only_what_identifies() {
+        assert_eq!(
+            normalise("Oxford University Press!"),
+            "oxford university press"
+        );
+        assert_eq!(normalise("  T&T  Clark "), "t t clark");
+        assert_eq!(normalise(""), "");
+    }
+}

parity/Cargo.toml

@@ -14,6 +14,7 @@ publish = false
 python = []
 
 [dependencies]
+bibclean.workspace = true
 booksniff.workspace = true
 chunkmd.workspace = true
 cuekit.workspace = true

parity/cases/bibclean/records.ndjson

@@ -0,0 +1,33 @@
+{"title": "The Human Condition", "author": "Arendt, Hannah", "isbn": "9780226586601"}
+{"title": "THE HUMAN CONDITION", "author": "Arendt, Hannah", "isbn": null}
+{"title": "Untitled", "author": "Arguing About War", "isbn": null}
+{"title": "Untitled", "author": "Default", "isbn": null}
+{"title": "chapter1.pdf", "author": "Unknown", "isbn": null}
+{"title": "A:\\pwecent2005.wpd", "author": "", "isbn": null}
+{"title": "/home/matt/book", "author": "1993", "isbn": null}
+{"title": "Microsoft Word - ch3.doc", "author": "Oxford University Press", "isbn": null}
+{"title": "Board_Advisors_etc", "author": "C. S. Lewis & Madeleine L'Engle", "isbn": null}
+{"title": "The Fall_ And the Rise_txt", "author": "Plato", "isbn": null}
+{"title": "heidegger areopagite", "author": "Smith Jr", "isbn": null}
+{"title": "Lossky s Argument", "author": "Kierkegaard's Conception of God", "isbn": null}
+{"title": "Rama", "author": "Arthur C. Clarke & Gentry Lee", "isbn": null}
+{"title": "MIT", "author": "An Ethnohistorical Dictionary China", "isbn": null}
+{"title": "ÊTRE ET TEMPS", "author": "Heidegger, Martin", "isbn": null}
+{"title": "ÉTUDES SUR LE MOYEN ÂGE", "author": "Gilson, Étienne", "isbn": null}
+{"title": "DON'T LOOK NOW", "author": "du Maurier, Daphne", "isbn": null}
+{"title": "THE WELL-BUILT WHEEL", "author": "", "isbn": null}
+{"title": "The (Second) Estate", "author": "Bloch, Marc", "isbn": null}
+{"title": null, "author": null, "isbn": null}
+{"title": "", "author": "", "isbn": ""}
+{"title": "  spaced  ", "author": "  Arendt, Hannah  ", "isbn": "978-0-226-58660-1"}
+{"title": "A Book", "author": "anonymous", "isbn": "080213825X"}
+{"title": "Scan 001", "author": "administrator", "isbn": "1234567890123"}
+{"title": "xxx", "author": "xxxx", "isbn": null}
+{"title": "The Document", "author": "Oxford, Michael", "isbn": null}
+{"title": "Full Page Fax Print", "author": "user", "isbn": null}
+{"title": "Tam Sayfa Faks Yazdırma", "author": "Routledge", "isbn": null}
+{"title": "A Book", "author": "Arendt", "isbn": null, "path": "/library/9780198237907.pdf"}
+{"title": "A Book", "author": "Arendt", "isbn": null, "path": "/library/20231115123456.pdf"}
+{"title": "A Book", "author": "Arendt", "isbn": null, "path": "Aquinas - 080213825X.epub"}
+{"title": "Ἀθῆναι καὶ Μόσχα", "author": "Θουκυδίδης", "isbn": null}
+{"title": "東京の手紙", "author": "夏目漱石", "isbn": null}

parity/golden/bibclean/records.ndjson

@@ -0,0 +1,33 @@
+{"clean_underscores": "The Human Condition", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": [], "severity": 0, "title_case": "The Human Condition", "title_in_author": false, "valid_isbn": true}
+{"clean_underscores": "THE HUMAN CONDITION", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "The Human Condition", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Untitled", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "arguing about war", "reasons": ["placeholder_title", "title_in_author", "missing_isbn"], "severity": 7, "title_case": "Untitled", "title_in_author": true, "valid_isbn": false}
+{"clean_underscores": "Untitled", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "default", "reasons": ["placeholder_title", "placeholder_author", "missing_isbn"], "severity": 8, "title_case": "Untitled", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "chapter1.", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "unknown", "reasons": ["unknown_author", "filename_title", "lowercase_title", "missing_isbn"], "severity": 8, "title_case": "Chapter1.pdf", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A:\\pwecent2005.wpd", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "path_title", "filename_title", "missing_isbn"], "severity": 10, "title_case": "A:\\pwecent2005.wpd", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "/home/matt/book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "1993", "reasons": ["path_title", "numeric_author", "lowercase_title", "missing_isbn"], "severity": 10, "title_case": "/Home/matt/book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Microsoft Word - ch3.doc", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "oxford university press", "reasons": ["filename_title", "junk_title", "publisher_as_author", "missing_isbn"], "severity": 9, "title_case": "Microsoft Word - Ch3.doc", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Board Advisors etc", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "c s lewis madeleine l engle", "reasons": ["underscore_title", "missing_isbn"], "severity": 1, "title_case": "Board_advisors_etc", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "The Fall: And the Rise", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "plato", "reasons": ["underscore_title", "missing_isbn"], "severity": 1, "title_case": "The Fall_ and the Rise_txt", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "heidegger areopagite", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "smith jr", "reasons": ["lowercase_title", "missing_isbn"], "severity": 2, "title_case": "Heidegger Areopagite", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Lossky s Argument", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "kierkegaard s conception of god", "reasons": ["missing_apostrophe", "missing_isbn"], "severity": 2, "title_case": "Lossky S Argument", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Rama", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arthur c clarke gentry lee", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Rama", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "MIT", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "an ethnohistorical dictionary china", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Mit", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "ÊTRE ET TEMPS", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "heidegger martin", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Être Et Temps", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "ÉTUDES SUR LE MOYEN ÂGE", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "gilson tienne", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Études Sur Le Moyen Âge", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "DON'T LOOK NOW", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "du maurier daphne", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Don't Look Now", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "THE WELL-BUILT WHEEL", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "allcaps_title", "missing_isbn"], "severity": 5, "title_case": "The Well-Built Wheel", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "The (Second) Estate", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "bloch marc", "reasons": ["missing_isbn"], "severity": 0, "title_case": "The (Second) Estate", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": null, "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "missing_isbn"], "severity": 3, "title_case": null, "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "missing_isbn"], "severity": 3, "title_case": "", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "spaced", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": [], "severity": 0, "title_case": "Spaced", "title_in_author": false, "valid_isbn": true}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "anonymous", "reasons": ["unknown_author"], "severity": 3, "title_case": "A Book", "title_in_author": false, "valid_isbn": true}
+{"clean_underscores": "Scan 001", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "administrator", "reasons": ["placeholder_author"], "severity": 4, "title_case": "Scan 001", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "xxx", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "xxxx", "reasons": ["placeholder_title", "placeholder_author", "missing_isbn"], "severity": 8, "title_case": "Xxx", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "The Document", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "oxford michael", "reasons": ["missing_isbn"], "severity": 0, "title_case": "The Document", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Full Page Fax Print", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "user", "reasons": ["placeholder_author", "junk_title", "missing_isbn"], "severity": 7, "title_case": "Full Page Fax Print", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Tam Sayfa Faks Yazdırma", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "routledge", "reasons": ["junk_title", "publisher_as_author", "missing_isbn"], "severity": 6, "title_case": "Tam Sayfa Faks Yazdırma", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": "9780198237907", "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": "080213825X", "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Ἀθῆναι καὶ Μόσχα", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Ἀθῆναι Καὶ Μόσχα", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "東京の手紙", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "", "reasons": ["missing_isbn"], "severity": 0, "title_case": "東京の手紙", "title_in_author": false, "valid_isbn": false}

parity/oracle/bibclean.py

@@ -0,0 +1,48 @@
+"""Audit catalogue records the way the Python pipeline audits them.
+
+Reads one `{"title", "author", "isbn", "path"}` object per line and writes
+what each of the metadata functions makes of it.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+        python3 oracle/bibclean.py < cases/bibclean/records.ndjson
+"""
+
+import json
+import os
+import sys
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from book_pipeline import metadata  # noqa: E402
+
+
+def main() -> int:
+    out = sys.stdout
+    for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+        if not line.strip():
+            continue
+        book = json.loads(line)
+        title = book.get("title")
+        author = book.get("author")
+        isbn = book.get("isbn")
+        reasons = metadata.problem_reasons(title, author, isbn)
+        record = {
+            "reasons": reasons,
+            "severity": metadata.severity(reasons),
+            "title_in_author": metadata.title_in_author(title, author),
+            "looks_like_person": metadata._looks_like_person(author or ""),
+            "is_shouting": metadata.is_shouting(title),
+            "title_case": metadata.title_case(title) if title is not None else None,
+            "clean_underscores": metadata.clean_underscores(title) if title is not None else None,
+            "valid_isbn": metadata.valid_isbn(isbn),
+            "isbn_from_filename": metadata.isbn_from_filename(book.get("path")),
+            "normalise": metadata._norm(author),
+        }
+        out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 10] = [
+    let jobs: [Job; 11] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
@@ -30,6 +30,7 @@ fn main() -> ExitCode {
         ("rrf", rrf),
         ("cuepack", cuepack),
         ("booksniff", booksniff),
+        ("bibclean", bibclean),
     ];
 
     for (name, job) in jobs {
@@ -73,6 +74,18 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn bibclean() -> Written {
+    let dir = parity::golden("bibclean");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/bibclean/records.ndjson");
+    let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let judgements = parity::run_oracle("bibclean.py", &cases)?;
+    let path = dir.join("records.ndjson");
+    std::fs::write(&path, judgements).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn booksniff() -> Written {
     let dir = parity::golden("booksniff");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/bibclean.rs

@@ -0,0 +1,123 @@
+//! Every judgement about a catalogue record must match the Python's.
+//!
+//! These decide what a cleanup run touches. A reason that fires here and not
+//! there means a record is rewritten that should have been left alone, which
+//! is the one failure mode a metadata tool cannot recover from.
+
+use serde_json::Value;
+
+fn records(path: &str) -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> {
+    record.get(name).and_then(Value::as_str)
+}
+
+#[test]
+fn every_judgement_matches_python() {
+    let cases = records("cases/bibclean/records.ndjson");
+    let golden = records("golden/bibclean/records.ndjson");
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    let mut wrong: Vec<String> = Vec::new();
+    for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() {
+        let title = field(case, "title");
+        let author = field(case, "author");
+        let isbn = field(case, "isbn");
+
+        let mut differs = |what: &str, ours: String, theirs: &Value| {
+            wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}"));
+        };
+
+        let reasons = bibclean::reasons::problems(title, author, isbn);
+        let their_reasons: Vec<String> = theirs["reasons"]
+            .as_array()
+            .expect("a list")
+            .iter()
+            .map(|reason| reason.as_str().expect("a name").to_string())
+            .collect();
+        if reasons != their_reasons {
+            differs("reasons", format!("{reasons:?}"), &theirs["reasons"]);
+        }
+
+        let severity = bibclean::reasons::severity(&reasons);
+        if severity as u64 != theirs["severity"].as_u64().expect("a number") {
+            differs("severity", severity.to_string(), &theirs["severity"]);
+        }
+
+        let ours = bibclean::reasons::title_in_author(title, author);
+        if ours != theirs["title_in_author"].as_bool().expect("a verdict") {
+            differs(
+                "title_in_author",
+                ours.to_string(),
+                &theirs["title_in_author"],
+            );
+        }
+
+        let ours = bibclean::reasons::looks_like_person(author.unwrap_or(""));
+        if ours != theirs["looks_like_person"].as_bool().expect("a verdict") {
+            differs(
+                "looks_like_person",
+                ours.to_string(),
+                &theirs["looks_like_person"],
+            );
+        }
+
+        let ours = bibclean::fix::is_shouting(title.unwrap_or(""));
+        if ours != theirs["is_shouting"].as_bool().expect("a verdict") {
+            differs("is_shouting", ours.to_string(), &theirs["is_shouting"]);
+        }
+
+        if let Some(title) = title {
+            let ours = bibclean::fix::title_case(title);
+            if Some(ours.as_str()) != theirs["title_case"].as_str() {
+                differs("title_case", ours, &theirs["title_case"]);
+            }
+            let ours = bibclean::fix::clean_underscores(title);
+            if Some(ours.as_str()) != theirs["clean_underscores"].as_str() {
+                differs("clean_underscores", ours, &theirs["clean_underscores"]);
+            }
+        }
+
+        let ours = bibclean::isbn::valid(isbn.unwrap_or(""));
+        if ours != theirs["valid_isbn"].as_bool().expect("a verdict") {
+            differs("valid_isbn", ours.to_string(), &theirs["valid_isbn"]);
+        }
+
+        let ours = field(case, "path").and_then(bibclean::isbn::from_filename);
+        if ours.as_deref() != theirs["isbn_from_filename"].as_str() {
+            differs(
+                "isbn_from_filename",
+                format!("{ours:?}"),
+                &theirs["isbn_from_filename"],
+            );
+        }
+
+        let ours = bibclean::reasons::normalise(author.unwrap_or(""));
+        if Some(ours.as_str()) != theirs["normalise"].as_str() {
+            differs("normalise", ours, &theirs["normalise"]);
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} judgements differ:\n{}",
+        wrong.len(),
+        wrong.join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/bibclean/records.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("bibclean.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("bibclean/records.ndjson"));
+}