bibclean: record faults, weighed, with the repairs that need no lookup

parity/Cargo.toml

@@ -14,6 +14,7 @@ publish = false
 python = []
 
 [dependencies]
+bibclean.workspace = true
 booksniff.workspace = true
 chunkmd.workspace = true
 cuekit.workspace = true

parity/cases/bibclean/records.ndjson

@@ -0,0 +1,33 @@
+{"title": "The Human Condition", "author": "Arendt, Hannah", "isbn": "9780226586601"}
+{"title": "THE HUMAN CONDITION", "author": "Arendt, Hannah", "isbn": null}
+{"title": "Untitled", "author": "Arguing About War", "isbn": null}
+{"title": "Untitled", "author": "Default", "isbn": null}
+{"title": "chapter1.pdf", "author": "Unknown", "isbn": null}
+{"title": "A:\\pwecent2005.wpd", "author": "", "isbn": null}
+{"title": "/home/matt/book", "author": "1993", "isbn": null}
+{"title": "Microsoft Word - ch3.doc", "author": "Oxford University Press", "isbn": null}
+{"title": "Board_Advisors_etc", "author": "C. S. Lewis & Madeleine L'Engle", "isbn": null}
+{"title": "The Fall_ And the Rise_txt", "author": "Plato", "isbn": null}
+{"title": "heidegger areopagite", "author": "Smith Jr", "isbn": null}
+{"title": "Lossky s Argument", "author": "Kierkegaard's Conception of God", "isbn": null}
+{"title": "Rama", "author": "Arthur C. Clarke & Gentry Lee", "isbn": null}
+{"title": "MIT", "author": "An Ethnohistorical Dictionary China", "isbn": null}
+{"title": "ÊTRE ET TEMPS", "author": "Heidegger, Martin", "isbn": null}
+{"title": "ÉTUDES SUR LE MOYEN ÂGE", "author": "Gilson, Étienne", "isbn": null}
+{"title": "DON'T LOOK NOW", "author": "du Maurier, Daphne", "isbn": null}
+{"title": "THE WELL-BUILT WHEEL", "author": "", "isbn": null}
+{"title": "The (Second) Estate", "author": "Bloch, Marc", "isbn": null}
+{"title": null, "author": null, "isbn": null}
+{"title": "", "author": "", "isbn": ""}
+{"title": "  spaced  ", "author": "  Arendt, Hannah  ", "isbn": "978-0-226-58660-1"}
+{"title": "A Book", "author": "anonymous", "isbn": "080213825X"}
+{"title": "Scan 001", "author": "administrator", "isbn": "1234567890123"}
+{"title": "xxx", "author": "xxxx", "isbn": null}
+{"title": "The Document", "author": "Oxford, Michael", "isbn": null}
+{"title": "Full Page Fax Print", "author": "user", "isbn": null}
+{"title": "Tam Sayfa Faks Yazdırma", "author": "Routledge", "isbn": null}
+{"title": "A Book", "author": "Arendt", "isbn": null, "path": "/library/9780198237907.pdf"}
+{"title": "A Book", "author": "Arendt", "isbn": null, "path": "/library/20231115123456.pdf"}
+{"title": "A Book", "author": "Arendt", "isbn": null, "path": "Aquinas - 080213825X.epub"}
+{"title": "Ἀθῆναι καὶ Μόσχα", "author": "Θουκυδίδης", "isbn": null}
+{"title": "東京の手紙", "author": "夏目漱石", "isbn": null}

parity/golden/bibclean/records.ndjson

@@ -0,0 +1,33 @@
+{"clean_underscores": "The Human Condition", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": [], "severity": 0, "title_case": "The Human Condition", "title_in_author": false, "valid_isbn": true}
+{"clean_underscores": "THE HUMAN CONDITION", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "The Human Condition", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Untitled", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "arguing about war", "reasons": ["placeholder_title", "title_in_author", "missing_isbn"], "severity": 7, "title_case": "Untitled", "title_in_author": true, "valid_isbn": false}
+{"clean_underscores": "Untitled", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "default", "reasons": ["placeholder_title", "placeholder_author", "missing_isbn"], "severity": 8, "title_case": "Untitled", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "chapter1.", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "unknown", "reasons": ["unknown_author", "filename_title", "lowercase_title", "missing_isbn"], "severity": 8, "title_case": "Chapter1.pdf", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A:\\pwecent2005.wpd", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "path_title", "filename_title", "missing_isbn"], "severity": 10, "title_case": "A:\\pwecent2005.wpd", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "/home/matt/book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "1993", "reasons": ["path_title", "numeric_author", "lowercase_title", "missing_isbn"], "severity": 10, "title_case": "/Home/matt/book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Microsoft Word - ch3.doc", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "oxford university press", "reasons": ["filename_title", "junk_title", "publisher_as_author", "missing_isbn"], "severity": 9, "title_case": "Microsoft Word - Ch3.doc", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Board Advisors etc", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "c s lewis madeleine l engle", "reasons": ["underscore_title", "missing_isbn"], "severity": 1, "title_case": "Board_advisors_etc", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "The Fall: And the Rise", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "plato", "reasons": ["underscore_title", "missing_isbn"], "severity": 1, "title_case": "The Fall_ and the Rise_txt", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "heidegger areopagite", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "smith jr", "reasons": ["lowercase_title", "missing_isbn"], "severity": 2, "title_case": "Heidegger Areopagite", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Lossky s Argument", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "kierkegaard s conception of god", "reasons": ["missing_apostrophe", "missing_isbn"], "severity": 2, "title_case": "Lossky S Argument", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Rama", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arthur c clarke gentry lee", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Rama", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "MIT", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "an ethnohistorical dictionary china", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Mit", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "ÊTRE ET TEMPS", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "heidegger martin", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Être Et Temps", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "ÉTUDES SUR LE MOYEN ÂGE", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "gilson tienne", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Études Sur Le Moyen Âge", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "DON'T LOOK NOW", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": true, "normalise": "du maurier daphne", "reasons": ["allcaps_title", "missing_isbn"], "severity": 2, "title_case": "Don't Look Now", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "THE WELL-BUILT WHEEL", "is_shouting": true, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "allcaps_title", "missing_isbn"], "severity": 5, "title_case": "The Well-Built Wheel", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "The (Second) Estate", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "bloch marc", "reasons": ["missing_isbn"], "severity": 0, "title_case": "The (Second) Estate", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": null, "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "missing_isbn"], "severity": 3, "title_case": null, "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": false, "normalise": "", "reasons": ["unknown_author", "missing_isbn"], "severity": 3, "title_case": "", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "spaced", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt hannah", "reasons": [], "severity": 0, "title_case": "Spaced", "title_in_author": false, "valid_isbn": true}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "anonymous", "reasons": ["unknown_author"], "severity": 3, "title_case": "A Book", "title_in_author": false, "valid_isbn": true}
+{"clean_underscores": "Scan 001", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "administrator", "reasons": ["placeholder_author"], "severity": 4, "title_case": "Scan 001", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "xxx", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "xxxx", "reasons": ["placeholder_title", "placeholder_author", "missing_isbn"], "severity": 8, "title_case": "Xxx", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "The Document", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "oxford michael", "reasons": ["missing_isbn"], "severity": 0, "title_case": "The Document", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Full Page Fax Print", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "user", "reasons": ["placeholder_author", "junk_title", "missing_isbn"], "severity": 7, "title_case": "Full Page Fax Print", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Tam Sayfa Faks Yazdırma", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "routledge", "reasons": ["junk_title", "publisher_as_author", "missing_isbn"], "severity": 6, "title_case": "Tam Sayfa Faks Yazdırma", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": "9780198237907", "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "A Book", "is_shouting": false, "isbn_from_filename": "080213825X", "looks_like_person": true, "normalise": "arendt", "reasons": ["missing_isbn"], "severity": 0, "title_case": "A Book", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "Ἀθῆναι καὶ Μόσχα", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "", "reasons": ["missing_isbn"], "severity": 0, "title_case": "Ἀθῆναι Καὶ Μόσχα", "title_in_author": false, "valid_isbn": false}
+{"clean_underscores": "東京の手紙", "is_shouting": false, "isbn_from_filename": null, "looks_like_person": true, "normalise": "", "reasons": ["missing_isbn"], "severity": 0, "title_case": "東京の手紙", "title_in_author": false, "valid_isbn": false}

parity/oracle/bibclean.py

@@ -0,0 +1,48 @@
+"""Audit catalogue records the way the Python pipeline audits them.
+
+Reads one `{"title", "author", "isbn", "path"}` object per line and writes
+what each of the metadata functions makes of it.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+        python3 oracle/bibclean.py < cases/bibclean/records.ndjson
+"""
+
+import json
+import os
+import sys
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from book_pipeline import metadata  # noqa: E402
+
+
+def main() -> int:
+    out = sys.stdout
+    for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+        if not line.strip():
+            continue
+        book = json.loads(line)
+        title = book.get("title")
+        author = book.get("author")
+        isbn = book.get("isbn")
+        reasons = metadata.problem_reasons(title, author, isbn)
+        record = {
+            "reasons": reasons,
+            "severity": metadata.severity(reasons),
+            "title_in_author": metadata.title_in_author(title, author),
+            "looks_like_person": metadata._looks_like_person(author or ""),
+            "is_shouting": metadata.is_shouting(title),
+            "title_case": metadata.title_case(title) if title is not None else None,
+            "clean_underscores": metadata.clean_underscores(title) if title is not None else None,
+            "valid_isbn": metadata.valid_isbn(isbn),
+            "isbn_from_filename": metadata.isbn_from_filename(book.get("path")),
+            "normalise": metadata._norm(author),
+        }
+        out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 10] = [
+    let jobs: [Job; 11] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
@@ -30,6 +30,7 @@ fn main() -> ExitCode {
         ("rrf", rrf),
         ("cuepack", cuepack),
         ("booksniff", booksniff),
+        ("bibclean", bibclean),
     ];
 
     for (name, job) in jobs {
@@ -73,6 +74,18 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn bibclean() -> Written {
+    let dir = parity::golden("bibclean");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/bibclean/records.ndjson");
+    let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let judgements = parity::run_oracle("bibclean.py", &cases)?;
+    let path = dir.join("records.ndjson");
+    std::fs::write(&path, judgements).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn booksniff() -> Written {
     let dir = parity::golden("booksniff");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/bibclean.rs

@@ -0,0 +1,123 @@
+//! Every judgement about a catalogue record must match the Python's.
+//!
+//! These decide what a cleanup run touches. A reason that fires here and not
+//! there means a record is rewritten that should have been left alone, which
+//! is the one failure mode a metadata tool cannot recover from.
+
+use serde_json::Value;
+
+fn records(path: &str) -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> {
+    record.get(name).and_then(Value::as_str)
+}
+
+#[test]
+fn every_judgement_matches_python() {
+    let cases = records("cases/bibclean/records.ndjson");
+    let golden = records("golden/bibclean/records.ndjson");
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    let mut wrong: Vec<String> = Vec::new();
+    for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() {
+        let title = field(case, "title");
+        let author = field(case, "author");
+        let isbn = field(case, "isbn");
+
+        let mut differs = |what: &str, ours: String, theirs: &Value| {
+            wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}"));
+        };
+
+        let reasons = bibclean::reasons::problems(title, author, isbn);
+        let their_reasons: Vec<String> = theirs["reasons"]
+            .as_array()
+            .expect("a list")
+            .iter()
+            .map(|reason| reason.as_str().expect("a name").to_string())
+            .collect();
+        if reasons != their_reasons {
+            differs("reasons", format!("{reasons:?}"), &theirs["reasons"]);
+        }
+
+        let severity = bibclean::reasons::severity(&reasons);
+        if severity as u64 != theirs["severity"].as_u64().expect("a number") {
+            differs("severity", severity.to_string(), &theirs["severity"]);
+        }
+
+        let ours = bibclean::reasons::title_in_author(title, author);
+        if ours != theirs["title_in_author"].as_bool().expect("a verdict") {
+            differs(
+                "title_in_author",
+                ours.to_string(),
+                &theirs["title_in_author"],
+            );
+        }
+
+        let ours = bibclean::reasons::looks_like_person(author.unwrap_or(""));
+        if ours != theirs["looks_like_person"].as_bool().expect("a verdict") {
+            differs(
+                "looks_like_person",
+                ours.to_string(),
+                &theirs["looks_like_person"],
+            );
+        }
+
+        let ours = bibclean::fix::is_shouting(title.unwrap_or(""));
+        if ours != theirs["is_shouting"].as_bool().expect("a verdict") {
+            differs("is_shouting", ours.to_string(), &theirs["is_shouting"]);
+        }
+
+        if let Some(title) = title {
+            let ours = bibclean::fix::title_case(title);
+            if Some(ours.as_str()) != theirs["title_case"].as_str() {
+                differs("title_case", ours, &theirs["title_case"]);
+            }
+            let ours = bibclean::fix::clean_underscores(title);
+            if Some(ours.as_str()) != theirs["clean_underscores"].as_str() {
+                differs("clean_underscores", ours, &theirs["clean_underscores"]);
+            }
+        }
+
+        let ours = bibclean::isbn::valid(isbn.unwrap_or(""));
+        if ours != theirs["valid_isbn"].as_bool().expect("a verdict") {
+            differs("valid_isbn", ours.to_string(), &theirs["valid_isbn"]);
+        }
+
+        let ours = field(case, "path").and_then(bibclean::isbn::from_filename);
+        if ours.as_deref() != theirs["isbn_from_filename"].as_str() {
+            differs(
+                "isbn_from_filename",
+                format!("{ours:?}"),
+                &theirs["isbn_from_filename"],
+            );
+        }
+
+        let ours = bibclean::reasons::normalise(author.unwrap_or(""));
+        if Some(ours.as_str()) != theirs["normalise"].as_str() {
+            differs("normalise", ours, &theirs["normalise"]);
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} judgements differ:\n{}",
+        wrong.len(),
+        wrong.join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/bibclean/records.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("bibclean.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("bibclean/records.ndjson"));
+}