bibclean: slugs, and whether the one a record has still names it

parity/cases/slug/records.ndjson

@@ -0,0 +1,25 @@
+{"slug": "william-zinsser-on-writing-well-3450", "author": "William Zinsser", "title": "On Writing Well", "id": 3450}
+{"slug": "william-zinsser-on-writing-well-3450", "author": "Laura Nader", "title": "Weaponizing Anthropology", "id": 3450}
+{"slug": "unknown-mind-and-brain-1234", "author": "Zinsser, William", "title": "Mind and Brain", "id": 1234}
+{"slug": "unknown-file-c-documents-and-settings-hasici-dokumen-88", "author": "Unknown", "title": "Mind and Brain: Chaos and Quantum Mechanics", "id": 88}
+{"slug": "smith-john-the-mill-z-lib-org-99", "author": "Smith, John", "title": "The Mill and the Valley", "id": 99}
+{"slug": "microsoft-word-mat-doc-1", "author": "Microsoft Word", "title": "mat doc", "id": 1}
+{"slug": "unknown-1234", "author": "Unknown Author", "title": null, "id": 1234}
+{"slug": "unknown-proceedings-of-the-society-vol-9-77", "author": "Aristotelian Society", "title": "Proceedings of the Society", "id": 77}
+{"slug": "unknown-the-land-of-iran-55", "author": "the land of iran (Cambridge University Press", "title": "The Land of Iran", "id": 55}
+{"slug": "old-testament-volume-ii-mark-sheridan-12", "author": "Old Testament, Volume II) (Mark Sheridan", "title": "Genesis 12-50", "id": 12}
+{"slug": "a-a-long-j-a-mansfeld-keimpe-algra-carl-a-huffman-edward-h-7", "author": "A. A. Long & J. A. Mansfeld & Keimpe Algra & Carl A. Huffman & Edward H. Hussey", "title": "The Cambridge Companion to Early Greek Philosophy", "id": 7}
+{"slug": "book-7023", "author": "()", "title": "Mind and Brain", "id": 7023}
+{"slug": "test-3-8", "author": "test", "title": "A Real Book About Something", "id": 8}
+{"slug": "jaroslav-pelikan-christianity-and-classical-culture-the-me-44", "author": "Jaroslav Pelikan", "title": "Christianity and Classical Culture: The Metamorphosis of Natural Theology in the Christian Encounter with Hellenism", "id": 44}
+{"slug": "etre-et-temps-9", "author": "Heidegger, Martin", "title": "Être et Temps", "id": 9}
+{"slug": "athenai-kai-moskha-10", "author": "Θουκυδίδης", "title": "Ἀθῆναι καὶ Μόσχα", "id": 10}
+{"slug": "dong-jing-noshou-zhi-11", "author": "夏目漱石", "title": "東京の手紙", "id": 11}
+{"slug": "kierkegaard-s-conception-of-god-13", "author": "Kierkegaard, Søren", "title": "Kierkegaard's Conception of God", "id": 13}
+{"slug": "12000-years-14", "author": "Smith, J.", "title": "12,000 Years of History", "id": 14}
+{"slug": "tom-jerry-15", "author": "Hanna & Barbera", "title": "Tom & Jerry", "id": 15}
+{"slug": "an-introduction-to-analysis-16", "author": null, "title": "An Introduction to Historical Analysis", "id": 16}
+{"slug": "bishop-christopher-m-pattern-recognition-17", "author": "Bishop, Christopher M.", "title": "Pattern Recognition and Machine Learning", "id": 17}
+{"slug": "miranda-lundy-quadrivium-18", "author": "Miranda Lundy, Daud Sutton, Anthony Ashton, Jason Martineau", "title": "Quadrivium", "id": 18}
+{"slug": "", "author": "Smith, John", "title": "A Book", "id": 19}
+{"slug": "guardians-guardians-3190", "author": "Guardians", "title": "Guardians", "id": 3190}

parity/golden/slug/records.ndjson

@@ -0,0 +1,25 @@
+{"lead_author": "William Zinsser", "proposed": null, "reasons": [], "slug_for": "william-zinsser-on-writing-well-3450", "slugify": "on-writing-well", "tier": null}
+{"lead_author": "Laura Nader", "proposed": "laura-nader-weaponizing-anthropology-3450", "reasons": ["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"], "slug_for": "laura-nader-weaponizing-anthropology-3450", "slugify": "weaponizing-anthropology", "tier": "wrong-book"}
+{"lead_author": "Zinsser, William", "proposed": "zinsser-william-mind-and-brain-1234", "reasons": ["slug opens with placeholder author 'unknown'"], "slug_for": "zinsser-william-mind-and-brain-1234", "slugify": "mind-and-brain", "tier": "placeholder-author"}
+{"lead_author": "Unknown", "proposed": "unknown-mind-and-brain-chaos-and-quantum-mechanics-88", "reasons": ["slug carries a filename or mirror artefact"], "slug_for": "unknown-mind-and-brain-chaos-and-quantum-mechanics-88", "slugify": "mind-and-brain-chaos-and-quantum-mechanics", "tier": "filename-artefact"}
+{"lead_author": "Smith, John", "proposed": "smith-john-the-mill-and-the-valley-99", "reasons": ["slug carries a filename or mirror artefact"], "slug_for": "smith-john-the-mill-and-the-valley-99", "slugify": "the-mill-and-the-valley", "tier": "filename-artefact"}
+{"lead_author": "Microsoft Word", "proposed": "microsoft-word-mat-doc-1", "reasons": ["slug carries a filename or mirror artefact", "recomputes to the same slug: fix the metadata first"], "slug_for": "microsoft-word-mat-doc-1", "slugify": "mat-doc", "tier": "noop-metadata-junk"}
+{"lead_author": "Unknown Author", "proposed": "unknown-author-1234", "reasons": ["slug opens with placeholder author 'unknown'", "new slug is no better: fix the metadata first"], "slug_for": "unknown-author-1234", "slugify": "", "tier": "defer-new-slug-junk"}
+{"lead_author": "Aristotelian Society", "proposed": "aristotelian-society-proceedings-of-the-society-77", "reasons": ["slug opens with placeholder author 'unknown'", "current slug carries a volume marker the new one loses"], "slug_for": "aristotelian-society-proceedings-of-the-society-77", "slugify": "proceedings-of-the-society", "tier": "review-loses-volume"}
+{"lead_author": "the land of iran (Cambridge University Press", "proposed": "the-land-of-iran-cambridge-the-land-of-iran-55", "reasons": ["slug opens with placeholder author 'unknown'", "author field looks like spilled title/series text: 'the land of iran (Cambridge University Press'"], "slug_for": "the-land-of-iran-cambridge-the-land-of-iran-55", "slugify": "the-land-of-iran", "tier": "defer-new-slug-junk"}
+{"lead_author": "Old Testament, Volume II) (Mark Sheridan", "proposed": null, "reasons": [], "slug_for": "old-testament-volume-ii-mark-genesis-12-50-12", "slugify": "genesis-12-50", "tier": null}
+{"lead_author": "A. A. Long", "proposed": null, "reasons": [], "slug_for": "a-a-long-the-cambridge-companion-to-early-greek-philosophy-7", "slugify": "the-cambridge-companion-to-early-greek-philosophy", "tier": null}
+{"lead_author": "()", "proposed": "mind-and-brain-7023", "reasons": ["slug opens with placeholder author 'book'"], "slug_for": "mind-and-brain-7023", "slugify": "mind-and-brain", "tier": "placeholder-author"}
+{"lead_author": "test", "proposed": "test-a-real-book-about-something-8", "reasons": ["slug opens with placeholder author 'test'"], "slug_for": "test-a-real-book-about-something-8", "slugify": "a-real-book-about-something", "tier": "placeholder-author"}
+{"lead_author": "Jaroslav Pelikan", "proposed": null, "reasons": [], "slug_for": "jaroslav-pelikan-christianity-and-classical-culture-the-metamorphosis-of-natural-44", "slugify": "christianity-and-classical-culture-the-metamorphosis-of-natural-theology-in-the-christian-encounter-with-hellenism", "tier": null}
+{"lead_author": "Heidegger, Martin", "proposed": null, "reasons": [], "slug_for": "heidegger-martin-etre-et-temps-9", "slugify": "etre-et-temps", "tier": null}
+{"lead_author": "Θουκυδίδης", "proposed": null, "reasons": [], "slug_for": "thoukudides-athenai-kai-moskha-10", "slugify": "athenai-kai-moskha", "tier": null}
+{"lead_author": "夏目漱石", "proposed": null, "reasons": [], "slug_for": "xia-mu-shu-shi-dong-jing-noshou-zhi-11", "slugify": "dong-jing-noshou-zhi", "tier": null}
+{"lead_author": "Kierkegaard, Søren", "proposed": null, "reasons": [], "slug_for": "kierkegaard-soren-kierkegaard-s-conception-of-god-13", "slugify": "kierkegaard-s-conception-of-god", "tier": null}
+{"lead_author": "Smith, J.", "proposed": null, "reasons": [], "slug_for": "smith-j-12000-years-of-history-14", "slugify": "12000-years-of-history", "tier": null}
+{"lead_author": "Hanna", "proposed": null, "reasons": [], "slug_for": "hanna-tom-jerry-15", "slugify": "tom-jerry", "tier": null}
+{"lead_author": "", "proposed": null, "reasons": [], "slug_for": "an-introduction-to-historical-analysis-16", "slugify": "an-introduction-to-historical-analysis", "tier": null}
+{"lead_author": "Bishop, Christopher M.", "proposed": null, "reasons": [], "slug_for": "bishop-christopher-m-pattern-recognition-and-machine-learning-17", "slugify": "pattern-recognition-and-machine-learning", "tier": null}
+{"lead_author": "Miranda Lundy", "proposed": null, "reasons": [], "slug_for": "miranda-lundy-quadrivium-18", "slugify": "quadrivium", "tier": null}
+{"lead_author": "Smith, John", "proposed": "smith-john-a-book-19", "reasons": ["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"], "slug_for": "smith-john-a-book-19", "slugify": "a-book", "tier": "wrong-book"}
+{"lead_author": "Guardians", "proposed": null, "reasons": [], "slug_for": "guardians-guardians-3190", "slugify": "guardians", "tier": null}

parity/oracle/slug.py

@@ -0,0 +1,50 @@
+"""Judge slugs the way the Python pipeline judges them.
+
+Reads one `{"slug", "author", "title", "id"}` object per line and writes what
+the slug module makes of each.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+        python3 oracle/slug.py < cases/slug/records.ndjson
+"""
+
+import json
+import os
+import sys
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from slugify import slugify  # noqa: E402
+
+from book_pipeline import slugs  # noqa: E402
+
+
+def main() -> int:
+    out = sys.stdout
+    for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+        if not line.strip():
+            continue
+        record = json.loads(line)
+        author, title, book_id = record.get("author"), record.get("title"), record["id"]
+        proposal = slugs.evaluate("x", record["slug"], author, title, book_id)
+        out.write(
+            json.dumps(
+                {
+                    "slugify": slugify(title or ""),
+                    "slug_for": slugs.slug_for(author, title, book_id),
+                    "lead_author": slugs._lead_author(author),
+                    "tier": proposal.tier if proposal else None,
+                    "proposed": proposal.proposed_slug if proposal else None,
+                    "reasons": proposal.reasons if proposal else [],
+                },
+                ensure_ascii=False,
+                sort_keys=True,
+            )
+            + "\n"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 11] = [
+    let jobs: [Job; 12] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
@@ -31,6 +31,7 @@ fn main() -> ExitCode {
         ("cuepack", cuepack),
         ("booksniff", booksniff),
         ("bibclean", bibclean),
+        ("slug", slug),
     ];
 
     for (name, job) in jobs {
@@ -74,6 +75,18 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn slug() -> Written {
+    let dir = parity::golden("slug");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/slug/records.ndjson");
+    let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let judgements = parity::run_oracle("slug.py", &cases)?;
+    let path = dir.join("records.ndjson");
+    std::fs::write(&path, judgements).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn bibclean() -> Written {
     let dir = parity::golden("bibclean");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/slug.rs

@@ -0,0 +1,100 @@
+//! Slugs, and the judgement about whether one should be replaced.
+//!
+//! A slug is a public identifier: it reaches citations and research notes and
+//! outlives whatever produced it. Generating a different one than the system
+//! being replaced would rename every book in a corpus, and judging one
+//! differently would rename the wrong ones.
+
+use serde_json::Value;
+
+fn records(path: &str) -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> {
+    record.get(name).and_then(Value::as_str)
+}
+
+#[test]
+fn every_slug_and_verdict_matches_python() {
+    let cases = records("cases/slug/records.ndjson");
+    let golden = records("golden/slug/records.ndjson");
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    let mut wrong: Vec<String> = Vec::new();
+    for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() {
+        let author = field(case, "author");
+        let title = field(case, "title");
+        let slug = field(case, "slug").unwrap_or("");
+        let id = case["id"].as_i64().expect("an id");
+
+        let mut differs = |what: &str, ours: String, theirs: &Value| {
+            wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}"));
+        };
+
+        let ours = bibclean::slug::slugify(title.unwrap_or(""), 0, false);
+        if Some(ours.as_str()) != theirs["slugify"].as_str() {
+            differs("slugify", ours, &theirs["slugify"]);
+        }
+
+        let ours = bibclean::slug::lead_author(author.unwrap_or(""));
+        if Some(ours.as_str()) != theirs["lead_author"].as_str() {
+            differs("lead_author", ours, &theirs["lead_author"]);
+        }
+
+        let ours = bibclean::slug::slug_for(author, title, id);
+        if Some(ours.as_str()) != theirs["slug_for"].as_str() {
+            differs("slug_for", ours, &theirs["slug_for"]);
+        }
+
+        let proposal = bibclean::slug::evaluate(slug, author, title, id);
+        let ours_tier = proposal.as_ref().map(|found| found.tier.as_str());
+        if ours_tier != theirs["tier"].as_str() {
+            differs("tier", format!("{ours_tier:?}"), &theirs["tier"]);
+        }
+
+        let ours_proposed = proposal.as_ref().map(|found| found.proposed.as_str());
+        if ours_proposed != theirs["proposed"].as_str() {
+            differs(
+                "proposed",
+                format!("{ours_proposed:?}"),
+                &theirs["proposed"],
+            );
+        }
+
+        let ours_reasons: Vec<String> = proposal
+            .as_ref()
+            .map(|found| found.reasons.clone())
+            .unwrap_or_default();
+        let their_reasons: Vec<String> = theirs["reasons"]
+            .as_array()
+            .expect("a list")
+            .iter()
+            .map(|reason| reason.as_str().expect("a reason").to_string())
+            .collect();
+        if ours_reasons != their_reasons {
+            differs("reasons", format!("{ours_reasons:?}"), &theirs["reasons"]);
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} judgements differ:\n{}",
+        wrong.len(),
+        wrong.join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/slug/records.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("slug.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("slug/records.ndjson"));
+}