From 3c16892ebf9e12cbbd0c97c1f40473deaf692cef Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 14:29:12 -0400 Subject: [PATCH] bibclean: a name from the book's own pages, where the record has none --- crates/bibclean/README.md | 37 ++ crates/bibclean/src/author.rs | 508 ++++++++++++++++++++++++++++ crates/bibclean/src/lib.rs | 53 +++ crates/bibclean/src/main.rs | 12 +- parity/cases/author/records.ndjson | 30 ++ parity/golden/author/records.ndjson | 30 ++ parity/oracle/author.py | 55 +++ parity/src/bin/regen.rs | 15 +- parity/tests/author.rs | 102 ++++++ 9 files changed, 836 insertions(+), 6 deletions(-) create mode 100644 crates/bibclean/src/author.rs create mode 100644 parity/cases/author/records.ndjson create mode 100644 parity/golden/author/records.ndjson create mode 100644 parity/oracle/author.py create mode 100644 parity/tests/author.rs diff --git a/crates/bibclean/README.md b/crates/bibclean/README.md index 89b7257..f4cb194 100644 --- a/crates/bibclean/README.md +++ b/crates/bibclean/README.md @@ -91,6 +91,43 @@ through this and back into whatever it came from. {"title":"THE HUMAN CONDITION","author":"Arendt, Hannah","path":"/library/9780226586601.pdf"} ``` +## Recovering a missing author + +A book says who wrote it on its own copyright page. Hand in the opening pages +as `front_matter` and a record whose author is missing gets a name proposed, +with the evidence and how far it is worth trusting: + +``` +$ bibclean authors.ndjson +{"v":1,"title":"The Icon and the Axe","author":"Unknown",…,"reasons":["unknown_author","missing_isbn"],"severity":3,"changes":[],"author_proposal":{"author":"James H. Billington","method":"front_matter_copyright","confidence":0.9,"evidence":"…by James H. Billington","isbn":"0394708461"}} +{"v":1,"title":"Tragedy and Hope by Carroll Quigley","author":"",…,"reasons":["unknown_author"],"severity":3,"changes":[],"author_proposal":{"author":"Carroll Quigley","method":"title_by","confidence":0.7,"evidence":"by Carroll Quigley","isbn":null}} +``` + +Four signals, cheapest and most trustworthy first: + +| Method | Confidence | Read from | +|---|---|---| +| `front_matter_copyright` | 0.9 | "Copyright © 1966 by …" | +| `front_matter_loc` | 0.8 | The cataloguing block's main entry, written surname first | +| `title_by` | 0.7 | "… by Author Name" at the end of the title | +| `title_camel` | 0.5 | A run-together name at the front of the title | + +An ISBN found on the same page rides along with the name, because the two are +facts about the same book. + +A candidate that reads as a publisher rather than a person is refused — an +imprint holds the copyright on a great many books and is the author of none of +them — as is one with a digit in it, one that opens with an article, and one +too long to be a name. Shouting is fixed, bare initials get their full stops, +and co-authors are kept: `John Schuster and John Sutton` becomes +`John Schuster & John Sutton`. + +Nothing is applied. A record that already has an author is not second-guessed; +the proposal is only ever made where there is a gap. + +The front matter is read and never written back — it is evidence handed in, +not part of the record. + ## Slugs A slug is a public identifier — it reaches citations and notes, and outlives diff --git a/crates/bibclean/src/author.rs b/crates/bibclean/src/author.rs new file mode 100644 index 0000000..91286c1 --- /dev/null +++ b/crates/bibclean/src/author.rs @@ -0,0 +1,508 @@ +//! Recovering an author a record does not have. +//! +//! All of it local: what the book says about itself on its own first pages, +//! and what its title says. No lookup, no network, and nothing applied — a +//! name is proposed with the evidence that produced it and how far it is +//! worth trusting. + +use std::sync::LazyLock; + +use fancy_regex::Regex as Boundaried; +use regex::Regex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use textkit::pattern::{SPACE, WORD_END, WORD_START}; + +fn pattern(source: &str) -> Regex { + Regex::new(source).expect("a valid pattern") +} + +/// Words that mark a candidate as title or publisher bleed rather than a +/// person. Without them the looser heuristics propose the imprint. +static NOT_A_PERSON: LazyLock = LazyLock::new(|| { + Boundaried::new(&format!( + r"(?i){WORD_START}(press|edition|introduction|university|volume|reprint|copyright|published|printed|rights|reserved|library|congress|isbn|inc|ltd|company|regents|trustees|foundation|institute|society|association|board|council|committee|department|ministry|editors?|trust|publishing|publishers|books|publications|sons|llc|gmbh|verlag|all|limited){WORD_END}" + )) + .expect("a valid pattern") +}); + +/// One token of a name: a capitalised word, a bare initial, or a nobiliary +/// particle that appears in the middle of one. +static NAME_TOKEN: LazyLock = LazyLock::new(|| { + pattern( + r"^(?:[A-Z][A-Za-z’'\-]+\.?|[A-Z]\.|(?:von|van|de|den|der|del|della|di|da|la|le|du|ter|ten))$", + ) +}); + +/// Where several authors are joined. +static AUTHOR_SEPARATOR: LazyLock = + LazyLock::new(|| pattern(&format!(r"(?i){SPACE}+(?:and|&){SPACE}+"))); + +/// "et al", "and others", and everything after. +static ET_AL: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"(?i)[{SPACE},]+(?:et{SPACE}+al\.?|and{SPACE}+others).*$" + )) +}); + +static SPACE_RUN: LazyLock = LazyLock::new(|| pattern(&format!(r"{SPACE}+"))); + +/// An ISBN announced on the copyright page. +static ISBN: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"ISBN[:{SPACE}]*((?:97[89][{SPACE}-]?)?(?:[0-9][{SPACE}-]?){{9}}[0-9Xx])" + )) +}); + +/// The strongest thing a book says about who wrote it. +static COPYRIGHT: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"(?i)Copyright{SPACE}*(?:©|\(c\))?{SPACE}*\d{{4}}{SPACE}+by{SPACE}+(.{{2,90}})" + )) +}); + +/// The cataloguing block a national library prints on the copyright page. +static CATALOGUING: LazyLock = LazyLock::new(|| { + pattern(&format!( + r"(?i)Cataloging.{{0,40}}Publication(?:{SPACE}+Data)?" + )) +}); + +/// Its main entry, which is the author, written surname first. +static MAIN_ENTRY: LazyLock = + LazyLock::new(|| pattern(r"([A-Z][A-Za-z'’-]+),\s+([A-Z][A-Za-z.ّ'’ -]{1,40}?)[,.]")); + +/// "… by Author Name" at the end of a title. +static TITLE_BY: LazyLock = LazyLock::new(|| { + Boundaried::new(&format!( + r"(?i){WORD_START}by{SPACE}+([A-Z][A-Za-z.'’ -]{{2,60}})$" + )) + .expect("a valid pattern") +}); + +/// A title that opens with the author's name run together. +static TITLE_CAMEL: LazyLock = LazyLock::new(|| pattern(r"^([A-Z][a-z]+[A-Z][a-z]+)[-\s]")); + +static CAMEL_PARTS: LazyLock = LazyLock::new(|| pattern(r"[A-Z][a-z]+")); + +/// Roman numerals that are a suffix on a name rather than shouting. +const ROMAN_SUFFIXES: &[&str] = &["II", "III", "IV", "V", "VI"]; + +/// Characters trimmed from the ends of a candidate. +const EDGES: &[char] = &[' ', '.', ',', ';', ':', '—', '-', '\t', '\n']; + +/// How far into a book's front matter to look. +pub const FRONT_MATTER_CHARS: usize = 8000; + +/// Digits that are digits to Python and are not decimal. +/// +/// A candidate name containing any digit is refused, and "any digit" there +/// means superscripts and circled numbers too — the ones a scanned page picks +/// up from a footnote marker. +const DIGIT_RANGES: &[(char, char)] = &[ + ('\u{b2}', '\u{b3}'), + ('\u{b9}', '\u{b9}'), + ('\u{1369}', '\u{1371}'), + ('\u{19da}', '\u{19da}'), + ('\u{2070}', '\u{2070}'), + ('\u{2074}', '\u{2079}'), + ('\u{2080}', '\u{2089}'), + ('\u{2460}', '\u{2468}'), + ('\u{2474}', '\u{247c}'), + ('\u{2488}', '\u{2490}'), + ('\u{24ea}', '\u{24ea}'), + ('\u{24f5}', '\u{24fd}'), + ('\u{24ff}', '\u{24ff}'), + ('\u{2776}', '\u{277e}'), + ('\u{2780}', '\u{2788}'), + ('\u{278a}', '\u{2792}'), + ('\u{10a40}', '\u{10a43}'), + ('\u{10e60}', '\u{10e68}'), + ('\u{11052}', '\u{1105a}'), + ('\u{1f100}', '\u{1f10a}'), +]; + +/// A name, where it came from, and how far it is worth trusting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Suggestion { + /// The name, normalised. + pub author: String, + /// Which signal produced it. + pub method: String, + /// How far to trust it, from zero to one. + #[serde(with = "corpus_core::finite")] + #[schemars(with = "f64")] + pub confidence: f64, + /// The text it was read from, so a person can check it. + pub evidence: String, + /// An ISBN found alongside it, if there was one. + pub isbn: Option, +} + +/// The best name a record's own text and title can offer, if any. +/// +/// Candidates are gathered cheapest and most trustworthy first, and the most +/// confident wins. An ISBN found in the front matter is carried onto the +/// winner even when the winner came from the title, because the two are +/// facts about the same book. +pub fn best(title: &str, front_matter: Option<&str>) -> Option { + let mut candidates: Vec = Vec::new(); + if let Some(text) = front_matter { + candidates.extend(from_front_matter(text)); + } + candidates.extend(from_title(title)); + + let isbn = candidates + .iter() + .find_map(|candidate| candidate.isbn.clone()); + // The first of the most confident, which is the earliest and therefore + // the cheapest signal that scored highest. + let mut winner = candidates.into_iter().reduce(|best, candidate| { + match candidate.confidence > best.confidence { + true => candidate, + false => best, + } + })?; + if winner.isbn.is_none() { + winner.isbn = isbn; + } + Some(winner) +} + +/// Names a book's own front matter offers. +pub fn from_front_matter(text: &str) -> Vec { + let isbn = ISBN + .captures(text) + .and_then(|found| normalise_isbn(&found[1])); + let mut out = Vec::new(); + + // Strongest: the book saying who holds its copyright. + if let Some(found) = COPYRIGHT.captures(text) + && let Some(name) = clean(&name_prefix(&found[1])) + && name.split_whitespace().count() >= 2 + { + // Two words at least: a lone surname is not an attribution. + out.push(Suggestion { + evidence: format!("…by {name}"), + author: name, + method: "front_matter_copyright".into(), + confidence: 0.9, + isbn: isbn.clone(), + }); + } + + // The cataloguing block's main entry, written surname first. The + // boilerplate is consumed first so the window starts at the name rather + // than at the last word of the heading. + if let Some(heading) = CATALOGUING.find(text) { + let from = heading.end(); + let window: String = text[from..].chars().take(300).collect(); + if let Some(found) = MAIN_ENTRY.captures(&window) + && let Some(name) = clean(&format!("{} {}", &found[2], &found[1])) + { + out.push(Suggestion { + author: name, + method: "front_matter_loc".into(), + confidence: 0.8, + evidence: found[0] + .trim_matches(textkit::is_space) + .chars() + .take(200) + .collect(), + isbn, + }); + } + } + out +} + +/// Names the title string itself offers. +pub fn from_title(title: &str) -> Vec { + let mut out = Vec::new(); + + if let Ok(Some(found)) = TITLE_BY.captures(title) + && let Some(name) = clean(&found[1]) + { + out.push(Suggestion { + author: name, + method: "title_by".into(), + confidence: 0.7, + evidence: found[0].trim_matches(textkit::is_space).to_string(), + isbn: None, + }); + } + + if let Some(found) = TITLE_CAMEL.captures(title) + && let Some(split) = split_camel(&found[1]) + { + out.push(Suggestion { + author: split, + method: "title_camel".into(), + confidence: 0.5, + evidence: found[1].to_string(), + isbn: None, + }); + } + out +} + +/// The leading run of name tokens, stopping at prose. +/// +/// "James H. Billington Printed in the…" is "James H. Billington". The run is +/// capped so that a sentence which happens to start with capitals cannot be +/// swallowed whole, and a word ending in a full stop that is not an initial +/// ends the name — it is the end of a sentence. +pub fn name_prefix(text: &str) -> String { + let mut tokens: Vec<&str> = Vec::new(); + for raw in text.split_whitespace() { + let token = raw.trim_matches([',', ';', ':', '(', ')']); + let core = token.trim_matches('.'); + if core.is_empty() || NOT_A_PERSON.is_match(core).unwrap_or(false) { + break; + } + if !NAME_TOKEN.is_match(token) { + break; + } + tokens.push(token); + if token.ends_with('.') && core.chars().count() > 1 { + break; + } + if tokens.len() >= 5 { + break; + } + } + tokens.join(" ") +} + +/// Normalise a candidate, or refuse it if it does not read as a person. +pub fn clean(raw: &str) -> Option { + let name = SPACE_RUN.replace_all(raw, " "); + let name = name.trim_matches(|c: char| EDGES.contains(&c)); + if name.is_empty() || name.chars().any(is_digit) { + return None; + } + if NOT_A_PERSON.is_match(name).unwrap_or(false) { + return None; + } + normalise(name) +} + +/// Normalise a name, keeping co-authors joined. +/// +/// `None` when any one author fails the shape a person's name has: one to +/// five words, capitalised, and not opening with an article. +pub fn normalise(raw: &str) -> Option { + let name = ET_AL.replace(raw, ""); + let name = name.trim_matches(textkit::is_space); + + let authors: Vec = AUTHOR_SEPARATOR + .split(name) + .map(|segment| segment.trim_matches(textkit::is_space)) + .filter(|segment| !segment.is_empty()) + .map(normalise_tokens) + .collect(); + if authors.is_empty() { + return None; + } + + for author in &authors { + let words: Vec<&str> = author.split_whitespace().collect(); + if author.is_empty() || !(1..=5).contains(&words.len()) { + return None; + } + let first = author.chars().next()?; + if !first.is_uppercase() || words[0].to_lowercase() == "the" { + return None; + } + } + Some(authors.join(" & ")) +} + +/// Fix shouting and bare initials within one name. +fn normalise_tokens(segment: &str) -> String { + segment + .split_whitespace() + .map(|token| { + let core = token.trim_matches('.'); + let shouting = core.chars().all(char::is_alphabetic) + && !core.is_empty() + && core.chars().all(|c| !c.is_lowercase()) + && core.chars().any(char::is_uppercase) + && core.chars().count() > 1 + && !ROMAN_SUFFIXES.contains(&core); + if shouting { + let mut fixed = capitalise(core); + if token.ends_with('.') { + fixed.push('.'); + } + return fixed; + } + if core.chars().count() == 1 + && token.chars().count() == 1 + && core.chars().all(|c| c.is_alphabetic() && c.is_uppercase()) + { + return format!("{token}."); + } + token.to_string() + }) + .collect::>() + .join(" ") +} + +fn capitalise(word: &str) -> String { + let mut chars = word.chars(); + match chars.next() { + None => String::new(), + Some(first) => first.to_uppercase().collect::() + &chars.as_str().to_lowercase(), + } +} + +/// A run-together name split back apart, when it is two or three parts. +fn split_camel(token: &str) -> Option { + let parts: Vec<&str> = CAMEL_PARTS + .find_iter(token) + .map(|found| found.as_str()) + .collect(); + let rejoined: String = parts.concat(); + ((2..=3).contains(&parts.len()) && rejoined == token).then(|| parts.join(" ")) +} + +fn normalise_isbn(raw: &str) -> Option { + let digits = crate::isbn::digits(raw); + matches!(digits.len(), 10 | 13).then(|| digits.to_uppercase()) +} + +/// Whether a character is a digit, by Python's definition — which counts the +/// superscripts and the circled numbers that a decimal test does not. +fn is_digit(c: char) -> bool { + c.to_digit(10).is_some() + || DIGIT_RANGES + .iter() + .any(|(from, to)| (*from..=*to).contains(&c)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_copyright_line_names_the_author() { + let text = "Copyright © 1999 by James H. Billington\nPrinted in the United States"; + let found = from_front_matter(text); + assert_eq!(found[0].author, "James H. Billington"); + assert_eq!(found[0].method, "front_matter_copyright"); + assert_eq!(found[0].confidence, 0.9); + } + + #[test] + fn a_cataloguing_block_names_the_author_surname_first() { + let text = "Library of Congress Cataloging-in-Publication Data\n\ + Billington, James H., 1929-\n The icon and the axe."; + let found = from_front_matter(text); + assert_eq!(found[0].author, "James H. Billington"); + assert_eq!(found[0].method, "front_matter_loc"); + } + + #[test] + fn an_isbn_on_the_same_page_comes_with_the_name() { + let text = "Copyright © 1999 by James Billington\nISBN 978-0-19-823790-7"; + assert_eq!( + from_front_matter(text)[0].isbn.as_deref(), + Some("9780198237907") + ); + } + + #[test] + fn a_title_that_names_its_author_offers_it() { + let found = from_title("The Icon and the Axe by James Billington"); + assert_eq!(found[0].author, "James Billington"); + assert_eq!(found[0].method, "title_by"); + } + + #[test] + fn a_run_together_name_at_the_front_of_a_title_is_split() { + let found = from_title("CarrollQuigley Tragedy and Hope"); + assert_eq!(found[0].author, "Carroll Quigley"); + assert_eq!(found[0].confidence, 0.5); + } + + #[test] + fn the_most_confident_candidate_wins_and_keeps_the_isbn() { + let front = "Copyright © 1999 by James Billington\nISBN 0198237901"; + let found = best("Tragedy and Hope by Carroll Quigley", Some(front)).expect("a name"); + assert_eq!(found.author, "James Billington"); + assert_eq!(found.isbn.as_deref(), Some("0198237901")); + } + + #[test] + fn an_isbn_alone_is_not_carried_anywhere() { + // The ISBN rides along with a name the front matter produced. Front + // matter that names nobody produces no candidate for it to ride on. + let front = "ISBN 0198237901\nno attribution here"; + let found = best("Tragedy and Hope by Carroll Quigley", Some(front)).expect("a name"); + assert_eq!(found.method, "title_by"); + assert_eq!(found.isbn, None); + } + + #[test] + fn a_publisher_is_not_a_person() { + assert_eq!(clean("Oxford University Press"), None); + assert_eq!(clean("The Regents of the University"), None); + assert!(from_front_matter("Copyright © 1999 by Cambridge University Press").is_empty()); + } + + #[test] + fn a_name_with_a_number_in_it_is_refused() { + assert_eq!(clean("James Billington 1929"), None); + assert_eq!(clean("James\u{b2} Billington"), None); + } + + #[test] + fn shouting_and_bare_initials_are_fixed() { + assert_eq!(clean("ARISTOTLE").as_deref(), Some("Aristotle")); + assert_eq!(clean("Barbara A Kerr").as_deref(), Some("Barbara A. Kerr")); + assert_eq!(clean("Smith Jr. II").as_deref(), Some("Smith Jr. II")); + } + + #[test] + fn co_authors_are_kept_and_joined() { + assert_eq!( + clean("John Schuster and John Sutton").as_deref(), + Some("John Schuster & John Sutton") + ); + } + + #[test] + fn everything_after_et_al_is_dropped() { + assert_eq!(clean("Smith et al.").as_deref(), Some("Smith")); + assert_eq!(clean("Smith and others").as_deref(), Some("Smith")); + // Refused before the trailing text is ever reached: a candidate with + // a digit in it is not a name. + assert_eq!(clean("Smith, and others, 1999"), None); + } + + #[test] + fn something_that_is_not_a_name_is_refused() { + assert_eq!(clean(""), None); + assert_eq!(clean("the icon and the axe"), None); + assert_eq!( + clean("A Very Long Name With Far Too Many Words In It"), + None + ); + } + + #[test] + fn a_name_run_stops_at_prose() { + assert_eq!( + name_prefix("James H. Billington Printed in the United States"), + "James H. Billington" + ); + assert_eq!(name_prefix("Ludwig von Mises wrote"), "Ludwig von Mises"); + } + + #[test] + fn nothing_to_go_on_offers_nothing() { + assert_eq!(best("An Ordinary Title", None), None); + assert_eq!(best("", Some("")), None); + } +} diff --git a/crates/bibclean/src/lib.rs b/crates/bibclean/src/lib.rs index 94d688c..8ca8617 100644 --- a/crates/bibclean/src/lib.rs +++ b/crates/bibclean/src/lib.rs @@ -11,6 +11,7 @@ //! which is the difference between a tool that cleans a catalogue and one //! that damages it in a new way. +pub mod author; mod entities4; pub mod fix; pub mod isbn; @@ -49,6 +50,17 @@ pub struct Book { /// A number that makes a slug unique. Needed to judge or regenerate one. #[serde(default)] pub id: Option, + /// The opening pages of the work, if the caller has them. + /// + /// A book says who wrote it on its own copyright page, which is the + /// cheapest and most trustworthy answer there is for a record whose + /// author is missing. + /// + /// Read but never written back: it is evidence handed in, not part of + /// the record, and several thousand characters of it on every line would + /// bury everything else. + #[serde(default, skip_serializing)] + pub front_matter: Option, /// Everything else the record carried. #[serde(flatten)] #[schemars(with = "Map")] @@ -90,6 +102,12 @@ pub struct Cleaned { pub severity: u32, /// What was changed, and why. pub changes: Vec, + /// A name for a record that has none, with the evidence for it. + /// + /// Proposed rather than applied, because the evidence differs in strength + /// and a wrong name is worse than an honest gap. + #[serde(skip_serializing_if = "Option::is_none")] + pub author_proposal: Option, /// What should happen to the record's slug, when it has one to judge. /// /// Named apart from the record's own `slug`, which is carried through @@ -172,10 +190,24 @@ pub fn clean(book: &Book) -> Cleaned { _ => None, }; + // Only for a record that has no author to overwrite. A name recovered + // from the text is evidence, not an improvement on one somebody entered. + let author_proposal = reasons + .iter() + .any(|reason| reason == "unknown_author" || reason == "placeholder_author") + .then(|| { + author::best( + book.title.as_deref().unwrap_or(""), + book.front_matter.as_deref(), + ) + }) + .flatten(); + Cleaned { severity: reasons::severity(&reasons), reasons, changes, + author_proposal, slug_proposal: proposal, book: cleaned, } @@ -194,6 +226,7 @@ mod tests { path: None, slug: None, id: None, + front_matter: None, extra: Map::new(), } } @@ -247,6 +280,26 @@ mod tests { assert_eq!(clean(&book).book.isbn, None); } + #[test] + fn a_record_with_no_author_gets_one_proposed_from_its_own_pages() { + let mut book = book("The Icon and the Axe", "Unknown"); + book.front_matter = Some("Copyright © 1966 by James H. Billington".into()); + + let cleaned = clean(&book); + let proposal = cleaned.author_proposal.expect("a proposal"); + assert_eq!(proposal.author, "James H. Billington"); + assert_eq!(proposal.method, "front_matter_copyright"); + // Proposed, never applied. + assert_eq!(cleaned.book.author.as_deref(), Some("Unknown")); + } + + #[test] + fn a_record_that_has_an_author_is_not_second_guessed() { + let mut book = book("The Icon and the Axe", "Billington, James H."); + book.front_matter = Some("Copyright © 1966 by Somebody Else".into()); + assert!(clean(&book).author_proposal.is_none()); + } + #[test] fn a_slug_is_judged_when_there_is_one_to_judge() { let mut book = book("Weaponizing Anthropology", "Laura Nader"); diff --git a/crates/bibclean/src/main.rs b/crates/bibclean/src/main.rs index 47ca25c..a6fa6a5 100644 --- a/crates/bibclean/src/main.rs +++ b/crates/bibclean/src/main.rs @@ -9,10 +9,10 @@ use corpus_core::{exit, ndjson}; /// Audit catalogue records and repair what can be repaired without a lookup. /// -/// Reads one JSON record per line — `title`, `author`, `isbn` and `path` are -/// read, everything else is carried through untouched — and writes each one -/// back with the faults it has, how much they weigh, and a line for every -/// change made and why. +/// Reads one JSON record per line — `title`, `author`, `isbn`, `path`, +/// `slug`, `id` and `front_matter` are read, everything else is carried +/// through untouched — and writes each one back with the faults it has, how +/// much they weigh, and a line for every change made and why. /// /// Nothing is rewritten silently. A run can be read before it is agreed to, /// which is the difference between cleaning a catalogue and damaging it in a @@ -45,7 +45,9 @@ struct Args { /// should be replaced does, even when nothing else is wrong, because the /// identifier is the part other things have written down. fn needs_attention(cleaned: &bibclean::Cleaned) -> bool { - bibclean::reasons::worth_fixing(&cleaned.reasons) || cleaned.slug_proposal.is_some() + bibclean::reasons::worth_fixing(&cleaned.reasons) + || cleaned.slug_proposal.is_some() + || cleaned.author_proposal.is_some() } fn main() -> ExitCode { diff --git a/parity/cases/author/records.ndjson b/parity/cases/author/records.ndjson new file mode 100644 index 0000000..da3fc68 --- /dev/null +++ b/parity/cases/author/records.ndjson @@ -0,0 +1,30 @@ +{"title": "The Icon and the Axe", "front_matter": "THE ICON AND THE AXE\n\nCopyright © 1966 by James H. Billington\nAll rights reserved under International and Pan-American Copyright\nConventions. Published in the United States by Vintage Books.\nISBN 0-394-70846-1\n", "candidate": "James H. Billington"} +{"title": "The Icon and the Axe", "front_matter": "Library of Congress Cataloging-in-Publication Data\n\nBillington, James H., 1929-\n The icon and the axe : an interpretive history of Russian culture /\n James H. Billington.\n p. cm.\n Includes bibliographical references and index.\n ISBN 978-0-394-70846-8\n", "candidate": "Billington, James H."} +{"title": "The Icon and the Axe", "front_matter": "THE ICON AND THE AXE\n\nCopyright © 1966 by James H. Billington\nAll rights reserved under International and Pan-American Copyright\nConventions. Published in the United States by Vintage Books.\nISBN 0-394-70846-1\n\nLibrary of Congress Cataloging-in-Publication Data\n\nBillington, James H., 1929-\n The icon and the axe : an interpretive history of Russian culture /\n James H. Billington.\n p. cm.\n Includes bibliographical references and index.\n ISBN 978-0-394-70846-8\n", "candidate": null} +{"title": "The Icon and the Axe", "front_matter": "Copyright © 2001 by Cambridge University Press\nAll rights reserved. No part of this publication may be reproduced.\n", "candidate": "Cambridge University Press"} +{"title": "Tragedy and Hope by Carroll Quigley", "front_matter": null, "candidate": "Carroll Quigley"} +{"title": "CarrollQuigley-Tragedy and Hope", "front_matter": null, "candidate": null} +{"title": "CarrollQuigley Tragedy and Hope", "front_matter": null, "candidate": null} +{"title": "An Ordinary Title", "front_matter": null, "candidate": null} +{"title": "Tragedy and Hope by Carroll Quigley", "front_matter": "THE ICON AND THE AXE\n\nCopyright © 1966 by James H. Billington\nAll rights reserved under International and Pan-American Copyright\nConventions. Published in the United States by Vintage Books.\nISBN 0-394-70846-1\n", "candidate": null} +{"title": "", "front_matter": "", "candidate": ""} +{"title": "x", "front_matter": null, "candidate": "ARISTOTLE"} +{"title": "x", "front_matter": null, "candidate": "Barbara A Kerr"} +{"title": "x", "front_matter": null, "candidate": "John Schuster and John Sutton"} +{"title": "x", "front_matter": null, "candidate": "Smith et al."} +{"title": "x", "front_matter": null, "candidate": "Smith and others"} +{"title": "x", "front_matter": null, "candidate": "Smith, and others, 1999"} +{"title": "x", "front_matter": null, "candidate": "Ludwig von Mises"} +{"title": "x", "front_matter": null, "candidate": "the icon and the axe"} +{"title": "x", "front_matter": null, "candidate": "A Very Long Name With Far Too Many Words"} +{"title": "x", "front_matter": null, "candidate": "James H. Billington Printed in the United States"} +{"title": "x", "front_matter": null, "candidate": "Smith Jr. II"} +{"title": "x", "front_matter": null, "candidate": "O'Brien, Flann"} +{"title": "x", "front_matter": null, "candidate": "Søren Kierkegaard"} +{"title": "x", "front_matter": null, "candidate": "James² Billington"} +{"title": "x", "front_matter": null, "candidate": "Éric Voegelin"} +{"title": "x", "front_matter": null, "candidate": " spaced out name "} +{"title": "x", "front_matter": "Copyright © 1999 by Smith", "candidate": null} +{"title": "x", "front_matter": "Copyright (c) 1999 by Jane Q. Public. All rights reserved", "candidate": null} +{"title": "x", "front_matter": "Cataloging in Publication Data\nO'Brien, Flann, 1911-1966.\n At swim.", "candidate": null} +{"title": "x", "front_matter": "ISBN 9780198237907 and nothing else", "candidate": null} diff --git a/parity/golden/author/records.ndjson b/parity/golden/author/records.ndjson new file mode 100644 index 0000000..a08c0f0 --- /dev/null +++ b/parity/golden/author/records.ndjson @@ -0,0 +1,30 @@ +{"best": {"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, "clean": "James H. Billington", "front_matter": [{"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}], "name_prefix": "James H. Billington", "title": []} +{"best": {"author": "James H. Billington", "confidence": 0.8, "evidence": "Billington, James H.", "isbn": "9780394708468", "method": "front_matter_loc"}, "clean": "Billington, James H.", "front_matter": [{"author": "James H. Billington", "confidence": 0.8, "evidence": "Billington, James H.", "isbn": "9780394708468", "method": "front_matter_loc"}], "name_prefix": "Billington James H.", "title": []} +{"best": {"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, "clean": null, "front_matter": [{"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, {"author": "James H. Billington", "confidence": 0.8, "evidence": "Billington, James H.", "isbn": "0394708461", "method": "front_matter_loc"}], "name_prefix": null, "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "Cambridge", "title": []} +{"best": {"author": "Carroll Quigley", "confidence": 0.7, "evidence": "by Carroll Quigley", "isbn": null, "method": "title_by"}, "clean": "Carroll Quigley", "front_matter": [], "name_prefix": "Carroll Quigley", "title": [{"author": "Carroll Quigley", "confidence": 0.7, "evidence": "by Carroll Quigley", "isbn": null, "method": "title_by"}]} +{"best": {"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}, "clean": null, "front_matter": [], "name_prefix": null, "title": [{"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}]} +{"best": {"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}, "clean": null, "front_matter": [], "name_prefix": null, "title": [{"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}]} +{"best": null, "clean": null, "front_matter": [], "name_prefix": null, "title": []} +{"best": {"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, "clean": null, "front_matter": [{"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}], "name_prefix": null, "title": [{"author": "Carroll Quigley", "confidence": 0.7, "evidence": "by Carroll Quigley", "isbn": null, "method": "title_by"}]} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": "Aristotle", "front_matter": [], "name_prefix": "ARISTOTLE", "title": []} +{"best": null, "clean": "Barbara A. Kerr", "front_matter": [], "name_prefix": "Barbara", "title": []} +{"best": null, "clean": "John Schuster & John Sutton", "front_matter": [], "name_prefix": "John Schuster", "title": []} +{"best": null, "clean": "Smith", "front_matter": [], "name_prefix": "Smith", "title": []} +{"best": null, "clean": "Smith", "front_matter": [], "name_prefix": "Smith", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "Smith", "title": []} +{"best": null, "clean": "Ludwig von Mises", "front_matter": [], "name_prefix": "Ludwig von Mises", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "James H. Billington", "title": []} +{"best": null, "clean": "Smith Jr. II", "front_matter": [], "name_prefix": "Smith Jr.", "title": []} +{"best": null, "clean": "O'Brien, Flann", "front_matter": [], "name_prefix": "O'Brien Flann", "title": []} +{"best": null, "clean": "Søren Kierkegaard", "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": "Éric Voegelin", "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": null, "title": []} +{"best": {"author": "Jane Q. Public", "confidence": 0.9, "evidence": "…by Jane Q. Public", "isbn": null, "method": "front_matter_copyright"}, "clean": null, "front_matter": [{"author": "Jane Q. Public", "confidence": 0.9, "evidence": "…by Jane Q. Public", "isbn": null, "method": "front_matter_copyright"}], "name_prefix": null, "title": []} +{"best": {"author": "Flann O'Brien", "confidence": 0.8, "evidence": "O'Brien, Flann,", "isbn": null, "method": "front_matter_loc"}, "clean": null, "front_matter": [{"author": "Flann O'Brien", "confidence": 0.8, "evidence": "O'Brien, Flann,", "isbn": null, "method": "front_matter_loc"}], "name_prefix": null, "title": []} +{"best": null, "clean": null, "front_matter": [], "name_prefix": null, "title": []} diff --git a/parity/oracle/author.py b/parity/oracle/author.py new file mode 100644 index 0000000..55590fb --- /dev/null +++ b/parity/oracle/author.py @@ -0,0 +1,55 @@ +"""Recover authors the way the Python pipeline recovers them. + +Reads one `{"title", "front_matter", "candidate"}` object per line and writes +what each of the author functions makes of it. + + CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \ + python3 oracle/author.py < cases/author/records.ndjson +""" + +import json +import os +import sys + +for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep): + if path: + sys.path.insert(0, path) + +from book_pipeline import authors # noqa: E402 + + +def as_dict(suggestion) -> dict | None: + if suggestion is None: + return None + return { + "author": suggestion.author, + "method": suggestion.method, + "confidence": suggestion.confidence, + "evidence": suggestion.evidence, + "isbn": suggestion.isbn, + } + + +def main() -> int: + out = sys.stdout + for line in sys.stdin.buffer.read().decode("utf-8").split("\n"): + if not line.strip(): + continue + case = json.loads(line) + title = case.get("title") or "" + front = case.get("front_matter") + candidate = case.get("candidate") + + record = { + "front_matter": [as_dict(s) for s in authors.mine_front_matter(front or "")], + "title": [as_dict(s) for s in authors.mine_title(title)], + "best": as_dict(authors.best_suggestion(title, front)), + "clean": authors._clean_name(candidate) if candidate is not None else None, + "name_prefix": authors._take_name_prefix(candidate) if candidate is not None else None, + } + out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index 73ff37f..3547ae6 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 12] = [ + let jobs: [Job; 13] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -32,6 +32,7 @@ fn main() -> ExitCode { ("booksniff", booksniff), ("bibclean", bibclean), ("slug", slug), + ("author", author), ]; for (name, job) in jobs { @@ -75,6 +76,18 @@ fn unescape() -> Written { Ok(vec![path.display().to_string()]) } +fn author() -> Written { + let dir = parity::golden("author"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/author/records.ndjson"); + let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let judgements = parity::run_oracle("author.py", &cases)?; + let path = dir.join("records.ndjson"); + std::fs::write(&path, judgements).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn slug() -> Written { let dir = parity::golden("slug"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/author.rs b/parity/tests/author.rs new file mode 100644 index 0000000..015b3bd --- /dev/null +++ b/parity/tests/author.rs @@ -0,0 +1,102 @@ +//! Author recovery must match the Python exactly. +//! +//! A recovered name is written back into a catalogue. Recovering a different +//! one — a publisher, a title fragment, a name with the initials in the wrong +//! place — puts a wrong fact where there was an honest gap, which is worse +//! than the gap. + +use serde_json::Value; + +fn records(path: &str) -> Vec { + let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file"); + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> { + record.get(name).and_then(Value::as_str) +} + +/// A suggestion as JSON, so the two sides compare field for field. +fn as_value(found: &bibclean::author::Suggestion) -> Value { + serde_json::json!({ + "author": found.author, + "method": found.method, + "confidence": found.confidence, + "evidence": found.evidence, + "isbn": found.isbn, + }) +} + +fn list(found: &[bibclean::author::Suggestion]) -> Value { + Value::Array(found.iter().map(as_value).collect()) +} + +#[test] +fn every_author_judgement_matches_python() { + let cases = records("cases/author/records.ndjson"); + let golden = records("golden/author/records.ndjson"); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + let mut wrong: Vec = Vec::new(); + for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() { + let title = field(case, "title").unwrap_or(""); + let front = field(case, "front_matter"); + let candidate = field(case, "candidate"); + + let mut differs = |what: &str, ours: String, theirs: &Value| { + wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}")); + }; + + let ours = list(&bibclean::author::from_front_matter(front.unwrap_or(""))); + if &ours != &theirs["front_matter"] { + differs("front_matter", ours.to_string(), &theirs["front_matter"]); + } + + let ours = list(&bibclean::author::from_title(title)); + if &ours != &theirs["title"] { + differs("title", ours.to_string(), &theirs["title"]); + } + + let ours = bibclean::author::best(title, front) + .as_ref() + .map_or(Value::Null, as_value); + if &ours != &theirs["best"] { + differs("best", ours.to_string(), &theirs["best"]); + } + + if let Some(candidate) = candidate { + let ours = bibclean::author::clean(candidate); + if ours.as_deref() != theirs["clean"].as_str() { + differs("clean", format!("{ours:?}"), &theirs["clean"]); + } + let ours = bibclean::author::name_prefix(candidate); + if Some(ours.as_str()) != theirs["name_prefix"].as_str() { + differs("name_prefix", ours, &theirs["name_prefix"]); + } + } + } + + assert!( + wrong.is_empty(), + "{} judgements differ:\n{}", + wrong.len(), + wrong + .iter() + .map(|line| line.chars().take(300).collect::()) + .collect::>() + .join("\n") + ); +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/author/records.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("author.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("author/records.ndjson")); +}