bibclean: a name from the book's own pages, where the record has none

crates/bibclean/src/author.rs

@@ -0,0 +1,508 @@
+//! Recovering an author a record does not have.
+//!
+//! All of it local: what the book says about itself on its own first pages,
+//! and what its title says. No lookup, no network, and nothing applied — a
+//! name is proposed with the evidence that produced it and how far it is
+//! worth trusting.
+
+use std::sync::LazyLock;
+
+use fancy_regex::Regex as Boundaried;
+use regex::Regex;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use textkit::pattern::{SPACE, WORD_END, WORD_START};
+
+fn pattern(source: &str) -> Regex {
+    Regex::new(source).expect("a valid pattern")
+}
+
+/// Words that mark a candidate as title or publisher bleed rather than a
+/// person. Without them the looser heuristics propose the imprint.
+static NOT_A_PERSON: LazyLock<Boundaried> = LazyLock::new(|| {
+    Boundaried::new(&format!(
+        r"(?i){WORD_START}(press|edition|introduction|university|volume|reprint|copyright|published|printed|rights|reserved|library|congress|isbn|inc|ltd|company|regents|trustees|foundation|institute|society|association|board|council|committee|department|ministry|editors?|trust|publishing|publishers|books|publications|sons|llc|gmbh|verlag|all|limited){WORD_END}"
+    ))
+    .expect("a valid pattern")
+});
+
+/// One token of a name: a capitalised word, a bare initial, or a nobiliary
+/// particle that appears in the middle of one.
+static NAME_TOKEN: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(
+        r"^(?:[A-Z][A-Za-z’'\-]+\.?|[A-Z]\.|(?:von|van|de|den|der|del|della|di|da|la|le|du|ter|ten))$",
+    )
+});
+
+/// Where several authors are joined.
+static AUTHOR_SEPARATOR: LazyLock<Regex> =
+    LazyLock::new(|| pattern(&format!(r"(?i){SPACE}+(?:and|&){SPACE}+")));
+
+/// "et al", "and others", and everything after.
+static ET_AL: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(&format!(
+        r"(?i)[{SPACE},]+(?:et{SPACE}+al\.?|and{SPACE}+others).*$"
+    ))
+});
+
+static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(&format!(r"{SPACE}+")));
+
+/// An ISBN announced on the copyright page.
+static ISBN: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(&format!(
+        r"ISBN[:{SPACE}]*((?:97[89][{SPACE}-]?)?(?:[0-9][{SPACE}-]?){{9}}[0-9Xx])"
+    ))
+});
+
+/// The strongest thing a book says about who wrote it.
+static COPYRIGHT: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(&format!(
+        r"(?i)Copyright{SPACE}*(?:©|\(c\))?{SPACE}*\d{{4}}{SPACE}+by{SPACE}+(.{{2,90}})"
+    ))
+});
+
+/// The cataloguing block a national library prints on the copyright page.
+static CATALOGUING: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(&format!(
+        r"(?i)Cataloging.{{0,40}}Publication(?:{SPACE}+Data)?"
+    ))
+});
+
+/// Its main entry, which is the author, written surname first.
+static MAIN_ENTRY: LazyLock<Regex> =
+    LazyLock::new(|| pattern(r"([A-Z][A-Za-z'’-]+),\s+([A-Z][A-Za-z.ّ'’ -]{1,40}?)[,.]"));
+
+/// "… by Author Name" at the end of a title.
+static TITLE_BY: LazyLock<Boundaried> = LazyLock::new(|| {
+    Boundaried::new(&format!(
+        r"(?i){WORD_START}by{SPACE}+([A-Z][A-Za-z.'’ -]{{2,60}})$"
+    ))
+    .expect("a valid pattern")
+});
+
+/// A title that opens with the author's name run together.
+static TITLE_CAMEL: LazyLock<Regex> = LazyLock::new(|| pattern(r"^([A-Z][a-z]+[A-Z][a-z]+)[-\s]"));
+
+static CAMEL_PARTS: LazyLock<Regex> = LazyLock::new(|| pattern(r"[A-Z][a-z]+"));
+
+/// Roman numerals that are a suffix on a name rather than shouting.
+const ROMAN_SUFFIXES: &[&str] = &["II", "III", "IV", "V", "VI"];
+
+/// Characters trimmed from the ends of a candidate.
+const EDGES: &[char] = &[' ', '.', ',', ';', ':', '—', '-', '\t', '\n'];
+
+/// How far into a book's front matter to look.
+pub const FRONT_MATTER_CHARS: usize = 8000;
+
+/// Digits that are digits to Python and are not decimal.
+///
+/// A candidate name containing any digit is refused, and "any digit" there
+/// means superscripts and circled numbers too — the ones a scanned page picks
+/// up from a footnote marker.
+const DIGIT_RANGES: &[(char, char)] = &[
+    ('\u{b2}', '\u{b3}'),
+    ('\u{b9}', '\u{b9}'),
+    ('\u{1369}', '\u{1371}'),
+    ('\u{19da}', '\u{19da}'),
+    ('\u{2070}', '\u{2070}'),
+    ('\u{2074}', '\u{2079}'),
+    ('\u{2080}', '\u{2089}'),
+    ('\u{2460}', '\u{2468}'),
+    ('\u{2474}', '\u{247c}'),
+    ('\u{2488}', '\u{2490}'),
+    ('\u{24ea}', '\u{24ea}'),
+    ('\u{24f5}', '\u{24fd}'),
+    ('\u{24ff}', '\u{24ff}'),
+    ('\u{2776}', '\u{277e}'),
+    ('\u{2780}', '\u{2788}'),
+    ('\u{278a}', '\u{2792}'),
+    ('\u{10a40}', '\u{10a43}'),
+    ('\u{10e60}', '\u{10e68}'),
+    ('\u{11052}', '\u{1105a}'),
+    ('\u{1f100}', '\u{1f10a}'),
+];
+
+/// A name, where it came from, and how far it is worth trusting.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Suggestion {
+    /// The name, normalised.
+    pub author: String,
+    /// Which signal produced it.
+    pub method: String,
+    /// How far to trust it, from zero to one.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub confidence: f64,
+    /// The text it was read from, so a person can check it.
+    pub evidence: String,
+    /// An ISBN found alongside it, if there was one.
+    pub isbn: Option<String>,
+}
+
+/// The best name a record's own text and title can offer, if any.
+///
+/// Candidates are gathered cheapest and most trustworthy first, and the most
+/// confident wins. An ISBN found in the front matter is carried onto the
+/// winner even when the winner came from the title, because the two are
+/// facts about the same book.
+pub fn best(title: &str, front_matter: Option<&str>) -> Option<Suggestion> {
+    let mut candidates: Vec<Suggestion> = Vec::new();
+    if let Some(text) = front_matter {
+        candidates.extend(from_front_matter(text));
+    }
+    candidates.extend(from_title(title));
+
+    let isbn = candidates
+        .iter()
+        .find_map(|candidate| candidate.isbn.clone());
+    // The first of the most confident, which is the earliest and therefore
+    // the cheapest signal that scored highest.
+    let mut winner = candidates.into_iter().reduce(|best, candidate| {
+        match candidate.confidence > best.confidence {
+            true => candidate,
+            false => best,
+        }
+    })?;
+    if winner.isbn.is_none() {
+        winner.isbn = isbn;
+    }
+    Some(winner)
+}
+
+/// Names a book's own front matter offers.
+pub fn from_front_matter(text: &str) -> Vec<Suggestion> {
+    let isbn = ISBN
+        .captures(text)
+        .and_then(|found| normalise_isbn(&found[1]));
+    let mut out = Vec::new();
+
+    // Strongest: the book saying who holds its copyright.
+    if let Some(found) = COPYRIGHT.captures(text)
+        && let Some(name) = clean(&name_prefix(&found[1]))
+        && name.split_whitespace().count() >= 2
+    {
+        // Two words at least: a lone surname is not an attribution.
+        out.push(Suggestion {
+            evidence: format!("…by {name}"),
+            author: name,
+            method: "front_matter_copyright".into(),
+            confidence: 0.9,
+            isbn: isbn.clone(),
+        });
+    }
+
+    // The cataloguing block's main entry, written surname first. The
+    // boilerplate is consumed first so the window starts at the name rather
+    // than at the last word of the heading.
+    if let Some(heading) = CATALOGUING.find(text) {
+        let from = heading.end();
+        let window: String = text[from..].chars().take(300).collect();
+        if let Some(found) = MAIN_ENTRY.captures(&window)
+            && let Some(name) = clean(&format!("{} {}", &found[2], &found[1]))
+        {
+            out.push(Suggestion {
+                author: name,
+                method: "front_matter_loc".into(),
+                confidence: 0.8,
+                evidence: found[0]
+                    .trim_matches(textkit::is_space)
+                    .chars()
+                    .take(200)
+                    .collect(),
+                isbn,
+            });
+        }
+    }
+    out
+}
+
+/// Names the title string itself offers.
+pub fn from_title(title: &str) -> Vec<Suggestion> {
+    let mut out = Vec::new();
+
+    if let Ok(Some(found)) = TITLE_BY.captures(title)
+        && let Some(name) = clean(&found[1])
+    {
+        out.push(Suggestion {
+            author: name,
+            method: "title_by".into(),
+            confidence: 0.7,
+            evidence: found[0].trim_matches(textkit::is_space).to_string(),
+            isbn: None,
+        });
+    }
+
+    if let Some(found) = TITLE_CAMEL.captures(title)
+        && let Some(split) = split_camel(&found[1])
+    {
+        out.push(Suggestion {
+            author: split,
+            method: "title_camel".into(),
+            confidence: 0.5,
+            evidence: found[1].to_string(),
+            isbn: None,
+        });
+    }
+    out
+}
+
+/// The leading run of name tokens, stopping at prose.
+///
+/// "James H. Billington Printed in the…" is "James H. Billington". The run is
+/// capped so that a sentence which happens to start with capitals cannot be
+/// swallowed whole, and a word ending in a full stop that is not an initial
+/// ends the name — it is the end of a sentence.
+pub fn name_prefix(text: &str) -> String {
+    let mut tokens: Vec<&str> = Vec::new();
+    for raw in text.split_whitespace() {
+        let token = raw.trim_matches([',', ';', ':', '(', ')']);
+        let core = token.trim_matches('.');
+        if core.is_empty() || NOT_A_PERSON.is_match(core).unwrap_or(false) {
+            break;
+        }
+        if !NAME_TOKEN.is_match(token) {
+            break;
+        }
+        tokens.push(token);
+        if token.ends_with('.') && core.chars().count() > 1 {
+            break;
+        }
+        if tokens.len() >= 5 {
+            break;
+        }
+    }
+    tokens.join(" ")
+}
+
+/// Normalise a candidate, or refuse it if it does not read as a person.
+pub fn clean(raw: &str) -> Option<String> {
+    let name = SPACE_RUN.replace_all(raw, " ");
+    let name = name.trim_matches(|c: char| EDGES.contains(&c));
+    if name.is_empty() || name.chars().any(is_digit) {
+        return None;
+    }
+    if NOT_A_PERSON.is_match(name).unwrap_or(false) {
+        return None;
+    }
+    normalise(name)
+}
+
+/// Normalise a name, keeping co-authors joined.
+///
+/// `None` when any one author fails the shape a person's name has: one to
+/// five words, capitalised, and not opening with an article.
+pub fn normalise(raw: &str) -> Option<String> {
+    let name = ET_AL.replace(raw, "");
+    let name = name.trim_matches(textkit::is_space);
+
+    let authors: Vec<String> = AUTHOR_SEPARATOR
+        .split(name)
+        .map(|segment| segment.trim_matches(textkit::is_space))
+        .filter(|segment| !segment.is_empty())
+        .map(normalise_tokens)
+        .collect();
+    if authors.is_empty() {
+        return None;
+    }
+
+    for author in &authors {
+        let words: Vec<&str> = author.split_whitespace().collect();
+        if author.is_empty() || !(1..=5).contains(&words.len()) {
+            return None;
+        }
+        let first = author.chars().next()?;
+        if !first.is_uppercase() || words[0].to_lowercase() == "the" {
+            return None;
+        }
+    }
+    Some(authors.join(" & "))
+}
+
+/// Fix shouting and bare initials within one name.
+fn normalise_tokens(segment: &str) -> String {
+    segment
+        .split_whitespace()
+        .map(|token| {
+            let core = token.trim_matches('.');
+            let shouting = core.chars().all(char::is_alphabetic)
+                && !core.is_empty()
+                && core.chars().all(|c| !c.is_lowercase())
+                && core.chars().any(char::is_uppercase)
+                && core.chars().count() > 1
+                && !ROMAN_SUFFIXES.contains(&core);
+            if shouting {
+                let mut fixed = capitalise(core);
+                if token.ends_with('.') {
+                    fixed.push('.');
+                }
+                return fixed;
+            }
+            if core.chars().count() == 1
+                && token.chars().count() == 1
+                && core.chars().all(|c| c.is_alphabetic() && c.is_uppercase())
+            {
+                return format!("{token}.");
+            }
+            token.to_string()
+        })
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
+fn capitalise(word: &str) -> String {
+    let mut chars = word.chars();
+    match chars.next() {
+        None => String::new(),
+        Some(first) => first.to_uppercase().collect::<String>() + &chars.as_str().to_lowercase(),
+    }
+}
+
+/// A run-together name split back apart, when it is two or three parts.
+fn split_camel(token: &str) -> Option<String> {
+    let parts: Vec<&str> = CAMEL_PARTS
+        .find_iter(token)
+        .map(|found| found.as_str())
+        .collect();
+    let rejoined: String = parts.concat();
+    ((2..=3).contains(&parts.len()) && rejoined == token).then(|| parts.join(" "))
+}
+
+fn normalise_isbn(raw: &str) -> Option<String> {
+    let digits = crate::isbn::digits(raw);
+    matches!(digits.len(), 10 | 13).then(|| digits.to_uppercase())
+}
+
+/// Whether a character is a digit, by Python's definition — which counts the
+/// superscripts and the circled numbers that a decimal test does not.
+fn is_digit(c: char) -> bool {
+    c.to_digit(10).is_some()
+        || DIGIT_RANGES
+            .iter()
+            .any(|(from, to)| (*from..=*to).contains(&c))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_copyright_line_names_the_author() {
+        let text = "Copyright © 1999 by James H. Billington\nPrinted in the United States";
+        let found = from_front_matter(text);
+        assert_eq!(found[0].author, "James H. Billington");
+        assert_eq!(found[0].method, "front_matter_copyright");
+        assert_eq!(found[0].confidence, 0.9);
+    }
+
+    #[test]
+    fn a_cataloguing_block_names_the_author_surname_first() {
+        let text = "Library of Congress Cataloging-in-Publication Data\n\
+                    Billington, James H., 1929-\n  The icon and the axe.";
+        let found = from_front_matter(text);
+        assert_eq!(found[0].author, "James H. Billington");
+        assert_eq!(found[0].method, "front_matter_loc");
+    }
+
+    #[test]
+    fn an_isbn_on_the_same_page_comes_with_the_name() {
+        let text = "Copyright © 1999 by James Billington\nISBN 978-0-19-823790-7";
+        assert_eq!(
+            from_front_matter(text)[0].isbn.as_deref(),
+            Some("9780198237907")
+        );
+    }
+
+    #[test]
+    fn a_title_that_names_its_author_offers_it() {
+        let found = from_title("The Icon and the Axe by James Billington");
+        assert_eq!(found[0].author, "James Billington");
+        assert_eq!(found[0].method, "title_by");
+    }
+
+    #[test]
+    fn a_run_together_name_at_the_front_of_a_title_is_split() {
+        let found = from_title("CarrollQuigley Tragedy and Hope");
+        assert_eq!(found[0].author, "Carroll Quigley");
+        assert_eq!(found[0].confidence, 0.5);
+    }
+
+    #[test]
+    fn the_most_confident_candidate_wins_and_keeps_the_isbn() {
+        let front = "Copyright © 1999 by James Billington\nISBN 0198237901";
+        let found = best("Tragedy and Hope by Carroll Quigley", Some(front)).expect("a name");
+        assert_eq!(found.author, "James Billington");
+        assert_eq!(found.isbn.as_deref(), Some("0198237901"));
+    }
+
+    #[test]
+    fn an_isbn_alone_is_not_carried_anywhere() {
+        // The ISBN rides along with a name the front matter produced. Front
+        // matter that names nobody produces no candidate for it to ride on.
+        let front = "ISBN 0198237901\nno attribution here";
+        let found = best("Tragedy and Hope by Carroll Quigley", Some(front)).expect("a name");
+        assert_eq!(found.method, "title_by");
+        assert_eq!(found.isbn, None);
+    }
+
+    #[test]
+    fn a_publisher_is_not_a_person() {
+        assert_eq!(clean("Oxford University Press"), None);
+        assert_eq!(clean("The Regents of the University"), None);
+        assert!(from_front_matter("Copyright © 1999 by Cambridge University Press").is_empty());
+    }
+
+    #[test]
+    fn a_name_with_a_number_in_it_is_refused() {
+        assert_eq!(clean("James Billington 1929"), None);
+        assert_eq!(clean("James\u{b2} Billington"), None);
+    }
+
+    #[test]
+    fn shouting_and_bare_initials_are_fixed() {
+        assert_eq!(clean("ARISTOTLE").as_deref(), Some("Aristotle"));
+        assert_eq!(clean("Barbara A Kerr").as_deref(), Some("Barbara A. Kerr"));
+        assert_eq!(clean("Smith Jr. II").as_deref(), Some("Smith Jr. II"));
+    }
+
+    #[test]
+    fn co_authors_are_kept_and_joined() {
+        assert_eq!(
+            clean("John Schuster and John Sutton").as_deref(),
+            Some("John Schuster & John Sutton")
+        );
+    }
+
+    #[test]
+    fn everything_after_et_al_is_dropped() {
+        assert_eq!(clean("Smith et al.").as_deref(), Some("Smith"));
+        assert_eq!(clean("Smith and others").as_deref(), Some("Smith"));
+        // Refused before the trailing text is ever reached: a candidate with
+        // a digit in it is not a name.
+        assert_eq!(clean("Smith, and others, 1999"), None);
+    }
+
+    #[test]
+    fn something_that_is_not_a_name_is_refused() {
+        assert_eq!(clean(""), None);
+        assert_eq!(clean("the icon and the axe"), None);
+        assert_eq!(
+            clean("A Very Long Name With Far Too Many Words In It"),
+            None
+        );
+    }
+
+    #[test]
+    fn a_name_run_stops_at_prose() {
+        assert_eq!(
+            name_prefix("James H. Billington Printed in the United States"),
+            "James H. Billington"
+        );
+        assert_eq!(name_prefix("Ludwig von Mises wrote"), "Ludwig von Mises");
+    }
+
+    #[test]
+    fn nothing_to_go_on_offers_nothing() {
+        assert_eq!(best("An Ordinary Title", None), None);
+        assert_eq!(best("", Some("")), None);
+    }
+}

crates/bibclean/src/lib.rs

@@ -11,6 +11,7 @@
 //! which is the difference between a tool that cleans a catalogue and one
 //! that damages it in a new way.
 
+pub mod author;
 mod entities4;
 pub mod fix;
 pub mod isbn;
@@ -49,6 +50,17 @@ pub struct Book {
     /// A number that makes a slug unique. Needed to judge or regenerate one.
     #[serde(default)]
     pub id: Option<i64>,
+    /// The opening pages of the work, if the caller has them.
+    ///
+    /// A book says who wrote it on its own copyright page, which is the
+    /// cheapest and most trustworthy answer there is for a record whose
+    /// author is missing.
+    ///
+    /// Read but never written back: it is evidence handed in, not part of
+    /// the record, and several thousand characters of it on every line would
+    /// bury everything else.
+    #[serde(default, skip_serializing)]
+    pub front_matter: Option<String>,
     /// Everything else the record carried.
     #[serde(flatten)]
     #[schemars(with = "Map<String, Value>")]
@@ -90,6 +102,12 @@ pub struct Cleaned {
     pub severity: u32,
     /// What was changed, and why.
     pub changes: Vec<Change>,
+    /// A name for a record that has none, with the evidence for it.
+    ///
+    /// Proposed rather than applied, because the evidence differs in strength
+    /// and a wrong name is worse than an honest gap.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub author_proposal: Option<author::Suggestion>,
     /// What should happen to the record's slug, when it has one to judge.
     ///
     /// Named apart from the record's own `slug`, which is carried through
@@ -172,10 +190,24 @@ pub fn clean(book: &Book) -> Cleaned {
         _ => None,
     };
 
+    // Only for a record that has no author to overwrite. A name recovered
+    // from the text is evidence, not an improvement on one somebody entered.
+    let author_proposal = reasons
+        .iter()
+        .any(|reason| reason == "unknown_author" || reason == "placeholder_author")
+        .then(|| {
+            author::best(
+                book.title.as_deref().unwrap_or(""),
+                book.front_matter.as_deref(),
+            )
+        })
+        .flatten();
+
     Cleaned {
         severity: reasons::severity(&reasons),
         reasons,
         changes,
+        author_proposal,
         slug_proposal: proposal,
         book: cleaned,
     }
@@ -194,6 +226,7 @@ mod tests {
             path: None,
             slug: None,
             id: None,
+            front_matter: None,
             extra: Map::new(),
         }
     }
@@ -247,6 +280,26 @@ mod tests {
         assert_eq!(clean(&book).book.isbn, None);
     }
 
+    #[test]
+    fn a_record_with_no_author_gets_one_proposed_from_its_own_pages() {
+        let mut book = book("The Icon and the Axe", "Unknown");
+        book.front_matter = Some("Copyright © 1966 by James H. Billington".into());
+
+        let cleaned = clean(&book);
+        let proposal = cleaned.author_proposal.expect("a proposal");
+        assert_eq!(proposal.author, "James H. Billington");
+        assert_eq!(proposal.method, "front_matter_copyright");
+        // Proposed, never applied.
+        assert_eq!(cleaned.book.author.as_deref(), Some("Unknown"));
+    }
+
+    #[test]
+    fn a_record_that_has_an_author_is_not_second_guessed() {
+        let mut book = book("The Icon and the Axe", "Billington, James H.");
+        book.front_matter = Some("Copyright © 1966 by Somebody Else".into());
+        assert!(clean(&book).author_proposal.is_none());
+    }
+
     #[test]
     fn a_slug_is_judged_when_there_is_one_to_judge() {
         let mut book = book("Weaponizing Anthropology", "Laura Nader");

crates/bibclean/src/main.rs

@@ -9,10 +9,10 @@ use corpus_core::{exit, ndjson};
 
 /// Audit catalogue records and repair what can be repaired without a lookup.
 ///
-/// Reads one JSON record per line — `title`, `author`, `isbn` and `path` are
-/// read, everything else is carried through untouched — and writes each one
-/// back with the faults it has, how much they weigh, and a line for every
-/// change made and why.
+/// Reads one JSON record per line — `title`, `author`, `isbn`, `path`,
+/// `slug`, `id` and `front_matter` are read, everything else is carried
+/// through untouched — and writes each one back with the faults it has, how
+/// much they weigh, and a line for every change made and why.
 ///
 /// Nothing is rewritten silently. A run can be read before it is agreed to,
 /// which is the difference between cleaning a catalogue and damaging it in a
@@ -45,7 +45,9 @@ struct Args {
 /// should be replaced does, even when nothing else is wrong, because the
 /// identifier is the part other things have written down.
 fn needs_attention(cleaned: &bibclean::Cleaned) -> bool {
-    bibclean::reasons::worth_fixing(&cleaned.reasons) || cleaned.slug_proposal.is_some()
+    bibclean::reasons::worth_fixing(&cleaned.reasons)
+        || cleaned.slug_proposal.is_some()
+        || cleaned.author_proposal.is_some()
 }
 
 fn main() -> ExitCode {