mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
bibclean: a name from the book's own pages, where the record has none
This commit is contained in:
funkpower
@@ -0,0 +1,30 @@
|
||||
{"title": "The Icon and the Axe", "front_matter": "THE ICON AND THE AXE\n\nCopyright © 1966 by James H. Billington\nAll rights reserved under International and Pan-American Copyright\nConventions. Published in the United States by Vintage Books.\nISBN 0-394-70846-1\n", "candidate": "James H. Billington"}
|
||||
{"title": "The Icon and the Axe", "front_matter": "Library of Congress Cataloging-in-Publication Data\n\nBillington, James H., 1929-\n The icon and the axe : an interpretive history of Russian culture /\n James H. Billington.\n p. cm.\n Includes bibliographical references and index.\n ISBN 978-0-394-70846-8\n", "candidate": "Billington, James H."}
|
||||
{"title": "The Icon and the Axe", "front_matter": "THE ICON AND THE AXE\n\nCopyright © 1966 by James H. Billington\nAll rights reserved under International and Pan-American Copyright\nConventions. Published in the United States by Vintage Books.\nISBN 0-394-70846-1\n\nLibrary of Congress Cataloging-in-Publication Data\n\nBillington, James H., 1929-\n The icon and the axe : an interpretive history of Russian culture /\n James H. Billington.\n p. cm.\n Includes bibliographical references and index.\n ISBN 978-0-394-70846-8\n", "candidate": null}
|
||||
{"title": "The Icon and the Axe", "front_matter": "Copyright © 2001 by Cambridge University Press\nAll rights reserved. No part of this publication may be reproduced.\n", "candidate": "Cambridge University Press"}
|
||||
{"title": "Tragedy and Hope by Carroll Quigley", "front_matter": null, "candidate": "Carroll Quigley"}
|
||||
{"title": "CarrollQuigley-Tragedy and Hope", "front_matter": null, "candidate": null}
|
||||
{"title": "CarrollQuigley Tragedy and Hope", "front_matter": null, "candidate": null}
|
||||
{"title": "An Ordinary Title", "front_matter": null, "candidate": null}
|
||||
{"title": "Tragedy and Hope by Carroll Quigley", "front_matter": "THE ICON AND THE AXE\n\nCopyright © 1966 by James H. Billington\nAll rights reserved under International and Pan-American Copyright\nConventions. Published in the United States by Vintage Books.\nISBN 0-394-70846-1\n", "candidate": null}
|
||||
{"title": "", "front_matter": "", "candidate": ""}
|
||||
{"title": "x", "front_matter": null, "candidate": "ARISTOTLE"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Barbara A Kerr"}
|
||||
{"title": "x", "front_matter": null, "candidate": "John Schuster and John Sutton"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Smith et al."}
|
||||
{"title": "x", "front_matter": null, "candidate": "Smith and others"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Smith, and others, 1999"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Ludwig von Mises"}
|
||||
{"title": "x", "front_matter": null, "candidate": "the icon and the axe"}
|
||||
{"title": "x", "front_matter": null, "candidate": "A Very Long Name With Far Too Many Words"}
|
||||
{"title": "x", "front_matter": null, "candidate": "James H. Billington Printed in the United States"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Smith Jr. II"}
|
||||
{"title": "x", "front_matter": null, "candidate": "O'Brien, Flann"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Søren Kierkegaard"}
|
||||
{"title": "x", "front_matter": null, "candidate": "James² Billington"}
|
||||
{"title": "x", "front_matter": null, "candidate": "Éric Voegelin"}
|
||||
{"title": "x", "front_matter": null, "candidate": " spaced out name "}
|
||||
{"title": "x", "front_matter": "Copyright © 1999 by Smith", "candidate": null}
|
||||
{"title": "x", "front_matter": "Copyright (c) 1999 by Jane Q. Public. All rights reserved", "candidate": null}
|
||||
{"title": "x", "front_matter": "Cataloging in Publication Data\nO'Brien, Flann, 1911-1966.\n At swim.", "candidate": null}
|
||||
{"title": "x", "front_matter": "ISBN 9780198237907 and nothing else", "candidate": null}
|
||||
@@ -0,0 +1,30 @@
|
||||
{"best": {"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, "clean": "James H. Billington", "front_matter": [{"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}], "name_prefix": "James H. Billington", "title": []}
|
||||
{"best": {"author": "James H. Billington", "confidence": 0.8, "evidence": "Billington, James H.", "isbn": "9780394708468", "method": "front_matter_loc"}, "clean": "Billington, James H.", "front_matter": [{"author": "James H. Billington", "confidence": 0.8, "evidence": "Billington, James H.", "isbn": "9780394708468", "method": "front_matter_loc"}], "name_prefix": "Billington James H.", "title": []}
|
||||
{"best": {"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, "clean": null, "front_matter": [{"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, {"author": "James H. Billington", "confidence": 0.8, "evidence": "Billington, James H.", "isbn": "0394708461", "method": "front_matter_loc"}], "name_prefix": null, "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "Cambridge", "title": []}
|
||||
{"best": {"author": "Carroll Quigley", "confidence": 0.7, "evidence": "by Carroll Quigley", "isbn": null, "method": "title_by"}, "clean": "Carroll Quigley", "front_matter": [], "name_prefix": "Carroll Quigley", "title": [{"author": "Carroll Quigley", "confidence": 0.7, "evidence": "by Carroll Quigley", "isbn": null, "method": "title_by"}]}
|
||||
{"best": {"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}, "clean": null, "front_matter": [], "name_prefix": null, "title": [{"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}]}
|
||||
{"best": {"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}, "clean": null, "front_matter": [], "name_prefix": null, "title": [{"author": "Carroll Quigley", "confidence": 0.5, "evidence": "CarrollQuigley", "isbn": null, "method": "title_camel"}]}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": null, "title": []}
|
||||
{"best": {"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}, "clean": null, "front_matter": [{"author": "James H. Billington", "confidence": 0.9, "evidence": "…by James H. Billington", "isbn": "0394708461", "method": "front_matter_copyright"}], "name_prefix": null, "title": [{"author": "Carroll Quigley", "confidence": 0.7, "evidence": "by Carroll Quigley", "isbn": null, "method": "title_by"}]}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": "Aristotle", "front_matter": [], "name_prefix": "ARISTOTLE", "title": []}
|
||||
{"best": null, "clean": "Barbara A. Kerr", "front_matter": [], "name_prefix": "Barbara", "title": []}
|
||||
{"best": null, "clean": "John Schuster & John Sutton", "front_matter": [], "name_prefix": "John Schuster", "title": []}
|
||||
{"best": null, "clean": "Smith", "front_matter": [], "name_prefix": "Smith", "title": []}
|
||||
{"best": null, "clean": "Smith", "front_matter": [], "name_prefix": "Smith", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "Smith", "title": []}
|
||||
{"best": null, "clean": "Ludwig von Mises", "front_matter": [], "name_prefix": "Ludwig von Mises", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "James H. Billington", "title": []}
|
||||
{"best": null, "clean": "Smith Jr. II", "front_matter": [], "name_prefix": "Smith Jr.", "title": []}
|
||||
{"best": null, "clean": "O'Brien, Flann", "front_matter": [], "name_prefix": "O'Brien Flann", "title": []}
|
||||
{"best": null, "clean": "Søren Kierkegaard", "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": "Éric Voegelin", "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": "", "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": null, "title": []}
|
||||
{"best": {"author": "Jane Q. Public", "confidence": 0.9, "evidence": "…by Jane Q. Public", "isbn": null, "method": "front_matter_copyright"}, "clean": null, "front_matter": [{"author": "Jane Q. Public", "confidence": 0.9, "evidence": "…by Jane Q. Public", "isbn": null, "method": "front_matter_copyright"}], "name_prefix": null, "title": []}
|
||||
{"best": {"author": "Flann O'Brien", "confidence": 0.8, "evidence": "O'Brien, Flann,", "isbn": null, "method": "front_matter_loc"}, "clean": null, "front_matter": [{"author": "Flann O'Brien", "confidence": 0.8, "evidence": "O'Brien, Flann,", "isbn": null, "method": "front_matter_loc"}], "name_prefix": null, "title": []}
|
||||
{"best": null, "clean": null, "front_matter": [], "name_prefix": null, "title": []}
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Recover authors the way the Python pipeline recovers them.
|
||||
|
||||
Reads one `{"title", "front_matter", "candidate"}` object per line and writes
|
||||
what each of the author functions makes of it.
|
||||
|
||||
CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
|
||||
python3 oracle/author.py < cases/author/records.ndjson
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
|
||||
if path:
|
||||
sys.path.insert(0, path)
|
||||
|
||||
from book_pipeline import authors # noqa: E402
|
||||
|
||||
|
||||
def as_dict(suggestion) -> dict | None:
|
||||
if suggestion is None:
|
||||
return None
|
||||
return {
|
||||
"author": suggestion.author,
|
||||
"method": suggestion.method,
|
||||
"confidence": suggestion.confidence,
|
||||
"evidence": suggestion.evidence,
|
||||
"isbn": suggestion.isbn,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
out = sys.stdout
|
||||
for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
case = json.loads(line)
|
||||
title = case.get("title") or ""
|
||||
front = case.get("front_matter")
|
||||
candidate = case.get("candidate")
|
||||
|
||||
record = {
|
||||
"front_matter": [as_dict(s) for s in authors.mine_front_matter(front or "")],
|
||||
"title": [as_dict(s) for s in authors.mine_title(title)],
|
||||
"best": as_dict(authors.best_suggestion(title, front)),
|
||||
"clean": authors._clean_name(candidate) if candidate is not None else None,
|
||||
"name_prefix": authors._take_name_prefix(candidate) if candidate is not None else None,
|
||||
}
|
||||
out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+14
-1
@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
|
||||
type Job = (&'static str, fn() -> Written);
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let jobs: [Job; 12] = [
|
||||
let jobs: [Job; 13] = [
|
||||
("tokens", tokens),
|
||||
("unescape", unescape),
|
||||
("chunkmd", chunkmd),
|
||||
@@ -32,6 +32,7 @@ fn main() -> ExitCode {
|
||||
("booksniff", booksniff),
|
||||
("bibclean", bibclean),
|
||||
("slug", slug),
|
||||
("author", author),
|
||||
];
|
||||
|
||||
for (name, job) in jobs {
|
||||
@@ -75,6 +76,18 @@ fn unescape() -> Written {
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn author() -> Written {
|
||||
let dir = parity::golden("author");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
let cases = parity::dir().join("cases/author/records.ndjson");
|
||||
let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
|
||||
let judgements = parity::run_oracle("author.py", &cases)?;
|
||||
let path = dir.join("records.ndjson");
|
||||
std::fs::write(&path, judgements).map_err(|e| e.to_string())?;
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn slug() -> Written {
|
||||
let dir = parity::golden("slug");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//! Author recovery must match the Python exactly.
|
||||
//!
|
||||
//! A recovered name is written back into a catalogue. Recovering a different
|
||||
//! one — a publisher, a title fragment, a name with the initials in the wrong
|
||||
//! place — puts a wrong fact where there was an honest gap, which is worse
|
||||
//! than the gap.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
fn records(path: &str) -> Vec<Value> {
|
||||
let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file");
|
||||
text.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| serde_json::from_str(line).expect("valid JSON"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> {
|
||||
record.get(name).and_then(Value::as_str)
|
||||
}
|
||||
|
||||
/// A suggestion as JSON, so the two sides compare field for field.
|
||||
fn as_value(found: &bibclean::author::Suggestion) -> Value {
|
||||
serde_json::json!({
|
||||
"author": found.author,
|
||||
"method": found.method,
|
||||
"confidence": found.confidence,
|
||||
"evidence": found.evidence,
|
||||
"isbn": found.isbn,
|
||||
})
|
||||
}
|
||||
|
||||
fn list(found: &[bibclean::author::Suggestion]) -> Value {
|
||||
Value::Array(found.iter().map(as_value).collect())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_author_judgement_matches_python() {
|
||||
let cases = records("cases/author/records.ndjson");
|
||||
let golden = records("golden/author/records.ndjson");
|
||||
assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
|
||||
|
||||
let mut wrong: Vec<String> = Vec::new();
|
||||
for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() {
|
||||
let title = field(case, "title").unwrap_or("");
|
||||
let front = field(case, "front_matter");
|
||||
let candidate = field(case, "candidate");
|
||||
|
||||
let mut differs = |what: &str, ours: String, theirs: &Value| {
|
||||
wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}"));
|
||||
};
|
||||
|
||||
let ours = list(&bibclean::author::from_front_matter(front.unwrap_or("")));
|
||||
if &ours != &theirs["front_matter"] {
|
||||
differs("front_matter", ours.to_string(), &theirs["front_matter"]);
|
||||
}
|
||||
|
||||
let ours = list(&bibclean::author::from_title(title));
|
||||
if &ours != &theirs["title"] {
|
||||
differs("title", ours.to_string(), &theirs["title"]);
|
||||
}
|
||||
|
||||
let ours = bibclean::author::best(title, front)
|
||||
.as_ref()
|
||||
.map_or(Value::Null, as_value);
|
||||
if &ours != &theirs["best"] {
|
||||
differs("best", ours.to_string(), &theirs["best"]);
|
||||
}
|
||||
|
||||
if let Some(candidate) = candidate {
|
||||
let ours = bibclean::author::clean(candidate);
|
||||
if ours.as_deref() != theirs["clean"].as_str() {
|
||||
differs("clean", format!("{ours:?}"), &theirs["clean"]);
|
||||
}
|
||||
let ours = bibclean::author::name_prefix(candidate);
|
||||
if Some(ours.as_str()) != theirs["name_prefix"].as_str() {
|
||||
differs("name_prefix", ours, &theirs["name_prefix"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
wrong.is_empty(),
|
||||
"{} judgements differ:\n{}",
|
||||
wrong.len(),
|
||||
wrong
|
||||
.iter()
|
||||
.map(|line| line.chars().take(300).collect::<String>())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Catches a golden file drifting away from a Python that has moved on.
|
||||
#[test]
|
||||
#[cfg(feature = "python")]
|
||||
fn the_golden_file_still_matches_live_python() {
|
||||
let cases = std::fs::read_to_string(parity::dir().join("cases/author/records.ndjson"))
|
||||
.expect("a readable case file");
|
||||
let live = parity::run_oracle("author.py", &cases).expect("the oracle runs");
|
||||
assert_eq!(live, parity::read_golden_text("author/records.ndjson"));
|
||||
}
|
||||
Reference in New Issue
Block a user