diff --git a/README.md b/README.md index a3040a7..fb6224e 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Each tool installs by name, so you can take one without taking the rest. | [`booksniff`](crates/booksniff/README.md) | What a file actually is, whatever its name claims | | [`pdftriage`](crates/pdftriage/README.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed | | [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change | +| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary | | [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out | | [`chunkmd`](crates/chunkmd/README.md) | Markdown into passages that carry their page, chapter and role | | [`cuenorm`](crates/cuenorm/README.md) | Any subtitle format in, one normalised cue stream out | diff --git a/crates/bibclean/README.md b/crates/bibclean/README.md index f4cb194..62016e1 100644 --- a/crates/bibclean/README.md +++ b/crates/bibclean/README.md @@ -2,6 +2,10 @@ Catalogue records audited and repaired, with a reason for every change. +Two commands: `bibclean` for the records, and +[`vocabmerge`](#vocabmerge--the-other-command) for the headings they are filed +under. + Part of [catena](../../README.md). ## What it does @@ -35,7 +39,7 @@ cargo install --git https://git.pumpin.online/funkpower/catena bibclean ## Example -Five records, four of which have something wrong with them: +Five records in, four of them wanting attention: ``` $ bibclean --flagged books.ndjson @@ -200,6 +204,90 @@ an author field is wrong while "Oxford, Michael" is a person. Everything else is named and left alone, because the fix needs a lookup and a wrong guess in a catalogue is harder to find than a missing value. +## vocabmerge — the other command + +A catalogue's headings accumulate spellings. The same person is entered +surname-first once and given-name-first twice; a publisher appears with and +without its legal suffix; a subject grows a stray bracket. Each spelling is a +separate heading, so the works under one are invisible from the others. + +`vocabmerge` reads a list of headings and says what should happen to them: + +``` +$ vocabmerge --kind author authors.ndjson +{"kind":"author","action":"merge","keep":"Fyodor Dostoevsky","drop":"Dostoevsky, Fyodor","keep_id":3,"drop_id":1,"keep_count":11,"drop_count":3,"method":"last_first","confidence":0.9,"evidence":"3 books as 'Dostoevsky, Fyodor' → 'Fyodor Dostoevsky' (11)"} +{"kind":"author","action":"merge","keep":"Fyodor Dostoevsky","drop":"Dostoevsky| Fyodor","keep_id":3,"drop_id":2,"keep_count":11,"drop_count":1,"method":"last_first","confidence":0.9,"evidence":"1 books as 'Dostoevsky| Fyodor' → 'Fyodor Dostoevsky' (11)"} +``` + +Subjects get the same treatment, plus the headings that are not subjects at +all: + +``` +$ vocabmerge --kind tag tags.ndjson +{"kind":"tag","action":"merge","keep":"Philosophy","drop":"[Philosophy]",…,"method":"tag_norm","confidence":0.9,…} +{"kind":"tag","action":"merge","keep":"Philosophy","drop":"Philosophy.",…,"method":"tag_norm","confidence":0.9,…} +{"kind":"tag","action":"delete","keep":null,"drop":"z-lib.org",…,"method":"junk_tag:format-or-watermark","confidence":0.92,"evidence":"format-or-watermark: 'z-lib.org' on 300 books"} +{"kind":"tag","action":"delete","keep":null,"drop":"9780198237907",…,"method":"junk_tag:isbn","confidence":0.92,"evidence":"isbn: '9780198237907' on 1 books"} +``` + +`18th Century` and `Philosophy` are left alone: the deletion rules are +high-precision on purpose, because a real subject swept up by one is a shelf +nobody can find again. + +### Input + +One heading per line — an `id`, a `name`, and the `count` of works under it: + +```json +{"id":1,"name":"Dostoevsky, Fyodor","count":3} +``` + +### Options + +| Option | Effect | +|---|---| +| `--kind ` | What kind of heading the list holds | +| `--rules ` | Vocabulary specific to this collection | + +The rules file is JSON, and today holds one thing: + +```json +{"device_prefixes": ["alexskobo"]} +``` + +An e-reader stamps its own name onto every book it touches. The generic device +names are built in; one named after its owner is a fact about a single +collection, so it is supplied rather than compiled in. + +### Exit codes + +| Code | Meaning | +|---|---| +| `0` | Nothing to propose | +| `1` | Something to propose | +| `64` | The command line was wrong | +| `65` | A heading could not be read | +| `74` | The input could not be read | + +### How headings are grouped + +| Kind | Joined when | +|---|---| +| `author` | They normalise to the same string, **or** they are the same words in a different order and both read as a person's name | +| `publisher` | They are the same once the legal suffixes and the words `press`, `books`, `university` and the like are removed | +| `tag` | They normalise to the same string, brackets and trailing stops ignored | + +Author groups join both rules at once, deliberately. Otherwise two +surname-first spellings merge onto each other and neither ever reaches the +given-name-first heading that most of the works are actually under. + +The heading kept is the one with the most works, then the cleaner spelling of +it — shouting, a trailing separator, a legal suffix and a leading bracket all +count against a heading, and spaced initials count for one. + +A respelling is trusted further than an inversion: `strict_norm` at 0.95, +`last_first` at 0.9, a publisher family at 0.8, a junk subject at 0.92. + ## As a library ```rust @@ -213,7 +301,8 @@ if bibclean::reasons::worth_fixing(&cleaned.reasons) { The pieces are usable on their own: `bibclean::isbn::valid`, `bibclean::isbn::from_filename`, `bibclean::fix::title_case`, -`bibclean::matching::same_work`, `bibclean::slug::slug_for`. +`bibclean::matching::same_work`, `bibclean::slug::slug_for`, +`bibclean::vocab::cluster`. `same_work` is worth knowing about separately. It compares vocabulary rather than characters, because character similarity is not usable for titles: "The diff --git a/crates/bibclean/src/bin/vocabmerge.rs b/crates/bibclean/src/bin/vocabmerge.rs new file mode 100644 index 0000000..d7b37f5 --- /dev/null +++ b/crates/bibclean/src/bin/vocabmerge.rs @@ -0,0 +1,115 @@ +//! Finds the near-duplicates in a catalogue's vocabulary. + +use std::path::PathBuf; +use std::process::ExitCode; + +use bibclean::vocab::{Item, Kind, Rules}; +use clap::Parser; +use corpus_core::{exit, ndjson}; + +/// Read a list of catalogue headings and write what should happen to them. +/// +/// One JSON object per line — an `id`, a `name`, and the `count` of works +/// filed under it — and one proposal per line out: which heading to keep, +/// which to lose, by what rule, and how far to trust it. +/// +/// Nothing is merged. Two headings that normalise to the same string are +/// usually the same thing and occasionally are not, and a catalogue that has +/// merged two real people is harder to repair than one that never did. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 nothing to propose, 1 something to propose, 65 a heading could +not be read, 74 the input could not be read. + +Example: + + vocabmerge --kind author < authors.ndjson | jq -r '\"\\(.drop)\\t→\\t\\(.keep)\"'" +)] +struct Args { + /// Heading list to read. Reads stdin when absent. + path: Option, + + /// What kind of heading the list holds. + #[arg(long, value_enum)] + kind: Heading, + + /// A JSON file of vocabulary specific to this collection. + /// + /// `{"device_prefixes": ["…"]}` — the names of e-readers that stamp + /// themselves onto every book they touch. The generic ones are built in; + /// a device named after its owner is a fact about one collection and + /// belongs here. + #[arg(long, value_name = "FILE")] + rules: Option, +} + +#[derive(Clone, Copy, clap::ValueEnum)] +enum Heading { + /// A person who wrote something. + Author, + /// A publisher or an imprint. + Publisher, + /// A subject heading. + Tag, +} + +impl From for Kind { + fn from(heading: Heading) -> Self { + match heading { + Heading::Author => Kind::Author, + Heading::Publisher => Kind::Publisher, + Heading::Tag => Kind::Tag, + } + } +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + + let rules = match args.rules.as_deref() { + None => Rules::default(), + Some(path) => match read_rules(path) { + Ok(rules) => rules, + Err((code, why)) => { + eprintln!("vocabmerge: {why}"); + return ExitCode::from(code); + } + }, + }; + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("vocabmerge: {why}"); + return ExitCode::from(code); + } + }; + let items: Vec = match ndjson::read_any(text.as_bytes()).collect() { + Ok(items) => items, + Err(e) => { + eprintln!("vocabmerge: {e}"); + return ExitCode::from(exit::DATA); + } + }; + + let proposals = bibclean::vocab::cluster(args.kind.into(), &items, &rules); + let anything = !proposals.is_empty(); + + let wrote = catena_cli::stream("vocabmerge", &proposals); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + ExitCode::from(if anything { exit::NO } else { exit::OK }) +} + +fn read_rules(path: &std::path::Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| (exit::IO, format!("{}: {e}", path.display())))?; + serde_json::from_str(&text).map_err(|e| (exit::DATA, format!("{}: {e}", path.display()))) +} diff --git a/crates/bibclean/src/lib.rs b/crates/bibclean/src/lib.rs index 8ca8617..ccf8212 100644 --- a/crates/bibclean/src/lib.rs +++ b/crates/bibclean/src/lib.rs @@ -18,6 +18,7 @@ pub mod isbn; pub mod matching; pub mod reasons; pub mod slug; +pub mod vocab; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/crates/bibclean/src/vocab.rs b/crates/bibclean/src/vocab.rs new file mode 100644 index 0000000..c180ed8 --- /dev/null +++ b/crates/bibclean/src/vocab.rs @@ -0,0 +1,753 @@ +//! Finding the near-duplicates in a catalogue's vocabulary. +//! +//! An author, a publisher or a subject heading accumulates spellings: the +//! same person entered surname-first once and given-name-first twice, a +//! publisher with and without its legal suffix, a subject with a stray +//! bracket. Each spelling is a separate heading, so the books under one are +//! invisible from the others. +//! +//! What comes out is a proposal per alias, never a merge. Two headings that +//! normalise to the same string are usually the same thing and occasionally +//! are not, and a catalogue that has merged two real people is harder to +//! repair than one that never did. + +use std::collections::HashMap; +use std::sync::LazyLock; + +use fancy_regex::Regex as Boundaried; +use regex::Regex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use textkit::pattern::{WORD_END, WORD_START}; +use unicode_normalization::UnicodeNormalization; + +fn pattern(source: &str) -> Regex { + Regex::new(source).expect("a valid pattern") +} + +fn bounded(source: &str) -> Boundaried { + Boundaried::new(source).expect("a valid pattern") +} + +/// Suffixes that say what a company is rather than who it is. +static LEGAL_SUFFIX: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?i){WORD_START}(inc|ltd|llc|gmbh|co|company|limited|usa|uk){WORD_END}" + )) +}); + +/// Everything in a publisher's name that is not the publisher. +static PUBLISHER_NOISE: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?i){WORD_START}(inc|ltd|llc|gmbh|co|company|limited|usa|uk|press|publishing|publishers|books|media|group|university){WORD_END}" + )) +}); + +/// A role attached to a name, which is not part of the name. +static AUTHOR_ROLE: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?i){WORD_START}(editors?|eds?|translator|trans|compiler|author){WORD_END}" + )) +}); + +/// Headings that are a file format or a watermark rather than a subject. +static FORMAT_OR_WATERMARK: LazyLock = LazyLock::new(|| { + pattern( + r"(?i)^(z-?lib(\.org)?|libgen|annas?-?archive|ocr|untitled( document)?|full page fax print|tam sayfa faks|microsoft word|ebook|pdf|epub|mobi|azw3?|kindle|unknown|n/?a|none|untagged|#genre|#)$", + ) +}); + +/// A catalogue field that was exploded into the subject list. +static CATALOGUE_FIELD: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?i)^(#\s*)?(isbn([-\s]?1[03])?|asin|publication date|number of pages|publisher|genre){WORD_END}" + )) +}); + +static ISBN_IN_TAG: LazyLock = LazyLock::new(|| { + bounded(&format!( + r"(?:97[89][\s\-–—]?)?(?:[0-9][\s\-–—]?){{9}}[0-9Xx]{WORD_END}" + )) +}); + +static URL: LazyLock = LazyLock::new(|| pattern(r"(?i)https?://")); +static YEAR_ONLY: LazyLock = LazyLock::new(|| pattern(r"^\d{4}\.?$")); +static ISO_DATE: LazyLock = LazyLock::new(|| pattern(r"^\d{4}-\d{2}-\d{2}")); +static BARE_NUMBER: LazyLock = LazyLock::new(|| pattern(r"^\d{1,6}$")); +static TIMESTAMP: LazyLock = LazyLock::new(|| pattern(r"^\d{1,2}:\d{2}(:\d{2})?")); +static PUNCTUATION_ONLY: LazyLock = LazyLock::new(|| pattern(r"^[\W_]+$")); +static SPACE_RUN: LazyLock = LazyLock::new(|| pattern(r"\s+")); +static NOT_ALPHANUMERIC: LazyLock = LazyLock::new(|| pattern(r"[^a-z0-9]+")); +static INITIALS_TOGETHER: LazyLock = LazyLock::new(|| pattern(r"[A-Z]\.[A-Z]")); +static INITIALS_SPACED: LazyLock = LazyLock::new(|| pattern(r"[A-Z]\.\s[A-Z]")); + +/// Suffixes on a person's name that are not part of who they are. +const NAME_SUFFIXES: &[&str] = &["ii", "iii", "iv", "jr", "md", "phd", "sr"]; + +/// Device names that an e-reader stamps onto every book it touches. +/// +/// The generic ones only. A device named after its owner is a fact about one +/// collection, which is what `Rules` is for. +pub const DEVICE_PREFIXES: &[&str] = &["internet", "kindle", "kobo"]; + +/// A heading a catalogue holds, and how many works are under it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Item { + /// The catalogue's own identifier for the heading. + pub id: i64, + /// The heading, as written. + pub name: String, + /// How many works are filed under it. + #[serde(default)] + pub count: i64, +} + +/// What kind of heading a list holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum Kind { + /// A person who wrote something. + Author, + /// A publisher or an imprint. + Publisher, + /// A subject heading. + Tag, +} + +impl Kind { + /// What this kind is called. + pub fn name(self) -> &'static str { + match self { + Kind::Author => "author", + Kind::Publisher => "publisher", + Kind::Tag => "tag", + } + } +} + +/// What should happen to one heading. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Proposal { + /// Which kind of heading this is. + pub kind: String, + /// `merge` or `delete`. + pub action: String, + /// The heading to keep, when there is one. + pub keep: Option, + /// The heading to lose. + pub drop: String, + /// The kept heading's identifier. + pub keep_id: Option, + /// The lost heading's identifier. + pub drop_id: i64, + /// How many works are under the kept heading. + pub keep_count: i64, + /// How many are under the lost one. + pub drop_count: i64, + /// Which rule produced this. + pub method: String, + /// How far to trust it, from zero to one. + #[serde(with = "corpus_core::finite")] + #[schemars(with = "f64")] + pub confidence: f64, + /// What a person needs to see to agree with it. + pub evidence: String, +} + +/// Collection-specific vocabulary, supplied rather than compiled in. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Rules { + /// Extra device prefixes, beyond the generic ones, that mark a heading + /// as an e-reader's stamp rather than a subject. + /// + /// A device named after its owner belongs here and not in the program. + #[serde(default)] + pub device_prefixes: Vec, +} + +/// Every proposal a list of headings suggests. +/// +/// Groups of near-duplicates first, then — for subjects — the headings that +/// are not subjects at all and should simply go. +pub fn cluster(kind: Kind, items: &[Item], rules: &Rules) -> Vec { + let mut out = Vec::new(); + let mut grouped: Vec = Vec::new(); + + for (method, group) in groups(kind, items) { + out.extend(proposals_from_group(kind, &group, method)); + grouped.extend(group.iter().map(|item| item.id)); + } + if kind == Kind::Tag { + out.extend(junk_deletes(items, &grouped, rules)); + } + out +} + +/// The near-duplicate groups, each with the rule that joined them. +pub fn groups(kind: Kind, items: &[Item]) -> Vec<(&'static str, Vec)> { + match kind { + Kind::Author => author_components(items) + .into_iter() + .filter(|group| distinct_names(group) >= 2) + .map(|group| ("author", group)) + .collect(), + Kind::Publisher => keyed_groups(items, publisher_key) + .into_iter() + .map(|group| ("publisher_family", group)) + .collect(), + Kind::Tag => keyed_groups(items, tag_key) + .into_iter() + .map(|group| ("tag_norm", group)) + .collect(), + } +} + +/// The heading to keep out of a group: the one with the most works, then the +/// cleaner spelling of it. +pub fn pick_canonical(kind: Kind, group: &[Item]) -> Item { + // The first of the highest-scoring, so a tie goes to the heading the + // catalogue listed first. + group + .iter() + .cloned() + .reduce(|best, item| match score(kind, &item) > score(kind, &best) { + true => item, + false => best, + }) + .expect("a group is never empty") +} + +fn score(kind: Kind, item: &Item) -> (i64, usize) { + let name = &item.name; + let mut score = item.count * 10; + + if is_shouting(name) && name.contains(' ') { + score -= 6; + } + match kind { + Kind::Author => { + if name.contains(',') || name.contains('|') { + score -= 8; + } + if name + .trim_end_matches(textkit::is_space) + .ends_with([';', '|', ',']) + { + score -= 10; + } + if INITIALS_TOGETHER.is_match(name) { + score -= 1; + } + if INITIALS_SPACED.is_match(name) { + score += 3; + } + } + Kind::Publisher => { + if LEGAL_SUFFIX.is_match(name).unwrap_or(false) { + score -= 4; + } + if name.contains(',') { + score -= 2; + } + } + Kind::Tag => { + if name.starts_with(['[', '*', '&']) { + score -= 10; + } + if name.ends_with('.') { + score -= 4; + } + } + } + (score, name.chars().count()) +} + +/// Whether a heading is in capitals throughout, by Python's reading: at least +/// one cased letter, and no lowercase one. +fn is_shouting(name: &str) -> bool { + name.chars().any(char::is_uppercase) && !name.chars().any(char::is_lowercase) +} + +/// One proposal per alias in a group. +pub fn proposals_from_group(kind: Kind, group: &[Item], method: &str) -> Vec { + let keep = pick_canonical(kind, group); + let canon = keep.name.clone(); + + group + .iter() + .filter(|item| item.name != canon) + .map(|item| { + let same = normalise(&item.name) == normalise(&canon); + let (row_method, confidence) = match method { + // An author group is joined two ways at once, and which one + // joined this row decides how far to trust it: a heading that + // normalises to the canonical one is the same string written + // differently, where an inverted name is an inference. + "author" => match same { + true => ("strict_norm", 0.95), + false => ("last_first", 0.9), + }, + "publisher_family" => ("publisher_family", 0.8), + other => (other, 0.9), + }; + Proposal { + kind: kind.name().to_string(), + action: "merge".into(), + keep: Some(canon.clone()), + drop: item.name.clone(), + keep_id: Some(keep.id), + drop_id: item.id, + keep_count: keep.count, + drop_count: item.count, + method: row_method.to_string(), + confidence, + evidence: format!( + "{} books as '{}' → '{}' ({})", + item.count, item.name, canon, keep.count + ), + } + }) + .collect() +} + +/// Why a subject heading should be deleted, or `None` to keep it. +/// +/// High precision only: identifiers, catalogue leftovers, links, blurbs used +/// as headings, bare years and page counts. A real subject — a century, a +/// discipline, a war — must never match. +pub fn junk_reason(name: &str, rules: &Rules) -> Option<&'static str> { + let raw = name.trim_matches(textkit::is_space); + if raw.is_empty() { + return Some("empty"); + } + if PUNCTUATION_ONLY.is_match(raw) { + return Some("punctuation-only"); + } + let unbracketed = raw.trim_matches(['[', ']', '*', '&']); + if FORMAT_OR_WATERMARK.is_match(raw) || FORMAT_OR_WATERMARK.is_match(unbracketed) { + return Some("format-or-watermark"); + } + if raw.starts_with('#') || CATALOGUE_FIELD.is_match(raw).unwrap_or(false) { + return Some("catalog-field"); + } + if URL.is_match(raw) { + return Some("url"); + } + let digits: String = raw.chars().filter(char::is_ascii_digit).collect(); + if ISBN_IN_TAG.is_match(raw).unwrap_or(false) + && (raw.to_uppercase().starts_with("ISBN") + || raw.chars().take(3).collect::() == "978" + || raw.chars().take(3).collect::() == "979" + || digits.len() >= 10) + { + return Some("isbn"); + } + if YEAR_ONLY.is_match(raw) || ISO_DATE.is_match(raw) { + return Some("date"); + } + if BARE_NUMBER.is_match(raw) { + return Some("numeric-id"); + } + if TIMESTAMP.is_match(raw) { + return Some("timestamp"); + } + if is_device_tag(raw, rules) { + return Some("device-tag"); + } + if raw.chars().count() > 80 { + return Some("blurb"); + } + if raw.matches(';').count() >= 3 { + return Some("exploded-list"); + } + None +} + +fn is_device_tag(raw: &str, rules: &Rules) -> bool { + let Some(rest) = raw.strip_prefix('[') else { + return false; + }; + let rest = rest.to_lowercase(); + DEVICE_PREFIXES + .iter() + .map(|prefix| prefix.to_string()) + .chain(rules.device_prefixes.iter().map(|p| p.to_lowercase())) + .any(|prefix| rest.starts_with(&prefix)) +} + +fn junk_deletes(items: &[Item], grouped: &[i64], rules: &Rules) -> Vec { + items + .iter() + .filter(|item| !grouped.contains(&item.id)) + .filter_map(|item| { + let why = junk_reason(&item.name, rules)?; + Some(Proposal { + kind: "tag".into(), + action: "delete".into(), + keep: None, + drop: item.name.clone(), + keep_id: None, + drop_id: item.id, + keep_count: 0, + drop_count: item.count, + method: format!("junk_tag:{why}"), + confidence: 0.92, + evidence: format!("{why}: '{}' on {} books", item.name, item.count), + }) + }) + .collect() +} + +/// A heading reduced to the letters and digits in it, accents removed. +pub fn normalise(name: &str) -> String { + let stripped: String = name + .nfkd() + .filter(|c| !unicode_normalization::char::is_combining_mark(*c)) + .collect(); + let lowered = stripped.to_lowercase().replace('|', " "); + NOT_ALPHANUMERIC + .replace_all(&lowered, " ") + .trim_matches(textkit::is_space) + .to_string() +} + +/// The words of a person's name, with roles and suffixes taken out. +pub fn author_tokens(name: &str) -> Vec { + let normalised = normalise(name); + let without_roles = AUTHOR_ROLE.replace_all(&normalised, " "); + without_roles + .split_whitespace() + .filter(|token| NAME_SUFFIXES.binary_search(token).is_err()) + .map(str::to_string) + .collect() +} + +/// A publisher reduced to what actually names it. +pub fn publisher_key(name: &str) -> String { + let normalised = normalise(name); + let without_noise = PUBLISHER_NOISE.replace_all(&normalised, " "); + SPACE_RUN + .replace_all(&without_noise, " ") + .trim_matches(textkit::is_space) + .to_string() +} + +/// A subject heading reduced to what it is about. +pub fn tag_key(name: &str) -> String { + let trimmed = name.trim_matches(textkit::is_space); + let unbracketed = trimmed.trim_matches(['[', ']', '*', '&']); + let trimmed = unbracketed.trim_matches(textkit::is_space); + normalise(trimmed.trim_end_matches('.')) +} + +/// Groups of two or more distinct headings sharing a key, in the order the +/// keys were first seen. +fn keyed_groups(items: &[Item], key: fn(&str) -> String) -> Vec> { + let mut order: Vec = Vec::new(); + let mut buckets: HashMap> = HashMap::new(); + + for item in items { + let key = key(&item.name); + if key.is_empty() { + continue; + } + buckets + .entry(key.clone()) + .or_insert_with(|| { + order.push(key); + Vec::new() + }) + .push(item.clone()); + } + order + .into_iter() + .filter_map(|key| buckets.remove(&key)) + .filter(|group| distinct_names(group) >= 2) + .collect() +} + +/// Author groups, joining spelling matches and inverted names together. +/// +/// Joined rather than kept apart, so the heading that is kept is chosen from +/// the whole set. Otherwise two surname-first spellings merge onto each other +/// and neither ever reaches the given-name-first one that most of the works +/// are actually under. +fn author_components(items: &[Item]) -> Vec> { + let mut parent: HashMap = items.iter().map(|item| (item.id, item.id)).collect(); + + fn find(parent: &mut HashMap, of: i64) -> i64 { + let mut at = of; + while parent[&at] != at { + let grandparent = parent[&parent[&at]]; + parent.insert(at, grandparent); + at = grandparent; + } + at + } + fn union(parent: &mut HashMap, a: i64, b: i64) { + let (a, b) = (find(parent, a), find(parent, b)); + if a != b { + parent.insert(b, a); + } + } + + // Same spelling once the accents and punctuation are gone. + for group in keyed_buckets(items, normalise) { + for item in &group[1..] { + union(&mut parent, group[0].id, item.id); + } + } + + // Same words in a different order, for the headings that read as people. + let inverted = keyed_buckets(items, |name| { + if !crate::reasons::looks_like_person(&name.replace('|', ",")) { + return String::new(); + } + let mut tokens = author_tokens(name); + tokens.sort(); + match (2..=5).contains(&tokens.len()) { + true => tokens.join(" "), + false => String::new(), + } + }); + for group in inverted { + for item in &group[1..] { + union(&mut parent, group[0].id, item.id); + } + } + + let mut order: Vec = Vec::new(); + let mut components: HashMap> = HashMap::new(); + for item in items { + let root = find(&mut parent, item.id); + components + .entry(root) + .or_insert_with(|| { + order.push(root); + Vec::new() + }) + .push(item.clone()); + } + order + .into_iter() + .filter_map(|root| components.remove(&root)) + .collect() +} + +/// Every bucket of a key, including the ones with a single member, in +/// first-seen order. +fn keyed_buckets(items: &[Item], key: impl Fn(&str) -> String) -> Vec> { + let mut order: Vec = Vec::new(); + let mut buckets: HashMap> = HashMap::new(); + for item in items { + let key = key(&item.name); + if key.is_empty() { + continue; + } + buckets + .entry(key.clone()) + .or_insert_with(|| { + order.push(key); + Vec::new() + }) + .push(item.clone()); + } + order + .into_iter() + .filter_map(|key| buckets.remove(&key)) + .collect() +} + +fn distinct_names(group: &[Item]) -> usize { + let mut names: Vec<&str> = group.iter().map(|item| item.name.as_str()).collect(); + names.sort_unstable(); + names.dedup(); + names.len() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item(id: i64, name: &str, count: i64) -> Item { + Item { + id, + name: name.into(), + count, + } + } + + #[test] + fn the_suffix_list_is_sorted_so_it_can_be_searched() { + assert!(NAME_SUFFIXES.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn normalising_removes_accents_and_punctuation() { + assert_eq!(normalise("Dostoevsky| Fyodor"), "dostoevsky fyodor"); + assert_eq!(normalise("Étienne Gilson"), "etienne gilson"); + assert_eq!(normalise(""), ""); + } + + #[test] + fn a_publisher_is_reduced_to_what_names_it() { + assert_eq!(publisher_key("Oxford University Press, Inc."), "oxford"); + assert_eq!(publisher_key("Oxford University Press"), "oxford"); + assert_eq!(publisher_key("Routledge"), "routledge"); + } + + #[test] + fn a_subject_is_reduced_to_what_it_is_about() { + assert_eq!(tag_key("[Philosophy]"), "philosophy"); + assert_eq!(tag_key("Philosophy."), "philosophy"); + assert_eq!(tag_key(" *Philosophy "), "philosophy"); + } + + #[test] + fn a_role_is_not_part_of_a_name() { + assert_eq!(author_tokens("Smith, John (editor)"), ["smith", "john"]); + assert_eq!(author_tokens("Smith, John Jr."), ["smith", "john"]); + } + + #[test] + fn the_heading_with_the_most_works_is_the_one_kept() { + let group = [ + item(1, "Dostoevsky, Fyodor", 3), + item(2, "Fyodor Dostoevsky", 11), + ]; + assert_eq!( + pick_canonical(Kind::Author, &group).name, + "Fyodor Dostoevsky" + ); + } + + #[test] + fn an_inverted_name_and_a_stray_separator_join_the_same_group() { + let items = [ + item(1, "Dostoevsky, Fyodor", 3), + item(2, "Dostoevsky| Fyodor", 1), + item(3, "Fyodor Dostoevsky", 11), + ]; + let proposals = cluster(Kind::Author, &items, &Rules::default()); + assert_eq!(proposals.len(), 2); + assert!( + proposals + .iter() + .all(|p| p.keep.as_deref() == Some("Fyodor Dostoevsky")), + "{proposals:#?}" + ); + // Neither surname-first spelling normalises to the given-name-first + // heading, so both are inferences rather than respellings. + assert!( + proposals.iter().all(|p| p.method == "last_first"), + "{proposals:#?}" + ); + } + + #[test] + fn a_respelling_is_trusted_further_than_an_inversion() { + let items = [ + item(1, "Fyodor Dostoevsky", 11), + item(2, "Fyodor Dostoëvsky", 2), + item(3, "Dostoevsky, Fyodor", 3), + ]; + let proposals = cluster(Kind::Author, &items, &Rules::default()); + let respelling = proposals + .iter() + .find(|p| p.drop == "Fyodor Dostoëvsky") + .expect("a proposal"); + let inversion = proposals + .iter() + .find(|p| p.drop == "Dostoevsky, Fyodor") + .expect("a proposal"); + assert_eq!(respelling.method, "strict_norm"); + assert_eq!(respelling.confidence, 0.95); + assert_eq!(inversion.method, "last_first"); + assert_eq!(inversion.confidence, 0.9); + } + + #[test] + fn a_publisher_family_merges_onto_its_commonest_spelling() { + let items = [ + item(1, "Oxford University Press", 40), + item(2, "Oxford University Press, Inc.", 2), + item(3, "Routledge", 9), + ]; + let proposals = cluster(Kind::Publisher, &items, &Rules::default()); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].drop, "Oxford University Press, Inc."); + assert_eq!( + proposals[0].keep.as_deref(), + Some("Oxford University Press") + ); + assert_eq!(proposals[0].confidence, 0.8); + } + + #[test] + fn a_heading_on_its_own_is_left_alone() { + let items = [item(1, "Philosophy", 30)]; + assert!(cluster(Kind::Tag, &items, &Rules::default()).is_empty()); + } + + #[test] + fn a_subject_that_is_not_a_subject_is_proposed_for_deletion() { + let rules = Rules::default(); + for (name, why) in [ + ("", "empty"), + ("---", "punctuation-only"), + ("z-lib.org", "format-or-watermark"), + ("[kindle]", "format-or-watermark"), + ("#genre", "format-or-watermark"), + ("ISBN-13", "catalog-field"), + ("https://example.org/x", "url"), + ("9780198237907", "isbn"), + ("1998", "date"), + ("2024-01-15", "date"), + ("451", "numeric-id"), + ("1:23:45", "timestamp"), + ("[kobo] favourites", "device-tag"), + ("a;b;c;d", "exploded-list"), + ] { + assert_eq!(junk_reason(name, &rules), Some(why), "{name:?}"); + } + } + + #[test] + fn a_real_subject_is_never_junk() { + let rules = Rules::default(); + for name in [ + "18th Century", + "Philosophy", + "World War, 1939-1945", + "Church history", + "Russia — History — Revolution, 1917-1921", + ] { + assert_eq!(junk_reason(name, &rules), None, "{name:?}"); + } + } + + #[test] + fn a_device_named_after_its_owner_comes_from_the_rules() { + let rules = Rules { + device_prefixes: vec!["alexskobo".into()], + }; + assert_eq!(junk_reason("[alexskobo] to read", &Rules::default()), None); + assert_eq!( + junk_reason("[alexskobo] to read", &rules), + Some("device-tag") + ); + } + + #[test] + fn a_heading_already_in_a_group_is_not_also_deleted() { + let items = [item(1, "Philosophy", 30), item(2, "[Philosophy]", 1)]; + let proposals = cluster(Kind::Tag, &items, &Rules::default()); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].action, "merge"); + } + + #[test] + fn nothing_in_nothing_out() { + assert!(cluster(Kind::Author, &[], &Rules::default()).is_empty()); + } +} diff --git a/crates/booksniff/README.md b/crates/booksniff/README.md index 845665f..edbe14c 100644 --- a/crates/booksniff/README.md +++ b/crates/booksniff/README.md @@ -37,7 +37,7 @@ cargo install --git https://git.pumpin.online/funkpower/catena booksniff ## Example -Three files, one of which is not what it says it is: +`atlas.pdf` is a DjVu that somebody renamed: ``` $ booksniff --plain book.pdf atlas.pdf chapter.md diff --git a/crates/corpus-core/src/ndjson.rs b/crates/corpus-core/src/ndjson.rs index b17259e..1c68a87 100644 --- a/crates/corpus-core/src/ndjson.rs +++ b/crates/corpus-core/src/ndjson.rs @@ -100,6 +100,23 @@ where read_with_limit(reader, MAX_LINE) } +/// Read a stream of anything JSON, one per line, without checking a version. +/// +/// For a stream that is not one of this suite's records — a list exported +/// from somebody else's catalogue, say. Everything else about the reading is +/// the same: the line limit, the blank lines, the byte-order mark, and +/// stopping at the first line that does not parse. +pub fn read_any(reader: R) -> impl Iterator> +where + T: serde::de::DeserializeOwned, + R: BufRead, +{ + stop_at_first_error(lines(reader, MAX_LINE).map(|line| { + let (at, line) = line?; + serde_json::from_str(&line).map_err(|source| Error::Parse { line: at, source }) + })) +} + /// Read a stream of records, one per line, refusing any line longer than /// `limit` bytes. /// @@ -107,11 +124,45 @@ where /// still reads, and a byte-order mark at the very front is ignored. The /// iterator ends at the first error: a bad line means the producer and the /// consumer disagree about the contract, and reading on would bury that. -pub fn read_with_limit(mut reader: R, limit: usize) -> impl Iterator> +pub fn read_with_limit(reader: R, limit: usize) -> impl Iterator> where T: Record, R: BufRead, { + stop_at_first_error(lines(reader, limit).map(|line| { + let (at, line) = line?; + match serde_json::from_str::(&line) { + Err(source) => Err(Error::Parse { line: at, source }), + Ok(record) if record.version() != crate::V => Err(Error::Version { + line: at, + found: record.version(), + expected: crate::V, + }), + Ok(record) => Ok(record), + } + })) +} + +/// End a stream at its first error. +/// +/// A bad line means the producer and the consumer disagree about the +/// contract, and reading on would bury that. +fn stop_at_first_error( + records: impl Iterator>, +) -> impl Iterator> { + let mut stopped = false; + records.take_while(move |record| { + let carry_on = !stopped; + stopped |= record.is_err(); + carry_on + }) +} + +/// The lines of a stream, numbered, bounded, and without the blank ones. +fn lines( + mut reader: R, + limit: usize, +) -> impl Iterator> { let mut line_no = 0u64; let mut buf = Vec::new(); let mut done = false; @@ -171,25 +222,7 @@ where if line.trim().is_empty() { continue; } - - return Some(match serde_json::from_str::(line) { - Err(source) => { - done = true; - Err(Error::Parse { - line: line_no, - source, - }) - } - Ok(record) if record.version() != crate::V => { - done = true; - Err(Error::Version { - line: line_no, - found: record.version(), - expected: crate::V, - }) - } - Ok(record) => Ok(record), - }); + return Some(Ok((line_no, line.to_string()))); } }) } @@ -295,6 +328,33 @@ mod tests { } } + #[test] + fn a_stream_that_is_not_one_of_these_records_reads_without_a_version() { + #[derive(Debug, PartialEq, serde::Deserialize)] + struct Heading { + name: String, + } + let input = "{\"name\":\"Philosophy\"}\n{\"name\":\"History\"}\n"; + let headings: Vec = read_any(input.as_bytes()) + .collect::>() + .unwrap(); + assert_eq!(headings.len(), 2); + assert_eq!(headings[1].name, "History"); + } + + #[test] + fn an_unversioned_stream_still_stops_at_a_bad_line() { + #[derive(Debug, serde::Deserialize)] + struct Heading { + #[allow(dead_code)] + name: String, + } + let mut stream = read_any::("{\"name\":\"x\"}\nnot json\n".as_bytes()); + assert!(stream.next().unwrap().is_ok()); + assert!(stream.next().unwrap().is_err()); + assert!(stream.next().is_none()); + } + #[test] fn a_record_from_a_future_version_is_refused() { let input = "{\"v\":2,\"id\":\"a\",\"channel\":\"fts\",\"rank\":1,\"score\":1.0}\n"; diff --git a/parity/cases/vocab/lists.ndjson b/parity/cases/vocab/lists.ndjson new file mode 100644 index 0000000..a1d8238 --- /dev/null +++ b/parity/cases/vocab/lists.ndjson @@ -0,0 +1,15 @@ +{"kind": "author", "items": [{"id": 1, "name": "Dostoevsky, Fyodor", "count": 3}, {"id": 2, "name": "Dostoevsky| Fyodor", "count": 1}, {"id": 3, "name": "Fyodor Dostoevsky", "count": 11}]} +{"kind": "author", "items": [{"id": 1, "name": "Fyodor Dostoevsky", "count": 11}, {"id": 2, "name": "Fyodor Dostoëvsky", "count": 2}, {"id": 3, "name": "Dostoevsky, Fyodor", "count": 3}]} +{"kind": "author", "items": [{"id": 1, "name": "C.S. Lewis", "count": 8}, {"id": 2, "name": "C. S. Lewis", "count": 12}, {"id": 3, "name": "Lewis, C. S.", "count": 4}, {"id": 4, "name": "CLIVE STAPLES LEWIS", "count": 1}]} +{"kind": "author", "items": [{"id": 1, "name": "Smith, John (editor)", "count": 2}, {"id": 2, "name": "John Smith", "count": 5}, {"id": 3, "name": "Smith, John Jr.", "count": 1}]} +{"kind": "author", "items": [{"id": 1, "name": "Plato", "count": 30}, {"id": 2, "name": "Aristotle", "count": 25}]} +{"kind": "author", "items": [{"id": 1, "name": "Arendt, Hannah;", "count": 1}, {"id": 2, "name": "Hannah Arendt", "count": 9}]} +{"kind": "author", "items": [{"id": 1, "name": "Étienne Gilson", "count": 4}, {"id": 2, "name": "Etienne Gilson", "count": 2}]} +{"kind": "publisher", "items": [{"id": 1, "name": "Oxford University Press", "count": 40}, {"id": 2, "name": "Oxford University Press, Inc.", "count": 2}, {"id": 3, "name": "Oxford Univ. Press", "count": 1}, {"id": 4, "name": "Routledge", "count": 9}]} +{"kind": "publisher", "items": [{"id": 1, "name": "Penguin Books", "count": 20}, {"id": 2, "name": "Penguin Books Ltd", "count": 3}, {"id": 3, "name": "Penguin", "count": 7}]} +{"kind": "publisher", "items": [{"id": 1, "name": "Brill", "count": 5}]} +{"kind": "tag", "items": [{"id": 1, "name": "Philosophy", "count": 30}, {"id": 2, "name": "[Philosophy]", "count": 1}, {"id": 3, "name": "Philosophy.", "count": 2}, {"id": 4, "name": "philosophy", "count": 4}]} +{"kind": "tag", "items": [{"id": 1, "name": "18th Century", "count": 12}, {"id": 2, "name": "z-lib.org", "count": 300}, {"id": 3, "name": "9780198237907", "count": 1}, {"id": 4, "name": "https://example.org/x", "count": 2}, {"id": 5, "name": "1998", "count": 5}, {"id": 6, "name": "2024-01-15", "count": 1}, {"id": 7, "name": "451", "count": 2}, {"id": 8, "name": "1:23:45", "count": 1}, {"id": 9, "name": "[kobo] favourites", "count": 4}, {"id": 10, "name": "a;b;c;d", "count": 1}, {"id": 11, "name": "---", "count": 1}, {"id": 12, "name": "#genre", "count": 9}, {"id": 13, "name": "ISBN-13", "count": 3}, {"id": 14, "name": "World War, 1939-1945", "count": 40}]} +{"kind": "tag", "items": [{"id": 1, "name": "A very long blurb used as a tag that goes on and on and on and should not be here at all really", "count": 1}, {"id": 2, "name": "Church history", "count": 8}]} +{"kind": "tag", "items": [{"id": 1, "name": "[alexskobo] to read", "count": 6}, {"id": 2, "name": "Fiction", "count": 40}]} +{"kind": "author", "items": []} diff --git a/parity/golden/vocab/lists.ndjson b/parity/golden/vocab/lists.ndjson new file mode 100644 index 0000000..306ddd5 --- /dev/null +++ b/parity/golden/vocab/lists.ndjson @@ -0,0 +1,15 @@ +{"canonical": "Fyodor Dostoevsky", "keys": [{"author_tokens": ["dostoevsky", "fyodor"], "junk": null, "norm": "dostoevsky fyodor", "pub_key": "dostoevsky fyodor", "tag_key": "dostoevsky fyodor"}, {"author_tokens": ["dostoevsky", "fyodor"], "junk": null, "norm": "dostoevsky fyodor", "pub_key": "dostoevsky fyodor", "tag_key": "dostoevsky fyodor"}, {"author_tokens": ["fyodor", "dostoevsky"], "junk": null, "norm": "fyodor dostoevsky", "pub_key": "fyodor dostoevsky", "tag_key": "fyodor dostoevsky"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "Dostoevsky, Fyodor", "drop_count": 3, "drop_id": 1, "evidence": "3 books as 'Dostoevsky, Fyodor' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 3, "method": "last_first"}, {"action": "merge", "confidence": 0.9, "drop": "Dostoevsky| Fyodor", "drop_count": 1, "drop_id": 2, "evidence": "1 books as 'Dostoevsky| Fyodor' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 3, "method": "last_first"}]} +{"canonical": "Fyodor Dostoevsky", "keys": [{"author_tokens": ["fyodor", "dostoevsky"], "junk": null, "norm": "fyodor dostoevsky", "pub_key": "fyodor dostoevsky", "tag_key": "fyodor dostoevsky"}, {"author_tokens": ["fyodor", "dostoevsky"], "junk": null, "norm": "fyodor dostoevsky", "pub_key": "fyodor dostoevsky", "tag_key": "fyodor dostoevsky"}, {"author_tokens": ["dostoevsky", "fyodor"], "junk": null, "norm": "dostoevsky fyodor", "pub_key": "dostoevsky fyodor", "tag_key": "dostoevsky fyodor"}], "proposals": [{"action": "merge", "confidence": 0.95, "drop": "Fyodor Dostoëvsky", "drop_count": 2, "drop_id": 2, "evidence": "2 books as 'Fyodor Dostoëvsky' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 1, "method": "strict_norm"}, {"action": "merge", "confidence": 0.9, "drop": "Dostoevsky, Fyodor", "drop_count": 3, "drop_id": 3, "evidence": "3 books as 'Dostoevsky, Fyodor' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 1, "method": "last_first"}]} +{"canonical": "C. S. Lewis", "keys": [{"author_tokens": ["c", "s", "lewis"], "junk": null, "norm": "c s lewis", "pub_key": "c s lewis", "tag_key": "c s lewis"}, {"author_tokens": ["c", "s", "lewis"], "junk": null, "norm": "c s lewis", "pub_key": "c s lewis", "tag_key": "c s lewis"}, {"author_tokens": ["lewis", "c", "s"], "junk": null, "norm": "lewis c s", "pub_key": "lewis c s", "tag_key": "lewis c s"}, {"author_tokens": ["clive", "staples", "lewis"], "junk": null, "norm": "clive staples lewis", "pub_key": "clive staples lewis", "tag_key": "clive staples lewis"}], "proposals": [{"action": "merge", "confidence": 0.95, "drop": "C.S. Lewis", "drop_count": 8, "drop_id": 1, "evidence": "8 books as 'C.S. Lewis' → 'C. S. Lewis' (12)", "keep": "C. S. Lewis", "keep_count": 12, "keep_id": 2, "method": "strict_norm"}, {"action": "merge", "confidence": 0.9, "drop": "Lewis, C. S.", "drop_count": 4, "drop_id": 3, "evidence": "4 books as 'Lewis, C. S.' → 'C. S. Lewis' (12)", "keep": "C. S. Lewis", "keep_count": 12, "keep_id": 2, "method": "last_first"}]} +{"canonical": "John Smith", "keys": [{"author_tokens": ["smith", "john"], "junk": null, "norm": "smith john editor", "pub_key": "smith john editor", "tag_key": "smith john editor"}, {"author_tokens": ["john", "smith"], "junk": null, "norm": "john smith", "pub_key": "john smith", "tag_key": "john smith"}, {"author_tokens": ["smith", "john"], "junk": null, "norm": "smith john jr", "pub_key": "smith john jr", "tag_key": "smith john jr"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "Smith, John (editor)", "drop_count": 2, "drop_id": 1, "evidence": "2 books as 'Smith, John (editor)' → 'John Smith' (5)", "keep": "John Smith", "keep_count": 5, "keep_id": 2, "method": "last_first"}, {"action": "merge", "confidence": 0.9, "drop": "Smith, John Jr.", "drop_count": 1, "drop_id": 3, "evidence": "1 books as 'Smith, John Jr.' → 'John Smith' (5)", "keep": "John Smith", "keep_count": 5, "keep_id": 2, "method": "last_first"}]} +{"canonical": "Plato", "keys": [{"author_tokens": ["plato"], "junk": null, "norm": "plato", "pub_key": "plato", "tag_key": "plato"}, {"author_tokens": ["aristotle"], "junk": null, "norm": "aristotle", "pub_key": "aristotle", "tag_key": "aristotle"}], "proposals": []} +{"canonical": "Hannah Arendt", "keys": [{"author_tokens": ["arendt", "hannah"], "junk": null, "norm": "arendt hannah", "pub_key": "arendt hannah", "tag_key": "arendt hannah"}, {"author_tokens": ["hannah", "arendt"], "junk": null, "norm": "hannah arendt", "pub_key": "hannah arendt", "tag_key": "hannah arendt"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "Arendt, Hannah;", "drop_count": 1, "drop_id": 1, "evidence": "1 books as 'Arendt, Hannah;' → 'Hannah Arendt' (9)", "keep": "Hannah Arendt", "keep_count": 9, "keep_id": 2, "method": "last_first"}]} +{"canonical": "Étienne Gilson", "keys": [{"author_tokens": ["etienne", "gilson"], "junk": null, "norm": "etienne gilson", "pub_key": "etienne gilson", "tag_key": "etienne gilson"}, {"author_tokens": ["etienne", "gilson"], "junk": null, "norm": "etienne gilson", "pub_key": "etienne gilson", "tag_key": "etienne gilson"}], "proposals": [{"action": "merge", "confidence": 0.95, "drop": "Etienne Gilson", "drop_count": 2, "drop_id": 2, "evidence": "2 books as 'Etienne Gilson' → 'Étienne Gilson' (4)", "keep": "Étienne Gilson", "keep_count": 4, "keep_id": 1, "method": "strict_norm"}]} +{"canonical": "Oxford University Press", "keys": [{"author_tokens": ["oxford", "university", "press"], "junk": null, "norm": "oxford university press", "pub_key": "oxford", "tag_key": "oxford university press"}, {"author_tokens": ["oxford", "university", "press", "inc"], "junk": null, "norm": "oxford university press inc", "pub_key": "oxford", "tag_key": "oxford university press inc"}, {"author_tokens": ["oxford", "univ", "press"], "junk": null, "norm": "oxford univ press", "pub_key": "oxford univ", "tag_key": "oxford univ press"}, {"author_tokens": ["routledge"], "junk": null, "norm": "routledge", "pub_key": "routledge", "tag_key": "routledge"}], "proposals": [{"action": "merge", "confidence": 0.8, "drop": "Oxford University Press, Inc.", "drop_count": 2, "drop_id": 2, "evidence": "2 books as 'Oxford University Press, Inc.' → 'Oxford University Press' (40)", "keep": "Oxford University Press", "keep_count": 40, "keep_id": 1, "method": "publisher_family"}]} +{"canonical": "Penguin Books", "keys": [{"author_tokens": ["penguin", "books"], "junk": null, "norm": "penguin books", "pub_key": "penguin", "tag_key": "penguin books"}, {"author_tokens": ["penguin", "books", "ltd"], "junk": null, "norm": "penguin books ltd", "pub_key": "penguin", "tag_key": "penguin books ltd"}, {"author_tokens": ["penguin"], "junk": null, "norm": "penguin", "pub_key": "penguin", "tag_key": "penguin"}], "proposals": [{"action": "merge", "confidence": 0.8, "drop": "Penguin Books Ltd", "drop_count": 3, "drop_id": 2, "evidence": "3 books as 'Penguin Books Ltd' → 'Penguin Books' (20)", "keep": "Penguin Books", "keep_count": 20, "keep_id": 1, "method": "publisher_family"}, {"action": "merge", "confidence": 0.8, "drop": "Penguin", "drop_count": 7, "drop_id": 3, "evidence": "7 books as 'Penguin' → 'Penguin Books' (20)", "keep": "Penguin Books", "keep_count": 20, "keep_id": 1, "method": "publisher_family"}]} +{"canonical": "Brill", "keys": [{"author_tokens": ["brill"], "junk": null, "norm": "brill", "pub_key": "brill", "tag_key": "brill"}], "proposals": []} +{"canonical": "Philosophy", "keys": [{"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}, {"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}, {"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}, {"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "[Philosophy]", "drop_count": 1, "drop_id": 2, "evidence": "1 books as '[Philosophy]' → 'Philosophy' (30)", "keep": "Philosophy", "keep_count": 30, "keep_id": 1, "method": "tag_norm"}, {"action": "merge", "confidence": 0.9, "drop": "Philosophy.", "drop_count": 2, "drop_id": 3, "evidence": "2 books as 'Philosophy.' → 'Philosophy' (30)", "keep": "Philosophy", "keep_count": 30, "keep_id": 1, "method": "tag_norm"}, {"action": "merge", "confidence": 0.9, "drop": "philosophy", "drop_count": 4, "drop_id": 4, "evidence": "4 books as 'philosophy' → 'Philosophy' (30)", "keep": "Philosophy", "keep_count": 30, "keep_id": 1, "method": "tag_norm"}]} +{"canonical": "z-lib.org", "keys": [{"author_tokens": ["18th", "century"], "junk": null, "norm": "18th century", "pub_key": "18th century", "tag_key": "18th century"}, {"author_tokens": ["z", "lib", "org"], "junk": "format-or-watermark", "norm": "z lib org", "pub_key": "z lib org", "tag_key": "z lib org"}, {"author_tokens": ["9780198237907"], "junk": "isbn", "norm": "9780198237907", "pub_key": "9780198237907", "tag_key": "9780198237907"}, {"author_tokens": ["https", "example", "org", "x"], "junk": "url", "norm": "https example org x", "pub_key": "https example org x", "tag_key": "https example org x"}, {"author_tokens": ["1998"], "junk": "date", "norm": "1998", "pub_key": "1998", "tag_key": "1998"}, {"author_tokens": ["2024", "01", "15"], "junk": "date", "norm": "2024 01 15", "pub_key": "2024 01 15", "tag_key": "2024 01 15"}, {"author_tokens": ["451"], "junk": "numeric-id", "norm": "451", "pub_key": "451", "tag_key": "451"}, {"author_tokens": ["1", "23", "45"], "junk": "timestamp", "norm": "1 23 45", "pub_key": "1 23 45", "tag_key": "1 23 45"}, {"author_tokens": ["kobo", "favourites"], "junk": "device-tag", "norm": "kobo favourites", "pub_key": "kobo favourites", "tag_key": "kobo favourites"}, {"author_tokens": ["a", "b", "c", "d"], "junk": "exploded-list", "norm": "a b c d", "pub_key": "a b c d", "tag_key": "a b c d"}, {"author_tokens": [], "junk": "punctuation-only", "norm": "", "pub_key": "", "tag_key": ""}, {"author_tokens": ["genre"], "junk": "format-or-watermark", "norm": "genre", "pub_key": "genre", "tag_key": "genre"}, {"author_tokens": ["isbn", "13"], "junk": "catalog-field", "norm": "isbn 13", "pub_key": "isbn 13", "tag_key": "isbn 13"}, {"author_tokens": ["world", "war", "1939", "1945"], "junk": null, "norm": "world war 1939 1945", "pub_key": "world war 1939 1945", "tag_key": "world war 1939 1945"}], "proposals": [{"action": "delete", "confidence": 0.92, "drop": "z-lib.org", "drop_count": 300, "drop_id": 2, "evidence": "format-or-watermark: 'z-lib.org' on 300 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:format-or-watermark"}, {"action": "delete", "confidence": 0.92, "drop": "9780198237907", "drop_count": 1, "drop_id": 3, "evidence": "isbn: '9780198237907' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:isbn"}, {"action": "delete", "confidence": 0.92, "drop": "https://example.org/x", "drop_count": 2, "drop_id": 4, "evidence": "url: 'https://example.org/x' on 2 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:url"}, {"action": "delete", "confidence": 0.92, "drop": "1998", "drop_count": 5, "drop_id": 5, "evidence": "date: '1998' on 5 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:date"}, {"action": "delete", "confidence": 0.92, "drop": "2024-01-15", "drop_count": 1, "drop_id": 6, "evidence": "date: '2024-01-15' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:date"}, {"action": "delete", "confidence": 0.92, "drop": "451", "drop_count": 2, "drop_id": 7, "evidence": "numeric-id: '451' on 2 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:numeric-id"}, {"action": "delete", "confidence": 0.92, "drop": "1:23:45", "drop_count": 1, "drop_id": 8, "evidence": "timestamp: '1:23:45' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:timestamp"}, {"action": "delete", "confidence": 0.92, "drop": "[kobo] favourites", "drop_count": 4, "drop_id": 9, "evidence": "device-tag: '[kobo] favourites' on 4 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:device-tag"}, {"action": "delete", "confidence": 0.92, "drop": "a;b;c;d", "drop_count": 1, "drop_id": 10, "evidence": "exploded-list: 'a;b;c;d' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:exploded-list"}, {"action": "delete", "confidence": 0.92, "drop": "---", "drop_count": 1, "drop_id": 11, "evidence": "punctuation-only: '---' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:punctuation-only"}, {"action": "delete", "confidence": 0.92, "drop": "#genre", "drop_count": 9, "drop_id": 12, "evidence": "format-or-watermark: '#genre' on 9 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:format-or-watermark"}, {"action": "delete", "confidence": 0.92, "drop": "ISBN-13", "drop_count": 3, "drop_id": 13, "evidence": "catalog-field: 'ISBN-13' on 3 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:catalog-field"}]} +{"canonical": "Church history", "keys": [{"author_tokens": ["a", "very", "long", "blurb", "used", "as", "a", "tag", "that", "goes", "on", "and", "on", "and", "on", "and", "should", "not", "be", "here", "at", "all", "really"], "junk": "blurb", "norm": "a very long blurb used as a tag that goes on and on and on and should not be here at all really", "pub_key": "a very long blurb used as a tag that goes on and on and on and should not be here at all really", "tag_key": "a very long blurb used as a tag that goes on and on and on and should not be here at all really"}, {"author_tokens": ["church", "history"], "junk": null, "norm": "church history", "pub_key": "church history", "tag_key": "church history"}], "proposals": [{"action": "delete", "confidence": 0.92, "drop": "A very long blurb used as a tag that goes on and on and on and should not be here at all really", "drop_count": 1, "drop_id": 1, "evidence": "blurb: 'A very long blurb used as a tag that goes on and on and on and should not be here at all really' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:blurb"}]} +{"canonical": "Fiction", "keys": [{"author_tokens": ["alexskobo", "to", "read"], "junk": "device-tag", "norm": "alexskobo to read", "pub_key": "alexskobo to read", "tag_key": "alexskobo to read"}, {"author_tokens": ["fiction"], "junk": null, "norm": "fiction", "pub_key": "fiction", "tag_key": "fiction"}], "proposals": [{"action": "delete", "confidence": 0.92, "drop": "[alexskobo] to read", "drop_count": 6, "drop_id": 1, "evidence": "device-tag: '[alexskobo] to read' on 6 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:device-tag"}]} +{"canonical": null, "keys": [], "proposals": []} diff --git a/parity/oracle/vocab.py b/parity/oracle/vocab.py new file mode 100644 index 0000000..e48f0c8 --- /dev/null +++ b/parity/oracle/vocab.py @@ -0,0 +1,73 @@ +"""Cluster vocabulary the way the Python pipeline clusters it. + +Reads one `{"kind", "items": [{"id","name","count"}]}` object per line and +writes the proposals each list produces. + + CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \ + python3 oracle/vocab.py < cases/vocab/lists.ndjson +""" + +import json +import os +import sys + +for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep): + if path: + sys.path.insert(0, path) + +from book_pipeline import vocab # noqa: E402 + + +def main() -> int: + out = sys.stdout + for line in sys.stdin.buffer.read().decode("utf-8").split("\n"): + if not line.strip(): + continue + case = json.loads(line) + items = [ + vocab.VocabItem(calibre_id=i["id"], name=i["name"], book_count=i["count"]) + for i in case["items"] + ] + proposals = vocab.cluster(case["kind"], items) + out.write( + json.dumps( + { + "keys": [ + { + "norm": vocab._norm(i.name), + "author_tokens": list(vocab._author_tokens(i.name)), + "pub_key": vocab._pub_key(i.name), + "tag_key": vocab._tag_key(i.name), + "junk": vocab.junk_tag_reason(i.name), + } + for i in items + ], + "canonical": ( + vocab.pick_canonical(case["kind"], items).name if items else None + ), + "proposals": [ + { + "action": p.action, + "keep": p.keep_name, + "drop": p.drop_name, + "keep_id": p.keep_calibre_id, + "drop_id": p.drop_calibre_id, + "keep_count": p.book_count_keep, + "drop_count": p.book_count_drop, + "method": p.method, + "confidence": p.confidence, + "evidence": p.evidence, + } + for p in proposals + ], + }, + ensure_ascii=False, + sort_keys=True, + ) + + "\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index 3547ae6..d81e949 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 13] = [ + let jobs: [Job; 14] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -33,6 +33,7 @@ fn main() -> ExitCode { ("bibclean", bibclean), ("slug", slug), ("author", author), + ("vocab", vocab), ]; for (name, job) in jobs { @@ -76,6 +77,18 @@ fn unescape() -> Written { Ok(vec![path.display().to_string()]) } +fn vocab() -> Written { + let dir = parity::golden("vocab"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/vocab/lists.ndjson"); + let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let judgements = parity::run_oracle("vocab.py", &cases)?; + let path = dir.join("lists.ndjson"); + std::fs::write(&path, judgements).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn author() -> Written { let dir = parity::golden("author"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/vocab.rs b/parity/tests/vocab.rs new file mode 100644 index 0000000..e6ac5c6 --- /dev/null +++ b/parity/tests/vocab.rs @@ -0,0 +1,139 @@ +//! Vocabulary clustering must match the Python exactly. +//! +//! A merge proposal is a suggestion to collapse two catalogue headings into +//! one. Proposing a different set than the system being replaced means either +//! merging two real people or leaving a person split in half, and the first +//! is much harder to undo than it is to do. + +use bibclean::vocab::{Item, Kind, Rules}; +use serde_json::Value; + +fn records(path: &str) -> Vec { + let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file"); + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn kind_of(case: &Value) -> Kind { + match case["kind"].as_str().expect("a kind") { + "author" => Kind::Author, + "publisher" => Kind::Publisher, + "tag" => Kind::Tag, + other => panic!("unknown kind {other}"), + } +} + +fn items_of(case: &Value) -> Vec { + case["items"] + .as_array() + .expect("a list") + .iter() + .map(|item| Item { + id: item["id"].as_i64().expect("an id"), + name: item["name"].as_str().expect("a name").to_string(), + count: item["count"].as_i64().expect("a count"), + }) + .collect() +} + +/// The device named after its owner is supplied rather than compiled in, so +/// the oracle's built-in list has to be handed over for the comparison. +fn rules() -> Rules { + Rules { + device_prefixes: vec!["alexskobo".into()], + } +} + +#[test] +fn every_cluster_matches_python() { + let cases = records("cases/vocab/lists.ndjson"); + let golden = records("golden/vocab/lists.ndjson"); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + let mut wrong: Vec = Vec::new(); + for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() { + let kind = kind_of(case); + let items = items_of(case); + + let mut differs = |what: &str, ours: String, theirs: &Value| { + wrong.push(format!("list {at} {what}: python {theirs}, rust {ours}")); + }; + + // The keys every grouping rests on. + for (item, theirs) in items.iter().zip(theirs["keys"].as_array().expect("a list")) { + let ours = bibclean::vocab::normalise(&item.name); + if Some(ours.as_str()) != theirs["norm"].as_str() { + differs("norm", ours, &theirs["norm"]); + } + let ours = Value::from(bibclean::vocab::author_tokens(&item.name)); + if ours != theirs["author_tokens"] { + differs("author_tokens", ours.to_string(), &theirs["author_tokens"]); + } + let ours = bibclean::vocab::publisher_key(&item.name); + if Some(ours.as_str()) != theirs["pub_key"].as_str() { + differs("pub_key", ours, &theirs["pub_key"]); + } + let ours = bibclean::vocab::tag_key(&item.name); + if Some(ours.as_str()) != theirs["tag_key"].as_str() { + differs("tag_key", ours, &theirs["tag_key"]); + } + let ours = bibclean::vocab::junk_reason(&item.name, &rules()); + if ours != theirs["junk"].as_str() { + differs("junk", format!("{ours:?}"), &theirs["junk"]); + } + } + + if !items.is_empty() { + let ours = bibclean::vocab::pick_canonical(kind, &items).name; + if Some(ours.as_str()) != theirs["canonical"].as_str() { + differs("canonical", ours, &theirs["canonical"]); + } + } + + let ours: Vec = bibclean::vocab::cluster(kind, &items, &rules()) + .iter() + .map(|proposal| { + serde_json::json!({ + "action": proposal.action, + "keep": proposal.keep, + "drop": proposal.drop, + "keep_id": proposal.keep_id, + "drop_id": proposal.drop_id, + "keep_count": proposal.keep_count, + "drop_count": proposal.drop_count, + "method": proposal.method, + "confidence": proposal.confidence, + "evidence": proposal.evidence, + }) + }) + .collect(); + let ours = Value::Array(ours); + if ours != theirs["proposals"] { + differs("proposals", ours.to_string(), &theirs["proposals"]); + } + } + + assert!( + wrong.is_empty(), + "{} judgements differ:\n{}", + wrong.len(), + wrong + .iter() + .take(10) + .map(|line| line.chars().take(400).collect::()) + .collect::>() + .join("\n") + ); +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/vocab/lists.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("vocab.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("vocab/lists.ndjson")); +}