From 14e1fcf3ef029165c2ca8c3753bb58a43e2b940d Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 14:23:36 -0400 Subject: [PATCH] bibclean: slugs, and whether the one a record has still names it --- Cargo.lock | 23 + Cargo.toml | 2 + crates/bibclean/Cargo.toml | 2 + crates/bibclean/README.md | 46 +- crates/bibclean/src/entities4.rs | 261 ++++++++++++ crates/bibclean/src/lib.rs | 68 +++ crates/bibclean/src/main.rs | 13 +- crates/bibclean/src/slug.rs | 677 ++++++++++++++++++++++++++++++ crates/cueseg/src/lib.rs | 26 +- crates/textkit/src/lib.rs | 2 +- crates/textkit/src/number.rs | 41 ++ parity/cases/slug/records.ndjson | 25 ++ parity/golden/slug/records.ndjson | 25 ++ parity/oracle/slug.py | 50 +++ parity/src/bin/regen.rs | 15 +- parity/tests/slug.rs | 100 +++++ 16 files changed, 1342 insertions(+), 34 deletions(-) create mode 100644 crates/bibclean/src/entities4.rs create mode 100644 crates/bibclean/src/slug.rs create mode 100644 parity/cases/slug/records.ndjson create mode 100644 parity/golden/slug/records.ndjson create mode 100644 parity/oracle/slug.py create mode 100644 parity/tests/slug.rs diff --git a/Cargo.lock b/Cargo.lock index eb7595c..67a2eb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,12 +92,14 @@ dependencies = [ "catena-cli", "clap", "corpus-core", + "deunicode", "fancy-regex 0.16.2", "regex", "schemars", "serde", "serde_json", "textkit", + "unicode-normalization", ] [[package]] @@ -332,6 +334,12 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "deunicode" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" + [[package]] name = "displaydoc" version = "0.2.7" @@ -1070,6 +1078,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -1082,6 +1096,15 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 6f62e59..65d38b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } clap = { version = "4", features = ["derive"] } +deunicode = "1" fancy-regex = "0.16" regex = "1" schemars = "1" @@ -36,6 +37,7 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["float_roundtrip"] } thiserror = "2" unicode-general-category = "1" +unicode-normalization = "0.1" ureq = { version = "3", features = ["json"] } tiktoken-rs = "0.12" diff --git a/crates/bibclean/Cargo.toml b/crates/bibclean/Cargo.toml index 825496d..adb2eae 100644 --- a/crates/bibclean/Cargo.toml +++ b/crates/bibclean/Cargo.toml @@ -11,12 +11,14 @@ repository.workspace = true catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true +deunicode.workspace = true fancy-regex.workspace = true regex.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true textkit.workspace = true +unicode-normalization.workspace = true [lints] workspace = true diff --git a/crates/bibclean/README.md b/crates/bibclean/README.md index e41c6e4..89b7257 100644 --- a/crates/bibclean/README.md +++ b/crates/bibclean/README.md @@ -76,21 +76,57 @@ lacks one. | Code | Meaning | |---|---| | `0` | Nothing needed attention | -| `1` | Something did | +| `1` | Something did — a fault worth fixing, or a slug worth replacing | | `64` | The command line was wrong | | `65` | A record could not be read | | `74` | The file could not be read | ## Input -One JSON record per line. `title`, `author`, `isbn` and `path` are read; -everything else is carried through untouched, so a record can go through this -and back into whatever it came from. +One JSON record per line. `title`, `author`, `isbn`, `path`, `slug` and `id` +are read; everything else is carried through untouched, so a record can go +through this and back into whatever it came from. ```json {"title":"THE HUMAN CONDITION","author":"Arendt, Hannah","path":"/library/9780226586601.pdf"} ``` +## Slugs + +A slug is a public identifier — it reaches citations and notes, and outlives +whatever produced it. When a record carries one, along with the `id` that +makes it unique, its slug is judged too: + +``` +$ bibclean slugs.ndjson +{"v":1,"title":"Weaponizing Anthropology","author":"Laura Nader",…,"slug":"william-zinsser-on-writing-well-3451","id":3451,"reasons":[],"severity":0,"changes":[],"slug_proposal":{"current":"william-zinsser-on-writing-well-3451","proposed":"laura-nader-weaponizing-anthropology-3451","tier":"wrong-book","reasons":["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"]}} +{"v":1,"title":"Mind and Brain","author":"Zinsser, William",…,"slug":"unknown-mind-and-brain-1234","id":1234,"reasons":[],"severity":0,"changes":[],"slug_proposal":{"current":"unknown-mind-and-brain-1234","proposed":"zinsser-william-mind-and-brain-1234","tier":"placeholder-author","reasons":["slug opens with placeholder author 'unknown'"]}} +``` + +Nothing is renamed. A proposal comes back beside the slug the record has, and +the **tier** says how much the evidence is worth: + +| Tier | Evidence | Safe to apply in bulk? | +|---|---|---| +| `placeholder-author` | The slug opens with a placeholder, so it was minted without an author | yes | +| `filename-artefact` | The slug carries a mirror suffix, a wrapper format or a path | yes | +| `wrong-book` | Too few of the slug's words appear in the record at all | no | +| `noop-metadata-junk` | Regenerating gives the same slug: the record is the problem | no | +| `defer-new-slug-junk` | The new slug would be no better than the old one | no | +| `review-loses-volume` | The current slug names a volume the new one would lose | no | +| `review-low-recall` | The new slug recovers too little of the title | no | + +The first two rest on evidence inside the slug itself and cannot be wrong +about it. The rest are *disagreements* between the slug and the record, and a +disagreement does not say which side is at fault — which is exactly the case +where renaming automatically makes things worse. + +Only the safe tiers: + +```sh +bibclean books.ndjson | jq -c 'select(.slug_proposal.tier | IN("placeholder-author","filename-artefact"))' +``` + ## The faults it names | Reason | Weight | What it means | @@ -140,7 +176,7 @@ if bibclean::reasons::worth_fixing(&cleaned.reasons) { The pieces are usable on their own: `bibclean::isbn::valid`, `bibclean::isbn::from_filename`, `bibclean::fix::title_case`, -`bibclean::matching::same_work`. +`bibclean::matching::same_work`, `bibclean::slug::slug_for`. `same_work` is worth knowing about separately. It compares vocabulary rather than characters, because character similarity is not usable for titles: "The diff --git a/crates/bibclean/src/entities4.rs b/crates/bibclean/src/entities4.rs new file mode 100644 index 0000000..6091c71 --- /dev/null +++ b/crates/bibclean/src/entities4.rs @@ -0,0 +1,261 @@ +//! The named character references a slug generator decodes. +//! +//! The HTML 4 set, which is smaller than the HTML 5 one and is what the +//! slug generators in wide use recognise. A name outside it is left as +//! text, and becomes a word in the slug like any other. + +/// Named references, sorted by name, without their semicolons. +pub static NAMED: &[(&str, char)] = &[ + ("AElig", '\u{c6}'), + ("Aacute", '\u{c1}'), + ("Acirc", '\u{c2}'), + ("Agrave", '\u{c0}'), + ("Alpha", '\u{391}'), + ("Aring", '\u{c5}'), + ("Atilde", '\u{c3}'), + ("Auml", '\u{c4}'), + ("Beta", '\u{392}'), + ("Ccedil", '\u{c7}'), + ("Chi", '\u{3a7}'), + ("Dagger", '\u{2021}'), + ("Delta", '\u{394}'), + ("ETH", '\u{d0}'), + ("Eacute", '\u{c9}'), + ("Ecirc", '\u{ca}'), + ("Egrave", '\u{c8}'), + ("Epsilon", '\u{395}'), + ("Eta", '\u{397}'), + ("Euml", '\u{cb}'), + ("Gamma", '\u{393}'), + ("Iacute", '\u{cd}'), + ("Icirc", '\u{ce}'), + ("Igrave", '\u{cc}'), + ("Iota", '\u{399}'), + ("Iuml", '\u{cf}'), + ("Kappa", '\u{39a}'), + ("Lambda", '\u{39b}'), + ("Mu", '\u{39c}'), + ("Ntilde", '\u{d1}'), + ("Nu", '\u{39d}'), + ("OElig", '\u{152}'), + ("Oacute", '\u{d3}'), + ("Ocirc", '\u{d4}'), + ("Ograve", '\u{d2}'), + ("Omega", '\u{3a9}'), + ("Omicron", '\u{39f}'), + ("Oslash", '\u{d8}'), + ("Otilde", '\u{d5}'), + ("Ouml", '\u{d6}'), + ("Phi", '\u{3a6}'), + ("Pi", '\u{3a0}'), + ("Prime", '\u{2033}'), + ("Psi", '\u{3a8}'), + ("Rho", '\u{3a1}'), + ("Scaron", '\u{160}'), + ("Sigma", '\u{3a3}'), + ("THORN", '\u{de}'), + ("Tau", '\u{3a4}'), + ("Theta", '\u{398}'), + ("Uacute", '\u{da}'), + ("Ucirc", '\u{db}'), + ("Ugrave", '\u{d9}'), + ("Upsilon", '\u{3a5}'), + ("Uuml", '\u{dc}'), + ("Xi", '\u{39e}'), + ("Yacute", '\u{dd}'), + ("Yuml", '\u{178}'), + ("Zeta", '\u{396}'), + ("aacute", '\u{e1}'), + ("acirc", '\u{e2}'), + ("acute", '\u{b4}'), + ("aelig", '\u{e6}'), + ("agrave", '\u{e0}'), + ("alefsym", '\u{2135}'), + ("alpha", '\u{3b1}'), + ("amp", '&'), + ("and", '\u{2227}'), + ("ang", '\u{2220}'), + ("aring", '\u{e5}'), + ("asymp", '\u{2248}'), + ("atilde", '\u{e3}'), + ("auml", '\u{e4}'), + ("bdquo", '\u{201e}'), + ("beta", '\u{3b2}'), + ("brvbar", '\u{a6}'), + ("bull", '\u{2022}'), + ("cap", '\u{2229}'), + ("ccedil", '\u{e7}'), + ("cedil", '\u{b8}'), + ("cent", '\u{a2}'), + ("chi", '\u{3c7}'), + ("circ", '\u{2c6}'), + ("clubs", '\u{2663}'), + ("cong", '\u{2245}'), + ("copy", '\u{a9}'), + ("crarr", '\u{21b5}'), + ("cup", '\u{222a}'), + ("curren", '\u{a4}'), + ("dArr", '\u{21d3}'), + ("dagger", '\u{2020}'), + ("darr", '\u{2193}'), + ("deg", '\u{b0}'), + ("delta", '\u{3b4}'), + ("diams", '\u{2666}'), + ("divide", '\u{f7}'), + ("eacute", '\u{e9}'), + ("ecirc", '\u{ea}'), + ("egrave", '\u{e8}'), + ("empty", '\u{2205}'), + ("emsp", '\u{2003}'), + ("ensp", '\u{2002}'), + ("epsilon", '\u{3b5}'), + ("equiv", '\u{2261}'), + ("eta", '\u{3b7}'), + ("eth", '\u{f0}'), + ("euml", '\u{eb}'), + ("euro", '\u{20ac}'), + ("exist", '\u{2203}'), + ("fnof", '\u{192}'), + ("forall", '\u{2200}'), + ("frac12", '\u{bd}'), + ("frac14", '\u{bc}'), + ("frac34", '\u{be}'), + ("frasl", '\u{2044}'), + ("gamma", '\u{3b3}'), + ("ge", '\u{2265}'), + ("gt", '>'), + ("hArr", '\u{21d4}'), + ("harr", '\u{2194}'), + ("hearts", '\u{2665}'), + ("hellip", '\u{2026}'), + ("iacute", '\u{ed}'), + ("icirc", '\u{ee}'), + ("iexcl", '\u{a1}'), + ("igrave", '\u{ec}'), + ("image", '\u{2111}'), + ("infin", '\u{221e}'), + ("int", '\u{222b}'), + ("iota", '\u{3b9}'), + ("iquest", '\u{bf}'), + ("isin", '\u{2208}'), + ("iuml", '\u{ef}'), + ("kappa", '\u{3ba}'), + ("lArr", '\u{21d0}'), + ("lambda", '\u{3bb}'), + ("lang", '\u{2329}'), + ("laquo", '\u{ab}'), + ("larr", '\u{2190}'), + ("lceil", '\u{2308}'), + ("ldquo", '\u{201c}'), + ("le", '\u{2264}'), + ("lfloor", '\u{230a}'), + ("lowast", '\u{2217}'), + ("loz", '\u{25ca}'), + ("lrm", '\u{200e}'), + ("lsaquo", '\u{2039}'), + ("lsquo", '\u{2018}'), + ("lt", '<'), + ("macr", '\u{af}'), + ("mdash", '\u{2014}'), + ("micro", '\u{b5}'), + ("middot", '\u{b7}'), + ("minus", '\u{2212}'), + ("mu", '\u{3bc}'), + ("nabla", '\u{2207}'), + ("nbsp", '\u{a0}'), + ("ndash", '\u{2013}'), + ("ne", '\u{2260}'), + ("ni", '\u{220b}'), + ("not", '\u{ac}'), + ("notin", '\u{2209}'), + ("nsub", '\u{2284}'), + ("ntilde", '\u{f1}'), + ("nu", '\u{3bd}'), + ("oacute", '\u{f3}'), + ("ocirc", '\u{f4}'), + ("oelig", '\u{153}'), + ("ograve", '\u{f2}'), + ("oline", '\u{203e}'), + ("omega", '\u{3c9}'), + ("omicron", '\u{3bf}'), + ("oplus", '\u{2295}'), + ("or", '\u{2228}'), + ("ordf", '\u{aa}'), + ("ordm", '\u{ba}'), + ("oslash", '\u{f8}'), + ("otilde", '\u{f5}'), + ("otimes", '\u{2297}'), + ("ouml", '\u{f6}'), + ("para", '\u{b6}'), + ("part", '\u{2202}'), + ("permil", '\u{2030}'), + ("perp", '\u{22a5}'), + ("phi", '\u{3c6}'), + ("pi", '\u{3c0}'), + ("piv", '\u{3d6}'), + ("plusmn", '\u{b1}'), + ("pound", '\u{a3}'), + ("prime", '\u{2032}'), + ("prod", '\u{220f}'), + ("prop", '\u{221d}'), + ("psi", '\u{3c8}'), + ("quot", '"'), + ("rArr", '\u{21d2}'), + ("radic", '\u{221a}'), + ("rang", '\u{232a}'), + ("raquo", '\u{bb}'), + ("rarr", '\u{2192}'), + ("rceil", '\u{2309}'), + ("rdquo", '\u{201d}'), + ("real", '\u{211c}'), + ("reg", '\u{ae}'), + ("rfloor", '\u{230b}'), + ("rho", '\u{3c1}'), + ("rlm", '\u{200f}'), + ("rsaquo", '\u{203a}'), + ("rsquo", '\u{2019}'), + ("sbquo", '\u{201a}'), + ("scaron", '\u{161}'), + ("sdot", '\u{22c5}'), + ("sect", '\u{a7}'), + ("shy", '\u{ad}'), + ("sigma", '\u{3c3}'), + ("sigmaf", '\u{3c2}'), + ("sim", '\u{223c}'), + ("spades", '\u{2660}'), + ("sub", '\u{2282}'), + ("sube", '\u{2286}'), + ("sum", '\u{2211}'), + ("sup", '\u{2283}'), + ("sup1", '\u{b9}'), + ("sup2", '\u{b2}'), + ("sup3", '\u{b3}'), + ("supe", '\u{2287}'), + ("szlig", '\u{df}'), + ("tau", '\u{3c4}'), + ("there4", '\u{2234}'), + ("theta", '\u{3b8}'), + ("thetasym", '\u{3d1}'), + ("thinsp", '\u{2009}'), + ("thorn", '\u{fe}'), + ("tilde", '\u{2dc}'), + ("times", '\u{d7}'), + ("trade", '\u{2122}'), + ("uArr", '\u{21d1}'), + ("uacute", '\u{fa}'), + ("uarr", '\u{2191}'), + ("ucirc", '\u{fb}'), + ("ugrave", '\u{f9}'), + ("uml", '\u{a8}'), + ("upsih", '\u{3d2}'), + ("upsilon", '\u{3c5}'), + ("uuml", '\u{fc}'), + ("weierp", '\u{2118}'), + ("xi", '\u{3be}'), + ("yacute", '\u{fd}'), + ("yen", '\u{a5}'), + ("yuml", '\u{ff}'), + ("zeta", '\u{3b6}'), + ("zwj", '\u{200d}'), + ("zwnj", '\u{200c}'), +]; diff --git a/crates/bibclean/src/lib.rs b/crates/bibclean/src/lib.rs index 11cd78f..94d688c 100644 --- a/crates/bibclean/src/lib.rs +++ b/crates/bibclean/src/lib.rs @@ -11,10 +11,12 @@ //! which is the difference between a tool that cleans a catalogue and one //! that damages it in a new way. +mod entities4; pub mod fix; pub mod isbn; pub mod matching; pub mod reasons; +pub mod slug; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -41,6 +43,12 @@ pub struct Book { /// Where the file is, which is often where a missing ISBN is hiding. #[serde(default)] pub path: Option, + /// The public identifier the record already has, if it has one. + #[serde(default)] + pub slug: Option, + /// A number that makes a slug unique. Needed to judge or regenerate one. + #[serde(default)] + pub id: Option, /// Everything else the record carried. #[serde(flatten)] #[schemars(with = "Map")] @@ -82,6 +90,17 @@ pub struct Cleaned { pub severity: u32, /// What was changed, and why. pub changes: Vec, + /// What should happen to the record's slug, when it has one to judge. + /// + /// Named apart from the record's own `slug`, which is carried through + /// beside it — the two would otherwise be one field and the proposal + /// would quietly replace the fact. + /// + /// Never applied as a change: a slug is a public identifier that other + /// things have already written down, so moving one is a decision with + /// consequences outside this record. + #[serde(skip_serializing_if = "Option::is_none")] + pub slug_proposal: Option, } /// Audit a record and make the repairs that need no lookup. @@ -143,10 +162,21 @@ pub fn clean(book: &Book) -> Cleaned { cleaned.isbn = Some(found); } + // Judged against the record as it arrived, not as it was repaired: the + // question is whether the slug describes the book, and a title this just + // title-cased describes the same book either way. + let proposal = match (book.slug.as_deref(), book.id) { + (Some(slug), Some(id)) => { + slug::evaluate(slug, book.author.as_deref(), book.title.as_deref(), id) + } + _ => None, + }; + Cleaned { severity: reasons::severity(&reasons), reasons, changes, + slug_proposal: proposal, book: cleaned, } } @@ -162,6 +192,8 @@ mod tests { author: Some(author.into()), isbn: None, path: None, + slug: None, + id: None, extra: Map::new(), } } @@ -215,6 +247,42 @@ mod tests { assert_eq!(clean(&book).book.isbn, None); } + #[test] + fn a_slug_is_judged_when_there_is_one_to_judge() { + let mut book = book("Weaponizing Anthropology", "Laura Nader"); + book.slug = Some("william-zinsser-on-writing-well-3450".into()); + book.id = Some(3450); + + let cleaned = clean(&book); + let proposal = cleaned.slug_proposal.expect("a proposal"); + assert_eq!(proposal.tier, "wrong-book"); + assert_eq!( + proposal.proposed, + "laura-nader-weaponizing-anthropology-3450" + ); + // Proposed, never applied. + assert_eq!( + cleaned.book.slug.as_deref(), + Some("william-zinsser-on-writing-well-3450") + ); + } + + #[test] + fn the_record_keeps_its_own_slug_beside_the_proposal() { + let mut book = book("Weaponizing Anthropology", "Laura Nader"); + book.slug = Some("william-zinsser-on-writing-well-3450".into()); + book.id = Some(3450); + + let json = serde_json::to_value(clean(&book)).expect("a record serialises"); + assert_eq!(json["slug"], "william-zinsser-on-writing-well-3450"); + assert_eq!(json["slug_proposal"]["tier"], "wrong-book"); + } + + #[test] + fn a_record_with_no_slug_gets_no_proposal() { + assert!(clean(&book("A Book", "An Author")).slug_proposal.is_none()); + } + #[test] fn fields_it_does_not_know_about_are_carried_through() { let mut book = book("A Book", "An Author"); diff --git a/crates/bibclean/src/main.rs b/crates/bibclean/src/main.rs index e106fe1..47ca25c 100644 --- a/crates/bibclean/src/main.rs +++ b/crates/bibclean/src/main.rs @@ -39,6 +39,15 @@ struct Args { flagged: bool, } +/// Whether anything about a record wants a person's attention. +/// +/// A missing ISBN alone does not: nearly every book lacks one. A slug that +/// should be replaced does, even when nothing else is wrong, because the +/// identifier is the part other things have written down. +fn needs_attention(cleaned: &bibclean::Cleaned) -> bool { + bibclean::reasons::worth_fixing(&cleaned.reasons) || cleaned.slug_proposal.is_some() +} + fn main() -> ExitCode { let args: Args = match catena_cli::parse() { Ok(args) => args, @@ -65,8 +74,8 @@ fn main() -> ExitCode { let cleaned: Vec = books .iter() .map(bibclean::clean) - .inspect(|cleaned| flagged |= bibclean::reasons::worth_fixing(&cleaned.reasons)) - .filter(|cleaned| !args.flagged || bibclean::reasons::worth_fixing(&cleaned.reasons)) + .inspect(|cleaned| flagged |= needs_attention(cleaned)) + .filter(|cleaned| !args.flagged || needs_attention(cleaned)) .collect(); let wrote = catena_cli::stream("bibclean", &cleaned); diff --git a/crates/bibclean/src/slug.rs b/crates/bibclean/src/slug.rs new file mode 100644 index 0000000..d5dc15d --- /dev/null +++ b/crates/bibclean/src/slug.rs @@ -0,0 +1,677 @@ +//! Turning a record into a readable identifier, and judging the one it has. + +use std::sync::LazyLock; + +use regex::Regex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use unicode_normalization::UnicodeNormalization; + +use crate::entities4::NAMED; + +/// Apostrophes. The ones written in a title become a separator; the ones +/// transliteration produces afterwards — a Cyrillic hard sign becomes one — +/// are dropped, so they do not each turn into a dash of their own. +static QUOTES: LazyLock = LazyLock::new(|| Regex::new(r"[']+").expect("valid")); + +/// A named character reference. +static NAMED_REFERENCE: LazyLock = + LazyLock::new(|| Regex::new(r"&([a-zA-Z][a-zA-Z0-9]*);").expect("valid")); + +static DECIMAL_REFERENCE: LazyLock = + LazyLock::new(|| Regex::new(r"&#(\d+);").expect("valid")); + +static HEX_REFERENCE: LazyLock = + LazyLock::new(|| Regex::new(r"&#x([\da-fA-F]+);").expect("valid")); + +/// A comma inside a number, which is a thousands separator rather than a word +/// break: `12,000` is one number. +static THOUSANDS: LazyLock = LazyLock::new(|| Regex::new(r"(\d),(\d)").expect("valid")); + +/// Everything a slug may not contain. +static DISALLOWED: LazyLock = + LazyLock::new(|| Regex::new(r"[^-a-zA-Z0-9]+").expect("valid")); + +static DASH_RUN: LazyLock = LazyLock::new(|| Regex::new(r"-{2,}").expect("valid")); + +/// How many characters a whole slug may take. +pub const MAX_LENGTH: usize = 80; + +/// How many of them the lead author may take. +/// +/// The title always needs room. Measured against a review queue: a +/// five-editor volume otherwise consumed the whole budget and left a slug +/// naming only people, which cannot be recognised as a book. +pub const AUTHOR_BUDGET: usize = 30; + +/// Reduce a text to a slug. +/// +/// Transliterated to ASCII, lowercased, everything else turned into a +/// separator, and truncated at a word if a maximum is given. +pub fn slugify(text: &str, max_length: usize, word_boundary: bool) -> String { + let text = QUOTES.replace_all(text, "-"); + let text: String = deunicode::deunicode(&text.nfkd().collect::()); + + let text = NAMED_REFERENCE.replace_all(&text, |found: ®ex::Captures| { + match NAMED.binary_search_by_key(&&found[1], |(name, _)| name) { + Ok(at) => NAMED[at].1.to_string(), + Err(_) => found[0].to_string(), + } + }); + let text = DECIMAL_REFERENCE.replace_all(&text, |found: ®ex::Captures| { + character(found[1].parse::().ok(), &found[0]) + }); + let text = HEX_REFERENCE.replace_all(&text, |found: ®ex::Captures| { + character(u32::from_str_radix(&found[1], 16).ok(), &found[0]) + }); + + let text: String = text.nfkd().collect::().to_lowercase(); + let text = QUOTES.replace_all(&text, ""); + let text = THOUSANDS.replace_all(&text, "$1$2"); + let text = DISALLOWED.replace_all(&text, "-"); + let text = DASH_RUN.replace_all(&text, "-"); + let text = text.trim_matches('-'); + + match max_length { + 0 => text.to_string(), + limit => truncate(text, limit, word_boundary), + } +} + +fn character(code: Option, whole: &str) -> String { + code.and_then(char::from_u32) + .map_or_else(|| whole.to_string(), String::from) +} + +/// Cut a slug to `limit` characters. +/// +/// At a word, when asked — and a word that does not fit is skipped rather +/// than ending the slug, so a long word in the middle of a title does not +/// cost every word after it. `an-introduction-to-historical-analysis` cut to +/// twenty-eight is `an-introduction-to-analysis`. +fn truncate(text: &str, limit: usize, word_boundary: bool) -> String { + if text.len() < limit { + return text.to_string(); + } + if !word_boundary { + return text[..floor_boundary(text, limit)] + .trim_matches('-') + .to_string(); + } + if !text.contains('-') { + return text[..floor_boundary(text, limit)].to_string(); + } + + let mut kept = String::new(); + for word in text.split('-') { + if word.is_empty() { + continue; + } + let would_be = kept.len() + word.len(); + if would_be < limit { + kept.push_str(word); + kept.push('-'); + } else if would_be == limit { + kept.push_str(word); + break; + } + } + if kept.is_empty() { + kept = text[..floor_boundary(text, limit)].to_string(); + } + kept.trim_matches('-').to_string() +} + +/// The largest character boundary at or below `at`, so a cut never lands +/// inside a character. +fn floor_boundary(text: &str, at: usize) -> usize { + let mut at = at.min(text.len()); + while at > 0 && !text.is_char_boundary(at) { + at -= 1; + } + at +} + +/// The first name in an author field, which may list several. +/// +/// An edited volume can carry five names, and the whole list ahead of the +/// title spends the entire budget on people. An identifier needs one name and +/// the work; the rest are in the record. +pub fn lead_author(author: &str) -> String { + static SEPARATOR: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\s*[&;]\s*|\s+and\s+").expect("valid")); + + let author = author.trim_matches(textkit::is_space); + let lead = SEPARATOR.splitn(author, 2).next().unwrap_or(""); + // Commas are ambiguous: "Bishop, Christopher M." is one person, but + // "Miranda Lundy, Daud Sutton, Anthony Ashton" is three — and taking the + // whole list truncates it mid-name at the budget. Two or more commas + // means a list; one means a surname first. + let lead = match lead.matches(',').count() >= 2 { + true => lead.split_once(',').map_or(lead, |(first, _)| first), + false => lead, + }; + lead.trim_matches(|c: char| textkit::is_space(c) || c == ',') + .to_string() +} + +/// The slug a record's current fields imply. +/// +/// The author is reduced to its lead name and given a bounded share of the +/// budget, so the title is always represented. +pub fn slug_for(author: Option<&str>, title: Option<&str>, id: i64) -> String { + let lead = slugify(&lead_author(author.unwrap_or("")), AUTHOR_BUDGET, true); + let room = MAX_LENGTH.saturating_sub(if lead.is_empty() { 0 } else { lead.len() + 1 }); + let work = slugify(title.unwrap_or(""), room.max(20), true); + + let base = [lead.as_str(), work.as_str()] + .into_iter() + .filter(|part| !part.is_empty()) + .collect::>() + .join("-"); + match base.is_empty() { + true => format!("book-{id}"), + false => format!("{base}-{id}"), + } +} + +/// Head tokens that mean the slug was minted without an author. +/// +/// The slug's first token comes from the author field, so a placeholder there +/// is direct evidence the slug was made blind — independent of whatever the +/// record says now. +const PLACEHOLDER_AUTHORS: &[&str] = &[ + "admin", + "administrator", + "anonymous", + "book", + "default", + "guest", + "imac", + "na", + "none", + "someauthor", + "test", + "unknown", + "untitled", + "user", +]; + +/// Words that carry no weight when deciding whether a slug and a record are +/// describing the same book. +const STOPWORDS: &[&str] = &[ + "a", "an", "and", "as", "at", "by", "de", "for", "from", "in", "is", "la", "le", "of", "on", + "or", "the", "to", "with", +]; + +/// Below this share of the slug's meaningful tokens appearing in the current +/// title or author, the slug is describing some other book. +/// +/// Deliberately low: above it sit ordinary drifts — a subtitle dropped, an +/// editor removed — and a disagreement this size is held for review either +/// way. +const WRONG_BOOK_PRECISION: f64 = 0.34; + +/// A new slug that recovers less than this much of the current title is not +/// clearly an improvement. +const MIN_TITLE_RECALL: f64 = 0.35; + +/// Traces of the file a book arrived as, rather than of the book: mirror +/// suffixes, a wrapper format, a word processor's name, a Windows path. +static ARTEFACT: LazyLock = LazyLock::new(|| { + Regex::new( + r"z-lib|1lib|zlibrary|djvu|microsoft-word|documents-and-settings|(?:^|-)(?:pdf|doc|docx|txt|tmp|epub|mobi|azw3|rtf|prnt|qxd|qxp|fm)(?:-|$)", + ) + .expect("a valid pattern") +}); + +/// A volume marker. +/// +/// One in the current slug is worth more than a clean slug without: several +/// volumes of a proceedings share a title, and their existing slugs are what +/// tell them apart. +static VOLUME: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^|-)(?:vol|volume|part|book|bk|no|nr|tome|band)-?(?:\d+|[ivxl]+)(?:-|$)") + .expect("a valid pattern") +}); + +/// An author field that has swallowed part of the title or the series. +/// +/// A slug built from that names nothing, and it is junk of a kind the +/// placeholder and artefact tests cannot see, because the text is neither a +/// handle nor a filename. +static SPILLED_AUTHOR: LazyLock = LazyLock::new(|| { + use textkit::pattern::{WORD_END, WORD_START}; + fancy_regex::Regex::new(&format!( + r"(?i)[()\[\]]|{WORD_START}volume{WORD_END}|{WORD_START}an introduction{WORD_END}" + )) + .expect("a valid pattern") +}); + +/// Tiers that rest on evidence inside the slug itself, and so can be applied +/// without a person reading them. +/// +/// A placeholder author or a filename artefact is a property of the slug and +/// cannot be wrong about itself. Everything else is a *disagreement* between +/// the slug and the record, and a disagreement does not say which side is at +/// fault. +pub const SAFE_TIERS: &[&str] = &["filename-artefact", "placeholder-author"]; + +/// What should happen to a record's slug. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Proposal { + /// The slug the record has. + pub current: String, + /// The slug its fields imply. + pub proposed: String, + /// How much the evidence is worth; see [`SAFE_TIERS`]. + pub tier: String, + /// Why, in the order the evidence was found. + pub reasons: Vec, +} + +impl Proposal { + /// Whether this can be applied without a person reading it. + pub fn is_safe(&self) -> bool { + SAFE_TIERS.binary_search(&self.tier.as_str()).is_ok() + } +} + +/// Judge a record's slug. `None` when there is nothing wrong with it. +/// +/// What comes back is a tier rather than a verdict, because the three kinds +/// of evidence differ in kind and only two of them are about the slug alone. +pub fn evaluate( + current: &str, + author: Option<&str>, + title: Option<&str>, + id: i64, +) -> Option { + let body = strip_id(current); + let body_tokens = tokens(body); + let title_tokens: Vec = tokens(&slugify(title.unwrap_or(""), 0, false)) + .into_iter() + .map(str::to_string) + .collect(); + let author_tokens: Vec = tokens(&slugify(author.unwrap_or(""), 0, false)) + .into_iter() + .map(str::to_string) + .collect(); + + let meaningful: Vec<&str> = body_tokens + .iter() + .copied() + .filter(|token| STOPWORDS.binary_search(token).is_err() && token.chars().count() > 1) + .collect(); + let precision = match meaningful.is_empty() { + true => 0.0, + false => { + let matched = meaningful + .iter() + .filter(|token| { + title_tokens.iter().any(|t| t == *token) + || author_tokens.iter().any(|t| t == *token) + }) + .count(); + matched as f64 / meaningful.len() as f64 + } + }; + + let author_text = author.unwrap_or("").trim_matches(textkit::is_space); + let author_known = !author_text.is_empty() && author.unwrap_or("") != "Unknown"; + + let mut reasons = Vec::new(); + let tier = if body_tokens + .first() + .is_some_and(|head| PLACEHOLDER_AUTHORS.binary_search(head).is_ok()) + && author_known + { + reasons.push(format!( + "slug opens with placeholder author '{}'", + body_tokens[0] + )); + "placeholder-author" + } else if ARTEFACT.is_match(body) { + reasons.push("slug carries a filename or mirror artefact".to_string()); + "filename-artefact" + } else if precision < WRONG_BOOK_PRECISION { + reasons.push(format!( + "only {} of slug tokens appear in the current title/author — slug and metadata disagree", + textkit::percent(precision) + )); + "wrong-book" + } else { + return None; + }; + + let proposed = slug_for(author, title, id); + let proposed_body = strip_id(&proposed); + let proposed_tokens = tokens(proposed_body); + + if proposed == current { + reasons.push("recomputes to the same slug: fix the metadata first".into()); + return Some(Proposal { + current: current.into(), + proposed, + tier: "noop-metadata-junk".into(), + reasons, + }); + } + + let recall = |slug_tokens: &[&str]| -> f64 { + match title_tokens.is_empty() { + true => 0.0, + false => { + let found = title_tokens + .iter() + .filter(|token| slug_tokens.iter().any(|t| t == token)) + .count(); + found as f64 / title_tokens.len() as f64 + } + } + }; + let proposed_recall = recall(&proposed_tokens); + let recovers_title = proposed_recall > recall(&body_tokens); + + // Is the proposal better than what is there, even if imperfect? A slug + // still saying "unknown" because the author genuinely is, but now naming + // the book, beats one naming a temporary directory. So the test is not + // "is the new slug clean" but "does it recover title the current one + // lacks, without carrying an artefact of its own". + let still_placeholder = proposed_tokens + .first() + .is_some_and(|head| PLACEHOLDER_AUTHORS.binary_search(head).is_ok()); + if ARTEFACT.is_match(proposed_body) || (still_placeholder && !recovers_title) { + reasons.push("new slug is no better: fix the metadata first".into()); + return Some(Proposal { + current: current.into(), + proposed, + tier: "defer-new-slug-junk".into(), + reasons, + }); + } + + // Only when the author actually reaches the slug: an author field of + // nothing but brackets slugifies to nothing, so the proposal leads with + // the title and is fine. + let lead_in_slug = slugify(&lead_author(author_text), AUTHOR_BUDGET, true); + if !lead_in_slug.is_empty() && SPILLED_AUTHOR.is_match(author_text).unwrap_or(false) { + reasons.push(format!( + "author field looks like spilled title/series text: '{author_text}'" + )); + return Some(Proposal { + current: current.into(), + proposed, + tier: "defer-new-slug-junk".into(), + reasons, + }); + } + + if VOLUME.is_match(body) && !VOLUME.is_match(proposed_body) { + reasons.push("current slug carries a volume marker the new one loses".into()); + return Some(Proposal { + current: current.into(), + proposed, + tier: "review-loses-volume".into(), + reasons, + }); + } + + // Measured on the proposal itself rather than recomputed from the fields: + // the generator budgets the author separately, so a formula that + // re-derives the length describes a slug it no longer builds. + let truncated = proposed_body.len() >= MAX_LENGTH - 1; + if truncated || proposed_recall < MIN_TITLE_RECALL { + reasons.push(format!( + "new slug recovers only {} of the title", + textkit::percent(proposed_recall) + )); + return Some(Proposal { + current: current.into(), + proposed, + tier: "review-low-recall".into(), + reasons, + }); + } + + Some(Proposal { + current: current.into(), + proposed, + tier: tier.into(), + reasons, + }) +} + +/// A slug without its trailing numeric id. +fn strip_id(slug: &str) -> &str { + static TRAILING_ID: LazyLock = LazyLock::new(|| Regex::new(r"-\d+$").expect("valid")); + match TRAILING_ID.find(slug) { + Some(found) if found.end() == slug.len() => &slug[..found.start()], + _ => slug, + } +} + +fn tokens(slug: &str) -> Vec<&str> { + slug.split('-').filter(|token| !token.is_empty()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_entity_table_is_sorted_so_it_can_be_searched() { + assert!(NAMED.windows(2).all(|pair| pair[0].0 < pair[1].0)); + } + + #[test] + fn ordinary_text_slugifies() { + assert_eq!(slugify("William Zinsser", 0, false), "william-zinsser"); + assert_eq!( + slugify("Hello---World & Friends", 0, false), + "hello-world-friends" + ); + assert_eq!(slugify("", 0, false), ""); + } + + #[test] + fn a_written_apostrophe_is_a_word_break() { + assert_eq!( + slugify("Kierkegaard's Conception", 0, false), + "kierkegaard-s-conception" + ); + } + + #[test] + fn an_apostrophe_transliteration_produced_is_not() { + // The Cyrillic hard sign becomes one, and a dash for each would make + // a slug of fragments. + assert_eq!(slugify("Олегъ", 0, false), "oleg"); + } + + #[test] + fn other_scripts_are_transliterated() { + assert_eq!(slugify("Être et Temps", 0, false), "etre-et-temps"); + assert_eq!(slugify("café naïve", 0, false), "cafe-naive"); + assert_eq!(slugify("Straße", 0, false), "strasse"); + assert_eq!(slugify("Ἀθῆναι καὶ Μόσχα", 0, false), "athenai-kai-moskha"); + } + + #[test] + fn a_thousands_separator_does_not_split_a_number() { + assert_eq!(slugify("12,000 Years", 0, false), "12000-years"); + } + + #[test] + fn character_references_are_decoded() { + assert_eq!(slugify("Tom & Jerry", 0, false), "tom-jerry"); + assert_eq!(slugify("café", 0, false), "cafe"); + assert_eq!(slugify("café", 0, false), "cafe"); + assert_eq!(slugify("¬aname; here", 0, false), "notaname-here"); + } + + #[test] + fn a_cut_lands_on_a_word() { + assert_eq!( + slugify("An Introduction to Historical Analysis", 28, true), + "an-introduction-to-analysis" + ); + assert_eq!( + slugify("An Introduction to Historical Analysis", 28, false), + "an-introduction-to-historica" + ); + } + + #[test] + fn a_lead_author_is_one_person() { + assert_eq!(lead_author("A. A. Long & J. A. Mansfeld"), "A. A. Long"); + assert_eq!( + lead_author("Clarke, Arthur C. and Lee, Gentry"), + "Clarke, Arthur C." + ); + assert_eq!( + lead_author("Bishop, Christopher M."), + "Bishop, Christopher M." + ); + assert_eq!( + lead_author("Miranda Lundy, Daud Sutton, Anthony Ashton"), + "Miranda Lundy" + ); + assert_eq!(lead_author(""), ""); + } + + #[test] + fn a_slug_names_a_person_and_a_work() { + assert_eq!( + slug_for(Some("William Zinsser"), Some("On Writing Well"), 3450), + "william-zinsser-on-writing-well-3450" + ); + } + + #[test] + fn a_slug_always_has_room_for_the_title() { + let slug = slug_for( + Some("A. A. Long & J. A. Mansfeld & Keimpe Algra & Carl A. Huffman"), + Some("The Cambridge Companion to Early Greek Philosophy"), + 1234, + ); + assert!(slug.contains("cambridge"), "{slug}"); + assert!(slug.len() <= MAX_LENGTH + 6, "{slug}"); + } + + #[test] + fn a_record_with_nothing_usable_still_gets_a_slug() { + assert_eq!(slug_for(Some("()"), Some(""), 7023), "book-7023"); + assert_eq!(slug_for(None, None, 1), "book-1"); + } + + #[test] + fn a_slug_that_agrees_with_its_record_is_left_alone() { + assert_eq!( + evaluate( + "william-zinsser-on-writing-well-3450", + Some("William Zinsser"), + Some("On Writing Well"), + 3450 + ), + None + ); + } + + #[test] + fn a_slug_minted_without_an_author_is_safe_to_replace() { + let found = evaluate( + "unknown-mind-and-brain-1234", + Some("Zinsser, William"), + Some("Mind and Brain"), + 1234, + ) + .expect("a proposal"); + assert_eq!(found.tier, "placeholder-author"); + assert!(found.is_safe()); + assert_eq!(found.proposed, "zinsser-william-mind-and-brain-1234"); + } + + #[test] + fn a_slug_carrying_a_filename_artefact_is_safe_to_replace() { + let found = evaluate( + "smith-john-the-mill-z-lib-org-99", + Some("Smith, John"), + Some("The Mill and the Valley"), + 99, + ) + .expect("a proposal"); + assert_eq!(found.tier, "filename-artefact"); + assert!(found.is_safe()); + } + + #[test] + fn a_slug_naming_another_book_is_held_for_review() { + let found = evaluate( + "william-zinsser-on-writing-well-3450", + Some("Laura Nader"), + Some("Weaponizing Anthropology"), + 3450, + ) + .expect("a proposal"); + assert_eq!(found.tier, "wrong-book"); + assert!( + !found.is_safe(), + "a disagreement does not say which side is wrong" + ); + } + + #[test] + fn a_proposal_that_changes_nothing_says_to_fix_the_record() { + // The record's own fields carry the artefact, so regenerating from + // them reproduces it exactly. + let found = evaluate( + "microsoft-word-mat-doc-1", + Some("Microsoft Word"), + Some("mat doc"), + 1, + ) + .expect("a proposal"); + assert_eq!(found.tier, "noop-metadata-junk"); + assert_eq!(found.proposed, found.current); + } + + #[test] + fn a_new_slug_that_is_still_a_placeholder_is_deferred() { + let found = + evaluate("unknown-1234", Some("Unknown Author"), None, 1234).expect("a proposal"); + assert_eq!(found.tier, "defer-new-slug-junk"); + } + + #[test] + fn a_volume_marker_the_new_slug_would_lose_is_held_for_review() { + let found = evaluate( + "unknown-proceedings-of-the-society-vol-9-77", + Some("Aristotelian Society"), + Some("Proceedings of the Society"), + 77, + ) + .expect("a proposal"); + assert_eq!(found.tier, "review-loses-volume"); + } + + #[test] + fn an_author_field_holding_title_text_is_not_used_to_rename() { + let found = evaluate( + "unknown-the-land-of-iran-55", + Some("the land of iran (Cambridge University Press"), + Some("The Land of Iran"), + 55, + ) + .expect("a proposal"); + assert_eq!(found.tier, "defer-new-slug-junk"); + } + + #[test] + fn an_author_that_slugifies_to_nothing_leaves_the_title_leading() { + assert_eq!( + slug_for(Some("()"), Some("Mind and Brain"), 7023), + "mind-and-brain-7023" + ); + } +} diff --git a/crates/cueseg/src/lib.rs b/crates/cueseg/src/lib.rs index bd1a2d0..bd6e196 100644 --- a/crates/cueseg/src/lib.rs +++ b/crates/cueseg/src/lib.rs @@ -64,38 +64,14 @@ pub fn segment(cues: &[Cue], duration: f64) -> Vec { /// How many chapters a recording of this length should have. pub fn target_chapter_count(duration: f64) -> usize { - let target = round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER); + let target = textkit::round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER); (target as usize).clamp(MIN_CHAPTERS, MAX_CHAPTERS) } -/// Round half to even, which is what the Python this was ported from does -/// and what Rust's own `round` does not: `round(2.5)` is 2 there and 3 here, -/// and a recording exactly two and a half chapters long would otherwise get a -/// different number of them. -fn round_half_to_even(value: f64) -> f64 { - let rounded = value.round(); - if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 { - rounded - value.signum() - } else { - rounded - } -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn rounding_goes_to_even_on_a_half() { - assert_eq!(round_half_to_even(0.5), 0.0); - assert_eq!(round_half_to_even(1.5), 2.0); - assert_eq!(round_half_to_even(2.5), 2.0); - assert_eq!(round_half_to_even(3.5), 4.0); - assert_eq!(round_half_to_even(-2.5), -2.0); - assert_eq!(round_half_to_even(2.4), 2.0); - assert_eq!(round_half_to_even(2.6), 3.0); - } - #[test] fn the_chapter_count_follows_the_length_within_bounds() { assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS); diff --git a/crates/textkit/src/lib.rs b/crates/textkit/src/lib.rs index 0d28cab..acd2ce5 100644 --- a/crates/textkit/src/lib.rs +++ b/crates/textkit/src/lib.rs @@ -15,6 +15,6 @@ pub mod pattern; pub mod segment; pub mod tokens; -pub use number::decimal; +pub use number::{decimal, percent, round_half_to_even}; pub use segment::{is_space, paragraphs, sentences}; pub use tokens::{count, decode_lossy, encode, split_at_token_limit}; diff --git a/crates/textkit/src/number.rs b/crates/textkit/src/number.rs index 7711f91..3df247e 100644 --- a/crates/textkit/src/number.rs +++ b/crates/textkit/src/number.rs @@ -47,6 +47,28 @@ fn digit_value(c: char) -> Option { ten.then_some(value) } +/// Round half to even, which is what Python's `round` does and what Rust's +/// does not: `round(2.5)` is 2 there and 3 here. +/// +/// It matters wherever a rounded number decides something — how many +/// chapters a recording gets, what a percentage in a report reads as — and +/// the two rules disagree exactly on the values that land on a boundary, +/// which is where a disagreement is most likely to be noticed. +pub fn round_half_to_even(value: f64) -> f64 { + let rounded = value.round(); + if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 { + rounded - value.signum() + } else { + rounded + } +} + +/// A fraction as a whole-number percentage, rounded the same way Python +/// rounds it. +pub fn percent(fraction: f64) -> String { + format!("{}%", round_half_to_even(fraction * 100.0)) +} + #[cfg(test)] mod tests { use super::*; @@ -84,6 +106,25 @@ mod tests { assert_eq!(decimal("page"), 0); } + #[test] + fn rounding_goes_to_even_on_a_half() { + assert_eq!(round_half_to_even(0.5), 0.0); + assert_eq!(round_half_to_even(1.5), 2.0); + assert_eq!(round_half_to_even(2.5), 2.0); + assert_eq!(round_half_to_even(-2.5), -2.0); + assert_eq!(round_half_to_even(2.4), 2.0); + } + + #[test] + fn a_percentage_reads_as_a_whole_number() { + assert_eq!(percent(0.0), "0%"); + // Not 33: a third of a percent is a hair above the halfway point + // once it is a binary float, so it rounds up rather than to even. + assert_eq!(percent(0.335), "34%"); + assert_eq!(percent(1.0), "100%"); + assert_eq!(percent(0.125), "12%"); + } + #[test] fn an_absurd_number_saturates_rather_than_wrapping() { assert_eq!(decimal(&"9".repeat(40)), u32::MAX); diff --git a/parity/cases/slug/records.ndjson b/parity/cases/slug/records.ndjson new file mode 100644 index 0000000..3f66def --- /dev/null +++ b/parity/cases/slug/records.ndjson @@ -0,0 +1,25 @@ +{"slug": "william-zinsser-on-writing-well-3450", "author": "William Zinsser", "title": "On Writing Well", "id": 3450} +{"slug": "william-zinsser-on-writing-well-3450", "author": "Laura Nader", "title": "Weaponizing Anthropology", "id": 3450} +{"slug": "unknown-mind-and-brain-1234", "author": "Zinsser, William", "title": "Mind and Brain", "id": 1234} +{"slug": "unknown-file-c-documents-and-settings-hasici-dokumen-88", "author": "Unknown", "title": "Mind and Brain: Chaos and Quantum Mechanics", "id": 88} +{"slug": "smith-john-the-mill-z-lib-org-99", "author": "Smith, John", "title": "The Mill and the Valley", "id": 99} +{"slug": "microsoft-word-mat-doc-1", "author": "Microsoft Word", "title": "mat doc", "id": 1} +{"slug": "unknown-1234", "author": "Unknown Author", "title": null, "id": 1234} +{"slug": "unknown-proceedings-of-the-society-vol-9-77", "author": "Aristotelian Society", "title": "Proceedings of the Society", "id": 77} +{"slug": "unknown-the-land-of-iran-55", "author": "the land of iran (Cambridge University Press", "title": "The Land of Iran", "id": 55} +{"slug": "old-testament-volume-ii-mark-sheridan-12", "author": "Old Testament, Volume II) (Mark Sheridan", "title": "Genesis 12-50", "id": 12} +{"slug": "a-a-long-j-a-mansfeld-keimpe-algra-carl-a-huffman-edward-h-7", "author": "A. A. Long & J. A. Mansfeld & Keimpe Algra & Carl A. Huffman & Edward H. Hussey", "title": "The Cambridge Companion to Early Greek Philosophy", "id": 7} +{"slug": "book-7023", "author": "()", "title": "Mind and Brain", "id": 7023} +{"slug": "test-3-8", "author": "test", "title": "A Real Book About Something", "id": 8} +{"slug": "jaroslav-pelikan-christianity-and-classical-culture-the-me-44", "author": "Jaroslav Pelikan", "title": "Christianity and Classical Culture: The Metamorphosis of Natural Theology in the Christian Encounter with Hellenism", "id": 44} +{"slug": "etre-et-temps-9", "author": "Heidegger, Martin", "title": "Être et Temps", "id": 9} +{"slug": "athenai-kai-moskha-10", "author": "Θουκυδίδης", "title": "Ἀθῆναι καὶ Μόσχα", "id": 10} +{"slug": "dong-jing-noshou-zhi-11", "author": "夏目漱石", "title": "東京の手紙", "id": 11} +{"slug": "kierkegaard-s-conception-of-god-13", "author": "Kierkegaard, Søren", "title": "Kierkegaard's Conception of God", "id": 13} +{"slug": "12000-years-14", "author": "Smith, J.", "title": "12,000 Years of History", "id": 14} +{"slug": "tom-jerry-15", "author": "Hanna & Barbera", "title": "Tom & Jerry", "id": 15} +{"slug": "an-introduction-to-analysis-16", "author": null, "title": "An Introduction to Historical Analysis", "id": 16} +{"slug": "bishop-christopher-m-pattern-recognition-17", "author": "Bishop, Christopher M.", "title": "Pattern Recognition and Machine Learning", "id": 17} +{"slug": "miranda-lundy-quadrivium-18", "author": "Miranda Lundy, Daud Sutton, Anthony Ashton, Jason Martineau", "title": "Quadrivium", "id": 18} +{"slug": "", "author": "Smith, John", "title": "A Book", "id": 19} +{"slug": "guardians-guardians-3190", "author": "Guardians", "title": "Guardians", "id": 3190} diff --git a/parity/golden/slug/records.ndjson b/parity/golden/slug/records.ndjson new file mode 100644 index 0000000..0b1dc55 --- /dev/null +++ b/parity/golden/slug/records.ndjson @@ -0,0 +1,25 @@ +{"lead_author": "William Zinsser", "proposed": null, "reasons": [], "slug_for": "william-zinsser-on-writing-well-3450", "slugify": "on-writing-well", "tier": null} +{"lead_author": "Laura Nader", "proposed": "laura-nader-weaponizing-anthropology-3450", "reasons": ["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"], "slug_for": "laura-nader-weaponizing-anthropology-3450", "slugify": "weaponizing-anthropology", "tier": "wrong-book"} +{"lead_author": "Zinsser, William", "proposed": "zinsser-william-mind-and-brain-1234", "reasons": ["slug opens with placeholder author 'unknown'"], "slug_for": "zinsser-william-mind-and-brain-1234", "slugify": "mind-and-brain", "tier": "placeholder-author"} +{"lead_author": "Unknown", "proposed": "unknown-mind-and-brain-chaos-and-quantum-mechanics-88", "reasons": ["slug carries a filename or mirror artefact"], "slug_for": "unknown-mind-and-brain-chaos-and-quantum-mechanics-88", "slugify": "mind-and-brain-chaos-and-quantum-mechanics", "tier": "filename-artefact"} +{"lead_author": "Smith, John", "proposed": "smith-john-the-mill-and-the-valley-99", "reasons": ["slug carries a filename or mirror artefact"], "slug_for": "smith-john-the-mill-and-the-valley-99", "slugify": "the-mill-and-the-valley", "tier": "filename-artefact"} +{"lead_author": "Microsoft Word", "proposed": "microsoft-word-mat-doc-1", "reasons": ["slug carries a filename or mirror artefact", "recomputes to the same slug: fix the metadata first"], "slug_for": "microsoft-word-mat-doc-1", "slugify": "mat-doc", "tier": "noop-metadata-junk"} +{"lead_author": "Unknown Author", "proposed": "unknown-author-1234", "reasons": ["slug opens with placeholder author 'unknown'", "new slug is no better: fix the metadata first"], "slug_for": "unknown-author-1234", "slugify": "", "tier": "defer-new-slug-junk"} +{"lead_author": "Aristotelian Society", "proposed": "aristotelian-society-proceedings-of-the-society-77", "reasons": ["slug opens with placeholder author 'unknown'", "current slug carries a volume marker the new one loses"], "slug_for": "aristotelian-society-proceedings-of-the-society-77", "slugify": "proceedings-of-the-society", "tier": "review-loses-volume"} +{"lead_author": "the land of iran (Cambridge University Press", "proposed": "the-land-of-iran-cambridge-the-land-of-iran-55", "reasons": ["slug opens with placeholder author 'unknown'", "author field looks like spilled title/series text: 'the land of iran (Cambridge University Press'"], "slug_for": "the-land-of-iran-cambridge-the-land-of-iran-55", "slugify": "the-land-of-iran", "tier": "defer-new-slug-junk"} +{"lead_author": "Old Testament, Volume II) (Mark Sheridan", "proposed": null, "reasons": [], "slug_for": "old-testament-volume-ii-mark-genesis-12-50-12", "slugify": "genesis-12-50", "tier": null} +{"lead_author": "A. A. Long", "proposed": null, "reasons": [], "slug_for": "a-a-long-the-cambridge-companion-to-early-greek-philosophy-7", "slugify": "the-cambridge-companion-to-early-greek-philosophy", "tier": null} +{"lead_author": "()", "proposed": "mind-and-brain-7023", "reasons": ["slug opens with placeholder author 'book'"], "slug_for": "mind-and-brain-7023", "slugify": "mind-and-brain", "tier": "placeholder-author"} +{"lead_author": "test", "proposed": "test-a-real-book-about-something-8", "reasons": ["slug opens with placeholder author 'test'"], "slug_for": "test-a-real-book-about-something-8", "slugify": "a-real-book-about-something", "tier": "placeholder-author"} +{"lead_author": "Jaroslav Pelikan", "proposed": null, "reasons": [], "slug_for": "jaroslav-pelikan-christianity-and-classical-culture-the-metamorphosis-of-natural-44", "slugify": "christianity-and-classical-culture-the-metamorphosis-of-natural-theology-in-the-christian-encounter-with-hellenism", "tier": null} +{"lead_author": "Heidegger, Martin", "proposed": null, "reasons": [], "slug_for": "heidegger-martin-etre-et-temps-9", "slugify": "etre-et-temps", "tier": null} +{"lead_author": "Θουκυδίδης", "proposed": null, "reasons": [], "slug_for": "thoukudides-athenai-kai-moskha-10", "slugify": "athenai-kai-moskha", "tier": null} +{"lead_author": "夏目漱石", "proposed": null, "reasons": [], "slug_for": "xia-mu-shu-shi-dong-jing-noshou-zhi-11", "slugify": "dong-jing-noshou-zhi", "tier": null} +{"lead_author": "Kierkegaard, Søren", "proposed": null, "reasons": [], "slug_for": "kierkegaard-soren-kierkegaard-s-conception-of-god-13", "slugify": "kierkegaard-s-conception-of-god", "tier": null} +{"lead_author": "Smith, J.", "proposed": null, "reasons": [], "slug_for": "smith-j-12000-years-of-history-14", "slugify": "12000-years-of-history", "tier": null} +{"lead_author": "Hanna", "proposed": null, "reasons": [], "slug_for": "hanna-tom-jerry-15", "slugify": "tom-jerry", "tier": null} +{"lead_author": "", "proposed": null, "reasons": [], "slug_for": "an-introduction-to-historical-analysis-16", "slugify": "an-introduction-to-historical-analysis", "tier": null} +{"lead_author": "Bishop, Christopher M.", "proposed": null, "reasons": [], "slug_for": "bishop-christopher-m-pattern-recognition-and-machine-learning-17", "slugify": "pattern-recognition-and-machine-learning", "tier": null} +{"lead_author": "Miranda Lundy", "proposed": null, "reasons": [], "slug_for": "miranda-lundy-quadrivium-18", "slugify": "quadrivium", "tier": null} +{"lead_author": "Smith, John", "proposed": "smith-john-a-book-19", "reasons": ["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"], "slug_for": "smith-john-a-book-19", "slugify": "a-book", "tier": "wrong-book"} +{"lead_author": "Guardians", "proposed": null, "reasons": [], "slug_for": "guardians-guardians-3190", "slugify": "guardians", "tier": null} diff --git a/parity/oracle/slug.py b/parity/oracle/slug.py new file mode 100644 index 0000000..c83375e --- /dev/null +++ b/parity/oracle/slug.py @@ -0,0 +1,50 @@ +"""Judge slugs the way the Python pipeline judges them. + +Reads one `{"slug", "author", "title", "id"}` object per line and writes what +the slug module makes of each. + + CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \ + python3 oracle/slug.py < cases/slug/records.ndjson +""" + +import json +import os +import sys + +for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep): + if path: + sys.path.insert(0, path) + +from slugify import slugify # noqa: E402 + +from book_pipeline import slugs # noqa: E402 + + +def main() -> int: + out = sys.stdout + for line in sys.stdin.buffer.read().decode("utf-8").split("\n"): + if not line.strip(): + continue + record = json.loads(line) + author, title, book_id = record.get("author"), record.get("title"), record["id"] + proposal = slugs.evaluate("x", record["slug"], author, title, book_id) + out.write( + json.dumps( + { + "slugify": slugify(title or ""), + "slug_for": slugs.slug_for(author, title, book_id), + "lead_author": slugs._lead_author(author), + "tier": proposal.tier if proposal else None, + "proposed": proposal.proposed_slug if proposal else None, + "reasons": proposal.reasons if proposal else [], + }, + ensure_ascii=False, + sort_keys=True, + ) + + "\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index 1adde87..73ff37f 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 11] = [ + let jobs: [Job; 12] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -31,6 +31,7 @@ fn main() -> ExitCode { ("cuepack", cuepack), ("booksniff", booksniff), ("bibclean", bibclean), + ("slug", slug), ]; for (name, job) in jobs { @@ -74,6 +75,18 @@ fn unescape() -> Written { Ok(vec![path.display().to_string()]) } +fn slug() -> Written { + let dir = parity::golden("slug"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/slug/records.ndjson"); + let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let judgements = parity::run_oracle("slug.py", &cases)?; + let path = dir.join("records.ndjson"); + std::fs::write(&path, judgements).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn bibclean() -> Written { let dir = parity::golden("bibclean"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/slug.rs b/parity/tests/slug.rs new file mode 100644 index 0000000..35cd1e3 --- /dev/null +++ b/parity/tests/slug.rs @@ -0,0 +1,100 @@ +//! Slugs, and the judgement about whether one should be replaced. +//! +//! A slug is a public identifier: it reaches citations and research notes and +//! outlives whatever produced it. Generating a different one than the system +//! being replaced would rename every book in a corpus, and judging one +//! differently would rename the wrong ones. + +use serde_json::Value; + +fn records(path: &str) -> Vec { + let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file"); + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> { + record.get(name).and_then(Value::as_str) +} + +#[test] +fn every_slug_and_verdict_matches_python() { + let cases = records("cases/slug/records.ndjson"); + let golden = records("golden/slug/records.ndjson"); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + let mut wrong: Vec = Vec::new(); + for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() { + let author = field(case, "author"); + let title = field(case, "title"); + let slug = field(case, "slug").unwrap_or(""); + let id = case["id"].as_i64().expect("an id"); + + let mut differs = |what: &str, ours: String, theirs: &Value| { + wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}")); + }; + + let ours = bibclean::slug::slugify(title.unwrap_or(""), 0, false); + if Some(ours.as_str()) != theirs["slugify"].as_str() { + differs("slugify", ours, &theirs["slugify"]); + } + + let ours = bibclean::slug::lead_author(author.unwrap_or("")); + if Some(ours.as_str()) != theirs["lead_author"].as_str() { + differs("lead_author", ours, &theirs["lead_author"]); + } + + let ours = bibclean::slug::slug_for(author, title, id); + if Some(ours.as_str()) != theirs["slug_for"].as_str() { + differs("slug_for", ours, &theirs["slug_for"]); + } + + let proposal = bibclean::slug::evaluate(slug, author, title, id); + let ours_tier = proposal.as_ref().map(|found| found.tier.as_str()); + if ours_tier != theirs["tier"].as_str() { + differs("tier", format!("{ours_tier:?}"), &theirs["tier"]); + } + + let ours_proposed = proposal.as_ref().map(|found| found.proposed.as_str()); + if ours_proposed != theirs["proposed"].as_str() { + differs( + "proposed", + format!("{ours_proposed:?}"), + &theirs["proposed"], + ); + } + + let ours_reasons: Vec = proposal + .as_ref() + .map(|found| found.reasons.clone()) + .unwrap_or_default(); + let their_reasons: Vec = theirs["reasons"] + .as_array() + .expect("a list") + .iter() + .map(|reason| reason.as_str().expect("a reason").to_string()) + .collect(); + if ours_reasons != their_reasons { + differs("reasons", format!("{ours_reasons:?}"), &theirs["reasons"]); + } + } + + assert!( + wrong.is_empty(), + "{} judgements differ:\n{}", + wrong.len(), + wrong.join("\n") + ); +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/slug/records.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("slug.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("slug/records.ndjson")); +}