mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
bibclean: slugs, and whether the one a record has still names it
This commit is contained in:
funkpower
Generated
+23
@@ -92,12 +92,14 @@ dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"deunicode",
|
||||
"fancy-regex 0.16.2",
|
||||
"regex",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -332,6 +334,12 @@ dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deunicode"
|
||||
version = "1.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.7"
|
||||
@@ -1070,6 +1078,12 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-general-category"
|
||||
version = "1.1.0"
|
||||
@@ -1082,6 +1096,15 @@ version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
|
||||
@@ -26,6 +26,7 @@ booksniff = { path = "crates/booksniff", version = "0.1.0" }
|
||||
emb = { path = "crates/emb", version = "0.1.0" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
deunicode = "1"
|
||||
fancy-regex = "0.16"
|
||||
regex = "1"
|
||||
schemars = "1"
|
||||
@@ -36,6 +37,7 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_json = { version = "1", features = ["float_roundtrip"] }
|
||||
thiserror = "2"
|
||||
unicode-general-category = "1"
|
||||
unicode-normalization = "0.1"
|
||||
ureq = { version = "3", features = ["json"] }
|
||||
tiktoken-rs = "0.12"
|
||||
|
||||
|
||||
@@ -11,12 +11,14 @@ repository.workspace = true
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
deunicode.workspace = true
|
||||
fancy-regex.workspace = true
|
||||
regex.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
textkit.workspace = true
|
||||
unicode-normalization.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -76,21 +76,57 @@ lacks one.
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| `0` | Nothing needed attention |
|
||||
| `1` | Something did |
|
||||
| `1` | Something did — a fault worth fixing, or a slug worth replacing |
|
||||
| `64` | The command line was wrong |
|
||||
| `65` | A record could not be read |
|
||||
| `74` | The file could not be read |
|
||||
|
||||
## Input
|
||||
|
||||
One JSON record per line. `title`, `author`, `isbn` and `path` are read;
|
||||
everything else is carried through untouched, so a record can go through this
|
||||
and back into whatever it came from.
|
||||
One JSON record per line. `title`, `author`, `isbn`, `path`, `slug` and `id`
|
||||
are read; everything else is carried through untouched, so a record can go
|
||||
through this and back into whatever it came from.
|
||||
|
||||
```json
|
||||
{"title":"THE HUMAN CONDITION","author":"Arendt, Hannah","path":"/library/9780226586601.pdf"}
|
||||
```
|
||||
|
||||
## Slugs
|
||||
|
||||
A slug is a public identifier — it reaches citations and notes, and outlives
|
||||
whatever produced it. When a record carries one, along with the `id` that
|
||||
makes it unique, its slug is judged too:
|
||||
|
||||
```
|
||||
$ bibclean slugs.ndjson
|
||||
{"v":1,"title":"Weaponizing Anthropology","author":"Laura Nader",…,"slug":"william-zinsser-on-writing-well-3451","id":3451,"reasons":[],"severity":0,"changes":[],"slug_proposal":{"current":"william-zinsser-on-writing-well-3451","proposed":"laura-nader-weaponizing-anthropology-3451","tier":"wrong-book","reasons":["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"]}}
|
||||
{"v":1,"title":"Mind and Brain","author":"Zinsser, William",…,"slug":"unknown-mind-and-brain-1234","id":1234,"reasons":[],"severity":0,"changes":[],"slug_proposal":{"current":"unknown-mind-and-brain-1234","proposed":"zinsser-william-mind-and-brain-1234","tier":"placeholder-author","reasons":["slug opens with placeholder author 'unknown'"]}}
|
||||
```
|
||||
|
||||
Nothing is renamed. A proposal comes back beside the slug the record has, and
|
||||
the **tier** says how much the evidence is worth:
|
||||
|
||||
| Tier | Evidence | Safe to apply in bulk? |
|
||||
|---|---|---|
|
||||
| `placeholder-author` | The slug opens with a placeholder, so it was minted without an author | yes |
|
||||
| `filename-artefact` | The slug carries a mirror suffix, a wrapper format or a path | yes |
|
||||
| `wrong-book` | Too few of the slug's words appear in the record at all | no |
|
||||
| `noop-metadata-junk` | Regenerating gives the same slug: the record is the problem | no |
|
||||
| `defer-new-slug-junk` | The new slug would be no better than the old one | no |
|
||||
| `review-loses-volume` | The current slug names a volume the new one would lose | no |
|
||||
| `review-low-recall` | The new slug recovers too little of the title | no |
|
||||
|
||||
The first two rest on evidence inside the slug itself and cannot be wrong
|
||||
about it. The rest are *disagreements* between the slug and the record, and a
|
||||
disagreement does not say which side is at fault — which is exactly the case
|
||||
where renaming automatically makes things worse.
|
||||
|
||||
Only the safe tiers:
|
||||
|
||||
```sh
|
||||
bibclean books.ndjson | jq -c 'select(.slug_proposal.tier | IN("placeholder-author","filename-artefact"))'
|
||||
```
|
||||
|
||||
## The faults it names
|
||||
|
||||
| Reason | Weight | What it means |
|
||||
@@ -140,7 +176,7 @@ if bibclean::reasons::worth_fixing(&cleaned.reasons) {
|
||||
|
||||
The pieces are usable on their own: `bibclean::isbn::valid`,
|
||||
`bibclean::isbn::from_filename`, `bibclean::fix::title_case`,
|
||||
`bibclean::matching::same_work`.
|
||||
`bibclean::matching::same_work`, `bibclean::slug::slug_for`.
|
||||
|
||||
`same_work` is worth knowing about separately. It compares vocabulary rather
|
||||
than characters, because character similarity is not usable for titles: "The
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
//! The named character references a slug generator decodes.
|
||||
//!
|
||||
//! The HTML 4 set, which is smaller than the HTML 5 one and is what the
|
||||
//! slug generators in wide use recognise. A name outside it is left as
|
||||
//! text, and becomes a word in the slug like any other.
|
||||
|
||||
/// Named references, sorted by name, without their semicolons.
|
||||
pub static NAMED: &[(&str, char)] = &[
|
||||
("AElig", '\u{c6}'),
|
||||
("Aacute", '\u{c1}'),
|
||||
("Acirc", '\u{c2}'),
|
||||
("Agrave", '\u{c0}'),
|
||||
("Alpha", '\u{391}'),
|
||||
("Aring", '\u{c5}'),
|
||||
("Atilde", '\u{c3}'),
|
||||
("Auml", '\u{c4}'),
|
||||
("Beta", '\u{392}'),
|
||||
("Ccedil", '\u{c7}'),
|
||||
("Chi", '\u{3a7}'),
|
||||
("Dagger", '\u{2021}'),
|
||||
("Delta", '\u{394}'),
|
||||
("ETH", '\u{d0}'),
|
||||
("Eacute", '\u{c9}'),
|
||||
("Ecirc", '\u{ca}'),
|
||||
("Egrave", '\u{c8}'),
|
||||
("Epsilon", '\u{395}'),
|
||||
("Eta", '\u{397}'),
|
||||
("Euml", '\u{cb}'),
|
||||
("Gamma", '\u{393}'),
|
||||
("Iacute", '\u{cd}'),
|
||||
("Icirc", '\u{ce}'),
|
||||
("Igrave", '\u{cc}'),
|
||||
("Iota", '\u{399}'),
|
||||
("Iuml", '\u{cf}'),
|
||||
("Kappa", '\u{39a}'),
|
||||
("Lambda", '\u{39b}'),
|
||||
("Mu", '\u{39c}'),
|
||||
("Ntilde", '\u{d1}'),
|
||||
("Nu", '\u{39d}'),
|
||||
("OElig", '\u{152}'),
|
||||
("Oacute", '\u{d3}'),
|
||||
("Ocirc", '\u{d4}'),
|
||||
("Ograve", '\u{d2}'),
|
||||
("Omega", '\u{3a9}'),
|
||||
("Omicron", '\u{39f}'),
|
||||
("Oslash", '\u{d8}'),
|
||||
("Otilde", '\u{d5}'),
|
||||
("Ouml", '\u{d6}'),
|
||||
("Phi", '\u{3a6}'),
|
||||
("Pi", '\u{3a0}'),
|
||||
("Prime", '\u{2033}'),
|
||||
("Psi", '\u{3a8}'),
|
||||
("Rho", '\u{3a1}'),
|
||||
("Scaron", '\u{160}'),
|
||||
("Sigma", '\u{3a3}'),
|
||||
("THORN", '\u{de}'),
|
||||
("Tau", '\u{3a4}'),
|
||||
("Theta", '\u{398}'),
|
||||
("Uacute", '\u{da}'),
|
||||
("Ucirc", '\u{db}'),
|
||||
("Ugrave", '\u{d9}'),
|
||||
("Upsilon", '\u{3a5}'),
|
||||
("Uuml", '\u{dc}'),
|
||||
("Xi", '\u{39e}'),
|
||||
("Yacute", '\u{dd}'),
|
||||
("Yuml", '\u{178}'),
|
||||
("Zeta", '\u{396}'),
|
||||
("aacute", '\u{e1}'),
|
||||
("acirc", '\u{e2}'),
|
||||
("acute", '\u{b4}'),
|
||||
("aelig", '\u{e6}'),
|
||||
("agrave", '\u{e0}'),
|
||||
("alefsym", '\u{2135}'),
|
||||
("alpha", '\u{3b1}'),
|
||||
("amp", '&'),
|
||||
("and", '\u{2227}'),
|
||||
("ang", '\u{2220}'),
|
||||
("aring", '\u{e5}'),
|
||||
("asymp", '\u{2248}'),
|
||||
("atilde", '\u{e3}'),
|
||||
("auml", '\u{e4}'),
|
||||
("bdquo", '\u{201e}'),
|
||||
("beta", '\u{3b2}'),
|
||||
("brvbar", '\u{a6}'),
|
||||
("bull", '\u{2022}'),
|
||||
("cap", '\u{2229}'),
|
||||
("ccedil", '\u{e7}'),
|
||||
("cedil", '\u{b8}'),
|
||||
("cent", '\u{a2}'),
|
||||
("chi", '\u{3c7}'),
|
||||
("circ", '\u{2c6}'),
|
||||
("clubs", '\u{2663}'),
|
||||
("cong", '\u{2245}'),
|
||||
("copy", '\u{a9}'),
|
||||
("crarr", '\u{21b5}'),
|
||||
("cup", '\u{222a}'),
|
||||
("curren", '\u{a4}'),
|
||||
("dArr", '\u{21d3}'),
|
||||
("dagger", '\u{2020}'),
|
||||
("darr", '\u{2193}'),
|
||||
("deg", '\u{b0}'),
|
||||
("delta", '\u{3b4}'),
|
||||
("diams", '\u{2666}'),
|
||||
("divide", '\u{f7}'),
|
||||
("eacute", '\u{e9}'),
|
||||
("ecirc", '\u{ea}'),
|
||||
("egrave", '\u{e8}'),
|
||||
("empty", '\u{2205}'),
|
||||
("emsp", '\u{2003}'),
|
||||
("ensp", '\u{2002}'),
|
||||
("epsilon", '\u{3b5}'),
|
||||
("equiv", '\u{2261}'),
|
||||
("eta", '\u{3b7}'),
|
||||
("eth", '\u{f0}'),
|
||||
("euml", '\u{eb}'),
|
||||
("euro", '\u{20ac}'),
|
||||
("exist", '\u{2203}'),
|
||||
("fnof", '\u{192}'),
|
||||
("forall", '\u{2200}'),
|
||||
("frac12", '\u{bd}'),
|
||||
("frac14", '\u{bc}'),
|
||||
("frac34", '\u{be}'),
|
||||
("frasl", '\u{2044}'),
|
||||
("gamma", '\u{3b3}'),
|
||||
("ge", '\u{2265}'),
|
||||
("gt", '>'),
|
||||
("hArr", '\u{21d4}'),
|
||||
("harr", '\u{2194}'),
|
||||
("hearts", '\u{2665}'),
|
||||
("hellip", '\u{2026}'),
|
||||
("iacute", '\u{ed}'),
|
||||
("icirc", '\u{ee}'),
|
||||
("iexcl", '\u{a1}'),
|
||||
("igrave", '\u{ec}'),
|
||||
("image", '\u{2111}'),
|
||||
("infin", '\u{221e}'),
|
||||
("int", '\u{222b}'),
|
||||
("iota", '\u{3b9}'),
|
||||
("iquest", '\u{bf}'),
|
||||
("isin", '\u{2208}'),
|
||||
("iuml", '\u{ef}'),
|
||||
("kappa", '\u{3ba}'),
|
||||
("lArr", '\u{21d0}'),
|
||||
("lambda", '\u{3bb}'),
|
||||
("lang", '\u{2329}'),
|
||||
("laquo", '\u{ab}'),
|
||||
("larr", '\u{2190}'),
|
||||
("lceil", '\u{2308}'),
|
||||
("ldquo", '\u{201c}'),
|
||||
("le", '\u{2264}'),
|
||||
("lfloor", '\u{230a}'),
|
||||
("lowast", '\u{2217}'),
|
||||
("loz", '\u{25ca}'),
|
||||
("lrm", '\u{200e}'),
|
||||
("lsaquo", '\u{2039}'),
|
||||
("lsquo", '\u{2018}'),
|
||||
("lt", '<'),
|
||||
("macr", '\u{af}'),
|
||||
("mdash", '\u{2014}'),
|
||||
("micro", '\u{b5}'),
|
||||
("middot", '\u{b7}'),
|
||||
("minus", '\u{2212}'),
|
||||
("mu", '\u{3bc}'),
|
||||
("nabla", '\u{2207}'),
|
||||
("nbsp", '\u{a0}'),
|
||||
("ndash", '\u{2013}'),
|
||||
("ne", '\u{2260}'),
|
||||
("ni", '\u{220b}'),
|
||||
("not", '\u{ac}'),
|
||||
("notin", '\u{2209}'),
|
||||
("nsub", '\u{2284}'),
|
||||
("ntilde", '\u{f1}'),
|
||||
("nu", '\u{3bd}'),
|
||||
("oacute", '\u{f3}'),
|
||||
("ocirc", '\u{f4}'),
|
||||
("oelig", '\u{153}'),
|
||||
("ograve", '\u{f2}'),
|
||||
("oline", '\u{203e}'),
|
||||
("omega", '\u{3c9}'),
|
||||
("omicron", '\u{3bf}'),
|
||||
("oplus", '\u{2295}'),
|
||||
("or", '\u{2228}'),
|
||||
("ordf", '\u{aa}'),
|
||||
("ordm", '\u{ba}'),
|
||||
("oslash", '\u{f8}'),
|
||||
("otilde", '\u{f5}'),
|
||||
("otimes", '\u{2297}'),
|
||||
("ouml", '\u{f6}'),
|
||||
("para", '\u{b6}'),
|
||||
("part", '\u{2202}'),
|
||||
("permil", '\u{2030}'),
|
||||
("perp", '\u{22a5}'),
|
||||
("phi", '\u{3c6}'),
|
||||
("pi", '\u{3c0}'),
|
||||
("piv", '\u{3d6}'),
|
||||
("plusmn", '\u{b1}'),
|
||||
("pound", '\u{a3}'),
|
||||
("prime", '\u{2032}'),
|
||||
("prod", '\u{220f}'),
|
||||
("prop", '\u{221d}'),
|
||||
("psi", '\u{3c8}'),
|
||||
("quot", '"'),
|
||||
("rArr", '\u{21d2}'),
|
||||
("radic", '\u{221a}'),
|
||||
("rang", '\u{232a}'),
|
||||
("raquo", '\u{bb}'),
|
||||
("rarr", '\u{2192}'),
|
||||
("rceil", '\u{2309}'),
|
||||
("rdquo", '\u{201d}'),
|
||||
("real", '\u{211c}'),
|
||||
("reg", '\u{ae}'),
|
||||
("rfloor", '\u{230b}'),
|
||||
("rho", '\u{3c1}'),
|
||||
("rlm", '\u{200f}'),
|
||||
("rsaquo", '\u{203a}'),
|
||||
("rsquo", '\u{2019}'),
|
||||
("sbquo", '\u{201a}'),
|
||||
("scaron", '\u{161}'),
|
||||
("sdot", '\u{22c5}'),
|
||||
("sect", '\u{a7}'),
|
||||
("shy", '\u{ad}'),
|
||||
("sigma", '\u{3c3}'),
|
||||
("sigmaf", '\u{3c2}'),
|
||||
("sim", '\u{223c}'),
|
||||
("spades", '\u{2660}'),
|
||||
("sub", '\u{2282}'),
|
||||
("sube", '\u{2286}'),
|
||||
("sum", '\u{2211}'),
|
||||
("sup", '\u{2283}'),
|
||||
("sup1", '\u{b9}'),
|
||||
("sup2", '\u{b2}'),
|
||||
("sup3", '\u{b3}'),
|
||||
("supe", '\u{2287}'),
|
||||
("szlig", '\u{df}'),
|
||||
("tau", '\u{3c4}'),
|
||||
("there4", '\u{2234}'),
|
||||
("theta", '\u{3b8}'),
|
||||
("thetasym", '\u{3d1}'),
|
||||
("thinsp", '\u{2009}'),
|
||||
("thorn", '\u{fe}'),
|
||||
("tilde", '\u{2dc}'),
|
||||
("times", '\u{d7}'),
|
||||
("trade", '\u{2122}'),
|
||||
("uArr", '\u{21d1}'),
|
||||
("uacute", '\u{fa}'),
|
||||
("uarr", '\u{2191}'),
|
||||
("ucirc", '\u{fb}'),
|
||||
("ugrave", '\u{f9}'),
|
||||
("uml", '\u{a8}'),
|
||||
("upsih", '\u{3d2}'),
|
||||
("upsilon", '\u{3c5}'),
|
||||
("uuml", '\u{fc}'),
|
||||
("weierp", '\u{2118}'),
|
||||
("xi", '\u{3be}'),
|
||||
("yacute", '\u{fd}'),
|
||||
("yen", '\u{a5}'),
|
||||
("yuml", '\u{ff}'),
|
||||
("zeta", '\u{3b6}'),
|
||||
("zwj", '\u{200d}'),
|
||||
("zwnj", '\u{200c}'),
|
||||
];
|
||||
@@ -11,10 +11,12 @@
|
||||
//! which is the difference between a tool that cleans a catalogue and one
|
||||
//! that damages it in a new way.
|
||||
|
||||
mod entities4;
|
||||
pub mod fix;
|
||||
pub mod isbn;
|
||||
pub mod matching;
|
||||
pub mod reasons;
|
||||
pub mod slug;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -41,6 +43,12 @@ pub struct Book {
|
||||
/// Where the file is, which is often where a missing ISBN is hiding.
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
/// The public identifier the record already has, if it has one.
|
||||
#[serde(default)]
|
||||
pub slug: Option<String>,
|
||||
/// A number that makes a slug unique. Needed to judge or regenerate one.
|
||||
#[serde(default)]
|
||||
pub id: Option<i64>,
|
||||
/// Everything else the record carried.
|
||||
#[serde(flatten)]
|
||||
#[schemars(with = "Map<String, Value>")]
|
||||
@@ -82,6 +90,17 @@ pub struct Cleaned {
|
||||
pub severity: u32,
|
||||
/// What was changed, and why.
|
||||
pub changes: Vec<Change>,
|
||||
/// What should happen to the record's slug, when it has one to judge.
|
||||
///
|
||||
/// Named apart from the record's own `slug`, which is carried through
|
||||
/// beside it — the two would otherwise be one field and the proposal
|
||||
/// would quietly replace the fact.
|
||||
///
|
||||
/// Never applied as a change: a slug is a public identifier that other
|
||||
/// things have already written down, so moving one is a decision with
|
||||
/// consequences outside this record.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub slug_proposal: Option<slug::Proposal>,
|
||||
}
|
||||
|
||||
/// Audit a record and make the repairs that need no lookup.
|
||||
@@ -143,10 +162,21 @@ pub fn clean(book: &Book) -> Cleaned {
|
||||
cleaned.isbn = Some(found);
|
||||
}
|
||||
|
||||
// Judged against the record as it arrived, not as it was repaired: the
|
||||
// question is whether the slug describes the book, and a title this just
|
||||
// title-cased describes the same book either way.
|
||||
let proposal = match (book.slug.as_deref(), book.id) {
|
||||
(Some(slug), Some(id)) => {
|
||||
slug::evaluate(slug, book.author.as_deref(), book.title.as_deref(), id)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Cleaned {
|
||||
severity: reasons::severity(&reasons),
|
||||
reasons,
|
||||
changes,
|
||||
slug_proposal: proposal,
|
||||
book: cleaned,
|
||||
}
|
||||
}
|
||||
@@ -162,6 +192,8 @@ mod tests {
|
||||
author: Some(author.into()),
|
||||
isbn: None,
|
||||
path: None,
|
||||
slug: None,
|
||||
id: None,
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
@@ -215,6 +247,42 @@ mod tests {
|
||||
assert_eq!(clean(&book).book.isbn, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_is_judged_when_there_is_one_to_judge() {
|
||||
let mut book = book("Weaponizing Anthropology", "Laura Nader");
|
||||
book.slug = Some("william-zinsser-on-writing-well-3450".into());
|
||||
book.id = Some(3450);
|
||||
|
||||
let cleaned = clean(&book);
|
||||
let proposal = cleaned.slug_proposal.expect("a proposal");
|
||||
assert_eq!(proposal.tier, "wrong-book");
|
||||
assert_eq!(
|
||||
proposal.proposed,
|
||||
"laura-nader-weaponizing-anthropology-3450"
|
||||
);
|
||||
// Proposed, never applied.
|
||||
assert_eq!(
|
||||
cleaned.book.slug.as_deref(),
|
||||
Some("william-zinsser-on-writing-well-3450")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_record_keeps_its_own_slug_beside_the_proposal() {
|
||||
let mut book = book("Weaponizing Anthropology", "Laura Nader");
|
||||
book.slug = Some("william-zinsser-on-writing-well-3450".into());
|
||||
book.id = Some(3450);
|
||||
|
||||
let json = serde_json::to_value(clean(&book)).expect("a record serialises");
|
||||
assert_eq!(json["slug"], "william-zinsser-on-writing-well-3450");
|
||||
assert_eq!(json["slug_proposal"]["tier"], "wrong-book");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_with_no_slug_gets_no_proposal() {
|
||||
assert!(clean(&book("A Book", "An Author")).slug_proposal.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fields_it_does_not_know_about_are_carried_through() {
|
||||
let mut book = book("A Book", "An Author");
|
||||
|
||||
@@ -39,6 +39,15 @@ struct Args {
|
||||
flagged: bool,
|
||||
}
|
||||
|
||||
/// Whether anything about a record wants a person's attention.
|
||||
///
|
||||
/// A missing ISBN alone does not: nearly every book lacks one. A slug that
|
||||
/// should be replaced does, even when nothing else is wrong, because the
|
||||
/// identifier is the part other things have written down.
|
||||
fn needs_attention(cleaned: &bibclean::Cleaned) -> bool {
|
||||
bibclean::reasons::worth_fixing(&cleaned.reasons) || cleaned.slug_proposal.is_some()
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
@@ -65,8 +74,8 @@ fn main() -> ExitCode {
|
||||
let cleaned: Vec<bibclean::Cleaned> = books
|
||||
.iter()
|
||||
.map(bibclean::clean)
|
||||
.inspect(|cleaned| flagged |= bibclean::reasons::worth_fixing(&cleaned.reasons))
|
||||
.filter(|cleaned| !args.flagged || bibclean::reasons::worth_fixing(&cleaned.reasons))
|
||||
.inspect(|cleaned| flagged |= needs_attention(cleaned))
|
||||
.filter(|cleaned| !args.flagged || needs_attention(cleaned))
|
||||
.collect();
|
||||
|
||||
let wrote = catena_cli::stream("bibclean", &cleaned);
|
||||
|
||||
@@ -0,0 +1,677 @@
|
||||
//! Turning a record into a readable identifier, and judging the one it has.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use crate::entities4::NAMED;
|
||||
|
||||
/// Apostrophes. The ones written in a title become a separator; the ones
|
||||
/// transliteration produces afterwards — a Cyrillic hard sign becomes one —
|
||||
/// are dropped, so they do not each turn into a dash of their own.
|
||||
static QUOTES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[']+").expect("valid"));
|
||||
|
||||
/// A named character reference.
|
||||
static NAMED_REFERENCE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"&([a-zA-Z][a-zA-Z0-9]*);").expect("valid"));
|
||||
|
||||
static DECIMAL_REFERENCE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"&#(\d+);").expect("valid"));
|
||||
|
||||
static HEX_REFERENCE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"&#x([\da-fA-F]+);").expect("valid"));
|
||||
|
||||
/// A comma inside a number, which is a thousands separator rather than a word
|
||||
/// break: `12,000` is one number.
|
||||
static THOUSANDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\d),(\d)").expect("valid"));
|
||||
|
||||
/// Everything a slug may not contain.
|
||||
static DISALLOWED: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[^-a-zA-Z0-9]+").expect("valid"));
|
||||
|
||||
static DASH_RUN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"-{2,}").expect("valid"));
|
||||
|
||||
/// How many characters a whole slug may take.
|
||||
pub const MAX_LENGTH: usize = 80;
|
||||
|
||||
/// How many of them the lead author may take.
|
||||
///
|
||||
/// The title always needs room. Measured against a review queue: a
|
||||
/// five-editor volume otherwise consumed the whole budget and left a slug
|
||||
/// naming only people, which cannot be recognised as a book.
|
||||
pub const AUTHOR_BUDGET: usize = 30;
|
||||
|
||||
/// Reduce a text to a slug.
|
||||
///
|
||||
/// Transliterated to ASCII, lowercased, everything else turned into a
|
||||
/// separator, and truncated at a word if a maximum is given.
|
||||
pub fn slugify(text: &str, max_length: usize, word_boundary: bool) -> String {
|
||||
let text = QUOTES.replace_all(text, "-");
|
||||
let text: String = deunicode::deunicode(&text.nfkd().collect::<String>());
|
||||
|
||||
let text = NAMED_REFERENCE.replace_all(&text, |found: ®ex::Captures| {
|
||||
match NAMED.binary_search_by_key(&&found[1], |(name, _)| name) {
|
||||
Ok(at) => NAMED[at].1.to_string(),
|
||||
Err(_) => found[0].to_string(),
|
||||
}
|
||||
});
|
||||
let text = DECIMAL_REFERENCE.replace_all(&text, |found: ®ex::Captures| {
|
||||
character(found[1].parse::<u32>().ok(), &found[0])
|
||||
});
|
||||
let text = HEX_REFERENCE.replace_all(&text, |found: ®ex::Captures| {
|
||||
character(u32::from_str_radix(&found[1], 16).ok(), &found[0])
|
||||
});
|
||||
|
||||
let text: String = text.nfkd().collect::<String>().to_lowercase();
|
||||
let text = QUOTES.replace_all(&text, "");
|
||||
let text = THOUSANDS.replace_all(&text, "$1$2");
|
||||
let text = DISALLOWED.replace_all(&text, "-");
|
||||
let text = DASH_RUN.replace_all(&text, "-");
|
||||
let text = text.trim_matches('-');
|
||||
|
||||
match max_length {
|
||||
0 => text.to_string(),
|
||||
limit => truncate(text, limit, word_boundary),
|
||||
}
|
||||
}
|
||||
|
||||
fn character(code: Option<u32>, whole: &str) -> String {
|
||||
code.and_then(char::from_u32)
|
||||
.map_or_else(|| whole.to_string(), String::from)
|
||||
}
|
||||
|
||||
/// Cut a slug to `limit` characters.
|
||||
///
|
||||
/// At a word, when asked — and a word that does not fit is skipped rather
|
||||
/// than ending the slug, so a long word in the middle of a title does not
|
||||
/// cost every word after it. `an-introduction-to-historical-analysis` cut to
|
||||
/// twenty-eight is `an-introduction-to-analysis`.
|
||||
fn truncate(text: &str, limit: usize, word_boundary: bool) -> String {
|
||||
if text.len() < limit {
|
||||
return text.to_string();
|
||||
}
|
||||
if !word_boundary {
|
||||
return text[..floor_boundary(text, limit)]
|
||||
.trim_matches('-')
|
||||
.to_string();
|
||||
}
|
||||
if !text.contains('-') {
|
||||
return text[..floor_boundary(text, limit)].to_string();
|
||||
}
|
||||
|
||||
let mut kept = String::new();
|
||||
for word in text.split('-') {
|
||||
if word.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let would_be = kept.len() + word.len();
|
||||
if would_be < limit {
|
||||
kept.push_str(word);
|
||||
kept.push('-');
|
||||
} else if would_be == limit {
|
||||
kept.push_str(word);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if kept.is_empty() {
|
||||
kept = text[..floor_boundary(text, limit)].to_string();
|
||||
}
|
||||
kept.trim_matches('-').to_string()
|
||||
}
|
||||
|
||||
/// The largest character boundary at or below `at`, so a cut never lands
|
||||
/// inside a character.
|
||||
fn floor_boundary(text: &str, at: usize) -> usize {
|
||||
let mut at = at.min(text.len());
|
||||
while at > 0 && !text.is_char_boundary(at) {
|
||||
at -= 1;
|
||||
}
|
||||
at
|
||||
}
|
||||
|
||||
/// The first name in an author field, which may list several.
|
||||
///
|
||||
/// An edited volume can carry five names, and the whole list ahead of the
|
||||
/// title spends the entire budget on people. An identifier needs one name and
|
||||
/// the work; the rest are in the record.
|
||||
pub fn lead_author(author: &str) -> String {
|
||||
static SEPARATOR: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\s*[&;]\s*|\s+and\s+").expect("valid"));
|
||||
|
||||
let author = author.trim_matches(textkit::is_space);
|
||||
let lead = SEPARATOR.splitn(author, 2).next().unwrap_or("");
|
||||
// Commas are ambiguous: "Bishop, Christopher M." is one person, but
|
||||
// "Miranda Lundy, Daud Sutton, Anthony Ashton" is three — and taking the
|
||||
// whole list truncates it mid-name at the budget. Two or more commas
|
||||
// means a list; one means a surname first.
|
||||
let lead = match lead.matches(',').count() >= 2 {
|
||||
true => lead.split_once(',').map_or(lead, |(first, _)| first),
|
||||
false => lead,
|
||||
};
|
||||
lead.trim_matches(|c: char| textkit::is_space(c) || c == ',')
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// The slug a record's current fields imply.
|
||||
///
|
||||
/// The author is reduced to its lead name and given a bounded share of the
|
||||
/// budget, so the title is always represented.
|
||||
pub fn slug_for(author: Option<&str>, title: Option<&str>, id: i64) -> String {
|
||||
let lead = slugify(&lead_author(author.unwrap_or("")), AUTHOR_BUDGET, true);
|
||||
let room = MAX_LENGTH.saturating_sub(if lead.is_empty() { 0 } else { lead.len() + 1 });
|
||||
let work = slugify(title.unwrap_or(""), room.max(20), true);
|
||||
|
||||
let base = [lead.as_str(), work.as_str()]
|
||||
.into_iter()
|
||||
.filter(|part| !part.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join("-");
|
||||
match base.is_empty() {
|
||||
true => format!("book-{id}"),
|
||||
false => format!("{base}-{id}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Head tokens that mean the slug was minted without an author.
|
||||
///
|
||||
/// The slug's first token comes from the author field, so a placeholder there
|
||||
/// is direct evidence the slug was made blind — independent of whatever the
|
||||
/// record says now.
|
||||
const PLACEHOLDER_AUTHORS: &[&str] = &[
|
||||
"admin",
|
||||
"administrator",
|
||||
"anonymous",
|
||||
"book",
|
||||
"default",
|
||||
"guest",
|
||||
"imac",
|
||||
"na",
|
||||
"none",
|
||||
"someauthor",
|
||||
"test",
|
||||
"unknown",
|
||||
"untitled",
|
||||
"user",
|
||||
];
|
||||
|
||||
/// Words that carry no weight when deciding whether a slug and a record are
|
||||
/// describing the same book.
|
||||
const STOPWORDS: &[&str] = &[
|
||||
"a", "an", "and", "as", "at", "by", "de", "for", "from", "in", "is", "la", "le", "of", "on",
|
||||
"or", "the", "to", "with",
|
||||
];
|
||||
|
||||
/// Below this share of the slug's meaningful tokens appearing in the current
|
||||
/// title or author, the slug is describing some other book.
|
||||
///
|
||||
/// Deliberately low: above it sit ordinary drifts — a subtitle dropped, an
|
||||
/// editor removed — and a disagreement this size is held for review either
|
||||
/// way.
|
||||
const WRONG_BOOK_PRECISION: f64 = 0.34;
|
||||
|
||||
/// A new slug that recovers less than this much of the current title is not
|
||||
/// clearly an improvement.
|
||||
const MIN_TITLE_RECALL: f64 = 0.35;
|
||||
|
||||
/// Traces of the file a book arrived as, rather than of the book: mirror
|
||||
/// suffixes, a wrapper format, a word processor's name, a Windows path.
|
||||
static ARTEFACT: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"z-lib|1lib|zlibrary|djvu|microsoft-word|documents-and-settings|(?:^|-)(?:pdf|doc|docx|txt|tmp|epub|mobi|azw3|rtf|prnt|qxd|qxp|fm)(?:-|$)",
|
||||
)
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// A volume marker.
|
||||
///
|
||||
/// One in the current slug is worth more than a clean slug without: several
|
||||
/// volumes of a proceedings share a title, and their existing slugs are what
|
||||
/// tell them apart.
|
||||
static VOLUME: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?:^|-)(?:vol|volume|part|book|bk|no|nr|tome|band)-?(?:\d+|[ivxl]+)(?:-|$)")
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// An author field that has swallowed part of the title or the series.
|
||||
///
|
||||
/// A slug built from that names nothing, and it is junk of a kind the
|
||||
/// placeholder and artefact tests cannot see, because the text is neither a
|
||||
/// handle nor a filename.
|
||||
static SPILLED_AUTHOR: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
|
||||
use textkit::pattern::{WORD_END, WORD_START};
|
||||
fancy_regex::Regex::new(&format!(
|
||||
r"(?i)[()\[\]]|{WORD_START}volume{WORD_END}|{WORD_START}an introduction{WORD_END}"
|
||||
))
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// Tiers that rest on evidence inside the slug itself, and so can be applied
|
||||
/// without a person reading them.
|
||||
///
|
||||
/// A placeholder author or a filename artefact is a property of the slug and
|
||||
/// cannot be wrong about itself. Everything else is a *disagreement* between
|
||||
/// the slug and the record, and a disagreement does not say which side is at
|
||||
/// fault.
|
||||
pub const SAFE_TIERS: &[&str] = &["filename-artefact", "placeholder-author"];
|
||||
|
||||
/// What should happen to a record's slug.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Proposal {
|
||||
/// The slug the record has.
|
||||
pub current: String,
|
||||
/// The slug its fields imply.
|
||||
pub proposed: String,
|
||||
/// How much the evidence is worth; see [`SAFE_TIERS`].
|
||||
pub tier: String,
|
||||
/// Why, in the order the evidence was found.
|
||||
pub reasons: Vec<String>,
|
||||
}
|
||||
|
||||
impl Proposal {
|
||||
/// Whether this can be applied without a person reading it.
|
||||
pub fn is_safe(&self) -> bool {
|
||||
SAFE_TIERS.binary_search(&self.tier.as_str()).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Judge a record's slug. `None` when there is nothing wrong with it.
|
||||
///
|
||||
/// What comes back is a tier rather than a verdict, because the three kinds
|
||||
/// of evidence differ in kind and only two of them are about the slug alone.
|
||||
pub fn evaluate(
|
||||
current: &str,
|
||||
author: Option<&str>,
|
||||
title: Option<&str>,
|
||||
id: i64,
|
||||
) -> Option<Proposal> {
|
||||
let body = strip_id(current);
|
||||
let body_tokens = tokens(body);
|
||||
let title_tokens: Vec<String> = tokens(&slugify(title.unwrap_or(""), 0, false))
|
||||
.into_iter()
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
let author_tokens: Vec<String> = tokens(&slugify(author.unwrap_or(""), 0, false))
|
||||
.into_iter()
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
|
||||
let meaningful: Vec<&str> = body_tokens
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|token| STOPWORDS.binary_search(token).is_err() && token.chars().count() > 1)
|
||||
.collect();
|
||||
let precision = match meaningful.is_empty() {
|
||||
true => 0.0,
|
||||
false => {
|
||||
let matched = meaningful
|
||||
.iter()
|
||||
.filter(|token| {
|
||||
title_tokens.iter().any(|t| t == *token)
|
||||
|| author_tokens.iter().any(|t| t == *token)
|
||||
})
|
||||
.count();
|
||||
matched as f64 / meaningful.len() as f64
|
||||
}
|
||||
};
|
||||
|
||||
let author_text = author.unwrap_or("").trim_matches(textkit::is_space);
|
||||
let author_known = !author_text.is_empty() && author.unwrap_or("") != "Unknown";
|
||||
|
||||
let mut reasons = Vec::new();
|
||||
let tier = if body_tokens
|
||||
.first()
|
||||
.is_some_and(|head| PLACEHOLDER_AUTHORS.binary_search(head).is_ok())
|
||||
&& author_known
|
||||
{
|
||||
reasons.push(format!(
|
||||
"slug opens with placeholder author '{}'",
|
||||
body_tokens[0]
|
||||
));
|
||||
"placeholder-author"
|
||||
} else if ARTEFACT.is_match(body) {
|
||||
reasons.push("slug carries a filename or mirror artefact".to_string());
|
||||
"filename-artefact"
|
||||
} else if precision < WRONG_BOOK_PRECISION {
|
||||
reasons.push(format!(
|
||||
"only {} of slug tokens appear in the current title/author — slug and metadata disagree",
|
||||
textkit::percent(precision)
|
||||
));
|
||||
"wrong-book"
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let proposed = slug_for(author, title, id);
|
||||
let proposed_body = strip_id(&proposed);
|
||||
let proposed_tokens = tokens(proposed_body);
|
||||
|
||||
if proposed == current {
|
||||
reasons.push("recomputes to the same slug: fix the metadata first".into());
|
||||
return Some(Proposal {
|
||||
current: current.into(),
|
||||
proposed,
|
||||
tier: "noop-metadata-junk".into(),
|
||||
reasons,
|
||||
});
|
||||
}
|
||||
|
||||
let recall = |slug_tokens: &[&str]| -> f64 {
|
||||
match title_tokens.is_empty() {
|
||||
true => 0.0,
|
||||
false => {
|
||||
let found = title_tokens
|
||||
.iter()
|
||||
.filter(|token| slug_tokens.iter().any(|t| t == token))
|
||||
.count();
|
||||
found as f64 / title_tokens.len() as f64
|
||||
}
|
||||
}
|
||||
};
|
||||
let proposed_recall = recall(&proposed_tokens);
|
||||
let recovers_title = proposed_recall > recall(&body_tokens);
|
||||
|
||||
// Is the proposal better than what is there, even if imperfect? A slug
|
||||
// still saying "unknown" because the author genuinely is, but now naming
|
||||
// the book, beats one naming a temporary directory. So the test is not
|
||||
// "is the new slug clean" but "does it recover title the current one
|
||||
// lacks, without carrying an artefact of its own".
|
||||
let still_placeholder = proposed_tokens
|
||||
.first()
|
||||
.is_some_and(|head| PLACEHOLDER_AUTHORS.binary_search(head).is_ok());
|
||||
if ARTEFACT.is_match(proposed_body) || (still_placeholder && !recovers_title) {
|
||||
reasons.push("new slug is no better: fix the metadata first".into());
|
||||
return Some(Proposal {
|
||||
current: current.into(),
|
||||
proposed,
|
||||
tier: "defer-new-slug-junk".into(),
|
||||
reasons,
|
||||
});
|
||||
}
|
||||
|
||||
// Only when the author actually reaches the slug: an author field of
|
||||
// nothing but brackets slugifies to nothing, so the proposal leads with
|
||||
// the title and is fine.
|
||||
let lead_in_slug = slugify(&lead_author(author_text), AUTHOR_BUDGET, true);
|
||||
if !lead_in_slug.is_empty() && SPILLED_AUTHOR.is_match(author_text).unwrap_or(false) {
|
||||
reasons.push(format!(
|
||||
"author field looks like spilled title/series text: '{author_text}'"
|
||||
));
|
||||
return Some(Proposal {
|
||||
current: current.into(),
|
||||
proposed,
|
||||
tier: "defer-new-slug-junk".into(),
|
||||
reasons,
|
||||
});
|
||||
}
|
||||
|
||||
if VOLUME.is_match(body) && !VOLUME.is_match(proposed_body) {
|
||||
reasons.push("current slug carries a volume marker the new one loses".into());
|
||||
return Some(Proposal {
|
||||
current: current.into(),
|
||||
proposed,
|
||||
tier: "review-loses-volume".into(),
|
||||
reasons,
|
||||
});
|
||||
}
|
||||
|
||||
// Measured on the proposal itself rather than recomputed from the fields:
|
||||
// the generator budgets the author separately, so a formula that
|
||||
// re-derives the length describes a slug it no longer builds.
|
||||
let truncated = proposed_body.len() >= MAX_LENGTH - 1;
|
||||
if truncated || proposed_recall < MIN_TITLE_RECALL {
|
||||
reasons.push(format!(
|
||||
"new slug recovers only {} of the title",
|
||||
textkit::percent(proposed_recall)
|
||||
));
|
||||
return Some(Proposal {
|
||||
current: current.into(),
|
||||
proposed,
|
||||
tier: "review-low-recall".into(),
|
||||
reasons,
|
||||
});
|
||||
}
|
||||
|
||||
Some(Proposal {
|
||||
current: current.into(),
|
||||
proposed,
|
||||
tier: tier.into(),
|
||||
reasons,
|
||||
})
|
||||
}
|
||||
|
||||
/// A slug without its trailing numeric id.
|
||||
fn strip_id(slug: &str) -> &str {
|
||||
static TRAILING_ID: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"-\d+$").expect("valid"));
|
||||
match TRAILING_ID.find(slug) {
|
||||
Some(found) if found.end() == slug.len() => &slug[..found.start()],
|
||||
_ => slug,
|
||||
}
|
||||
}
|
||||
|
||||
fn tokens(slug: &str) -> Vec<&str> {
|
||||
slug.split('-').filter(|token| !token.is_empty()).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_entity_table_is_sorted_so_it_can_be_searched() {
|
||||
assert!(NAMED.windows(2).all(|pair| pair[0].0 < pair[1].0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_slugifies() {
|
||||
assert_eq!(slugify("William Zinsser", 0, false), "william-zinsser");
|
||||
assert_eq!(
|
||||
slugify("Hello---World & Friends", 0, false),
|
||||
"hello-world-friends"
|
||||
);
|
||||
assert_eq!(slugify("", 0, false), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_written_apostrophe_is_a_word_break() {
|
||||
assert_eq!(
|
||||
slugify("Kierkegaard's Conception", 0, false),
|
||||
"kierkegaard-s-conception"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_apostrophe_transliteration_produced_is_not() {
|
||||
// The Cyrillic hard sign becomes one, and a dash for each would make
|
||||
// a slug of fragments.
|
||||
assert_eq!(slugify("Олегъ", 0, false), "oleg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_scripts_are_transliterated() {
|
||||
assert_eq!(slugify("Être et Temps", 0, false), "etre-et-temps");
|
||||
assert_eq!(slugify("café naïve", 0, false), "cafe-naive");
|
||||
assert_eq!(slugify("Straße", 0, false), "strasse");
|
||||
assert_eq!(slugify("Ἀθῆναι καὶ Μόσχα", 0, false), "athenai-kai-moskha");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_thousands_separator_does_not_split_a_number() {
|
||||
assert_eq!(slugify("12,000 Years", 0, false), "12000-years");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn character_references_are_decoded() {
|
||||
assert_eq!(slugify("Tom & Jerry", 0, false), "tom-jerry");
|
||||
assert_eq!(slugify("café", 0, false), "cafe");
|
||||
assert_eq!(slugify("café", 0, false), "cafe");
|
||||
assert_eq!(slugify("¬aname; here", 0, false), "notaname-here");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cut_lands_on_a_word() {
|
||||
assert_eq!(
|
||||
slugify("An Introduction to Historical Analysis", 28, true),
|
||||
"an-introduction-to-analysis"
|
||||
);
|
||||
assert_eq!(
|
||||
slugify("An Introduction to Historical Analysis", 28, false),
|
||||
"an-introduction-to-historica"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lead_author_is_one_person() {
|
||||
assert_eq!(lead_author("A. A. Long & J. A. Mansfeld"), "A. A. Long");
|
||||
assert_eq!(
|
||||
lead_author("Clarke, Arthur C. and Lee, Gentry"),
|
||||
"Clarke, Arthur C."
|
||||
);
|
||||
assert_eq!(
|
||||
lead_author("Bishop, Christopher M."),
|
||||
"Bishop, Christopher M."
|
||||
);
|
||||
assert_eq!(
|
||||
lead_author("Miranda Lundy, Daud Sutton, Anthony Ashton"),
|
||||
"Miranda Lundy"
|
||||
);
|
||||
assert_eq!(lead_author(""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_names_a_person_and_a_work() {
|
||||
assert_eq!(
|
||||
slug_for(Some("William Zinsser"), Some("On Writing Well"), 3450),
|
||||
"william-zinsser-on-writing-well-3450"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_always_has_room_for_the_title() {
|
||||
let slug = slug_for(
|
||||
Some("A. A. Long & J. A. Mansfeld & Keimpe Algra & Carl A. Huffman"),
|
||||
Some("The Cambridge Companion to Early Greek Philosophy"),
|
||||
1234,
|
||||
);
|
||||
assert!(slug.contains("cambridge"), "{slug}");
|
||||
assert!(slug.len() <= MAX_LENGTH + 6, "{slug}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_with_nothing_usable_still_gets_a_slug() {
|
||||
assert_eq!(slug_for(Some("()"), Some(""), 7023), "book-7023");
|
||||
assert_eq!(slug_for(None, None, 1), "book-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_that_agrees_with_its_record_is_left_alone() {
|
||||
assert_eq!(
|
||||
evaluate(
|
||||
"william-zinsser-on-writing-well-3450",
|
||||
Some("William Zinsser"),
|
||||
Some("On Writing Well"),
|
||||
3450
|
||||
),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_minted_without_an_author_is_safe_to_replace() {
|
||||
let found = evaluate(
|
||||
"unknown-mind-and-brain-1234",
|
||||
Some("Zinsser, William"),
|
||||
Some("Mind and Brain"),
|
||||
1234,
|
||||
)
|
||||
.expect("a proposal");
|
||||
assert_eq!(found.tier, "placeholder-author");
|
||||
assert!(found.is_safe());
|
||||
assert_eq!(found.proposed, "zinsser-william-mind-and-brain-1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_carrying_a_filename_artefact_is_safe_to_replace() {
|
||||
let found = evaluate(
|
||||
"smith-john-the-mill-z-lib-org-99",
|
||||
Some("Smith, John"),
|
||||
Some("The Mill and the Valley"),
|
||||
99,
|
||||
)
|
||||
.expect("a proposal");
|
||||
assert_eq!(found.tier, "filename-artefact");
|
||||
assert!(found.is_safe());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slug_naming_another_book_is_held_for_review() {
|
||||
let found = evaluate(
|
||||
"william-zinsser-on-writing-well-3450",
|
||||
Some("Laura Nader"),
|
||||
Some("Weaponizing Anthropology"),
|
||||
3450,
|
||||
)
|
||||
.expect("a proposal");
|
||||
assert_eq!(found.tier, "wrong-book");
|
||||
assert!(
|
||||
!found.is_safe(),
|
||||
"a disagreement does not say which side is wrong"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_proposal_that_changes_nothing_says_to_fix_the_record() {
|
||||
// The record's own fields carry the artefact, so regenerating from
|
||||
// them reproduces it exactly.
|
||||
let found = evaluate(
|
||||
"microsoft-word-mat-doc-1",
|
||||
Some("Microsoft Word"),
|
||||
Some("mat doc"),
|
||||
1,
|
||||
)
|
||||
.expect("a proposal");
|
||||
assert_eq!(found.tier, "noop-metadata-junk");
|
||||
assert_eq!(found.proposed, found.current);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_slug_that_is_still_a_placeholder_is_deferred() {
|
||||
let found =
|
||||
evaluate("unknown-1234", Some("Unknown Author"), None, 1234).expect("a proposal");
|
||||
assert_eq!(found.tier, "defer-new-slug-junk");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_volume_marker_the_new_slug_would_lose_is_held_for_review() {
|
||||
let found = evaluate(
|
||||
"unknown-proceedings-of-the-society-vol-9-77",
|
||||
Some("Aristotelian Society"),
|
||||
Some("Proceedings of the Society"),
|
||||
77,
|
||||
)
|
||||
.expect("a proposal");
|
||||
assert_eq!(found.tier, "review-loses-volume");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_author_field_holding_title_text_is_not_used_to_rename() {
|
||||
let found = evaluate(
|
||||
"unknown-the-land-of-iran-55",
|
||||
Some("the land of iran (Cambridge University Press"),
|
||||
Some("The Land of Iran"),
|
||||
55,
|
||||
)
|
||||
.expect("a proposal");
|
||||
assert_eq!(found.tier, "defer-new-slug-junk");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_author_that_slugifies_to_nothing_leaves_the_title_leading() {
|
||||
assert_eq!(
|
||||
slug_for(Some("()"), Some("Mind and Brain"), 7023),
|
||||
"mind-and-brain-7023"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -64,38 +64,14 @@ pub fn segment(cues: &[Cue], duration: f64) -> Vec<Segment> {
|
||||
|
||||
/// How many chapters a recording of this length should have.
|
||||
pub fn target_chapter_count(duration: f64) -> usize {
|
||||
let target = round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER);
|
||||
let target = textkit::round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER);
|
||||
(target as usize).clamp(MIN_CHAPTERS, MAX_CHAPTERS)
|
||||
}
|
||||
|
||||
/// Round half to even, which is what the Python this was ported from does
|
||||
/// and what Rust's own `round` does not: `round(2.5)` is 2 there and 3 here,
|
||||
/// and a recording exactly two and a half chapters long would otherwise get a
|
||||
/// different number of them.
|
||||
fn round_half_to_even(value: f64) -> f64 {
|
||||
let rounded = value.round();
|
||||
if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 {
|
||||
rounded - value.signum()
|
||||
} else {
|
||||
rounded
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rounding_goes_to_even_on_a_half() {
|
||||
assert_eq!(round_half_to_even(0.5), 0.0);
|
||||
assert_eq!(round_half_to_even(1.5), 2.0);
|
||||
assert_eq!(round_half_to_even(2.5), 2.0);
|
||||
assert_eq!(round_half_to_even(3.5), 4.0);
|
||||
assert_eq!(round_half_to_even(-2.5), -2.0);
|
||||
assert_eq!(round_half_to_even(2.4), 2.0);
|
||||
assert_eq!(round_half_to_even(2.6), 3.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_chapter_count_follows_the_length_within_bounds() {
|
||||
assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS);
|
||||
|
||||
@@ -15,6 +15,6 @@ pub mod pattern;
|
||||
pub mod segment;
|
||||
pub mod tokens;
|
||||
|
||||
pub use number::decimal;
|
||||
pub use number::{decimal, percent, round_half_to_even};
|
||||
pub use segment::{is_space, paragraphs, sentences};
|
||||
pub use tokens::{count, decode_lossy, encode, split_at_token_limit};
|
||||
|
||||
@@ -47,6 +47,28 @@ fn digit_value(c: char) -> Option<u32> {
|
||||
ten.then_some(value)
|
||||
}
|
||||
|
||||
/// Round half to even, which is what Python's `round` does and what Rust's
|
||||
/// does not: `round(2.5)` is 2 there and 3 here.
|
||||
///
|
||||
/// It matters wherever a rounded number decides something — how many
|
||||
/// chapters a recording gets, what a percentage in a report reads as — and
|
||||
/// the two rules disagree exactly on the values that land on a boundary,
|
||||
/// which is where a disagreement is most likely to be noticed.
|
||||
pub fn round_half_to_even(value: f64) -> f64 {
|
||||
let rounded = value.round();
|
||||
if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 {
|
||||
rounded - value.signum()
|
||||
} else {
|
||||
rounded
|
||||
}
|
||||
}
|
||||
|
||||
/// A fraction as a whole-number percentage, rounded the same way Python
|
||||
/// rounds it.
|
||||
pub fn percent(fraction: f64) -> String {
|
||||
format!("{}%", round_half_to_even(fraction * 100.0))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -84,6 +106,25 @@ mod tests {
|
||||
assert_eq!(decimal("page"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rounding_goes_to_even_on_a_half() {
|
||||
assert_eq!(round_half_to_even(0.5), 0.0);
|
||||
assert_eq!(round_half_to_even(1.5), 2.0);
|
||||
assert_eq!(round_half_to_even(2.5), 2.0);
|
||||
assert_eq!(round_half_to_even(-2.5), -2.0);
|
||||
assert_eq!(round_half_to_even(2.4), 2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_percentage_reads_as_a_whole_number() {
|
||||
assert_eq!(percent(0.0), "0%");
|
||||
// Not 33: a third of a percent is a hair above the halfway point
|
||||
// once it is a binary float, so it rounds up rather than to even.
|
||||
assert_eq!(percent(0.335), "34%");
|
||||
assert_eq!(percent(1.0), "100%");
|
||||
assert_eq!(percent(0.125), "12%");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absurd_number_saturates_rather_than_wrapping() {
|
||||
assert_eq!(decimal(&"9".repeat(40)), u32::MAX);
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
{"slug": "william-zinsser-on-writing-well-3450", "author": "William Zinsser", "title": "On Writing Well", "id": 3450}
|
||||
{"slug": "william-zinsser-on-writing-well-3450", "author": "Laura Nader", "title": "Weaponizing Anthropology", "id": 3450}
|
||||
{"slug": "unknown-mind-and-brain-1234", "author": "Zinsser, William", "title": "Mind and Brain", "id": 1234}
|
||||
{"slug": "unknown-file-c-documents-and-settings-hasici-dokumen-88", "author": "Unknown", "title": "Mind and Brain: Chaos and Quantum Mechanics", "id": 88}
|
||||
{"slug": "smith-john-the-mill-z-lib-org-99", "author": "Smith, John", "title": "The Mill and the Valley", "id": 99}
|
||||
{"slug": "microsoft-word-mat-doc-1", "author": "Microsoft Word", "title": "mat doc", "id": 1}
|
||||
{"slug": "unknown-1234", "author": "Unknown Author", "title": null, "id": 1234}
|
||||
{"slug": "unknown-proceedings-of-the-society-vol-9-77", "author": "Aristotelian Society", "title": "Proceedings of the Society", "id": 77}
|
||||
{"slug": "unknown-the-land-of-iran-55", "author": "the land of iran (Cambridge University Press", "title": "The Land of Iran", "id": 55}
|
||||
{"slug": "old-testament-volume-ii-mark-sheridan-12", "author": "Old Testament, Volume II) (Mark Sheridan", "title": "Genesis 12-50", "id": 12}
|
||||
{"slug": "a-a-long-j-a-mansfeld-keimpe-algra-carl-a-huffman-edward-h-7", "author": "A. A. Long & J. A. Mansfeld & Keimpe Algra & Carl A. Huffman & Edward H. Hussey", "title": "The Cambridge Companion to Early Greek Philosophy", "id": 7}
|
||||
{"slug": "book-7023", "author": "()", "title": "Mind and Brain", "id": 7023}
|
||||
{"slug": "test-3-8", "author": "test", "title": "A Real Book About Something", "id": 8}
|
||||
{"slug": "jaroslav-pelikan-christianity-and-classical-culture-the-me-44", "author": "Jaroslav Pelikan", "title": "Christianity and Classical Culture: The Metamorphosis of Natural Theology in the Christian Encounter with Hellenism", "id": 44}
|
||||
{"slug": "etre-et-temps-9", "author": "Heidegger, Martin", "title": "Être et Temps", "id": 9}
|
||||
{"slug": "athenai-kai-moskha-10", "author": "Θουκυδίδης", "title": "Ἀθῆναι καὶ Μόσχα", "id": 10}
|
||||
{"slug": "dong-jing-noshou-zhi-11", "author": "夏目漱石", "title": "東京の手紙", "id": 11}
|
||||
{"slug": "kierkegaard-s-conception-of-god-13", "author": "Kierkegaard, Søren", "title": "Kierkegaard's Conception of God", "id": 13}
|
||||
{"slug": "12000-years-14", "author": "Smith, J.", "title": "12,000 Years of History", "id": 14}
|
||||
{"slug": "tom-jerry-15", "author": "Hanna & Barbera", "title": "Tom & Jerry", "id": 15}
|
||||
{"slug": "an-introduction-to-analysis-16", "author": null, "title": "An Introduction to Historical Analysis", "id": 16}
|
||||
{"slug": "bishop-christopher-m-pattern-recognition-17", "author": "Bishop, Christopher M.", "title": "Pattern Recognition and Machine Learning", "id": 17}
|
||||
{"slug": "miranda-lundy-quadrivium-18", "author": "Miranda Lundy, Daud Sutton, Anthony Ashton, Jason Martineau", "title": "Quadrivium", "id": 18}
|
||||
{"slug": "", "author": "Smith, John", "title": "A Book", "id": 19}
|
||||
{"slug": "guardians-guardians-3190", "author": "Guardians", "title": "Guardians", "id": 3190}
|
||||
@@ -0,0 +1,25 @@
|
||||
{"lead_author": "William Zinsser", "proposed": null, "reasons": [], "slug_for": "william-zinsser-on-writing-well-3450", "slugify": "on-writing-well", "tier": null}
|
||||
{"lead_author": "Laura Nader", "proposed": "laura-nader-weaponizing-anthropology-3450", "reasons": ["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"], "slug_for": "laura-nader-weaponizing-anthropology-3450", "slugify": "weaponizing-anthropology", "tier": "wrong-book"}
|
||||
{"lead_author": "Zinsser, William", "proposed": "zinsser-william-mind-and-brain-1234", "reasons": ["slug opens with placeholder author 'unknown'"], "slug_for": "zinsser-william-mind-and-brain-1234", "slugify": "mind-and-brain", "tier": "placeholder-author"}
|
||||
{"lead_author": "Unknown", "proposed": "unknown-mind-and-brain-chaos-and-quantum-mechanics-88", "reasons": ["slug carries a filename or mirror artefact"], "slug_for": "unknown-mind-and-brain-chaos-and-quantum-mechanics-88", "slugify": "mind-and-brain-chaos-and-quantum-mechanics", "tier": "filename-artefact"}
|
||||
{"lead_author": "Smith, John", "proposed": "smith-john-the-mill-and-the-valley-99", "reasons": ["slug carries a filename or mirror artefact"], "slug_for": "smith-john-the-mill-and-the-valley-99", "slugify": "the-mill-and-the-valley", "tier": "filename-artefact"}
|
||||
{"lead_author": "Microsoft Word", "proposed": "microsoft-word-mat-doc-1", "reasons": ["slug carries a filename or mirror artefact", "recomputes to the same slug: fix the metadata first"], "slug_for": "microsoft-word-mat-doc-1", "slugify": "mat-doc", "tier": "noop-metadata-junk"}
|
||||
{"lead_author": "Unknown Author", "proposed": "unknown-author-1234", "reasons": ["slug opens with placeholder author 'unknown'", "new slug is no better: fix the metadata first"], "slug_for": "unknown-author-1234", "slugify": "", "tier": "defer-new-slug-junk"}
|
||||
{"lead_author": "Aristotelian Society", "proposed": "aristotelian-society-proceedings-of-the-society-77", "reasons": ["slug opens with placeholder author 'unknown'", "current slug carries a volume marker the new one loses"], "slug_for": "aristotelian-society-proceedings-of-the-society-77", "slugify": "proceedings-of-the-society", "tier": "review-loses-volume"}
|
||||
{"lead_author": "the land of iran (Cambridge University Press", "proposed": "the-land-of-iran-cambridge-the-land-of-iran-55", "reasons": ["slug opens with placeholder author 'unknown'", "author field looks like spilled title/series text: 'the land of iran (Cambridge University Press'"], "slug_for": "the-land-of-iran-cambridge-the-land-of-iran-55", "slugify": "the-land-of-iran", "tier": "defer-new-slug-junk"}
|
||||
{"lead_author": "Old Testament, Volume II) (Mark Sheridan", "proposed": null, "reasons": [], "slug_for": "old-testament-volume-ii-mark-genesis-12-50-12", "slugify": "genesis-12-50", "tier": null}
|
||||
{"lead_author": "A. A. Long", "proposed": null, "reasons": [], "slug_for": "a-a-long-the-cambridge-companion-to-early-greek-philosophy-7", "slugify": "the-cambridge-companion-to-early-greek-philosophy", "tier": null}
|
||||
{"lead_author": "()", "proposed": "mind-and-brain-7023", "reasons": ["slug opens with placeholder author 'book'"], "slug_for": "mind-and-brain-7023", "slugify": "mind-and-brain", "tier": "placeholder-author"}
|
||||
{"lead_author": "test", "proposed": "test-a-real-book-about-something-8", "reasons": ["slug opens with placeholder author 'test'"], "slug_for": "test-a-real-book-about-something-8", "slugify": "a-real-book-about-something", "tier": "placeholder-author"}
|
||||
{"lead_author": "Jaroslav Pelikan", "proposed": null, "reasons": [], "slug_for": "jaroslav-pelikan-christianity-and-classical-culture-the-metamorphosis-of-natural-44", "slugify": "christianity-and-classical-culture-the-metamorphosis-of-natural-theology-in-the-christian-encounter-with-hellenism", "tier": null}
|
||||
{"lead_author": "Heidegger, Martin", "proposed": null, "reasons": [], "slug_for": "heidegger-martin-etre-et-temps-9", "slugify": "etre-et-temps", "tier": null}
|
||||
{"lead_author": "Θουκυδίδης", "proposed": null, "reasons": [], "slug_for": "thoukudides-athenai-kai-moskha-10", "slugify": "athenai-kai-moskha", "tier": null}
|
||||
{"lead_author": "夏目漱石", "proposed": null, "reasons": [], "slug_for": "xia-mu-shu-shi-dong-jing-noshou-zhi-11", "slugify": "dong-jing-noshou-zhi", "tier": null}
|
||||
{"lead_author": "Kierkegaard, Søren", "proposed": null, "reasons": [], "slug_for": "kierkegaard-soren-kierkegaard-s-conception-of-god-13", "slugify": "kierkegaard-s-conception-of-god", "tier": null}
|
||||
{"lead_author": "Smith, J.", "proposed": null, "reasons": [], "slug_for": "smith-j-12000-years-of-history-14", "slugify": "12000-years-of-history", "tier": null}
|
||||
{"lead_author": "Hanna", "proposed": null, "reasons": [], "slug_for": "hanna-tom-jerry-15", "slugify": "tom-jerry", "tier": null}
|
||||
{"lead_author": "", "proposed": null, "reasons": [], "slug_for": "an-introduction-to-historical-analysis-16", "slugify": "an-introduction-to-historical-analysis", "tier": null}
|
||||
{"lead_author": "Bishop, Christopher M.", "proposed": null, "reasons": [], "slug_for": "bishop-christopher-m-pattern-recognition-and-machine-learning-17", "slugify": "pattern-recognition-and-machine-learning", "tier": null}
|
||||
{"lead_author": "Miranda Lundy", "proposed": null, "reasons": [], "slug_for": "miranda-lundy-quadrivium-18", "slugify": "quadrivium", "tier": null}
|
||||
{"lead_author": "Smith, John", "proposed": "smith-john-a-book-19", "reasons": ["only 0% of slug tokens appear in the current title/author — slug and metadata disagree"], "slug_for": "smith-john-a-book-19", "slugify": "a-book", "tier": "wrong-book"}
|
||||
{"lead_author": "Guardians", "proposed": null, "reasons": [], "slug_for": "guardians-guardians-3190", "slugify": "guardians", "tier": null}
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Judge slugs the way the Python pipeline judges them.
|
||||
|
||||
Reads one `{"slug", "author", "title", "id"}` object per line and writes what
|
||||
the slug module makes of each.
|
||||
|
||||
CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
|
||||
python3 oracle/slug.py < cases/slug/records.ndjson
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
|
||||
if path:
|
||||
sys.path.insert(0, path)
|
||||
|
||||
from slugify import slugify # noqa: E402
|
||||
|
||||
from book_pipeline import slugs # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
out = sys.stdout
|
||||
for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
record = json.loads(line)
|
||||
author, title, book_id = record.get("author"), record.get("title"), record["id"]
|
||||
proposal = slugs.evaluate("x", record["slug"], author, title, book_id)
|
||||
out.write(
|
||||
json.dumps(
|
||||
{
|
||||
"slugify": slugify(title or ""),
|
||||
"slug_for": slugs.slug_for(author, title, book_id),
|
||||
"lead_author": slugs._lead_author(author),
|
||||
"tier": proposal.tier if proposal else None,
|
||||
"proposed": proposal.proposed_slug if proposal else None,
|
||||
"reasons": proposal.reasons if proposal else [],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+14
-1
@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
|
||||
type Job = (&'static str, fn() -> Written);
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let jobs: [Job; 11] = [
|
||||
let jobs: [Job; 12] = [
|
||||
("tokens", tokens),
|
||||
("unescape", unescape),
|
||||
("chunkmd", chunkmd),
|
||||
@@ -31,6 +31,7 @@ fn main() -> ExitCode {
|
||||
("cuepack", cuepack),
|
||||
("booksniff", booksniff),
|
||||
("bibclean", bibclean),
|
||||
("slug", slug),
|
||||
];
|
||||
|
||||
for (name, job) in jobs {
|
||||
@@ -74,6 +75,18 @@ fn unescape() -> Written {
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn slug() -> Written {
|
||||
let dir = parity::golden("slug");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
let cases = parity::dir().join("cases/slug/records.ndjson");
|
||||
let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
|
||||
let judgements = parity::run_oracle("slug.py", &cases)?;
|
||||
let path = dir.join("records.ndjson");
|
||||
std::fs::write(&path, judgements).map_err(|e| e.to_string())?;
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn bibclean() -> Written {
|
||||
let dir = parity::golden("bibclean");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
//! Slugs, and the judgement about whether one should be replaced.
|
||||
//!
|
||||
//! A slug is a public identifier: it reaches citations and research notes and
|
||||
//! outlives whatever produced it. Generating a different one than the system
|
||||
//! being replaced would rename every book in a corpus, and judging one
|
||||
//! differently would rename the wrong ones.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
fn records(path: &str) -> Vec<Value> {
|
||||
let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file");
|
||||
text.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| serde_json::from_str(line).expect("valid JSON"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn field<'a>(record: &'a Value, name: &str) -> Option<&'a str> {
|
||||
record.get(name).and_then(Value::as_str)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_slug_and_verdict_matches_python() {
|
||||
let cases = records("cases/slug/records.ndjson");
|
||||
let golden = records("golden/slug/records.ndjson");
|
||||
assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
|
||||
|
||||
let mut wrong: Vec<String> = Vec::new();
|
||||
for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() {
|
||||
let author = field(case, "author");
|
||||
let title = field(case, "title");
|
||||
let slug = field(case, "slug").unwrap_or("");
|
||||
let id = case["id"].as_i64().expect("an id");
|
||||
|
||||
let mut differs = |what: &str, ours: String, theirs: &Value| {
|
||||
wrong.push(format!("record {at} {what}: python {theirs}, rust {ours}"));
|
||||
};
|
||||
|
||||
let ours = bibclean::slug::slugify(title.unwrap_or(""), 0, false);
|
||||
if Some(ours.as_str()) != theirs["slugify"].as_str() {
|
||||
differs("slugify", ours, &theirs["slugify"]);
|
||||
}
|
||||
|
||||
let ours = bibclean::slug::lead_author(author.unwrap_or(""));
|
||||
if Some(ours.as_str()) != theirs["lead_author"].as_str() {
|
||||
differs("lead_author", ours, &theirs["lead_author"]);
|
||||
}
|
||||
|
||||
let ours = bibclean::slug::slug_for(author, title, id);
|
||||
if Some(ours.as_str()) != theirs["slug_for"].as_str() {
|
||||
differs("slug_for", ours, &theirs["slug_for"]);
|
||||
}
|
||||
|
||||
let proposal = bibclean::slug::evaluate(slug, author, title, id);
|
||||
let ours_tier = proposal.as_ref().map(|found| found.tier.as_str());
|
||||
if ours_tier != theirs["tier"].as_str() {
|
||||
differs("tier", format!("{ours_tier:?}"), &theirs["tier"]);
|
||||
}
|
||||
|
||||
let ours_proposed = proposal.as_ref().map(|found| found.proposed.as_str());
|
||||
if ours_proposed != theirs["proposed"].as_str() {
|
||||
differs(
|
||||
"proposed",
|
||||
format!("{ours_proposed:?}"),
|
||||
&theirs["proposed"],
|
||||
);
|
||||
}
|
||||
|
||||
let ours_reasons: Vec<String> = proposal
|
||||
.as_ref()
|
||||
.map(|found| found.reasons.clone())
|
||||
.unwrap_or_default();
|
||||
let their_reasons: Vec<String> = theirs["reasons"]
|
||||
.as_array()
|
||||
.expect("a list")
|
||||
.iter()
|
||||
.map(|reason| reason.as_str().expect("a reason").to_string())
|
||||
.collect();
|
||||
if ours_reasons != their_reasons {
|
||||
differs("reasons", format!("{ours_reasons:?}"), &theirs["reasons"]);
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
wrong.is_empty(),
|
||||
"{} judgements differ:\n{}",
|
||||
wrong.len(),
|
||||
wrong.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Catches a golden file drifting away from a Python that has moved on.
|
||||
#[test]
|
||||
#[cfg(feature = "python")]
|
||||
fn the_golden_file_still_matches_live_python() {
|
||||
let cases = std::fs::read_to_string(parity::dir().join("cases/slug/records.ndjson"))
|
||||
.expect("a readable case file");
|
||||
let live = parity::run_oracle("slug.py", &cases).expect("the oracle runs");
|
||||
assert_eq!(live, parity::read_golden_text("slug/records.ndjson"));
|
||||
}
|
||||
Reference in New Issue
Block a user