diff --git a/Cargo.lock b/Cargo.lock index 3f0aba5..176742f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,7 @@ dependencies = [ "serde", "serde-wasm-bindgen", "serde_json", + "seriesof", "textkit", "wasm-bindgen", ] @@ -1013,6 +1014,7 @@ dependencies = [ "pdftriage", "rrf", "serde_json", + "seriesof", "textkit", ] @@ -1420,6 +1422,20 @@ dependencies = [ "zmij", ] +[[package]] +name = "seriesof" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "fancy-regex 0.16.2", + "schemars", + "serde", + "serde_json", + "textkit", +] + [[package]] name = "sha2" version = "0.10.9" diff --git a/Cargo.toml b/Cargo.toml index a4da8e6..665a27c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ bibclean = { path = "crates/bibclean", version = "0.1.0" } avplan = { path = "crates/avplan", version = "0.1.0" } pdfgutter = { path = "crates/pdfgutter", version = "0.1.0" } catena = { path = "crates/bindings", version = "0.1.0" } +seriesof = { path = "crates/seriesof", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } diff --git a/README.md b/README.md index 465e896..52d164b 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Each tool installs by name, so you can take one without taking the rest. | [`pdftriage`](crates/pdftriage/README.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed | | [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change | | [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary | +| [`seriesof`](crates/seriesof/README.md) | The series a book belongs to, and which volume this is | | [`mdscrub`](crates/mdscrub/README.md) | Converted markdown in, clean markdown out | | [`mdsplit`](crates/mdsplit/README.md) | One markdown file into the works inside it | | [`chunkmd`](crates/chunkmd/README.md) | Markdown into passages that carry their page, chapter and role | diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index 3e59ef3..a7064dc 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -36,6 +36,7 @@ cuepack.workspace = true mdscrub.workspace = true mdsplit.workspace = true rrf.workspace = true +seriesof.workspace = true serde.workspace = true serde_json.workspace = true textkit.workspace = true diff --git a/crates/bindings/README.md b/crates/bindings/README.md index e136f2d..8589261 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -67,6 +67,7 @@ except ValueError as why: | `split_collection` | [`mdsplit`](../mdsplit/README.md) | | `fuse_hits` | [`rrf`](../rrf/README.md) | | `identify` | [`booksniff`](../booksniff/README.md) | +| `series_of` | [`seriesof`](../seriesof/README.md) | The tools that are not here are the ones a binding would gain nothing over a subprocess for: they read files, shell out to another program, or talk to a diff --git a/crates/bindings/npm/README.md b/crates/bindings/npm/README.md index c44e691..1ab762c 100644 --- a/crates/bindings/npm/README.md +++ b/crates/bindings/npm/README.md @@ -48,6 +48,7 @@ try { | `splitCollection` | One markdown file into the works inside it | | `fuseHits` | Several ranked lists into one | | `identify` | What a file is, from its first bytes | +| `seriesOf` | The series a book belongs to, and which volume this is | The tools that read files or call another program are not here: a browser has neither, and a binding that pretends otherwise fails at run time rather than diff --git a/crates/bindings/npm/typecheck.ts b/crates/bindings/npm/typecheck.ts index fef0c35..3d3fa16 100644 --- a/crates/bindings/npm/typecheck.ts +++ b/crates/bindings/npm/typecheck.ts @@ -3,8 +3,8 @@ // check, and nothing else would notice. import { chunkMarkdown, scrub, countTokens, parseSubtitles, packCues, - splitCollection, fuseHits, identify, - type Chunk, type Cue, type Section, type Hit, type Doc, + splitCollection, fuseHits, identify, seriesOf, + type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, } from "./dist/web/catena.js"; const chunks: Chunk[] = chunkMarkdown("# One\n\nbody\n", 512, 64, 50, 768); @@ -35,4 +35,8 @@ parseSubtitles("x"); packCues(cues); fuseHits(hits); -export const checked = [page, chapter, flags, clean, tokens, start, packed, slug, fused, kind, notes]; +const proposal: SeriesProposal | undefined = seriesOf("The Philokalia Vol 3"); +const volume: number | undefined = proposal?.volume; +seriesOf("Aquinas (Companion)", "Cambridge"); + +export const checked = [volume, page, chapter, flags, clean, tokens, start, packed, slug, fused, kind, notes]; diff --git a/crates/bindings/python/catena/__init__.py b/crates/bindings/python/catena/__init__.py index bdecad5..c72d8cb 100644 --- a/crates/bindings/python/catena/__init__.py +++ b/crates/bindings/python/catena/__init__.py @@ -19,6 +19,7 @@ from ._catena import ( pack_cues, parse_subtitles, scrub, + series_of, split_collection, ) @@ -31,5 +32,6 @@ __all__ = [ "pack_cues", "parse_subtitles", "scrub", + "series_of", "split_collection", ] diff --git a/crates/bindings/python/catena/__init__.pyi b/crates/bindings/python/catena/__init__.pyi index c3bd903..482c74c 100644 --- a/crates/bindings/python/catena/__init__.pyi +++ b/crates/bindings/python/catena/__init__.pyi @@ -30,3 +30,6 @@ def fuse_hits( limit: int | None = ..., ) -> list[dict[str, Any]]: ... def identify(name: str, data: bytes) -> dict[str, Any]: ... +def series_of( + title: str, author: str | None = ... +) -> dict[str, Any] | None: ... diff --git a/crates/bindings/src/capi.rs b/crates/bindings/src/capi.rs index c95f166..ee8e193 100644 --- a/crates/bindings/src/capi.rs +++ b/crates/bindings/src/capi.rs @@ -356,6 +356,35 @@ pub unsafe extern "C" fn catena_identify( give_json(&crate::identify(name, bytes)) } +/// The series a title belongs to. Returns a JSON proposal, or NULL when the +/// title says nothing about a series — which is most titles, and not an +/// error: `catena_last_error` is NULL in that case. +/// +/// # Safety +/// +/// `title` must be NUL-terminated UTF-8; `author` likewise, or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_series_of( + title: *const c_char, + author: *const c_char, +) -> *mut c_char { + clear_error(); + let Some(title) = (unsafe { borrow(title, "title") }) else { + return ptr::null_mut(); + }; + let author = match author.is_null() { + true => None, + false => match unsafe { borrow(author, "author") } { + Some(author) => Some(author), + None => return ptr::null_mut(), + }, + }; + match crate::series_of(title, author) { + Some(found) => give_json(&found), + None => ptr::null_mut(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -455,6 +484,18 @@ mod tests { assert!(chunks.contains("the mill stood at the northern end")); } + #[test] + fn a_title_with_no_series_is_null_and_not_an_error() { + let title = c"Being and Time"; + assert!(through(|| unsafe { catena_series_of(title.as_ptr(), ptr::null()) }).is_none()); + assert_eq!(error(), None, "no series is an answer, not a failure"); + + let series = c"The Philokalia Vol 3"; + let json = through(|| unsafe { catena_series_of(series.as_ptr(), ptr::null()) }) + .expect("a proposal"); + assert!(json.contains("\"volume\":3")); + } + #[test] fn the_abi_version_is_the_one_the_header_declares() { assert_eq!(catena_abi_version(), 1); diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index c730c5c..b299cd5 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -176,6 +176,11 @@ pub fn identify(name: &str, bytes: &[u8]) -> Doc { booksniff::sniff(&bytes, name) } +/// The series a title belongs to, and which volume of it this is. +pub fn series_of(title: &str, author: Option<&str>) -> Option { + seriesof::series_of(title, author) +} + #[cfg(test)] mod tests { use super::*; @@ -232,6 +237,14 @@ mod tests { assert_eq!(channel_named("lexical"), None); } + #[test] + fn a_series_comes_back_the_same_as_the_library_reads_it() { + let found = series_of("The Philokalia Vol 3", None).expect("a series"); + assert_eq!(found.series, "The Philokalia"); + assert_eq!(found.volume, Some(3)); + assert_eq!(series_of("Being and Time", None), None); + } + #[test] fn bytes_are_identified_the_same_way_a_file_is() { let pdf = b"%PDF-1.4\n"; diff --git a/crates/bindings/src/python.rs b/crates/bindings/src/python.rs index 9ec8307..259990b 100644 --- a/crates/bindings/src/python.rs +++ b/crates/bindings/src/python.rs @@ -192,6 +192,22 @@ fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult( + py: Python<'py>, + title: &str, + author: Option<&str>, +) -> PyResult>> { + match crate::series_of(title, author) { + Some(found) => Ok(Some(to_py(py, &found)?)), + None => Ok(None), + } +} + /// The module Python imports. #[pymodule] fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -204,5 +220,6 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(split_collection, module)?)?; module.add_function(wrap_pyfunction!(fuse_hits, module)?)?; module.add_function(wrap_pyfunction!(identify, module)?)?; + module.add_function(wrap_pyfunction!(series_of, module)?)?; Ok(()) } diff --git a/crates/bindings/src/wasm.rs b/crates/bindings/src/wasm.rs index 083ac14..228064d 100644 --- a/crates/bindings/src/wasm.rs +++ b/crates/bindings/src/wasm.rs @@ -70,6 +70,20 @@ export interface Hit { payload: unknown; } +/** What a title says about its series. */ +export interface SeriesProposal { + v: number; + title: string; + author?: string; + series: string; + /** Which volume, where the title said. Absent for membership without order. */ + volume?: number; + /** A title reconstructed from publisher shorthand, where one reliably can be. */ + proposed_title?: string; + /** "volume" or "publisher". */ + method: string; +} + /** What a file is. */ export interface Doc { v: number; @@ -208,6 +222,17 @@ pub fn identify(name: &str, data: &[u8]) -> Result { to_js(&crate::identify(name, data)) } +/// The series a title belongs to, and which volume of it this is. +/// +/// `undefined` when the title says nothing about a series. +#[wasm_bindgen(js_name = seriesOf, skip_typescript)] +pub fn series_of(title: &str, author: Option) -> Result { + match crate::series_of(title, author.as_deref()) { + Some(found) => to_js(&found), + None => Ok(JsValue::UNDEFINED), + } +} + /// Declarations for the functions whose return type the generator cannot /// infer, so a caller gets `Chunk[]` rather than `any`. #[wasm_bindgen(typescript_custom_section)] @@ -218,4 +243,5 @@ export function packCues(cues: Cue[], seconds?: number, max_tokens?: number): Ch export function splitCollection(markdown: string): Section[]; export function fuseHits(hits: Hit[], k?: number, weights?: Record, limit?: number): Hit[]; export function identify(name: string, data: Uint8Array): Doc; +export function seriesOf(title: string, author?: string): SeriesProposal | undefined; "#; diff --git a/crates/seriesof/Cargo.toml b/crates/seriesof/Cargo.toml new file mode 100644 index 0000000..b0b524c --- /dev/null +++ b/crates/seriesof/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "seriesof" +description = "The series a book belongs to, and which volume of it this is." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +fancy-regex.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +textkit.workspace = true + +[lints] +workspace = true diff --git a/crates/seriesof/README.md b/crates/seriesof/README.md new file mode 100644 index 0000000..a677973 --- /dev/null +++ b/crates/seriesof/README.md @@ -0,0 +1,161 @@ +# seriesof + +The series a book belongs to, and which volume of it this is. + +Part of [catena](../../README.md). + +## What it does + +Reads a series and a volume off a title, and says nothing about the books it +cannot place. + +A shelf groups by series and volume, and most catalogues have neither set on +most books. What that looks like from the reading chair: four records all +reading "A History of Philosophy" with nothing to say which volume any of them +is, and five volumes of one work sorting as five unrelated books. + +## When you would use it + +- **Before importing a shelf** into anything that browses by series. +- **Auditing a catalogue** that was built from filenames, where the series was + in the title all along and nobody read it. +- **Finding the volumes of a work you already have**, which is a query you + cannot ask until the series field exists. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena seriesof +``` + +## Example + +``` +$ seriesof catalogue.ndjson | jq -r '"\(.series)\t#\(.volume // "—")\t\(.method)"' +The Philokalia #3 volume +A History of Philosophy #4 volume +The Art of Computer Programming #3 volume +Cambridge Companions #— publisher +Very Short Introductions #— publisher +Blackwell Companions #— publisher +Raven Son #1 volume +``` + +The whole record, for a title that carries a volume: + +``` +$ seriesof catalogue.ndjson | head -1 +{"v":1,"title":"The Philokalia Vol 3","author":"Palmer","series":"The Philokalia","volume":3,"method":"volume"} +``` + +And for publisher shorthand, which proposes a title as well: + +``` +$ seriesof catalogue.ndjson | jq -c 'select(.method == "publisher") | {series, proposed_title}' | head -2 +{"series":"Cambridge Companions","proposed_title":"The Cambridge Companion to Aquinas"} +{"series":"Very Short Introductions","proposed_title":"Logic: A Very Short Introduction"} +``` + +Plain titles, for a directory rather than a catalogue: + +``` +$ ls *.epub | sed 's/\.epub$//' | seriesof --titles +{"v":1,"title":"The Philokalia Vol 3","series":"The Philokalia","volume":3,"method":"volume"} +``` + +Nothing to propose is an answer: + +``` +$ printf '{"title":"Being and Time"}\n' | seriesof +$ echo $? +1 +``` + +Applying only what you believe: + +```sh +seriesof catalogue.ndjson \ + | jq -r 'select(.volume) | "UPDATE book SET series = \(.series|@json), series_index = \(.volume) WHERE title = \(.title|@json);"' \ + | psql +``` + +## Options + +| Option | Effect | +|---|---| +| `--titles` | Read plain titles, one per line, rather than records | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Something was proposed | +| `1` | Nothing was | +| `64` | The command line was wrong | +| `65` | The input was not records | +| `74` | The file could not be read | + +## How it decides + +Two shapes are recoverable, and they are not the same kind of fact. + +### A volume marker is a position + +"The Philokalia Vol 3", "Collected Works, Volume 9", "(Raven Son Book 1)". +The unit word is required — `Vol`, `Volume`, `Book`, `Part`, `Pt`, `Tome`, +`Bk` — because a bare trailing number is far more often a year, an edition or +noise. Arabic and roman numerals both count, in any script, up to two hundred. + +The series is **what comes before the marker**. Whatever follows is that +volume's own subtitle: "The Art of Computer Programming, Volume 3: Sorting and +Searching" is volume 3 *of* the former, and gluing the halves back together +gives a title nobody published. + +Except where the convention puts the series inside a bracket — "The Song of +the Sirin (Raven Son Book 1)" is book one of *Raven Son*, not of "The Song of +the Sirin (Raven Son". + +A marker with nothing usable before it is left alone. "Vol. 2. PLATO, +EPICTETUS, MARCUS AURELIUS" is a volume of something whose name is not in the +field at all, and filing it under its own contents is worse than not filing +it. + +### Publisher shorthand is only membership + +A title of "Aquinas (Companion)" against an author of "Cambridge" means the +publisher landed in the author column and the collection name was flattened +into a parenthetical. The work is *The Cambridge Companion to Aquinas*. + +These get a series and **no number**, deliberately. Cambridge Companions are +not numbered, nobody knows what "#3" would mean, and cataloguing software +defaults a missing index to 1 — so filing them as a numbered series makes +every one display as volume one, asserting an order the data does not have. + +A title is only reconstructed where it reliably can be. Cambridge and +Routledge front theirs with the imprint; Blackwell does not — its Companions +to Philosophy are "A Companion to Nietzsche", while a few of its religion +titles genuinely are "The Blackwell Companion to …". The shorthand cannot tell +which, so Blackwell gets its series and no title claim. + +Recognised: Cambridge, Oxford, Blackwell, Wiley Blackwell, Routledge, SUNY, +Palgrave and Springer, across companions, concise companions, handbooks, +guidebooks, Very Short Introductions, Contemporary Debates, contemporary +introductions, anthologies and studies. + +## As a library + +```rust +let found = seriesof::series_of("The Philokalia Vol 3", None).expect("a series"); +assert_eq!(found.series, "The Philokalia"); +assert_eq!(found.volume, Some(3)); +``` + +`parse_volume` and `publisher_series` answer each half on its own, and +`roman_or_int` reads a numeral in any script. + +## See also + +- [`bibclean`](../bibclean/README.md) — the rest of what a catalogue record + needs before it is worth keeping. +- [`booksniff`](../booksniff/README.md) — what the file actually is. diff --git a/crates/seriesof/src/lib.rs b/crates/seriesof/src/lib.rs new file mode 100644 index 0000000..9652cd7 --- /dev/null +++ b/crates/seriesof/src/lib.rs @@ -0,0 +1,147 @@ +//! The series a book belongs to, and which volume of it this is. +//! +//! A shelf groups by series and volume, and most catalogues have neither set +//! on most books. The consequence is visible rather than theoretical: four +//! records all reading "A History of Philosophy" with nothing to say which +//! volume any of them is, and five volumes of one work sorting as five +//! unrelated books. +//! +//! Two shapes are recoverable from text the catalogue already holds, and they +//! are not the same kind of fact. +//! +//! A **volume marker** in the title — "The Philokalia Vol 3", "Collected +//! Works, Volume 9", "(Raven Son Book 1)" — is a *position*, and gives both a +//! series and a number. +//! +//! **Publisher shorthand** — a title of "Aquinas (Companion)" against an +//! author of "Cambridge" — is only *membership*. Cambridge Companions are not +//! numbered, so filing one under a numbered series makes every one of them +//! display as volume one, asserting an order the data does not have. Those +//! get a series and no number. + +pub mod publisher; +pub mod volume; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub use publisher::publisher_series; +pub use volume::{parse_volume, roman_or_int}; + +/// What a title turned out to say about its series. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Proposal { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// The title it was read from. + pub title: String, + /// The author it was read against, where one was given. + #[serde(skip_serializing_if = "Option::is_none")] + pub author: Option, + /// The series the book belongs to. + pub series: String, + /// Which volume, where the title said. Absent for a series that is + /// membership rather than position. + #[serde(skip_serializing_if = "Option::is_none")] + pub volume: Option, + /// A title reconstructed from the shorthand, where one reliably can be. + #[serde(skip_serializing_if = "Option::is_none")] + pub proposed_title: Option, + /// Which of the two shapes this was. + pub method: Method, +} + +/// How a series was arrived at. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Method { + /// A volume marker in the title: a position within a series. + Volume, + /// Publisher shorthand in a parenthetical: membership, not position. + Publisher, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Proposal { + fn version(&self) -> u32 { + self.v + } +} + +/// What a title and author say about the series, if anything. +/// +/// The volume marker is looked for first: it carries a number, and a number +/// is worth more than membership. +pub fn series_of(title: &str, author: Option<&str>) -> Option { + if let Some((series, volume)) = parse_volume(title) { + return Some(Proposal { + v: corpus_core::V, + title: title.to_string(), + author: author.map(str::to_string), + series, + volume: Some(volume), + proposed_title: None, + method: Method::Volume, + }); + } + + let (proposed_title, series) = publisher_series(title, author?)?; + Some(Proposal { + v: corpus_core::V, + title: title.to_string(), + author: author.map(str::to_string), + series, + volume: None, + proposed_title, + method: Method::Publisher, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_volume_marker_gives_a_series_and_a_number() { + let found = series_of("The Philokalia Vol 3", None).expect("a series"); + assert_eq!(found.series, "The Philokalia"); + assert_eq!(found.volume, Some(3)); + assert_eq!(found.method, Method::Volume); + } + + #[test] + fn publisher_shorthand_gives_membership_and_no_number() { + let found = series_of("Aquinas (Companion)", Some("Cambridge")).expect("a series"); + assert_eq!(found.series, "Cambridge Companions"); + assert_eq!(found.volume, None, "membership is not a position"); + assert_eq!( + found.proposed_title.as_deref(), + Some("The Cambridge Companion to Aquinas") + ); + } + + #[test] + fn a_volume_marker_wins_over_a_parenthetical() { + // Both could match; the one carrying a number is worth more. + let found = series_of("Aquinas (Companion) Vol 2", Some("Cambridge")).expect("a series"); + assert_eq!(found.method, Method::Volume); + assert_eq!(found.volume, Some(2)); + } + + #[test] + fn an_ordinary_title_proposes_nothing() { + assert_eq!(series_of("Being and Time", Some("Heidegger")), None); + assert_eq!(series_of("", None), None); + } + + #[test] + fn publisher_shorthand_needs_the_publisher() { + // The same title against anybody else is just a title. + assert_eq!(series_of("Aquinas (Companion)", Some("Kerr")), None); + assert_eq!(series_of("Aquinas (Companion)", None), None); + } +} diff --git a/crates/seriesof/src/main.rs b/crates/seriesof/src/main.rs new file mode 100644 index 0000000..4511539 --- /dev/null +++ b/crates/seriesof/src/main.rs @@ -0,0 +1,96 @@ +//! The series a book belongs to, and which volume of it this is. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::exit; + +/// Read a series and a volume off a title, one record per proposal. +/// +/// A shelf groups by series and volume, and most catalogues have neither. Four +/// records all reading "A History of Philosophy" with nothing to say which +/// volume any of them is, is what that looks like from the reading chair. +/// +/// Reads one JSON record per line with a `title` and optionally an `author`, +/// or plain titles with `--titles`. Says nothing about the books it cannot +/// place. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 something was proposed, 1 nothing was, 65 the input was not +records, 74 the file could not be read. + +Example: + + seriesof catalogue.ndjson | jq -r '\"\\(.series) #\\(.volume // \"-\")\"' + ls *.epub | sed 's/\\.epub$//' | seriesof --titles" +)] +struct Args { + /// Records to read. Reads stdin when absent. + path: Option, + + /// Read plain titles, one per line, rather than records. + #[arg(long)] + titles: bool, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("seriesof: {why}"); + return ExitCode::from(code); + } + }; + + let mut proposals = Vec::new(); + for (number, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let (title, author) = match args.titles { + true => (line.trim().to_string(), None), + false => match read_record(line) { + Ok(pair) => pair, + Err(why) => { + eprintln!("seriesof: line {}: {why}", number + 1); + return ExitCode::from(exit::DATA); + } + }, + }; + if let Some(proposal) = seriesof::series_of(&title, author.as_deref()) { + proposals.push(proposal); + } + } + + // Nothing to propose is an answer, and the usual one for a catalogue that + // has already been through this. + if proposals.is_empty() { + return ExitCode::from(exit::NO); + } + catena_cli::stream("seriesof", &proposals) +} + +/// The title and author out of one record. +fn read_record(line: &str) -> Result<(String, Option), String> { + let value: serde_json::Value = + serde_json::from_str(line).map_err(|e| format!("not a record: {e}"))?; + let title = value + .get("title") + .and_then(serde_json::Value::as_str) + .ok_or("no title")? + .to_string(); + let author = value + .get("author") + .and_then(serde_json::Value::as_str) + .map(str::to_string); + Ok((title, author)) +} diff --git a/crates/seriesof/src/publisher.rs b/crates/seriesof/src/publisher.rs new file mode 100644 index 0000000..e362cbc --- /dev/null +++ b/crates/seriesof/src/publisher.rs @@ -0,0 +1,265 @@ +//! Publisher-collection shorthand: "Aquinas (Companion)" against "Cambridge". +//! +//! The publishers and their collections are general reference data — a +//! Cambridge Companion is the same thing on every shelf in the world — of the +//! same kind as the publisher names elsewhere in this suite. + +use std::collections::HashMap; +use std::sync::LazyLock; + +use fancy_regex::Regex; + +/// A trailing parenthetical short enough to be a collection name. +static PAREN_KIND: LazyLock = + LazyLock::new(|| Regex::new(r"\(([^)]{3,40})\)\s*$").expect("a valid pattern")); + +/// A trailing year, parenthesised or not, that a filename picked up. +static TRAILING_YEAR: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:\s*[\[(]\s*(?:19|20)\d{2}\s*[\])]\s*|\s*[.,\-]\s*(?:19|20)\d{2}\s*)$") + .expect("a valid pattern") +}); + +/// A second trailing parenthetical, once the first has been taken as the kind. +static TRAILING_PAREN: LazyLock = + LazyLock::new(|| Regex::new(r"\s*[\[(][^\])]{0,40}[\])]\s*$").expect("a valid pattern")); + +/// The publishers this shorthand appears with, and the name their collections +/// are published under. +static PUBLISHERS: LazyLock> = LazyLock::new(|| { + [ + ("cambridge", "Cambridge"), + ("oxford", "Oxford"), + ("blackwell", "Blackwell"), + ("wiley blackwell", "Blackwell"), + ("routledge", "Routledge"), + ("suny", "SUNY"), + ("palgrave", "Palgrave"), + ("springer", "Springer"), + ] + .into_iter() + .collect() +}); + +/// What each kind of collection is called, and what its volumes are titled. +/// +/// The title is `None` where the real one is not derivable from the shorthand +/// — an "Anthology" or a "Studies" volume is not titled by any one pattern, +/// and guessing produces things nobody published. For those only the series +/// is proposed, which is the half that fixes browsing. +static KINDS: LazyLock, &'static str)>> = + LazyLock::new(|| { + [ + ("companion", (None, "{pub} Companions")), + ("companions", (None, "{pub} Companions")), + ("concise", (None, "{pub} Concise Companions")), + ("handbook", (None, "{pub} Handbooks")), + ( + "guidebook", + ( + Some("{pub} Philosophy Guidebook to {topic}"), + "{pub} Philosophy Guidebooks", + ), + ), + ( + "very short introduction", + ( + Some("{topic}: A Very Short Introduction"), + "Very Short Introductions", + ), + ), + ( + "contemporary debates", + ( + Some("Contemporary Debates in {topic}"), + "Contemporary Debates", + ), + ), + ( + "contemporary introductions", + (None, "{pub} Contemporary Introductions"), + ), + ("anthology", (None, "{pub} Anthologies")), + ("studies", (None, "{pub} Studies")), + ] + .into_iter() + .collect() + }); + +/// Titles that depend on who published them. +/// +/// The houses do not name their volumes alike, and getting it wrong renames a +/// book. Cambridge and Routledge front theirs with the imprint; Blackwell does +/// not — its Companions to Philosophy are "A Companion to Nietzsche", while a +/// few of its religion titles genuinely are "The Blackwell Companion to …". +/// That is unresolvable from the shorthand, so Blackwell gets its series and +/// no title claim. +static PUBLISHER_TITLES: LazyLock> = + LazyLock::new(|| { + [ + ( + ("Cambridge", "companion"), + "The Cambridge Companion to {topic}", + ), + ( + ("Cambridge", "companions"), + "The Cambridge Companion to {topic}", + ), + ( + ("Routledge", "companion"), + "The Routledge Companion to {topic}", + ), + ( + ("Routledge", "companions"), + "The Routledge Companion to {topic}", + ), + (("Oxford", "handbook"), "The Oxford Handbook of {topic}"), + ] + .into_iter() + .collect() + }); + +/// Reconstruct a title and a series from publisher shorthand. +/// +/// Gives `(title, series)`, where the title is `None` for the kinds whose +/// real titles do not follow one pattern. `None` altogether when the title +/// carries no recognised parenthetical, or the author is not a publisher this +/// shorthand shows up with. +pub fn publisher_series(title: &str, author: &str) -> Option<(Option, String)> { + let title = title.trim(); + // Lowercased and trimmed, but runs of space inside are left alone: the + // table is matched against exactly, and a doubled space in an author + // field is as likely to be two different names run together as a typo. + let author = author.trim().to_lowercase(); + if title.is_empty() { + return None; + } + let publisher = *PUBLISHERS.get(author.as_str())?; + + let found = PAREN_KIND.captures(title).ok().flatten()?; + let kind = collapse(&found.get(1)?.as_str().trim().to_lowercase()); + let (title_template, series_template) = *KINDS.get(kind.as_str())?; + + let trim = |text: &str| { + text.trim_matches(|c: char| " ,:;.-–—".contains(c)) + .to_string() + }; + let topic = trim(&title[..found.get(0)?.start()]); + let topic = trim(&TRAILING_YEAR.replace(&topic, "")); + let topic = trim(&TRAILING_PAREN.replace(&topic, "")); + if topic.chars().count() < 2 { + return None; + } + + // The per-publisher title first; the per-kind one only covers collections + // that name themselves the same way whoever publishes them. + let template = PUBLISHER_TITLES + .get(&(publisher, kind.as_str())) + .copied() + .or(title_template); + Some(( + template.map(|template| fill(template, publisher, &topic)), + fill(series_template, publisher, &topic), + )) +} + +/// A template with its two placeholders filled. +fn fill(template: &str, publisher: &str, topic: &str) -> String { + template + .replace("{pub}", publisher) + .replace("{topic}", topic) +} + +/// Runs of whitespace down to one space. +fn collapse(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_cambridge_companion_is_reconstructed_whole() { + assert_eq!( + publisher_series("Aquinas (Companion)", "Cambridge"), + Some(( + Some("The Cambridge Companion to Aquinas".to_string()), + "Cambridge Companions".to_string() + )) + ); + } + + #[test] + fn blackwell_gets_its_series_and_no_title_claim() { + // Its companions are "A Companion to Nietzsche", not "The Blackwell + // Companion to Nietzsche", and the shorthand cannot tell which. + assert_eq!( + publisher_series("Nietzsche (Companion)", "Blackwell"), + Some((None, "Blackwell Companions".to_string())) + ); + } + + #[test] + fn a_collection_that_names_itself_the_same_way_everywhere_needs_no_publisher() { + assert_eq!( + publisher_series("Logic (Very Short Introduction)", "Oxford"), + Some(( + Some("Logic: A Very Short Introduction".to_string()), + "Very Short Introductions".to_string() + )) + ); + } + + #[test] + fn a_trailing_year_is_not_part_of_the_topic() { + for title in [ + "Christian Ethics (2004) (Companion)", + "Christian Ethics.2009 (Companion)", + "Christian Ethics [2006] (Companion)", + ] { + let (proposed, _) = publisher_series(title, "Cambridge").expect("a series"); + assert_eq!( + proposed.as_deref(), + Some("The Cambridge Companion to Christian Ethics"), + "{title}" + ); + } + } + + #[test] + fn a_second_parenthetical_is_not_part_of_the_topic_either() { + let (proposed, _) = + publisher_series("British Drama [Contemporary] (Companion)", "Cambridge") + .expect("a series"); + assert_eq!( + proposed.as_deref(), + Some("The Cambridge Companion to British Drama") + ); + } + + #[test] + fn an_author_who_is_not_a_publisher_proposes_nothing() { + assert_eq!(publisher_series("Aquinas (Companion)", "Fergus Kerr"), None); + } + + #[test] + fn a_parenthetical_that_is_not_a_collection_proposes_nothing() { + assert_eq!( + publisher_series("Aquinas (Second Edition)", "Cambridge"), + None + ); + assert_eq!(publisher_series("Aquinas", "Cambridge"), None); + } + + #[test] + fn a_shorthand_with_no_topic_left_proposes_nothing() { + assert_eq!(publisher_series("(Companion)", "Cambridge"), None); + } + + #[test] + fn the_publisher_is_read_however_it_is_cased_or_padded() { + assert!(publisher_series("Ethics (Companion)", " WILEY BLACKWELL ").is_some()); + // But a doubled space inside is a different string, not a typo to fix. + assert!(publisher_series("Ethics (Companion)", "wiley blackwell").is_none()); + } +} diff --git a/crates/seriesof/src/volume.rs b/crates/seriesof/src/volume.rs new file mode 100644 index 0000000..ebc6240 --- /dev/null +++ b/crates/seriesof/src/volume.rs @@ -0,0 +1,243 @@ +//! A volume marker in a title, and the series it implies. + +use std::collections::HashMap; +use std::sync::LazyLock; + +use fancy_regex::Regex; + +/// The roman numerals a volume is ever written in. +/// +/// Twenty is where real collected editions stop; past that the numeral is far +/// more often a century or a chapter than a volume. +static ROMAN: LazyLock> = LazyLock::new(|| { + [ + ("i", 1), + ("ii", 2), + ("iii", 3), + ("iv", 4), + ("v", 5), + ("vi", 6), + ("vii", 7), + ("viii", 8), + ("ix", 9), + ("x", 10), + ("xi", 11), + ("xii", 12), + ("xiii", 13), + ("xiv", 14), + ("xv", 15), + ("xvi", 16), + ("xvii", 17), + ("xviii", 18), + ("xix", 19), + ("xx", 20), + ] + .into_iter() + .collect() +}); + +/// "Vol 3", "Volume IX", "Book 1", "Part 2", "Tome II". +/// +/// The unit word is required. A bare trailing number is far more often a +/// year, an edition, or noise than a volume. +static VOLUME: LazyLock = LazyLock::new(|| { + Regex::new(r"(?ix) [\s,:;.\-–—(\[]* \b(?:vol(?:ume)?s?|bk|book|part|pt|tome)\b [\s.:\#\-]* (?P\d{1,3}|[ivxlc]{1,7}) \b[\s)\]]*") + .expect("a valid pattern") +}); + +/// The largest volume number worth believing. +/// +/// Past two hundred it is a page, a year or a catalogue number that happens +/// to sit after the word "book". +const MAX_VOLUME: u32 = 200; + +/// Parse `9` or `IX` to nine. `None` for anything that is neither. +/// +/// Digits in any script, because the pattern that finds them matches the +/// Unicode class and a catalogue in Arabic writes its volumes in Arabic +/// figures. +pub fn roman_or_int(text: &str) -> Option { + let text = text.trim().to_lowercase(); + if text.is_empty() { + return None; + } + if text.chars().all(char::is_numeric) { + let number = textkit::decimal(&text); + return match (1..=MAX_VOLUME).contains(&number) { + true => Some(number), + false => None, + }; + } + ROMAN.get(text.as_str()).copied() +} + +/// Split a volume marker off a title, giving the series and the number. +/// +/// `None` when there is no marker, or when removing it leaves nothing that +/// could be a series name. +pub fn parse_volume(title: &str) -> Option<(String, u32)> { + let title = title.trim(); + if title.is_empty() { + return None; + } + + let found = VOLUME.captures(title).ok().flatten()?; + let whole = found.get(0)?; + let volume = roman_or_int(found.name("num")?.as_str())?; + + // What comes before the marker is the series. Whatever follows is this + // volume's own subtitle — "The Art of Computer Programming, Volume 3: + // Sorting and Searching" is volume 3 *of* the former, and gluing the + // halves back together gives a title nobody wrote. + let mut name = &title[..whole.start()]; + + // The trailing-parenthetical convention puts the series inside the + // brackets: "The Song of the Sirin (Raven Son Book 1)" is book one of + // "Raven Son", not of "The Song of the Sirin (Raven Son". + if let Some(open) = name.rfind(['(', '[']) { + let after = &name[open..]; + if !after.contains(')') && !after.contains(']') { + name = &name[open + 1..]; + } + } + + let name = collapse_spaces(name); + let name = name.trim_matches(|c: char| " ,:;.-–—([".contains(c)); + let name = name.trim_end_matches(|c: char| " ,:;.-–—".contains(c)); + + // Nothing before the marker means the title never named a series. "Vol. + // 2. PLATO, EPICTETUS, MARCUS AURELIUS" is a volume of something whose + // name is simply not in the field, and filing it under its own contents + // is worse than leaving it unfiled. + if name.chars().count() < 4 || !has_word(name) { + return None; + } + Some((name.to_string(), volume)) +} + +/// Runs of whitespace down to one space. +fn collapse_spaces(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut spaces = 0usize; + for c in text.chars() { + match c.is_whitespace() { + true => spaces += 1, + false => { + if spaces > 0 { + out.push(' '); + spaces = 0; + } + out.push(c); + } + } + } + if spaces > 0 { + out.push(' '); + } + out +} + +/// Whether there are three letters in a row anywhere, which is the least a +/// series name can be and still be a name. +fn has_word(text: &str) -> bool { + let mut run = 0; + for c in text.chars() { + run = match c.is_ascii_alphabetic() { + true => run + 1, + false => 0, + }; + if run >= 3 { + return true; + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arabic_and_roman_both_read_as_numbers() { + assert_eq!(roman_or_int("9"), Some(9)); + assert_eq!(roman_or_int("IX"), Some(9)); + assert_eq!(roman_or_int(" ix "), Some(9)); + assert_eq!(roman_or_int("xx"), Some(20)); + } + + #[test] + fn what_is_not_a_volume_number_is_not_one() { + assert_eq!(roman_or_int(""), None); + assert_eq!(roman_or_int("0"), None); + assert_eq!(roman_or_int("201"), None); + assert_eq!(roman_or_int("xxi"), None, "past twenty is not a volume"); + assert_eq!(roman_or_int("nine"), None); + } + + #[test] + fn a_volume_written_in_another_script_is_still_a_volume() { + assert_eq!(roman_or_int("٣"), Some(3)); + assert_eq!(roman_or_int("١٢"), Some(12)); + // A fraction is numeric and is not a digit run. + assert_eq!(roman_or_int("½"), None); + } + + #[test] + fn the_series_is_what_comes_before_the_marker() { + assert_eq!( + parse_volume("The Philokalia Vol 3"), + Some(("The Philokalia".to_string(), 3)) + ); + assert_eq!( + parse_volume("Collected Works of C.G. Jung, Volume 9"), + Some(("Collected Works of C.G. Jung".to_string(), 9)) + ); + } + + #[test] + fn a_volumes_own_subtitle_is_not_part_of_the_series_name() { + assert_eq!( + parse_volume("The Art of Computer Programming, Volume 3: Sorting and Searching"), + Some(("The Art of Computer Programming".to_string(), 3)) + ); + } + + #[test] + fn a_series_named_inside_a_parenthetical_is_the_series() { + assert_eq!( + parse_volume("The Song of the Sirin (Raven Son Book 1)"), + Some(("Raven Son".to_string(), 1)) + ); + } + + #[test] + fn a_title_that_never_named_a_series_is_left_unfiled() { + // A Harvard Classics volume, whose series name is not in the field. + assert_eq!( + parse_volume("Vol. 2. PLATO, EPICTETUS, MARCUS AURELIUS"), + None + ); + assert_eq!(parse_volume("Book 1"), None); + } + + #[test] + fn a_bare_number_is_not_a_volume() { + // Far more often a year, an edition, or noise. + assert_eq!(parse_volume("Being and Time 1927"), None); + assert_eq!(parse_volume("Ethics 2"), None); + } + + #[test] + fn nothing_in_nothing_out() { + assert_eq!(parse_volume(""), None); + assert_eq!(parse_volume(" "), None); + } + + #[test] + fn roman_volumes_read_the_same_as_arabic_ones() { + assert_eq!( + parse_volume("A History of Philosophy, Vol. IV"), + Some(("A History of Philosophy".to_string(), 4)) + ); + } +} diff --git a/parity/Cargo.toml b/parity/Cargo.toml index 1d2b958..43c95c4 100644 --- a/parity/Cargo.toml +++ b/parity/Cargo.toml @@ -24,6 +24,7 @@ mdscrub.workspace = true mdsplit.workspace = true avplan.workspace = true pdfgutter.workspace = true +seriesof.workspace = true pdftriage.workspace = true rrf.workspace = true corpus-core.workspace = true diff --git a/parity/cases/seriesof/cases.ndjson b/parity/cases/seriesof/cases.ndjson new file mode 100644 index 0000000..c5f9077 --- /dev/null +++ b/parity/cases/seriesof/cases.ndjson @@ -0,0 +1,80 @@ +{"name": "number-9-0", "kind": "number", "text": "9"} +{"name": "number-IX-1", "kind": "number", "text": "IX"} +{"name": "number-ix-2", "kind": "number", "text": "ix"} +{"name": "number-ix-3", "kind": "number", "text": " ix "} +{"name": "number-XX-4", "kind": "number", "text": "XX"} +{"name": "number-0-5", "kind": "number", "text": "0"} +{"name": "number-201-6", "kind": "number", "text": "201"} +{"name": "number-200-7", "kind": "number", "text": "200"} +{"name": "number-1-8", "kind": "number", "text": "1"} +{"name": "number-xxi-9", "kind": "number", "text": "xxi"} +{"name": "number-nine-10", "kind": "number", "text": "nine"} +{"name": "number-blank-11", "kind": "number", "text": ""} +{"name": "number-blank-12", "kind": "number", "text": " "} +{"name": "number-i-13", "kind": "number", "text": "i"} +{"name": "number-III-14", "kind": "number", "text": "III"} +{"name": "number-c-15", "kind": "number", "text": "c"} +{"name": "number-l-16", "kind": "number", "text": "l"} +{"name": "number-12.5-17", "kind": "number", "text": "12.5"} +{"name": "number-07-18", "kind": "number", "text": "07"} +{"name": "number-\u0663-19", "kind": "number", "text": "\u0663"} +{"name": "volume-20", "kind": "volume", "title": "The Philokalia Vol 3"} +{"name": "volume-21", "kind": "volume", "title": "Collected Works of C.G. Jung, Volume 9"} +{"name": "volume-22", "kind": "volume", "title": "The Art of Computer Programming, Volume 3: Sorting and Searching"} +{"name": "volume-23", "kind": "volume", "title": "The Song of the Sirin (Raven Son Book 1)"} +{"name": "volume-24", "kind": "volume", "title": "Vol. 2. PLATO, EPICTETUS, MARCUS AURELIUS"} +{"name": "volume-25", "kind": "volume", "title": "Book 1"} +{"name": "volume-26", "kind": "volume", "title": "Being and Time 1927"} +{"name": "volume-27", "kind": "volume", "title": "Ethics 2"} +{"name": "volume-28", "kind": "volume", "title": "A History of Philosophy, Vol. IV"} +{"name": "volume-29", "kind": "volume", "title": "A History of Philosophy Volume IV: Modern Philosophy"} +{"name": "volume-30", "kind": "volume", "title": "The Lord of the Rings Part 2"} +{"name": "volume-31", "kind": "volume", "title": "Summa Theologica, Tome II"} +{"name": "volume-32", "kind": "volume", "title": "Institutes Bk 4"} +{"name": "volume-33", "kind": "volume", "title": "Harry Potter (Book 7)"} +{"name": "volume-34", "kind": "volume", "title": "The Decline and Fall vol. xx"} +{"name": "volume-35", "kind": "volume", "title": "The Decline and Fall vol. xxi"} +{"name": "volume-36", "kind": "volume", "title": "Works pt 3"} +{"name": "volume-37", "kind": "volume", "title": "Volume 1"} +{"name": "volume-38", "kind": "volume", "title": ""} +{"name": "volume-39", "kind": "volume", "title": " "} +{"name": "volume-40", "kind": "volume", "title": "The Republic"} +{"name": "volume-41", "kind": "volume", "title": "Selected Works Vol 201"} +{"name": "volume-42", "kind": "volume", "title": "Selected Works Vol 0"} +{"name": "volume-43", "kind": "volume", "title": "abc Vol 1"} +{"name": "volume-44", "kind": "volume", "title": "abcd Vol 1"} +{"name": "volume-45", "kind": "volume", "title": "(Raven Son Book 12)"} +{"name": "volume-46", "kind": "volume", "title": "Sirin [Raven Son Book 3]"} +{"name": "volume-47", "kind": "volume", "title": "Title (Series Book 2) extra"} +{"name": "volume-48", "kind": "volume", "title": "Jung \u2014 Volume 9"} +{"name": "volume-49", "kind": "volume", "title": "Jung \u2013 Volume 9"} +{"name": "volume-50", "kind": "volume", "title": "Nicomachean Ethics, Books VIII and IX"} +{"name": "volume-51", "kind": "volume", "title": "The Cambridge History, Volume II: The Middle Ages"} +{"name": "publisher-52", "kind": "publisher", "title": "Aquinas (Companion)", "author": "Cambridge"} +{"name": "publisher-53", "kind": "publisher", "title": "Nietzsche (Companion)", "author": "Blackwell"} +{"name": "publisher-54", "kind": "publisher", "title": "Logic (Very Short Introduction)", "author": "Oxford"} +{"name": "publisher-55", "kind": "publisher", "title": "Ethics (Handbook)", "author": "Oxford"} +{"name": "publisher-56", "kind": "publisher", "title": "Ethics (Companion)", "author": "Routledge"} +{"name": "publisher-57", "kind": "publisher", "title": "Christian Ethics (2004) (Companion)", "author": "Cambridge"} +{"name": "publisher-58", "kind": "publisher", "title": "Christian Ethics.2009 (Companion)", "author": "Cambridge"} +{"name": "publisher-59", "kind": "publisher", "title": "Christian Ethics [2006] (Companion)", "author": "Cambridge"} +{"name": "publisher-60", "kind": "publisher", "title": "British Drama [Contemporary] (Companion)", "author": "Cambridge"} +{"name": "publisher-61", "kind": "publisher", "title": "Aquinas (Second Edition)", "author": "Cambridge"} +{"name": "publisher-62", "kind": "publisher", "title": "Aquinas", "author": "Cambridge"} +{"name": "publisher-63", "kind": "publisher", "title": "Aquinas (Companion)", "author": "Fergus Kerr"} +{"name": "publisher-64", "kind": "publisher", "title": "Aquinas (Companion)", "author": null} +{"name": "publisher-65", "kind": "publisher", "title": "(Companion)", "author": "Cambridge"} +{"name": "publisher-66", "kind": "publisher", "title": "Ethics (Companion)", "author": " WILEY BLACKWELL "} +{"name": "publisher-67", "kind": "publisher", "title": "Ethics (Companions)", "author": "Cambridge"} +{"name": "publisher-68", "kind": "publisher", "title": "Kant (Guidebook)", "author": "Routledge"} +{"name": "publisher-69", "kind": "publisher", "title": "Mind (Contemporary Debates)", "author": "Blackwell"} +{"name": "publisher-70", "kind": "publisher", "title": "Philosophy (Anthology)", "author": "Blackwell"} +{"name": "publisher-71", "kind": "publisher", "title": "Theology (Studies)", "author": "SUNY"} +{"name": "publisher-72", "kind": "publisher", "title": "Hegel (Concise)", "author": "Blackwell"} +{"name": "publisher-73", "kind": "publisher", "title": "Ethics (Contemporary Introductions)", "author": "Routledge"} +{"name": "publisher-74", "kind": "publisher", "title": "X (Companion)", "author": "Cambridge"} +{"name": "publisher-75", "kind": "publisher", "title": "Ethics (companion)", "author": "cambridge"} +{"name": "publisher-76", "kind": "publisher", "title": "Ethics ( Companion )", "author": "Cambridge"} +{"name": "publisher-77", "kind": "publisher", "title": "Ethics (Handbook)", "author": "Palgrave"} +{"name": "publisher-78", "kind": "publisher", "title": "Ethics (Handbook)", "author": "Springer"} +{"name": "publisher-79", "kind": "publisher", "title": "", "author": "Cambridge"} diff --git a/parity/golden/seriesof/cases.ndjson b/parity/golden/seriesof/cases.ndjson new file mode 100644 index 0000000..43a97d7 --- /dev/null +++ b/parity/golden/seriesof/cases.ndjson @@ -0,0 +1,80 @@ +{"kind": "number", "name": "number-9-0", "number": 9} +{"kind": "number", "name": "number-IX-1", "number": 9} +{"kind": "number", "name": "number-ix-2", "number": 9} +{"kind": "number", "name": "number-ix-3", "number": 9} +{"kind": "number", "name": "number-XX-4", "number": 20} +{"kind": "number", "name": "number-0-5", "number": null} +{"kind": "number", "name": "number-201-6", "number": null} +{"kind": "number", "name": "number-200-7", "number": 200} +{"kind": "number", "name": "number-1-8", "number": 1} +{"kind": "number", "name": "number-xxi-9", "number": null} +{"kind": "number", "name": "number-nine-10", "number": null} +{"kind": "number", "name": "number-blank-11", "number": null} +{"kind": "number", "name": "number-blank-12", "number": null} +{"kind": "number", "name": "number-i-13", "number": 1} +{"kind": "number", "name": "number-III-14", "number": 3} +{"kind": "number", "name": "number-c-15", "number": null} +{"kind": "number", "name": "number-l-16", "number": null} +{"kind": "number", "name": "number-12.5-17", "number": null} +{"kind": "number", "name": "number-07-18", "number": 7} +{"kind": "number", "name": "number-\u0663-19", "number": 3} +{"kind": "volume", "name": "volume-20", "series": "The Philokalia", "volume": 3} +{"kind": "volume", "name": "volume-21", "series": "Collected Works of C.G. Jung", "volume": 9} +{"kind": "volume", "name": "volume-22", "series": "The Art of Computer Programming", "volume": 3} +{"kind": "volume", "name": "volume-23", "series": "Raven Son", "volume": 1} +{"kind": "volume", "name": "volume-24", "series": null, "volume": null} +{"kind": "volume", "name": "volume-25", "series": null, "volume": null} +{"kind": "volume", "name": "volume-26", "series": null, "volume": null} +{"kind": "volume", "name": "volume-27", "series": null, "volume": null} +{"kind": "volume", "name": "volume-28", "series": "A History of Philosophy", "volume": 4} +{"kind": "volume", "name": "volume-29", "series": "A History of Philosophy", "volume": 4} +{"kind": "volume", "name": "volume-30", "series": "The Lord of the Rings", "volume": 2} +{"kind": "volume", "name": "volume-31", "series": "Summa Theologica", "volume": 2} +{"kind": "volume", "name": "volume-32", "series": "Institutes", "volume": 4} +{"kind": "volume", "name": "volume-33", "series": "Harry Potter", "volume": 7} +{"kind": "volume", "name": "volume-34", "series": "The Decline and Fall", "volume": 20} +{"kind": "volume", "name": "volume-35", "series": null, "volume": null} +{"kind": "volume", "name": "volume-36", "series": "Works", "volume": 3} +{"kind": "volume", "name": "volume-37", "series": null, "volume": null} +{"kind": "volume", "name": "volume-38", "series": null, "volume": null} +{"kind": "volume", "name": "volume-39", "series": null, "volume": null} +{"kind": "volume", "name": "volume-40", "series": null, "volume": null} +{"kind": "volume", "name": "volume-41", "series": null, "volume": null} +{"kind": "volume", "name": "volume-42", "series": null, "volume": null} +{"kind": "volume", "name": "volume-43", "series": null, "volume": null} +{"kind": "volume", "name": "volume-44", "series": "abcd", "volume": 1} +{"kind": "volume", "name": "volume-45", "series": "Raven Son", "volume": 12} +{"kind": "volume", "name": "volume-46", "series": "Raven Son", "volume": 3} +{"kind": "volume", "name": "volume-47", "series": "Series", "volume": 2} +{"kind": "volume", "name": "volume-48", "series": "Jung", "volume": 9} +{"kind": "volume", "name": "volume-49", "series": "Jung", "volume": 9} +{"kind": "volume", "name": "volume-50", "series": null, "volume": null} +{"kind": "volume", "name": "volume-51", "series": "The Cambridge History", "volume": 2} +{"kind": "publisher", "name": "publisher-52", "proposed_title": "The Cambridge Companion to Aquinas", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-53", "proposed_title": null, "series": "Blackwell Companions"} +{"kind": "publisher", "name": "publisher-54", "proposed_title": "Logic: A Very Short Introduction", "series": "Very Short Introductions"} +{"kind": "publisher", "name": "publisher-55", "proposed_title": "The Oxford Handbook of Ethics", "series": "Oxford Handbooks"} +{"kind": "publisher", "name": "publisher-56", "proposed_title": "The Routledge Companion to Ethics", "series": "Routledge Companions"} +{"kind": "publisher", "name": "publisher-57", "proposed_title": "The Cambridge Companion to Christian Ethics", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-58", "proposed_title": "The Cambridge Companion to Christian Ethics", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-59", "proposed_title": "The Cambridge Companion to Christian Ethics", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-60", "proposed_title": "The Cambridge Companion to British Drama", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-61", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-62", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-63", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-64", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-65", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-66", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-67", "proposed_title": "The Cambridge Companion to Ethics", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-68", "proposed_title": "Routledge Philosophy Guidebook to Kant", "series": "Routledge Philosophy Guidebooks"} +{"kind": "publisher", "name": "publisher-69", "proposed_title": "Contemporary Debates in Mind", "series": "Contemporary Debates"} +{"kind": "publisher", "name": "publisher-70", "proposed_title": null, "series": "Blackwell Anthologies"} +{"kind": "publisher", "name": "publisher-71", "proposed_title": null, "series": "SUNY Studies"} +{"kind": "publisher", "name": "publisher-72", "proposed_title": null, "series": "Blackwell Concise Companions"} +{"kind": "publisher", "name": "publisher-73", "proposed_title": null, "series": "Routledge Contemporary Introductions"} +{"kind": "publisher", "name": "publisher-74", "proposed_title": null, "series": null} +{"kind": "publisher", "name": "publisher-75", "proposed_title": "The Cambridge Companion to Ethics", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-76", "proposed_title": "The Cambridge Companion to Ethics", "series": "Cambridge Companions"} +{"kind": "publisher", "name": "publisher-77", "proposed_title": null, "series": "Palgrave Handbooks"} +{"kind": "publisher", "name": "publisher-78", "proposed_title": null, "series": "Springer Handbooks"} +{"kind": "publisher", "name": "publisher-79", "proposed_title": null, "series": null} diff --git a/parity/oracle/series.py b/parity/oracle/series.py new file mode 100644 index 0000000..e5ba3d7 --- /dev/null +++ b/parity/oracle/series.py @@ -0,0 +1,44 @@ +"""Read a series and a volume off a title the way the book pipeline did. + +The functions the module exports are imported rather than copied — it has no +import-time side effects, unlike the modules the other oracles copy from. + + python3 oracle/series.py < cases.ndjson + +One case per line: {name, title, author}. Writes one JSON answer per line. +""" + +from __future__ import annotations + +import json +import sys + +from book_pipeline.series import ( # noqa: E402 + parse_publisher_series, + parse_volume, + roman_or_int, +) + + +def answer(case: dict) -> dict: + kind = case["kind"] + if kind == "number": + return {"number": roman_or_int(case["text"])} + if kind == "volume": + found = parse_volume(case["title"]) + return {"series": found[0] if found else None, + "volume": found[1] if found else None} + if kind == "publisher": + found = parse_publisher_series(case["title"], case.get("author")) + return {"proposed_title": found[0] if found else None, + "series": found[1] if found else None} + raise SystemExit(f"no such case: {kind}") + + +for line in sys.stdin: + if not line.strip(): + continue + case = json.loads(line) + out = {"name": case["name"], "kind": case["kind"]} + out.update(answer(case)) + sys.stdout.write(json.dumps(out, sort_keys=True) + "\n") diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index b01aca9..202cedd 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 17] = [ + let jobs: [Job; 18] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -37,6 +37,7 @@ fn main() -> ExitCode { ("mdsplit", mdsplit), ("avplan", avplan), ("pdfgutter", pdfgutter), + ("seriesof", seriesof), ]; for (name, job) in jobs { @@ -219,6 +220,18 @@ fn cueseg() -> Written { Ok(written) } +fn seriesof() -> Written { + let dir = parity::golden("seriesof"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/seriesof/cases.ndjson"); + let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let answers = parity::run_oracle("series.py", &text)?; + let path = dir.join("cases.ndjson"); + std::fs::write(&path, answers).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn pdfgutter() -> Written { let dir = parity::golden("pdfgutter"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/seriesof.rs b/parity/tests/seriesof.rs new file mode 100644 index 0000000..8c2bff5 --- /dev/null +++ b/parity/tests/seriesof.rs @@ -0,0 +1,124 @@ +//! Every series and volume read off a title must match the Python exactly. +//! +//! The cost of getting it wrong is a shelf that looks right and is not: a +//! volume filed under the wrong series disappears from the work it belongs +//! to, and a title reconstructed from the wrong template renames a book to +//! something nobody published. + +use serde_json::Value; + +fn records(text: &str) -> Vec { + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn cases() -> Vec { + let text = std::fs::read_to_string(parity::dir().join("cases/seriesof/cases.ndjson")) + .expect("a readable case file"); + records(&text) +} + +/// A string field, where `null` and absent mean the same thing. +fn text(value: &Value) -> Option<&str> { + value.as_str() +} + +#[test] +fn every_series_matches_python() { + let golden = records(&parity::read_golden_text("seriesof/cases.ndjson")); + let cases = cases(); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + let mut wrong: Vec = Vec::new(); + for (case, theirs) in cases.iter().zip(&golden) { + let name = case["name"].as_str().expect("a name"); + let kind = case["kind"].as_str().expect("a kind"); + assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order"); + + let mut note = |field: &str, ours: String, theirs: &Value| { + wrong.push(format!("{name}.{field}: python {theirs}, rust {ours}")); + }; + + match kind { + "number" => { + let ours = seriesof::roman_or_int(case["text"].as_str().expect("text")); + let theirs_number = theirs["number"].as_u64().map(|n| n as u32); + if ours != theirs_number { + note("number", format!("{ours:?}"), &theirs["number"]); + } + } + "volume" => { + let found = seriesof::parse_volume(case["title"].as_str().expect("a title")); + let (series, volume) = match &found { + Some((series, volume)) => (Some(series.as_str()), Some(*volume)), + None => (None, None), + }; + if series != text(&theirs["series"]) { + note("series", format!("{series:?}"), &theirs["series"]); + } + let theirs_volume = theirs["volume"].as_u64().map(|n| n as u32); + if volume != theirs_volume { + note("volume", format!("{volume:?}"), &theirs["volume"]); + } + } + "publisher" => { + let author = case["author"].as_str(); + let found = author.and_then(|author| { + seriesof::publisher_series(case["title"].as_str().expect("a title"), author) + }); + let (proposed, series) = match &found { + Some((proposed, series)) => (proposed.as_deref(), Some(series.as_str())), + None => (None, None), + }; + if proposed != text(&theirs["proposed_title"]) { + note( + "proposed_title", + format!("{proposed:?}"), + &theirs["proposed_title"], + ); + } + if series != text(&theirs["series"]) { + note("series", format!("{series:?}"), &theirs["series"]); + } + } + other => panic!("nothing answers a {other} case"), + } + } + + assert!( + wrong.is_empty(), + "{} answers differ:\n{}", + wrong.len(), + wrong + .iter() + .take(20) + .map(|line| line.chars().take(200).collect::()) + .collect::>() + .join("\n") + ); +} + +/// All three questions have to be asked, or the file proves less than it +/// looks like it does. +#[test] +fn every_kind_of_case_is_present() { + let cases = cases(); + for kind in ["number", "volume", "publisher"] { + assert!( + cases.iter().any(|case| case["kind"] == kind), + "no {kind} case" + ); + } +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/seriesof/cases.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("series.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("seriesof/cases.ndjson")); +}