diff --git a/crates/colophon/README.md b/crates/colophon/README.md index 83d6830..3c80213 100644 --- a/crates/colophon/README.md +++ b/crates/colophon/README.md @@ -21,6 +21,38 @@ Every field names the page it was read off. Nothing is inferred: a scan that begins at the preface printed no title page, and the record of that is an empty one with every field listed in `unsettled`. +## A shelf of your own + +A cataloguing block states subjects the way a national library does: + +```text +Political science – Early works to 1800 +English literature—Middle English, 1100–1500—History and criticism +``` + +Nobody browses by those. `--shelf` takes the closed list you actually file by +and returns the headings from it that the book's own subjects hit: + +```sh +colophon --shelf shelf.txt book.md +``` + +```text +# one heading per line; `= terms` gives the words that match it, +# and a heading with no terms matches its own name. +Byzantine History = byzantium, byzantine, constantinople +Political Science = political science, politics +Ethics +``` + +The longest term wins, so `Ancient Philosophy` beats `Philosophy` and a period +of thought does not collapse into its parent. Matching is on whole words after +punctuation is folded away, because a block separates with em dashes, en +dashes, full stops and hyphens, and OCR turns any of those into any other. + +Headings that match nothing come back under `unshelved`, kept as printed. A +subject quietly dropped reads exactly like a book that printed none. + ## When you would use it - **Cataloguing scans and secondhand files**, where the embedded metadata is diff --git a/crates/colophon/src/lib.rs b/crates/colophon/src/lib.rs index ec21eb7..68da040 100644 --- a/crates/colophon/src/lib.rs +++ b/crates/colophon/src/lib.rs @@ -38,6 +38,7 @@ pub mod cip; pub mod imprint; pub mod page; pub mod prompt; +pub mod shelf; pub mod title; use std::collections::BTreeMap; @@ -109,8 +110,18 @@ pub struct Printed { pub place: Option, /// The year of publication, preferring first publication. pub year: Option, - /// The subject headings the cataloguing block filed it under. + /// The subject headings the cataloguing block filed it under, as printed. pub subjects: Vec, + /// Those headings filed onto a caller's own shelf, when one was given. + /// + /// Empty without `--shelf`, and empty too when the book's subjects match + /// nothing on it — which is an answer, and the reason `unshelved` exists. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub shelved: Vec, + /// Printed headings that matched nothing on the shelf, kept verbatim so a + /// subject is never silently dropped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub unshelved: Vec, /// Which page each field was read off. pub read_from: BTreeMap, /// The fields a catalogue wants that this book did not print. @@ -197,6 +208,24 @@ fn squash(s: &str) -> String { .collect() } +/// Read a book's front matter, and file its subjects onto a shelf. +/// +/// `budget` is how many pages in to read; [`PAGES`] is the sensible default. +/// Passing a [`Shelf`](shelf::Shelf) fills `shelved` and `unshelved`; without +/// one, `subjects` is still read and both stay empty. +pub fn read_onto(text: &str, budget: usize, shelf: Option<&shelf::Shelf>) -> Printed { + let mut out = read(text, budget); + if let Some(shelf) = shelf.filter(|s| !s.is_empty()) { + let filed = shelf.file(&out.subjects); + if !filed.on.is_empty() { + out.note("shelved", Source::Cip); + } + out.shelved = filed.on; + out.unshelved = filed.unfiled; + } + out +} + /// Read a book's front matter. /// /// `budget` is how many pages in to read; [`PAGES`] is the sensible default. @@ -214,6 +243,8 @@ pub fn read(text: &str, budget: usize) -> Printed { place: None, year: None, subjects: Vec::new(), + shelved: Vec::new(), + unshelved: Vec::new(), read_from: BTreeMap::new(), unsettled: Vec::new(), pages_read: pages.len(), diff --git a/crates/colophon/src/main.rs b/crates/colophon/src/main.rs index cafb12d..cb4cae6 100644 --- a/crates/colophon/src/main.rs +++ b/crates/colophon/src/main.rs @@ -53,6 +53,16 @@ struct Args { /// How many pages in from the front to read. #[arg(long, value_name = "COUNT", default_value_t = colophon::PAGES)] pages: usize, + + /// A closed list of headings to file the book's printed subjects under. + /// + /// One heading per line, optionally `Heading = term, term` to give the + /// words that should match it; a heading with no terms matches its own + /// name. A cataloguing block states subjects in Library of Congress form, + /// which is no use as a shelf, so this says what the shelf is. What + /// matches nothing comes back under `unshelved` rather than being dropped. + #[arg(long, value_name = "FILE")] + shelf: Option, } fn main() -> ExitCode { @@ -73,6 +83,26 @@ fn main() -> ExitCode { return ExitCode::from(exit::USAGE); } + // Read the shelf before any book, so a bad path fails at once rather than + // after the whole library has been read with no shelf applied. + let shelf = match args.shelf.as_deref() { + None => None, + Some(path) => match std::fs::read_to_string(path) { + Ok(text) => { + let s = colophon::shelf::Shelf::parse(&text); + if s.is_empty() { + eprintln!("colophon: {} lists no headings", path.display()); + return ExitCode::from(exit::USAGE); + } + Some(s) + } + Err(e) => { + eprintln!("colophon: {}: {e}", path.display()); + return ExitCode::from(exit::USAGE); + } + }, + }; + let inputs: Vec> = if args.paths.is_empty() { vec![None] } else { @@ -89,7 +119,7 @@ fn main() -> ExitCode { return ExitCode::from(code); } }; - let mut record = colophon::read(&text, args.pages); + let mut record = colophon::read_onto(&text, args.pages, shelf.as_ref()); record.path = path.as_deref().map(|p| p.display().to_string()); records.push(record); last_text = text; diff --git a/crates/colophon/src/shelf.rs b/crates/colophon/src/shelf.rs new file mode 100644 index 0000000..0a77fd0 --- /dev/null +++ b/crates/colophon/src/shelf.rs @@ -0,0 +1,264 @@ +//! Filing a book's printed subject headings onto a shelf someone actually has. +//! +//! A cataloguing block states subjects in Library of Congress form, which is +//! precise, deep, and no use as a shelf. Real blocks from real books read: +//! +//! ```text +//! Political science – Early works to 1800 +//! English literature—Middle English, 1100–1500—History and criticism +//! Methodology. 2.Anthropic principle +//! Hobbes, Thomas, 1588–1679. Leviathan +//! ``` +//! +//! Nobody browses by those. A library is browsed by a short closed list its +//! owner chose, and the job here is to get from the first to the second. +//! +//! The mapping is deliberately dumb: a heading is matched against the terms of +//! the shelf, longest first, and the first hit wins. It infers nothing, invents +//! nothing, and returns nothing at all rather than guess — a heading that +//! matches no term is reported unfiled, with its raw text kept, because a +//! subject silently dropped is indistinguishable from a book with no subjects. + +use std::collections::BTreeSet; + +/// A closed list of headings, with the words that file a book under each. +/// +/// Parsed from a plain text file: one heading per line, optionally followed by +/// `=` and the comma-separated terms that should match it. A heading with no +/// terms matches its own name. +/// +/// ```text +/// Byzantine History = byzantium, byzantine, constantinople, eastern roman +/// Patristics = church fathers, patristic, patrology +/// Ethics +/// ``` +#[derive(Debug, Clone, Default)] +pub struct Shelf { + /// (term, heading), longest term first so the most specific wins. + terms: Vec<(String, String)>, + headings: BTreeSet, +} + +impl Shelf { + /// Read a shelf from the list format above. Blank lines and `#` comments + /// are skipped. + pub fn parse(text: &str) -> Self { + let mut terms: Vec<(String, String)> = Vec::new(); + let mut headings = BTreeSet::new(); + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (heading, rest) = match line.split_once('=') { + Some((h, r)) => (h.trim(), r.trim()), + None => (line, ""), + }; + if heading.is_empty() { + continue; + } + headings.insert(heading.to_string()); + terms.push((fold(heading), heading.to_string())); + for term in rest.split(',') { + let term = fold(term); + if !term.is_empty() { + terms.push((term, heading.to_string())); + } + } + } + // Longest first: "ancient philosophy" must beat "philosophy", or every + // period of thought files under the parent and the shelf does no work. + terms.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0))); + terms.dedup_by(|a, b| a.0 == b.0); + Self { terms, headings } + } + + /// Whether the shelf lists no headings, in which case it files nothing. + pub fn is_empty(&self) -> bool { + self.terms.is_empty() + } + + /// Whether a name is one of the shelf's own headings. + pub fn has(&self, heading: &str) -> bool { + self.headings.contains(heading) + } + + /// The heading a single raw subject files under, if any. + pub fn file_one(&self, raw: &str) -> Option<&str> { + let hay = fold(raw); + if hay.is_empty() { + return None; + } + self.terms + .iter() + .find(|(term, _)| contains_word(&hay, term)) + .map(|(_, heading)| heading.as_str()) + } + + /// File a book's raw headings, in the shelf's own order, without repeats. + /// + /// Returns the headings that matched and the raw subjects that did not, so + /// a caller can tell "this book is not about anything on the shelf" from + /// "this book printed no subjects". + pub fn file(&self, raw: &[String]) -> Filed { + let mut on = Vec::new(); + let mut unfiled = Vec::new(); + for subject in raw { + match self.file_one(subject) { + Some(h) => { + if !on.iter().any(|x: &String| x == h) { + on.push(h.to_string()); + } + } + None => unfiled.push(subject.clone()), + } + } + Filed { on, unfiled } + } +} + +/// What a shelf made of a book's printed subjects. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Filed { + /// Headings from the shelf, in the order the book's subjects hit them. + pub on: Vec, + /// Subjects that matched no term, kept verbatim. + pub unfiled: Vec, +} + +/// Lowercase, and reduce every run of non-alphanumerics to one space. +/// +/// Cataloguing blocks separate with em dashes, en dashes, full stops and plain +/// hyphens, sometimes three of them in one heading, and OCR turns any of those +/// into any other. None of that should decide whether a book gets shelved. +fn fold(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut space = true; // no leading space + for c in s.chars() { + if c.is_alphanumeric() { + for l in c.to_lowercase() { + out.push(l); + } + space = false; + } else if !space { + out.push(' '); + space = true; + } + } + while out.ends_with(' ') { + out.pop(); + } + out +} + +/// Whether `needle` appears in `hay` on whole-word boundaries. +/// +/// Substring matching would file "Islam" under any book mentioning "Islamic" +/// — which is right — but would also file "Art" under "Cartography", which is +/// not. Both strings are already folded, so the boundary is a space. +fn contains_word(hay: &str, needle: &str) -> bool { + if needle.is_empty() || needle.len() > hay.len() { + return false; + } + hay.match_indices(needle).any(|(i, _)| { + let before = i == 0 || hay.as_bytes()[i - 1] == b' '; + let end = i + needle.len(); + let after = end == hay.len() || hay.as_bytes()[end] == b' '; + before && after + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn shelf() -> Shelf { + Shelf::parse( + "# a shelf\n\ + Philosophy = philosophy, philosophical\n\ + Ancient Philosophy = ancient philosophy, plato, aristotle\n\ + Political Science = political science, politics\n\ + Theology = theology, christianity\n\ + Byzantine History = byzantium, byzantine, constantinople\n\ + Ethics\n", + ) + } + + #[test] + fn a_heading_with_no_terms_matches_its_own_name() { + assert_eq!(shelf().file_one("Ethics — early works"), Some("Ethics")); + } + + #[test] + fn the_most_specific_term_wins() { + // Both "ancient philosophy" and "philosophy" match; the longer one has + // to win or every period files under the parent. + assert_eq!( + shelf().file_one("Ancient philosophy—History"), + Some("Ancient Philosophy") + ); + } + + #[test] + fn real_cataloguing_headings_file() { + let s = shelf(); + for (raw, want) in [ + ("Political science – Early works to 1800", "Political Science"), + ("Political science — Philosophy", "Political Science"), + ("Byzantine Empire—History—527-1081", "Byzantine History"), + ("Christianity and culture", "Theology"), + ] { + assert_eq!(s.file_one(raw), Some(want), "{raw}"); + } + } + + #[test] + fn a_dash_is_a_dash_however_it_was_printed() { + // em, en, hyphen, and the double hyphen OCR leaves behind. + let s = shelf(); + for raw in [ + "Philosophy—Introductions", + "Philosophy – Introductions", + "Philosophy - Introductions", + "Philosophy -- Introductions", + "Philosophy. Introductions", + ] { + assert_eq!(s.file_one(raw), Some("Philosophy"), "{raw}"); + } + } + + #[test] + fn a_term_must_match_a_whole_word() { + // "politics" must not file a book on "geopolitics", and "art" must not + // file one on cartography. + let s = Shelf::parse("Art\nPolitical Science = politics\n"); + assert_eq!(s.file_one("Cartography—History"), None); + assert_eq!(s.file_one("Art—Byzantine"), Some("Art")); + } + + #[test] + fn what_matches_nothing_is_reported_rather_than_dropped() { + let filed = shelf().file(&[ + "Political science – Early works to 1800".into(), + "Selectivity (Psychology)".into(), + ]); + assert_eq!(filed.on, ["Political Science"]); + assert_eq!(filed.unfiled, ["Selectivity (Psychology)"]); + } + + #[test] + fn a_heading_is_not_repeated() { + let filed = shelf().file(&[ + "Philosophy—Ancient".into(), + "Philosophy—Modern".into(), + ]); + assert_eq!(filed.on, ["Philosophy"]); + } + + #[test] + fn an_empty_shelf_files_nothing_and_says_so() { + let s = Shelf::parse(" \n# nothing but a comment\n"); + assert!(s.is_empty()); + assert_eq!(s.file(&["Philosophy".into()]).unfiled, ["Philosophy"]); + } +}