From ba8ec817350aa2db11928a0dbac071957a42a6ba Mon Sep 17 00:00:00 2001 From: funkpower Date: Mon, 21 Sep 2026 09:05:43 -0400 Subject: [PATCH] colophon: the record a book prints in its own front matter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A catalogue's worst records are the ones whose file says nothing true about itself: a PDF titled `Untitled` by an author called `Oxford`, which is a publisher, from a filename that was never anything but a filename. What those books do have is pages. The title page carries the title as the publisher set it; the copyright page carries the imprint, the year, the ISBN, and in most English books since the 1970s the cataloguing block, which is a catalogue record a librarian already made, subject headings and all. So this reads the pages. Every field names the page it came from, and a book that printed nothing comes back empty with its fields in `unsettled` rather than filled with something plausible — an extraction that starts at chapter one has no front matter in it, and saying so is the answer. Two pages look like a title page and are not, and both are how a catalogue comes to name the wrong book. A series advertisement lists other titles in the shape a title page uses, over as many as four pages with only the first announcing itself. A translation's cataloguing block prints the original title in brackets above the real one. Both are found and refused. Three things the pages give that no file does: the subject headings a librarian assigned, an ISBN that exists only in print — which turns a book with no identifier into one `bibfetch` can look up — and a title in the case the publisher meant, because a half title prints unbroken what the title page breaks across two lines and sets in capitals. `--prompt` and `--merge` are citefind's: a question about exactly the fields the pages left unsettled, composed here and answered wherever the caller keeps a model, with the advertising taken out of what the model is shown. A field a model answered is marked `model`, so a record always says which of it was read and which was inferred. Measured against 600 books of the home corpus: the cataloguing block answered for 21% of them and agreed with the catalogue 89% of the time, most of the residue being the catalogue's error rather than this one's. Title pages answered for another 41%. The remaining 38% print nothing readable, and are what the prompt is for. --- Cargo.lock | 14 ++ Cargo.toml | 1 + README.md | 1 + crates/colophon/Cargo.toml | 21 ++ crates/colophon/README.md | 119 ++++++++++ crates/colophon/src/cip.rs | 274 ++++++++++++++++++++++ crates/colophon/src/imprint.rs | 104 +++++++++ crates/colophon/src/lib.rs | 331 +++++++++++++++++++++++++++ crates/colophon/src/main.rs | 137 +++++++++++ crates/colophon/src/page.rs | 278 ++++++++++++++++++++++ crates/colophon/src/prompt.rs | 194 ++++++++++++++++ crates/colophon/src/title.rs | 291 +++++++++++++++++++++++ crates/colophon/tests/frontmatter.rs | 299 ++++++++++++++++++++++++ 13 files changed, 2064 insertions(+) create mode 100644 crates/colophon/Cargo.toml create mode 100644 crates/colophon/README.md create mode 100644 crates/colophon/src/cip.rs create mode 100644 crates/colophon/src/imprint.rs create mode 100644 crates/colophon/src/lib.rs create mode 100644 crates/colophon/src/main.rs create mode 100644 crates/colophon/src/page.rs create mode 100644 crates/colophon/src/prompt.rs create mode 100644 crates/colophon/src/title.rs create mode 100644 crates/colophon/tests/frontmatter.rs diff --git a/Cargo.lock b/Cargo.lock index e79f47c..5923e37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -395,6 +395,20 @@ dependencies = [ "roff", ] +[[package]] +name = "colophon" +version = "0.1.0" +dependencies = [ + "bibclean", + "catena-cli", + "clap", + "corpus-core", + "regex", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "colorchoice" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index ffe0b52..2591b19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ langid = { path = "crates/langid", version = "0.1.0" } tocfind = { path = "crates/tocfind", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } citefind = { path = "crates/citefind", version = "0.1.0" } +colophon = { path = "crates/colophon", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } clap = { version = "4", features = ["derive"] } diff --git a/README.md b/README.md index 48359d7..c9cdee9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Each tool installs by name, so you can take one without taking the rest. |---|---| | [`booksniff`](crates/booksniff/README.md) | What a file actually is, whatever its name claims | | [`bookmeta`](crates/bookmeta/README.md) | The metadata a file already carries | +| [`colophon`](crates/colophon/README.md) | The record a book prints in its own front matter | | [`tocfind`](crates/tocfind/README.md) | A book's own table of contents | | [`langid`](crates/langid/README.md) | What language a document is in | | [`pdfgutter`](crates/pdfgutter/README.md) | Books scanned two pages to a sheet, split back into pages | diff --git a/crates/colophon/Cargo.toml b/crates/colophon/Cargo.toml new file mode 100644 index 0000000..0ccf590 --- /dev/null +++ b/crates/colophon/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "colophon" +description = "The record a book prints in its own front matter." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +bibclean.workspace = true +regex.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/colophon/README.md b/crates/colophon/README.md new file mode 100644 index 0000000..83d6830 --- /dev/null +++ b/crates/colophon/README.md @@ -0,0 +1,119 @@ +# colophon + +The record a book prints in its own front matter. + +Part of [catena](../../README.md). + +## What it does + +A book states what it is twice before the text starts. The title page carries +the title as the publisher set it and the name of whoever wrote it. The +copyright page carries the imprint, the year, the ISBN, and — in most +English-language books printed since the 1970s — the cataloguing block, which +is a catalogue record a librarian made from the manuscript, subject headings +and all. + +So this reads the pages rather than the file. A PDF whose embedded title is +`Untitled` and whose filename says `Oxford` opens on *The First World War: A +Very Short Introduction*, by Michael Howard, and that is what comes back. + +Every field names the page it was read off. Nothing is inferred: a scan that +begins at the preface printed no title page, and the record of that is an +empty one with every field listed in `unsettled`. + +## When you would use it + +- **Cataloguing scans and secondhand files**, where the embedded metadata is + whatever the last tool to touch the file happened to write. +- **Checking a catalogue against the books themselves.** A record that + disagrees with the book's own title page is a record worth looking at. +- **Recovering an ISBN** that only exists in print, which turns a book with + no identifier into one [`bibfetch`](../bibfetch/README.md) can look up. +- **Getting subject headings** that nobody had to invent, from the + cataloguing block of the books that print one. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena colophon +``` + +## Example + +``` +$ colophon first-world-war.md +{"v":1,"path":"first-world-war.md","title":"The First World War","subtitle":"A Very Short Introduction","authors":["Michael Howard"],"editors":[],"isbns":["9780192803535"],"publisher":"Oxford University Press","place":"Oxford","year":2002,"subjects":[],"read_from":{"authors":"title_page","isbns":"copyright_page","publisher":"copyright_page","subtitle":"title_page","title":"title_page","year":"copyright_page"},"unsettled":["subjects"],"pages_read":16} +``` + +A library at once, and the books that say nothing about themselves: + +```sh +colophon library/*.md | jq -r 'select(.title == null) | .path' +``` + +An ISBN that only exists in print, looked up: + +```sh +colophon book.md | jq -r '.isbns[]' | idnorm | bibfetch --polite you@example.org +``` + +## What it will not read + +Two pages in a book's front matter look like a title page and are not, and +both of them are how a catalogue ends up naming the wrong book. They are +found and refused. + +The **series advertisement** lists other titles in the same series in the +same shape a title page uses — a work in capitals, a person beside it — +sometimes for four pages, with only the first saying what it is. The +**uniform title** in a translation's cataloguing block is the original +title, printed in brackets above the real one as a filing heading. + +## Asking a model about the rest + +Roughly a third of books print nothing this can read: an extraction that +starts at chapter one has no front matter in it at all. `--prompt` writes a +question about exactly the fields left `unsettled`, and `--merge` folds the +answer back in. + +```sh +colophon --prompt book.md | ollama run qwen3:8b > reply.json +colophon --merge reply.json book.md +``` + +Nothing is sent from here: the question goes to stdout, and the model is +yours. What a model answers is marked `model` in `read_from`, so a record +always says which of its fields were read off a page and which were +answered. + +The pages the question shows have had the advertising taken out of them +first. Handing a model a series list is how a book becomes catalogued as +whichever of two hundred other titles the model's eye fell on. + +## Where this stops + +It reports what the book printed, not what is true. `THE HUMAN CONDITION` is +reported in the capitals it was set in, and straightening that is +[`bibclean`](../bibclean/README.md)'s job. What a file's own metadata claims +is [`bookmeta`](../bookmeta/README.md)'s. What a register says about the +ISBN is [`bibfetch`](../bibfetch/README.md)'s. Which series a book belongs to +is [`seriesof`](../seriesof/README.md)'s. + +## Options + +| Option | Effect | +|---|---| +| `--pages ` | How many pages in from the front to read. Default `16` | +| `--prompt` | Write the question to put to a model about what the pages left unsettled | +| `--merge ` | Fill in from a model's reply and write the completed record | +| `--quiet` | Say nothing; the exit code is the answer | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | The book printed something about itself | +| `1` | It printed nothing this could read | +| `65` | The input was not readable as text | +| `74` | The file could not be read | diff --git a/crates/colophon/src/cip.rs b/crates/colophon/src/cip.rs new file mode 100644 index 0000000..7260164 --- /dev/null +++ b/crates/colophon/src/cip.rs @@ -0,0 +1,274 @@ +//! The cataloguing-in-publication block, which is a catalogue record the +//! publisher printed inside the book. +//! +//! Since the 1970s most English-language publishers have printed the record +//! the Library of Congress made from the manuscript on the copyright page, +//! and since 2013 they have printed it in labelled fields. Either way it +//! names the author as a library files him, the title with its subtitle, the +//! ISBN, and the subject headings the book was classified under — which is +//! the one place in a book where its subjects are stated rather than +//! inferred. +//! +//! It is a record, so it is read as one. Nothing here guesses. + +use std::sync::LazyLock; + +use regex::Regex; + +/// What the block said. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Cip { + /// Title, without its subtitle. + pub title: Option, + /// Subtitle, where the title statement had one. + pub subtitle: Option, + /// Authors, in reading order rather than filing order. + pub authors: Vec, + /// Editors, where the work is an edited one. + pub editors: Vec, + /// ISBNs printed inside the block. + pub isbns: Vec, + /// Subject headings the book was classified under. + pub subjects: Vec, +} + +impl Cip { + fn is_empty(&self) -> bool { + self.title.is_none() && self.authors.is_empty() && self.subjects.is_empty() + } +} + +static HEAD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)catalog(?:u)?ing[-\s]in[-\s]publication(?:\s+data)?").expect("a valid pattern") +}); +/// `B105.C477D46 1991` / `126—dc20` — the call number, which ends the block. +static CALL_NUMBER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^([A-Z]{1,3}\d{1,4}(\.\w+)*\s+\d{4}|\d{3}(\.\d+)?\s*[—–-]{1,2}\s*dc\d{2})") + .expect("a valid pattern") +}); +/// A name as a library files it: `Howard, Michael, 1922- author.` +static FILED_NAME: LazyLock = LazyLock::new(|| { + Regex::new( + r"^(?P[A-Z][\p{L}'’\-]+(?:\s+[A-Z][\p{L}'’\-]+)?),\s+(?P[A-Z][\p{L}'’\-\.]*(?:\s+[A-Z][\p{L}'’\-\.]*){0,3})\ +(?:,\s*(?:\d{4}\s*[-–]\s*\d{0,4}|b\.\s*\d{4}|d\.\s*\d{4}))?\ +(?:,?\s*(?:author|editor|translator|compiler)s?\.?)?\.?$", + ) + .expect("a valid pattern") +}); +/// `[Sur la traduction. English]` — the uniform title of a translation, +/// printed above the title it was translated into. +static UNIFORM_TITLE: LazyLock = + LazyLock::new(|| Regex::new(r"^\\?\[.*\\?\]\.?$").expect("a valid pattern")); +static LABEL: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(names|title|other titles|description|identifiers|subjects|classification|uniform title)\s*:\s*(.*)$") + .expect("a valid pattern") +}); +/// `1. Consciousness. 2. Philosophy of mind. I. Title.` +static NUMBERED_SUBJECTS: LazyLock = + LazyLock::new(|| Regex::new(r"^\s*1\.\s+\S").expect("a valid pattern")); +static SUBJECT_SPLIT: LazyLock = + LazyLock::new(|| Regex::new(r"\s*\b\d{1,2}\.\s+").expect("a valid pattern")); +/// The added entries a subject list ends with: `I. Title. II. Series.` +static ADDED_ENTRY: LazyLock = + LazyLock::new(|| Regex::new(r"\s+[IVX]+\.\s+.*$").expect("a valid pattern")); +static ISBN_LINE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)isbn[:\s]*([0-9][0-9\s\-–]{8,20}[0-9Xx])").expect("a valid pattern") +}); +/// The responsibility statement's separator, and what OCR makes of it. +/// +/// A slash between the title and who wrote it is a solidus in a serif face, +/// and page scanners read it as a capital I or a lowercase l about as often +/// as they read it correctly. +static RESPONSIBILITY: LazyLock = + LazyLock::new(|| Regex::new(r"\s+[/Il\\|]\s+").expect("a valid pattern")); +static EDITION_TAIL: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\s*[.,]?\s*[—–-]{1,2}\s*(\d+(st|nd|rd|th)|first|second|third|new|rev\.?|revised)\b.*$") + .expect("a valid pattern") +}); + +fn tidy(s: &str) -> String { + s.trim() + .trim_matches(|c: char| c == '\\' || c == '"') + .trim() + .trim_end_matches(['.', ',', ';', ':', '/']) + .trim() + .to_string() +} + +/// Split a statement of responsibility into who wrote and who edited. +fn people(statement: &str) -> (Vec, Vec) { + let mut authors = Vec::new(); + let mut editors = Vec::new(); + for part in statement.split(';') { + let part = part.trim(); + let lower = part.to_lowercase(); + let (target, rest) = if let Some(rest) = lower.strip_prefix("edited by ") { + (&mut editors, &part[part.len() - rest.len()..]) + } else if lower.starts_with("translated by ") + || lower.starts_with("with ") + || lower.starts_with("foreword by ") + || lower.starts_with("introduction by ") + { + continue; + } else { + (&mut authors, part) + }; + for name in rest.split(" and ").flat_map(|n| n.split(',')) { + let name = tidy(name); + if name.len() >= 3 && name.split_whitespace().count() <= 5 { + target.push(name); + } + } + } + (authors, editors) +} + +fn filed_to_reading(line: &str) -> Option { + let caps = FILED_NAME.captures(line)?; + Some(format!( + "{} {}", + tidy(caps.name("first")?.as_str()), + tidy(caps.name("last")?.as_str()) + )) +} + +fn split_title(statement: &str) -> (Option, Option, Vec, Vec) { + let mut pieces = RESPONSIBILITY.splitn(statement, 2); + let title_part = pieces.next().unwrap_or_default(); + let (authors, editors) = pieces.next().map(people).unwrap_or_default(); + let title_part = EDITION_TAIL.replace(title_part, ""); + let mut halves = title_part.splitn(2, " : "); + let title = tidy(halves.next().unwrap_or_default()); + let subtitle = halves.next().map(tidy).filter(|s| !s.is_empty()); + ( + Some(title).filter(|t| t.len() >= 2), + subtitle, + authors, + editors, + ) +} + +fn subjects_from(line: &str) -> Vec { + let trimmed = ADDED_ENTRY.replace(line, ""); + SUBJECT_SPLIT + .split(&trimmed) + .map(tidy) + .filter(|s| s.len() >= 3 && s.split_whitespace().count() <= 12) + .collect() +} + +fn isbns_from(line: &str) -> Vec { + ISBN_LINE + .captures_iter(line) + .filter_map(|c| { + let digits = bibclean::isbn::digits(&c[1]); + bibclean::isbn::valid(&digits).then_some(digits) + }) + .collect() +} + +/// Read the CIP block out of a book's front matter, if it printed one. +pub fn read(front: &str) -> Option { + let start = HEAD.find(front)?.end(); + let mut block: Vec<&str> = Vec::new(); + for line in front[start..].lines().skip(1) { + let line = line.trim(); + if line.is_empty() { + if block.len() > 2 { + break; + } + continue; + } + if CALL_NUMBER.is_match(line) { + break; + } + block.push(line); + if block.len() >= 24 { + break; + } + } + if block.is_empty() { + return None; + } + + let mut cip = Cip::default(); + let labelled = block.iter().any(|l| { + LABEL + .captures(l) + .is_some_and(|c| matches!(c[1].to_lowercase().as_str(), "names" | "title" | "subjects")) + }); + + if labelled { + for line in &block { + let Some(caps) = LABEL.captures(line) else { + continue; + }; + let value = caps[2].trim(); + match caps[1].to_lowercase().as_str() { + "names" => { + for name in value.split('|') { + if let Some(read) = filed_to_reading(tidy(name).as_str()) { + cip.authors.push(read); + } + } + } + "title" => { + let (title, subtitle, authors, editors) = split_title(value); + cip.title = title; + cip.subtitle = subtitle; + if cip.authors.is_empty() { + cip.authors = authors; + } + cip.editors = editors; + } + "identifiers" => cip.isbns.extend(isbns_from(value)), + "subjects" => { + let value = value.trim_start_matches("LCSH:").trim(); + for subject in value.split('|') { + let subject = tidy(subject); + if subject.len() >= 3 { + cip.subjects.push(subject); + } + } + } + _ => {} + } + } + } else { + let mut seen_title = false; + for line in &block { + if UNIFORM_TITLE.is_match(line) { + continue; + } + if !seen_title { + if let Some(name) = filed_to_reading(line) { + cip.authors.push(name); + continue; + } + if ISBN_LINE.is_match(line) || line.to_lowercase().starts_with("isbn") { + cip.isbns.extend(isbns_from(line)); + continue; + } + if RESPONSIBILITY.is_match(line) || line.contains(" : ") { + let (title, subtitle, authors, editors) = split_title(line); + cip.title = title; + cip.subtitle = subtitle; + if cip.authors.is_empty() { + cip.authors = authors; + } + cip.editors = editors; + seen_title = true; + continue; + } + } + if NUMBERED_SUBJECTS.is_match(line) { + cip.subjects = subjects_from(line); + } + cip.isbns.extend(isbns_from(line)); + } + } + + cip.isbns.sort(); + cip.isbns.dedup(); + if cip.is_empty() { None } else { Some(cip) } +} diff --git a/crates/colophon/src/imprint.rs b/crates/colophon/src/imprint.rs new file mode 100644 index 0000000..e6817cc --- /dev/null +++ b/crates/colophon/src/imprint.rs @@ -0,0 +1,104 @@ +//! The copyright page: who published it, where, when, and under what ISBN. +//! +//! Everything on this page is formula. The same dozen sentences appear in +//! every book printed in English, which is what makes them worth matching +//! exactly rather than interpreting. + +use std::sync::LazyLock; + +use regex::Regex; + +/// What the copyright page stated. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Imprint { + /// The press. + pub publisher: Option, + /// Where it publishes from, where the page named one place. + pub place: Option, + /// The year of this edition, preferring first publication. + pub year: Option, + /// Every checksum-valid ISBN printed on the page, in the order printed. + pub isbns: Vec, +} + +static PUBLISHED_BY: LazyLock = LazyLock::new(|| { + Regex::new(r"(?im)^\s*(?:first\s+)?published(?:\s+(?:in\s+)?(?:1[5-9]|20)\d{2})?\s+by\s+(?P[^,.\n]{3,60})") + .expect("a valid pattern") +}); +static PRESS_LINE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?im)^\s*(?P[A-Z][^\n]{2,58}?(?:University Press|Press|Publishers|Publishing|Books|Verlag))\s*[,.]?\s*$") + .expect("a valid pattern") +}); +static FIRST_PUBLISHED: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)first published[^.\n]{0,40}?((?:1[5-9]|20)\d{2})").expect("a valid pattern") +}); +static COPYRIGHT_YEAR: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)(?:©|\(c\)|copyright)\s*(?:by\s+)?((?:1[5-9]|20)\d{2})") + .expect("a valid pattern") +}); +static ISBN: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)isbn(?:[-\s]?1[03])?[:\s]*([0-9][0-9\s\-–—]{8,24}[0-9Xx])") + .expect("a valid pattern") +}); +/// `Oxford : Oxford University Press, 2007` — the labelled CIP's description, +/// which states all three at once. +static DESCRIPTION: LazyLock = LazyLock::new(|| { + Regex::new(r"(?im)^\s*Description\s*:\s*(?P[^:|\n]{2,40}?)\s*:\s*(?P[^,|\n]{3,60}?)\s*,\s*(?P(?:1[5-9]|20)\d{2})") + .expect("a valid pattern") +}); + +fn tidy(s: &str) -> String { + s.trim() + .trim_end_matches([',', '.', ';']) + .trim() + .to_string() +} + +/// Every checksum-valid ISBN announced in a piece of text. +pub fn isbns(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for caps in ISBN.captures_iter(text) { + let digits = bibclean::isbn::digits(&caps[1]); + if matches!(digits.len(), 10 | 13) + && bibclean::isbn::valid(&digits) + && !out.contains(&digits) + { + out.push(digits); + } + } + out +} + +/// Read the copyright page. +pub fn read(text: &str) -> Imprint { + let mut out = Imprint { + isbns: isbns(text), + ..Imprint::default() + }; + + if let Some(caps) = DESCRIPTION.captures(text) { + out.place = Some(tidy(&caps["place"])); + out.publisher = Some(tidy(&caps["who"])); + out.year = caps["year"].parse().ok(); + return out; + } + + out.publisher = PUBLISHED_BY + .captures(text) + .or_else(|| PRESS_LINE.captures(text)) + .map(|caps| tidy(&caps["who"])); + out.year = FIRST_PUBLISHED + .captures(text) + .or_else(|| COPYRIGHT_YEAR.captures(text)) + .and_then(|caps| caps[1].parse().ok()); + out.place = text + .lines() + .map(str::trim) + .find(|line| crate::title::PLACE.is_match(line) && line.split_whitespace().count() <= 4) + .and_then(|line| { + crate::title::PLACE + .find(line) + .map(|m| tidy(m.as_str().trim_end_matches(|c: char| !c.is_alphabetic()))) + }); + out +} diff --git a/crates/colophon/src/lib.rs b/crates/colophon/src/lib.rs new file mode 100644 index 0000000..ec21eb7 --- /dev/null +++ b/crates/colophon/src/lib.rs @@ -0,0 +1,331 @@ +//! The record a book prints in its own front matter. +//! +//! A book states its own title twice before the text starts, and states it +//! properly: on the title page, set by the publisher, and again in the +//! cataloguing block on the copyright page, where a librarian has already +//! decided what the title is, who the author is, and what the book is about. +//! A file's embedded metadata may say `Untitled` and a filename may say +//! `Oxford`, but the printed page says *The First World War: A Very Short +//! Introduction*, by Michael Howard. +//! +//! So this reads the pages, not the file. What comes back is what the book +//! printed, with the page it was printed on named for every field, and +//! nothing at all where the book printed nothing — a scan that starts at the +//! preface has no title page, and the honest record of that is an empty one. +//! +//! Two traps are worth naming, because both of them are how a reader that +//! does not know better ends up cataloguing the wrong book: +//! +//! - **The series advertisement.** A paperback's front matter lists two +//! hundred other books in the same series, in the same shape a title page +//! uses: a work in capitals, a person beside it. A title taken off that +//! page belongs to somebody else. Those pages are found and refused, +//! including the ones that continue the list without repeating its heading. +//! - **The uniform title.** A translation's cataloguing block prints the +//! original title in brackets above the real one. It is a heading, not the +//! title of the book in your hand. +//! +//! What this does not do is decide whether what the book printed is any +//! good. `THE HUMAN CONDITION` is reported in the capitals it was set in; +//! straightening that is [`bibclean`]'s job. Nor does it look anything up: +//! an ISBN read off the copyright page is a key, and [`bibfetch`] is what +//! turns a key into a catalogue record. +//! +//! [`bibclean`]: https://git.pumpin.online/funkpower/catena +//! [`bibfetch`]: https://git.pumpin.online/funkpower/catena + +pub mod cip; +pub mod imprint; +pub mod page; +pub mod prompt; +pub mod title; + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// How many pages in from the front to read. +/// +/// Front matter that has not finished by here is not front matter. Reading +/// further costs nothing but finds chapter headings, which look enough like +/// titles to be believed. +pub const PAGES: usize = 16; + +/// Which page a field was read off. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Source { + /// The cataloguing-in-publication block: a catalogue record, printed. + Cip, + /// The title page, as the publisher set it. + TitlePage, + /// The copyright page, outside the cataloguing block. + CopyrightPage, + /// A model, answering about the pages this could not read. Never + /// written by a read: only [`prompt::merge`] puts it on a record. + Model, +} + +impl Source { + /// What it is called. + pub fn name(self) -> &'static str { + match self { + Source::Cip => "cip", + Source::TitlePage => "title_page", + Source::CopyrightPage => "copyright_page", + Source::Model => "model", + } + } +} + +/// What one book printed about itself. +/// +/// Every field is written on every record, empty where the book said +/// nothing, and every field that was read names the page it came from in +/// `read_from`. The fields a catalogue needs and this could not find are +/// listed in `unsettled`, so an absence is legible rather than inferred from +/// a missing key. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Printed { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// The file this was read from, where it was read from one. + pub path: Option, + /// The title, without its subtitle. + pub title: Option, + /// The subtitle, where the page set one. + pub subtitle: Option, + /// Whoever the book says wrote it, in reading order. + pub authors: Vec, + /// Whoever the book says edited it. + pub editors: Vec, + /// Checksum-valid ISBNs printed in the front matter, as printed. + pub isbns: Vec, + /// The press. + pub publisher: Option, + /// Where it publishes from. + pub place: Option, + /// The year of publication, preferring first publication. + pub year: Option, + /// The subject headings the cataloguing block filed it under. + pub subjects: Vec, + /// Which page each field was read off. + pub read_from: BTreeMap, + /// The fields a catalogue wants that this book did not print. + pub unsettled: Vec, + /// How many pages were read. + pub pages_read: usize, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Printed { + fn version(&self) -> u32 { + self.v + } +} + +impl Printed { + /// Did the book print anything at all? + pub fn is_empty(&self) -> bool { + self.title.is_none() + && self.authors.is_empty() + && self.isbns.is_empty() + && self.subjects.is_empty() + } + + fn note(&mut self, field: &str, source: Source) { + self.read_from.insert(field.to_string(), source); + } +} + +/// Which page holds the title, given where the copyright page is. +/// +/// The title page is the last page before the copyright page that is a page +/// of set lines rather than a list or a contents — publishers have put it +/// there for two centuries. Where there is no copyright page to count back +/// from, the first such page before the text starts is all there is to go on. +fn title_page(pages: &[page::Page], copyright: Option) -> Option { + let usable = |i: usize| pages.get(i).is_some_and(|p| p.kind == page::Kind::Title); + if let Some(c) = copyright { + if let Some(found) = (0..c).rev().find(|i| usable(*i)) { + return Some(found); + } + } + pages + .iter() + .position(|p| p.kind == page::Kind::Text) + .map_or_else( + || (0..pages.len()).find(|i| usable(*i)), + |end| (0..end).find(|i| usable(*i)), + ) +} + +/// The longest printing of the same title anywhere in the front matter. +/// +/// A title page breaks its title across lines and sets it in capitals; the +/// half title two leaves earlier prints the whole of it on one line, with +/// its subtitle, in the case the publisher meant. They are the same title, +/// and the unbroken one is the one to keep. +fn longest_printing<'a>(pages: &'a [page::Page], title: &str) -> Option<&'a str> { + let wanted = squash(title); + if wanted.len() < 4 { + return None; + } + pages + .iter() + .filter(|p| matches!(p.kind, page::Kind::Title | page::Kind::Copyright)) + .flat_map(|p| p.lines.iter()) + .filter(|line| { + let candidate = squash(line); + candidate.starts_with(&wanted) && candidate.len() > wanted.len() + }) + .min_by_key(|line| line.len()) + .map(String::as_str) +} + +/// A string reduced to the letters and digits in it, for comparing two +/// printings of one title across a change of case and punctuation. +fn squash(s: &str) -> String { + s.chars() + .filter(|c| c.is_alphanumeric()) + .flat_map(char::to_lowercase) + .collect() +} + +/// Read a book's front matter. +/// +/// `budget` is how many pages in to read; [`PAGES`] is the sensible default. +pub fn read(text: &str, budget: usize) -> Printed { + let pages = page::split(text, budget); + let mut out = Printed { + v: version(), + path: None, + title: None, + subtitle: None, + authors: Vec::new(), + editors: Vec::new(), + isbns: Vec::new(), + publisher: None, + place: None, + year: None, + subjects: Vec::new(), + read_from: BTreeMap::new(), + unsettled: Vec::new(), + pages_read: pages.len(), + }; + + let front: String = pages + .iter() + .map(|p| p.text.as_str()) + .collect::>() + .join("\n"); + let copyright = pages.iter().position(|p| p.kind == page::Kind::Copyright); + + // The cataloguing block is a record somebody else already made, so it + // outranks the title page wherever the two disagree. + if let Some(found) = cip::read(&front) { + if let Some(t) = found.title { + out.title = Some(t); + out.note("title", Source::Cip); + } + if let Some(s) = found.subtitle { + out.subtitle = Some(s); + out.note("subtitle", Source::Cip); + } + if !found.authors.is_empty() { + out.authors = found.authors; + out.note("authors", Source::Cip); + } + if !found.editors.is_empty() { + out.editors = found.editors; + out.note("editors", Source::Cip); + } + if !found.subjects.is_empty() { + out.subjects = found.subjects; + out.note("subjects", Source::Cip); + } + out.isbns = found.isbns; + if !out.isbns.is_empty() { + out.note("isbns", Source::Cip); + } + } + + if let Some(page) = title_page(&pages, copyright).and_then(|i| title::read(&pages[i].lines)) { + if out.title.is_none() { + out.title = match page + .title + .as_deref() + .and_then(|t| longest_printing(&pages, t)) + { + Some(longer) => { + let (title, subtitle) = longer + .split_once(" : ") + .or_else(|| longer.split_once(": ")) + .map_or((longer.to_string(), None), |(head, tail)| { + (head.trim().to_string(), Some(tail.trim().to_string())) + }); + out.subtitle = subtitle; + Some(title) + } + None => page.title, + }; + if out.title.is_some() { + out.note("title", Source::TitlePage); + } + if out.subtitle.is_none() && page.subtitle.is_some() { + out.subtitle = page.subtitle; + out.note("subtitle", Source::TitlePage); + } + } + if out.authors.is_empty() && !page.authors.is_empty() { + out.authors = page.authors; + out.note("authors", Source::TitlePage); + } + if out.editors.is_empty() && !page.editors.is_empty() { + out.editors = page.editors; + out.note("editors", Source::TitlePage); + } + } + + let imprint_text = copyright.map_or(front.as_str(), |i| pages[i].text.as_str()); + let stated = imprint::read(imprint_text); + for isbn in stated.isbns { + if !out.isbns.contains(&isbn) { + out.isbns.push(isbn); + out.read_from + .entry("isbns".into()) + .or_insert(Source::CopyrightPage); + } + } + if let Some(publisher) = stated.publisher { + out.publisher = Some(publisher); + out.note("publisher", Source::CopyrightPage); + } + if let Some(place) = stated.place { + out.place = Some(place); + out.note("place", Source::CopyrightPage); + } + if let Some(year) = stated.year { + out.year = Some(year); + out.note("year", Source::CopyrightPage); + } + + for (field, missing) in [ + ("title", out.title.is_none()), + ("authors", out.authors.is_empty() && out.editors.is_empty()), + ("isbns", out.isbns.is_empty()), + ("publisher", out.publisher.is_none()), + ("year", out.year.is_none()), + ("subjects", out.subjects.is_empty()), + ] { + if missing { + out.unsettled.push(field.to_string()); + } + } + out +} diff --git a/crates/colophon/src/main.rs b/crates/colophon/src/main.rs new file mode 100644 index 0000000..cafb12d --- /dev/null +++ b/crates/colophon/src/main.rs @@ -0,0 +1,137 @@ +//! The record a book prints in its own front matter. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::exit; + +/// Read what a book prints about itself in its opening pages. +/// +/// The input is the book as markdown — the whole of it, or as much of the +/// front as you care to pass. The title page and the cataloguing block on +/// the copyright page are read, and what they state comes back as one +/// record, with the page each field was read off named in `read_from`. +/// +/// Nothing is inferred. A scan that starts at chapter one prints no title +/// page, and what comes back then is a record with nothing in it and every +/// field named in `unsettled` — which is the answer, and a usable one. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 the book printed something about itself, 1 it printed nothing +this could read, 65 the input was not readable as text, 74 the file could not +be read. + +Example: + + colophon book.md | jq -r '[.title, (.authors|join(\", \"))] | @tsv' + colophon *.md | jq -r 'select(.unsettled | index(\"title\")) | .path' + colophon book.md | jq -r '.isbns[]' | idnorm | bibfetch" +)] +struct Args { + /// The books to read. Standard input when none are given. + paths: Vec, + + /// Say nothing; the exit code is the answer. + #[arg(long, short)] + quiet: bool, + + /// Write the question to put to a model about the fields these pages + /// did not settle, instead of the record. Nothing is sent: it goes to + /// stdout. One book at a time, because the pages differ. + #[arg(long)] + prompt: bool, + + /// Fill in what the pages did not settle from a model's reply, written + /// earlier by `--prompt`, and write the completed record. + #[arg(long, value_name = "FILE")] + merge: Option, + + /// How many pages in from the front to read. + #[arg(long, value_name = "COUNT", default_value_t = colophon::PAGES)] + pages: usize, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + if args.pages == 0 { + eprintln!("colophon: --pages 0 would read nothing"); + return ExitCode::from(exit::USAGE); + } + if args.prompt && args.merge.is_some() { + eprintln!("colophon: --prompt composes a question and --merge answers one"); + return ExitCode::from(exit::USAGE); + } + if (args.prompt || args.merge.is_some()) && args.paths.len() > 1 { + eprintln!("colophon: one book at a time for --prompt and --merge"); + return ExitCode::from(exit::USAGE); + } + + let inputs: Vec> = if args.paths.is_empty() { + vec![None] + } else { + args.paths.iter().cloned().map(Some).collect() + }; + + let mut records = Vec::with_capacity(inputs.len()); + let mut last_text = String::new(); + for path in inputs { + let text = match catena_cli::read_text(path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("colophon: {why}"); + return ExitCode::from(code); + } + }; + let mut record = colophon::read(&text, args.pages); + record.path = path.as_deref().map(|p| p.display().to_string()); + records.push(record); + last_text = text; + } + + if args.prompt { + let record = &records[0]; + return match colophon::prompt::compose(record, &last_text, args.pages) { + // Nothing left to ask is not a failure, and a question nobody + // needs would waste a model's time. + None => ExitCode::from(exit::NO), + Some(question) => catena_cli::text("colophon", &question), + }; + } + if let Some(path) = args.merge.as_deref() { + let reply = match std::fs::read_to_string(path) { + Ok(reply) => reply, + Err(e) => { + eprintln!("colophon: {}: {e}", path.display()); + return ExitCode::from(exit::IO); + } + }; + match colophon::prompt::merge(&mut records[0], &reply) { + Err(why) => { + eprintln!("colophon: {why}"); + return ExitCode::from(exit::DATA); + } + Ok(0) => eprintln!("colophon: the reply filled nothing in"), + Ok(filled) => eprintln!("colophon: {filled} field(s) filled in from the reply"), + } + } + + let anything = records.iter().any(|r| !r.is_empty()); + if args.quiet { + return ExitCode::from(if anything { exit::OK } else { exit::NO }); + } + if !anything { + eprintln!("colophon: nothing in these pages says what the book is"); + for record in &records { + let _ = catena_cli::stream("colophon", std::slice::from_ref(record)); + } + return ExitCode::from(exit::NO); + } + catena_cli::stream("colophon", &records) +} diff --git a/crates/colophon/src/page.rs b/crates/colophon/src/page.rs new file mode 100644 index 0000000..4527df6 --- /dev/null +++ b/crates/colophon/src/page.rs @@ -0,0 +1,278 @@ +//! The opening of a book, cut into pages and each page named for what it is. +//! +//! Front matter is not prose and does not read like it. A title page is four +//! lines and no sentence; a copyright page is a dense block of formulae that +//! recur from publisher to publisher; a series advertisement is a list of +//! other people's books. Telling them apart is most of the work, because the +//! title that matters is on one of them and the titles that will mislead you +//! are on another. + +use std::sync::LazyLock; + +use regex::Regex; + +/// What a page of front matter is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// Nothing legible on it. + Blank, + /// Other books, advertised. The trap: it names two hundred works that + /// are not this one, in the same shapes a title page uses. + SeriesList, + /// Publication facts: the imprint, the copyright, the ISBN, the CIP block. + Copyright, + /// The book's own table of contents. + Contents, + /// Few lines, no sentences — a title page, a half title, or a dedication. + Title, + /// Running text. The front matter is over. + Text, +} + +/// One page, cleaned of what the converter added. +#[derive(Debug, Clone)] +pub struct Page { + /// Page number as the marker gave it, or position when there was none. + pub number: u32, + /// Legible lines, in order, with markup and typesetting slugs removed. + pub lines: Vec, + /// The page's text as it arrived, for the block readers. + pub text: String, + /// What the page is. + pub kind: Kind, +} + +static PAGE_MARKER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?im)^\s*\s*$").expect("a valid pattern") +}); +static IMAGE: LazyLock = + LazyLock::new(|| Regex::new(r"!\[[^\]]*\]\([^)]*\)").expect("a valid pattern")); +static HEADING: LazyLock = + LazyLock::new(|| Regex::new(r"^#{1,6}\s+").expect("a valid pattern")); +/// `DESA01 19/9/06 6:02 PM Page i` — the typesetter's slug, printed by +/// accident and carried by every page of some books. +static SLUG: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^[a-z]{2,10}[-_ ]?\d{1,6}\s+\d{1,2}/\d{1,2}/\d{2,4}.*page\s+[ivxlcdm\d]+\s*$") + .expect("a valid pattern") +}); +/// A converted EPUB's navigation: `[Begin Reading](dedication.xhtml#ded)`, +/// and the page links some converters leave behind. +static NAVIGATION: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(<|\[[^\]]*\]\([^)]*\)|https?://|www\.|previous page|next page|page_\w+)") + .expect("a valid pattern") +}); +static FOLIO: LazyLock = + LazyLock::new(|| Regex::new(r"^[\d\s.\-–—ivxlcdmIVXLCDM|]+$").expect("a valid pattern")); +static BLANK_NOTE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(this page (has been )?intentionally left blank|blank page)\.?$") + .expect("a valid pattern") +}); +static LIST_CUE: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)(available now|available soon|also by|also in this series|also available|\ +other titles|other books|forthcoming|in the same series|praise for|advance praise|\ +very short introductions|series editors?|titles in this series)", + ) + .expect("a valid pattern") +}); +static COPYRIGHT_CUE: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)(all rights reserved|first published|first edition|copyright\s*[©®(]?|[©®]\s*\d{4}|\(c\)\s*\d{4}|\ +library of congress|british library|catalog(u)?ing[- ]in[- ]publication|\ +printed (in|and bound)|isbn|no part of this (book|publication)|\ +typeset (in|by)|a cip (record|catalogue))", + ) + .expect("a valid pattern") +}); +static CONTENTS_CUE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(table of )?contents$|\.{4,}\s*\d+\s*$").expect("a valid pattern") +}); +/// A line of a series list: a work, then whoever wrote it, on one line. +static LISTED_WORK: LazyLock = LazyLock::new(|| { + Regex::new(r"^[A-Z][A-Za-z'’\-]*(?:[ ,][A-Z][A-Za-z'’\-]*){0,7}\s+[A-Z][a-z'’\-]+\s+[A-Z][A-Za-z'’\-]+\.?$") + .expect("a valid pattern") +}); +/// A person's name alone on a line. +pub static PERSON: LazyLock = LazyLock::new(|| { + Regex::new(r"^(?:[A-Z][A-Za-z'’\-\.]{0,20}\s+){1,3}[A-Z][A-Za-z'’\-]{1,25}\.?$") + .expect("a valid pattern") +}); + +/// Letters spread across a line by a PDF's glyph positions, put back together. +/// +/// `A N E W T R A N SL AT ION` is one word to a reader and eleven tokens to a +/// parser, and the line it ruins is usually the title. +pub fn despread(line: &str) -> String { + let tokens: Vec<&str> = line.split_whitespace().collect(); + if tokens.len() < 4 { + return line.to_string(); + } + let singles = tokens.iter().filter(|t| t.chars().count() == 1).count(); + if singles * 2 < tokens.len() { + return line.to_string(); + } + let mut out: Vec = Vec::new(); + let mut run = String::new(); + for token in tokens { + if token.chars().count() <= 3 && token.chars().all(char::is_alphabetic) { + run.push_str(token); + } else { + if !run.is_empty() { + out.push(std::mem::take(&mut run)); + } + out.push(token.to_string()); + } + } + if !run.is_empty() { + out.push(run); + } + out.join(" ") +} + +/// Is the line so damaged that reading it would invent something? +/// +/// A font whose glyphs map into the private use area comes out of extraction +/// as a line of unassigned code points. It looks like text to a length check +/// and is not text. +fn illegible(line: &str) -> bool { + let total = line.chars().filter(|c| !c.is_whitespace()).count(); + if total == 0 { + return true; + } + let bad = line + .chars() + .filter(|c| { + let n = *c as u32; + (0xE000..=0xF8FF).contains(&n) || *c == '\u{FFFD}' + }) + .count(); + bad * 5 > total +} + +fn clean_line(raw: &str) -> Option { + let without_images = IMAGE.replace_all(raw, " "); + let stripped = HEADING.replace(without_images.trim(), ""); + let line = despread( + stripped + .trim() + .trim_matches(|c| c == '*' || c == '_' || c == '|') + .trim(), + ); + if line.is_empty() + || illegible(&line) + || SLUG.is_match(&line) + || NAVIGATION.is_match(&line) + || FOLIO.is_match(&line) + || BLANK_NOTE.is_match(&line) + { + return None; + } + Some(line) +} + +fn looks_like_prose(line: &str) -> bool { + let words = line.split_whitespace().count(); + words > 14 || (words > 8 && line.trim_end().ends_with(['.', ';', ','])) +} + +fn classify(lines: &[String], text: &str) -> Kind { + if lines.is_empty() { + return Kind::Blank; + } + if COPYRIGHT_CUE.find_iter(text).count() >= 2 { + return Kind::Copyright; + } + if LIST_CUE.is_match(text) && lines.len() >= 5 { + return Kind::SeriesList; + } + let listed = lines.iter().filter(|l| LISTED_WORK.is_match(l)).count(); + if lines.len() >= 6 && listed * 5 >= lines.len() * 2 { + return Kind::SeriesList; + } + if lines.iter().take(3).any(|l| CONTENTS_CUE.is_match(l)) + || lines.iter().filter(|l| CONTENTS_CUE.is_match(l)).count() >= 4 + { + return Kind::Contents; + } + if lines.len() <= 12 && !lines.iter().any(|l| looks_like_prose(l)) { + return Kind::Title; + } + Kind::Text +} + +/// Cut the opening into pages and name each one. +/// +/// Page markers are used where the converter left them. Where it did not — +/// an EPUB has no pages to mark — the blank lines are the only boundary +/// there is, and blocks stand in for pages. That is weaker, which is why a +/// book converted from EPUB usually comes back with nothing but its ISBN: +/// the metadata is in its OPF, and [`bookmeta`] reads that. +/// +/// [`bookmeta`]: https://git.pumpin.online/funkpower/catena +pub fn split(text: &str, budget: usize) -> Vec { + let mut pages = Vec::new(); + if PAGE_MARKER.is_match(text) { + let mut last_end = 0; + let mut number = 0u32; + for m in PAGE_MARKER.captures_iter(text) { + let whole = m.get(0).expect("group zero is always there"); + if number > 0 || whole.start() > 0 { + push(&mut pages, number, &text[last_end..whole.start()]); + } + number = m[1].parse().unwrap_or(number + 1); + last_end = whole.end(); + if pages.len() > budget { + break; + } + } + push(&mut pages, number, &text[last_end..]); + } else { + for (i, block) in text.split("\n\n").enumerate() { + push(&mut pages, i as u32 + 1, block); + if pages.len() > budget { + break; + } + } + } + pages.truncate(budget); + continuations(&mut pages); + pages +} + +fn push(pages: &mut Vec, number: u32, text: &str) { + let lines: Vec = text.lines().filter_map(clean_line).collect(); + let kind = classify(&lines, text); + pages.push(Page { + number, + lines, + text: text.to_string(), + kind, + }); +} + +/// A series list runs over several pages and only the first one announces +/// itself. The pages after it are the same list and have to be refused too, +/// or the title taken off one of them is another book's. +fn continuations(pages: &mut [Page]) { + for i in 1..pages.len() { + if pages[i - 1].kind != Kind::SeriesList { + continue; + } + let page = &pages[i]; + if page.kind == Kind::Blank || page.lines.len() < 6 { + continue; + } + // Only a page that is itself a list continues one. Counting bare + // names here would take a title page for advertising, because a + // title set in capitals and broken across lines is, line by line, + // the same shape as a person's name. + let listed = page + .lines + .iter() + .filter(|l| LISTED_WORK.is_match(l)) + .count(); + if page.lines.len() >= 6 && listed * 2 >= page.lines.len() { + pages[i].kind = Kind::SeriesList; + } + } +} diff --git a/crates/colophon/src/prompt.rs b/crates/colophon/src/prompt.rs new file mode 100644 index 0000000..6c3bb7b --- /dev/null +++ b/crates/colophon/src/prompt.rs @@ -0,0 +1,194 @@ +//! A question to put to a model, for what the pages did not settle. +//! +//! Nothing here sends anything. The prompt is composed and printed, and what +//! happens to it is the caller's business — their model, their machine, +//! their GPU. The tool stays what every tool here is: the same output for +//! the same input, on every machine, with no network and nothing installed. +//! +//! What the model is shown is not the front matter as it arrived. The pages +//! that advertise other books are taken out first, because handing them to a +//! model is how a book ends up catalogued as whichever of two hundred other +//! titles the model's eye fell on. It is shown the pages that describe this +//! book, told what was already read off them, and asked only for the rest. + +use crate::{Printed, Source, page}; + +/// The version of the prompt below. +/// +/// Bumped when the wording changes in a way that could change an answer, so +/// a caller keeping results can tell which prompt produced them. +pub const PROMPT_VERSION: u32 = 1; + +/// How much of the front matter to show. Enough for a title page, a +/// copyright page and a contents; not enough to wander into the text. +const SHOWN: usize = 6000; + +/// The fields a model may be asked for. +const ASKABLE: &[&str] = &["title", "authors", "publisher", "year", "subjects"]; + +fn known(record: &Printed) -> String { + let mut out = String::new(); + if let Some(title) = &record.title { + out.push_str(&format!("title already read: {title}\n")); + } + if !record.authors.is_empty() { + out.push_str(&format!( + "authors already read: {}\n", + record.authors.join("; ") + )); + } + if !record.isbns.is_empty() { + out.push_str(&format!( + "isbn printed in the book: {}\n", + record.isbns.join(", ") + )); + } + if let Some(publisher) = &record.publisher { + out.push_str(&format!("publisher already read: {publisher}\n")); + } + out +} + +/// The front matter with the advertising taken out. +pub fn readable_pages(text: &str, budget: usize) -> String { + let mut out = String::new(); + for page in page::split(text, budget) { + if matches!(page.kind, page::Kind::SeriesList | page::Kind::Blank) || page.lines.is_empty() + { + continue; + } + out.push_str(&format!("--- page {}\n", page.number)); + for line in &page.lines { + out.push_str(line); + out.push('\n'); + } + out.push('\n'); + if out.len() >= SHOWN { + break; + } + } + out.truncate(SHOWN); + out +} + +/// Compose the question, or decline to. +/// +/// `None` when the pages settled everything a model could help with, because +/// there is nothing to ask and a call to a model would be waste. +pub fn compose(record: &Printed, text: &str, budget: usize) -> Option { + let wanted: Vec<&str> = ASKABLE + .iter() + .copied() + .filter(|field| record.unsettled.iter().any(|f| f == field)) + .collect(); + if wanted.is_empty() { + return None; + } + let pages = readable_pages(text, budget); + if pages.trim().is_empty() { + return None; + } + + let mut out = String::from( + "Below are the opening pages of one book, as text. Pages advertising \ + other books have already been removed.\n\n", + ); + out.push_str(&pages); + out.push('\n'); + let already = known(record); + if !already.is_empty() { + out.push_str("Read off these pages already:\n"); + out.push_str(&already); + out.push('\n'); + } + out.push_str(&format!("wanted: {}\n\n", wanted.join(", "))); + out.push_str( + "Reply with one JSON object and nothing else, using only the keys \ + listed as wanted. `title` is the book's own title, without its \ + subtitle. `authors` is a list of the people who wrote it, as the \ + book writes their names. `publisher` is the press. `year` is a \ + four-digit number. `subjects` is a list of at most five subject \ + headings for what the book is about.\n\n\ + Answer only from the pages above. Use null for anything they do not \ + settle: a guess is worse than an absence, because nobody downstream \ + can tell them apart. A title listed beside other titles is another \ + book's, not this one's.\n", + ); + Some(out) +} + +/// Fill in what the pages did not settle from a model's reply. +/// +/// Only the fields that were unsettled are filled, and each one is marked as +/// having come from a model — a record that cannot say which of its fields +/// were read and which were answered is a record nobody can apply a policy +/// to. Returns how many fields were filled. +pub fn merge(record: &mut Printed, reply: &str) -> Result { + let start = reply.find('{').ok_or("the reply holds no JSON object")?; + let end = reply.rfind('}').ok_or("the reply holds no JSON object")?; + let parsed: serde_json::Value = serde_json::from_str(&reply[start..=end]) + .map_err(|e| format!("the reply is not JSON: {e}"))?; + let object = parsed.as_object().ok_or("the reply is not a JSON object")?; + + let mut filled = 0; + let wanted = |record: &Printed, field: &str| record.unsettled.iter().any(|f| f == field); + + if wanted(record, "title") { + if let Some(title) = object.get("title").and_then(|v| v.as_str()) { + let title = title.trim(); + if !title.is_empty() { + record.title = Some(title.to_string()); + record.read_from.insert("title".into(), Source::Model); + filled += 1; + } + } + } + if wanted(record, "publisher") { + if let Some(publisher) = object.get("publisher").and_then(|v| v.as_str()) { + let publisher = publisher.trim(); + if !publisher.is_empty() { + record.publisher = Some(publisher.to_string()); + record.read_from.insert("publisher".into(), Source::Model); + filled += 1; + } + } + } + if wanted(record, "year") { + if let Some(year) = object.get("year").and_then(serde_json::Value::as_u64) { + if (1400..=2200).contains(&year) { + record.year = Some(year as u16); + record.read_from.insert("year".into(), Source::Model); + filled += 1; + } + } + } + for (field, into) in [("authors", true), ("subjects", false)] { + if !wanted(record, field) { + continue; + } + let Some(list) = object.get(field).and_then(|v| v.as_array()) else { + continue; + }; + let values: Vec = list + .iter() + .filter_map(|v| v.as_str()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + if values.is_empty() { + continue; + } + if into { + record.authors = values; + } else { + record.subjects = values; + } + record.read_from.insert(field.into(), Source::Model); + filled += 1; + } + + record + .unsettled + .retain(|field| !record.read_from.contains_key(field)); + Ok(filled) +} diff --git a/crates/colophon/src/title.rs b/crates/colophon/src/title.rs new file mode 100644 index 0000000..dc74dbb --- /dev/null +++ b/crates/colophon/src/title.rs @@ -0,0 +1,291 @@ +//! The title page, read as the block it is. +//! +//! A title page is laid out rather than written: the title, sometimes broken +//! across lines by nothing but the width of the page, then whoever wrote it, +//! then the publisher at the foot. Reading it is a matter of knowing where +//! the title stops, which is at the first line that is a person or a press. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::page::PERSON; + +/// What a title page printed. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct TitlePage { + /// The title, with lines the typesetter broke put back together. + pub title: Option, + /// What followed it, where the page gave a subtitle. + pub subtitle: Option, + /// Whoever it says wrote the book. + pub authors: Vec, + /// Whoever it says edited it. + pub editors: Vec, +} + +/// `(1714)`, `, 1922-2019` — when a title page dates its author. +static LIFESPAN: LazyLock = LazyLock::new(|| { + Regex::new( + r"\s*[(,]\s*(?:1[0-9]|20)\d{2}\s*(?:[-\u{2013}]\s*(?:(?:1[0-9]|20)\d{2})?)?\s*\)?\s*$", + ) + .expect("a valid pattern") +}); +/// `350 BC`, `1971` — a date alone on a line, which is not a title. +static BARE_DATE: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)^\d{1,4}\s*(bc|ad|bce|ce)?\.?$").expect("a valid pattern")); +/// `Edited by`, with whoever it was set on the line beneath. +static ROLE_ONLY: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(edited|translated|written|selected|compiled)\s+by:?$") + .expect("a valid pattern") +}); +static BYLINE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(?Pby|edited by|written by|translated by|selected by|compiled by)\s+(?P.{3,90})$") + .expect("a valid pattern") +}); +/// A press, an imprint, or the line that announces one. The foot of the page. +pub static IMPRINT: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)\b(university press|[a-z]+ press\b|publish(?:er|ers|ing)\b|publications\b|\ +verlag\b|éditions\b|editions\b|books? (?:ltd|inc|group)\b|\ +routledge|blackwell|wiley|springer|palgrave|macmillan|penguin|\ +random house|harpercollins|w\.? ?w\.? norton|doubleday|bloomsbury|eerdmans|\ +brill\b|sage\b|vintage\b|knopf|faber|crossway|zondervan|clarendon|\ +foundation\b|institute\b|society\b|trust\b|distributed by|\ +cover (design|art|photograph)|jacket design|thank you for buying|\ +typeset|designed by|illustrated by|printed by|\ +copyright|all rights reserved|printed in)\b", + ) + .expect("a valid pattern") +}); +/// A city a publisher puts on a title page, alone on its line. +pub static PLACE: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)^(oxford|cambridge|london|new york|new haven|chicago|boston|princeton|\ +berlin|paris|munich|rome|madrid|grand rapids|edinburgh|dublin|toronto|sydney|\ +melbourne|minneapolis|philadelphia|san francisco|los angeles|leiden|amsterdam|\ +copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*", + ) + .expect("a valid pattern") +}); +/// Words that name a part of a book rather than the book. +static STRUCTURAL: &[&str] = &[ + "abbreviations", + "about the author", + "acknowledgements", + "acknowledgments", + "afterword", + "appendix", + "bibliography", + "contents", + "dedication", + "epigraph", + "epilogue", + "foreword", + "frontispiece", + "glossary", + "half title", + "illustrations", + "index", + "introduction", + "list of contributors", + "list of illustrations", + "maps", + "notes", + "postscript", + "preface", + "prologue", + "table of contents", + "title page", + "translator's note", +]; + +fn is_structural(line: &str) -> bool { + let norm: String = line + .chars() + .filter(|c| c.is_alphanumeric() || c.is_whitespace()) + .collect::() + .trim() + .to_lowercase(); + STRUCTURAL.binary_search(&norm.as_str()).is_ok() +} + +fn people(who: &str) -> Vec { + who.split(&[';', '&'][..]) + .flat_map(|part| part.split(" and ")) + .map(|name| name.trim().trim_end_matches(['.', ',']).trim().to_string()) + .filter(|name| name.len() >= 3 && name.split_whitespace().count() <= 5) + .collect() +} + +/// Words a line ends on when the typesetter broke a title across it. +/// +/// `Esotericism in` / `Western Culture` is one title set on two lines, and +/// nothing but the dangling preposition says so. +const CONTINUES: &[&str] = &[ + "a", "an", "and", "at", "between", "by", "for", "from", "in", "into", "of", "on", "or", "the", + "to", "under", "with", "without", +]; + +/// Endings no surname has and every abstract noun does. +/// +/// Two capitalised words look exactly like a name — `Martin Cohen` and +/// `Political Philosophy` are the same shape — and on a title page the +/// second one is the top half of the title. English morphology is what +/// tells them apart without a dictionary of names to consult. +const NOT_A_NAME: &[&str] = &[ + "ance", "ence", "ics", "ism", "ity", "ment", "ness", "ography", "ology", "osophy", "sion", + "tion", "ture", +]; + +fn could_be_a_person(line: &str) -> bool { + let line = LIFESPAN.replace(line, ""); + let line = line.trim().trim_end_matches('.'); + if !PERSON.is_match(line) { + return false; + } + let tokens: Vec<&str> = line.split_whitespace().collect(); + if tokens.len() > 4 { + return false; + } + !tokens.iter().any(|token| { + let word = token.trim_end_matches('.').to_lowercase(); + word.len() > 4 && NOT_A_NAME.iter().any(|end| word.ends_with(end)) + }) +} + +fn wraps(line: &str) -> bool { + if line.ends_with('-') { + return true; + } + line.rsplit_once(char::is_whitespace) + .is_some_and(|(_, last)| CONTINUES.contains(&last.to_lowercase().as_str())) +} + +/// Put a title the page broke across lines back together. +fn unwrap_lines(block: &[&str]) -> Vec { + let mut out: Vec = Vec::new(); + for line in block { + match out.last_mut() { + Some(previous) if wraps(previous) => { + if previous.ends_with('-') { + previous.pop(); + previous.push_str(line); + } else { + previous.push(' '); + previous.push_str(line); + } + } + _ => out.push((*line).to_string()), + } + } + out +} + +/// Read a title page, or decline to. +/// +/// The page is taken as a block that ends at the imprint — a press, a place, +/// or the first line of running prose. What is left is the title, whoever +/// wrote it, and nothing else, because that is all a title page holds. +/// +/// Where the byline sits is the publisher's choice and both choices are +/// common, so it is looked for at the top of the block and at the bottom and +/// nowhere between: a name in the middle of a title page is a title that +/// happens to be two capitalised words. +/// +/// Declines when the block is empty or runs past five lines, at which point +/// this is a page of something else and a title taken off it would be a +/// guess with a page number attached. +pub fn read(lines: &[String]) -> Option { + let mut out = TitlePage::default(); + let mut block: Vec<&str> = Vec::new(); + let mut waiting: Option = None; + + for line in lines.iter().take(14) { + // `Edited by` on its own line, with the names set under it. + if let Some(role) = waiting.take() { + let who = people(line); + if !who.is_empty() { + if role.starts_with("edited") { + out.editors.extend(who); + } else if !role.starts_with("translated") { + out.authors.extend(who); + } + continue; + } + } + if is_structural(line) || BARE_DATE.is_match(line) { + if block.is_empty() { + continue; + } + break; + } + if let Some(caps) = BYLINE.captures(line) { + let role = caps["role"].to_lowercase(); + let who = people(&caps["who"]); + if who.is_empty() { + waiting = Some(role); + continue; + } + if role.starts_with("edited") { + out.editors.extend(who); + } else if !role.starts_with("translated") { + out.authors.extend(who); + } + continue; + } + if ROLE_ONLY.is_match(line) { + waiting = Some(line.to_lowercase()); + continue; + } + if IMPRINT.is_match(line) || PLACE.is_match(line) { + if block.is_empty() { + continue; + } + break; + } + block.push(line); + if block.len() > 5 { + return None; + } + } + + if block.is_empty() { + return None; + } + + // The byline is above the title or below it, never inside it. + if out.authors.is_empty() && out.editors.is_empty() && block.len() > 1 { + if could_be_a_person(block[0]) { + out.authors = people(block[0]); + block.remove(0); + } else if could_be_a_person(block[block.len() - 1]) { + out.authors = people(block[block.len() - 1]); + block.pop(); + } + } + + let units = unwrap_lines(&block); + let mut title = units.first()?.clone(); + let mut subtitle = units.get(1).cloned(); + let split = title + .split_once(" : ") + .or_else(|| title.split_once(": ")) + .map(|(head, tail)| (head.trim().to_string(), tail.trim().to_string())); + if let Some((head, tail)) = split { + title = head; + subtitle = Some(tail); + } + + // A fragment is not a title. Two letters of one, left behind by a font + // the extractor could not read, is worse than nothing: it is a title + // page's authority attached to noise. + out.title = Some(title).filter(|t| { + t.chars().filter(|c| c.is_alphabetic()).count() >= 4 + && t.split_whitespace() + .any(|w| w.chars().filter(|c| c.is_alphabetic()).count() >= 3) + }); + out.subtitle = subtitle.filter(|s| s.len() >= 2); + out.title.as_ref()?; + Some(out) +} diff --git a/crates/colophon/tests/frontmatter.rs b/crates/colophon/tests/frontmatter.rs new file mode 100644 index 0000000..0c3c4d2 --- /dev/null +++ b/crates/colophon/tests/frontmatter.rs @@ -0,0 +1,299 @@ +//! What real front matter looks like, and what has to come off it. +//! +//! Every case here is a shape taken from a book that was catalogued wrongly +//! without it. + +use colophon::{PAGES, Source, read}; + +/// A paperback series prints two hundred other titles before its own, in the +/// same shape a title page uses. Catalogued off that page, this book is by +/// whichever author the reader's eye landed on. +#[test] +fn a_series_advertisement_is_not_the_title_page() { + let front = "\ + + +The First World War: A Very Short Introduction + + + +Very Short Introductions available now: +ANARCHISM Colin Ward +ANCIENT EGYPT Ian Shaw +ARISTOTLE Jonathan Barnes +THE ELEMENTS Philip Ball +CONSCIOUSNESS Susan Blackmore +COSMOLOGY Peter Coles + + + +CLASSICS Mary Beard +DADA AND SURREALISM David Hopkins +DARWIN Jonathan Howard +THE DEAD SEA SCROLLS Timothy Lim +RUSSIAN LITERATURE Catriona Kelly + + + +Michael Howard + +THE FIRST +WORLD WAR + +A Very Short Introduction + + + +Great Clarendon Street, Oxford ox2 6dp +Oxford University Press is a department of the University of Oxford. +First published 2002 +All rights reserved +ISBN 978-0-19-280353-5 +"; + let found = read(front, PAGES); + // The title page breaks the title across two lines and sets it in + // capitals; the half title prints the whole of it, cased as the + // publisher meant. Both are this book, and the unbroken one is kept. + assert_eq!(found.title.as_deref(), Some("The First World War")); + assert_eq!(found.subtitle.as_deref(), Some("A Very Short Introduction")); + assert_eq!(found.authors, vec!["Michael Howard"]); + assert_eq!(found.isbns, vec!["9780192803535"]); + assert_eq!(found.year, Some(2002)); + assert_eq!(found.read_from.get("title"), Some(&Source::TitlePage)); +} + +/// The cataloguing block is a catalogue record somebody else already made. +/// The solidus between the title and who wrote it is read as a capital I by +/// every scanner ever built. +#[test] +fn a_cataloguing_block_outranks_the_page_it_is_printed_on() { + let front = "\ + + +Consciousness Explained + +Daniel C. Dennett + + + +Copyright © 1991 by Daniel C. Dennett +All rights reserved +First published 1991 +Library of Congress Cataloging-in-Publication Data +Dennett, Daniel C. + Consciousness explained I Daniel C. Dennett. — 1st ed. + p. cm. + Includes bibliographical references and index. + ISBN 0-316-18065-3 + 1. Consciousness. 2. Philosophy of mind. I. Title. + B105.C477D46 1991 + 126—dc20 +"; + let found = read(front, PAGES); + assert_eq!(found.title.as_deref(), Some("Consciousness explained")); + assert_eq!(found.authors, vec!["Daniel C. Dennett"]); + assert_eq!(found.read_from.get("title"), Some(&Source::Cip)); + assert_eq!(found.subjects, vec!["Consciousness", "Philosophy of mind"]); + assert_eq!(found.isbns, vec!["0316180653"]); +} + +/// Since 2013 the block comes in labelled fields, and the subject headings +/// with it — the one place a book states what it is about. +#[test] +fn the_labelled_block_gives_its_subjects() { + let front = "\ + + +Library of Congress Cataloging-in-Publication Data +Names: Howard, Michael, 1922- author. +Title: The First World War : a very short introduction / Michael Howard. +Description: Oxford : Oxford University Press, 2007. +Identifiers: ISBN 9780192803535 +Subjects: LCSH: World War, 1914-1918. | Europe--History--1871-1918. +All rights reserved +"; + let found = read(front, PAGES); + assert_eq!(found.title.as_deref(), Some("The First World War")); + assert_eq!(found.subtitle.as_deref(), Some("a very short introduction")); + assert_eq!(found.authors, vec!["Michael Howard"]); + assert_eq!(found.publisher.as_deref(), Some("Oxford University Press")); + assert_eq!(found.place.as_deref(), Some("Oxford")); + assert_eq!(found.year, Some(2007)); + assert_eq!(found.subjects.len(), 2); +} + +/// A translation's block prints the original title in brackets above the +/// real one. It is a heading, not the book in your hand. +#[test] +fn a_uniform_title_is_not_the_title() { + let front = "\ + + +All rights reserved +Library of Congress Cataloging-in-Publication Data +Ricoeur, Paul. + [Sur la traduction. English] + On translation / Paul Ricoeur ; translated by Eileen Brennan. + ISBN 0-415-35777-X +"; + let found = read(front, PAGES); + assert_eq!(found.title.as_deref(), Some("On translation")); + assert_eq!(found.authors, vec!["Paul Ricoeur"]); +} + +/// A title broken across two lines by the width of the page, with the second +/// half looking exactly like a person's name. +#[test] +fn a_wrapped_title_is_one_title() { + let front = "\ + + +Esotericism in +Western Culture + +Counter-Normativity and +Rejected Knowledge + +Wouter J. Hanegraaff + + + +BLOOMSBURY ACADEMIC +First published 2012 +All rights reserved +"; + let found = read(front, PAGES); + assert_eq!( + found.title.as_deref(), + Some("Esotericism in Western Culture") + ); + assert_eq!( + found.subtitle.as_deref(), + Some("Counter-Normativity and Rejected Knowledge") + ); + assert_eq!(found.authors, vec!["Wouter J. Hanegraaff"]); +} + +/// Two capitalised words are a name or they are half a title, and English +/// morphology is the only thing on the page that says which. +#[test] +fn an_abstract_noun_is_not_a_person() { + let front = "\ + + +Political Philosophy +From Plato to Mao +Martin Cohen +Pluto Press +LONDON + + + +First published 2001 by Pluto Press +All rights reserved +"; + let found = read(front, PAGES); + assert_eq!(found.title.as_deref(), Some("Political Philosophy")); + assert_eq!(found.subtitle.as_deref(), Some("From Plato to Mao")); + assert_eq!(found.authors, vec!["Martin Cohen"]); + assert_eq!(found.publisher.as_deref(), Some("Pluto Press")); +} + +/// A scan that starts at the preface prints no title page, and the honest +/// record of that is an empty one. +#[test] +fn a_book_that_says_nothing_about_itself_is_reported_as_saying_nothing() { + let front = "\ + + +Preface + +Some mental states have qualitative phenomenal characters, and since the +dawn of Behaviorism some philosophers have doubted the ability of any +physicalist account of the mental to accommodate this fact. +"; + let found = read(front, PAGES); + assert!(found.is_empty()); + assert!(found.title.is_none()); + for field in ["title", "authors", "isbns", "publisher", "year", "subjects"] { + assert!( + found.unsettled.iter().any(|f| f == field), + "{field} should be unsettled" + ); + } +} + +/// An ISBN whose check digit disagrees is a mistyped one, and reporting it +/// would hand the next stage a key that opens nothing. +#[test] +fn an_isbn_that_fails_its_check_digit_is_not_reported() { + let front = "\ + + +All rights reserved +First published 1999 +ISBN 978-0-19-280353-9 +"; + let found = read(front, PAGES); + assert!(found.isbns.is_empty()); +} + +/// What a model answers is filled in only where the pages said nothing, and +/// is marked as having come from a model. +#[test] +fn a_reply_fills_only_what_the_pages_left() { + let front = "\ + + +Esotericism in +Western Culture + +Wouter J. Hanegraaff + + + +All rights reserved +First published 2012 +"; + let mut found = read(front, PAGES); + assert!(found.subjects.is_empty()); + let filled = colophon::prompt::merge( + &mut found, + "{\"title\": \"Something Else\", \"subjects\": [\"Occultism\", \"Europe\"]}", + ) + .expect("a readable reply"); + assert_eq!(filled, 1); + assert_eq!( + found.title.as_deref(), + Some("Esotericism in Western Culture") + ); + assert_eq!(found.subjects, vec!["Occultism", "Europe"]); + assert_eq!(found.read_from.get("subjects"), Some(&Source::Model)); + assert!(!found.unsettled.iter().any(|f| f == "subjects")); +} + +/// The question a model is asked never includes the pages that would +/// mislead it. +#[test] +fn the_question_leaves_out_the_advertising() { + let front = "\ + + +Very Short Introductions available now: +ANARCHISM Colin Ward +THE ELEMENTS Philip Ball +CONSCIOUSNESS Susan Blackmore +ARISTOTLE Jonathan Barnes +COSMOLOGY Peter Coles + + + +Some pages of a preface that names no title at all, running on for long +enough that the classifier takes it for text rather than a title page. +"; + let found = read(front, PAGES); + let question = colophon::prompt::compose(&found, front, PAGES).expect("a question"); + assert!(!question.contains("THE ELEMENTS")); + assert!(!question.contains("Philip Ball")); + assert!(question.contains("wanted: title")); +}