diff --git a/Cargo.lock b/Cargo.lock index 704704a..5082389 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,6 +84,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -185,6 +194,20 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bookmeta" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "lopdf", + "schemars", + "serde", + "serde_json", + "zip", +] + [[package]] name = "booksniff" version = "0.1.0" @@ -418,6 +441,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.1.7" @@ -486,6 +515,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "deunicode" version = "1.6.2" @@ -2062,6 +2102,23 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.8" @@ -2073,3 +2130,15 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index cae7b78..5ba60b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ catena = { path = "crates/bindings", version = "0.1.0" } seriesof = { path = "crates/seriesof", version = "0.1.0" } idnorm = { path = "crates/idnorm", version = "0.1.0" } bibfetch = { path = "crates/bibfetch", version = "0.1.0" } +bookmeta = { path = "crates/bookmeta", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } @@ -42,6 +43,9 @@ fancy-regex = "0.16" # 0.38 is the last release that builds on the pinned toolchain; the ones # after it need a newer compiler than the rest of this asks for. lopdf = { version = "0.38", default-features = false } +# Reading one small member out of an EPUB. Default features pull in four +# compression backends; an EPUB is deflate or nothing. +zip = { version = "2", default-features = false, features = ["deflate"] } regex = "1" schemars = "1" serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index 0e86e85..9ecebc0 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Each tool installs by name, so you can take one without taking the rest. | | | |---|---| | [`booksniff`](crates/booksniff/README.md) | What a file actually is, whatever its name claims | +| [`bookmeta`](crates/bookmeta/README.md) | The metadata a file already carries | | [`pdfgutter`](crates/pdfgutter/README.md) | Books scanned two pages to a sheet, split back into pages | | [`pdftriage`](crates/pdftriage/README.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed | | [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change | diff --git a/crates/bookmeta/Cargo.toml b/crates/bookmeta/Cargo.toml new file mode 100644 index 0000000..0ad0879 --- /dev/null +++ b/crates/bookmeta/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "bookmeta" +description = "The metadata a file already carries: title, author, publisher, identifiers." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +lopdf.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +zip.workspace = true + +[lints] +workspace = true diff --git a/crates/bookmeta/README.md b/crates/bookmeta/README.md new file mode 100644 index 0000000..2234f16 --- /dev/null +++ b/crates/bookmeta/README.md @@ -0,0 +1,157 @@ +# bookmeta + +The metadata a file already carries. + +Part of [catena](../../README.md). + +## What it does + +Every ebook format has somewhere to put the title, and almost every file in a +library has something written there — by the publisher, by the converter, or +by whoever last touched it. Reading it costs nothing, needs no network, and is +the first thing a catalogue wants. + +What comes back is **what the file says, not what is true**. A PDF titled +`Microsoft Word - ch3.doc` is reported with that title and a note saying where +it came from. Deciding that is rubbish is [`bibclean`](../bibclean/README.md)'s +job; finding out what is actually true is [`bibfetch`](../bibfetch/README.md)'s. + +## When you would use it + +- **Cataloguing a directory** you have just been given, before anything else. +- **Filling a library** without shelling out to calibre for every file. +- **Finding the books that say nothing about themselves**, which are the ones + worth a person's attention. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena bookmeta +``` + +## Example + +``` +$ bookmeta library/*.epub | jq -r '[.title, (.authors|join(", "))] | @tsv' +Hagakure - The Way of the Samurai Yamamoto Tsunetomo +The Complete Stories Flannery O'Connor +A Collection of Essays George Orwell +``` + +The whole record: + +``` +$ bookmeta philokalia.epub +{"v":1,"path":"philokalia.epub","source":"opf","title":"The Philokalia, Volume 3","authors":["G. E. H. Palmer","Kallistos Ware"],"publisher":"Faber & Faber","date":"1984-09-17","language":"en","series":"The Philokalia","series_index":"3","identifiers":{"isbn":"9780571175256"},"subjects":["Orthodox Christianity"]} +``` + +The ones that said nothing: + +```sh +bookmeta library/* | jq -r 'select(.title == null) | .path' +``` + +Feeding a catalogue: + +```sh +bookmeta library/*.epub \ + | jq -c '{title, author: .authors[0], isbn: .identifiers.isbn}' \ + | bibclean +``` + +## Options + +| Option | Effect | +|---|---| +| `--prompt` | Write a question about what the file did not say, instead of the record | +| `--merge` | Read a model's reply on stdin and fill in what was missing | +| `--excerpt ` | Text to show the model alongside the metadata | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Every file said something | +| `1` | At least one said nothing | +| `64` | The command line was wrong | +| `65` | A model's reply could not be read | +| `74` | A file could not be read | + +## Where it reads from + +| Format | Where | +|---|---| +| EPUB | The package document named by `META-INF/container.xml` — Dublin Core, plus the `` entries Calibre adds for a series | +| PDF | The trailer's information dictionary | + +An EPUB whose container is malformed falls back to the first `.opf` in the +archive, because a converter that wrote a bad container often still wrote a +usable package. + +Identifiers come back under the scheme they are: `urn:isbn:…` and a bare ISBN +both become `isbn`, a UUID becomes `uuid`, anything else `other`. PDF dates +are reduced to the day where they parse and kept verbatim where they do not, +because a date nobody can read is still evidence. + +## Asking a model + +Some files say nothing useful. `--prompt` writes the question and **sends +nothing** — where it goes is your business, your model, your machine: + +``` +$ bookmeta --prompt --excerpt opening.txt scan.pdf +You are cataloguing a book. Below is what its file says about itself, and the +opening of the text where one was available. + +Known: + path: scan.pdf + title: Microsoft Word - ch3.doc + +Missing: authors, publisher, date, language +… +Reply with one JSON object and nothing else. … Use null for anything the text +does not settle — a guess is worse than an absence, because nobody downstream +can tell them apart. +``` + +Then put it wherever you like and hand the answer back: + +```sh +bookmeta --prompt scan.pdf | ollama run qwen3.5:9b | bookmeta --merge scan.pdf +``` + +Four things make this safe to depend on: + +- **Only what was missing is filled.** A model that volunteers a better title + for a book whose title was already known is overruled: what the file says is + evidence, what a model says is a guess. +- **A null is an absence**, not a value. So is an empty string. +- **The prompt is versioned** and printed where you can read it. A record + filled this way carries a note saying which prompt version did it. +- **A reply that is not one JSON object is refused**, not half-applied. Prose + and code fences around the object are fine. + +Nothing here reaches the network, so this tool keeps [the contract](../../docs/contract.md) +whole: same input, same output, every run and every machine. + +## As a library + +```rust +let meta = bookmeta::read(std::path::Path::new("philokalia.epub")); +assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3")); + +if let Some(prompt) = bookmeta::prompt::compose(&meta, Some(opening)) { + // send it wherever you like, then: + // bookmeta::prompt::merge(&mut meta, &reply)?; +} +``` + +`bookmeta::epub` and `bookmeta::pdf` read one format each, and +`Metadata::missing` says which fields a record still wants. + +## See also + +- [`booksniff`](../booksniff/README.md) — what the file actually is. +- [`bibclean`](../bibclean/README.md) — whether what it says is any good. +- [`bibfetch`](../bibfetch/README.md) — what a register says instead. diff --git a/crates/bookmeta/src/epub.rs b/crates/bookmeta/src/epub.rs new file mode 100644 index 0000000..d434b82 --- /dev/null +++ b/crates/bookmeta/src/epub.rs @@ -0,0 +1,347 @@ +//! An EPUB's package document, which is where the publisher put it. +//! +//! The archive names a package document in `META-INF/container.xml`, and that +//! document carries Dublin Core elements: title, creator, publisher, date, +//! language, identifier, subject. Calibre adds its own `` entries for +//! the series, which nothing in the standard has a place for. + +use std::io::Read; + +use crate::{Metadata, Source}; + +/// How large a package document is worth reading. +/// +/// A real one is a few kilobytes. Anything enormous is either not a package +/// document or is trying to be a decompression bomb. +const MAX_PACKAGE: u64 = 4 * 1024 * 1024; + +/// Read what an EPUB says about itself. +pub fn read(path: &str, bytes: &[u8]) -> Metadata { + let mut meta = Metadata::empty(path, Source::Opf); + + let cursor = std::io::Cursor::new(bytes); + let Ok(mut archive) = zip::ZipArchive::new(cursor) else { + meta.source = Source::None.name().to_string(); + meta.note("a zip whose directory could not be read"); + return meta; + }; + + let Some(package_path) = package_path(&mut archive) else { + meta.source = Source::None.name().to_string(); + meta.note("no package document, so not an EPUB"); + return meta; + }; + let Some(package) = member(&mut archive, &package_path) else { + meta.source = Source::None.name().to_string(); + meta.note(format!("{package_path} could not be read")); + return meta; + }; + + fill(&mut meta, &package); + if meta.is_empty() { + meta.note("a package document with nothing filled in"); + } + meta +} + +/// Where the package document lives, per `META-INF/container.xml`. +/// +/// Falls back to the first `.opf` in the archive, because a converter that +/// wrote a malformed container often still wrote a usable package. +fn package_path( + archive: &mut zip::ZipArchive, +) -> Option { + if let Some(container) = member(archive, "META-INF/container.xml") { + if let Some(found) = attribute(&container, "rootfile", "full-path") { + return Some(found); + } + } + (0..archive.len()) + .filter_map(|at| { + archive + .by_index(at) + .ok() + .map(|file| file.name().to_string()) + }) + .find(|name| name.to_lowercase().ends_with(".opf")) +} + +/// One member's text, where it is small enough to be worth reading. +fn member( + archive: &mut zip::ZipArchive, + name: &str, +) -> Option { + let file = archive.by_name(name).ok()?; + if file.size() > MAX_PACKAGE { + return None; + } + let mut text = String::new(); + file.take(MAX_PACKAGE).read_to_string(&mut text).ok()?; + Some(text) +} + +/// Fill a record from a package document. +fn fill(meta: &mut Metadata, package: &str) { + meta.title = first(package, "title"); + meta.authors = creators(package); + meta.publisher = first(package, "publisher"); + meta.date = first(package, "date"); + meta.language = first(package, "language"); + meta.subjects = all(package, "subject"); + + for (scheme, value) in identifiers(package) { + meta.identifiers.insert(scheme, value); + } + meta.series = meta_content(package, "calibre:series"); + meta.series_index = meta_content(package, "calibre:series_index"); +} + +/// The text of the first `dc:` element, namespace prefix or not. +fn first(package: &str, name: &str) -> Option { + all(package, name).into_iter().next() +} + +/// The text of every `dc:` element. +fn all(package: &str, name: &str) -> Vec { + let mut out = Vec::new(); + for prefix in ["dc:", ""] { + let open = format!("<{prefix}{name}"); + let close = format!(""); + let mut rest = package; + while let Some(at) = rest.find(&open) { + let after = &rest[at + open.len()..]; + // `', ' ', '\t', '\n', '\r', '/']) { + rest = after; + continue; + } + let Some(gt) = after.find('>') else { break }; + let Some(end) = after[gt + 1..].find(&close) else { + break; + }; + let text = unescape(after[gt + 1..gt + 1 + end].trim()); + if !text.is_empty() { + out.push(text); + } + rest = &after[gt + 1 + end..]; + } + if !out.is_empty() { + return out; + } + } + out +} + +/// The people credited, with `file-as` spellings left out. +/// +/// A package lists `` for authors and `` for +/// everyone else; only the first is a credit a catalogue wants. +fn creators(package: &str) -> Vec { + let mut out: Vec = Vec::new(); + for name in all(package, "creator") { + if !out.contains(&name) { + out.push(name); + } + } + out +} + +/// Every identifier, by scheme. +/// +/// The scheme is in an `opf:scheme` attribute where the publisher bothered, +/// and otherwise has to be read off the value: `urn:isbn:…`, a bare ISBN, a +/// UUID. +fn identifiers(package: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + for raw in all(package, "identifier") { + let value = raw.trim(); + let lower = value.to_lowercase(); + let (scheme, id) = if let Some(rest) = lower.strip_prefix("urn:isbn:") { + ("isbn", rest.to_string()) + } else if let Some(rest) = lower.strip_prefix("urn:uuid:") { + ("uuid", rest.to_string()) + } else if let Some(rest) = lower.strip_prefix("urn:doi:") { + ("doi", rest.to_string()) + } else if lower.starts_with("10.") && lower.contains('/') { + ("doi", value.to_string()) + } else if is_isbn(value) { + ("isbn", digits_of(value)) + } else if lower.len() == 36 && lower.matches('-').count() == 4 { + ("uuid", lower.clone()) + } else { + ("other", value.to_string()) + }; + if !id.is_empty() { + out.push((scheme.to_string(), id)); + } + } + out +} + +/// Whether a string is ten or thirteen digits once its hyphens come off. +fn is_isbn(value: &str) -> bool { + let digits = digits_of(value); + matches!(digits.len(), 10 | 13) + && value + .chars() + .all(|c| c.is_ascii_digit() || "- xX".contains(c)) +} + +fn digits_of(value: &str) -> String { + value + .chars() + .filter(|c| c.is_ascii_digit() || matches!(c, 'x' | 'X')) + .collect() +} + +/// The `content` of a ``, which is where Calibre puts a series. +fn meta_content(package: &str, name: &str) -> Option { + let mut rest = package; + while let Some(at) = rest.find("')?; + let tag = &after[..end]; + if tag.contains(&format!("\"{name}\"")) || tag.contains(&format!("'{name}'")) { + if let Some(content) = attribute_in(tag, "content") { + return Some(unescape(&content)); + } + } + rest = &after[end..]; + } + None +} + +/// An attribute of the first element with a given name. +fn attribute(xml: &str, element: &str, name: &str) -> Option { + let at = xml.find(&format!("<{element}"))?; + let end = xml[at..].find('>')? + at; + attribute_in(&xml[at..end], name) +} + +/// An attribute out of one tag's text. +fn attribute_in(tag: &str, name: &str) -> Option { + for quote in ['"', '\''] { + let needle = format!("{name}={quote}"); + if let Some(at) = tag.find(&needle) { + let rest = &tag[at + needle.len()..]; + if let Some(end) = rest.find(quote) { + return Some(rest[..end].to_string()); + } + } + } + None +} + +/// The five XML entities, which is all a package document uses. +fn unescape(text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +#[cfg(test)] +mod tests { + use super::*; + + const PACKAGE: &str = r#" + + + The Philokalia, Volume 3 + G. E. H. Palmer + Kallistos Ware + A Typesetter + Faber & Faber + 1984-09-17 + en + Orthodox Christianity + Asceticism + 978-0-571-17525-6 + urn:uuid:8f1a2b3c-4d5e-6f70-8192-a3b4c5d6e7f8 + + + +"#; + + fn read_package() -> Metadata { + let mut meta = Metadata::empty("x.epub", Source::Opf); + fill(&mut meta, PACKAGE); + meta + } + + #[test] + fn the_dublin_core_elements_come_out() { + let meta = read_package(); + assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3")); + assert_eq!(meta.publisher.as_deref(), Some("Faber & Faber")); + assert_eq!(meta.date.as_deref(), Some("1984-09-17")); + assert_eq!(meta.language.as_deref(), Some("en")); + } + + #[test] + fn every_author_is_kept_in_order_and_contributors_are_not_authors() { + let meta = read_package(); + assert_eq!(meta.authors, ["G. E. H. Palmer", "Kallistos Ware"]); + } + + #[test] + fn identifiers_come_back_under_the_scheme_they_are() { + let meta = read_package(); + assert_eq!( + meta.identifiers.get("isbn").map(String::as_str), + Some("9780571175256") + ); + assert_eq!( + meta.identifiers.get("uuid").map(String::as_str), + Some("8f1a2b3c-4d5e-6f70-8192-a3b4c5d6e7f8") + ); + } + + #[test] + fn a_series_is_read_from_the_meta_entries_calibre_adds() { + let meta = read_package(); + assert_eq!(meta.series.as_deref(), Some("The Philokalia")); + assert_eq!(meta.series_index.as_deref(), Some("3")); + } + + #[test] + fn subjects_come_out_in_order() { + assert_eq!( + read_package().subjects, + ["Orthodox Christianity", "Asceticism"] + ); + } + + #[test] + fn entities_are_undone_once_and_not_twice() { + // `&lt;` is a literal "<", not a "<". + assert_eq!(unescape("Faber & Faber"), "Faber & Faber"); + assert_eq!(unescape("a &lt; b"), "a < b"); + } + + #[test] + fn a_package_without_the_dublin_core_prefix_still_reads() { + let bare = + "BareNobody"; + let mut meta = Metadata::empty("x.epub", Source::Opf); + fill(&mut meta, bare); + assert_eq!(meta.title.as_deref(), Some("Bare")); + assert_eq!(meta.authors, ["Nobody"]); + } + + #[test] + fn an_element_that_merely_starts_the_same_way_is_not_the_element() { + let tricky = "noYes"; + let mut meta = Metadata::empty("x.epub", Source::Opf); + fill(&mut meta, tricky); + assert_eq!(meta.title.as_deref(), Some("Yes")); + } + + #[test] + fn what_is_not_a_zip_says_so_rather_than_guessing() { + let meta = read("x.epub", b"not a zip at all"); + assert_eq!(meta.source, "none"); + assert!(!meta.notes.is_empty()); + } +} diff --git a/crates/bookmeta/src/lib.rs b/crates/bookmeta/src/lib.rs new file mode 100644 index 0000000..871525c --- /dev/null +++ b/crates/bookmeta/src/lib.rs @@ -0,0 +1,216 @@ +//! The metadata a file already carries. +//! +//! Every ebook format has somewhere to put the title, and almost every file +//! in a library has something written there — put by the publisher, by the +//! converter, or by whoever last touched it. Reading it costs nothing and +//! needs no network, and it is the first thing a catalogue wants. +//! +//! What comes back is what the file says, not what is true. A PDF whose title +//! is `Microsoft Word - chapter3.doc` is reported with that title and a note +//! saying where it came from; deciding it is rubbish is +//! [`bibclean`](https://git.pumpin.online/funkpower/catena)'s job, and +//! looking up what is actually true is [`bibfetch`]'s. +//! +//! [`bibfetch`]: https://git.pumpin.online/funkpower/catena + +pub mod epub; +pub mod pdf; +pub mod prompt; + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Where a file's metadata was read from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Source { + /// An EPUB's package document, which is where the publisher put it. + Opf, + /// A PDF's information dictionary. + PdfInfo, + /// Nothing was found to read. + None, +} + +impl Source { + /// What it is called. + pub fn name(self) -> &'static str { + match self { + Source::Opf => "opf", + Source::PdfInfo => "pdf_info", + Source::None => "none", + } + } +} + +/// What a file says about itself. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Metadata { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// The file it was read from. + pub path: String, + /// Where in the file it came from. + pub source: String, + /// The title, as written. + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + /// The people credited, in the order the file lists them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub authors: Vec, + /// Who published it. + #[serde(skip_serializing_if = "Option::is_none")] + pub publisher: Option, + /// The publication date, in whatever form the file wrote it. + #[serde(skip_serializing_if = "Option::is_none")] + pub date: Option, + /// The language, as the file declares it. + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + /// The series it belongs to. + #[serde(skip_serializing_if = "Option::is_none")] + pub series: Option, + /// Which volume of the series. + #[serde(skip_serializing_if = "Option::is_none")] + pub series_index: Option, + /// Identifiers the file carries, by scheme: `isbn`, `doi`, `uuid` and + /// whatever else the publisher wrote. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub identifiers: BTreeMap, + /// Subject headings, as written. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub subjects: Vec, + /// What was worth saying about the reading. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Metadata { + fn version(&self) -> u32 { + self.v + } +} + +impl Metadata { + /// An empty record for a file nothing could be read from. + pub fn empty(path: &str, source: Source) -> Self { + Metadata { + v: corpus_core::V, + path: path.to_string(), + source: source.name().to_string(), + ..Metadata::default() + } + } + + /// Add a note. + pub fn note(&mut self, note: impl Into) { + self.notes.push(note.into()); + } + + /// The fields a catalogue needs that this record has not got. + /// + /// What a model could be asked about, and the only thing worth asking it: + /// everything else is already known. + pub fn missing(&self) -> Vec<&'static str> { + let mut out = Vec::new(); + if self.title.as_deref().unwrap_or("").trim().is_empty() { + out.push("title"); + } + if self.authors.is_empty() { + out.push("authors"); + } + if self.publisher.is_none() { + out.push("publisher"); + } + if self.date.is_none() { + out.push("date"); + } + if self.language.is_none() { + out.push("language"); + } + out + } + + /// Whether anything at all was found. + pub fn is_empty(&self) -> bool { + self.title.is_none() + && self.authors.is_empty() + && self.publisher.is_none() + && self.date.is_none() + && self.language.is_none() + && self.identifiers.is_empty() + && self.subjects.is_empty() + } +} + +/// Read whatever a file carries, choosing by what the bytes say it is. +pub fn read(path: &std::path::Path) -> Metadata { + let name = path.display().to_string(); + match std::fs::read(path) { + Err(e) => { + let mut meta = Metadata::empty(&name, Source::None); + meta.note(format!("could not be read: {e}")); + meta + } + Ok(bytes) => from_bytes(&name, &bytes), + } +} + +/// The same, for bytes already in hand. +pub fn from_bytes(path: &str, bytes: &[u8]) -> Metadata { + if bytes.starts_with(b"PK\x03\x04") { + return epub::read(path, bytes); + } + if bytes.starts_with(b"%PDF") { + return pdf::read(path, bytes); + } + let mut meta = Metadata::empty(path, Source::None); + meta.note("not a format this reads metadata out of"); + meta +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_format_with_nowhere_to_put_metadata_says_so() { + let meta = from_bytes("notes.txt", b"just some text"); + assert_eq!(meta.source, "none"); + assert!(meta.is_empty()); + assert_eq!(meta.notes, ["not a format this reads metadata out of"]); + } + + #[test] + fn what_is_missing_is_what_a_model_could_be_asked() { + let mut meta = Metadata::empty("x.epub", Source::Opf); + assert_eq!( + meta.missing(), + ["title", "authors", "publisher", "date", "language"] + ); + meta.title = Some("The Philokalia".into()); + meta.authors = vec!["Palmer".into()]; + assert_eq!(meta.missing(), ["publisher", "date", "language"]); + } + + #[test] + fn a_title_of_whitespace_is_no_title() { + let mut meta = Metadata::empty("x.epub", Source::Opf); + meta.title = Some(" ".into()); + assert!(meta.missing().contains(&"title")); + } + + #[test] + fn a_file_that_cannot_be_opened_is_a_record_with_a_reason() { + let meta = read(std::path::Path::new("/definitely/not/here.epub")); + assert_eq!(meta.source, "none"); + assert!(meta.notes[0].starts_with("could not be read:")); + } +} diff --git a/crates/bookmeta/src/main.rs b/crates/bookmeta/src/main.rs new file mode 100644 index 0000000..d68c83c --- /dev/null +++ b/crates/bookmeta/src/main.rs @@ -0,0 +1,138 @@ +//! The metadata a file already carries. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::exit; + +/// Read what a file says about itself, one record per file. +/// +/// Every ebook format has somewhere to put the title, and almost every file +/// in a library has something written there. Reading it costs nothing and +/// needs no network, and it is the first thing a catalogue wants. +/// +/// What comes back is what the file says, not what is true. A PDF titled +/// `Microsoft Word - ch3.doc` is reported with that title and a note saying +/// where it came from. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 every file said something, 1 at least one said nothing, +74 a file could not be read. + +Example: + + bookmeta library/*.epub | jq -r '[.title, (.authors|join(\", \"))] | @tsv' + bookmeta --prompt scan.pdf | ollama run qwen3:8b | bookmeta --merge scan.pdf" +)] +struct Args { + /// Files to read. + paths: Vec, + + /// Write the question to put to a model about what the file did not say, + /// instead of the record. Nothing is sent: the prompt goes to stdout. + #[arg(long)] + prompt: bool, + + /// Read a model's reply on stdin and fill in what was missing, writing + /// the completed record. + #[arg(long)] + merge: bool, + + /// Text to show the model, where the file itself is not worth quoting. + /// Defaults to nothing, which asks about the metadata alone. + #[arg(long, value_name = "FILE")] + excerpt: Option, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + if args.paths.is_empty() { + eprintln!("bookmeta: no files given"); + return ExitCode::from(exit::USAGE); + } + if args.prompt && args.merge { + eprintln!("bookmeta: --prompt composes a question and --merge answers one"); + return ExitCode::from(exit::USAGE); + } + + let excerpt = match args.excerpt.as_deref() { + None => None, + Some(path) => match std::fs::read_to_string(path) { + Ok(text) => Some(text), + Err(e) => { + eprintln!("bookmeta: {}: {e}", path.display()); + return ExitCode::from(exit::IO); + } + }, + }; + + // Both of the model-facing modes are about one file: a prompt for two + // books at once gets an answer about neither. + if args.prompt || args.merge { + if args.paths.len() != 1 { + eprintln!("bookmeta: --prompt and --merge take one file at a time"); + return ExitCode::from(exit::USAGE); + } + return one(&args, &args.paths[0], excerpt.as_deref()); + } + + let mut records = Vec::with_capacity(args.paths.len()); + let mut silent = 0usize; + for path in &args.paths { + let meta = bookmeta::read(path); + if meta.is_empty() { + silent += 1; + } + records.push(meta); + } + + let wrote = catena_cli::stream("bookmeta", &records); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + ExitCode::from(match silent { + 0 => exit::OK, + _ => exit::NO, + }) +} + +/// The two modes that put a question to somebody else's model. +fn one(args: &Args, path: &std::path::Path, excerpt: Option<&str>) -> ExitCode { + let mut meta = bookmeta::read(path); + + if args.prompt { + return match bookmeta::prompt::compose(&meta, excerpt) { + // Nothing missing is not a failure, and printing a prompt + // nobody needs would waste a model's time. + None => ExitCode::from(exit::NO), + Some(prompt) => catena_cli::text("bookmeta", &prompt), + }; + } + + let reply = match catena_cli::read_text(None) { + Ok(reply) => reply, + Err((code, why)) => { + eprintln!("bookmeta: {why}"); + return ExitCode::from(code); + } + }; + match bookmeta::prompt::merge(&mut meta, &reply) { + Err(why) => { + eprintln!("bookmeta: {why}"); + ExitCode::from(exit::DATA) + } + Ok(filled) => { + if filled.is_empty() { + eprintln!("bookmeta: the reply filled nothing in"); + } + catena_cli::stream("bookmeta", std::slice::from_ref(&meta)) + } + } +} diff --git a/crates/bookmeta/src/pdf.rs b/crates/bookmeta/src/pdf.rs new file mode 100644 index 0000000..4f687d6 --- /dev/null +++ b/crates/bookmeta/src/pdf.rs @@ -0,0 +1,188 @@ +//! A PDF's information dictionary. +//! +//! Older PDFs put the title and author in a dictionary named by the trailer. +//! Newer ones prefer XMP, an RDF document embedded as a stream, and most +//! carry both. The dictionary is read here because it is where a converter +//! writes, and what it writes is usually the only thing in the file. +//! +//! What is written there is often rubbish — `Microsoft Word - ch3.doc`, a +//! DTP scratch path, the name of whoever's licence produced it. That is +//! reported as it stands, with a note saying where it came from. + +use lopdf::{Document, Object}; + +use crate::{Metadata, Source}; + +/// Read what a PDF says about itself. +pub fn read(path: &str, bytes: &[u8]) -> Metadata { + let mut meta = Metadata::empty(path, Source::PdfInfo); + + let document = match Document::load_mem(bytes) { + Ok(document) => document, + Err(e) => { + meta.source = Source::None.name().to_string(); + meta.note(format!("could not be parsed: {e}")); + return meta; + } + }; + if document.is_encrypted() { + meta.source = Source::None.name().to_string(); + meta.note("encrypted, so nothing can be read out of it"); + return meta; + } + + let Some(info) = info_dictionary(&document) else { + meta.source = Source::None.name().to_string(); + meta.note("no information dictionary"); + return meta; + }; + + meta.title = text(&document, &info, b"Title"); + meta.authors = text(&document, &info, b"Author") + .map(|author| split_authors(&author)) + .unwrap_or_default(); + meta.date = text(&document, &info, b"CreationDate").map(|date| tidy_date(&date)); + meta.subjects = text(&document, &info, b"Keywords") + .map(|keywords| split_keywords(&keywords)) + .unwrap_or_default(); + + if let Some(producer) = text(&document, &info, b"Producer") { + meta.note(format!("produced by {producer}")); + } + if meta.is_empty() { + meta.note("an information dictionary with nothing filled in"); + } + meta +} + +/// The trailer's `/Info` dictionary, dereferenced. +fn info_dictionary(document: &Document) -> Option { + let info = document.trailer.get(b"Info").ok()?; + let (_, object) = document.dereference(info).ok()?; + object.as_dict().ok().cloned() +} + +/// One entry, as text, with the two PDF string encodings both handled. +fn text(document: &Document, info: &lopdf::Dictionary, key: &[u8]) -> Option { + let object = info.get(key).ok()?; + let (_, object) = document.dereference(object).ok()?; + let bytes = match object { + Object::String(bytes, _) => bytes, + _ => return None, + }; + let decoded = decode(bytes); + let trimmed = decoded.trim(); + match trimmed.is_empty() { + true => None, + false => Some(trimmed.to_string()), + } +} + +/// PDFDocEncoding or UTF-16, told apart by the byte-order mark the spec +/// requires on the latter. +fn decode(bytes: &[u8]) -> String { + if bytes.starts_with(&[0xFE, 0xFF]) { + let units: Vec = bytes[2..] + .chunks_exact(2) + .map(|pair| u16::from_be_bytes([pair[0], pair[1]])) + .collect(); + return String::from_utf16_lossy(&units); + } + // PDFDocEncoding agrees with Latin-1 over everything a title uses. + bytes.iter().map(|byte| *byte as char).collect() +} + +/// Authors, as a producer felt like writing them. +/// +/// There is no convention: semicolons, commas, ampersands and the word "and" +/// all appear. A comma is left alone, because "Last, First" is one person far +/// more often than two. +fn split_authors(author: &str) -> Vec { + let parts: Vec = author + .split([';', '&']) + .flat_map(|part| part.split(" and ")) + .map(|part| part.trim().to_string()) + .filter(|part| !part.is_empty()) + .collect(); + match parts.is_empty() { + true => Vec::new(), + false => parts, + } +} + +/// Keywords, which are comma or semicolon separated by convention. +fn split_keywords(keywords: &str) -> Vec { + keywords + .split([',', ';']) + .map(|word| word.trim().to_string()) + .filter(|word| !word.is_empty()) + .collect() +} + +/// A PDF date, which is `D:YYYYMMDDHHmmSS` and an offset, reduced to the day. +/// +/// Left as written when it is not that shape, because a date nobody can parse +/// is still evidence. +fn tidy_date(date: &str) -> String { + let digits: String = date + .trim_start_matches("D:") + .chars() + .take_while(char::is_ascii_digit) + .collect(); + match digits.len() >= 8 { + true => format!("{}-{}-{}", &digits[0..4], &digits[4..6], &digits[6..8]), + false => date.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_pdf_date_is_reduced_to_the_day() { + assert_eq!(tidy_date("D:19840917120000+01'00'"), "1984-09-17"); + assert_eq!(tidy_date("D:20240102"), "2024-01-02"); + } + + #[test] + fn a_date_nobody_can_parse_is_kept_as_written() { + assert_eq!(tidy_date("sometime in 1984"), "sometime in 1984"); + assert_eq!(tidy_date("D:2024"), "D:2024"); + } + + #[test] + fn authors_are_split_on_what_producers_actually_use() { + assert_eq!(split_authors("Palmer; Ware"), ["Palmer", "Ware"]); + assert_eq!(split_authors("Palmer & Ware"), ["Palmer", "Ware"]); + assert_eq!(split_authors("Palmer and Ware"), ["Palmer", "Ware"]); + } + + #[test] + fn a_comma_is_not_a_separator_because_last_first_is_one_person() { + assert_eq!(split_authors("Dostoevsky, Fyodor"), ["Dostoevsky, Fyodor"]); + } + + #[test] + fn utf16_titles_are_decoded_and_latin1_ones_are_too() { + let utf16 = [0xFE, 0xFF, 0x00, 0x53, 0x00, 0xE9]; + assert_eq!(decode(&utf16), "Sé"); + assert_eq!(decode(b"Plain"), "Plain"); + } + + #[test] + fn keywords_split_on_either_separator() { + assert_eq!( + split_keywords("history, war ; peace"), + ["history", "war", "peace"] + ); + assert_eq!(split_keywords(""), Vec::::new()); + } + + #[test] + fn what_is_not_a_pdf_says_so() { + let meta = read("x.pdf", b"%PDF-1.4 but truncated"); + assert_eq!(meta.source, "none"); + assert!(!meta.notes.is_empty()); + } +} diff --git a/crates/bookmeta/src/prompt.rs b/crates/bookmeta/src/prompt.rs new file mode 100644 index 0000000..561f101 --- /dev/null +++ b/crates/bookmeta/src/prompt.rs @@ -0,0 +1,298 @@ +//! A question to put to a model, for the fields a file did not answer. +//! +//! Nothing here sends anything. The prompt is composed and printed, and what +//! happens to it is the caller's business — their model, their machine, their +//! GPU. That keeps this tool what every other tool here is: the same output +//! for the same input, on every run and every machine, with no network and +//! nothing to install. +//! +//! The prompt is part of the interface, not an implementation detail. It is +//! versioned with the records and printed where anyone can read it, because a +//! prompt that changes quietly is worse than one you cannot see at all. + +use crate::Metadata; + +/// The version of the prompt below. +/// +/// Bumped when the wording changes in a way that could change an answer, so a +/// caller keeping results can tell which prompt produced them. +pub const PROMPT_VERSION: u32 = 1; + +/// How much of a document to show a model. +/// +/// The front matter is where a title page is. Past a few thousand characters +/// it is the body, which says nothing about who published it. +pub const EXCERPT_CHARS: usize = 3000; + +/// Compose the question for what a file did not say about itself. +/// +/// `excerpt` is the opening of the document, where there is one to hand; +/// without it the prompt asks about the metadata alone, which is worth less +/// but still worth something when the title is a scratch filename. +/// +/// `None` when the file answered everything, because there is nothing to ask +/// and a call to a model would be waste. +pub fn compose(meta: &Metadata, excerpt: Option<&str>) -> Option { + let missing = meta.missing(); + if missing.is_empty() { + return None; + } + + let mut out = String::new(); + out.push_str( + "You are cataloguing a book. Below is what its file says about \ + itself, and the opening of the text where one was available.\n\n", + ); + out.push_str("Known:\n"); + out.push_str(&known(meta)); + out.push_str("\nMissing: "); + out.push_str(&missing.join(", ")); + out.push_str("\n\n"); + + if let Some(excerpt) = excerpt { + let excerpt: String = excerpt.chars().take(EXCERPT_CHARS).collect(); + out.push_str("Opening of the text:\n---\n"); + out.push_str(excerpt.trim()); + out.push_str("\n---\n\n"); + } + + out.push_str( + "Reply with one JSON object and nothing else. Use only these keys, \ + and only for fields listed as missing: title, authors, publisher, \ + date, language. `authors` is a list. `language` is a two-letter \ + code. Use null for anything the text does not settle — a guess is \ + worse than an absence, because nobody downstream can tell them \ + apart.\n", + ); + Some(out) +} + +/// The fields that are already known, as the prompt shows them. +fn known(meta: &Metadata) -> String { + let mut out = String::new(); + let mut line = |name: &str, value: &str| { + if !value.is_empty() { + out.push_str(&format!(" {name}: {value}\n")); + } + }; + line("path", &meta.path); + line("title", meta.title.as_deref().unwrap_or("")); + line("authors", &meta.authors.join("; ")); + line("publisher", meta.publisher.as_deref().unwrap_or("")); + line("date", meta.date.as_deref().unwrap_or("")); + line("language", meta.language.as_deref().unwrap_or("")); + for (scheme, value) in &meta.identifiers { + line(scheme, value); + } + out +} + +/// Take a model's reply and fill in the fields that were missing. +/// +/// Only the missing ones: a model that volunteers a better title for a book +/// whose title was already known is overruled, because what the file says is +/// evidence and what a model says is a guess. A reply that is not one JSON +/// object is refused rather than half-applied. +pub fn merge(meta: &mut Metadata, reply: &str) -> Result, String> { + let json = object_in(reply).ok_or("no JSON object in the reply")?; + let value: serde_json::Value = + serde_json::from_str(json).map_err(|e| format!("not readable as JSON: {e}"))?; + + let missing = meta.missing(); + let mut filled = Vec::new(); + let text = |value: &serde_json::Value, key: &str| -> Option { + let found = value.get(key)?.as_str()?.trim().to_string(); + match found.is_empty() { + true => None, + false => Some(found), + } + }; + + if missing.contains(&"title") { + if let Some(title) = text(&value, "title") { + meta.title = Some(title); + filled.push("title"); + } + } + if missing.contains(&"authors") { + let authors: Vec = value + .get("authors") + .and_then(serde_json::Value::as_array) + .map(|list| { + list.iter() + .filter_map(|name| name.as_str()) + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect() + }) + .unwrap_or_default(); + if !authors.is_empty() { + meta.authors = authors; + filled.push("authors"); + } + } + for (key, slot) in [ + ("publisher", &mut meta.publisher), + ("date", &mut meta.date), + ("language", &mut meta.language), + ] { + if missing.contains(&key) { + if let Some(found) = text(&value, key) { + *slot = Some(found); + filled.push(match key { + "publisher" => "publisher", + "date" => "date", + _ => "language", + }); + } + } + } + + if !filled.is_empty() { + meta.note(format!( + "{} filled in from a model, prompt version {PROMPT_VERSION}", + filled.join(", ") + )); + } + Ok(filled) +} + +/// The first balanced `{…}` in a reply. +/// +/// Models wrap JSON in prose and in code fences however they were asked not +/// to, and refusing a good answer over a fence helps nobody. +fn object_in(reply: &str) -> Option<&str> { + let start = reply.find('{')?; + let bytes = reply.as_bytes(); + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + for at in start..bytes.len() { + let byte = bytes[at]; + if in_string { + match (escaped, byte) { + (true, _) => escaped = false, + (false, b'\\') => escaped = true, + (false, b'"') => in_string = false, + _ => {} + } + continue; + } + match byte { + b'"' => in_string = true, + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return Some(&reply[start..=at]); + } + } + _ => {} + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Source; + + fn bare() -> Metadata { + let mut meta = Metadata::empty("scan.pdf", Source::PdfInfo); + meta.title = Some("Microsoft Word - ch3.doc".into()); + meta + } + + #[test] + fn a_file_that_answered_everything_is_not_worth_asking_about() { + let mut meta = bare(); + meta.authors = vec!["Palmer".into()]; + meta.publisher = Some("Faber".into()); + meta.date = Some("1984".into()); + meta.language = Some("en".into()); + assert_eq!(compose(&meta, None), None); + } + + #[test] + fn the_prompt_says_what_is_known_and_what_is_wanted() { + let prompt = compose(&bare(), Some("THE PHILOKALIA\n\nVolume 3")).expect("a prompt"); + assert!(prompt.contains("Microsoft Word - ch3.doc"), "{prompt}"); + assert!( + prompt.contains("Missing: authors, publisher, date, language"), + "{prompt}" + ); + assert!(prompt.contains("THE PHILOKALIA"), "{prompt}"); + } + + #[test] + fn the_same_record_composes_the_same_prompt_every_time() { + assert_eq!(compose(&bare(), Some("x")), compose(&bare(), Some("x"))); + } + + #[test] + fn an_excerpt_is_cut_rather_than_sent_whole() { + let huge = "a".repeat(EXCERPT_CHARS * 3); + let prompt = compose(&bare(), Some(&huge)).expect("a prompt"); + assert!(prompt.len() < EXCERPT_CHARS * 2); + } + + #[test] + fn a_reply_fills_in_only_what_was_missing() { + let mut meta = bare(); + let filled = merge( + &mut meta, + r#"{"title":"The Philokalia","authors":["Palmer","Ware"],"publisher":"Faber","date":"1984","language":"en"}"#, + ) + .expect("a reply"); + // The title was present, however bad, so the model does not overrule it. + assert_eq!(meta.title.as_deref(), Some("Microsoft Word - ch3.doc")); + assert_eq!(meta.authors, ["Palmer", "Ware"]); + assert_eq!(filled, ["authors", "publisher", "date", "language"]); + } + + #[test] + fn a_null_is_an_absence_and_not_a_value() { + let mut meta = bare(); + let filled = merge( + &mut meta, + r#"{"authors":null,"publisher":"","date":"1984"}"#, + ) + .expect("a reply"); + assert_eq!(filled, ["date"]); + assert!(meta.authors.is_empty()); + assert_eq!(meta.publisher, None); + } + + #[test] + fn json_wrapped_in_prose_or_a_fence_is_still_found() { + for reply in [ + "Here you go:\n```json\n{\"date\":\"1984\"}\n```\nHope that helps.", + "{\"date\":\"1984\"}", + "Sure! {\"date\":\"1984\"}", + ] { + let mut meta = bare(); + assert_eq!( + merge(&mut meta, reply).expect("a reply"), + ["date"], + "{reply}" + ); + } + } + + #[test] + fn a_brace_inside_a_string_does_not_end_the_object() { + let mut meta = bare(); + merge(&mut meta, r#"{"publisher":"Faber } Faber","date":"1984"}"#).expect("a reply"); + assert_eq!(meta.publisher.as_deref(), Some("Faber } Faber")); + assert_eq!(meta.date.as_deref(), Some("1984")); + } + + #[test] + fn a_reply_that_is_not_json_is_refused_rather_than_half_applied() { + let mut meta = bare(); + assert!(merge(&mut meta, "I could not tell.").is_err()); + assert!(merge(&mut meta, "{not json}").is_err()); + assert!(meta.notes.is_empty()); + } +}