mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
bookmeta: the metadata a file already carries, and a question about the rest
This commit is contained in:
funkpower
Generated
+69
@@ -84,6 +84,15 @@ version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
|
||||
dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
@@ -185,6 +194,20 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bookmeta"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"lopdf",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.1.0"
|
||||
@@ -418,6 +441,12 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
@@ -486,6 +515,17 @@ dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deunicode"
|
||||
version = "1.6.2"
|
||||
@@ -2062,6 +2102,23 @@ dependencies = [
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"displaydoc",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"thiserror",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.8"
|
||||
@@ -2073,3 +2130,15 @@ name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
@@ -29,6 +29,7 @@ catena = { path = "crates/bindings", version = "0.1.0" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.1.0" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.1.0" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.1.0" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.1.0" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.1.0" }
|
||||
emb = { path = "crates/emb", version = "0.1.0" }
|
||||
|
||||
@@ -42,6 +43,9 @@ fancy-regex = "0.16"
|
||||
# 0.38 is the last release that builds on the pinned toolchain; the ones
|
||||
# after it need a newer compiler than the rest of this asks for.
|
||||
lopdf = { version = "0.38", default-features = false }
|
||||
# Reading one small member out of an EPUB. Default features pull in four
|
||||
# compression backends; an EPUB is deflate or nothing.
|
||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||
regex = "1"
|
||||
schemars = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
|
||||
@@ -24,6 +24,7 @@ Each tool installs by name, so you can take one without taking the rest.
|
||||
| | |
|
||||
|---|---|
|
||||
| [`booksniff`](crates/booksniff/README.md) | What a file actually is, whatever its name claims |
|
||||
| [`bookmeta`](crates/bookmeta/README.md) | The metadata a file already carries |
|
||||
| [`pdfgutter`](crates/pdfgutter/README.md) | Books scanned two pages to a sheet, split back into pages |
|
||||
| [`pdftriage`](crates/pdftriage/README.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed |
|
||||
| [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change |
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "bookmeta"
|
||||
description = "The metadata a file already carries: title, author, publisher, identifiers."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
lopdf.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
zip.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,157 @@
|
||||
# bookmeta
|
||||
|
||||
The metadata a file already carries.
|
||||
|
||||
Part of [catena](../../README.md).
|
||||
|
||||
## What it does
|
||||
|
||||
Every ebook format has somewhere to put the title, and almost every file in a
|
||||
library has something written there — by the publisher, by the converter, or
|
||||
by whoever last touched it. Reading it costs nothing, needs no network, and is
|
||||
the first thing a catalogue wants.
|
||||
|
||||
What comes back is **what the file says, not what is true**. A PDF titled
|
||||
`Microsoft Word - ch3.doc` is reported with that title and a note saying where
|
||||
it came from. Deciding that is rubbish is [`bibclean`](../bibclean/README.md)'s
|
||||
job; finding out what is actually true is [`bibfetch`](../bibfetch/README.md)'s.
|
||||
|
||||
## When you would use it
|
||||
|
||||
- **Cataloguing a directory** you have just been given, before anything else.
|
||||
- **Filling a library** without shelling out to calibre for every file.
|
||||
- **Finding the books that say nothing about themselves**, which are the ones
|
||||
worth a person's attention.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
cargo install --git https://git.pumpin.online/funkpower/catena bookmeta
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
$ bookmeta library/*.epub | jq -r '[.title, (.authors|join(", "))] | @tsv'
|
||||
Hagakure - The Way of the Samurai Yamamoto Tsunetomo
|
||||
The Complete Stories Flannery O'Connor
|
||||
A Collection of Essays George Orwell
|
||||
```
|
||||
|
||||
The whole record:
|
||||
|
||||
```
|
||||
$ bookmeta philokalia.epub
|
||||
{"v":1,"path":"philokalia.epub","source":"opf","title":"The Philokalia, Volume 3","authors":["G. E. H. Palmer","Kallistos Ware"],"publisher":"Faber & Faber","date":"1984-09-17","language":"en","series":"The Philokalia","series_index":"3","identifiers":{"isbn":"9780571175256"},"subjects":["Orthodox Christianity"]}
|
||||
```
|
||||
|
||||
The ones that said nothing:
|
||||
|
||||
```sh
|
||||
bookmeta library/* | jq -r 'select(.title == null) | .path'
|
||||
```
|
||||
|
||||
Feeding a catalogue:
|
||||
|
||||
```sh
|
||||
bookmeta library/*.epub \
|
||||
| jq -c '{title, author: .authors[0], isbn: .identifiers.isbn}' \
|
||||
| bibclean
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `--prompt` | Write a question about what the file did not say, instead of the record |
|
||||
| `--merge` | Read a model's reply on stdin and fill in what was missing |
|
||||
| `--excerpt <FILE>` | Text to show the model alongside the metadata |
|
||||
| `--man` | Write this page as a manual page, in roff |
|
||||
|
||||
## Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| `0` | Every file said something |
|
||||
| `1` | At least one said nothing |
|
||||
| `64` | The command line was wrong |
|
||||
| `65` | A model's reply could not be read |
|
||||
| `74` | A file could not be read |
|
||||
|
||||
## Where it reads from
|
||||
|
||||
| Format | Where |
|
||||
|---|---|
|
||||
| EPUB | The package document named by `META-INF/container.xml` — Dublin Core, plus the `<meta>` entries Calibre adds for a series |
|
||||
| PDF | The trailer's information dictionary |
|
||||
|
||||
An EPUB whose container is malformed falls back to the first `.opf` in the
|
||||
archive, because a converter that wrote a bad container often still wrote a
|
||||
usable package.
|
||||
|
||||
Identifiers come back under the scheme they are: `urn:isbn:…` and a bare ISBN
|
||||
both become `isbn`, a UUID becomes `uuid`, anything else `other`. PDF dates
|
||||
are reduced to the day where they parse and kept verbatim where they do not,
|
||||
because a date nobody can read is still evidence.
|
||||
|
||||
## Asking a model
|
||||
|
||||
Some files say nothing useful. `--prompt` writes the question and **sends
|
||||
nothing** — where it goes is your business, your model, your machine:
|
||||
|
||||
```
|
||||
$ bookmeta --prompt --excerpt opening.txt scan.pdf
|
||||
You are cataloguing a book. Below is what its file says about itself, and the
|
||||
opening of the text where one was available.
|
||||
|
||||
Known:
|
||||
path: scan.pdf
|
||||
title: Microsoft Word - ch3.doc
|
||||
|
||||
Missing: authors, publisher, date, language
|
||||
…
|
||||
Reply with one JSON object and nothing else. … Use null for anything the text
|
||||
does not settle — a guess is worse than an absence, because nobody downstream
|
||||
can tell them apart.
|
||||
```
|
||||
|
||||
Then put it wherever you like and hand the answer back:
|
||||
|
||||
```sh
|
||||
bookmeta --prompt scan.pdf | ollama run qwen3.5:9b | bookmeta --merge scan.pdf
|
||||
```
|
||||
|
||||
Four things make this safe to depend on:
|
||||
|
||||
- **Only what was missing is filled.** A model that volunteers a better title
|
||||
for a book whose title was already known is overruled: what the file says is
|
||||
evidence, what a model says is a guess.
|
||||
- **A null is an absence**, not a value. So is an empty string.
|
||||
- **The prompt is versioned** and printed where you can read it. A record
|
||||
filled this way carries a note saying which prompt version did it.
|
||||
- **A reply that is not one JSON object is refused**, not half-applied. Prose
|
||||
and code fences around the object are fine.
|
||||
|
||||
Nothing here reaches the network, so this tool keeps [the contract](../../docs/contract.md)
|
||||
whole: same input, same output, every run and every machine.
|
||||
|
||||
## As a library
|
||||
|
||||
```rust
|
||||
let meta = bookmeta::read(std::path::Path::new("philokalia.epub"));
|
||||
assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3"));
|
||||
|
||||
if let Some(prompt) = bookmeta::prompt::compose(&meta, Some(opening)) {
|
||||
// send it wherever you like, then:
|
||||
// bookmeta::prompt::merge(&mut meta, &reply)?;
|
||||
}
|
||||
```
|
||||
|
||||
`bookmeta::epub` and `bookmeta::pdf` read one format each, and
|
||||
`Metadata::missing` says which fields a record still wants.
|
||||
|
||||
## See also
|
||||
|
||||
- [`booksniff`](../booksniff/README.md) — what the file actually is.
|
||||
- [`bibclean`](../bibclean/README.md) — whether what it says is any good.
|
||||
- [`bibfetch`](../bibfetch/README.md) — what a register says instead.
|
||||
@@ -0,0 +1,347 @@
|
||||
//! An EPUB's package document, which is where the publisher put it.
|
||||
//!
|
||||
//! The archive names a package document in `META-INF/container.xml`, and that
|
||||
//! document carries Dublin Core elements: title, creator, publisher, date,
|
||||
//! language, identifier, subject. Calibre adds its own `<meta>` entries for
|
||||
//! the series, which nothing in the standard has a place for.
|
||||
|
||||
use std::io::Read;
|
||||
|
||||
use crate::{Metadata, Source};
|
||||
|
||||
/// How large a package document is worth reading.
|
||||
///
|
||||
/// A real one is a few kilobytes. Anything enormous is either not a package
|
||||
/// document or is trying to be a decompression bomb.
|
||||
const MAX_PACKAGE: u64 = 4 * 1024 * 1024;
|
||||
|
||||
/// Read what an EPUB says about itself.
|
||||
pub fn read(path: &str, bytes: &[u8]) -> Metadata {
|
||||
let mut meta = Metadata::empty(path, Source::Opf);
|
||||
|
||||
let cursor = std::io::Cursor::new(bytes);
|
||||
let Ok(mut archive) = zip::ZipArchive::new(cursor) else {
|
||||
meta.source = Source::None.name().to_string();
|
||||
meta.note("a zip whose directory could not be read");
|
||||
return meta;
|
||||
};
|
||||
|
||||
let Some(package_path) = package_path(&mut archive) else {
|
||||
meta.source = Source::None.name().to_string();
|
||||
meta.note("no package document, so not an EPUB");
|
||||
return meta;
|
||||
};
|
||||
let Some(package) = member(&mut archive, &package_path) else {
|
||||
meta.source = Source::None.name().to_string();
|
||||
meta.note(format!("{package_path} could not be read"));
|
||||
return meta;
|
||||
};
|
||||
|
||||
fill(&mut meta, &package);
|
||||
if meta.is_empty() {
|
||||
meta.note("a package document with nothing filled in");
|
||||
}
|
||||
meta
|
||||
}
|
||||
|
||||
/// Where the package document lives, per `META-INF/container.xml`.
|
||||
///
|
||||
/// Falls back to the first `.opf` in the archive, because a converter that
|
||||
/// wrote a malformed container often still wrote a usable package.
|
||||
fn package_path<R: std::io::Read + std::io::Seek>(
|
||||
archive: &mut zip::ZipArchive<R>,
|
||||
) -> Option<String> {
|
||||
if let Some(container) = member(archive, "META-INF/container.xml") {
|
||||
if let Some(found) = attribute(&container, "rootfile", "full-path") {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
(0..archive.len())
|
||||
.filter_map(|at| {
|
||||
archive
|
||||
.by_index(at)
|
||||
.ok()
|
||||
.map(|file| file.name().to_string())
|
||||
})
|
||||
.find(|name| name.to_lowercase().ends_with(".opf"))
|
||||
}
|
||||
|
||||
/// One member's text, where it is small enough to be worth reading.
|
||||
fn member<R: std::io::Read + std::io::Seek>(
|
||||
archive: &mut zip::ZipArchive<R>,
|
||||
name: &str,
|
||||
) -> Option<String> {
|
||||
let file = archive.by_name(name).ok()?;
|
||||
if file.size() > MAX_PACKAGE {
|
||||
return None;
|
||||
}
|
||||
let mut text = String::new();
|
||||
file.take(MAX_PACKAGE).read_to_string(&mut text).ok()?;
|
||||
Some(text)
|
||||
}
|
||||
|
||||
/// Fill a record from a package document.
|
||||
fn fill(meta: &mut Metadata, package: &str) {
|
||||
meta.title = first(package, "title");
|
||||
meta.authors = creators(package);
|
||||
meta.publisher = first(package, "publisher");
|
||||
meta.date = first(package, "date");
|
||||
meta.language = first(package, "language");
|
||||
meta.subjects = all(package, "subject");
|
||||
|
||||
for (scheme, value) in identifiers(package) {
|
||||
meta.identifiers.insert(scheme, value);
|
||||
}
|
||||
meta.series = meta_content(package, "calibre:series");
|
||||
meta.series_index = meta_content(package, "calibre:series_index");
|
||||
}
|
||||
|
||||
/// The text of the first `dc:<name>` element, namespace prefix or not.
|
||||
fn first(package: &str, name: &str) -> Option<String> {
|
||||
all(package, name).into_iter().next()
|
||||
}
|
||||
|
||||
/// The text of every `dc:<name>` element.
|
||||
fn all(package: &str, name: &str) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
for prefix in ["dc:", ""] {
|
||||
let open = format!("<{prefix}{name}");
|
||||
let close = format!("</{prefix}{name}>");
|
||||
let mut rest = package;
|
||||
while let Some(at) = rest.find(&open) {
|
||||
let after = &rest[at + open.len()..];
|
||||
// `<title` must not match `<titlepage`.
|
||||
if !after.starts_with(['>', ' ', '\t', '\n', '\r', '/']) {
|
||||
rest = after;
|
||||
continue;
|
||||
}
|
||||
let Some(gt) = after.find('>') else { break };
|
||||
let Some(end) = after[gt + 1..].find(&close) else {
|
||||
break;
|
||||
};
|
||||
let text = unescape(after[gt + 1..gt + 1 + end].trim());
|
||||
if !text.is_empty() {
|
||||
out.push(text);
|
||||
}
|
||||
rest = &after[gt + 1 + end..];
|
||||
}
|
||||
if !out.is_empty() {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The people credited, with `file-as` spellings left out.
|
||||
///
|
||||
/// A package lists `<dc:creator>` for authors and `<dc:contributor>` for
|
||||
/// everyone else; only the first is a credit a catalogue wants.
|
||||
fn creators(package: &str) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for name in all(package, "creator") {
|
||||
if !out.contains(&name) {
|
||||
out.push(name);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every identifier, by scheme.
|
||||
///
|
||||
/// The scheme is in an `opf:scheme` attribute where the publisher bothered,
|
||||
/// and otherwise has to be read off the value: `urn:isbn:…`, a bare ISBN, a
|
||||
/// UUID.
|
||||
fn identifiers(package: &str) -> Vec<(String, String)> {
|
||||
let mut out = Vec::new();
|
||||
for raw in all(package, "identifier") {
|
||||
let value = raw.trim();
|
||||
let lower = value.to_lowercase();
|
||||
let (scheme, id) = if let Some(rest) = lower.strip_prefix("urn:isbn:") {
|
||||
("isbn", rest.to_string())
|
||||
} else if let Some(rest) = lower.strip_prefix("urn:uuid:") {
|
||||
("uuid", rest.to_string())
|
||||
} else if let Some(rest) = lower.strip_prefix("urn:doi:") {
|
||||
("doi", rest.to_string())
|
||||
} else if lower.starts_with("10.") && lower.contains('/') {
|
||||
("doi", value.to_string())
|
||||
} else if is_isbn(value) {
|
||||
("isbn", digits_of(value))
|
||||
} else if lower.len() == 36 && lower.matches('-').count() == 4 {
|
||||
("uuid", lower.clone())
|
||||
} else {
|
||||
("other", value.to_string())
|
||||
};
|
||||
if !id.is_empty() {
|
||||
out.push((scheme.to_string(), id));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a string is ten or thirteen digits once its hyphens come off.
|
||||
fn is_isbn(value: &str) -> bool {
|
||||
let digits = digits_of(value);
|
||||
matches!(digits.len(), 10 | 13)
|
||||
&& value
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_digit() || "- xX".contains(c))
|
||||
}
|
||||
|
||||
fn digits_of(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit() || matches!(c, 'x' | 'X'))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The `content` of a `<meta name="…">`, which is where Calibre puts a series.
|
||||
fn meta_content(package: &str, name: &str) -> Option<String> {
|
||||
let mut rest = package;
|
||||
while let Some(at) = rest.find("<meta") {
|
||||
let after = &rest[at..];
|
||||
let end = after.find('>')?;
|
||||
let tag = &after[..end];
|
||||
if tag.contains(&format!("\"{name}\"")) || tag.contains(&format!("'{name}'")) {
|
||||
if let Some(content) = attribute_in(tag, "content") {
|
||||
return Some(unescape(&content));
|
||||
}
|
||||
}
|
||||
rest = &after[end..];
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// An attribute of the first element with a given name.
|
||||
fn attribute(xml: &str, element: &str, name: &str) -> Option<String> {
|
||||
let at = xml.find(&format!("<{element}"))?;
|
||||
let end = xml[at..].find('>')? + at;
|
||||
attribute_in(&xml[at..end], name)
|
||||
}
|
||||
|
||||
/// An attribute out of one tag's text.
|
||||
fn attribute_in(tag: &str, name: &str) -> Option<String> {
|
||||
for quote in ['"', '\''] {
|
||||
let needle = format!("{name}={quote}");
|
||||
if let Some(at) = tag.find(&needle) {
|
||||
let rest = &tag[at + needle.len()..];
|
||||
if let Some(end) = rest.find(quote) {
|
||||
return Some(rest[..end].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The five XML entities, which is all a package document uses.
|
||||
fn unescape(text: &str) -> String {
|
||||
text.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
.replace("&", "&")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const PACKAGE: &str = r#"<?xml version="1.0"?>
|
||||
<package xmlns:dc="http://purl.org/dc/elements/1.1/" version="3.0">
|
||||
<metadata>
|
||||
<dc:title>The Philokalia, Volume 3</dc:title>
|
||||
<dc:creator opf:role="aut">G. E. H. Palmer</dc:creator>
|
||||
<dc:creator opf:role="aut">Kallistos Ware</dc:creator>
|
||||
<dc:contributor>A Typesetter</dc:contributor>
|
||||
<dc:publisher>Faber & Faber</dc:publisher>
|
||||
<dc:date>1984-09-17</dc:date>
|
||||
<dc:language>en</dc:language>
|
||||
<dc:subject>Orthodox Christianity</dc:subject>
|
||||
<dc:subject>Asceticism</dc:subject>
|
||||
<dc:identifier opf:scheme="ISBN">978-0-571-17525-6</dc:identifier>
|
||||
<dc:identifier>urn:uuid:8f1a2b3c-4d5e-6f70-8192-a3b4c5d6e7f8</dc:identifier>
|
||||
<meta name="calibre:series" content="The Philokalia"/>
|
||||
<meta name="calibre:series_index" content="3"/>
|
||||
</metadata>
|
||||
</package>"#;
|
||||
|
||||
fn read_package() -> Metadata {
|
||||
let mut meta = Metadata::empty("x.epub", Source::Opf);
|
||||
fill(&mut meta, PACKAGE);
|
||||
meta
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_dublin_core_elements_come_out() {
|
||||
let meta = read_package();
|
||||
assert_eq!(meta.title.as_deref(), Some("The Philokalia, Volume 3"));
|
||||
assert_eq!(meta.publisher.as_deref(), Some("Faber & Faber"));
|
||||
assert_eq!(meta.date.as_deref(), Some("1984-09-17"));
|
||||
assert_eq!(meta.language.as_deref(), Some("en"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_author_is_kept_in_order_and_contributors_are_not_authors() {
|
||||
let meta = read_package();
|
||||
assert_eq!(meta.authors, ["G. E. H. Palmer", "Kallistos Ware"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifiers_come_back_under_the_scheme_they_are() {
|
||||
let meta = read_package();
|
||||
assert_eq!(
|
||||
meta.identifiers.get("isbn").map(String::as_str),
|
||||
Some("9780571175256")
|
||||
);
|
||||
assert_eq!(
|
||||
meta.identifiers.get("uuid").map(String::as_str),
|
||||
Some("8f1a2b3c-4d5e-6f70-8192-a3b4c5d6e7f8")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_series_is_read_from_the_meta_entries_calibre_adds() {
|
||||
let meta = read_package();
|
||||
assert_eq!(meta.series.as_deref(), Some("The Philokalia"));
|
||||
assert_eq!(meta.series_index.as_deref(), Some("3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subjects_come_out_in_order() {
|
||||
assert_eq!(
|
||||
read_package().subjects,
|
||||
["Orthodox Christianity", "Asceticism"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entities_are_undone_once_and_not_twice() {
|
||||
// `&lt;` is a literal "<", not a "<".
|
||||
assert_eq!(unescape("Faber & Faber"), "Faber & Faber");
|
||||
assert_eq!(unescape("a &lt; b"), "a < b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_package_without_the_dublin_core_prefix_still_reads() {
|
||||
let bare =
|
||||
"<package><metadata><title>Bare</title><creator>Nobody</creator></metadata></package>";
|
||||
let mut meta = Metadata::empty("x.epub", Source::Opf);
|
||||
fill(&mut meta, bare);
|
||||
assert_eq!(meta.title.as_deref(), Some("Bare"));
|
||||
assert_eq!(meta.authors, ["Nobody"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_element_that_merely_starts_the_same_way_is_not_the_element() {
|
||||
let tricky = "<package><titlepage>no</titlepage><dc:title>Yes</dc:title></package>";
|
||||
let mut meta = Metadata::empty("x.epub", Source::Opf);
|
||||
fill(&mut meta, tricky);
|
||||
assert_eq!(meta.title.as_deref(), Some("Yes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_zip_says_so_rather_than_guessing() {
|
||||
let meta = read("x.epub", b"not a zip at all");
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(!meta.notes.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
//! The metadata a file already carries.
|
||||
//!
|
||||
//! Every ebook format has somewhere to put the title, and almost every file
|
||||
//! in a library has something written there — put by the publisher, by the
|
||||
//! converter, or by whoever last touched it. Reading it costs nothing and
|
||||
//! needs no network, and it is the first thing a catalogue wants.
|
||||
//!
|
||||
//! What comes back is what the file says, not what is true. A PDF whose title
|
||||
//! is `Microsoft Word - chapter3.doc` is reported with that title and a note
|
||||
//! saying where it came from; deciding it is rubbish is
|
||||
//! [`bibclean`](https://git.pumpin.online/funkpower/catena)'s job, and
|
||||
//! looking up what is actually true is [`bibfetch`]'s.
|
||||
//!
|
||||
//! [`bibfetch`]: https://git.pumpin.online/funkpower/catena
|
||||
|
||||
pub mod epub;
|
||||
pub mod pdf;
|
||||
pub mod prompt;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Where a file's metadata was read from.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// An EPUB's package document, which is where the publisher put it.
|
||||
Opf,
|
||||
/// A PDF's information dictionary.
|
||||
PdfInfo,
|
||||
/// Nothing was found to read.
|
||||
None,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// What it is called.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Source::Opf => "opf",
|
||||
Source::PdfInfo => "pdf_info",
|
||||
Source::None => "none",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a file says about itself.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Metadata {
|
||||
/// Record version.
|
||||
#[serde(default = "version")]
|
||||
pub v: u32,
|
||||
/// The file it was read from.
|
||||
pub path: String,
|
||||
/// Where in the file it came from.
|
||||
pub source: String,
|
||||
/// The title, as written.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
/// The people credited, in the order the file lists them.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub authors: Vec<String>,
|
||||
/// Who published it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub publisher: Option<String>,
|
||||
/// The publication date, in whatever form the file wrote it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<String>,
|
||||
/// The language, as the file declares it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<String>,
|
||||
/// The series it belongs to.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub series: Option<String>,
|
||||
/// Which volume of the series.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub series_index: Option<String>,
|
||||
/// Identifiers the file carries, by scheme: `isbn`, `doi`, `uuid` and
|
||||
/// whatever else the publisher wrote.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub identifiers: BTreeMap<String, String>,
|
||||
/// Subject headings, as written.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub subjects: Vec<String>,
|
||||
/// What was worth saying about the reading.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub notes: Vec<String>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
corpus_core::V
|
||||
}
|
||||
|
||||
impl corpus_core::Record for Metadata {
|
||||
fn version(&self) -> u32 {
|
||||
self.v
|
||||
}
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
/// An empty record for a file nothing could be read from.
|
||||
pub fn empty(path: &str, source: Source) -> Self {
|
||||
Metadata {
|
||||
v: corpus_core::V,
|
||||
path: path.to_string(),
|
||||
source: source.name().to_string(),
|
||||
..Metadata::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a note.
|
||||
pub fn note(&mut self, note: impl Into<String>) {
|
||||
self.notes.push(note.into());
|
||||
}
|
||||
|
||||
/// The fields a catalogue needs that this record has not got.
|
||||
///
|
||||
/// What a model could be asked about, and the only thing worth asking it:
|
||||
/// everything else is already known.
|
||||
pub fn missing(&self) -> Vec<&'static str> {
|
||||
let mut out = Vec::new();
|
||||
if self.title.as_deref().unwrap_or("").trim().is_empty() {
|
||||
out.push("title");
|
||||
}
|
||||
if self.authors.is_empty() {
|
||||
out.push("authors");
|
||||
}
|
||||
if self.publisher.is_none() {
|
||||
out.push("publisher");
|
||||
}
|
||||
if self.date.is_none() {
|
||||
out.push("date");
|
||||
}
|
||||
if self.language.is_none() {
|
||||
out.push("language");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether anything at all was found.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.title.is_none()
|
||||
&& self.authors.is_empty()
|
||||
&& self.publisher.is_none()
|
||||
&& self.date.is_none()
|
||||
&& self.language.is_none()
|
||||
&& self.identifiers.is_empty()
|
||||
&& self.subjects.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Read whatever a file carries, choosing by what the bytes say it is.
|
||||
pub fn read(path: &std::path::Path) -> Metadata {
|
||||
let name = path.display().to_string();
|
||||
match std::fs::read(path) {
|
||||
Err(e) => {
|
||||
let mut meta = Metadata::empty(&name, Source::None);
|
||||
meta.note(format!("could not be read: {e}"));
|
||||
meta
|
||||
}
|
||||
Ok(bytes) => from_bytes(&name, &bytes),
|
||||
}
|
||||
}
|
||||
|
||||
/// The same, for bytes already in hand.
|
||||
pub fn from_bytes(path: &str, bytes: &[u8]) -> Metadata {
|
||||
if bytes.starts_with(b"PK\x03\x04") {
|
||||
return epub::read(path, bytes);
|
||||
}
|
||||
if bytes.starts_with(b"%PDF") {
|
||||
return pdf::read(path, bytes);
|
||||
}
|
||||
let mut meta = Metadata::empty(path, Source::None);
|
||||
meta.note("not a format this reads metadata out of");
|
||||
meta
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_format_with_nowhere_to_put_metadata_says_so() {
|
||||
let meta = from_bytes("notes.txt", b"just some text");
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(meta.is_empty());
|
||||
assert_eq!(meta.notes, ["not a format this reads metadata out of"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_missing_is_what_a_model_could_be_asked() {
|
||||
let mut meta = Metadata::empty("x.epub", Source::Opf);
|
||||
assert_eq!(
|
||||
meta.missing(),
|
||||
["title", "authors", "publisher", "date", "language"]
|
||||
);
|
||||
meta.title = Some("The Philokalia".into());
|
||||
meta.authors = vec!["Palmer".into()];
|
||||
assert_eq!(meta.missing(), ["publisher", "date", "language"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_of_whitespace_is_no_title() {
|
||||
let mut meta = Metadata::empty("x.epub", Source::Opf);
|
||||
meta.title = Some(" ".into());
|
||||
assert!(meta.missing().contains(&"title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_cannot_be_opened_is_a_record_with_a_reason() {
|
||||
let meta = read(std::path::Path::new("/definitely/not/here.epub"));
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(meta.notes[0].starts_with("could not be read:"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! The metadata a file already carries.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
|
||||
/// Read what a file says about itself, one record per file.
|
||||
///
|
||||
/// Every ebook format has somewhere to put the title, and almost every file
|
||||
/// in a library has something written there. Reading it costs nothing and
|
||||
/// needs no network, and it is the first thing a catalogue wants.
|
||||
///
|
||||
/// What comes back is what the file says, not what is true. A PDF titled
|
||||
/// `Microsoft Word - ch3.doc` is reported with that title and a note saying
|
||||
/// where it came from.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 every file said something, 1 at least one said nothing,
|
||||
74 a file could not be read.
|
||||
|
||||
Example:
|
||||
|
||||
bookmeta library/*.epub | jq -r '[.title, (.authors|join(\", \"))] | @tsv'
|
||||
bookmeta --prompt scan.pdf | ollama run qwen3:8b | bookmeta --merge scan.pdf"
|
||||
)]
|
||||
struct Args {
|
||||
/// Files to read.
|
||||
paths: Vec<PathBuf>,
|
||||
|
||||
/// Write the question to put to a model about what the file did not say,
|
||||
/// instead of the record. Nothing is sent: the prompt goes to stdout.
|
||||
#[arg(long)]
|
||||
prompt: bool,
|
||||
|
||||
/// Read a model's reply on stdin and fill in what was missing, writing
|
||||
/// the completed record.
|
||||
#[arg(long)]
|
||||
merge: bool,
|
||||
|
||||
/// Text to show the model, where the file itself is not worth quoting.
|
||||
/// Defaults to nothing, which asks about the metadata alone.
|
||||
#[arg(long, value_name = "FILE")]
|
||||
excerpt: Option<PathBuf>,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
if args.paths.is_empty() {
|
||||
eprintln!("bookmeta: no files given");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
if args.prompt && args.merge {
|
||||
eprintln!("bookmeta: --prompt composes a question and --merge answers one");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
|
||||
let excerpt = match args.excerpt.as_deref() {
|
||||
None => None,
|
||||
Some(path) => match std::fs::read_to_string(path) {
|
||||
Ok(text) => Some(text),
|
||||
Err(e) => {
|
||||
eprintln!("bookmeta: {}: {e}", path.display());
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Both of the model-facing modes are about one file: a prompt for two
|
||||
// books at once gets an answer about neither.
|
||||
if args.prompt || args.merge {
|
||||
if args.paths.len() != 1 {
|
||||
eprintln!("bookmeta: --prompt and --merge take one file at a time");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
return one(&args, &args.paths[0], excerpt.as_deref());
|
||||
}
|
||||
|
||||
let mut records = Vec::with_capacity(args.paths.len());
|
||||
let mut silent = 0usize;
|
||||
for path in &args.paths {
|
||||
let meta = bookmeta::read(path);
|
||||
if meta.is_empty() {
|
||||
silent += 1;
|
||||
}
|
||||
records.push(meta);
|
||||
}
|
||||
|
||||
let wrote = catena_cli::stream("bookmeta", &records);
|
||||
if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) {
|
||||
return wrote;
|
||||
}
|
||||
ExitCode::from(match silent {
|
||||
0 => exit::OK,
|
||||
_ => exit::NO,
|
||||
})
|
||||
}
|
||||
|
||||
/// The two modes that put a question to somebody else's model.
|
||||
fn one(args: &Args, path: &std::path::Path, excerpt: Option<&str>) -> ExitCode {
|
||||
let mut meta = bookmeta::read(path);
|
||||
|
||||
if args.prompt {
|
||||
return match bookmeta::prompt::compose(&meta, excerpt) {
|
||||
// Nothing missing is not a failure, and printing a prompt
|
||||
// nobody needs would waste a model's time.
|
||||
None => ExitCode::from(exit::NO),
|
||||
Some(prompt) => catena_cli::text("bookmeta", &prompt),
|
||||
};
|
||||
}
|
||||
|
||||
let reply = match catena_cli::read_text(None) {
|
||||
Ok(reply) => reply,
|
||||
Err((code, why)) => {
|
||||
eprintln!("bookmeta: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
match bookmeta::prompt::merge(&mut meta, &reply) {
|
||||
Err(why) => {
|
||||
eprintln!("bookmeta: {why}");
|
||||
ExitCode::from(exit::DATA)
|
||||
}
|
||||
Ok(filled) => {
|
||||
if filled.is_empty() {
|
||||
eprintln!("bookmeta: the reply filled nothing in");
|
||||
}
|
||||
catena_cli::stream("bookmeta", std::slice::from_ref(&meta))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
//! A PDF's information dictionary.
|
||||
//!
|
||||
//! Older PDFs put the title and author in a dictionary named by the trailer.
|
||||
//! Newer ones prefer XMP, an RDF document embedded as a stream, and most
|
||||
//! carry both. The dictionary is read here because it is where a converter
|
||||
//! writes, and what it writes is usually the only thing in the file.
|
||||
//!
|
||||
//! What is written there is often rubbish — `Microsoft Word - ch3.doc`, a
|
||||
//! DTP scratch path, the name of whoever's licence produced it. That is
|
||||
//! reported as it stands, with a note saying where it came from.
|
||||
|
||||
use lopdf::{Document, Object};
|
||||
|
||||
use crate::{Metadata, Source};
|
||||
|
||||
/// Read what a PDF says about itself.
|
||||
pub fn read(path: &str, bytes: &[u8]) -> Metadata {
|
||||
let mut meta = Metadata::empty(path, Source::PdfInfo);
|
||||
|
||||
let document = match Document::load_mem(bytes) {
|
||||
Ok(document) => document,
|
||||
Err(e) => {
|
||||
meta.source = Source::None.name().to_string();
|
||||
meta.note(format!("could not be parsed: {e}"));
|
||||
return meta;
|
||||
}
|
||||
};
|
||||
if document.is_encrypted() {
|
||||
meta.source = Source::None.name().to_string();
|
||||
meta.note("encrypted, so nothing can be read out of it");
|
||||
return meta;
|
||||
}
|
||||
|
||||
let Some(info) = info_dictionary(&document) else {
|
||||
meta.source = Source::None.name().to_string();
|
||||
meta.note("no information dictionary");
|
||||
return meta;
|
||||
};
|
||||
|
||||
meta.title = text(&document, &info, b"Title");
|
||||
meta.authors = text(&document, &info, b"Author")
|
||||
.map(|author| split_authors(&author))
|
||||
.unwrap_or_default();
|
||||
meta.date = text(&document, &info, b"CreationDate").map(|date| tidy_date(&date));
|
||||
meta.subjects = text(&document, &info, b"Keywords")
|
||||
.map(|keywords| split_keywords(&keywords))
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Some(producer) = text(&document, &info, b"Producer") {
|
||||
meta.note(format!("produced by {producer}"));
|
||||
}
|
||||
if meta.is_empty() {
|
||||
meta.note("an information dictionary with nothing filled in");
|
||||
}
|
||||
meta
|
||||
}
|
||||
|
||||
/// The trailer's `/Info` dictionary, dereferenced.
|
||||
fn info_dictionary(document: &Document) -> Option<lopdf::Dictionary> {
|
||||
let info = document.trailer.get(b"Info").ok()?;
|
||||
let (_, object) = document.dereference(info).ok()?;
|
||||
object.as_dict().ok().cloned()
|
||||
}
|
||||
|
||||
/// One entry, as text, with the two PDF string encodings both handled.
|
||||
fn text(document: &Document, info: &lopdf::Dictionary, key: &[u8]) -> Option<String> {
|
||||
let object = info.get(key).ok()?;
|
||||
let (_, object) = document.dereference(object).ok()?;
|
||||
let bytes = match object {
|
||||
Object::String(bytes, _) => bytes,
|
||||
_ => return None,
|
||||
};
|
||||
let decoded = decode(bytes);
|
||||
let trimmed = decoded.trim();
|
||||
match trimmed.is_empty() {
|
||||
true => None,
|
||||
false => Some(trimmed.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// PDFDocEncoding or UTF-16, told apart by the byte-order mark the spec
|
||||
/// requires on the latter.
|
||||
fn decode(bytes: &[u8]) -> String {
|
||||
if bytes.starts_with(&[0xFE, 0xFF]) {
|
||||
let units: Vec<u16> = bytes[2..]
|
||||
.chunks_exact(2)
|
||||
.map(|pair| u16::from_be_bytes([pair[0], pair[1]]))
|
||||
.collect();
|
||||
return String::from_utf16_lossy(&units);
|
||||
}
|
||||
// PDFDocEncoding agrees with Latin-1 over everything a title uses.
|
||||
bytes.iter().map(|byte| *byte as char).collect()
|
||||
}
|
||||
|
||||
/// Authors, as a producer felt like writing them.
|
||||
///
|
||||
/// There is no convention: semicolons, commas, ampersands and the word "and"
|
||||
/// all appear. A comma is left alone, because "Last, First" is one person far
|
||||
/// more often than two.
|
||||
fn split_authors(author: &str) -> Vec<String> {
|
||||
let parts: Vec<String> = author
|
||||
.split([';', '&'])
|
||||
.flat_map(|part| part.split(" and "))
|
||||
.map(|part| part.trim().to_string())
|
||||
.filter(|part| !part.is_empty())
|
||||
.collect();
|
||||
match parts.is_empty() {
|
||||
true => Vec::new(),
|
||||
false => parts,
|
||||
}
|
||||
}
|
||||
|
||||
/// Keywords, which are comma or semicolon separated by convention.
|
||||
fn split_keywords(keywords: &str) -> Vec<String> {
|
||||
keywords
|
||||
.split([',', ';'])
|
||||
.map(|word| word.trim().to_string())
|
||||
.filter(|word| !word.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A PDF date, which is `D:YYYYMMDDHHmmSS` and an offset, reduced to the day.
|
||||
///
|
||||
/// Left as written when it is not that shape, because a date nobody can parse
|
||||
/// is still evidence.
|
||||
fn tidy_date(date: &str) -> String {
|
||||
let digits: String = date
|
||||
.trim_start_matches("D:")
|
||||
.chars()
|
||||
.take_while(char::is_ascii_digit)
|
||||
.collect();
|
||||
match digits.len() >= 8 {
|
||||
true => format!("{}-{}-{}", &digits[0..4], &digits[4..6], &digits[6..8]),
|
||||
false => date.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_pdf_date_is_reduced_to_the_day() {
|
||||
assert_eq!(tidy_date("D:19840917120000+01'00'"), "1984-09-17");
|
||||
assert_eq!(tidy_date("D:20240102"), "2024-01-02");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_date_nobody_can_parse_is_kept_as_written() {
|
||||
assert_eq!(tidy_date("sometime in 1984"), "sometime in 1984");
|
||||
assert_eq!(tidy_date("D:2024"), "D:2024");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authors_are_split_on_what_producers_actually_use() {
|
||||
assert_eq!(split_authors("Palmer; Ware"), ["Palmer", "Ware"]);
|
||||
assert_eq!(split_authors("Palmer & Ware"), ["Palmer", "Ware"]);
|
||||
assert_eq!(split_authors("Palmer and Ware"), ["Palmer", "Ware"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_comma_is_not_a_separator_because_last_first_is_one_person() {
|
||||
assert_eq!(split_authors("Dostoevsky, Fyodor"), ["Dostoevsky, Fyodor"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf16_titles_are_decoded_and_latin1_ones_are_too() {
|
||||
let utf16 = [0xFE, 0xFF, 0x00, 0x53, 0x00, 0xE9];
|
||||
assert_eq!(decode(&utf16), "Sé");
|
||||
assert_eq!(decode(b"Plain"), "Plain");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keywords_split_on_either_separator() {
|
||||
assert_eq!(
|
||||
split_keywords("history, war ; peace"),
|
||||
["history", "war", "peace"]
|
||||
);
|
||||
assert_eq!(split_keywords(""), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_pdf_says_so() {
|
||||
let meta = read("x.pdf", b"%PDF-1.4 but truncated");
|
||||
assert_eq!(meta.source, "none");
|
||||
assert!(!meta.notes.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
//! A question to put to a model, for the fields a file did not answer.
|
||||
//!
|
||||
//! Nothing here sends anything. The prompt is composed and printed, and what
|
||||
//! happens to it is the caller's business — their model, their machine, their
|
||||
//! GPU. That keeps this tool what every other tool here is: the same output
|
||||
//! for the same input, on every run and every machine, with no network and
|
||||
//! nothing to install.
|
||||
//!
|
||||
//! The prompt is part of the interface, not an implementation detail. It is
|
||||
//! versioned with the records and printed where anyone can read it, because a
|
||||
//! prompt that changes quietly is worse than one you cannot see at all.
|
||||
|
||||
use crate::Metadata;
|
||||
|
||||
/// The version of the prompt below.
|
||||
///
|
||||
/// Bumped when the wording changes in a way that could change an answer, so a
|
||||
/// caller keeping results can tell which prompt produced them.
|
||||
pub const PROMPT_VERSION: u32 = 1;
|
||||
|
||||
/// How much of a document to show a model.
|
||||
///
|
||||
/// The front matter is where a title page is. Past a few thousand characters
|
||||
/// it is the body, which says nothing about who published it.
|
||||
pub const EXCERPT_CHARS: usize = 3000;
|
||||
|
||||
/// Compose the question for what a file did not say about itself.
|
||||
///
|
||||
/// `excerpt` is the opening of the document, where there is one to hand;
|
||||
/// without it the prompt asks about the metadata alone, which is worth less
|
||||
/// but still worth something when the title is a scratch filename.
|
||||
///
|
||||
/// `None` when the file answered everything, because there is nothing to ask
|
||||
/// and a call to a model would be waste.
|
||||
pub fn compose(meta: &Metadata, excerpt: Option<&str>) -> Option<String> {
|
||||
let missing = meta.missing();
|
||||
if missing.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut out = String::new();
|
||||
out.push_str(
|
||||
"You are cataloguing a book. Below is what its file says about \
|
||||
itself, and the opening of the text where one was available.\n\n",
|
||||
);
|
||||
out.push_str("Known:\n");
|
||||
out.push_str(&known(meta));
|
||||
out.push_str("\nMissing: ");
|
||||
out.push_str(&missing.join(", "));
|
||||
out.push_str("\n\n");
|
||||
|
||||
if let Some(excerpt) = excerpt {
|
||||
let excerpt: String = excerpt.chars().take(EXCERPT_CHARS).collect();
|
||||
out.push_str("Opening of the text:\n---\n");
|
||||
out.push_str(excerpt.trim());
|
||||
out.push_str("\n---\n\n");
|
||||
}
|
||||
|
||||
out.push_str(
|
||||
"Reply with one JSON object and nothing else. Use only these keys, \
|
||||
and only for fields listed as missing: title, authors, publisher, \
|
||||
date, language. `authors` is a list. `language` is a two-letter \
|
||||
code. Use null for anything the text does not settle — a guess is \
|
||||
worse than an absence, because nobody downstream can tell them \
|
||||
apart.\n",
|
||||
);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// The fields that are already known, as the prompt shows them.
|
||||
fn known(meta: &Metadata) -> String {
|
||||
let mut out = String::new();
|
||||
let mut line = |name: &str, value: &str| {
|
||||
if !value.is_empty() {
|
||||
out.push_str(&format!(" {name}: {value}\n"));
|
||||
}
|
||||
};
|
||||
line("path", &meta.path);
|
||||
line("title", meta.title.as_deref().unwrap_or(""));
|
||||
line("authors", &meta.authors.join("; "));
|
||||
line("publisher", meta.publisher.as_deref().unwrap_or(""));
|
||||
line("date", meta.date.as_deref().unwrap_or(""));
|
||||
line("language", meta.language.as_deref().unwrap_or(""));
|
||||
for (scheme, value) in &meta.identifiers {
|
||||
line(scheme, value);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Take a model's reply and fill in the fields that were missing.
|
||||
///
|
||||
/// Only the missing ones: a model that volunteers a better title for a book
|
||||
/// whose title was already known is overruled, because what the file says is
|
||||
/// evidence and what a model says is a guess. A reply that is not one JSON
|
||||
/// object is refused rather than half-applied.
|
||||
pub fn merge(meta: &mut Metadata, reply: &str) -> Result<Vec<&'static str>, String> {
|
||||
let json = object_in(reply).ok_or("no JSON object in the reply")?;
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(json).map_err(|e| format!("not readable as JSON: {e}"))?;
|
||||
|
||||
let missing = meta.missing();
|
||||
let mut filled = Vec::new();
|
||||
let text = |value: &serde_json::Value, key: &str| -> Option<String> {
|
||||
let found = value.get(key)?.as_str()?.trim().to_string();
|
||||
match found.is_empty() {
|
||||
true => None,
|
||||
false => Some(found),
|
||||
}
|
||||
};
|
||||
|
||||
if missing.contains(&"title") {
|
||||
if let Some(title) = text(&value, "title") {
|
||||
meta.title = Some(title);
|
||||
filled.push("title");
|
||||
}
|
||||
}
|
||||
if missing.contains(&"authors") {
|
||||
let authors: Vec<String> = value
|
||||
.get("authors")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.map(|list| {
|
||||
list.iter()
|
||||
.filter_map(|name| name.as_str())
|
||||
.map(|name| name.trim().to_string())
|
||||
.filter(|name| !name.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if !authors.is_empty() {
|
||||
meta.authors = authors;
|
||||
filled.push("authors");
|
||||
}
|
||||
}
|
||||
for (key, slot) in [
|
||||
("publisher", &mut meta.publisher),
|
||||
("date", &mut meta.date),
|
||||
("language", &mut meta.language),
|
||||
] {
|
||||
if missing.contains(&key) {
|
||||
if let Some(found) = text(&value, key) {
|
||||
*slot = Some(found);
|
||||
filled.push(match key {
|
||||
"publisher" => "publisher",
|
||||
"date" => "date",
|
||||
_ => "language",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !filled.is_empty() {
|
||||
meta.note(format!(
|
||||
"{} filled in from a model, prompt version {PROMPT_VERSION}",
|
||||
filled.join(", ")
|
||||
));
|
||||
}
|
||||
Ok(filled)
|
||||
}
|
||||
|
||||
/// The first balanced `{…}` in a reply.
|
||||
///
|
||||
/// Models wrap JSON in prose and in code fences however they were asked not
|
||||
/// to, and refusing a good answer over a fence helps nobody.
|
||||
fn object_in(reply: &str) -> Option<&str> {
|
||||
let start = reply.find('{')?;
|
||||
let bytes = reply.as_bytes();
|
||||
let mut depth = 0usize;
|
||||
let mut in_string = false;
|
||||
let mut escaped = false;
|
||||
for at in start..bytes.len() {
|
||||
let byte = bytes[at];
|
||||
if in_string {
|
||||
match (escaped, byte) {
|
||||
(true, _) => escaped = false,
|
||||
(false, b'\\') => escaped = true,
|
||||
(false, b'"') => in_string = false,
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match byte {
|
||||
b'"' => in_string = true,
|
||||
b'{' => depth += 1,
|
||||
b'}' => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
return Some(&reply[start..=at]);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::Source;
|
||||
|
||||
fn bare() -> Metadata {
|
||||
let mut meta = Metadata::empty("scan.pdf", Source::PdfInfo);
|
||||
meta.title = Some("Microsoft Word - ch3.doc".into());
|
||||
meta
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_answered_everything_is_not_worth_asking_about() {
|
||||
let mut meta = bare();
|
||||
meta.authors = vec!["Palmer".into()];
|
||||
meta.publisher = Some("Faber".into());
|
||||
meta.date = Some("1984".into());
|
||||
meta.language = Some("en".into());
|
||||
assert_eq!(compose(&meta, None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_prompt_says_what_is_known_and_what_is_wanted() {
|
||||
let prompt = compose(&bare(), Some("THE PHILOKALIA\n\nVolume 3")).expect("a prompt");
|
||||
assert!(prompt.contains("Microsoft Word - ch3.doc"), "{prompt}");
|
||||
assert!(
|
||||
prompt.contains("Missing: authors, publisher, date, language"),
|
||||
"{prompt}"
|
||||
);
|
||||
assert!(prompt.contains("THE PHILOKALIA"), "{prompt}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_record_composes_the_same_prompt_every_time() {
|
||||
assert_eq!(compose(&bare(), Some("x")), compose(&bare(), Some("x")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_excerpt_is_cut_rather_than_sent_whole() {
|
||||
let huge = "a".repeat(EXCERPT_CHARS * 3);
|
||||
let prompt = compose(&bare(), Some(&huge)).expect("a prompt");
|
||||
assert!(prompt.len() < EXCERPT_CHARS * 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reply_fills_in_only_what_was_missing() {
|
||||
let mut meta = bare();
|
||||
let filled = merge(
|
||||
&mut meta,
|
||||
r#"{"title":"The Philokalia","authors":["Palmer","Ware"],"publisher":"Faber","date":"1984","language":"en"}"#,
|
||||
)
|
||||
.expect("a reply");
|
||||
// The title was present, however bad, so the model does not overrule it.
|
||||
assert_eq!(meta.title.as_deref(), Some("Microsoft Word - ch3.doc"));
|
||||
assert_eq!(meta.authors, ["Palmer", "Ware"]);
|
||||
assert_eq!(filled, ["authors", "publisher", "date", "language"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_is_an_absence_and_not_a_value() {
|
||||
let mut meta = bare();
|
||||
let filled = merge(
|
||||
&mut meta,
|
||||
r#"{"authors":null,"publisher":"","date":"1984"}"#,
|
||||
)
|
||||
.expect("a reply");
|
||||
assert_eq!(filled, ["date"]);
|
||||
assert!(meta.authors.is_empty());
|
||||
assert_eq!(meta.publisher, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_wrapped_in_prose_or_a_fence_is_still_found() {
|
||||
for reply in [
|
||||
"Here you go:\n```json\n{\"date\":\"1984\"}\n```\nHope that helps.",
|
||||
"{\"date\":\"1984\"}",
|
||||
"Sure! {\"date\":\"1984\"}",
|
||||
] {
|
||||
let mut meta = bare();
|
||||
assert_eq!(
|
||||
merge(&mut meta, reply).expect("a reply"),
|
||||
["date"],
|
||||
"{reply}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_brace_inside_a_string_does_not_end_the_object() {
|
||||
let mut meta = bare();
|
||||
merge(&mut meta, r#"{"publisher":"Faber } Faber","date":"1984"}"#).expect("a reply");
|
||||
assert_eq!(meta.publisher.as_deref(), Some("Faber } Faber"));
|
||||
assert_eq!(meta.date.as_deref(), Some("1984"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reply_that_is_not_json_is_refused_rather_than_half_applied() {
|
||||
let mut meta = bare();
|
||||
assert!(merge(&mut meta, "I could not tell.").is_err());
|
||||
assert!(merge(&mut meta, "{not json}").is_err());
|
||||
assert!(meta.notes.is_empty());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user