booksniff, and one place for the parts every tool shares

2026-10-23 03:07:13 +00:00 · 2026-09-19 12:40:20 -04:00
parent 36afe3350b
commit 1071f0bed3
32 changed files with 1426 additions and 237 deletions
@@ -88,6 +88,17 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"

+[[package]]
+name = "booksniff"
+version = "0.1.0"
+dependencies = [
+ "catena-cli",
+ "clap",
+ "corpus-core",
+ "serde_json",
+ "textkit",
+]
+
 [[package]]
 name = "bstr"
 version = "1.13.1"
@@ -99,10 +110,20 @@ dependencies = [
 "serde_core",
 ]

+[[package]]
+name = "catena-cli"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "corpus-core",
+ "serde",
+]
+
 [[package]]
 name = "chunkmd"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "fancy-regex 0.16.2",
@@ -182,6 +203,7 @@ dependencies = [
 name = "cuenorm"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "cuekit",
@@ -191,6 +213,7 @@ dependencies = [
 name = "cuepack"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "cuekit",
@@ -201,6 +224,7 @@ dependencies = [
 name = "cueseg"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "cuekit",
@@ -268,6 +292,7 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 name = "mdscrub"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "regex",
@@ -290,6 +315,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 name = "parity"
 version = "0.1.0"
 dependencies = [
+ "booksniff",
 "chunkmd",
 "corpus-core",
 "cuekit",
@@ -306,6 +332,7 @@ dependencies = [
 name = "pdftriage"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "regex",
@@ -385,6 +412,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 name = "rrf"
 version = "0.1.0"
 dependencies = [
+ "catena-cli",
 "clap",
 "corpus-core",
 "serde_json",
@@ -10,6 +10,7 @@ license = "MPL-2.0"
 repository = "https://git.pumpin.online/funkpower/catena"

 [workspace.dependencies]
+catena-cli = { path = "crates/catena-cli", version = "0.1.0" }
 corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
 cuekit = { path = "crates/cuekit", version = "0.1.0" }
 textkit = { path = "crates/textkit", version = "0.1.0" }
@@ -20,6 +21,7 @@ cuenorm = { path = "crates/cuenorm", version = "0.1.0" }
 cueseg = { path = "crates/cueseg", version = "0.1.0" }
 rrf = { path = "crates/rrf", version = "0.1.0" }
 cuepack = { path = "crates/cuepack", version = "0.1.0" }
+booksniff = { path = "crates/booksniff", version = "0.1.0" }

 clap = { version = "4", features = ["derive"] }
 fancy-regex = "0.16"
@@ -53,6 +53,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done

 | | |
 |---|---|
+| `booksniff` | What a file actually is, whatever its name claims |
 | `mdscrub` | Converted markdown in, clean markdown out: residual HTML, control characters, headings, page markers |
 | `chunkmd` | Markdown into passages that carry their page, chapter and role |
 | `pdftriage` | Whether a PDF's text layer is healthy, garbled, sparse, permuted, or absent — and so whether OCR is worth its cost |
@@ -0,0 +1,20 @@
+[package]
+name = "booksniff"
+description = "Says what a file actually is, whatever its name claims."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+catena-cli.workspace = true
+clap.workspace = true
+corpus-core.workspace = true
+textkit.workspace = true
+
+[dependencies.serde_json]
+workspace = true
+
+[lints]
+workspace = true
@@ -0,0 +1,132 @@
+//! Reading a few bytes of a file without reading the file.
+//!
+//! Sniffing a format needs the first few bytes, and sometimes a structure at
+//! the end. A nine-hundred-megabyte scan should cost neither nine hundred
+//! megabytes of memory nor nine hundred megabytes of reading, so everything
+//! here works through a window rather than a buffer — and the same trait is
+//! implemented for a slice, so the tests need no files at all.
+
+use std::io::{Read, Seek, SeekFrom};
+
+/// Something a few bytes can be read out of, at a position.
+pub trait Bytes {
+    /// How long the whole thing is.
+    fn len(&self) -> u64;
+
+    /// Whether there is nothing at all.
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Read up to `into.len()` bytes from `offset`, returning how many were
+    /// read. Short at the end, zero past it.
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize>;
+
+    /// Read exactly `count` bytes from `offset`, or as many as there are.
+    fn slice(&self, offset: u64, count: usize) -> std::io::Result<Vec<u8>> {
+        let mut buffer = vec![0; count];
+        let read = self.read_at(offset, &mut buffer)?;
+        buffer.truncate(read);
+        Ok(buffer)
+    }
+
+    /// Whether the bytes at `offset` are exactly `magic`.
+    fn magic_at(&self, offset: u64, magic: &[u8]) -> bool {
+        self.slice(offset, magic.len())
+            .is_ok_and(|read| read == magic)
+    }
+}
+
+impl Bytes for &[u8] {
+    fn len(&self) -> u64 {
+        <[u8]>::len(self) as u64
+    }
+
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
+        let Ok(offset) = usize::try_from(offset) else {
+            return Ok(0);
+        };
+        if offset >= <[u8]>::len(self) {
+            return Ok(0);
+        }
+        let available = &self[offset..];
+        let take = available.len().min(into.len());
+        into[..take].copy_from_slice(&available[..take]);
+        Ok(take)
+    }
+}
+
+/// A file on disk, read through a window.
+pub struct File {
+    file: std::cell::RefCell<std::fs::File>,
+    len: u64,
+}
+
+impl File {
+    /// Open a file for sniffing.
+    pub fn open(path: &std::path::Path) -> std::io::Result<Self> {
+        let file = std::fs::File::open(path)?;
+        let len = file.metadata()?.len();
+        Ok(Self {
+            file: std::cell::RefCell::new(file),
+            len,
+        })
+    }
+}
+
+impl Bytes for File {
+    fn len(&self) -> u64 {
+        self.len
+    }
+
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
+        let mut file = self.file.borrow_mut();
+        file.seek(SeekFrom::Start(offset))?;
+        let mut read = 0;
+        while read < into.len() {
+            match file.read(&mut into[read..])? {
+                0 => break,
+                n => read += n,
+            }
+        }
+        Ok(read)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_slice_reads_from_the_middle() {
+        let bytes: &[u8] = b"hello world";
+        assert_eq!(bytes.slice(6, 5).unwrap(), b"world");
+        assert_eq!(Bytes::len(&bytes), 11);
+    }
+
+    #[test]
+    fn reading_past_the_end_is_short_rather_than_an_error() {
+        let bytes: &[u8] = b"abc";
+        assert_eq!(bytes.slice(1, 99).unwrap(), b"bc");
+        assert!(bytes.slice(99, 4).unwrap().is_empty());
+        assert!(bytes.slice(u64::MAX, 4).unwrap().is_empty());
+    }
+
+    #[test]
+    fn magic_matches_only_when_it_is_all_there() {
+        let bytes: &[u8] = b"%PDF-1.7";
+        assert!(bytes.magic_at(0, b"%PDF-"));
+        assert!(!bytes.magic_at(0, b"%PDG-"));
+        assert!(!bytes.magic_at(4, b"-1.7 more"));
+    }
+
+    #[test]
+    fn a_file_reads_the_same_as_a_slice() {
+        let path = std::env::temp_dir().join("booksniff-bytes-test");
+        std::fs::write(&path, b"hello world").expect("a writable path");
+        let file = File::open(&path).expect("the file opens");
+        assert_eq!(file.len(), 11);
+        assert_eq!(file.slice(6, 5).unwrap(), b"world");
+        assert!(file.magic_at(0, b"hello"));
+    }
+}
@@ -0,0 +1,335 @@
+//! Says what a file actually is, whatever its name claims.
+//!
+//! A library accumulates files that lie: a DjVu named `.pdf`, a Word document
+//! named `.epub`, an EPUB saved as `.zip` and then never opened again, a
+//! single PDF somebody zipped and renamed back to `.pdf`. Every one of those
+//! fails somewhere downstream, usually as an unhelpful error from a converter
+//! that was handed the wrong kind of file.
+//!
+//! The containers are unambiguous, so they are read rather than guessed at.
+
+pub mod bytes;
+pub mod mobi;
+pub mod zip;
+
+use corpus_core::{Doc, Kind};
+
+use crate::bytes::Bytes;
+
+/// DjVu's container magic: an IFF stream tagged for the format's originator.
+const DJVU: &[u8] = b"AT&TFORM";
+/// The OLE2 compound-file header every binary Word document starts with.
+const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
+/// A PDF announces its version in its first bytes.
+const PDF: &[u8] = b"%PDF-";
+
+/// Archive members that carry no content: resource forks and desktop clutter.
+const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
+
+/// Substrings in a name that mark a file as a sample rather than a book.
+const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
+
+/// Below this, a file is too small to be a book whatever it claims to be.
+const SMALLEST_BOOK: u64 = 4096;
+
+/// How much of the front is read to tell text from markup.
+const TEXT_WINDOW: usize = 4096;
+
+/// Identify a file.
+///
+/// `name` is used only for the extension it claims and for the words in it,
+/// both of which are reported as observations rather than believed.
+pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
+    let mut doc = Doc::new(name, Kind::Unknown, source.len() as i64);
+    doc.declared_ext = extension(name);
+
+    doc.kind = if source.magic_at(0, DJVU) {
+        Kind::Djvu
+    } else if source.magic_at(0, PDF) {
+        Kind::Pdf
+    } else if source.magic_at(0, OLE2) {
+        Kind::Ole2
+    } else if source.magic_at(0, zip::LOCAL_HEADER) {
+        archive(source, &mut doc)
+    } else if let Some(kind) = mobi::kind(source) {
+        kind
+    } else {
+        text(source, &mut doc)
+    };
+
+    if doc.kind == Kind::Unknown {
+        doc.note("no container or magic bytes this recognises");
+    }
+    if let Some(declared) = doc.declared_ext.as_deref()
+        && !names_the_same_thing(declared, doc.kind)
+    {
+        doc.note(format!(
+            "named .{declared}, but the bytes say {}",
+            spelling(doc.kind)
+        ));
+    }
+    if source.len() < SMALLEST_BOOK {
+        doc.note(format!(
+            "{} bytes, which is too small to be a book",
+            source.len()
+        ));
+    }
+    if let Some(marker) = sample_marker(name) {
+        doc.note(format!("the name says {marker:?}, so it may be a sample"));
+    }
+    doc
+}
+
+/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
+fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
+    let Some(names) = zip::members(source) else {
+        doc.note("a zip whose directory could not be read");
+        return Kind::Unknown;
+    };
+    let lower: Vec<String> = names.iter().map(|name| name.to_lowercase()).collect();
+
+    if lower.iter().any(|name| name == "mimetype")
+        && lower
+            .iter()
+            .any(|name| name.starts_with("meta-inf/container.xml"))
+    {
+        return Kind::Epub;
+    }
+    if let Some(pdf) = sole_pdf(&names) {
+        doc.note(format!("a zip holding one PDF, {pdf:?}"));
+        return Kind::ZipWrappedPdf;
+    }
+    if lower
+        .iter()
+        .any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
+    {
+        doc.note("a zip of HTML documents");
+        return Kind::Unknown;
+    }
+    doc.note(format!(
+        "a zip of {} members, none of them a book",
+        names.len()
+    ));
+    Kind::Unknown
+}
+
+/// The one PDF an archive exists to carry, if that is what it is.
+///
+/// Only an unambiguous wrapper qualifies. An archive holding several PDFs is
+/// a collection, and choosing among them would be a guess — better to say it
+/// is not a book than to name one chapter as the whole.
+pub fn sole_pdf(names: &[String]) -> Option<&str> {
+    let real: Vec<&String> = names
+        .iter()
+        .filter(|name| !name.ends_with('/'))
+        .filter(|name| {
+            let lower = name.to_lowercase();
+            !ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
+        })
+        .collect();
+    match real.as_slice() {
+        [only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
+        _ => None,
+    }
+}
+
+/// Text, markup, or nothing this knows.
+fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
+    let Ok(head) = source.slice(0, TEXT_WINDOW) else {
+        return Kind::Unknown;
+    };
+    if head.is_empty() {
+        doc.note("the file is empty");
+        return Kind::Unknown;
+    }
+    // A file with a null byte in its first few kilobytes is not text, and
+    // nothing else here is going to identify it either.
+    if head.contains(&0) {
+        return Kind::Unknown;
+    }
+    // The window may have cut a character in half, which is not the same
+    // thing as the file not being text.
+    let head = match std::str::from_utf8(&head) {
+        Ok(text) => text,
+        Err(e) if e.error_len().is_none() => {
+            std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
+        }
+        Err(_) => return Kind::Unknown,
+    };
+
+    let start = head
+        .trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}')
+        .to_lowercase();
+    if start.starts_with("<!doctype html")
+        || start.starts_with("<html")
+        || start.starts_with("<?xml")
+        || start.starts_with("<head")
+    {
+        return Kind::Html;
+    }
+    Kind::Txt
+}
+
+/// Whether an extension and a verdict are saying the same thing.
+fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
+    match kind {
+        Kind::Pdf => extension == "pdf",
+        Kind::Epub => extension == "epub",
+        Kind::Djvu => matches!(extension, "djvu" | "djv"),
+        Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
+        Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
+        Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
+        Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
+        Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
+        Kind::ZipWrappedPdf => extension == "zip",
+        // Nothing was recognised, so there is nothing to disagree with.
+        Kind::Unknown => true,
+    }
+}
+
+fn spelling(kind: Kind) -> &'static str {
+    match kind {
+        Kind::Pdf => "PDF",
+        Kind::Epub => "EPUB",
+        Kind::Djvu => "DjVu",
+        Kind::Azw3 => "AZW3",
+        Kind::Mobi => "MOBI",
+        Kind::Txt => "plain text",
+        Kind::Html => "HTML",
+        Kind::Ole2 => "an OLE2 document",
+        Kind::ZipWrappedPdf => "a zip holding one PDF",
+        Kind::Unknown => "nothing recognisable",
+    }
+}
+
+fn extension(name: &str) -> Option<String> {
+    std::path::Path::new(name)
+        .extension()
+        .map(|ext| ext.to_string_lossy().to_lowercase())
+}
+
+fn sample_marker(name: &str) -> Option<&'static str> {
+    let lower = std::path::Path::new(name)
+        .file_name()
+        .map(|name| name.to_string_lossy().to_lowercase())
+        .unwrap_or_default();
+    SAMPLE_MARKERS
+        .iter()
+        .find(|marker| lower.contains(**marker))
+        .copied()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn padded(head: &[u8]) -> Vec<u8> {
+        let mut out = head.to_vec();
+        out.resize(SMALLEST_BOOK as usize + 1, b'x');
+        out
+    }
+
+    fn kind_of(head: &[u8], name: &str) -> Kind {
+        sniff(&padded(head).as_slice(), name).kind
+    }
+
+    #[test]
+    fn the_magic_decides_not_the_name() {
+        assert_eq!(kind_of(b"%PDF-1.7\n", "book.epub"), Kind::Pdf);
+        assert_eq!(kind_of(DJVU, "scan.pdf"), Kind::Djvu);
+        assert_eq!(kind_of(OLE2, "book.epub"), Kind::Ole2);
+    }
+
+    #[test]
+    fn a_name_that_disagrees_with_the_bytes_is_noted() {
+        let doc = sniff(&padded(DJVU).as_slice(), "scan.pdf");
+        assert_eq!(doc.kind, Kind::Djvu);
+        assert!(
+            doc.notes.iter().any(|note| note.contains("named .pdf")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_name_that_agrees_is_not_remarked_on() {
+        let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "book.pdf");
+        assert!(doc.notes.is_empty(), "{:?}", doc.notes);
+    }
+
+    #[test]
+    fn text_and_markup_are_told_apart() {
+        assert_eq!(
+            kind_of(b"The mill stood at the northern end.", "a.txt"),
+            Kind::Txt
+        );
+        assert_eq!(kind_of(b"<!DOCTYPE html>\n<html>", "a.html"), Kind::Html);
+        assert_eq!(kind_of(b"\n\n  <html lang=\"en\">", "a.txt"), Kind::Html);
+        assert_eq!(kind_of(b"<?xml version=\"1.0\"?>", "a.xml"), Kind::Html);
+    }
+
+    #[test]
+    fn binary_rubbish_is_not_called_text() {
+        assert_eq!(kind_of(&[0x00, 0x01, 0x02, 0x03], "a.txt"), Kind::Unknown);
+        assert_eq!(kind_of(&[0xff, 0xfe, 0xfd], "a.txt"), Kind::Unknown);
+    }
+
+    #[test]
+    fn a_small_file_is_remarked_on() {
+        let doc = sniff(&b"%PDF-1.7".as_slice(), "tiny.pdf");
+        assert!(
+            doc.notes.iter().any(|note| note.contains("too small")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_name_that_says_sample_is_remarked_on() {
+        let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "Book - Sample Chapter.pdf");
+        assert!(
+            doc.notes.iter().any(|note| note.contains("sample")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn the_one_pdf_in_an_archive_is_the_book_and_several_are_not() {
+        let one = ["__MACOSX/._x".into(), "dir/".into(), "book.pdf".into()];
+        assert_eq!(sole_pdf(&one), Some("book.pdf"));
+
+        let several = ["one.pdf".to_string(), "two.pdf".to_string()];
+        assert_eq!(sole_pdf(&several), None);
+
+        let mixed = ["book.pdf".to_string(), "cover.jpg".to_string()];
+        assert_eq!(sole_pdf(&mixed), None);
+    }
+
+    #[test]
+    fn an_empty_file_is_unknown_and_says_so() {
+        let doc = sniff(&b"".as_slice(), "nothing.pdf");
+        assert_eq!(doc.kind, Kind::Unknown);
+        assert!(doc.notes.iter().any(|note| note.contains("empty")));
+        assert_eq!(doc.bytes, 0);
+    }
+
+    #[test]
+    fn the_declared_extension_is_reported_lowercased() {
+        assert_eq!(
+            sniff(&padded(b"%PDF-").as_slice(), "BOOK.PDF").declared_ext,
+            Some("pdf".to_string())
+        );
+        assert_eq!(
+            sniff(&padded(b"%PDF-").as_slice(), "no-extension").declared_ext,
+            None
+        );
+    }
+
+    #[test]
+    fn a_character_cut_in_half_by_the_window_does_not_make_it_binary() {
+        let mut head = "é".repeat(TEXT_WINDOW).into_bytes();
+        head.truncate(TEXT_WINDOW + 1);
+        assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
+    }
+}
@@ -0,0 +1,99 @@
+//! Says what a file actually is, whatever its name claims.
+
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use booksniff::bytes::File;
+use clap::Parser;
+use corpus_core::{Doc, Kind, exit, ndjson};
+
+/// Identify files by what is in them rather than by what they are called.
+///
+/// Writes one doc record per file. Reads only the few kilobytes it needs —
+/// the front of the file, and an archive's directory at the back — so a
+/// directory of scans costs the same as a directory of anything else.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Exit codes: 0 every file was identified, 1 at least one was not, 74 a file
+could not be read.
+
+Example:
+
+    booksniff library/*.pdf | jq -r 'select(.kind != \"pdf\") | .path'"
+)]
+struct Args {
+    /// Files to identify.
+    #[arg(required = true)]
+    paths: Vec<PathBuf>,
+
+    /// Print a line a person can read instead of the record.
+    #[arg(long)]
+    plain: bool,
+}
+
+fn main() -> ExitCode {
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
+    let mut out = std::io::BufWriter::new(std::io::stdout().lock());
+    let mut unknown = false;
+    let mut unreadable = false;
+
+    for path in &args.paths {
+        let name = path.display().to_string();
+        let doc = match File::open(path) {
+            Ok(file) => booksniff::sniff(&file, &name),
+            Err(e) => {
+                eprintln!("booksniff: {name}: {e}");
+                unreadable = true;
+                continue;
+            }
+        };
+        unknown |= doc.kind == Kind::Unknown;
+
+        let wrote = if args.plain {
+            plain(&mut out, &doc)
+        } else {
+            ndjson::write_line(&mut out, &doc).map_err(|e| match e {
+                ndjson::Error::Io(e) => e,
+                other => std::io::Error::other(other.to_string()),
+            })
+        };
+        if let Err(e) = wrote {
+            // A consumer that stopped reading is not a failure.
+            if e.kind() == std::io::ErrorKind::BrokenPipe {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("booksniff: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+
+    if let Err(e) = out.flush() {
+        if e.kind() == std::io::ErrorKind::BrokenPipe {
+            return ExitCode::from(exit::OK);
+        }
+        eprintln!("booksniff: {e}");
+        return ExitCode::from(exit::IO);
+    }
+
+    ExitCode::from(match (unreadable, unknown) {
+        (true, _) => exit::IO,
+        (false, true) => exit::NO,
+        (false, false) => exit::OK,
+    })
+}
+
+fn plain(out: &mut impl Write, doc: &Doc) -> std::io::Result<()> {
+    let kind = serde_json::to_string(&doc.kind).unwrap_or_default();
+    writeln!(out, "{}\t{}", kind.trim_matches('"'), doc.path)?;
+    for note in &doc.notes {
+        writeln!(out, "\t{note}")?;
+    }
+    Ok(())
+}
@@ -0,0 +1,108 @@
+//! Telling the two Kindle formats apart.
+//!
+//! Both are Palm databases with the same type and creator, so the extension
+//! is no help and neither is the first eight bytes. The difference is a
+//! version number inside the first record: six is the old format, eight is
+//! the one that carries real typesetting.
+
+use corpus_core::Kind;
+
+use crate::bytes::Bytes;
+
+/// Where a Palm database says what it holds.
+const TYPE_AT: u64 = 60;
+/// What a Kindle book says there.
+const BOOK_TYPE: &[u8] = b"BOOKMOBI";
+
+/// Where the first record's offset is written.
+const FIRST_RECORD_AT: u64 = 78;
+
+/// The MOBI header's own magic, at the start of the first record's header.
+const MOBI_MAGIC: &[u8] = b"MOBI";
+/// Where the MOBI header begins inside the first record.
+const MOBI_HEADER_AT: u64 = 16;
+/// How far into the MOBI header the format version is.
+const VERSION_AT: u64 = 36;
+/// The version that means the newer format.
+const KF8: u32 = 8;
+
+/// Which Kindle format this is, or `None` if it is not one.
+pub fn kind(source: &impl Bytes) -> Option<Kind> {
+    if !source.magic_at(TYPE_AT, BOOK_TYPE) {
+        return None;
+    }
+    Some(match version(source) {
+        Some(version) if version >= KF8 => Kind::Azw3,
+        _ => Kind::Mobi,
+    })
+}
+
+/// The format version in the first record, when it is readable.
+fn version(source: &impl Bytes) -> Option<u32> {
+    let record = big_endian(source, FIRST_RECORD_AT)? as u64;
+    let header = record + MOBI_HEADER_AT;
+    if !source.magic_at(header, MOBI_MAGIC) {
+        return None;
+    }
+    big_endian(source, header + VERSION_AT)
+}
+
+fn big_endian(source: &impl Bytes, at: u64) -> Option<u32> {
+    let bytes = source.slice(at, 4).ok()?;
+    (bytes.len() == 4).then(|| u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A Palm database holding one record with a MOBI header of `version`.
+    fn book(version: Option<u32>) -> Vec<u8> {
+        let record_at: u32 = 1024;
+        let mut out = vec![0u8; record_at as usize];
+        out[TYPE_AT as usize..TYPE_AT as usize + 8].copy_from_slice(BOOK_TYPE);
+        out[FIRST_RECORD_AT as usize..FIRST_RECORD_AT as usize + 4]
+            .copy_from_slice(&record_at.to_be_bytes());
+
+        if let Some(version) = version {
+            out.resize(record_at as usize + MOBI_HEADER_AT as usize, 0);
+            out.extend_from_slice(MOBI_MAGIC);
+            out.resize(
+                record_at as usize + MOBI_HEADER_AT as usize + VERSION_AT as usize,
+                0,
+            );
+            out.extend_from_slice(&version.to_be_bytes());
+        }
+        out
+    }
+
+    #[test]
+    fn the_newer_format_is_recognised() {
+        assert_eq!(kind(&book(Some(8)).as_slice()), Some(Kind::Azw3));
+        assert_eq!(kind(&book(Some(9)).as_slice()), Some(Kind::Azw3));
+    }
+
+    #[test]
+    fn the_older_format_is_recognised() {
+        assert_eq!(kind(&book(Some(6)).as_slice()), Some(Kind::Mobi));
+    }
+
+    #[test]
+    fn a_book_with_no_readable_version_is_taken_for_the_older_format() {
+        assert_eq!(kind(&book(None).as_slice()), Some(Kind::Mobi));
+    }
+
+    #[test]
+    fn something_that_is_not_a_kindle_book_is_not_one() {
+        assert_eq!(kind(&b"%PDF-1.7".as_slice()), None);
+        assert_eq!(kind(&vec![0u8; 4096].as_slice()), None);
+    }
+
+    #[test]
+    fn a_truncated_book_does_not_panic() {
+        let whole = book(Some(8));
+        for cut in 0..whole.len() {
+            let _ = kind(&&whole[..cut]);
+        }
+    }
+}
@@ -0,0 +1,158 @@
+//! Listing what is inside a zip, without unpacking any of it.
+//!
+//! An EPUB, an htmlz and a PDF someone zipped and renamed are all zips, and
+//! telling them apart needs only the member names. Those live in the central
+//! directory at the end of the file, so reading them costs a few kilobytes
+//! whatever the archive weighs — and nothing is ever decompressed, which
+//! means a malicious archive cannot expand into anything here.
+
+use crate::bytes::Bytes;
+
+/// The end-of-central-directory record's signature.
+const END_OF_DIRECTORY: &[u8] = b"PK\x05\x06";
+/// A central directory entry's signature.
+const ENTRY: &[u8] = b"PK\x01\x02";
+/// A local file header, which is what a zip starts with.
+pub const LOCAL_HEADER: &[u8] = b"PK\x03\x04";
+
+/// How far back from the end the trailing record may be.
+///
+/// Its own size plus the largest comment a zip may carry.
+const SEARCH_BACK: u64 = 22 + 0xffff;
+
+/// Most members this will list.
+///
+/// A book in an archive has tens of members; an archive with a hundred
+/// thousand is not a book, and reading all of its names would be the only
+/// expensive thing this program does.
+const MOST_MEMBERS: usize = 100_000;
+
+/// The names of an archive's members, or `None` if it is not a zip this can
+/// read.
+pub fn members(source: &impl Bytes) -> Option<Vec<String>> {
+    if !source.magic_at(0, LOCAL_HEADER) {
+        return None;
+    }
+    let (count, mut at) = directory_start(source)?;
+
+    let mut names = Vec::with_capacity(count.min(1024));
+    for _ in 0..count.min(MOST_MEMBERS) {
+        let header = source.slice(at, 46).ok()?;
+        if header.len() < 46 || header[..4] != *ENTRY {
+            break;
+        }
+        let name_len = u16::from_le_bytes([header[28], header[29]]) as u64;
+        let extra_len = u16::from_le_bytes([header[30], header[31]]) as u64;
+        let comment_len = u16::from_le_bytes([header[32], header[33]]) as u64;
+
+        let name = source.slice(at + 46, name_len as usize).ok()?;
+        // Member names are bytes, and an archive written on another machine
+        // may not have written UTF-8. What is wanted here is a comparison
+        // against ASCII, so anything else may as well be a replacement
+        // character.
+        names.push(String::from_utf8_lossy(&name).into_owned());
+        at += 46 + name_len + extra_len + comment_len;
+    }
+    Some(names)
+}
+
+/// How many entries the central directory has and where it starts.
+fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
+    let len = source.len();
+    let window = SEARCH_BACK.min(len);
+    let from = len - window;
+    let tail = source.slice(from, window as usize).ok()?;
+
+    // The record is at the end, after a comment of unknown length, so it is
+    // found by searching backwards for its signature.
+    let at = tail
+        .windows(4)
+        .rposition(|window| window == END_OF_DIRECTORY)?;
+    let record = &tail[at..];
+    if record.len() < 22 {
+        return None;
+    }
+    let count = u16::from_le_bytes([record[10], record[11]]) as usize;
+    let offset = u32::from_le_bytes([record[16], record[17], record[18], record[19]]) as u64;
+    (offset < len).then_some((count, offset))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Build a zip holding `names`, each with empty stored content.
+    fn zip(names: &[&str]) -> Vec<u8> {
+        let mut out: Vec<u8> = Vec::new();
+        let mut offsets = Vec::new();
+        for name in names {
+            offsets.push(out.len() as u32);
+            out.extend_from_slice(LOCAL_HEADER);
+            out.extend_from_slice(&[0; 22]); // version .. uncompressed size
+            out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+            out.extend_from_slice(&0u16.to_le_bytes());
+            out.extend_from_slice(name.as_bytes());
+        }
+        let directory_at = out.len() as u32;
+        for (name, offset) in names.iter().zip(&offsets) {
+            out.extend_from_slice(ENTRY);
+            out.extend_from_slice(&[0; 24]); // version .. uncompressed size
+            out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+            out.extend_from_slice(&0u16.to_le_bytes()); // extra
+            out.extend_from_slice(&0u16.to_le_bytes()); // comment
+            out.extend_from_slice(&[0; 8]); // disk, attributes
+            out.extend_from_slice(&offset.to_le_bytes());
+            out.extend_from_slice(name.as_bytes());
+        }
+        out.extend_from_slice(END_OF_DIRECTORY);
+        out.extend_from_slice(&[0; 4]); // disk numbers
+        out.extend_from_slice(&(names.len() as u16).to_le_bytes());
+        out.extend_from_slice(&(names.len() as u16).to_le_bytes());
+        out.extend_from_slice(&((out.len() as u32).saturating_sub(directory_at)).to_le_bytes());
+        out.extend_from_slice(&directory_at.to_le_bytes());
+        out.extend_from_slice(&0u16.to_le_bytes()); // comment length
+        out
+    }
+
+    #[test]
+    fn the_member_names_come_back() {
+        let archive = zip(&["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]);
+        let names = members(&archive.as_slice()).expect("a zip");
+        assert_eq!(
+            names,
+            ["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]
+        );
+    }
+
+    #[test]
+    fn an_archive_with_no_members_is_not_read_as_one() {
+        // It has no local header to start with, so nothing identifies it as
+        // an archive in the first place — and an empty archive is not a book
+        // under any reading.
+        assert_eq!(members(&zip(&[]).as_slice()), None);
+    }
+
+    #[test]
+    fn something_that_is_not_a_zip_is_not_read_as_one() {
+        assert_eq!(members(&b"%PDF-1.7".as_slice()), None);
+        assert_eq!(members(&b"".as_slice()), None);
+    }
+
+    #[test]
+    fn a_truncated_archive_does_not_panic() {
+        let archive = zip(&["a.pdf", "b.pdf"]);
+        for cut in 0..archive.len() {
+            let _ = members(&&archive[..cut]);
+        }
+    }
+
+    #[test]
+    fn a_count_that_lies_stops_at_the_first_bad_entry() {
+        let mut archive = zip(&["a.pdf"]);
+        let at = archive.len() - 12;
+        // Claim a thousand entries where there is one.
+        archive[at - 6..at - 4].copy_from_slice(&1000u16.to_le_bytes());
+        let names = members(&archive.as_slice()).expect("a zip");
+        assert_eq!(names, ["a.pdf"]);
+    }
+}
@@ -0,0 +1,95 @@
+//! The tool as a caller meets it: files in, records and an exit code out.
+
+use std::path::PathBuf;
+use std::process::{Command, Output};
+
+fn run(args: &[&str]) -> Output {
+    Command::new(env!("CARGO_BIN_EXE_booksniff"))
+        .args(args)
+        .output()
+        .expect("the binary should run")
+}
+
+fn written(name: &str, bytes: &[u8]) -> PathBuf {
+    let dir = std::env::temp_dir().join("booksniff-tests");
+    std::fs::create_dir_all(&dir).expect("a writable temp directory");
+    let path = dir.join(name);
+    std::fs::write(&path, bytes).expect("a writable path");
+    path
+}
+
+fn padded(head: &[u8]) -> Vec<u8> {
+    let mut out = head.to_vec();
+    out.resize(8192, b'x');
+    out
+}
+
+#[test]
+fn a_file_it_recognises_comes_back_as_a_record() {
+    let path = written("book.pdf", &padded(b"%PDF-1.7\n"));
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(0));
+
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["kind"], "pdf");
+    assert_eq!(doc["declared_ext"], "pdf");
+    assert_eq!(doc["bytes"], 8192);
+    assert!(out.stderr.is_empty());
+}
+
+#[test]
+fn a_file_it_cannot_place_answers_no() {
+    let path = written("mystery.bin", &padded(&[0x00, 0x01, 0x02]));
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(1), "unknown must answer no");
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["kind"], "unknown");
+}
+
+#[test]
+fn several_files_come_back_as_several_records() {
+    let one = written("one.pdf", &padded(b"%PDF-1.7"));
+    let two = written("two.djvu", &padded(b"AT&TFORM"));
+    let out = run(&[
+        one.to_str().expect("utf-8 path"),
+        two.to_str().expect("utf-8 path"),
+    ]);
+    let kinds: Vec<String> = String::from_utf8(out.stdout)
+        .expect("utf-8")
+        .lines()
+        .map(|line| {
+            serde_json::from_str::<serde_json::Value>(line).expect("valid JSON")["kind"].to_string()
+        })
+        .collect();
+    assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]);
+}
+
+#[test]
+fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() {
+    let path = written("kept.pdf", &padded(b"%PDF-1.7"));
+    let out = run(&[
+        "/definitely/not/here.pdf",
+        path.to_str().expect("utf-8 path"),
+    ]);
+    assert_eq!(out.status.code(), Some(74));
+    assert!(!out.stderr.is_empty());
+    // The readable file was still identified, on stdout, uncontaminated.
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["kind"], "pdf");
+}
+
+#[test]
+fn the_plain_form_is_for_a_person() {
+    let path = written("lying.pdf", &padded(b"AT&TFORM"));
+    let out = run(&["--plain", path.to_str().expect("utf-8 path")]);
+    let said = String::from_utf8(out.stdout).expect("utf-8");
+    assert!(said.starts_with("djvu\t"), "{said}");
+    assert!(said.contains("named .pdf"), "{said}");
+}
+
+#[test]
+fn no_files_named_is_a_usage_error() {
+    let out = run(&[]);
+    assert_eq!(out.status.code(), Some(64), "usage errors exit 64");
+    assert!(out.stdout.is_empty());
+}
@@ -0,0 +1,16 @@
+[package]
+name = "catena-cli"
+description = "The parts of a filter that every tool in this workspace does the same way."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+serde.workspace = true
+
+[lints]
+workspace = true
@@ -0,0 +1,162 @@
+//! The parts of a filter that every tool here does the same way.
+//!
+//! Not a framework — three functions, each of which exists because getting
+//! it wrong is easy and the consequence is a pipeline that misreports why it
+//! stopped:
+//!
+//! - a usage error exits 64, not whatever the argument parser felt like;
+//! - a consumer that closed the pipe is the end of the job, not a failure;
+//! - a record is flushed as it is written, so a slow stage downstream sees it
+//!   immediately.
+
+use std::io::{BufWriter, Write};
+use std::process::ExitCode;
+
+use corpus_core::{exit, ndjson};
+use serde::Serialize;
+
+/// Parse the command line, or exit.
+///
+/// `--help` and `--version` are not errors and leave with 0; anything else
+/// wrong with the arguments leaves with 64, which is what the rest of the
+/// suite means by "the command line was wrong".
+pub fn parse<T: clap::Parser>() -> Result<T, ExitCode> {
+    T::try_parse().map_err(|e| {
+        let _ = e.print();
+        match e.use_stderr() {
+            true => ExitCode::from(exit::USAGE),
+            false => ExitCode::from(exit::OK),
+        }
+    })
+}
+
+/// Write records to stdout as NDJSON, one per line, flushed as they go.
+pub fn stream<'a, T: Serialize + 'a>(
+    tool: &str,
+    records: impl IntoIterator<Item = &'a T>,
+) -> ExitCode {
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    for record in records {
+        match ndjson::write_line(&mut out, record) {
+            Ok(()) => {}
+            // A consumer that stopped reading is not a failure: `… | head` is
+            // a normal way to look at a stream.
+            Err(e) if e.is_broken_pipe() => return ExitCode::from(exit::OK),
+            Err(e) => {
+                eprintln!("{tool}: {e}");
+                return ExitCode::from(exit::IO);
+            }
+        }
+    }
+    finish(tool, out)
+}
+
+/// Write text to stdout and leave.
+pub fn text(tool: &str, text: &str) -> ExitCode {
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    match out.write_all(text.as_bytes()) {
+        Ok(()) => finish(tool, out),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("{tool}: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Flush what is buffered, treating a closed consumer as the end of the job.
+pub fn finish(tool: &str, mut out: impl Write) -> ExitCode {
+    match out.flush() {
+        Ok(()) => ExitCode::from(exit::OK),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("{tool}: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Read the whole input as text, telling a file that cannot be read from one
+/// that is not text.
+///
+/// The two are different answers: the first is an I/O failure and the second
+/// is bad input, and a caller routing on the exit code needs to tell them
+/// apart.
+pub fn read_text(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
+    let bytes = read_bytes(path)?;
+    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
+}
+
+/// Read the whole input, replacing anything that is not UTF-8.
+///
+/// For the formats that routinely arrive in another encoding — a subtitle
+/// file downloaded from anywhere is as likely to be Latin-1 — where a cue
+/// with one wrong character still has its timing and most of its words.
+pub fn read_lossy(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
+    Ok(String::from_utf8_lossy(&read_bytes(path)?).into_owned())
+}
+
+fn read_bytes(path: Option<&std::path::Path>) -> Result<Vec<u8>, (u8, String)> {
+    use std::io::Read as _;
+
+    match path {
+        Some(path) => {
+            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
+        }
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin()
+                .read_to_end(&mut bytes)
+                .map(|_| bytes)
+                .map_err(|e| (exit::IO, e.to_string()))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn reading_a_file_that_is_not_there_is_an_io_failure() {
+        let missing = std::path::Path::new("/definitely/not/here");
+        let (code, _) = read_text(Some(missing)).expect_err("should fail");
+        assert_eq!(code, exit::IO);
+    }
+
+    #[test]
+    fn reading_bytes_that_are_not_text_is_a_data_failure() {
+        let path = std::env::temp_dir().join("catena-cli-not-text");
+        std::fs::write(&path, [0xff, 0xfe]).expect("a writable path");
+        let (code, why) = read_text(Some(&path)).expect_err("should fail");
+        assert_eq!(code, exit::DATA);
+        assert!(why.contains("UTF-8"), "{why}");
+    }
+
+    #[test]
+    fn the_lossy_read_takes_what_the_strict_one_refuses() {
+        let path = std::env::temp_dir().join("catena-cli-lossy");
+        std::fs::write(&path, [b'a', 0xff, b'b']).expect("a writable path");
+        assert_eq!(read_lossy(Some(&path)).expect("should read"), "a\u{fffd}b");
+    }
+
+    #[test]
+    fn a_closed_consumer_is_the_end_of_the_job_and_not_a_failure() {
+        struct Closed;
+        impl Write for Closed {
+            fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
+                Ok(0)
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::BrokenPipe,
+                    "broken pipe",
+                ))
+            }
+        }
+        assert_eq!(
+            format!("{:?}", finish("test", Closed)),
+            format!("{:?}", ExitCode::from(exit::OK))
+        );
+    }
+}
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 fancy-regex.workspace = true
@@ -1,13 +1,12 @@
 //! Cuts markdown into retrievable passages that carry their page, chapter
 //! and role.

-use std::io::{BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::process::ExitCode;

 use chunkmd::Budgets;
 use clap::Parser;
-use corpus_core::{exit, ndjson};
+use corpus_core::exit;

 /// Cut markdown into chunks, one JSON record per line.
 ///
@@ -48,7 +47,10 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
    let budgets = Budgets {
        target: args.target,
        overlap: args.overlap,
@@ -60,7 +62,7 @@ fn main() -> ExitCode {
        return ExitCode::from(exit::USAGE);
    }

-    let markdown = match read(args.path.as_deref()) {
+    let markdown = match catena_cli::read_text(args.path.as_deref()) {
        Ok(markdown) => markdown,
        Err((code, why)) => {
            eprintln!("chunkmd: {why}");
@@ -68,26 +70,7 @@ fn main() -> ExitCode {
        }
    };

-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for chunk in chunkmd::chunk(&markdown, &budgets) {
-        if let Err(e) = ndjson::write_line(&mut out, &chunk) {
-            // A consumer that stopped reading is not a failure: `| head` is a
-            // normal way to look at a stream.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("chunkmd: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("chunkmd: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("chunkmd", &chunkmd::chunk(&markdown, &budgets))
 }

 /// Budgets that cannot produce anything sensible, rejected before reading
@@ -116,22 +99,3 @@ fn sane(budgets: &Budgets) -> Result<(), String> {
    }
    Ok(())
 }
-
-/// Read the input, distinguishing a file that cannot be read from one that
-/// is not text: the first is an I/O failure and the second is bad input, and
-/// a caller routing on the exit code needs to tell them apart.
-fn read(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
-    let bytes = match path {
-        Some(path) => {
-            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
-        }
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin()
-                .read_to_end(&mut bytes)
-                .map(|_| bytes)
-                .map_err(|e| (exit::IO, e.to_string()))
-        }
-    }?;
-    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
-}
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true
@@ -1,11 +1,9 @@
 //! Any subtitle format in, one normalised cue stream out.

-use std::io::{BufWriter, Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;

 use clap::Parser;
-use corpus_core::{exit, ndjson};

 /// Read a subtitle file — SubRip, WebVTT or SubStation Alpha — and write one
 /// cue per line as JSON.
@@ -35,13 +33,16 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };

-    let content = match read(args.path.as_deref()) {
+    let content = match catena_cli::read_lossy(args.path.as_deref()) {
        Ok(content) => content,
-        Err(e) => {
-            eprintln!("cuenorm: {e}");
-            return ExitCode::from(exit::IO);
+        Err((code, why)) => {
+            eprintln!("cuenorm: {why}");
+            return ExitCode::from(code);
        }
    };

@@ -52,42 +53,5 @@ fn main() -> ExitCode {
        cuekit::merge_repeats(&cues)
    };

-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for cue in &cues {
-        if let Err(e) = ndjson::write_line(&mut out, cue) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("cuenorm: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("cuenorm: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
-}
-
-/// Read the file as text, tolerating the encodings a subtitle file arrives in.
-///
-/// Not UTF-8 is common: a subtitle file downloaded from anywhere is as likely
-/// to be Latin-1, and refusing it would make the tool useless for exactly the
-/// archives that need it. Undecodable bytes become the replacement character
-/// rather than an error, because a cue with one wrong character still has its
-/// timing and most of its words.
-fn read(path: Option<&Path>) -> std::io::Result<String> {
-    let bytes = match path {
-        Some(path) => std::fs::read(path)?,
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin().read_to_end(&mut bytes)?;
-            bytes
-        }
-    };
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
+    catena_cli::stream("cuenorm", &cues)
 }
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true
@@ -1,6 +1,5 @@
 //! Packs cues into passages worth embedding.

-use std::io::{BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::process::ExitCode;

@@ -38,7 +37,10 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
    let budgets = Budgets {
        seconds: args.seconds,
        max_tokens: args.max_tokens,
@@ -48,11 +50,11 @@ fn main() -> ExitCode {
        return ExitCode::from(exit::USAGE);
    }

-    let content = match read(args.path.as_deref()) {
+    let content = match catena_cli::read_lossy(args.path.as_deref()) {
        Ok(content) => content,
-        Err(e) => {
-            eprintln!("cuepack: {e}");
-            return ExitCode::from(exit::IO);
+        Err((code, why)) => {
+            eprintln!("cuepack: {why}");
+            return ExitCode::from(code);
        }
    };

@@ -64,25 +66,7 @@ fn main() -> ExitCode {
        }
    };

-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for passage in cuepack::pack(&cues, &budgets) {
-        if let Err(e) = ndjson::write_line(&mut out, &passage) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("cuepack: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("cuepack: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("cuepack", &cuepack::pack(&cues, &budgets))
 }

 /// Read cues from either encoding: a cue stream begins with a JSON object
@@ -97,15 +81,3 @@ fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
    }
    Ok(cuekit::parse(content))
 }
-
-fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
-    let bytes = match path {
-        Some(path) => std::fs::read(path)?,
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin().read_to_end(&mut bytes)?;
-            bytes
-        }
-    };
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
-}
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true
@@ -1,7 +1,6 @@
 //! Turns a timed transcript into topic chapters.

-use std::io::{BufWriter, Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;

 use clap::Parser;
@@ -38,13 +37,16 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };

-    let content = match read(args.path.as_deref()) {
+    let content = match catena_cli::read_lossy(args.path.as_deref()) {
        Ok(content) => content,
-        Err(e) => {
-            eprintln!("cueseg: {e}");
-            return ExitCode::from(exit::IO);
+        Err((code, why)) => {
+            eprintln!("cueseg: {why}");
+            return ExitCode::from(code);
        }
    };

@@ -65,25 +67,7 @@ fn main() -> ExitCode {
        None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max),
    };

-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for chapter in cueseg::segment(&cues, duration) {
-        if let Err(e) = ndjson::write_line(&mut out, &chapter) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("cueseg: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("cueseg: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("cueseg", &cueseg::segment(&cues, duration))
 }

 /// Read cues from either encoding.
@@ -101,15 +85,3 @@ fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
    }
    Ok(cuekit::parse(content))
 }
-
-fn read(path: Option<&Path>) -> std::io::Result<String> {
-    let bytes = match path {
-        Some(path) => std::fs::read(path)?,
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin().read_to_end(&mut bytes)?;
-            bytes
-        }
-    };
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
-}
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 regex.workspace = true
@@ -1,11 +1,9 @@
 //! Cleans converted markdown.

-use std::io::{Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;

 use clap::Parser;
-use corpus_core::exit;

 /// Clean converted markdown: drop residual HTML and control characters,
 /// make headings readable, and spell every page boundary the same way.
@@ -27,9 +25,12 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };

-    let markdown = match read(args.path.as_deref()) {
+    let markdown = match catena_cli::read_text(args.path.as_deref()) {
        Ok(markdown) => markdown,
        Err((code, why)) => {
            eprintln!("mdscrub: {why}");
@@ -37,33 +38,5 @@ fn main() -> ExitCode {
        }
    };

-    let cleaned = mdscrub::scrub(&markdown);
-    match std::io::stdout().write_all(cleaned.as_bytes()) {
-        Ok(()) => ExitCode::from(exit::OK),
-        // A consumer that stopped reading is not a failure.
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("mdscrub: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
-}
-
-/// Read the input, distinguishing a file that cannot be read from one that
-/// is not text: the first is an I/O failure and the second is bad input, and
-/// a caller routing on the exit code needs to tell them apart.
-fn read(path: Option<&Path>) -> Result<String, (u8, String)> {
-    let bytes = match path {
-        Some(path) => {
-            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
-        }
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin()
-                .read_to_end(&mut bytes)
-                .map(|_| bytes)
-                .map_err(|e| (exit::IO, e.to_string()))
-        }
-    }?;
-    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
+    catena_cli::text("mdscrub", &mdscrub::scrub(&markdown))
 }
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 regex.workspace = true
@@ -49,7 +49,10 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };

    let bytes = match std::fs::metadata(&args.path) {
        Ok(meta) => meta.len() as i64,
@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true

 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 serde_json.workspace = true
@@ -1,6 +1,5 @@
 //! Reciprocal rank fusion: several ranked lists in, one ranked list out.

-use std::io::{BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::process::ExitCode;

@@ -49,7 +48,10 @@ struct Args {
 }

 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };

    let weights = match weights(&args) {
        Ok(weights) => weights,
@@ -59,24 +61,14 @@ fn main() -> ExitCode {
        }
    };

-    let hits: Result<Vec<Hit>, _> = match args.path.as_deref() {
-        Some(path) => match std::fs::File::open(path) {
-            Ok(file) => ndjson::read(std::io::BufReader::new(file)).collect(),
-            Err(e) => {
-                eprintln!("rrf: {}: {e}", path.display());
-                return ExitCode::from(exit::IO);
-            }
-        },
-        None => {
-            let mut text = Vec::new();
-            if let Err(e) = std::io::stdin().read_to_end(&mut text) {
-                eprintln!("rrf: {e}");
-                return ExitCode::from(exit::IO);
-            }
-            ndjson::read(text.as_slice()).collect()
+    let text = match catena_cli::read_text(args.path.as_deref()) {
+        Ok(text) => text,
+        Err((code, why)) => {
+            eprintln!("rrf: {why}");
+            return ExitCode::from(code);
        }
    };
-    let hits = match hits {
+    let hits = match ndjson::read(text.as_bytes()).collect::<Result<Vec<Hit>, _>>() {
        Ok(hits) => hits,
        Err(e) => {
            eprintln!("rrf: {e}");
@@ -91,25 +83,7 @@ fn main() -> ExitCode {
    let fused = rrf::fuse(&hits, &weights);
    let fused = &fused[..args.limit.unwrap_or(fused.len()).min(fused.len())];

-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for hit in fused {
-        if let Err(e) = ndjson::write_line(&mut out, hit) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("rrf: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("rrf: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("rrf", fused)
 }

 fn weights(args: &Args) -> Result<Weights, String> {
@@ -14,6 +14,7 @@ publish = false
 python = []

 [dependencies]
+booksniff.workspace = true
 chunkmd.workspace = true
 cuekit.workspace = true
 cuepack.workspace = true
@@ -0,0 +1,20 @@
+{"name": "plain.pdf", "hex": "255044462d312e370a25e2e3cfd30a78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878"}
+{"name": "scan.pdf", "hex": "41542654464f524d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}
+{"name": "book.epub", "hex": "d0cf11e0a1b11ae100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}
+{"name": "notes.doc", "hex": "5468652050726f6a20706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f63"}
+{"name": "empty.pdf", "hex": ""}
+{"name": "tiny.txt", "hex": "6869"}
+{"name": "prose.txt", "hex": "546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a"}
+{"name": "page.html", "hex": "3c21444f43545950452068746d6c3e0a3c68746d6c3e3c626f64793e746578743c2f626f64793e3c2f68746d6c3e"}
+{"name": "spine.xhtml", "hex": "3c3f786d6c2076657273696f6e3d22312e30223f3e0a3c68746d6c20786d6c6e733d2278223e3c626f64792f3e3c2f68746d6c3e"}
+{"name": "real.epub", "hex": "504b03041400000000009164335d6f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b03041400000000009164335d54995f930c0000000c000000160000004d4554412d494e462f636f6e7461696e65722e786d6c3c636f6e7461696e65722f3e504b03041400000000009164335d9b7d083a0a0000000a0000000e0000004f454250532f626f6f6b2e6f70663c7061636b6167652f3e504b010214031400000000009164335d6f61ab2c14000000140000000800000000000000000000008001000000006d696d6574797065504b010214031400000000009164335d54995f930c0000000c00000016000000000000000000000080013a0000004d4554412d494e462f636f6e7461696e65722e786d6c504b010214031400000000009164335d9b7d083a0a0000000a0000000e000000000000000000000080017a0000004f454250532f626f6f6b2e6f7066504b05060000000003000300b6000000b00000000000"}
+{"name": "saved.zip", "hex": "504b03041400000000009164335d6f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b03041400000000009164335d54995f930c0000000c000000160000004d4554412d494e462f636f6e7461696e65722e786d6c3c636f6e7461696e65722f3e504b010214031400000000009164335d6f61ab2c14000000140000000800000000000000000000008001000000006d696d6574797065504b010214031400000000009164335d54995f930c0000000c00000016000000000000000000000080013a0000004d4554412d494e462f636f6e7461696e65722e786d6c504b050600000000020002007a0000007a0000000000"}
+{"name": "wrapped.pdf", "hex": "504b03041400000000009164335da36258a4100000001000000008000000626f6f6b2e706466255044462d312e3720636f6e74656e74504b010214031400000000009164335da36258a41000000010000000080000000000000000000000800100000000626f6f6b2e706466504b0506000000000100010036000000360000000000"}
+{"name": "wrapped-noisy.zip", "hex": "504b03041400000000009164335d399cfb060400000004000000130000005f5f4d41434f53582f2e5f626f6f6b2e7064666a756e6b504b03041400000000009164335d399cfb060400000004000000090000002e44535f53746f72656a756e6b504b03041400000000009164335da36258a4100000001000000008000000626f6f6b2e706466255044462d312e3720636f6e74656e74504b010214031400000000009164335d399cfb0604000000040000001300000000000000000000008001000000005f5f4d41434f53582f2e5f626f6f6b2e706466504b010214031400000000009164335d399cfb0604000000040000000900000000000000000000008001350000002e44535f53746f7265504b010214031400000000009164335da36258a41000000010000000080000000000000000000000800160000000626f6f6b2e706466504b05060000000003000300ae000000960000000000"}
+{"name": "collection.zip", "hex": "504b03041400000000009164335d43beb7e80100000001000000070000006f6e652e70646661504b03041400000000009164335df9efbe7101000000010000000700000074776f2e70646662504b010214031400000000009164335d43beb7e801000000010000000700000000000000000000008001000000006f6e652e706466504b010214031400000000009164335df9efbe71010000000100000007000000000000000000000080012600000074776f2e706466504b050600000000020002006a0000004c0000000000"}
+{"name": "pages.zip", "hex": "504b03041400000000009164335d3ce3a64107000000070000000a000000696e6465782e68746d6c3c68746d6c2f3e504b03041400000000009164335d8316dc8c0100000001000000090000007374796c652e63737378504b010214031400000000009164335d3ce3a64107000000070000000a0000000000000000000000800100000000696e6465782e68746d6c504b010214031400000000009164335d8316dc8c010000000100000009000000000000000000000080012f0000007374796c652e637373504b050600000000020002006f000000570000000000"}
+{"name": "nothing.zip", "hex": "504b03041400000000009164335d8316dc8c01000000010000000a000000726561646d652e74787478504b010214031400000000009164335d8316dc8c01000000010000000a0000000000000000000000800100000000726561646d652e747874504b0506000000000100010038000000290000000000"}
+{"name": "uppercase.zip", "hex": "504b03041400000000009164335d18eb5070080000000800000008000000424f4f4b2e504446255044462d312e37504b010214031400000000009164335d18eb50700800000008000000080000000000000000000000800100000000424f4f4b2e504446504b05060000000001000100360000002e0000000000"}
+{"name": "subdir.zip", "hex": "504b03041400000000009164335d18eb507008000000080000000c0000006469722f626f6f6b2e706466255044462d312e37504b010214031400000000009164335d18eb507008000000080000000c00000000000000000000008001000000006469722f626f6f6b2e706466504b050600000000010001003a000000320000000000"}
+{"name": "truncated.zip", "hex": "504b03041400000000009164335d18eb5070080000000800000008000000626f6f6b2e7064662550"}
+{"name": "Book - Sample Chapter.pdf", "hex": "255044462d312e370a25e2e3cfd30a78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878"}
@@ -0,0 +1,20 @@
+{"djvu": false, "name": "plain.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": true, "name": "scan.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "book.epub", "ole2": true, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "notes.doc", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "empty.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "tiny.txt", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "prose.txt", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "page.html", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "spine.xhtml", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "real.epub", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "epub"}
+{"djvu": false, "name": "saved.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "epub"}
+{"djvu": false, "name": "wrapped.pdf", "ole2": false, "sole_pdf": "book.pdf", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "wrapped-noisy.zip", "ole2": false, "sole_pdf": "book.pdf", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "collection.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"}
+{"djvu": false, "name": "pages.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "htmlz"}
+{"djvu": false, "name": "nothing.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"}
+{"djvu": false, "name": "uppercase.zip", "ole2": false, "sole_pdf": "BOOK.PDF", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "subdir.zip", "ole2": false, "sole_pdf": "dir/book.pdf", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "truncated.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"}
+{"djvu": false, "name": "Book - Sample Chapter.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
@@ -0,0 +1,57 @@
+"""Identify containers the way the Python pipeline identifies them.
+
+Reads one `{"name", "hex"}` object per line, writes the bytes to a temporary
+file under that name, and reports what each of the Python's container tests
+makes of it.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+        python3 oracle/booksniff.py < cases/booksniff/files.ndjson
+"""
+
+import json
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from book_pipeline import convert  # noqa: E402
+
+
+def main() -> int:
+    out = sys.stdout
+    with tempfile.TemporaryDirectory() as tmp:
+        for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+            if not line.strip():
+                continue
+            case = json.loads(line)
+            path = Path(tmp) / case["name"]
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(bytes.fromhex(case["hex"]))
+
+            record = {
+                "name": case["name"],
+                "djvu": convert._is_djvu_document(path),
+                "zip": convert._is_zip_container(path),
+                "ole2": convert._is_ole2_document(path),
+                "zip_kind": None,
+                "sole_pdf": None,
+            }
+            if record["zip"]:
+                record["zip_kind"] = convert._classify_zip(path)
+                try:
+                    import zipfile
+
+                    with zipfile.ZipFile(path) as z:
+                        record["sole_pdf"] = convert._sole_pdf_member(z.namelist())
+                except Exception:
+                    record["sole_pdf"] = None
+            out.write(json.dumps(record, sort_keys=True) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);

 fn main() -> ExitCode {
-    let jobs: [Job; 9] = [
+    let jobs: [Job; 10] = [
        ("tokens", tokens),
        ("unescape", unescape),
        ("chunkmd", chunkmd),
@@ -29,6 +29,7 @@ fn main() -> ExitCode {
        ("cueseg", cueseg),
        ("rrf", rrf),
        ("cuepack", cuepack),
+        ("booksniff", booksniff),
    ];

    for (name, job) in jobs {
@@ -72,6 +73,18 @@ fn unescape() -> Written {
    Ok(vec![path.display().to_string()])
 }

+fn booksniff() -> Written {
+    let dir = parity::golden("booksniff");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/booksniff/files.ndjson");
+    let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let verdicts = parity::run_oracle("booksniff.py", &cases)?;
+    let path = dir.join("files.ndjson");
+    std::fs::write(&path, verdicts).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn cuepack() -> Written {
    let dir = parity::golden("cuepack");
    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
@@ -0,0 +1,93 @@
+//! Container identification must agree with the Python's.
+//!
+//! The consequence of getting this wrong is not a wrong record: it is a
+//! converter handed a file it cannot read, and a book that never enters the
+//! corpus with nothing but an unhelpful error to say why.
+
+use booksniff::bytes::Bytes;
+use corpus_core::Kind;
+use serde_json::Value;
+
+fn cases() -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join("cases/booksniff/files.ndjson"))
+        .expect("a readable case file");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn bytes_of(case: &Value) -> Vec<u8> {
+    let hex = case["hex"].as_str().expect("hex bytes");
+    (0..hex.len())
+        .step_by(2)
+        .map(|at| u8::from_str_radix(&hex[at..at + 2], 16).expect("valid hex"))
+        .collect()
+}
+
+#[test]
+fn every_container_is_read_the_same_as_python() {
+    let golden: Vec<Value> = parity::read_golden_text("booksniff/files.ndjson")
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect();
+    let cases = cases();
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    for (case, theirs) in cases.iter().zip(&golden) {
+        let name = case["name"].as_str().expect("a name");
+        assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
+        let bytes = bytes_of(case);
+        let source = bytes.as_slice();
+        let doc = booksniff::sniff(&source, name);
+
+        // The three magic-byte tests, each of which the Python answers
+        // directly.
+        assert_eq!(
+            doc.kind == Kind::Djvu,
+            theirs["djvu"].as_bool().expect("a verdict"),
+            "{name}: DjVu"
+        );
+        assert_eq!(
+            doc.kind == Kind::Ole2,
+            theirs["ole2"].as_bool().expect("a verdict"),
+            "{name}: OLE2"
+        );
+        assert_eq!(
+            source.magic_at(0, booksniff::zip::LOCAL_HEADER),
+            theirs["zip"].as_bool().expect("a verdict"),
+            "{name}: zip"
+        );
+
+        // What the archive holds, where the Python classifies one.
+        if let Some(kind) = theirs["zip_kind"].as_str() {
+            let ours = match doc.kind {
+                Kind::Epub => "epub",
+                Kind::ZipWrappedPdf => "pdf",
+                _ if doc.notes.iter().any(|note| note.contains("HTML")) => "htmlz",
+                _ => "other",
+            };
+            assert_eq!(ours, kind, "{name}: what the zip holds");
+        }
+
+        // And which member it would unwrap.
+        if let Some(names) = booksniff::zip::members(&source) {
+            assert_eq!(
+                booksniff::sole_pdf(&names),
+                theirs["sole_pdf"].as_str(),
+                "{name}: the sole PDF"
+            );
+        }
+    }
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/booksniff/files.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("booksniff.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("booksniff/files.ndjson"));
+}