booksniff, and one place for the parts every tool shares

crates/booksniff/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "booksniff"
+description = "Says what a file actually is, whatever its name claims."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+catena-cli.workspace = true
+clap.workspace = true
+corpus-core.workspace = true
+textkit.workspace = true
+
+[dependencies.serde_json]
+workspace = true
+
+[lints]
+workspace = true

crates/booksniff/src/bytes.rs

@@ -0,0 +1,132 @@
+//! Reading a few bytes of a file without reading the file.
+//!
+//! Sniffing a format needs the first few bytes, and sometimes a structure at
+//! the end. A nine-hundred-megabyte scan should cost neither nine hundred
+//! megabytes of memory nor nine hundred megabytes of reading, so everything
+//! here works through a window rather than a buffer — and the same trait is
+//! implemented for a slice, so the tests need no files at all.
+
+use std::io::{Read, Seek, SeekFrom};
+
+/// Something a few bytes can be read out of, at a position.
+pub trait Bytes {
+    /// How long the whole thing is.
+    fn len(&self) -> u64;
+
+    /// Whether there is nothing at all.
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Read up to `into.len()` bytes from `offset`, returning how many were
+    /// read. Short at the end, zero past it.
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize>;
+
+    /// Read exactly `count` bytes from `offset`, or as many as there are.
+    fn slice(&self, offset: u64, count: usize) -> std::io::Result<Vec<u8>> {
+        let mut buffer = vec![0; count];
+        let read = self.read_at(offset, &mut buffer)?;
+        buffer.truncate(read);
+        Ok(buffer)
+    }
+
+    /// Whether the bytes at `offset` are exactly `magic`.
+    fn magic_at(&self, offset: u64, magic: &[u8]) -> bool {
+        self.slice(offset, magic.len())
+            .is_ok_and(|read| read == magic)
+    }
+}
+
+impl Bytes for &[u8] {
+    fn len(&self) -> u64 {
+        <[u8]>::len(self) as u64
+    }
+
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
+        let Ok(offset) = usize::try_from(offset) else {
+            return Ok(0);
+        };
+        if offset >= <[u8]>::len(self) {
+            return Ok(0);
+        }
+        let available = &self[offset..];
+        let take = available.len().min(into.len());
+        into[..take].copy_from_slice(&available[..take]);
+        Ok(take)
+    }
+}
+
+/// A file on disk, read through a window.
+pub struct File {
+    file: std::cell::RefCell<std::fs::File>,
+    len: u64,
+}
+
+impl File {
+    /// Open a file for sniffing.
+    pub fn open(path: &std::path::Path) -> std::io::Result<Self> {
+        let file = std::fs::File::open(path)?;
+        let len = file.metadata()?.len();
+        Ok(Self {
+            file: std::cell::RefCell::new(file),
+            len,
+        })
+    }
+}
+
+impl Bytes for File {
+    fn len(&self) -> u64 {
+        self.len
+    }
+
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
+        let mut file = self.file.borrow_mut();
+        file.seek(SeekFrom::Start(offset))?;
+        let mut read = 0;
+        while read < into.len() {
+            match file.read(&mut into[read..])? {
+                0 => break,
+                n => read += n,
+            }
+        }
+        Ok(read)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_slice_reads_from_the_middle() {
+        let bytes: &[u8] = b"hello world";
+        assert_eq!(bytes.slice(6, 5).unwrap(), b"world");
+        assert_eq!(Bytes::len(&bytes), 11);
+    }
+
+    #[test]
+    fn reading_past_the_end_is_short_rather_than_an_error() {
+        let bytes: &[u8] = b"abc";
+        assert_eq!(bytes.slice(1, 99).unwrap(), b"bc");
+        assert!(bytes.slice(99, 4).unwrap().is_empty());
+        assert!(bytes.slice(u64::MAX, 4).unwrap().is_empty());
+    }
+
+    #[test]
+    fn magic_matches_only_when_it_is_all_there() {
+        let bytes: &[u8] = b"%PDF-1.7";
+        assert!(bytes.magic_at(0, b"%PDF-"));
+        assert!(!bytes.magic_at(0, b"%PDG-"));
+        assert!(!bytes.magic_at(4, b"-1.7 more"));
+    }
+
+    #[test]
+    fn a_file_reads_the_same_as_a_slice() {
+        let path = std::env::temp_dir().join("booksniff-bytes-test");
+        std::fs::write(&path, b"hello world").expect("a writable path");
+        let file = File::open(&path).expect("the file opens");
+        assert_eq!(file.len(), 11);
+        assert_eq!(file.slice(6, 5).unwrap(), b"world");
+        assert!(file.magic_at(0, b"hello"));
+    }
+}

crates/booksniff/src/lib.rs

@@ -0,0 +1,335 @@
+//! Says what a file actually is, whatever its name claims.
+//!
+//! A library accumulates files that lie: a DjVu named `.pdf`, a Word document
+//! named `.epub`, an EPUB saved as `.zip` and then never opened again, a
+//! single PDF somebody zipped and renamed back to `.pdf`. Every one of those
+//! fails somewhere downstream, usually as an unhelpful error from a converter
+//! that was handed the wrong kind of file.
+//!
+//! The containers are unambiguous, so they are read rather than guessed at.
+
+pub mod bytes;
+pub mod mobi;
+pub mod zip;
+
+use corpus_core::{Doc, Kind};
+
+use crate::bytes::Bytes;
+
+/// DjVu's container magic: an IFF stream tagged for the format's originator.
+const DJVU: &[u8] = b"AT&TFORM";
+/// The OLE2 compound-file header every binary Word document starts with.
+const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
+/// A PDF announces its version in its first bytes.
+const PDF: &[u8] = b"%PDF-";
+
+/// Archive members that carry no content: resource forks and desktop clutter.
+const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
+
+/// Substrings in a name that mark a file as a sample rather than a book.
+const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
+
+/// Below this, a file is too small to be a book whatever it claims to be.
+const SMALLEST_BOOK: u64 = 4096;
+
+/// How much of the front is read to tell text from markup.
+const TEXT_WINDOW: usize = 4096;
+
+/// Identify a file.
+///
+/// `name` is used only for the extension it claims and for the words in it,
+/// both of which are reported as observations rather than believed.
+pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
+    let mut doc = Doc::new(name, Kind::Unknown, source.len() as i64);
+    doc.declared_ext = extension(name);
+
+    doc.kind = if source.magic_at(0, DJVU) {
+        Kind::Djvu
+    } else if source.magic_at(0, PDF) {
+        Kind::Pdf
+    } else if source.magic_at(0, OLE2) {
+        Kind::Ole2
+    } else if source.magic_at(0, zip::LOCAL_HEADER) {
+        archive(source, &mut doc)
+    } else if let Some(kind) = mobi::kind(source) {
+        kind
+    } else {
+        text(source, &mut doc)
+    };
+
+    if doc.kind == Kind::Unknown {
+        doc.note("no container or magic bytes this recognises");
+    }
+    if let Some(declared) = doc.declared_ext.as_deref()
+        && !names_the_same_thing(declared, doc.kind)
+    {
+        doc.note(format!(
+            "named .{declared}, but the bytes say {}",
+            spelling(doc.kind)
+        ));
+    }
+    if source.len() < SMALLEST_BOOK {
+        doc.note(format!(
+            "{} bytes, which is too small to be a book",
+            source.len()
+        ));
+    }
+    if let Some(marker) = sample_marker(name) {
+        doc.note(format!("the name says {marker:?}, so it may be a sample"));
+    }
+    doc
+}
+
+/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
+fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
+    let Some(names) = zip::members(source) else {
+        doc.note("a zip whose directory could not be read");
+        return Kind::Unknown;
+    };
+    let lower: Vec<String> = names.iter().map(|name| name.to_lowercase()).collect();
+
+    if lower.iter().any(|name| name == "mimetype")
+        && lower
+            .iter()
+            .any(|name| name.starts_with("meta-inf/container.xml"))
+    {
+        return Kind::Epub;
+    }
+    if let Some(pdf) = sole_pdf(&names) {
+        doc.note(format!("a zip holding one PDF, {pdf:?}"));
+        return Kind::ZipWrappedPdf;
+    }
+    if lower
+        .iter()
+        .any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
+    {
+        doc.note("a zip of HTML documents");
+        return Kind::Unknown;
+    }
+    doc.note(format!(
+        "a zip of {} members, none of them a book",
+        names.len()
+    ));
+    Kind::Unknown
+}
+
+/// The one PDF an archive exists to carry, if that is what it is.
+///
+/// Only an unambiguous wrapper qualifies. An archive holding several PDFs is
+/// a collection, and choosing among them would be a guess — better to say it
+/// is not a book than to name one chapter as the whole.
+pub fn sole_pdf(names: &[String]) -> Option<&str> {
+    let real: Vec<&String> = names
+        .iter()
+        .filter(|name| !name.ends_with('/'))
+        .filter(|name| {
+            let lower = name.to_lowercase();
+            !ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
+        })
+        .collect();
+    match real.as_slice() {
+        [only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
+        _ => None,
+    }
+}
+
+/// Text, markup, or nothing this knows.
+fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
+    let Ok(head) = source.slice(0, TEXT_WINDOW) else {
+        return Kind::Unknown;
+    };
+    if head.is_empty() {
+        doc.note("the file is empty");
+        return Kind::Unknown;
+    }
+    // A file with a null byte in its first few kilobytes is not text, and
+    // nothing else here is going to identify it either.
+    if head.contains(&0) {
+        return Kind::Unknown;
+    }
+    // The window may have cut a character in half, which is not the same
+    // thing as the file not being text.
+    let head = match std::str::from_utf8(&head) {
+        Ok(text) => text,
+        Err(e) if e.error_len().is_none() => {
+            std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
+        }
+        Err(_) => return Kind::Unknown,
+    };
+
+    let start = head
+        .trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}')
+        .to_lowercase();
+    if start.starts_with("<!doctype html")
+        || start.starts_with("<html")
+        || start.starts_with("<?xml")
+        || start.starts_with("<head")
+    {
+        return Kind::Html;
+    }
+    Kind::Txt
+}
+
+/// Whether an extension and a verdict are saying the same thing.
+fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
+    match kind {
+        Kind::Pdf => extension == "pdf",
+        Kind::Epub => extension == "epub",
+        Kind::Djvu => matches!(extension, "djvu" | "djv"),
+        Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
+        Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
+        Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
+        Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
+        Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
+        Kind::ZipWrappedPdf => extension == "zip",
+        // Nothing was recognised, so there is nothing to disagree with.
+        Kind::Unknown => true,
+    }
+}
+
+fn spelling(kind: Kind) -> &'static str {
+    match kind {
+        Kind::Pdf => "PDF",
+        Kind::Epub => "EPUB",
+        Kind::Djvu => "DjVu",
+        Kind::Azw3 => "AZW3",
+        Kind::Mobi => "MOBI",
+        Kind::Txt => "plain text",
+        Kind::Html => "HTML",
+        Kind::Ole2 => "an OLE2 document",
+        Kind::ZipWrappedPdf => "a zip holding one PDF",
+        Kind::Unknown => "nothing recognisable",
+    }
+}
+
+fn extension(name: &str) -> Option<String> {
+    std::path::Path::new(name)
+        .extension()
+        .map(|ext| ext.to_string_lossy().to_lowercase())
+}
+
+fn sample_marker(name: &str) -> Option<&'static str> {
+    let lower = std::path::Path::new(name)
+        .file_name()
+        .map(|name| name.to_string_lossy().to_lowercase())
+        .unwrap_or_default();
+    SAMPLE_MARKERS
+        .iter()
+        .find(|marker| lower.contains(**marker))
+        .copied()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn padded(head: &[u8]) -> Vec<u8> {
+        let mut out = head.to_vec();
+        out.resize(SMALLEST_BOOK as usize + 1, b'x');
+        out
+    }
+
+    fn kind_of(head: &[u8], name: &str) -> Kind {
+        sniff(&padded(head).as_slice(), name).kind
+    }
+
+    #[test]
+    fn the_magic_decides_not_the_name() {
+        assert_eq!(kind_of(b"%PDF-1.7\n", "book.epub"), Kind::Pdf);
+        assert_eq!(kind_of(DJVU, "scan.pdf"), Kind::Djvu);
+        assert_eq!(kind_of(OLE2, "book.epub"), Kind::Ole2);
+    }
+
+    #[test]
+    fn a_name_that_disagrees_with_the_bytes_is_noted() {
+        let doc = sniff(&padded(DJVU).as_slice(), "scan.pdf");
+        assert_eq!(doc.kind, Kind::Djvu);
+        assert!(
+            doc.notes.iter().any(|note| note.contains("named .pdf")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_name_that_agrees_is_not_remarked_on() {
+        let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "book.pdf");
+        assert!(doc.notes.is_empty(), "{:?}", doc.notes);
+    }
+
+    #[test]
+    fn text_and_markup_are_told_apart() {
+        assert_eq!(
+            kind_of(b"The mill stood at the northern end.", "a.txt"),
+            Kind::Txt
+        );
+        assert_eq!(kind_of(b"<!DOCTYPE html>\n<html>", "a.html"), Kind::Html);
+        assert_eq!(kind_of(b"\n\n  <html lang=\"en\">", "a.txt"), Kind::Html);
+        assert_eq!(kind_of(b"<?xml version=\"1.0\"?>", "a.xml"), Kind::Html);
+    }
+
+    #[test]
+    fn binary_rubbish_is_not_called_text() {
+        assert_eq!(kind_of(&[0x00, 0x01, 0x02, 0x03], "a.txt"), Kind::Unknown);
+        assert_eq!(kind_of(&[0xff, 0xfe, 0xfd], "a.txt"), Kind::Unknown);
+    }
+
+    #[test]
+    fn a_small_file_is_remarked_on() {
+        let doc = sniff(&b"%PDF-1.7".as_slice(), "tiny.pdf");
+        assert!(
+            doc.notes.iter().any(|note| note.contains("too small")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_name_that_says_sample_is_remarked_on() {
+        let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "Book - Sample Chapter.pdf");
+        assert!(
+            doc.notes.iter().any(|note| note.contains("sample")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn the_one_pdf_in_an_archive_is_the_book_and_several_are_not() {
+        let one = ["__MACOSX/._x".into(), "dir/".into(), "book.pdf".into()];
+        assert_eq!(sole_pdf(&one), Some("book.pdf"));
+
+        let several = ["one.pdf".to_string(), "two.pdf".to_string()];
+        assert_eq!(sole_pdf(&several), None);
+
+        let mixed = ["book.pdf".to_string(), "cover.jpg".to_string()];
+        assert_eq!(sole_pdf(&mixed), None);
+    }
+
+    #[test]
+    fn an_empty_file_is_unknown_and_says_so() {
+        let doc = sniff(&b"".as_slice(), "nothing.pdf");
+        assert_eq!(doc.kind, Kind::Unknown);
+        assert!(doc.notes.iter().any(|note| note.contains("empty")));
+        assert_eq!(doc.bytes, 0);
+    }
+
+    #[test]
+    fn the_declared_extension_is_reported_lowercased() {
+        assert_eq!(
+            sniff(&padded(b"%PDF-").as_slice(), "BOOK.PDF").declared_ext,
+            Some("pdf".to_string())
+        );
+        assert_eq!(
+            sniff(&padded(b"%PDF-").as_slice(), "no-extension").declared_ext,
+            None
+        );
+    }
+
+    #[test]
+    fn a_character_cut_in_half_by_the_window_does_not_make_it_binary() {
+        let mut head = "é".repeat(TEXT_WINDOW).into_bytes();
+        head.truncate(TEXT_WINDOW + 1);
+        assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
+    }
+}

crates/booksniff/src/main.rs

@@ -0,0 +1,99 @@
+//! Says what a file actually is, whatever its name claims.
+
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use booksniff::bytes::File;
+use clap::Parser;
+use corpus_core::{Doc, Kind, exit, ndjson};
+
+/// Identify files by what is in them rather than by what they are called.
+///
+/// Writes one doc record per file. Reads only the few kilobytes it needs —
+/// the front of the file, and an archive's directory at the back — so a
+/// directory of scans costs the same as a directory of anything else.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Exit codes: 0 every file was identified, 1 at least one was not, 74 a file
+could not be read.
+
+Example:
+
+    booksniff library/*.pdf | jq -r 'select(.kind != \"pdf\") | .path'"
+)]
+struct Args {
+    /// Files to identify.
+    #[arg(required = true)]
+    paths: Vec<PathBuf>,
+
+    /// Print a line a person can read instead of the record.
+    #[arg(long)]
+    plain: bool,
+}
+
+fn main() -> ExitCode {
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
+    let mut out = std::io::BufWriter::new(std::io::stdout().lock());
+    let mut unknown = false;
+    let mut unreadable = false;
+
+    for path in &args.paths {
+        let name = path.display().to_string();
+        let doc = match File::open(path) {
+            Ok(file) => booksniff::sniff(&file, &name),
+            Err(e) => {
+                eprintln!("booksniff: {name}: {e}");
+                unreadable = true;
+                continue;
+            }
+        };
+        unknown |= doc.kind == Kind::Unknown;
+
+        let wrote = if args.plain {
+            plain(&mut out, &doc)
+        } else {
+            ndjson::write_line(&mut out, &doc).map_err(|e| match e {
+                ndjson::Error::Io(e) => e,
+                other => std::io::Error::other(other.to_string()),
+            })
+        };
+        if let Err(e) = wrote {
+            // A consumer that stopped reading is not a failure.
+            if e.kind() == std::io::ErrorKind::BrokenPipe {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("booksniff: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+
+    if let Err(e) = out.flush() {
+        if e.kind() == std::io::ErrorKind::BrokenPipe {
+            return ExitCode::from(exit::OK);
+        }
+        eprintln!("booksniff: {e}");
+        return ExitCode::from(exit::IO);
+    }
+
+    ExitCode::from(match (unreadable, unknown) {
+        (true, _) => exit::IO,
+        (false, true) => exit::NO,
+        (false, false) => exit::OK,
+    })
+}
+
+fn plain(out: &mut impl Write, doc: &Doc) -> std::io::Result<()> {
+    let kind = serde_json::to_string(&doc.kind).unwrap_or_default();
+    writeln!(out, "{}\t{}", kind.trim_matches('"'), doc.path)?;
+    for note in &doc.notes {
+        writeln!(out, "\t{note}")?;
+    }
+    Ok(())
+}

crates/booksniff/src/mobi.rs

@@ -0,0 +1,108 @@
+//! Telling the two Kindle formats apart.
+//!
+//! Both are Palm databases with the same type and creator, so the extension
+//! is no help and neither is the first eight bytes. The difference is a
+//! version number inside the first record: six is the old format, eight is
+//! the one that carries real typesetting.
+
+use corpus_core::Kind;
+
+use crate::bytes::Bytes;
+
+/// Where a Palm database says what it holds.
+const TYPE_AT: u64 = 60;
+/// What a Kindle book says there.
+const BOOK_TYPE: &[u8] = b"BOOKMOBI";
+
+/// Where the first record's offset is written.
+const FIRST_RECORD_AT: u64 = 78;
+
+/// The MOBI header's own magic, at the start of the first record's header.
+const MOBI_MAGIC: &[u8] = b"MOBI";
+/// Where the MOBI header begins inside the first record.
+const MOBI_HEADER_AT: u64 = 16;
+/// How far into the MOBI header the format version is.
+const VERSION_AT: u64 = 36;
+/// The version that means the newer format.
+const KF8: u32 = 8;
+
+/// Which Kindle format this is, or `None` if it is not one.
+pub fn kind(source: &impl Bytes) -> Option<Kind> {
+    if !source.magic_at(TYPE_AT, BOOK_TYPE) {
+        return None;
+    }
+    Some(match version(source) {
+        Some(version) if version >= KF8 => Kind::Azw3,
+        _ => Kind::Mobi,
+    })
+}
+
+/// The format version in the first record, when it is readable.
+fn version(source: &impl Bytes) -> Option<u32> {
+    let record = big_endian(source, FIRST_RECORD_AT)? as u64;
+    let header = record + MOBI_HEADER_AT;
+    if !source.magic_at(header, MOBI_MAGIC) {
+        return None;
+    }
+    big_endian(source, header + VERSION_AT)
+}
+
+fn big_endian(source: &impl Bytes, at: u64) -> Option<u32> {
+    let bytes = source.slice(at, 4).ok()?;
+    (bytes.len() == 4).then(|| u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A Palm database holding one record with a MOBI header of `version`.
+    fn book(version: Option<u32>) -> Vec<u8> {
+        let record_at: u32 = 1024;
+        let mut out = vec![0u8; record_at as usize];
+        out[TYPE_AT as usize..TYPE_AT as usize + 8].copy_from_slice(BOOK_TYPE);
+        out[FIRST_RECORD_AT as usize..FIRST_RECORD_AT as usize + 4]
+            .copy_from_slice(&record_at.to_be_bytes());
+
+        if let Some(version) = version {
+            out.resize(record_at as usize + MOBI_HEADER_AT as usize, 0);
+            out.extend_from_slice(MOBI_MAGIC);
+            out.resize(
+                record_at as usize + MOBI_HEADER_AT as usize + VERSION_AT as usize,
+                0,
+            );
+            out.extend_from_slice(&version.to_be_bytes());
+        }
+        out
+    }
+
+    #[test]
+    fn the_newer_format_is_recognised() {
+        assert_eq!(kind(&book(Some(8)).as_slice()), Some(Kind::Azw3));
+        assert_eq!(kind(&book(Some(9)).as_slice()), Some(Kind::Azw3));
+    }
+
+    #[test]
+    fn the_older_format_is_recognised() {
+        assert_eq!(kind(&book(Some(6)).as_slice()), Some(Kind::Mobi));
+    }
+
+    #[test]
+    fn a_book_with_no_readable_version_is_taken_for_the_older_format() {
+        assert_eq!(kind(&book(None).as_slice()), Some(Kind::Mobi));
+    }
+
+    #[test]
+    fn something_that_is_not_a_kindle_book_is_not_one() {
+        assert_eq!(kind(&b"%PDF-1.7".as_slice()), None);
+        assert_eq!(kind(&vec![0u8; 4096].as_slice()), None);
+    }
+
+    #[test]
+    fn a_truncated_book_does_not_panic() {
+        let whole = book(Some(8));
+        for cut in 0..whole.len() {
+            let _ = kind(&&whole[..cut]);
+        }
+    }
+}

crates/booksniff/src/zip.rs

@@ -0,0 +1,158 @@
+//! Listing what is inside a zip, without unpacking any of it.
+//!
+//! An EPUB, an htmlz and a PDF someone zipped and renamed are all zips, and
+//! telling them apart needs only the member names. Those live in the central
+//! directory at the end of the file, so reading them costs a few kilobytes
+//! whatever the archive weighs — and nothing is ever decompressed, which
+//! means a malicious archive cannot expand into anything here.
+
+use crate::bytes::Bytes;
+
+/// The end-of-central-directory record's signature.
+const END_OF_DIRECTORY: &[u8] = b"PK\x05\x06";
+/// A central directory entry's signature.
+const ENTRY: &[u8] = b"PK\x01\x02";
+/// A local file header, which is what a zip starts with.
+pub const LOCAL_HEADER: &[u8] = b"PK\x03\x04";
+
+/// How far back from the end the trailing record may be.
+///
+/// Its own size plus the largest comment a zip may carry.
+const SEARCH_BACK: u64 = 22 + 0xffff;
+
+/// Most members this will list.
+///
+/// A book in an archive has tens of members; an archive with a hundred
+/// thousand is not a book, and reading all of its names would be the only
+/// expensive thing this program does.
+const MOST_MEMBERS: usize = 100_000;
+
+/// The names of an archive's members, or `None` if it is not a zip this can
+/// read.
+pub fn members(source: &impl Bytes) -> Option<Vec<String>> {
+    if !source.magic_at(0, LOCAL_HEADER) {
+        return None;
+    }
+    let (count, mut at) = directory_start(source)?;
+
+    let mut names = Vec::with_capacity(count.min(1024));
+    for _ in 0..count.min(MOST_MEMBERS) {
+        let header = source.slice(at, 46).ok()?;
+        if header.len() < 46 || header[..4] != *ENTRY {
+            break;
+        }
+        let name_len = u16::from_le_bytes([header[28], header[29]]) as u64;
+        let extra_len = u16::from_le_bytes([header[30], header[31]]) as u64;
+        let comment_len = u16::from_le_bytes([header[32], header[33]]) as u64;
+
+        let name = source.slice(at + 46, name_len as usize).ok()?;
+        // Member names are bytes, and an archive written on another machine
+        // may not have written UTF-8. What is wanted here is a comparison
+        // against ASCII, so anything else may as well be a replacement
+        // character.
+        names.push(String::from_utf8_lossy(&name).into_owned());
+        at += 46 + name_len + extra_len + comment_len;
+    }
+    Some(names)
+}
+
+/// How many entries the central directory has and where it starts.
+fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
+    let len = source.len();
+    let window = SEARCH_BACK.min(len);
+    let from = len - window;
+    let tail = source.slice(from, window as usize).ok()?;
+
+    // The record is at the end, after a comment of unknown length, so it is
+    // found by searching backwards for its signature.
+    let at = tail
+        .windows(4)
+        .rposition(|window| window == END_OF_DIRECTORY)?;
+    let record = &tail[at..];
+    if record.len() < 22 {
+        return None;
+    }
+    let count = u16::from_le_bytes([record[10], record[11]]) as usize;
+    let offset = u32::from_le_bytes([record[16], record[17], record[18], record[19]]) as u64;
+    (offset < len).then_some((count, offset))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Build a zip holding `names`, each with empty stored content.
+    fn zip(names: &[&str]) -> Vec<u8> {
+        let mut out: Vec<u8> = Vec::new();
+        let mut offsets = Vec::new();
+        for name in names {
+            offsets.push(out.len() as u32);
+            out.extend_from_slice(LOCAL_HEADER);
+            out.extend_from_slice(&[0; 22]); // version .. uncompressed size
+            out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+            out.extend_from_slice(&0u16.to_le_bytes());
+            out.extend_from_slice(name.as_bytes());
+        }
+        let directory_at = out.len() as u32;
+        for (name, offset) in names.iter().zip(&offsets) {
+            out.extend_from_slice(ENTRY);
+            out.extend_from_slice(&[0; 24]); // version .. uncompressed size
+            out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+            out.extend_from_slice(&0u16.to_le_bytes()); // extra
+            out.extend_from_slice(&0u16.to_le_bytes()); // comment
+            out.extend_from_slice(&[0; 8]); // disk, attributes
+            out.extend_from_slice(&offset.to_le_bytes());
+            out.extend_from_slice(name.as_bytes());
+        }
+        out.extend_from_slice(END_OF_DIRECTORY);
+        out.extend_from_slice(&[0; 4]); // disk numbers
+        out.extend_from_slice(&(names.len() as u16).to_le_bytes());
+        out.extend_from_slice(&(names.len() as u16).to_le_bytes());
+        out.extend_from_slice(&((out.len() as u32).saturating_sub(directory_at)).to_le_bytes());
+        out.extend_from_slice(&directory_at.to_le_bytes());
+        out.extend_from_slice(&0u16.to_le_bytes()); // comment length
+        out
+    }
+
+    #[test]
+    fn the_member_names_come_back() {
+        let archive = zip(&["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]);
+        let names = members(&archive.as_slice()).expect("a zip");
+        assert_eq!(
+            names,
+            ["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]
+        );
+    }
+
+    #[test]
+    fn an_archive_with_no_members_is_not_read_as_one() {
+        // It has no local header to start with, so nothing identifies it as
+        // an archive in the first place — and an empty archive is not a book
+        // under any reading.
+        assert_eq!(members(&zip(&[]).as_slice()), None);
+    }
+
+    #[test]
+    fn something_that_is_not_a_zip_is_not_read_as_one() {
+        assert_eq!(members(&b"%PDF-1.7".as_slice()), None);
+        assert_eq!(members(&b"".as_slice()), None);
+    }
+
+    #[test]
+    fn a_truncated_archive_does_not_panic() {
+        let archive = zip(&["a.pdf", "b.pdf"]);
+        for cut in 0..archive.len() {
+            let _ = members(&&archive[..cut]);
+        }
+    }
+
+    #[test]
+    fn a_count_that_lies_stops_at_the_first_bad_entry() {
+        let mut archive = zip(&["a.pdf"]);
+        let at = archive.len() - 12;
+        // Claim a thousand entries where there is one.
+        archive[at - 6..at - 4].copy_from_slice(&1000u16.to_le_bytes());
+        let names = members(&archive.as_slice()).expect("a zip");
+        assert_eq!(names, ["a.pdf"]);
+    }
+}

crates/booksniff/tests/cli.rs

@@ -0,0 +1,95 @@
+//! The tool as a caller meets it: files in, records and an exit code out.
+
+use std::path::PathBuf;
+use std::process::{Command, Output};
+
+fn run(args: &[&str]) -> Output {
+    Command::new(env!("CARGO_BIN_EXE_booksniff"))
+        .args(args)
+        .output()
+        .expect("the binary should run")
+}
+
+fn written(name: &str, bytes: &[u8]) -> PathBuf {
+    let dir = std::env::temp_dir().join("booksniff-tests");
+    std::fs::create_dir_all(&dir).expect("a writable temp directory");
+    let path = dir.join(name);
+    std::fs::write(&path, bytes).expect("a writable path");
+    path
+}
+
+fn padded(head: &[u8]) -> Vec<u8> {
+    let mut out = head.to_vec();
+    out.resize(8192, b'x');
+    out
+}
+
+#[test]
+fn a_file_it_recognises_comes_back_as_a_record() {
+    let path = written("book.pdf", &padded(b"%PDF-1.7\n"));
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(0));
+
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["kind"], "pdf");
+    assert_eq!(doc["declared_ext"], "pdf");
+    assert_eq!(doc["bytes"], 8192);
+    assert!(out.stderr.is_empty());
+}
+
+#[test]
+fn a_file_it_cannot_place_answers_no() {
+    let path = written("mystery.bin", &padded(&[0x00, 0x01, 0x02]));
+    let out = run(&[path.to_str().expect("utf-8 path")]);
+    assert_eq!(out.status.code(), Some(1), "unknown must answer no");
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["kind"], "unknown");
+}
+
+#[test]
+fn several_files_come_back_as_several_records() {
+    let one = written("one.pdf", &padded(b"%PDF-1.7"));
+    let two = written("two.djvu", &padded(b"AT&TFORM"));
+    let out = run(&[
+        one.to_str().expect("utf-8 path"),
+        two.to_str().expect("utf-8 path"),
+    ]);
+    let kinds: Vec<String> = String::from_utf8(out.stdout)
+        .expect("utf-8")
+        .lines()
+        .map(|line| {
+            serde_json::from_str::<serde_json::Value>(line).expect("valid JSON")["kind"].to_string()
+        })
+        .collect();
+    assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]);
+}
+
+#[test]
+fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() {
+    let path = written("kept.pdf", &padded(b"%PDF-1.7"));
+    let out = run(&[
+        "/definitely/not/here.pdf",
+        path.to_str().expect("utf-8 path"),
+    ]);
+    assert_eq!(out.status.code(), Some(74));
+    assert!(!out.stderr.is_empty());
+    // The readable file was still identified, on stdout, uncontaminated.
+    let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
+    assert_eq!(doc["kind"], "pdf");
+}
+
+#[test]
+fn the_plain_form_is_for_a_person() {
+    let path = written("lying.pdf", &padded(b"AT&TFORM"));
+    let out = run(&["--plain", path.to_str().expect("utf-8 path")]);
+    let said = String::from_utf8(out.stdout).expect("utf-8");
+    assert!(said.starts_with("djvu\t"), "{said}");
+    assert!(said.contains("named .pdf"), "{said}");
+}
+
+#[test]
+fn no_files_named_is_a_usage_error() {
+    let out = run(&[]);
+    assert_eq!(out.status.code(), Some(64), "usage errors exit 64");
+    assert!(out.stdout.is_empty());
+}

crates/catena-cli/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "catena-cli"
+description = "The parts of a filter that every tool in this workspace does the same way."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+serde.workspace = true
+
+[lints]
+workspace = true

crates/catena-cli/src/lib.rs

@@ -0,0 +1,162 @@
+//! The parts of a filter that every tool here does the same way.
+//!
+//! Not a framework — three functions, each of which exists because getting
+//! it wrong is easy and the consequence is a pipeline that misreports why it
+//! stopped:
+//!
+//! - a usage error exits 64, not whatever the argument parser felt like;
+//! - a consumer that closed the pipe is the end of the job, not a failure;
+//! - a record is flushed as it is written, so a slow stage downstream sees it
+//!   immediately.
+
+use std::io::{BufWriter, Write};
+use std::process::ExitCode;
+
+use corpus_core::{exit, ndjson};
+use serde::Serialize;
+
+/// Parse the command line, or exit.
+///
+/// `--help` and `--version` are not errors and leave with 0; anything else
+/// wrong with the arguments leaves with 64, which is what the rest of the
+/// suite means by "the command line was wrong".
+pub fn parse<T: clap::Parser>() -> Result<T, ExitCode> {
+    T::try_parse().map_err(|e| {
+        let _ = e.print();
+        match e.use_stderr() {
+            true => ExitCode::from(exit::USAGE),
+            false => ExitCode::from(exit::OK),
+        }
+    })
+}
+
+/// Write records to stdout as NDJSON, one per line, flushed as they go.
+pub fn stream<'a, T: Serialize + 'a>(
+    tool: &str,
+    records: impl IntoIterator<Item = &'a T>,
+) -> ExitCode {
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    for record in records {
+        match ndjson::write_line(&mut out, record) {
+            Ok(()) => {}
+            // A consumer that stopped reading is not a failure: `… | head` is
+            // a normal way to look at a stream.
+            Err(e) if e.is_broken_pipe() => return ExitCode::from(exit::OK),
+            Err(e) => {
+                eprintln!("{tool}: {e}");
+                return ExitCode::from(exit::IO);
+            }
+        }
+    }
+    finish(tool, out)
+}
+
+/// Write text to stdout and leave.
+pub fn text(tool: &str, text: &str) -> ExitCode {
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    match out.write_all(text.as_bytes()) {
+        Ok(()) => finish(tool, out),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("{tool}: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Flush what is buffered, treating a closed consumer as the end of the job.
+pub fn finish(tool: &str, mut out: impl Write) -> ExitCode {
+    match out.flush() {
+        Ok(()) => ExitCode::from(exit::OK),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("{tool}: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Read the whole input as text, telling a file that cannot be read from one
+/// that is not text.
+///
+/// The two are different answers: the first is an I/O failure and the second
+/// is bad input, and a caller routing on the exit code needs to tell them
+/// apart.
+pub fn read_text(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
+    let bytes = read_bytes(path)?;
+    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
+}
+
+/// Read the whole input, replacing anything that is not UTF-8.
+///
+/// For the formats that routinely arrive in another encoding — a subtitle
+/// file downloaded from anywhere is as likely to be Latin-1 — where a cue
+/// with one wrong character still has its timing and most of its words.
+pub fn read_lossy(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
+    Ok(String::from_utf8_lossy(&read_bytes(path)?).into_owned())
+}
+
+fn read_bytes(path: Option<&std::path::Path>) -> Result<Vec<u8>, (u8, String)> {
+    use std::io::Read as _;
+
+    match path {
+        Some(path) => {
+            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
+        }
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin()
+                .read_to_end(&mut bytes)
+                .map(|_| bytes)
+                .map_err(|e| (exit::IO, e.to_string()))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn reading_a_file_that_is_not_there_is_an_io_failure() {
+        let missing = std::path::Path::new("/definitely/not/here");
+        let (code, _) = read_text(Some(missing)).expect_err("should fail");
+        assert_eq!(code, exit::IO);
+    }
+
+    #[test]
+    fn reading_bytes_that_are_not_text_is_a_data_failure() {
+        let path = std::env::temp_dir().join("catena-cli-not-text");
+        std::fs::write(&path, [0xff, 0xfe]).expect("a writable path");
+        let (code, why) = read_text(Some(&path)).expect_err("should fail");
+        assert_eq!(code, exit::DATA);
+        assert!(why.contains("UTF-8"), "{why}");
+    }
+
+    #[test]
+    fn the_lossy_read_takes_what_the_strict_one_refuses() {
+        let path = std::env::temp_dir().join("catena-cli-lossy");
+        std::fs::write(&path, [b'a', 0xff, b'b']).expect("a writable path");
+        assert_eq!(read_lossy(Some(&path)).expect("should read"), "a\u{fffd}b");
+    }
+
+    #[test]
+    fn a_closed_consumer_is_the_end_of_the_job_and_not_a_failure() {
+        struct Closed;
+        impl Write for Closed {
+            fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
+                Ok(0)
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::BrokenPipe,
+                    "broken pipe",
+                ))
+            }
+        }
+        assert_eq!(
+            format!("{:?}", finish("test", Closed)),
+            format!("{:?}", ExitCode::from(exit::OK))
+        );
+    }
+}

crates/chunkmd/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 fancy-regex.workspace = true

crates/chunkmd/src/main.rs

@@ -1,13 +1,12 @@
 //! Cuts markdown into retrievable passages that carry their page, chapter
 //! and role.
 
-use std::io::{BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::process::ExitCode;
 
 use chunkmd::Budgets;
 use clap::Parser;
-use corpus_core::{exit, ndjson};
+use corpus_core::exit;
 
 /// Cut markdown into chunks, one JSON record per line.
 ///
@@ -48,7 +47,10 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
     let budgets = Budgets {
         target: args.target,
         overlap: args.overlap,
@@ -60,7 +62,7 @@ fn main() -> ExitCode {
         return ExitCode::from(exit::USAGE);
     }
 
-    let markdown = match read(args.path.as_deref()) {
+    let markdown = match catena_cli::read_text(args.path.as_deref()) {
         Ok(markdown) => markdown,
         Err((code, why)) => {
             eprintln!("chunkmd: {why}");
@@ -68,26 +70,7 @@ fn main() -> ExitCode {
         }
     };
 
-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for chunk in chunkmd::chunk(&markdown, &budgets) {
-        if let Err(e) = ndjson::write_line(&mut out, &chunk) {
-            // A consumer that stopped reading is not a failure: `| head` is a
-            // normal way to look at a stream.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("chunkmd: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("chunkmd: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("chunkmd", &chunkmd::chunk(&markdown, &budgets))
 }
 
 /// Budgets that cannot produce anything sensible, rejected before reading
@@ -116,22 +99,3 @@ fn sane(budgets: &Budgets) -> Result<(), String> {
     }
     Ok(())
 }
-
-/// Read the input, distinguishing a file that cannot be read from one that
-/// is not text: the first is an I/O failure and the second is bad input, and
-/// a caller routing on the exit code needs to tell them apart.
-fn read(path: Option<&std::path::Path>) -> Result<String, (u8, String)> {
-    let bytes = match path {
-        Some(path) => {
-            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
-        }
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin()
-                .read_to_end(&mut bytes)
-                .map(|_| bytes)
-                .map_err(|e| (exit::IO, e.to_string()))
-        }
-    }?;
-    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
-}

crates/cuenorm/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true

crates/cuenorm/src/main.rs

@@ -1,11 +1,9 @@
 //! Any subtitle format in, one normalised cue stream out.
 
-use std::io::{BufWriter, Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;
 
 use clap::Parser;
-use corpus_core::{exit, ndjson};
 
 /// Read a subtitle file — SubRip, WebVTT or SubStation Alpha — and write one
 /// cue per line as JSON.
@@ -35,13 +33,16 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
 
-    let content = match read(args.path.as_deref()) {
+    let content = match catena_cli::read_lossy(args.path.as_deref()) {
         Ok(content) => content,
-        Err(e) => {
-            eprintln!("cuenorm: {e}");
-            return ExitCode::from(exit::IO);
+        Err((code, why)) => {
+            eprintln!("cuenorm: {why}");
+            return ExitCode::from(code);
         }
     };
 
@@ -52,42 +53,5 @@ fn main() -> ExitCode {
         cuekit::merge_repeats(&cues)
     };
 
-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for cue in &cues {
-        if let Err(e) = ndjson::write_line(&mut out, cue) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("cuenorm: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("cuenorm: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
-}
-
-/// Read the file as text, tolerating the encodings a subtitle file arrives in.
-///
-/// Not UTF-8 is common: a subtitle file downloaded from anywhere is as likely
-/// to be Latin-1, and refusing it would make the tool useless for exactly the
-/// archives that need it. Undecodable bytes become the replacement character
-/// rather than an error, because a cue with one wrong character still has its
-/// timing and most of its words.
-fn read(path: Option<&Path>) -> std::io::Result<String> {
-    let bytes = match path {
-        Some(path) => std::fs::read(path)?,
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin().read_to_end(&mut bytes)?;
-            bytes
-        }
-    };
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
+    catena_cli::stream("cuenorm", &cues)
 }

crates/cuepack/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true

crates/cuepack/src/main.rs

@@ -1,6 +1,5 @@
 //! Packs cues into passages worth embedding.
 
-use std::io::{BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::process::ExitCode;
 
@@ -38,7 +37,10 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
     let budgets = Budgets {
         seconds: args.seconds,
         max_tokens: args.max_tokens,
@@ -48,11 +50,11 @@ fn main() -> ExitCode {
         return ExitCode::from(exit::USAGE);
     }
 
-    let content = match read(args.path.as_deref()) {
+    let content = match catena_cli::read_lossy(args.path.as_deref()) {
         Ok(content) => content,
-        Err(e) => {
-            eprintln!("cuepack: {e}");
-            return ExitCode::from(exit::IO);
+        Err((code, why)) => {
+            eprintln!("cuepack: {why}");
+            return ExitCode::from(code);
         }
     };
 
@@ -64,25 +66,7 @@ fn main() -> ExitCode {
         }
     };
 
-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for passage in cuepack::pack(&cues, &budgets) {
-        if let Err(e) = ndjson::write_line(&mut out, &passage) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("cuepack: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("cuepack: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("cuepack", &cuepack::pack(&cues, &budgets))
 }
 
 /// Read cues from either encoding: a cue stream begins with a JSON object
@@ -97,15 +81,3 @@ fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
     }
     Ok(cuekit::parse(content))
 }
-
-fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
-    let bytes = match path {
-        Some(path) => std::fs::read(path)?,
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin().read_to_end(&mut bytes)?;
-            bytes
-        }
-    };
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
-}

crates/cueseg/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 cuekit.workspace = true

crates/cueseg/src/main.rs

@@ -1,7 +1,6 @@
 //! Turns a timed transcript into topic chapters.
 
-use std::io::{BufWriter, Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;
 
 use clap::Parser;
@@ -38,13 +37,16 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
 
-    let content = match read(args.path.as_deref()) {
+    let content = match catena_cli::read_lossy(args.path.as_deref()) {
         Ok(content) => content,
-        Err(e) => {
-            eprintln!("cueseg: {e}");
-            return ExitCode::from(exit::IO);
+        Err((code, why)) => {
+            eprintln!("cueseg: {why}");
+            return ExitCode::from(code);
         }
     };
 
@@ -65,25 +67,7 @@ fn main() -> ExitCode {
         None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max),
     };
 
-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for chapter in cueseg::segment(&cues, duration) {
-        if let Err(e) = ndjson::write_line(&mut out, &chapter) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("cueseg: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("cueseg: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("cueseg", &cueseg::segment(&cues, duration))
 }
 
 /// Read cues from either encoding.
@@ -101,15 +85,3 @@ fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
     }
     Ok(cuekit::parse(content))
 }
-
-fn read(path: Option<&Path>) -> std::io::Result<String> {
-    let bytes = match path {
-        Some(path) => std::fs::read(path)?,
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin().read_to_end(&mut bytes)?;
-            bytes
-        }
-    };
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
-}

crates/mdscrub/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 regex.workspace = true

crates/mdscrub/src/main.rs

@@ -1,11 +1,9 @@
 //! Cleans converted markdown.
 
-use std::io::{Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;
 
 use clap::Parser;
-use corpus_core::exit;
 
 /// Clean converted markdown: drop residual HTML and control characters,
 /// make headings readable, and spell every page boundary the same way.
@@ -27,9 +25,12 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
 
-    let markdown = match read(args.path.as_deref()) {
+    let markdown = match catena_cli::read_text(args.path.as_deref()) {
         Ok(markdown) => markdown,
         Err((code, why)) => {
             eprintln!("mdscrub: {why}");
@@ -37,33 +38,5 @@ fn main() -> ExitCode {
         }
     };
 
-    let cleaned = mdscrub::scrub(&markdown);
-    match std::io::stdout().write_all(cleaned.as_bytes()) {
-        Ok(()) => ExitCode::from(exit::OK),
-        // A consumer that stopped reading is not a failure.
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("mdscrub: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
-}
-
-/// Read the input, distinguishing a file that cannot be read from one that
-/// is not text: the first is an I/O failure and the second is bad input, and
-/// a caller routing on the exit code needs to tell them apart.
-fn read(path: Option<&Path>) -> Result<String, (u8, String)> {
-    let bytes = match path {
-        Some(path) => {
-            std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display())))
-        }
-        None => {
-            let mut bytes = Vec::new();
-            std::io::stdin()
-                .read_to_end(&mut bytes)
-                .map(|_| bytes)
-                .map_err(|e| (exit::IO, e.to_string()))
-        }
-    }?;
-    String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}")))
+    catena_cli::text("mdscrub", &mdscrub::scrub(&markdown))
 }

crates/pdftriage/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 regex.workspace = true

crates/pdftriage/src/main.rs

@@ -49,7 +49,10 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
 
     let bytes = match std::fs::metadata(&args.path) {
         Ok(meta) => meta.len() as i64,

crates/rrf/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+catena-cli.workspace = true
 clap.workspace = true
 corpus-core.workspace = true
 serde_json.workspace = true

crates/rrf/src/main.rs

@@ -1,6 +1,5 @@
 //! Reciprocal rank fusion: several ranked lists in, one ranked list out.
 
-use std::io::{BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::process::ExitCode;
 
@@ -49,7 +48,10 @@ struct Args {
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
 
     let weights = match weights(&args) {
         Ok(weights) => weights,
@@ -59,24 +61,14 @@ fn main() -> ExitCode {
         }
     };
 
-    let hits: Result<Vec<Hit>, _> = match args.path.as_deref() {
-        Some(path) => match std::fs::File::open(path) {
-            Ok(file) => ndjson::read(std::io::BufReader::new(file)).collect(),
-            Err(e) => {
-                eprintln!("rrf: {}: {e}", path.display());
-                return ExitCode::from(exit::IO);
-            }
-        },
-        None => {
-            let mut text = Vec::new();
-            if let Err(e) = std::io::stdin().read_to_end(&mut text) {
-                eprintln!("rrf: {e}");
-                return ExitCode::from(exit::IO);
-            }
-            ndjson::read(text.as_slice()).collect()
+    let text = match catena_cli::read_text(args.path.as_deref()) {
+        Ok(text) => text,
+        Err((code, why)) => {
+            eprintln!("rrf: {why}");
+            return ExitCode::from(code);
         }
     };
-    let hits = match hits {
+    let hits = match ndjson::read(text.as_bytes()).collect::<Result<Vec<Hit>, _>>() {
         Ok(hits) => hits,
         Err(e) => {
             eprintln!("rrf: {e}");
@@ -91,25 +83,7 @@ fn main() -> ExitCode {
     let fused = rrf::fuse(&hits, &weights);
     let fused = &fused[..args.limit.unwrap_or(fused.len()).min(fused.len())];
 
-    let mut out = BufWriter::new(std::io::stdout().lock());
-    for hit in fused {
-        if let Err(e) = ndjson::write_line(&mut out, hit) {
-            // A consumer that stopped reading is not a failure.
-            if e.is_broken_pipe() {
-                return ExitCode::from(exit::OK);
-            }
-            eprintln!("rrf: {e}");
-            return ExitCode::from(exit::IO);
-        }
-    }
-    match out.flush() {
-        Ok(()) => ExitCode::from(exit::OK),
-        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
-        Err(e) => {
-            eprintln!("rrf: {e}");
-            ExitCode::from(exit::IO)
-        }
-    }
+    catena_cli::stream("rrf", fused)
 }
 
 fn weights(args: &Args) -> Result<Weights, String> {