booksniff, and one place for the parts every tool shares

crates/booksniff/src/bytes.rs

@@ -0,0 +1,132 @@
+//! Reading a few bytes of a file without reading the file.
+//!
+//! Sniffing a format needs the first few bytes, and sometimes a structure at
+//! the end. A nine-hundred-megabyte scan should cost neither nine hundred
+//! megabytes of memory nor nine hundred megabytes of reading, so everything
+//! here works through a window rather than a buffer — and the same trait is
+//! implemented for a slice, so the tests need no files at all.
+
+use std::io::{Read, Seek, SeekFrom};
+
+/// Something a few bytes can be read out of, at a position.
+pub trait Bytes {
+    /// How long the whole thing is.
+    fn len(&self) -> u64;
+
+    /// Whether there is nothing at all.
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Read up to `into.len()` bytes from `offset`, returning how many were
+    /// read. Short at the end, zero past it.
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize>;
+
+    /// Read exactly `count` bytes from `offset`, or as many as there are.
+    fn slice(&self, offset: u64, count: usize) -> std::io::Result<Vec<u8>> {
+        let mut buffer = vec![0; count];
+        let read = self.read_at(offset, &mut buffer)?;
+        buffer.truncate(read);
+        Ok(buffer)
+    }
+
+    /// Whether the bytes at `offset` are exactly `magic`.
+    fn magic_at(&self, offset: u64, magic: &[u8]) -> bool {
+        self.slice(offset, magic.len())
+            .is_ok_and(|read| read == magic)
+    }
+}
+
+impl Bytes for &[u8] {
+    fn len(&self) -> u64 {
+        <[u8]>::len(self) as u64
+    }
+
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
+        let Ok(offset) = usize::try_from(offset) else {
+            return Ok(0);
+        };
+        if offset >= <[u8]>::len(self) {
+            return Ok(0);
+        }
+        let available = &self[offset..];
+        let take = available.len().min(into.len());
+        into[..take].copy_from_slice(&available[..take]);
+        Ok(take)
+    }
+}
+
+/// A file on disk, read through a window.
+pub struct File {
+    file: std::cell::RefCell<std::fs::File>,
+    len: u64,
+}
+
+impl File {
+    /// Open a file for sniffing.
+    pub fn open(path: &std::path::Path) -> std::io::Result<Self> {
+        let file = std::fs::File::open(path)?;
+        let len = file.metadata()?.len();
+        Ok(Self {
+            file: std::cell::RefCell::new(file),
+            len,
+        })
+    }
+}
+
+impl Bytes for File {
+    fn len(&self) -> u64 {
+        self.len
+    }
+
+    fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
+        let mut file = self.file.borrow_mut();
+        file.seek(SeekFrom::Start(offset))?;
+        let mut read = 0;
+        while read < into.len() {
+            match file.read(&mut into[read..])? {
+                0 => break,
+                n => read += n,
+            }
+        }
+        Ok(read)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_slice_reads_from_the_middle() {
+        let bytes: &[u8] = b"hello world";
+        assert_eq!(bytes.slice(6, 5).unwrap(), b"world");
+        assert_eq!(Bytes::len(&bytes), 11);
+    }
+
+    #[test]
+    fn reading_past_the_end_is_short_rather_than_an_error() {
+        let bytes: &[u8] = b"abc";
+        assert_eq!(bytes.slice(1, 99).unwrap(), b"bc");
+        assert!(bytes.slice(99, 4).unwrap().is_empty());
+        assert!(bytes.slice(u64::MAX, 4).unwrap().is_empty());
+    }
+
+    #[test]
+    fn magic_matches_only_when_it_is_all_there() {
+        let bytes: &[u8] = b"%PDF-1.7";
+        assert!(bytes.magic_at(0, b"%PDF-"));
+        assert!(!bytes.magic_at(0, b"%PDG-"));
+        assert!(!bytes.magic_at(4, b"-1.7 more"));
+    }
+
+    #[test]
+    fn a_file_reads_the_same_as_a_slice() {
+        let path = std::env::temp_dir().join("booksniff-bytes-test");
+        std::fs::write(&path, b"hello world").expect("a writable path");
+        let file = File::open(&path).expect("the file opens");
+        assert_eq!(file.len(), 11);
+        assert_eq!(file.slice(6, 5).unwrap(), b"world");
+        assert!(file.magic_at(0, b"hello"));
+    }
+}

crates/booksniff/src/lib.rs

@@ -0,0 +1,335 @@
+//! Says what a file actually is, whatever its name claims.
+//!
+//! A library accumulates files that lie: a DjVu named `.pdf`, a Word document
+//! named `.epub`, an EPUB saved as `.zip` and then never opened again, a
+//! single PDF somebody zipped and renamed back to `.pdf`. Every one of those
+//! fails somewhere downstream, usually as an unhelpful error from a converter
+//! that was handed the wrong kind of file.
+//!
+//! The containers are unambiguous, so they are read rather than guessed at.
+
+pub mod bytes;
+pub mod mobi;
+pub mod zip;
+
+use corpus_core::{Doc, Kind};
+
+use crate::bytes::Bytes;
+
+/// DjVu's container magic: an IFF stream tagged for the format's originator.
+const DJVU: &[u8] = b"AT&TFORM";
+/// The OLE2 compound-file header every binary Word document starts with.
+const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
+/// A PDF announces its version in its first bytes.
+const PDF: &[u8] = b"%PDF-";
+
+/// Archive members that carry no content: resource forks and desktop clutter.
+const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
+
+/// Substrings in a name that mark a file as a sample rather than a book.
+const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
+
+/// Below this, a file is too small to be a book whatever it claims to be.
+const SMALLEST_BOOK: u64 = 4096;
+
+/// How much of the front is read to tell text from markup.
+const TEXT_WINDOW: usize = 4096;
+
+/// Identify a file.
+///
+/// `name` is used only for the extension it claims and for the words in it,
+/// both of which are reported as observations rather than believed.
+pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
+    let mut doc = Doc::new(name, Kind::Unknown, source.len() as i64);
+    doc.declared_ext = extension(name);
+
+    doc.kind = if source.magic_at(0, DJVU) {
+        Kind::Djvu
+    } else if source.magic_at(0, PDF) {
+        Kind::Pdf
+    } else if source.magic_at(0, OLE2) {
+        Kind::Ole2
+    } else if source.magic_at(0, zip::LOCAL_HEADER) {
+        archive(source, &mut doc)
+    } else if let Some(kind) = mobi::kind(source) {
+        kind
+    } else {
+        text(source, &mut doc)
+    };
+
+    if doc.kind == Kind::Unknown {
+        doc.note("no container or magic bytes this recognises");
+    }
+    if let Some(declared) = doc.declared_ext.as_deref()
+        && !names_the_same_thing(declared, doc.kind)
+    {
+        doc.note(format!(
+            "named .{declared}, but the bytes say {}",
+            spelling(doc.kind)
+        ));
+    }
+    if source.len() < SMALLEST_BOOK {
+        doc.note(format!(
+            "{} bytes, which is too small to be a book",
+            source.len()
+        ));
+    }
+    if let Some(marker) = sample_marker(name) {
+        doc.note(format!("the name says {marker:?}, so it may be a sample"));
+    }
+    doc
+}
+
+/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
+fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
+    let Some(names) = zip::members(source) else {
+        doc.note("a zip whose directory could not be read");
+        return Kind::Unknown;
+    };
+    let lower: Vec<String> = names.iter().map(|name| name.to_lowercase()).collect();
+
+    if lower.iter().any(|name| name == "mimetype")
+        && lower
+            .iter()
+            .any(|name| name.starts_with("meta-inf/container.xml"))
+    {
+        return Kind::Epub;
+    }
+    if let Some(pdf) = sole_pdf(&names) {
+        doc.note(format!("a zip holding one PDF, {pdf:?}"));
+        return Kind::ZipWrappedPdf;
+    }
+    if lower
+        .iter()
+        .any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
+    {
+        doc.note("a zip of HTML documents");
+        return Kind::Unknown;
+    }
+    doc.note(format!(
+        "a zip of {} members, none of them a book",
+        names.len()
+    ));
+    Kind::Unknown
+}
+
+/// The one PDF an archive exists to carry, if that is what it is.
+///
+/// Only an unambiguous wrapper qualifies. An archive holding several PDFs is
+/// a collection, and choosing among them would be a guess — better to say it
+/// is not a book than to name one chapter as the whole.
+pub fn sole_pdf(names: &[String]) -> Option<&str> {
+    let real: Vec<&String> = names
+        .iter()
+        .filter(|name| !name.ends_with('/'))
+        .filter(|name| {
+            let lower = name.to_lowercase();
+            !ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
+        })
+        .collect();
+    match real.as_slice() {
+        [only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
+        _ => None,
+    }
+}
+
+/// Text, markup, or nothing this knows.
+fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
+    let Ok(head) = source.slice(0, TEXT_WINDOW) else {
+        return Kind::Unknown;
+    };
+    if head.is_empty() {
+        doc.note("the file is empty");
+        return Kind::Unknown;
+    }
+    // A file with a null byte in its first few kilobytes is not text, and
+    // nothing else here is going to identify it either.
+    if head.contains(&0) {
+        return Kind::Unknown;
+    }
+    // The window may have cut a character in half, which is not the same
+    // thing as the file not being text.
+    let head = match std::str::from_utf8(&head) {
+        Ok(text) => text,
+        Err(e) if e.error_len().is_none() => {
+            std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
+        }
+        Err(_) => return Kind::Unknown,
+    };
+
+    let start = head
+        .trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}')
+        .to_lowercase();
+    if start.starts_with("<!doctype html")
+        || start.starts_with("<html")
+        || start.starts_with("<?xml")
+        || start.starts_with("<head")
+    {
+        return Kind::Html;
+    }
+    Kind::Txt
+}
+
+/// Whether an extension and a verdict are saying the same thing.
+fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
+    match kind {
+        Kind::Pdf => extension == "pdf",
+        Kind::Epub => extension == "epub",
+        Kind::Djvu => matches!(extension, "djvu" | "djv"),
+        Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
+        Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
+        Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
+        Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
+        Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
+        Kind::ZipWrappedPdf => extension == "zip",
+        // Nothing was recognised, so there is nothing to disagree with.
+        Kind::Unknown => true,
+    }
+}
+
+fn spelling(kind: Kind) -> &'static str {
+    match kind {
+        Kind::Pdf => "PDF",
+        Kind::Epub => "EPUB",
+        Kind::Djvu => "DjVu",
+        Kind::Azw3 => "AZW3",
+        Kind::Mobi => "MOBI",
+        Kind::Txt => "plain text",
+        Kind::Html => "HTML",
+        Kind::Ole2 => "an OLE2 document",
+        Kind::ZipWrappedPdf => "a zip holding one PDF",
+        Kind::Unknown => "nothing recognisable",
+    }
+}
+
+fn extension(name: &str) -> Option<String> {
+    std::path::Path::new(name)
+        .extension()
+        .map(|ext| ext.to_string_lossy().to_lowercase())
+}
+
+fn sample_marker(name: &str) -> Option<&'static str> {
+    let lower = std::path::Path::new(name)
+        .file_name()
+        .map(|name| name.to_string_lossy().to_lowercase())
+        .unwrap_or_default();
+    SAMPLE_MARKERS
+        .iter()
+        .find(|marker| lower.contains(**marker))
+        .copied()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn padded(head: &[u8]) -> Vec<u8> {
+        let mut out = head.to_vec();
+        out.resize(SMALLEST_BOOK as usize + 1, b'x');
+        out
+    }
+
+    fn kind_of(head: &[u8], name: &str) -> Kind {
+        sniff(&padded(head).as_slice(), name).kind
+    }
+
+    #[test]
+    fn the_magic_decides_not_the_name() {
+        assert_eq!(kind_of(b"%PDF-1.7\n", "book.epub"), Kind::Pdf);
+        assert_eq!(kind_of(DJVU, "scan.pdf"), Kind::Djvu);
+        assert_eq!(kind_of(OLE2, "book.epub"), Kind::Ole2);
+    }
+
+    #[test]
+    fn a_name_that_disagrees_with_the_bytes_is_noted() {
+        let doc = sniff(&padded(DJVU).as_slice(), "scan.pdf");
+        assert_eq!(doc.kind, Kind::Djvu);
+        assert!(
+            doc.notes.iter().any(|note| note.contains("named .pdf")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_name_that_agrees_is_not_remarked_on() {
+        let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "book.pdf");
+        assert!(doc.notes.is_empty(), "{:?}", doc.notes);
+    }
+
+    #[test]
+    fn text_and_markup_are_told_apart() {
+        assert_eq!(
+            kind_of(b"The mill stood at the northern end.", "a.txt"),
+            Kind::Txt
+        );
+        assert_eq!(kind_of(b"<!DOCTYPE html>\n<html>", "a.html"), Kind::Html);
+        assert_eq!(kind_of(b"\n\n  <html lang=\"en\">", "a.txt"), Kind::Html);
+        assert_eq!(kind_of(b"<?xml version=\"1.0\"?>", "a.xml"), Kind::Html);
+    }
+
+    #[test]
+    fn binary_rubbish_is_not_called_text() {
+        assert_eq!(kind_of(&[0x00, 0x01, 0x02, 0x03], "a.txt"), Kind::Unknown);
+        assert_eq!(kind_of(&[0xff, 0xfe, 0xfd], "a.txt"), Kind::Unknown);
+    }
+
+    #[test]
+    fn a_small_file_is_remarked_on() {
+        let doc = sniff(&b"%PDF-1.7".as_slice(), "tiny.pdf");
+        assert!(
+            doc.notes.iter().any(|note| note.contains("too small")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn a_name_that_says_sample_is_remarked_on() {
+        let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "Book - Sample Chapter.pdf");
+        assert!(
+            doc.notes.iter().any(|note| note.contains("sample")),
+            "{:?}",
+            doc.notes
+        );
+    }
+
+    #[test]
+    fn the_one_pdf_in_an_archive_is_the_book_and_several_are_not() {
+        let one = ["__MACOSX/._x".into(), "dir/".into(), "book.pdf".into()];
+        assert_eq!(sole_pdf(&one), Some("book.pdf"));
+
+        let several = ["one.pdf".to_string(), "two.pdf".to_string()];
+        assert_eq!(sole_pdf(&several), None);
+
+        let mixed = ["book.pdf".to_string(), "cover.jpg".to_string()];
+        assert_eq!(sole_pdf(&mixed), None);
+    }
+
+    #[test]
+    fn an_empty_file_is_unknown_and_says_so() {
+        let doc = sniff(&b"".as_slice(), "nothing.pdf");
+        assert_eq!(doc.kind, Kind::Unknown);
+        assert!(doc.notes.iter().any(|note| note.contains("empty")));
+        assert_eq!(doc.bytes, 0);
+    }
+
+    #[test]
+    fn the_declared_extension_is_reported_lowercased() {
+        assert_eq!(
+            sniff(&padded(b"%PDF-").as_slice(), "BOOK.PDF").declared_ext,
+            Some("pdf".to_string())
+        );
+        assert_eq!(
+            sniff(&padded(b"%PDF-").as_slice(), "no-extension").declared_ext,
+            None
+        );
+    }
+
+    #[test]
+    fn a_character_cut_in_half_by_the_window_does_not_make_it_binary() {
+        let mut head = "é".repeat(TEXT_WINDOW).into_bytes();
+        head.truncate(TEXT_WINDOW + 1);
+        assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
+    }
+}

crates/booksniff/src/main.rs

@@ -0,0 +1,99 @@
+//! Says what a file actually is, whatever its name claims.
+
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use booksniff::bytes::File;
+use clap::Parser;
+use corpus_core::{Doc, Kind, exit, ndjson};
+
+/// Identify files by what is in them rather than by what they are called.
+///
+/// Writes one doc record per file. Reads only the few kilobytes it needs —
+/// the front of the file, and an archive's directory at the back — so a
+/// directory of scans costs the same as a directory of anything else.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Exit codes: 0 every file was identified, 1 at least one was not, 74 a file
+could not be read.
+
+Example:
+
+    booksniff library/*.pdf | jq -r 'select(.kind != \"pdf\") | .path'"
+)]
+struct Args {
+    /// Files to identify.
+    #[arg(required = true)]
+    paths: Vec<PathBuf>,
+
+    /// Print a line a person can read instead of the record.
+    #[arg(long)]
+    plain: bool,
+}
+
+fn main() -> ExitCode {
+    let args: Args = match catena_cli::parse() {
+        Ok(args) => args,
+        Err(code) => return code,
+    };
+    let mut out = std::io::BufWriter::new(std::io::stdout().lock());
+    let mut unknown = false;
+    let mut unreadable = false;
+
+    for path in &args.paths {
+        let name = path.display().to_string();
+        let doc = match File::open(path) {
+            Ok(file) => booksniff::sniff(&file, &name),
+            Err(e) => {
+                eprintln!("booksniff: {name}: {e}");
+                unreadable = true;
+                continue;
+            }
+        };
+        unknown |= doc.kind == Kind::Unknown;
+
+        let wrote = if args.plain {
+            plain(&mut out, &doc)
+        } else {
+            ndjson::write_line(&mut out, &doc).map_err(|e| match e {
+                ndjson::Error::Io(e) => e,
+                other => std::io::Error::other(other.to_string()),
+            })
+        };
+        if let Err(e) = wrote {
+            // A consumer that stopped reading is not a failure.
+            if e.kind() == std::io::ErrorKind::BrokenPipe {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("booksniff: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+
+    if let Err(e) = out.flush() {
+        if e.kind() == std::io::ErrorKind::BrokenPipe {
+            return ExitCode::from(exit::OK);
+        }
+        eprintln!("booksniff: {e}");
+        return ExitCode::from(exit::IO);
+    }
+
+    ExitCode::from(match (unreadable, unknown) {
+        (true, _) => exit::IO,
+        (false, true) => exit::NO,
+        (false, false) => exit::OK,
+    })
+}
+
+fn plain(out: &mut impl Write, doc: &Doc) -> std::io::Result<()> {
+    let kind = serde_json::to_string(&doc.kind).unwrap_or_default();
+    writeln!(out, "{}\t{}", kind.trim_matches('"'), doc.path)?;
+    for note in &doc.notes {
+        writeln!(out, "\t{note}")?;
+    }
+    Ok(())
+}

crates/booksniff/src/mobi.rs

@@ -0,0 +1,108 @@
+//! Telling the two Kindle formats apart.
+//!
+//! Both are Palm databases with the same type and creator, so the extension
+//! is no help and neither is the first eight bytes. The difference is a
+//! version number inside the first record: six is the old format, eight is
+//! the one that carries real typesetting.
+
+use corpus_core::Kind;
+
+use crate::bytes::Bytes;
+
+/// Where a Palm database says what it holds.
+const TYPE_AT: u64 = 60;
+/// What a Kindle book says there.
+const BOOK_TYPE: &[u8] = b"BOOKMOBI";
+
+/// Where the first record's offset is written.
+const FIRST_RECORD_AT: u64 = 78;
+
+/// The MOBI header's own magic, at the start of the first record's header.
+const MOBI_MAGIC: &[u8] = b"MOBI";
+/// Where the MOBI header begins inside the first record.
+const MOBI_HEADER_AT: u64 = 16;
+/// How far into the MOBI header the format version is.
+const VERSION_AT: u64 = 36;
+/// The version that means the newer format.
+const KF8: u32 = 8;
+
+/// Which Kindle format this is, or `None` if it is not one.
+pub fn kind(source: &impl Bytes) -> Option<Kind> {
+    if !source.magic_at(TYPE_AT, BOOK_TYPE) {
+        return None;
+    }
+    Some(match version(source) {
+        Some(version) if version >= KF8 => Kind::Azw3,
+        _ => Kind::Mobi,
+    })
+}
+
+/// The format version in the first record, when it is readable.
+fn version(source: &impl Bytes) -> Option<u32> {
+    let record = big_endian(source, FIRST_RECORD_AT)? as u64;
+    let header = record + MOBI_HEADER_AT;
+    if !source.magic_at(header, MOBI_MAGIC) {
+        return None;
+    }
+    big_endian(source, header + VERSION_AT)
+}
+
+fn big_endian(source: &impl Bytes, at: u64) -> Option<u32> {
+    let bytes = source.slice(at, 4).ok()?;
+    (bytes.len() == 4).then(|| u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A Palm database holding one record with a MOBI header of `version`.
+    fn book(version: Option<u32>) -> Vec<u8> {
+        let record_at: u32 = 1024;
+        let mut out = vec![0u8; record_at as usize];
+        out[TYPE_AT as usize..TYPE_AT as usize + 8].copy_from_slice(BOOK_TYPE);
+        out[FIRST_RECORD_AT as usize..FIRST_RECORD_AT as usize + 4]
+            .copy_from_slice(&record_at.to_be_bytes());
+
+        if let Some(version) = version {
+            out.resize(record_at as usize + MOBI_HEADER_AT as usize, 0);
+            out.extend_from_slice(MOBI_MAGIC);
+            out.resize(
+                record_at as usize + MOBI_HEADER_AT as usize + VERSION_AT as usize,
+                0,
+            );
+            out.extend_from_slice(&version.to_be_bytes());
+        }
+        out
+    }
+
+    #[test]
+    fn the_newer_format_is_recognised() {
+        assert_eq!(kind(&book(Some(8)).as_slice()), Some(Kind::Azw3));
+        assert_eq!(kind(&book(Some(9)).as_slice()), Some(Kind::Azw3));
+    }
+
+    #[test]
+    fn the_older_format_is_recognised() {
+        assert_eq!(kind(&book(Some(6)).as_slice()), Some(Kind::Mobi));
+    }
+
+    #[test]
+    fn a_book_with_no_readable_version_is_taken_for_the_older_format() {
+        assert_eq!(kind(&book(None).as_slice()), Some(Kind::Mobi));
+    }
+
+    #[test]
+    fn something_that_is_not_a_kindle_book_is_not_one() {
+        assert_eq!(kind(&b"%PDF-1.7".as_slice()), None);
+        assert_eq!(kind(&vec![0u8; 4096].as_slice()), None);
+    }
+
+    #[test]
+    fn a_truncated_book_does_not_panic() {
+        let whole = book(Some(8));
+        for cut in 0..whole.len() {
+            let _ = kind(&&whole[..cut]);
+        }
+    }
+}

crates/booksniff/src/zip.rs

@@ -0,0 +1,158 @@
+//! Listing what is inside a zip, without unpacking any of it.
+//!
+//! An EPUB, an htmlz and a PDF someone zipped and renamed are all zips, and
+//! telling them apart needs only the member names. Those live in the central
+//! directory at the end of the file, so reading them costs a few kilobytes
+//! whatever the archive weighs — and nothing is ever decompressed, which
+//! means a malicious archive cannot expand into anything here.
+
+use crate::bytes::Bytes;
+
+/// The end-of-central-directory record's signature.
+const END_OF_DIRECTORY: &[u8] = b"PK\x05\x06";
+/// A central directory entry's signature.
+const ENTRY: &[u8] = b"PK\x01\x02";
+/// A local file header, which is what a zip starts with.
+pub const LOCAL_HEADER: &[u8] = b"PK\x03\x04";
+
+/// How far back from the end the trailing record may be.
+///
+/// Its own size plus the largest comment a zip may carry.
+const SEARCH_BACK: u64 = 22 + 0xffff;
+
+/// Most members this will list.
+///
+/// A book in an archive has tens of members; an archive with a hundred
+/// thousand is not a book, and reading all of its names would be the only
+/// expensive thing this program does.
+const MOST_MEMBERS: usize = 100_000;
+
+/// The names of an archive's members, or `None` if it is not a zip this can
+/// read.
+pub fn members(source: &impl Bytes) -> Option<Vec<String>> {
+    if !source.magic_at(0, LOCAL_HEADER) {
+        return None;
+    }
+    let (count, mut at) = directory_start(source)?;
+
+    let mut names = Vec::with_capacity(count.min(1024));
+    for _ in 0..count.min(MOST_MEMBERS) {
+        let header = source.slice(at, 46).ok()?;
+        if header.len() < 46 || header[..4] != *ENTRY {
+            break;
+        }
+        let name_len = u16::from_le_bytes([header[28], header[29]]) as u64;
+        let extra_len = u16::from_le_bytes([header[30], header[31]]) as u64;
+        let comment_len = u16::from_le_bytes([header[32], header[33]]) as u64;
+
+        let name = source.slice(at + 46, name_len as usize).ok()?;
+        // Member names are bytes, and an archive written on another machine
+        // may not have written UTF-8. What is wanted here is a comparison
+        // against ASCII, so anything else may as well be a replacement
+        // character.
+        names.push(String::from_utf8_lossy(&name).into_owned());
+        at += 46 + name_len + extra_len + comment_len;
+    }
+    Some(names)
+}
+
+/// How many entries the central directory has and where it starts.
+fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
+    let len = source.len();
+    let window = SEARCH_BACK.min(len);
+    let from = len - window;
+    let tail = source.slice(from, window as usize).ok()?;
+
+    // The record is at the end, after a comment of unknown length, so it is
+    // found by searching backwards for its signature.
+    let at = tail
+        .windows(4)
+        .rposition(|window| window == END_OF_DIRECTORY)?;
+    let record = &tail[at..];
+    if record.len() < 22 {
+        return None;
+    }
+    let count = u16::from_le_bytes([record[10], record[11]]) as usize;
+    let offset = u32::from_le_bytes([record[16], record[17], record[18], record[19]]) as u64;
+    (offset < len).then_some((count, offset))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Build a zip holding `names`, each with empty stored content.
+    fn zip(names: &[&str]) -> Vec<u8> {
+        let mut out: Vec<u8> = Vec::new();
+        let mut offsets = Vec::new();
+        for name in names {
+            offsets.push(out.len() as u32);
+            out.extend_from_slice(LOCAL_HEADER);
+            out.extend_from_slice(&[0; 22]); // version .. uncompressed size
+            out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+            out.extend_from_slice(&0u16.to_le_bytes());
+            out.extend_from_slice(name.as_bytes());
+        }
+        let directory_at = out.len() as u32;
+        for (name, offset) in names.iter().zip(&offsets) {
+            out.extend_from_slice(ENTRY);
+            out.extend_from_slice(&[0; 24]); // version .. uncompressed size
+            out.extend_from_slice(&(name.len() as u16).to_le_bytes());
+            out.extend_from_slice(&0u16.to_le_bytes()); // extra
+            out.extend_from_slice(&0u16.to_le_bytes()); // comment
+            out.extend_from_slice(&[0; 8]); // disk, attributes
+            out.extend_from_slice(&offset.to_le_bytes());
+            out.extend_from_slice(name.as_bytes());
+        }
+        out.extend_from_slice(END_OF_DIRECTORY);
+        out.extend_from_slice(&[0; 4]); // disk numbers
+        out.extend_from_slice(&(names.len() as u16).to_le_bytes());
+        out.extend_from_slice(&(names.len() as u16).to_le_bytes());
+        out.extend_from_slice(&((out.len() as u32).saturating_sub(directory_at)).to_le_bytes());
+        out.extend_from_slice(&directory_at.to_le_bytes());
+        out.extend_from_slice(&0u16.to_le_bytes()); // comment length
+        out
+    }
+
+    #[test]
+    fn the_member_names_come_back() {
+        let archive = zip(&["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]);
+        let names = members(&archive.as_slice()).expect("a zip");
+        assert_eq!(
+            names,
+            ["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]
+        );
+    }
+
+    #[test]
+    fn an_archive_with_no_members_is_not_read_as_one() {
+        // It has no local header to start with, so nothing identifies it as
+        // an archive in the first place — and an empty archive is not a book
+        // under any reading.
+        assert_eq!(members(&zip(&[]).as_slice()), None);
+    }
+
+    #[test]
+    fn something_that_is_not_a_zip_is_not_read_as_one() {
+        assert_eq!(members(&b"%PDF-1.7".as_slice()), None);
+        assert_eq!(members(&b"".as_slice()), None);
+    }
+
+    #[test]
+    fn a_truncated_archive_does_not_panic() {
+        let archive = zip(&["a.pdf", "b.pdf"]);
+        for cut in 0..archive.len() {
+            let _ = members(&&archive[..cut]);
+        }
+    }
+
+    #[test]
+    fn a_count_that_lies_stops_at_the_first_bad_entry() {
+        let mut archive = zip(&["a.pdf"]);
+        let at = archive.len() - 12;
+        // Claim a thousand entries where there is one.
+        archive[at - 6..at - 4].copy_from_slice(&1000u16.to_le_bytes());
+        let names = members(&archive.as_slice()).expect("a zip");
+        assert_eq!(names, ["a.pdf"]);
+    }
+}