diff --git a/Cargo.lock b/Cargo.lock index 9ae7aea..89ec1a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,17 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "booksniff" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "serde_json", + "textkit", +] + [[package]] name = "bstr" version = "1.13.1" @@ -99,10 +110,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "catena-cli" +version = "0.1.0" +dependencies = [ + "clap", + "corpus-core", + "serde", +] + [[package]] name = "chunkmd" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "fancy-regex 0.16.2", @@ -182,6 +203,7 @@ dependencies = [ name = "cuenorm" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "cuekit", @@ -191,6 +213,7 @@ dependencies = [ name = "cuepack" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "cuekit", @@ -201,6 +224,7 @@ dependencies = [ name = "cueseg" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "cuekit", @@ -268,6 +292,7 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" name = "mdscrub" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "regex", @@ -290,6 +315,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" name = "parity" version = "0.1.0" dependencies = [ + "booksniff", "chunkmd", "corpus-core", "cuekit", @@ -306,6 +332,7 @@ dependencies = [ name = "pdftriage" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "regex", @@ -385,6 +412,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" name = "rrf" version = "0.1.0" dependencies = [ + "catena-cli", "clap", "corpus-core", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 015f843..f8a7710 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ license = "MPL-2.0" repository = "https://git.pumpin.online/funkpower/catena" [workspace.dependencies] +catena-cli = { path = "crates/catena-cli", version = "0.1.0" } corpus-core = { path = "crates/corpus-core", version = "0.1.0" } cuekit = { path = "crates/cuekit", version = "0.1.0" } textkit = { path = "crates/textkit", version = "0.1.0" } @@ -20,6 +21,7 @@ cuenorm = { path = "crates/cuenorm", version = "0.1.0" } cueseg = { path = "crates/cueseg", version = "0.1.0" } rrf = { path = "crates/rrf", version = "0.1.0" } cuepack = { path = "crates/cuepack", version = "0.1.0" } +booksniff = { path = "crates/booksniff", version = "0.1.0" } clap = { version = "4", features = ["derive"] } fancy-regex = "0.16" diff --git a/README.md b/README.md index 445d7e4..5b770d4 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done | | | |---|---| +| `booksniff` | What a file actually is, whatever its name claims | | `mdscrub` | Converted markdown in, clean markdown out: residual HTML, control characters, headings, page markers | | `chunkmd` | Markdown into passages that carry their page, chapter and role | | `pdftriage` | Whether a PDF's text layer is healthy, garbled, sparse, permuted, or absent — and so whether OCR is worth its cost | diff --git a/crates/booksniff/Cargo.toml b/crates/booksniff/Cargo.toml new file mode 100644 index 0000000..4ee120f --- /dev/null +++ b/crates/booksniff/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "booksniff" +description = "Says what a file actually is, whatever its name claims." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +textkit.workspace = true + +[dependencies.serde_json] +workspace = true + +[lints] +workspace = true diff --git a/crates/booksniff/src/bytes.rs b/crates/booksniff/src/bytes.rs new file mode 100644 index 0000000..cac7b98 --- /dev/null +++ b/crates/booksniff/src/bytes.rs @@ -0,0 +1,132 @@ +//! Reading a few bytes of a file without reading the file. +//! +//! Sniffing a format needs the first few bytes, and sometimes a structure at +//! the end. A nine-hundred-megabyte scan should cost neither nine hundred +//! megabytes of memory nor nine hundred megabytes of reading, so everything +//! here works through a window rather than a buffer — and the same trait is +//! implemented for a slice, so the tests need no files at all. + +use std::io::{Read, Seek, SeekFrom}; + +/// Something a few bytes can be read out of, at a position. +pub trait Bytes { + /// How long the whole thing is. + fn len(&self) -> u64; + + /// Whether there is nothing at all. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Read up to `into.len()` bytes from `offset`, returning how many were + /// read. Short at the end, zero past it. + fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result; + + /// Read exactly `count` bytes from `offset`, or as many as there are. + fn slice(&self, offset: u64, count: usize) -> std::io::Result> { + let mut buffer = vec![0; count]; + let read = self.read_at(offset, &mut buffer)?; + buffer.truncate(read); + Ok(buffer) + } + + /// Whether the bytes at `offset` are exactly `magic`. + fn magic_at(&self, offset: u64, magic: &[u8]) -> bool { + self.slice(offset, magic.len()) + .is_ok_and(|read| read == magic) + } +} + +impl Bytes for &[u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + + fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result { + let Ok(offset) = usize::try_from(offset) else { + return Ok(0); + }; + if offset >= <[u8]>::len(self) { + return Ok(0); + } + let available = &self[offset..]; + let take = available.len().min(into.len()); + into[..take].copy_from_slice(&available[..take]); + Ok(take) + } +} + +/// A file on disk, read through a window. +pub struct File { + file: std::cell::RefCell, + len: u64, +} + +impl File { + /// Open a file for sniffing. + pub fn open(path: &std::path::Path) -> std::io::Result { + let file = std::fs::File::open(path)?; + let len = file.metadata()?.len(); + Ok(Self { + file: std::cell::RefCell::new(file), + len, + }) + } +} + +impl Bytes for File { + fn len(&self) -> u64 { + self.len + } + + fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result { + let mut file = self.file.borrow_mut(); + file.seek(SeekFrom::Start(offset))?; + let mut read = 0; + while read < into.len() { + match file.read(&mut into[read..])? { + 0 => break, + n => read += n, + } + } + Ok(read) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_slice_reads_from_the_middle() { + let bytes: &[u8] = b"hello world"; + assert_eq!(bytes.slice(6, 5).unwrap(), b"world"); + assert_eq!(Bytes::len(&bytes), 11); + } + + #[test] + fn reading_past_the_end_is_short_rather_than_an_error() { + let bytes: &[u8] = b"abc"; + assert_eq!(bytes.slice(1, 99).unwrap(), b"bc"); + assert!(bytes.slice(99, 4).unwrap().is_empty()); + assert!(bytes.slice(u64::MAX, 4).unwrap().is_empty()); + } + + #[test] + fn magic_matches_only_when_it_is_all_there() { + let bytes: &[u8] = b"%PDF-1.7"; + assert!(bytes.magic_at(0, b"%PDF-")); + assert!(!bytes.magic_at(0, b"%PDG-")); + assert!(!bytes.magic_at(4, b"-1.7 more")); + } + + #[test] + fn a_file_reads_the_same_as_a_slice() { + let path = std::env::temp_dir().join("booksniff-bytes-test"); + std::fs::write(&path, b"hello world").expect("a writable path"); + let file = File::open(&path).expect("the file opens"); + assert_eq!(file.len(), 11); + assert_eq!(file.slice(6, 5).unwrap(), b"world"); + assert!(file.magic_at(0, b"hello")); + } +} diff --git a/crates/booksniff/src/lib.rs b/crates/booksniff/src/lib.rs new file mode 100644 index 0000000..e5b48f7 --- /dev/null +++ b/crates/booksniff/src/lib.rs @@ -0,0 +1,335 @@ +//! Says what a file actually is, whatever its name claims. +//! +//! A library accumulates files that lie: a DjVu named `.pdf`, a Word document +//! named `.epub`, an EPUB saved as `.zip` and then never opened again, a +//! single PDF somebody zipped and renamed back to `.pdf`. Every one of those +//! fails somewhere downstream, usually as an unhelpful error from a converter +//! that was handed the wrong kind of file. +//! +//! The containers are unambiguous, so they are read rather than guessed at. + +pub mod bytes; +pub mod mobi; +pub mod zip; + +use corpus_core::{Doc, Kind}; + +use crate::bytes::Bytes; + +/// DjVu's container magic: an IFF stream tagged for the format's originator. +const DJVU: &[u8] = b"AT&TFORM"; +/// The OLE2 compound-file header every binary Word document starts with. +const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]; +/// A PDF announces its version in its first bytes. +const PDF: &[u8] = b"%PDF-"; + +/// Archive members that carry no content: resource forks and desktop clutter. +const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"]; + +/// Substrings in a name that mark a file as a sample rather than a book. +const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"]; + +/// Below this, a file is too small to be a book whatever it claims to be. +const SMALLEST_BOOK: u64 = 4096; + +/// How much of the front is read to tell text from markup. +const TEXT_WINDOW: usize = 4096; + +/// Identify a file. +/// +/// `name` is used only for the extension it claims and for the words in it, +/// both of which are reported as observations rather than believed. +pub fn sniff(source: &impl Bytes, name: &str) -> Doc { + let mut doc = Doc::new(name, Kind::Unknown, source.len() as i64); + doc.declared_ext = extension(name); + + doc.kind = if source.magic_at(0, DJVU) { + Kind::Djvu + } else if source.magic_at(0, PDF) { + Kind::Pdf + } else if source.magic_at(0, OLE2) { + Kind::Ole2 + } else if source.magic_at(0, zip::LOCAL_HEADER) { + archive(source, &mut doc) + } else if let Some(kind) = mobi::kind(source) { + kind + } else { + text(source, &mut doc) + }; + + if doc.kind == Kind::Unknown { + doc.note("no container or magic bytes this recognises"); + } + if let Some(declared) = doc.declared_ext.as_deref() + && !names_the_same_thing(declared, doc.kind) + { + doc.note(format!( + "named .{declared}, but the bytes say {}", + spelling(doc.kind) + )); + } + if source.len() < SMALLEST_BOOK { + doc.note(format!( + "{} bytes, which is too small to be a book", + source.len() + )); + } + if let Some(marker) = sample_marker(name) { + doc.note(format!("the name says {marker:?}, so it may be a sample")); + } + doc +} + +/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages. +fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind { + let Some(names) = zip::members(source) else { + doc.note("a zip whose directory could not be read"); + return Kind::Unknown; + }; + let lower: Vec = names.iter().map(|name| name.to_lowercase()).collect(); + + if lower.iter().any(|name| name == "mimetype") + && lower + .iter() + .any(|name| name.starts_with("meta-inf/container.xml")) + { + return Kind::Epub; + } + if let Some(pdf) = sole_pdf(&names) { + doc.note(format!("a zip holding one PDF, {pdf:?}")); + return Kind::ZipWrappedPdf; + } + if lower + .iter() + .any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml")) + { + doc.note("a zip of HTML documents"); + return Kind::Unknown; + } + doc.note(format!( + "a zip of {} members, none of them a book", + names.len() + )); + Kind::Unknown +} + +/// The one PDF an archive exists to carry, if that is what it is. +/// +/// Only an unambiguous wrapper qualifies. An archive holding several PDFs is +/// a collection, and choosing among them would be a guess — better to say it +/// is not a book than to name one chapter as the whole. +pub fn sole_pdf(names: &[String]) -> Option<&str> { + let real: Vec<&String> = names + .iter() + .filter(|name| !name.ends_with('/')) + .filter(|name| { + let lower = name.to_lowercase(); + !ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise)) + }) + .collect(); + match real.as_slice() { + [only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()), + _ => None, + } +} + +/// Text, markup, or nothing this knows. +fn text(source: &impl Bytes, doc: &mut Doc) -> Kind { + let Ok(head) = source.slice(0, TEXT_WINDOW) else { + return Kind::Unknown; + }; + if head.is_empty() { + doc.note("the file is empty"); + return Kind::Unknown; + } + // A file with a null byte in its first few kilobytes is not text, and + // nothing else here is going to identify it either. + if head.contains(&0) { + return Kind::Unknown; + } + // The window may have cut a character in half, which is not the same + // thing as the file not being text. + let head = match std::str::from_utf8(&head) { + Ok(text) => text, + Err(e) if e.error_len().is_none() => { + std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix") + } + Err(_) => return Kind::Unknown, + }; + + let start = head + .trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}') + .to_lowercase(); + if start.starts_with(" bool { + match kind { + Kind::Pdf => extension == "pdf", + Kind::Epub => extension == "epub", + Kind::Djvu => matches!(extension, "djvu" | "djv"), + Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"), + Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"), + Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"), + Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"), + Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"), + Kind::ZipWrappedPdf => extension == "zip", + // Nothing was recognised, so there is nothing to disagree with. + Kind::Unknown => true, + } +} + +fn spelling(kind: Kind) -> &'static str { + match kind { + Kind::Pdf => "PDF", + Kind::Epub => "EPUB", + Kind::Djvu => "DjVu", + Kind::Azw3 => "AZW3", + Kind::Mobi => "MOBI", + Kind::Txt => "plain text", + Kind::Html => "HTML", + Kind::Ole2 => "an OLE2 document", + Kind::ZipWrappedPdf => "a zip holding one PDF", + Kind::Unknown => "nothing recognisable", + } +} + +fn extension(name: &str) -> Option { + std::path::Path::new(name) + .extension() + .map(|ext| ext.to_string_lossy().to_lowercase()) +} + +fn sample_marker(name: &str) -> Option<&'static str> { + let lower = std::path::Path::new(name) + .file_name() + .map(|name| name.to_string_lossy().to_lowercase()) + .unwrap_or_default(); + SAMPLE_MARKERS + .iter() + .find(|marker| lower.contains(**marker)) + .copied() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn padded(head: &[u8]) -> Vec { + let mut out = head.to_vec(); + out.resize(SMALLEST_BOOK as usize + 1, b'x'); + out + } + + fn kind_of(head: &[u8], name: &str) -> Kind { + sniff(&padded(head).as_slice(), name).kind + } + + #[test] + fn the_magic_decides_not_the_name() { + assert_eq!(kind_of(b"%PDF-1.7\n", "book.epub"), Kind::Pdf); + assert_eq!(kind_of(DJVU, "scan.pdf"), Kind::Djvu); + assert_eq!(kind_of(OLE2, "book.epub"), Kind::Ole2); + } + + #[test] + fn a_name_that_disagrees_with_the_bytes_is_noted() { + let doc = sniff(&padded(DJVU).as_slice(), "scan.pdf"); + assert_eq!(doc.kind, Kind::Djvu); + assert!( + doc.notes.iter().any(|note| note.contains("named .pdf")), + "{:?}", + doc.notes + ); + } + + #[test] + fn a_name_that_agrees_is_not_remarked_on() { + let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "book.pdf"); + assert!(doc.notes.is_empty(), "{:?}", doc.notes); + } + + #[test] + fn text_and_markup_are_told_apart() { + assert_eq!( + kind_of(b"The mill stood at the northern end.", "a.txt"), + Kind::Txt + ); + assert_eq!(kind_of(b"\n", "a.html"), Kind::Html); + assert_eq!(kind_of(b"\n\n ", "a.txt"), Kind::Html); + assert_eq!(kind_of(b"", "a.xml"), Kind::Html); + } + + #[test] + fn binary_rubbish_is_not_called_text() { + assert_eq!(kind_of(&[0x00, 0x01, 0x02, 0x03], "a.txt"), Kind::Unknown); + assert_eq!(kind_of(&[0xff, 0xfe, 0xfd], "a.txt"), Kind::Unknown); + } + + #[test] + fn a_small_file_is_remarked_on() { + let doc = sniff(&b"%PDF-1.7".as_slice(), "tiny.pdf"); + assert!( + doc.notes.iter().any(|note| note.contains("too small")), + "{:?}", + doc.notes + ); + } + + #[test] + fn a_name_that_says_sample_is_remarked_on() { + let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "Book - Sample Chapter.pdf"); + assert!( + doc.notes.iter().any(|note| note.contains("sample")), + "{:?}", + doc.notes + ); + } + + #[test] + fn the_one_pdf_in_an_archive_is_the_book_and_several_are_not() { + let one = ["__MACOSX/._x".into(), "dir/".into(), "book.pdf".into()]; + assert_eq!(sole_pdf(&one), Some("book.pdf")); + + let several = ["one.pdf".to_string(), "two.pdf".to_string()]; + assert_eq!(sole_pdf(&several), None); + + let mixed = ["book.pdf".to_string(), "cover.jpg".to_string()]; + assert_eq!(sole_pdf(&mixed), None); + } + + #[test] + fn an_empty_file_is_unknown_and_says_so() { + let doc = sniff(&b"".as_slice(), "nothing.pdf"); + assert_eq!(doc.kind, Kind::Unknown); + assert!(doc.notes.iter().any(|note| note.contains("empty"))); + assert_eq!(doc.bytes, 0); + } + + #[test] + fn the_declared_extension_is_reported_lowercased() { + assert_eq!( + sniff(&padded(b"%PDF-").as_slice(), "BOOK.PDF").declared_ext, + Some("pdf".to_string()) + ); + assert_eq!( + sniff(&padded(b"%PDF-").as_slice(), "no-extension").declared_ext, + None + ); + } + + #[test] + fn a_character_cut_in_half_by_the_window_does_not_make_it_binary() { + let mut head = "é".repeat(TEXT_WINDOW).into_bytes(); + head.truncate(TEXT_WINDOW + 1); + assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt); + } +} diff --git a/crates/booksniff/src/main.rs b/crates/booksniff/src/main.rs new file mode 100644 index 0000000..9257f2b --- /dev/null +++ b/crates/booksniff/src/main.rs @@ -0,0 +1,99 @@ +//! Says what a file actually is, whatever its name claims. + +use std::io::Write; +use std::path::PathBuf; +use std::process::ExitCode; + +use booksniff::bytes::File; +use clap::Parser; +use corpus_core::{Doc, Kind, exit, ndjson}; + +/// Identify files by what is in them rather than by what they are called. +/// +/// Writes one doc record per file. Reads only the few kilobytes it needs — +/// the front of the file, and an archive's directory at the back — so a +/// directory of scans costs the same as a directory of anything else. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 every file was identified, 1 at least one was not, 74 a file +could not be read. + +Example: + + booksniff library/*.pdf | jq -r 'select(.kind != \"pdf\") | .path'" +)] +struct Args { + /// Files to identify. + #[arg(required = true)] + paths: Vec, + + /// Print a line a person can read instead of the record. + #[arg(long)] + plain: bool, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + let mut out = std::io::BufWriter::new(std::io::stdout().lock()); + let mut unknown = false; + let mut unreadable = false; + + for path in &args.paths { + let name = path.display().to_string(); + let doc = match File::open(path) { + Ok(file) => booksniff::sniff(&file, &name), + Err(e) => { + eprintln!("booksniff: {name}: {e}"); + unreadable = true; + continue; + } + }; + unknown |= doc.kind == Kind::Unknown; + + let wrote = if args.plain { + plain(&mut out, &doc) + } else { + ndjson::write_line(&mut out, &doc).map_err(|e| match e { + ndjson::Error::Io(e) => e, + other => std::io::Error::other(other.to_string()), + }) + }; + if let Err(e) = wrote { + // A consumer that stopped reading is not a failure. + if e.kind() == std::io::ErrorKind::BrokenPipe { + return ExitCode::from(exit::OK); + } + eprintln!("booksniff: {e}"); + return ExitCode::from(exit::IO); + } + } + + if let Err(e) = out.flush() { + if e.kind() == std::io::ErrorKind::BrokenPipe { + return ExitCode::from(exit::OK); + } + eprintln!("booksniff: {e}"); + return ExitCode::from(exit::IO); + } + + ExitCode::from(match (unreadable, unknown) { + (true, _) => exit::IO, + (false, true) => exit::NO, + (false, false) => exit::OK, + }) +} + +fn plain(out: &mut impl Write, doc: &Doc) -> std::io::Result<()> { + let kind = serde_json::to_string(&doc.kind).unwrap_or_default(); + writeln!(out, "{}\t{}", kind.trim_matches('"'), doc.path)?; + for note in &doc.notes { + writeln!(out, "\t{note}")?; + } + Ok(()) +} diff --git a/crates/booksniff/src/mobi.rs b/crates/booksniff/src/mobi.rs new file mode 100644 index 0000000..256aaff --- /dev/null +++ b/crates/booksniff/src/mobi.rs @@ -0,0 +1,108 @@ +//! Telling the two Kindle formats apart. +//! +//! Both are Palm databases with the same type and creator, so the extension +//! is no help and neither is the first eight bytes. The difference is a +//! version number inside the first record: six is the old format, eight is +//! the one that carries real typesetting. + +use corpus_core::Kind; + +use crate::bytes::Bytes; + +/// Where a Palm database says what it holds. +const TYPE_AT: u64 = 60; +/// What a Kindle book says there. +const BOOK_TYPE: &[u8] = b"BOOKMOBI"; + +/// Where the first record's offset is written. +const FIRST_RECORD_AT: u64 = 78; + +/// The MOBI header's own magic, at the start of the first record's header. +const MOBI_MAGIC: &[u8] = b"MOBI"; +/// Where the MOBI header begins inside the first record. +const MOBI_HEADER_AT: u64 = 16; +/// How far into the MOBI header the format version is. +const VERSION_AT: u64 = 36; +/// The version that means the newer format. +const KF8: u32 = 8; + +/// Which Kindle format this is, or `None` if it is not one. +pub fn kind(source: &impl Bytes) -> Option { + if !source.magic_at(TYPE_AT, BOOK_TYPE) { + return None; + } + Some(match version(source) { + Some(version) if version >= KF8 => Kind::Azw3, + _ => Kind::Mobi, + }) +} + +/// The format version in the first record, when it is readable. +fn version(source: &impl Bytes) -> Option { + let record = big_endian(source, FIRST_RECORD_AT)? as u64; + let header = record + MOBI_HEADER_AT; + if !source.magic_at(header, MOBI_MAGIC) { + return None; + } + big_endian(source, header + VERSION_AT) +} + +fn big_endian(source: &impl Bytes, at: u64) -> Option { + let bytes = source.slice(at, 4).ok()?; + (bytes.len() == 4).then(|| u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A Palm database holding one record with a MOBI header of `version`. + fn book(version: Option) -> Vec { + let record_at: u32 = 1024; + let mut out = vec![0u8; record_at as usize]; + out[TYPE_AT as usize..TYPE_AT as usize + 8].copy_from_slice(BOOK_TYPE); + out[FIRST_RECORD_AT as usize..FIRST_RECORD_AT as usize + 4] + .copy_from_slice(&record_at.to_be_bytes()); + + if let Some(version) = version { + out.resize(record_at as usize + MOBI_HEADER_AT as usize, 0); + out.extend_from_slice(MOBI_MAGIC); + out.resize( + record_at as usize + MOBI_HEADER_AT as usize + VERSION_AT as usize, + 0, + ); + out.extend_from_slice(&version.to_be_bytes()); + } + out + } + + #[test] + fn the_newer_format_is_recognised() { + assert_eq!(kind(&book(Some(8)).as_slice()), Some(Kind::Azw3)); + assert_eq!(kind(&book(Some(9)).as_slice()), Some(Kind::Azw3)); + } + + #[test] + fn the_older_format_is_recognised() { + assert_eq!(kind(&book(Some(6)).as_slice()), Some(Kind::Mobi)); + } + + #[test] + fn a_book_with_no_readable_version_is_taken_for_the_older_format() { + assert_eq!(kind(&book(None).as_slice()), Some(Kind::Mobi)); + } + + #[test] + fn something_that_is_not_a_kindle_book_is_not_one() { + assert_eq!(kind(&b"%PDF-1.7".as_slice()), None); + assert_eq!(kind(&vec![0u8; 4096].as_slice()), None); + } + + #[test] + fn a_truncated_book_does_not_panic() { + let whole = book(Some(8)); + for cut in 0..whole.len() { + let _ = kind(&&whole[..cut]); + } + } +} diff --git a/crates/booksniff/src/zip.rs b/crates/booksniff/src/zip.rs new file mode 100644 index 0000000..e4c0322 --- /dev/null +++ b/crates/booksniff/src/zip.rs @@ -0,0 +1,158 @@ +//! Listing what is inside a zip, without unpacking any of it. +//! +//! An EPUB, an htmlz and a PDF someone zipped and renamed are all zips, and +//! telling them apart needs only the member names. Those live in the central +//! directory at the end of the file, so reading them costs a few kilobytes +//! whatever the archive weighs — and nothing is ever decompressed, which +//! means a malicious archive cannot expand into anything here. + +use crate::bytes::Bytes; + +/// The end-of-central-directory record's signature. +const END_OF_DIRECTORY: &[u8] = b"PK\x05\x06"; +/// A central directory entry's signature. +const ENTRY: &[u8] = b"PK\x01\x02"; +/// A local file header, which is what a zip starts with. +pub const LOCAL_HEADER: &[u8] = b"PK\x03\x04"; + +/// How far back from the end the trailing record may be. +/// +/// Its own size plus the largest comment a zip may carry. +const SEARCH_BACK: u64 = 22 + 0xffff; + +/// Most members this will list. +/// +/// A book in an archive has tens of members; an archive with a hundred +/// thousand is not a book, and reading all of its names would be the only +/// expensive thing this program does. +const MOST_MEMBERS: usize = 100_000; + +/// The names of an archive's members, or `None` if it is not a zip this can +/// read. +pub fn members(source: &impl Bytes) -> Option> { + if !source.magic_at(0, LOCAL_HEADER) { + return None; + } + let (count, mut at) = directory_start(source)?; + + let mut names = Vec::with_capacity(count.min(1024)); + for _ in 0..count.min(MOST_MEMBERS) { + let header = source.slice(at, 46).ok()?; + if header.len() < 46 || header[..4] != *ENTRY { + break; + } + let name_len = u16::from_le_bytes([header[28], header[29]]) as u64; + let extra_len = u16::from_le_bytes([header[30], header[31]]) as u64; + let comment_len = u16::from_le_bytes([header[32], header[33]]) as u64; + + let name = source.slice(at + 46, name_len as usize).ok()?; + // Member names are bytes, and an archive written on another machine + // may not have written UTF-8. What is wanted here is a comparison + // against ASCII, so anything else may as well be a replacement + // character. + names.push(String::from_utf8_lossy(&name).into_owned()); + at += 46 + name_len + extra_len + comment_len; + } + Some(names) +} + +/// How many entries the central directory has and where it starts. +fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> { + let len = source.len(); + let window = SEARCH_BACK.min(len); + let from = len - window; + let tail = source.slice(from, window as usize).ok()?; + + // The record is at the end, after a comment of unknown length, so it is + // found by searching backwards for its signature. + let at = tail + .windows(4) + .rposition(|window| window == END_OF_DIRECTORY)?; + let record = &tail[at..]; + if record.len() < 22 { + return None; + } + let count = u16::from_le_bytes([record[10], record[11]]) as usize; + let offset = u32::from_le_bytes([record[16], record[17], record[18], record[19]]) as u64; + (offset < len).then_some((count, offset)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a zip holding `names`, each with empty stored content. + fn zip(names: &[&str]) -> Vec { + let mut out: Vec = Vec::new(); + let mut offsets = Vec::new(); + for name in names { + offsets.push(out.len() as u32); + out.extend_from_slice(LOCAL_HEADER); + out.extend_from_slice(&[0; 22]); // version .. uncompressed size + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + } + let directory_at = out.len() as u32; + for (name, offset) in names.iter().zip(&offsets) { + out.extend_from_slice(ENTRY); + out.extend_from_slice(&[0; 24]); // version .. uncompressed size + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // extra + out.extend_from_slice(&0u16.to_le_bytes()); // comment + out.extend_from_slice(&[0; 8]); // disk, attributes + out.extend_from_slice(&offset.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + } + out.extend_from_slice(END_OF_DIRECTORY); + out.extend_from_slice(&[0; 4]); // disk numbers + out.extend_from_slice(&(names.len() as u16).to_le_bytes()); + out.extend_from_slice(&(names.len() as u16).to_le_bytes()); + out.extend_from_slice(&((out.len() as u32).saturating_sub(directory_at)).to_le_bytes()); + out.extend_from_slice(&directory_at.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // comment length + out + } + + #[test] + fn the_member_names_come_back() { + let archive = zip(&["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]); + let names = members(&archive.as_slice()).expect("a zip"); + assert_eq!( + names, + ["mimetype", "META-INF/container.xml", "OEBPS/book.opf"] + ); + } + + #[test] + fn an_archive_with_no_members_is_not_read_as_one() { + // It has no local header to start with, so nothing identifies it as + // an archive in the first place — and an empty archive is not a book + // under any reading. + assert_eq!(members(&zip(&[]).as_slice()), None); + } + + #[test] + fn something_that_is_not_a_zip_is_not_read_as_one() { + assert_eq!(members(&b"%PDF-1.7".as_slice()), None); + assert_eq!(members(&b"".as_slice()), None); + } + + #[test] + fn a_truncated_archive_does_not_panic() { + let archive = zip(&["a.pdf", "b.pdf"]); + for cut in 0..archive.len() { + let _ = members(&&archive[..cut]); + } + } + + #[test] + fn a_count_that_lies_stops_at_the_first_bad_entry() { + let mut archive = zip(&["a.pdf"]); + let at = archive.len() - 12; + // Claim a thousand entries where there is one. + archive[at - 6..at - 4].copy_from_slice(&1000u16.to_le_bytes()); + let names = members(&archive.as_slice()).expect("a zip"); + assert_eq!(names, ["a.pdf"]); + } +} diff --git a/crates/booksniff/tests/cli.rs b/crates/booksniff/tests/cli.rs new file mode 100644 index 0000000..a39c7ab --- /dev/null +++ b/crates/booksniff/tests/cli.rs @@ -0,0 +1,95 @@ +//! The tool as a caller meets it: files in, records and an exit code out. + +use std::path::PathBuf; +use std::process::{Command, Output}; + +fn run(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_booksniff")) + .args(args) + .output() + .expect("the binary should run") +} + +fn written(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join("booksniff-tests"); + std::fs::create_dir_all(&dir).expect("a writable temp directory"); + let path = dir.join(name); + std::fs::write(&path, bytes).expect("a writable path"); + path +} + +fn padded(head: &[u8]) -> Vec { + let mut out = head.to_vec(); + out.resize(8192, b'x'); + out +} + +#[test] +fn a_file_it_recognises_comes_back_as_a_record() { + let path = written("book.pdf", &padded(b"%PDF-1.7\n")); + let out = run(&[path.to_str().expect("utf-8 path")]); + assert_eq!(out.status.code(), Some(0)); + + let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record"); + assert_eq!(doc["kind"], "pdf"); + assert_eq!(doc["declared_ext"], "pdf"); + assert_eq!(doc["bytes"], 8192); + assert!(out.stderr.is_empty()); +} + +#[test] +fn a_file_it_cannot_place_answers_no() { + let path = written("mystery.bin", &padded(&[0x00, 0x01, 0x02])); + let out = run(&[path.to_str().expect("utf-8 path")]); + assert_eq!(out.status.code(), Some(1), "unknown must answer no"); + let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record"); + assert_eq!(doc["kind"], "unknown"); +} + +#[test] +fn several_files_come_back_as_several_records() { + let one = written("one.pdf", &padded(b"%PDF-1.7")); + let two = written("two.djvu", &padded(b"AT&TFORM")); + let out = run(&[ + one.to_str().expect("utf-8 path"), + two.to_str().expect("utf-8 path"), + ]); + let kinds: Vec = String::from_utf8(out.stdout) + .expect("utf-8") + .lines() + .map(|line| { + serde_json::from_str::(line).expect("valid JSON")["kind"].to_string() + }) + .collect(); + assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]); +} + +#[test] +fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() { + let path = written("kept.pdf", &padded(b"%PDF-1.7")); + let out = run(&[ + "/definitely/not/here.pdf", + path.to_str().expect("utf-8 path"), + ]); + assert_eq!(out.status.code(), Some(74)); + assert!(!out.stderr.is_empty()); + // The readable file was still identified, on stdout, uncontaminated. + let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record"); + assert_eq!(doc["kind"], "pdf"); +} + +#[test] +fn the_plain_form_is_for_a_person() { + let path = written("lying.pdf", &padded(b"AT&TFORM")); + let out = run(&["--plain", path.to_str().expect("utf-8 path")]); + let said = String::from_utf8(out.stdout).expect("utf-8"); + assert!(said.starts_with("djvu\t"), "{said}"); + assert!(said.contains("named .pdf"), "{said}"); +} + +#[test] +fn no_files_named_is_a_usage_error() { + let out = run(&[]); + assert_eq!(out.status.code(), Some(64), "usage errors exit 64"); + assert!(out.stdout.is_empty()); +} diff --git a/crates/catena-cli/Cargo.toml b/crates/catena-cli/Cargo.toml new file mode 100644 index 0000000..ad92357 --- /dev/null +++ b/crates/catena-cli/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "catena-cli" +description = "The parts of a filter that every tool in this workspace does the same way." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +clap.workspace = true +corpus-core.workspace = true +serde.workspace = true + +[lints] +workspace = true diff --git a/crates/catena-cli/src/lib.rs b/crates/catena-cli/src/lib.rs new file mode 100644 index 0000000..50d8710 --- /dev/null +++ b/crates/catena-cli/src/lib.rs @@ -0,0 +1,162 @@ +//! The parts of a filter that every tool here does the same way. +//! +//! Not a framework — three functions, each of which exists because getting +//! it wrong is easy and the consequence is a pipeline that misreports why it +//! stopped: +//! +//! - a usage error exits 64, not whatever the argument parser felt like; +//! - a consumer that closed the pipe is the end of the job, not a failure; +//! - a record is flushed as it is written, so a slow stage downstream sees it +//! immediately. + +use std::io::{BufWriter, Write}; +use std::process::ExitCode; + +use corpus_core::{exit, ndjson}; +use serde::Serialize; + +/// Parse the command line, or exit. +/// +/// `--help` and `--version` are not errors and leave with 0; anything else +/// wrong with the arguments leaves with 64, which is what the rest of the +/// suite means by "the command line was wrong". +pub fn parse() -> Result { + T::try_parse().map_err(|e| { + let _ = e.print(); + match e.use_stderr() { + true => ExitCode::from(exit::USAGE), + false => ExitCode::from(exit::OK), + } + }) +} + +/// Write records to stdout as NDJSON, one per line, flushed as they go. +pub fn stream<'a, T: Serialize + 'a>( + tool: &str, + records: impl IntoIterator, +) -> ExitCode { + let mut out = BufWriter::new(std::io::stdout().lock()); + for record in records { + match ndjson::write_line(&mut out, record) { + Ok(()) => {} + // A consumer that stopped reading is not a failure: `… | head` is + // a normal way to look at a stream. + Err(e) if e.is_broken_pipe() => return ExitCode::from(exit::OK), + Err(e) => { + eprintln!("{tool}: {e}"); + return ExitCode::from(exit::IO); + } + } + } + finish(tool, out) +} + +/// Write text to stdout and leave. +pub fn text(tool: &str, text: &str) -> ExitCode { + let mut out = BufWriter::new(std::io::stdout().lock()); + match out.write_all(text.as_bytes()) { + Ok(()) => finish(tool, out), + Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), + Err(e) => { + eprintln!("{tool}: {e}"); + ExitCode::from(exit::IO) + } + } +} + +/// Flush what is buffered, treating a closed consumer as the end of the job. +pub fn finish(tool: &str, mut out: impl Write) -> ExitCode { + match out.flush() { + Ok(()) => ExitCode::from(exit::OK), + Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), + Err(e) => { + eprintln!("{tool}: {e}"); + ExitCode::from(exit::IO) + } + } +} + +/// Read the whole input as text, telling a file that cannot be read from one +/// that is not text. +/// +/// The two are different answers: the first is an I/O failure and the second +/// is bad input, and a caller routing on the exit code needs to tell them +/// apart. +pub fn read_text(path: Option<&std::path::Path>) -> Result { + let bytes = read_bytes(path)?; + String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}"))) +} + +/// Read the whole input, replacing anything that is not UTF-8. +/// +/// For the formats that routinely arrive in another encoding — a subtitle +/// file downloaded from anywhere is as likely to be Latin-1 — where a cue +/// with one wrong character still has its timing and most of its words. +pub fn read_lossy(path: Option<&std::path::Path>) -> Result { + Ok(String::from_utf8_lossy(&read_bytes(path)?).into_owned()) +} + +fn read_bytes(path: Option<&std::path::Path>) -> Result, (u8, String)> { + use std::io::Read as _; + + match path { + Some(path) => { + std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display()))) + } + None => { + let mut bytes = Vec::new(); + std::io::stdin() + .read_to_end(&mut bytes) + .map(|_| bytes) + .map_err(|e| (exit::IO, e.to_string())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reading_a_file_that_is_not_there_is_an_io_failure() { + let missing = std::path::Path::new("/definitely/not/here"); + let (code, _) = read_text(Some(missing)).expect_err("should fail"); + assert_eq!(code, exit::IO); + } + + #[test] + fn reading_bytes_that_are_not_text_is_a_data_failure() { + let path = std::env::temp_dir().join("catena-cli-not-text"); + std::fs::write(&path, [0xff, 0xfe]).expect("a writable path"); + let (code, why) = read_text(Some(&path)).expect_err("should fail"); + assert_eq!(code, exit::DATA); + assert!(why.contains("UTF-8"), "{why}"); + } + + #[test] + fn the_lossy_read_takes_what_the_strict_one_refuses() { + let path = std::env::temp_dir().join("catena-cli-lossy"); + std::fs::write(&path, [b'a', 0xff, b'b']).expect("a writable path"); + assert_eq!(read_lossy(Some(&path)).expect("should read"), "a\u{fffd}b"); + } + + #[test] + fn a_closed_consumer_is_the_end_of_the_job_and_not_a_failure() { + struct Closed; + impl Write for Closed { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Ok(0) + } + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "broken pipe", + )) + } + } + assert_eq!( + format!("{:?}", finish("test", Closed)), + format!("{:?}", ExitCode::from(exit::OK)) + ); + } +} diff --git a/crates/chunkmd/Cargo.toml b/crates/chunkmd/Cargo.toml index e52c274..7546d16 100644 --- a/crates/chunkmd/Cargo.toml +++ b/crates/chunkmd/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true fancy-regex.workspace = true diff --git a/crates/chunkmd/src/main.rs b/crates/chunkmd/src/main.rs index 6b9e17d..58aed53 100644 --- a/crates/chunkmd/src/main.rs +++ b/crates/chunkmd/src/main.rs @@ -1,13 +1,12 @@ //! Cuts markdown into retrievable passages that carry their page, chapter //! and role. -use std::io::{BufWriter, Read, Write}; use std::path::PathBuf; use std::process::ExitCode; use chunkmd::Budgets; use clap::Parser; -use corpus_core::{exit, ndjson}; +use corpus_core::exit; /// Cut markdown into chunks, one JSON record per line. /// @@ -48,7 +47,10 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; let budgets = Budgets { target: args.target, overlap: args.overlap, @@ -60,7 +62,7 @@ fn main() -> ExitCode { return ExitCode::from(exit::USAGE); } - let markdown = match read(args.path.as_deref()) { + let markdown = match catena_cli::read_text(args.path.as_deref()) { Ok(markdown) => markdown, Err((code, why)) => { eprintln!("chunkmd: {why}"); @@ -68,26 +70,7 @@ fn main() -> ExitCode { } }; - let mut out = BufWriter::new(std::io::stdout().lock()); - for chunk in chunkmd::chunk(&markdown, &budgets) { - if let Err(e) = ndjson::write_line(&mut out, &chunk) { - // A consumer that stopped reading is not a failure: `| head` is a - // normal way to look at a stream. - if e.is_broken_pipe() { - return ExitCode::from(exit::OK); - } - eprintln!("chunkmd: {e}"); - return ExitCode::from(exit::IO); - } - } - match out.flush() { - Ok(()) => ExitCode::from(exit::OK), - Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), - Err(e) => { - eprintln!("chunkmd: {e}"); - ExitCode::from(exit::IO) - } - } + catena_cli::stream("chunkmd", &chunkmd::chunk(&markdown, &budgets)) } /// Budgets that cannot produce anything sensible, rejected before reading @@ -116,22 +99,3 @@ fn sane(budgets: &Budgets) -> Result<(), String> { } Ok(()) } - -/// Read the input, distinguishing a file that cannot be read from one that -/// is not text: the first is an I/O failure and the second is bad input, and -/// a caller routing on the exit code needs to tell them apart. -fn read(path: Option<&std::path::Path>) -> Result { - let bytes = match path { - Some(path) => { - std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display()))) - } - None => { - let mut bytes = Vec::new(); - std::io::stdin() - .read_to_end(&mut bytes) - .map(|_| bytes) - .map_err(|e| (exit::IO, e.to_string())) - } - }?; - String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}"))) -} diff --git a/crates/cuenorm/Cargo.toml b/crates/cuenorm/Cargo.toml index 86c0c3e..59a3f7d 100644 --- a/crates/cuenorm/Cargo.toml +++ b/crates/cuenorm/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true cuekit.workspace = true diff --git a/crates/cuenorm/src/main.rs b/crates/cuenorm/src/main.rs index 51eadc0..dab2a20 100644 --- a/crates/cuenorm/src/main.rs +++ b/crates/cuenorm/src/main.rs @@ -1,11 +1,9 @@ //! Any subtitle format in, one normalised cue stream out. -use std::io::{BufWriter, Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::ExitCode; use clap::Parser; -use corpus_core::{exit, ndjson}; /// Read a subtitle file — SubRip, WebVTT or SubStation Alpha — and write one /// cue per line as JSON. @@ -35,13 +33,16 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; - let content = match read(args.path.as_deref()) { + let content = match catena_cli::read_lossy(args.path.as_deref()) { Ok(content) => content, - Err(e) => { - eprintln!("cuenorm: {e}"); - return ExitCode::from(exit::IO); + Err((code, why)) => { + eprintln!("cuenorm: {why}"); + return ExitCode::from(code); } }; @@ -52,42 +53,5 @@ fn main() -> ExitCode { cuekit::merge_repeats(&cues) }; - let mut out = BufWriter::new(std::io::stdout().lock()); - for cue in &cues { - if let Err(e) = ndjson::write_line(&mut out, cue) { - // A consumer that stopped reading is not a failure. - if e.is_broken_pipe() { - return ExitCode::from(exit::OK); - } - eprintln!("cuenorm: {e}"); - return ExitCode::from(exit::IO); - } - } - match out.flush() { - Ok(()) => ExitCode::from(exit::OK), - Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), - Err(e) => { - eprintln!("cuenorm: {e}"); - ExitCode::from(exit::IO) - } - } -} - -/// Read the file as text, tolerating the encodings a subtitle file arrives in. -/// -/// Not UTF-8 is common: a subtitle file downloaded from anywhere is as likely -/// to be Latin-1, and refusing it would make the tool useless for exactly the -/// archives that need it. Undecodable bytes become the replacement character -/// rather than an error, because a cue with one wrong character still has its -/// timing and most of its words. -fn read(path: Option<&Path>) -> std::io::Result { - let bytes = match path { - Some(path) => std::fs::read(path)?, - None => { - let mut bytes = Vec::new(); - std::io::stdin().read_to_end(&mut bytes)?; - bytes - } - }; - Ok(String::from_utf8_lossy(&bytes).into_owned()) + catena_cli::stream("cuenorm", &cues) } diff --git a/crates/cuepack/Cargo.toml b/crates/cuepack/Cargo.toml index 623f331..ed309ab 100644 --- a/crates/cuepack/Cargo.toml +++ b/crates/cuepack/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true cuekit.workspace = true diff --git a/crates/cuepack/src/main.rs b/crates/cuepack/src/main.rs index 0ab5046..64a637e 100644 --- a/crates/cuepack/src/main.rs +++ b/crates/cuepack/src/main.rs @@ -1,6 +1,5 @@ //! Packs cues into passages worth embedding. -use std::io::{BufWriter, Read, Write}; use std::path::PathBuf; use std::process::ExitCode; @@ -38,7 +37,10 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; let budgets = Budgets { seconds: args.seconds, max_tokens: args.max_tokens, @@ -48,11 +50,11 @@ fn main() -> ExitCode { return ExitCode::from(exit::USAGE); } - let content = match read(args.path.as_deref()) { + let content = match catena_cli::read_lossy(args.path.as_deref()) { Ok(content) => content, - Err(e) => { - eprintln!("cuepack: {e}"); - return ExitCode::from(exit::IO); + Err((code, why)) => { + eprintln!("cuepack: {why}"); + return ExitCode::from(code); } }; @@ -64,25 +66,7 @@ fn main() -> ExitCode { } }; - let mut out = BufWriter::new(std::io::stdout().lock()); - for passage in cuepack::pack(&cues, &budgets) { - if let Err(e) = ndjson::write_line(&mut out, &passage) { - // A consumer that stopped reading is not a failure. - if e.is_broken_pipe() { - return ExitCode::from(exit::OK); - } - eprintln!("cuepack: {e}"); - return ExitCode::from(exit::IO); - } - } - match out.flush() { - Ok(()) => ExitCode::from(exit::OK), - Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), - Err(e) => { - eprintln!("cuepack: {e}"); - ExitCode::from(exit::IO) - } - } + catena_cli::stream("cuepack", &cuepack::pack(&cues, &budgets)) } /// Read cues from either encoding: a cue stream begins with a JSON object @@ -97,15 +81,3 @@ fn cues(content: &str) -> Result, ndjson::Error> { } Ok(cuekit::parse(content)) } - -fn read(path: Option<&std::path::Path>) -> std::io::Result { - let bytes = match path { - Some(path) => std::fs::read(path)?, - None => { - let mut bytes = Vec::new(); - std::io::stdin().read_to_end(&mut bytes)?; - bytes - } - }; - Ok(String::from_utf8_lossy(&bytes).into_owned()) -} diff --git a/crates/cueseg/Cargo.toml b/crates/cueseg/Cargo.toml index 6f6e210..f82f500 100644 --- a/crates/cueseg/Cargo.toml +++ b/crates/cueseg/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true cuekit.workspace = true diff --git a/crates/cueseg/src/main.rs b/crates/cueseg/src/main.rs index 13a276c..8befb91 100644 --- a/crates/cueseg/src/main.rs +++ b/crates/cueseg/src/main.rs @@ -1,7 +1,6 @@ //! Turns a timed transcript into topic chapters. -use std::io::{BufWriter, Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::ExitCode; use clap::Parser; @@ -38,13 +37,16 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; - let content = match read(args.path.as_deref()) { + let content = match catena_cli::read_lossy(args.path.as_deref()) { Ok(content) => content, - Err(e) => { - eprintln!("cueseg: {e}"); - return ExitCode::from(exit::IO); + Err((code, why)) => { + eprintln!("cueseg: {why}"); + return ExitCode::from(code); } }; @@ -65,25 +67,7 @@ fn main() -> ExitCode { None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max), }; - let mut out = BufWriter::new(std::io::stdout().lock()); - for chapter in cueseg::segment(&cues, duration) { - if let Err(e) = ndjson::write_line(&mut out, &chapter) { - // A consumer that stopped reading is not a failure. - if e.is_broken_pipe() { - return ExitCode::from(exit::OK); - } - eprintln!("cueseg: {e}"); - return ExitCode::from(exit::IO); - } - } - match out.flush() { - Ok(()) => ExitCode::from(exit::OK), - Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), - Err(e) => { - eprintln!("cueseg: {e}"); - ExitCode::from(exit::IO) - } - } + catena_cli::stream("cueseg", &cueseg::segment(&cues, duration)) } /// Read cues from either encoding. @@ -101,15 +85,3 @@ fn cues(content: &str) -> Result, ndjson::Error> { } Ok(cuekit::parse(content)) } - -fn read(path: Option<&Path>) -> std::io::Result { - let bytes = match path { - Some(path) => std::fs::read(path)?, - None => { - let mut bytes = Vec::new(); - std::io::stdin().read_to_end(&mut bytes)?; - bytes - } - }; - Ok(String::from_utf8_lossy(&bytes).into_owned()) -} diff --git a/crates/mdscrub/Cargo.toml b/crates/mdscrub/Cargo.toml index 82a9406..1cc0e55 100644 --- a/crates/mdscrub/Cargo.toml +++ b/crates/mdscrub/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true regex.workspace = true diff --git a/crates/mdscrub/src/main.rs b/crates/mdscrub/src/main.rs index 70959eb..e85e545 100644 --- a/crates/mdscrub/src/main.rs +++ b/crates/mdscrub/src/main.rs @@ -1,11 +1,9 @@ //! Cleans converted markdown. -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::ExitCode; use clap::Parser; -use corpus_core::exit; /// Clean converted markdown: drop residual HTML and control characters, /// make headings readable, and spell every page boundary the same way. @@ -27,9 +25,12 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; - let markdown = match read(args.path.as_deref()) { + let markdown = match catena_cli::read_text(args.path.as_deref()) { Ok(markdown) => markdown, Err((code, why)) => { eprintln!("mdscrub: {why}"); @@ -37,33 +38,5 @@ fn main() -> ExitCode { } }; - let cleaned = mdscrub::scrub(&markdown); - match std::io::stdout().write_all(cleaned.as_bytes()) { - Ok(()) => ExitCode::from(exit::OK), - // A consumer that stopped reading is not a failure. - Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), - Err(e) => { - eprintln!("mdscrub: {e}"); - ExitCode::from(exit::IO) - } - } -} - -/// Read the input, distinguishing a file that cannot be read from one that -/// is not text: the first is an I/O failure and the second is bad input, and -/// a caller routing on the exit code needs to tell them apart. -fn read(path: Option<&Path>) -> Result { - let bytes = match path { - Some(path) => { - std::fs::read(path).map_err(|e| (exit::IO, format!("{}: {e}", path.display()))) - } - None => { - let mut bytes = Vec::new(); - std::io::stdin() - .read_to_end(&mut bytes) - .map(|_| bytes) - .map_err(|e| (exit::IO, e.to_string())) - } - }?; - String::from_utf8(bytes).map_err(|e| (exit::DATA, format!("input is not UTF-8: {e}"))) + catena_cli::text("mdscrub", &mdscrub::scrub(&markdown)) } diff --git a/crates/pdftriage/Cargo.toml b/crates/pdftriage/Cargo.toml index f1c9060..ebe3b0e 100644 --- a/crates/pdftriage/Cargo.toml +++ b/crates/pdftriage/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true regex.workspace = true diff --git a/crates/pdftriage/src/main.rs b/crates/pdftriage/src/main.rs index 8540ef2..fc53b17 100644 --- a/crates/pdftriage/src/main.rs +++ b/crates/pdftriage/src/main.rs @@ -49,7 +49,10 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; let bytes = match std::fs::metadata(&args.path) { Ok(meta) => meta.len() as i64, diff --git a/crates/rrf/Cargo.toml b/crates/rrf/Cargo.toml index 4187525..a29e643 100644 --- a/crates/rrf/Cargo.toml +++ b/crates/rrf/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true repository.workspace = true [dependencies] +catena-cli.workspace = true clap.workspace = true corpus-core.workspace = true serde_json.workspace = true diff --git a/crates/rrf/src/main.rs b/crates/rrf/src/main.rs index 3522efe..8de8e67 100644 --- a/crates/rrf/src/main.rs +++ b/crates/rrf/src/main.rs @@ -1,6 +1,5 @@ //! Reciprocal rank fusion: several ranked lists in, one ranked list out. -use std::io::{BufWriter, Read, Write}; use std::path::PathBuf; use std::process::ExitCode; @@ -49,7 +48,10 @@ struct Args { } fn main() -> ExitCode { - let args = Args::parse(); + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; let weights = match weights(&args) { Ok(weights) => weights, @@ -59,24 +61,14 @@ fn main() -> ExitCode { } }; - let hits: Result, _> = match args.path.as_deref() { - Some(path) => match std::fs::File::open(path) { - Ok(file) => ndjson::read(std::io::BufReader::new(file)).collect(), - Err(e) => { - eprintln!("rrf: {}: {e}", path.display()); - return ExitCode::from(exit::IO); - } - }, - None => { - let mut text = Vec::new(); - if let Err(e) = std::io::stdin().read_to_end(&mut text) { - eprintln!("rrf: {e}"); - return ExitCode::from(exit::IO); - } - ndjson::read(text.as_slice()).collect() + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("rrf: {why}"); + return ExitCode::from(code); } }; - let hits = match hits { + let hits = match ndjson::read(text.as_bytes()).collect::, _>>() { Ok(hits) => hits, Err(e) => { eprintln!("rrf: {e}"); @@ -91,25 +83,7 @@ fn main() -> ExitCode { let fused = rrf::fuse(&hits, &weights); let fused = &fused[..args.limit.unwrap_or(fused.len()).min(fused.len())]; - let mut out = BufWriter::new(std::io::stdout().lock()); - for hit in fused { - if let Err(e) = ndjson::write_line(&mut out, hit) { - // A consumer that stopped reading is not a failure. - if e.is_broken_pipe() { - return ExitCode::from(exit::OK); - } - eprintln!("rrf: {e}"); - return ExitCode::from(exit::IO); - } - } - match out.flush() { - Ok(()) => ExitCode::from(exit::OK), - Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), - Err(e) => { - eprintln!("rrf: {e}"); - ExitCode::from(exit::IO) - } - } + catena_cli::stream("rrf", fused) } fn weights(args: &Args) -> Result { diff --git a/parity/Cargo.toml b/parity/Cargo.toml index 6dc27c5..6a0cde3 100644 --- a/parity/Cargo.toml +++ b/parity/Cargo.toml @@ -14,6 +14,7 @@ publish = false python = [] [dependencies] +booksniff.workspace = true chunkmd.workspace = true cuekit.workspace = true cuepack.workspace = true diff --git a/parity/cases/booksniff/files.ndjson b/parity/cases/booksniff/files.ndjson new file mode 100644 index 0000000..45943b2 --- /dev/null +++ b/parity/cases/booksniff/files.ndjson @@ -0,0 +1,20 @@ +{"name": "plain.pdf", "hex": "255044462d312e370a25e2e3cfd30a78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878"} +{"name": "scan.pdf", "hex": "41542654464f524d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"} +{"name": "book.epub", "hex": "d0cf11e0a1b11ae100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"} +{"name": "notes.doc", "hex": "5468652050726f6a20706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f63"} +{"name": "empty.pdf", "hex": ""} +{"name": "tiny.txt", "hex": "6869"} +{"name": "prose.txt", "hex": "546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a"} +{"name": "page.html", "hex": "3c21444f43545950452068746d6c3e0a3c68746d6c3e3c626f64793e746578743c2f626f64793e3c2f68746d6c3e"} +{"name": "spine.xhtml", "hex": "3c3f786d6c2076657273696f6e3d22312e30223f3e0a3c68746d6c20786d6c6e733d2278223e3c626f64792f3e3c2f68746d6c3e"} +{"name": "real.epub", "hex": "504b03041400000000009164335d6f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b03041400000000009164335d54995f930c0000000c000000160000004d4554412d494e462f636f6e7461696e65722e786d6c3c636f6e7461696e65722f3e504b03041400000000009164335d9b7d083a0a0000000a0000000e0000004f454250532f626f6f6b2e6f70663c7061636b6167652f3e504b010214031400000000009164335d6f61ab2c14000000140000000800000000000000000000008001000000006d696d6574797065504b010214031400000000009164335d54995f930c0000000c00000016000000000000000000000080013a0000004d4554412d494e462f636f6e7461696e65722e786d6c504b010214031400000000009164335d9b7d083a0a0000000a0000000e000000000000000000000080017a0000004f454250532f626f6f6b2e6f7066504b05060000000003000300b6000000b00000000000"} +{"name": "saved.zip", "hex": "504b03041400000000009164335d6f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b03041400000000009164335d54995f930c0000000c000000160000004d4554412d494e462f636f6e7461696e65722e786d6c3c636f6e7461696e65722f3e504b010214031400000000009164335d6f61ab2c14000000140000000800000000000000000000008001000000006d696d6574797065504b010214031400000000009164335d54995f930c0000000c00000016000000000000000000000080013a0000004d4554412d494e462f636f6e7461696e65722e786d6c504b050600000000020002007a0000007a0000000000"} +{"name": "wrapped.pdf", "hex": "504b03041400000000009164335da36258a4100000001000000008000000626f6f6b2e706466255044462d312e3720636f6e74656e74504b010214031400000000009164335da36258a41000000010000000080000000000000000000000800100000000626f6f6b2e706466504b0506000000000100010036000000360000000000"} +{"name": "wrapped-noisy.zip", "hex": "504b03041400000000009164335d399cfb060400000004000000130000005f5f4d41434f53582f2e5f626f6f6b2e7064666a756e6b504b03041400000000009164335d399cfb060400000004000000090000002e44535f53746f72656a756e6b504b03041400000000009164335da36258a4100000001000000008000000626f6f6b2e706466255044462d312e3720636f6e74656e74504b010214031400000000009164335d399cfb0604000000040000001300000000000000000000008001000000005f5f4d41434f53582f2e5f626f6f6b2e706466504b010214031400000000009164335d399cfb0604000000040000000900000000000000000000008001350000002e44535f53746f7265504b010214031400000000009164335da36258a41000000010000000080000000000000000000000800160000000626f6f6b2e706466504b05060000000003000300ae000000960000000000"} +{"name": "collection.zip", "hex": "504b03041400000000009164335d43beb7e80100000001000000070000006f6e652e70646661504b03041400000000009164335df9efbe7101000000010000000700000074776f2e70646662504b010214031400000000009164335d43beb7e801000000010000000700000000000000000000008001000000006f6e652e706466504b010214031400000000009164335df9efbe71010000000100000007000000000000000000000080012600000074776f2e706466504b050600000000020002006a0000004c0000000000"} +{"name": "pages.zip", "hex": "504b03041400000000009164335d3ce3a64107000000070000000a000000696e6465782e68746d6c3c68746d6c2f3e504b03041400000000009164335d8316dc8c0100000001000000090000007374796c652e63737378504b010214031400000000009164335d3ce3a64107000000070000000a0000000000000000000000800100000000696e6465782e68746d6c504b010214031400000000009164335d8316dc8c010000000100000009000000000000000000000080012f0000007374796c652e637373504b050600000000020002006f000000570000000000"} +{"name": "nothing.zip", "hex": "504b03041400000000009164335d8316dc8c01000000010000000a000000726561646d652e74787478504b010214031400000000009164335d8316dc8c01000000010000000a0000000000000000000000800100000000726561646d652e747874504b0506000000000100010038000000290000000000"} +{"name": "uppercase.zip", "hex": "504b03041400000000009164335d18eb5070080000000800000008000000424f4f4b2e504446255044462d312e37504b010214031400000000009164335d18eb50700800000008000000080000000000000000000000800100000000424f4f4b2e504446504b05060000000001000100360000002e0000000000"} +{"name": "subdir.zip", "hex": "504b03041400000000009164335d18eb507008000000080000000c0000006469722f626f6f6b2e706466255044462d312e37504b010214031400000000009164335d18eb507008000000080000000c00000000000000000000008001000000006469722f626f6f6b2e706466504b050600000000010001003a000000320000000000"} +{"name": "truncated.zip", "hex": "504b03041400000000009164335d18eb5070080000000800000008000000626f6f6b2e7064662550"} +{"name": "Book - Sample Chapter.pdf", "hex": "255044462d312e370a25e2e3cfd30a78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878"} diff --git a/parity/golden/booksniff/files.ndjson b/parity/golden/booksniff/files.ndjson new file mode 100644 index 0000000..885dc50 --- /dev/null +++ b/parity/golden/booksniff/files.ndjson @@ -0,0 +1,20 @@ +{"djvu": false, "name": "plain.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": true, "name": "scan.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "book.epub", "ole2": true, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "notes.doc", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "empty.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "tiny.txt", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "prose.txt", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "page.html", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "spine.xhtml", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} +{"djvu": false, "name": "real.epub", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "epub"} +{"djvu": false, "name": "saved.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "epub"} +{"djvu": false, "name": "wrapped.pdf", "ole2": false, "sole_pdf": "book.pdf", "zip": true, "zip_kind": "pdf"} +{"djvu": false, "name": "wrapped-noisy.zip", "ole2": false, "sole_pdf": "book.pdf", "zip": true, "zip_kind": "pdf"} +{"djvu": false, "name": "collection.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"} +{"djvu": false, "name": "pages.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "htmlz"} +{"djvu": false, "name": "nothing.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"} +{"djvu": false, "name": "uppercase.zip", "ole2": false, "sole_pdf": "BOOK.PDF", "zip": true, "zip_kind": "pdf"} +{"djvu": false, "name": "subdir.zip", "ole2": false, "sole_pdf": "dir/book.pdf", "zip": true, "zip_kind": "pdf"} +{"djvu": false, "name": "truncated.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"} +{"djvu": false, "name": "Book - Sample Chapter.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null} diff --git a/parity/oracle/booksniff.py b/parity/oracle/booksniff.py new file mode 100644 index 0000000..461ba3c --- /dev/null +++ b/parity/oracle/booksniff.py @@ -0,0 +1,57 @@ +"""Identify containers the way the Python pipeline identifies them. + +Reads one `{"name", "hex"}` object per line, writes the bytes to a temporary +file under that name, and reports what each of the Python's container tests +makes of it. + + CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \ + python3 oracle/booksniff.py < cases/booksniff/files.ndjson +""" + +import json +import os +import sys +import tempfile +from pathlib import Path + +for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep): + if path: + sys.path.insert(0, path) + +from book_pipeline import convert # noqa: E402 + + +def main() -> int: + out = sys.stdout + with tempfile.TemporaryDirectory() as tmp: + for line in sys.stdin.buffer.read().decode("utf-8").split("\n"): + if not line.strip(): + continue + case = json.loads(line) + path = Path(tmp) / case["name"] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(bytes.fromhex(case["hex"])) + + record = { + "name": case["name"], + "djvu": convert._is_djvu_document(path), + "zip": convert._is_zip_container(path), + "ole2": convert._is_ole2_document(path), + "zip_kind": None, + "sole_pdf": None, + } + if record["zip"]: + record["zip_kind"] = convert._classify_zip(path) + try: + import zipfile + + with zipfile.ZipFile(path) as z: + record["sole_pdf"] = convert._sole_pdf_member(z.namelist()) + except Exception: + record["sole_pdf"] = None + out.write(json.dumps(record, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index c257d38..d972d2e 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 9] = [ + let jobs: [Job; 10] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -29,6 +29,7 @@ fn main() -> ExitCode { ("cueseg", cueseg), ("rrf", rrf), ("cuepack", cuepack), + ("booksniff", booksniff), ]; for (name, job) in jobs { @@ -72,6 +73,18 @@ fn unescape() -> Written { Ok(vec![path.display().to_string()]) } +fn booksniff() -> Written { + let dir = parity::golden("booksniff"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/booksniff/files.ndjson"); + let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let verdicts = parity::run_oracle("booksniff.py", &cases)?; + let path = dir.join("files.ndjson"); + std::fs::write(&path, verdicts).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn cuepack() -> Written { let dir = parity::golden("cuepack"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/booksniff.rs b/parity/tests/booksniff.rs new file mode 100644 index 0000000..171929e --- /dev/null +++ b/parity/tests/booksniff.rs @@ -0,0 +1,93 @@ +//! Container identification must agree with the Python's. +//! +//! The consequence of getting this wrong is not a wrong record: it is a +//! converter handed a file it cannot read, and a book that never enters the +//! corpus with nothing but an unhelpful error to say why. + +use booksniff::bytes::Bytes; +use corpus_core::Kind; +use serde_json::Value; + +fn cases() -> Vec { + let text = std::fs::read_to_string(parity::dir().join("cases/booksniff/files.ndjson")) + .expect("a readable case file"); + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn bytes_of(case: &Value) -> Vec { + let hex = case["hex"].as_str().expect("hex bytes"); + (0..hex.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&hex[at..at + 2], 16).expect("valid hex")) + .collect() +} + +#[test] +fn every_container_is_read_the_same_as_python() { + let golden: Vec = parity::read_golden_text("booksniff/files.ndjson") + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect(); + let cases = cases(); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + for (case, theirs) in cases.iter().zip(&golden) { + let name = case["name"].as_str().expect("a name"); + assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order"); + let bytes = bytes_of(case); + let source = bytes.as_slice(); + let doc = booksniff::sniff(&source, name); + + // The three magic-byte tests, each of which the Python answers + // directly. + assert_eq!( + doc.kind == Kind::Djvu, + theirs["djvu"].as_bool().expect("a verdict"), + "{name}: DjVu" + ); + assert_eq!( + doc.kind == Kind::Ole2, + theirs["ole2"].as_bool().expect("a verdict"), + "{name}: OLE2" + ); + assert_eq!( + source.magic_at(0, booksniff::zip::LOCAL_HEADER), + theirs["zip"].as_bool().expect("a verdict"), + "{name}: zip" + ); + + // What the archive holds, where the Python classifies one. + if let Some(kind) = theirs["zip_kind"].as_str() { + let ours = match doc.kind { + Kind::Epub => "epub", + Kind::ZipWrappedPdf => "pdf", + _ if doc.notes.iter().any(|note| note.contains("HTML")) => "htmlz", + _ => "other", + }; + assert_eq!(ours, kind, "{name}: what the zip holds"); + } + + // And which member it would unwrap. + if let Some(names) = booksniff::zip::members(&source) { + assert_eq!( + booksniff::sole_pdf(&names), + theirs["sole_pdf"].as_str(), + "{name}: the sole PDF" + ); + } + } +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/booksniff/files.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("booksniff.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("booksniff/files.ndjson")); +}