mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
booksniff, and one place for the parts every tool shares
This commit is contained in:
funkpower
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "booksniff"
|
||||
description = "Says what a file actually is, whatever its name claims."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
textkit.workspace = true
|
||||
|
||||
[dependencies.serde_json]
|
||||
workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Reading a few bytes of a file without reading the file.
|
||||
//!
|
||||
//! Sniffing a format needs the first few bytes, and sometimes a structure at
|
||||
//! the end. A nine-hundred-megabyte scan should cost neither nine hundred
|
||||
//! megabytes of memory nor nine hundred megabytes of reading, so everything
|
||||
//! here works through a window rather than a buffer — and the same trait is
|
||||
//! implemented for a slice, so the tests need no files at all.
|
||||
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
|
||||
/// Something a few bytes can be read out of, at a position.
|
||||
pub trait Bytes {
|
||||
/// How long the whole thing is.
|
||||
fn len(&self) -> u64;
|
||||
|
||||
/// Whether there is nothing at all.
|
||||
fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Read up to `into.len()` bytes from `offset`, returning how many were
|
||||
/// read. Short at the end, zero past it.
|
||||
fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize>;
|
||||
|
||||
/// Read exactly `count` bytes from `offset`, or as many as there are.
|
||||
fn slice(&self, offset: u64, count: usize) -> std::io::Result<Vec<u8>> {
|
||||
let mut buffer = vec![0; count];
|
||||
let read = self.read_at(offset, &mut buffer)?;
|
||||
buffer.truncate(read);
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
/// Whether the bytes at `offset` are exactly `magic`.
|
||||
fn magic_at(&self, offset: u64, magic: &[u8]) -> bool {
|
||||
self.slice(offset, magic.len())
|
||||
.is_ok_and(|read| read == magic)
|
||||
}
|
||||
}
|
||||
|
||||
impl Bytes for &[u8] {
|
||||
fn len(&self) -> u64 {
|
||||
<[u8]>::len(self) as u64
|
||||
}
|
||||
|
||||
fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
|
||||
let Ok(offset) = usize::try_from(offset) else {
|
||||
return Ok(0);
|
||||
};
|
||||
if offset >= <[u8]>::len(self) {
|
||||
return Ok(0);
|
||||
}
|
||||
let available = &self[offset..];
|
||||
let take = available.len().min(into.len());
|
||||
into[..take].copy_from_slice(&available[..take]);
|
||||
Ok(take)
|
||||
}
|
||||
}
|
||||
|
||||
/// A file on disk, read through a window.
|
||||
pub struct File {
|
||||
file: std::cell::RefCell<std::fs::File>,
|
||||
len: u64,
|
||||
}
|
||||
|
||||
impl File {
|
||||
/// Open a file for sniffing.
|
||||
pub fn open(path: &std::path::Path) -> std::io::Result<Self> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
Ok(Self {
|
||||
file: std::cell::RefCell::new(file),
|
||||
len,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Bytes for File {
|
||||
fn len(&self) -> u64 {
|
||||
self.len
|
||||
}
|
||||
|
||||
fn read_at(&self, offset: u64, into: &mut [u8]) -> std::io::Result<usize> {
|
||||
let mut file = self.file.borrow_mut();
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
let mut read = 0;
|
||||
while read < into.len() {
|
||||
match file.read(&mut into[read..])? {
|
||||
0 => break,
|
||||
n => read += n,
|
||||
}
|
||||
}
|
||||
Ok(read)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_slice_reads_from_the_middle() {
|
||||
let bytes: &[u8] = b"hello world";
|
||||
assert_eq!(bytes.slice(6, 5).unwrap(), b"world");
|
||||
assert_eq!(Bytes::len(&bytes), 11);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reading_past_the_end_is_short_rather_than_an_error() {
|
||||
let bytes: &[u8] = b"abc";
|
||||
assert_eq!(bytes.slice(1, 99).unwrap(), b"bc");
|
||||
assert!(bytes.slice(99, 4).unwrap().is_empty());
|
||||
assert!(bytes.slice(u64::MAX, 4).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn magic_matches_only_when_it_is_all_there() {
|
||||
let bytes: &[u8] = b"%PDF-1.7";
|
||||
assert!(bytes.magic_at(0, b"%PDF-"));
|
||||
assert!(!bytes.magic_at(0, b"%PDG-"));
|
||||
assert!(!bytes.magic_at(4, b"-1.7 more"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_reads_the_same_as_a_slice() {
|
||||
let path = std::env::temp_dir().join("booksniff-bytes-test");
|
||||
std::fs::write(&path, b"hello world").expect("a writable path");
|
||||
let file = File::open(&path).expect("the file opens");
|
||||
assert_eq!(file.len(), 11);
|
||||
assert_eq!(file.slice(6, 5).unwrap(), b"world");
|
||||
assert!(file.magic_at(0, b"hello"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
//! Says what a file actually is, whatever its name claims.
|
||||
//!
|
||||
//! A library accumulates files that lie: a DjVu named `.pdf`, a Word document
|
||||
//! named `.epub`, an EPUB saved as `.zip` and then never opened again, a
|
||||
//! single PDF somebody zipped and renamed back to `.pdf`. Every one of those
|
||||
//! fails somewhere downstream, usually as an unhelpful error from a converter
|
||||
//! that was handed the wrong kind of file.
|
||||
//!
|
||||
//! The containers are unambiguous, so they are read rather than guessed at.
|
||||
|
||||
pub mod bytes;
|
||||
pub mod mobi;
|
||||
pub mod zip;
|
||||
|
||||
use corpus_core::{Doc, Kind};
|
||||
|
||||
use crate::bytes::Bytes;
|
||||
|
||||
/// DjVu's container magic: an IFF stream tagged for the format's originator.
|
||||
const DJVU: &[u8] = b"AT&TFORM";
|
||||
/// The OLE2 compound-file header every binary Word document starts with.
|
||||
const OLE2: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1];
|
||||
/// A PDF announces its version in its first bytes.
|
||||
const PDF: &[u8] = b"%PDF-";
|
||||
|
||||
/// Archive members that carry no content: resource forks and desktop clutter.
|
||||
const ARCHIVE_NOISE: &[&str] = &["__macosx/", ".ds_store", "thumbs.db"];
|
||||
|
||||
/// Substrings in a name that mark a file as a sample rather than a book.
|
||||
const SAMPLE_MARKERS: &[&str] = &["sample", "readme", "read me"];
|
||||
|
||||
/// Below this, a file is too small to be a book whatever it claims to be.
|
||||
const SMALLEST_BOOK: u64 = 4096;
|
||||
|
||||
/// How much of the front is read to tell text from markup.
|
||||
const TEXT_WINDOW: usize = 4096;
|
||||
|
||||
/// Identify a file.
|
||||
///
|
||||
/// `name` is used only for the extension it claims and for the words in it,
|
||||
/// both of which are reported as observations rather than believed.
|
||||
pub fn sniff(source: &impl Bytes, name: &str) -> Doc {
|
||||
let mut doc = Doc::new(name, Kind::Unknown, source.len() as i64);
|
||||
doc.declared_ext = extension(name);
|
||||
|
||||
doc.kind = if source.magic_at(0, DJVU) {
|
||||
Kind::Djvu
|
||||
} else if source.magic_at(0, PDF) {
|
||||
Kind::Pdf
|
||||
} else if source.magic_at(0, OLE2) {
|
||||
Kind::Ole2
|
||||
} else if source.magic_at(0, zip::LOCAL_HEADER) {
|
||||
archive(source, &mut doc)
|
||||
} else if let Some(kind) = mobi::kind(source) {
|
||||
kind
|
||||
} else {
|
||||
text(source, &mut doc)
|
||||
};
|
||||
|
||||
if doc.kind == Kind::Unknown {
|
||||
doc.note("no container or magic bytes this recognises");
|
||||
}
|
||||
if let Some(declared) = doc.declared_ext.as_deref()
|
||||
&& !names_the_same_thing(declared, doc.kind)
|
||||
{
|
||||
doc.note(format!(
|
||||
"named .{declared}, but the bytes say {}",
|
||||
spelling(doc.kind)
|
||||
));
|
||||
}
|
||||
if source.len() < SMALLEST_BOOK {
|
||||
doc.note(format!(
|
||||
"{} bytes, which is too small to be a book",
|
||||
source.len()
|
||||
));
|
||||
}
|
||||
if let Some(marker) = sample_marker(name) {
|
||||
doc.note(format!("the name says {marker:?}, so it may be a sample"));
|
||||
}
|
||||
doc
|
||||
}
|
||||
|
||||
/// What is inside a zip: an EPUB, a wrapped PDF, or a collection of pages.
|
||||
fn archive(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
let Some(names) = zip::members(source) else {
|
||||
doc.note("a zip whose directory could not be read");
|
||||
return Kind::Unknown;
|
||||
};
|
||||
let lower: Vec<String> = names.iter().map(|name| name.to_lowercase()).collect();
|
||||
|
||||
if lower.iter().any(|name| name == "mimetype")
|
||||
&& lower
|
||||
.iter()
|
||||
.any(|name| name.starts_with("meta-inf/container.xml"))
|
||||
{
|
||||
return Kind::Epub;
|
||||
}
|
||||
if let Some(pdf) = sole_pdf(&names) {
|
||||
doc.note(format!("a zip holding one PDF, {pdf:?}"));
|
||||
return Kind::ZipWrappedPdf;
|
||||
}
|
||||
if lower
|
||||
.iter()
|
||||
.any(|name| name.ends_with(".htm") || name.ends_with(".html") || name.ends_with(".xhtml"))
|
||||
{
|
||||
doc.note("a zip of HTML documents");
|
||||
return Kind::Unknown;
|
||||
}
|
||||
doc.note(format!(
|
||||
"a zip of {} members, none of them a book",
|
||||
names.len()
|
||||
));
|
||||
Kind::Unknown
|
||||
}
|
||||
|
||||
/// The one PDF an archive exists to carry, if that is what it is.
|
||||
///
|
||||
/// Only an unambiguous wrapper qualifies. An archive holding several PDFs is
|
||||
/// a collection, and choosing among them would be a guess — better to say it
|
||||
/// is not a book than to name one chapter as the whole.
|
||||
pub fn sole_pdf(names: &[String]) -> Option<&str> {
|
||||
let real: Vec<&String> = names
|
||||
.iter()
|
||||
.filter(|name| !name.ends_with('/'))
|
||||
.filter(|name| {
|
||||
let lower = name.to_lowercase();
|
||||
!ARCHIVE_NOISE.iter().any(|noise| lower.contains(noise))
|
||||
})
|
||||
.collect();
|
||||
match real.as_slice() {
|
||||
[only] if only.to_lowercase().ends_with(".pdf") => Some(only.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Text, markup, or nothing this knows.
|
||||
fn text(source: &impl Bytes, doc: &mut Doc) -> Kind {
|
||||
let Ok(head) = source.slice(0, TEXT_WINDOW) else {
|
||||
return Kind::Unknown;
|
||||
};
|
||||
if head.is_empty() {
|
||||
doc.note("the file is empty");
|
||||
return Kind::Unknown;
|
||||
}
|
||||
// A file with a null byte in its first few kilobytes is not text, and
|
||||
// nothing else here is going to identify it either.
|
||||
if head.contains(&0) {
|
||||
return Kind::Unknown;
|
||||
}
|
||||
// The window may have cut a character in half, which is not the same
|
||||
// thing as the file not being text.
|
||||
let head = match std::str::from_utf8(&head) {
|
||||
Ok(text) => text,
|
||||
Err(e) if e.error_len().is_none() => {
|
||||
std::str::from_utf8(&head[..e.valid_up_to()]).expect("a valid prefix")
|
||||
}
|
||||
Err(_) => return Kind::Unknown,
|
||||
};
|
||||
|
||||
let start = head
|
||||
.trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}')
|
||||
.to_lowercase();
|
||||
if start.starts_with("<!doctype html")
|
||||
|| start.starts_with("<html")
|
||||
|| start.starts_with("<?xml")
|
||||
|| start.starts_with("<head")
|
||||
{
|
||||
return Kind::Html;
|
||||
}
|
||||
Kind::Txt
|
||||
}
|
||||
|
||||
/// Whether an extension and a verdict are saying the same thing.
|
||||
fn names_the_same_thing(extension: &str, kind: Kind) -> bool {
|
||||
match kind {
|
||||
Kind::Pdf => extension == "pdf",
|
||||
Kind::Epub => extension == "epub",
|
||||
Kind::Djvu => matches!(extension, "djvu" | "djv"),
|
||||
Kind::Azw3 => matches!(extension, "azw3" | "azw" | "kfx"),
|
||||
Kind::Mobi => matches!(extension, "mobi" | "prc" | "azw"),
|
||||
Kind::Txt => matches!(extension, "txt" | "text" | "md" | "markdown"),
|
||||
Kind::Html => matches!(extension, "html" | "htm" | "xhtml" | "xml"),
|
||||
Kind::Ole2 => matches!(extension, "doc" | "xls" | "ppt"),
|
||||
Kind::ZipWrappedPdf => extension == "zip",
|
||||
// Nothing was recognised, so there is nothing to disagree with.
|
||||
Kind::Unknown => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn spelling(kind: Kind) -> &'static str {
|
||||
match kind {
|
||||
Kind::Pdf => "PDF",
|
||||
Kind::Epub => "EPUB",
|
||||
Kind::Djvu => "DjVu",
|
||||
Kind::Azw3 => "AZW3",
|
||||
Kind::Mobi => "MOBI",
|
||||
Kind::Txt => "plain text",
|
||||
Kind::Html => "HTML",
|
||||
Kind::Ole2 => "an OLE2 document",
|
||||
Kind::ZipWrappedPdf => "a zip holding one PDF",
|
||||
Kind::Unknown => "nothing recognisable",
|
||||
}
|
||||
}
|
||||
|
||||
fn extension(name: &str) -> Option<String> {
|
||||
std::path::Path::new(name)
|
||||
.extension()
|
||||
.map(|ext| ext.to_string_lossy().to_lowercase())
|
||||
}
|
||||
|
||||
fn sample_marker(name: &str) -> Option<&'static str> {
|
||||
let lower = std::path::Path::new(name)
|
||||
.file_name()
|
||||
.map(|name| name.to_string_lossy().to_lowercase())
|
||||
.unwrap_or_default();
|
||||
SAMPLE_MARKERS
|
||||
.iter()
|
||||
.find(|marker| lower.contains(**marker))
|
||||
.copied()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn padded(head: &[u8]) -> Vec<u8> {
|
||||
let mut out = head.to_vec();
|
||||
out.resize(SMALLEST_BOOK as usize + 1, b'x');
|
||||
out
|
||||
}
|
||||
|
||||
fn kind_of(head: &[u8], name: &str) -> Kind {
|
||||
sniff(&padded(head).as_slice(), name).kind
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_magic_decides_not_the_name() {
|
||||
assert_eq!(kind_of(b"%PDF-1.7\n", "book.epub"), Kind::Pdf);
|
||||
assert_eq!(kind_of(DJVU, "scan.pdf"), Kind::Djvu);
|
||||
assert_eq!(kind_of(OLE2, "book.epub"), Kind::Ole2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_that_disagrees_with_the_bytes_is_noted() {
|
||||
let doc = sniff(&padded(DJVU).as_slice(), "scan.pdf");
|
||||
assert_eq!(doc.kind, Kind::Djvu);
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("named .pdf")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_that_agrees_is_not_remarked_on() {
|
||||
let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "book.pdf");
|
||||
assert!(doc.notes.is_empty(), "{:?}", doc.notes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_and_markup_are_told_apart() {
|
||||
assert_eq!(
|
||||
kind_of(b"The mill stood at the northern end.", "a.txt"),
|
||||
Kind::Txt
|
||||
);
|
||||
assert_eq!(kind_of(b"<!DOCTYPE html>\n<html>", "a.html"), Kind::Html);
|
||||
assert_eq!(kind_of(b"\n\n <html lang=\"en\">", "a.txt"), Kind::Html);
|
||||
assert_eq!(kind_of(b"<?xml version=\"1.0\"?>", "a.xml"), Kind::Html);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binary_rubbish_is_not_called_text() {
|
||||
assert_eq!(kind_of(&[0x00, 0x01, 0x02, 0x03], "a.txt"), Kind::Unknown);
|
||||
assert_eq!(kind_of(&[0xff, 0xfe, 0xfd], "a.txt"), Kind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_small_file_is_remarked_on() {
|
||||
let doc = sniff(&b"%PDF-1.7".as_slice(), "tiny.pdf");
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("too small")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_that_says_sample_is_remarked_on() {
|
||||
let doc = sniff(&padded(b"%PDF-1.7").as_slice(), "Book - Sample Chapter.pdf");
|
||||
assert!(
|
||||
doc.notes.iter().any(|note| note.contains("sample")),
|
||||
"{:?}",
|
||||
doc.notes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_one_pdf_in_an_archive_is_the_book_and_several_are_not() {
|
||||
let one = ["__MACOSX/._x".into(), "dir/".into(), "book.pdf".into()];
|
||||
assert_eq!(sole_pdf(&one), Some("book.pdf"));
|
||||
|
||||
let several = ["one.pdf".to_string(), "two.pdf".to_string()];
|
||||
assert_eq!(sole_pdf(&several), None);
|
||||
|
||||
let mixed = ["book.pdf".to_string(), "cover.jpg".to_string()];
|
||||
assert_eq!(sole_pdf(&mixed), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_file_is_unknown_and_says_so() {
|
||||
let doc = sniff(&b"".as_slice(), "nothing.pdf");
|
||||
assert_eq!(doc.kind, Kind::Unknown);
|
||||
assert!(doc.notes.iter().any(|note| note.contains("empty")));
|
||||
assert_eq!(doc.bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_declared_extension_is_reported_lowercased() {
|
||||
assert_eq!(
|
||||
sniff(&padded(b"%PDF-").as_slice(), "BOOK.PDF").declared_ext,
|
||||
Some("pdf".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
sniff(&padded(b"%PDF-").as_slice(), "no-extension").declared_ext,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_character_cut_in_half_by_the_window_does_not_make_it_binary() {
|
||||
let mut head = "é".repeat(TEXT_WINDOW).into_bytes();
|
||||
head.truncate(TEXT_WINDOW + 1);
|
||||
assert_eq!(sniff(&head.as_slice(), "a.txt").kind, Kind::Txt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
//! Says what a file actually is, whatever its name claims.
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use booksniff::bytes::File;
|
||||
use clap::Parser;
|
||||
use corpus_core::{Doc, Kind, exit, ndjson};
|
||||
|
||||
/// Identify files by what is in them rather than by what they are called.
|
||||
///
|
||||
/// Writes one doc record per file. Reads only the few kilobytes it needs —
|
||||
/// the front of the file, and an archive's directory at the back — so a
|
||||
/// directory of scans costs the same as a directory of anything else.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 every file was identified, 1 at least one was not, 74 a file
|
||||
could not be read.
|
||||
|
||||
Example:
|
||||
|
||||
booksniff library/*.pdf | jq -r 'select(.kind != \"pdf\") | .path'"
|
||||
)]
|
||||
struct Args {
|
||||
/// Files to identify.
|
||||
#[arg(required = true)]
|
||||
paths: Vec<PathBuf>,
|
||||
|
||||
/// Print a line a person can read instead of the record.
|
||||
#[arg(long)]
|
||||
plain: bool,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
let mut out = std::io::BufWriter::new(std::io::stdout().lock());
|
||||
let mut unknown = false;
|
||||
let mut unreadable = false;
|
||||
|
||||
for path in &args.paths {
|
||||
let name = path.display().to_string();
|
||||
let doc = match File::open(path) {
|
||||
Ok(file) => booksniff::sniff(&file, &name),
|
||||
Err(e) => {
|
||||
eprintln!("booksniff: {name}: {e}");
|
||||
unreadable = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
unknown |= doc.kind == Kind::Unknown;
|
||||
|
||||
let wrote = if args.plain {
|
||||
plain(&mut out, &doc)
|
||||
} else {
|
||||
ndjson::write_line(&mut out, &doc).map_err(|e| match e {
|
||||
ndjson::Error::Io(e) => e,
|
||||
other => std::io::Error::other(other.to_string()),
|
||||
})
|
||||
};
|
||||
if let Err(e) = wrote {
|
||||
// A consumer that stopped reading is not a failure.
|
||||
if e.kind() == std::io::ErrorKind::BrokenPipe {
|
||||
return ExitCode::from(exit::OK);
|
||||
}
|
||||
eprintln!("booksniff: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = out.flush() {
|
||||
if e.kind() == std::io::ErrorKind::BrokenPipe {
|
||||
return ExitCode::from(exit::OK);
|
||||
}
|
||||
eprintln!("booksniff: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
|
||||
ExitCode::from(match (unreadable, unknown) {
|
||||
(true, _) => exit::IO,
|
||||
(false, true) => exit::NO,
|
||||
(false, false) => exit::OK,
|
||||
})
|
||||
}
|
||||
|
||||
fn plain(out: &mut impl Write, doc: &Doc) -> std::io::Result<()> {
|
||||
let kind = serde_json::to_string(&doc.kind).unwrap_or_default();
|
||||
writeln!(out, "{}\t{}", kind.trim_matches('"'), doc.path)?;
|
||||
for note in &doc.notes {
|
||||
writeln!(out, "\t{note}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
//! Telling the two Kindle formats apart.
|
||||
//!
|
||||
//! Both are Palm databases with the same type and creator, so the extension
|
||||
//! is no help and neither is the first eight bytes. The difference is a
|
||||
//! version number inside the first record: six is the old format, eight is
|
||||
//! the one that carries real typesetting.
|
||||
|
||||
use corpus_core::Kind;
|
||||
|
||||
use crate::bytes::Bytes;
|
||||
|
||||
/// Where a Palm database says what it holds.
|
||||
const TYPE_AT: u64 = 60;
|
||||
/// What a Kindle book says there.
|
||||
const BOOK_TYPE: &[u8] = b"BOOKMOBI";
|
||||
|
||||
/// Where the first record's offset is written.
|
||||
const FIRST_RECORD_AT: u64 = 78;
|
||||
|
||||
/// The MOBI header's own magic, at the start of the first record's header.
|
||||
const MOBI_MAGIC: &[u8] = b"MOBI";
|
||||
/// Where the MOBI header begins inside the first record.
|
||||
const MOBI_HEADER_AT: u64 = 16;
|
||||
/// How far into the MOBI header the format version is.
|
||||
const VERSION_AT: u64 = 36;
|
||||
/// The version that means the newer format.
|
||||
const KF8: u32 = 8;
|
||||
|
||||
/// Which Kindle format this is, or `None` if it is not one.
|
||||
pub fn kind(source: &impl Bytes) -> Option<Kind> {
|
||||
if !source.magic_at(TYPE_AT, BOOK_TYPE) {
|
||||
return None;
|
||||
}
|
||||
Some(match version(source) {
|
||||
Some(version) if version >= KF8 => Kind::Azw3,
|
||||
_ => Kind::Mobi,
|
||||
})
|
||||
}
|
||||
|
||||
/// The format version in the first record, when it is readable.
|
||||
fn version(source: &impl Bytes) -> Option<u32> {
|
||||
let record = big_endian(source, FIRST_RECORD_AT)? as u64;
|
||||
let header = record + MOBI_HEADER_AT;
|
||||
if !source.magic_at(header, MOBI_MAGIC) {
|
||||
return None;
|
||||
}
|
||||
big_endian(source, header + VERSION_AT)
|
||||
}
|
||||
|
||||
fn big_endian(source: &impl Bytes, at: u64) -> Option<u32> {
|
||||
let bytes = source.slice(at, 4).ok()?;
|
||||
(bytes.len() == 4).then(|| u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A Palm database holding one record with a MOBI header of `version`.
|
||||
fn book(version: Option<u32>) -> Vec<u8> {
|
||||
let record_at: u32 = 1024;
|
||||
let mut out = vec![0u8; record_at as usize];
|
||||
out[TYPE_AT as usize..TYPE_AT as usize + 8].copy_from_slice(BOOK_TYPE);
|
||||
out[FIRST_RECORD_AT as usize..FIRST_RECORD_AT as usize + 4]
|
||||
.copy_from_slice(&record_at.to_be_bytes());
|
||||
|
||||
if let Some(version) = version {
|
||||
out.resize(record_at as usize + MOBI_HEADER_AT as usize, 0);
|
||||
out.extend_from_slice(MOBI_MAGIC);
|
||||
out.resize(
|
||||
record_at as usize + MOBI_HEADER_AT as usize + VERSION_AT as usize,
|
||||
0,
|
||||
);
|
||||
out.extend_from_slice(&version.to_be_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_newer_format_is_recognised() {
|
||||
assert_eq!(kind(&book(Some(8)).as_slice()), Some(Kind::Azw3));
|
||||
assert_eq!(kind(&book(Some(9)).as_slice()), Some(Kind::Azw3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_older_format_is_recognised() {
|
||||
assert_eq!(kind(&book(Some(6)).as_slice()), Some(Kind::Mobi));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_book_with_no_readable_version_is_taken_for_the_older_format() {
|
||||
assert_eq!(kind(&book(None).as_slice()), Some(Kind::Mobi));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn something_that_is_not_a_kindle_book_is_not_one() {
|
||||
assert_eq!(kind(&b"%PDF-1.7".as_slice()), None);
|
||||
assert_eq!(kind(&vec![0u8; 4096].as_slice()), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_book_does_not_panic() {
|
||||
let whole = book(Some(8));
|
||||
for cut in 0..whole.len() {
|
||||
let _ = kind(&&whole[..cut]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
//! Listing what is inside a zip, without unpacking any of it.
|
||||
//!
|
||||
//! An EPUB, an htmlz and a PDF someone zipped and renamed are all zips, and
|
||||
//! telling them apart needs only the member names. Those live in the central
|
||||
//! directory at the end of the file, so reading them costs a few kilobytes
|
||||
//! whatever the archive weighs — and nothing is ever decompressed, which
|
||||
//! means a malicious archive cannot expand into anything here.
|
||||
|
||||
use crate::bytes::Bytes;
|
||||
|
||||
/// The end-of-central-directory record's signature.
|
||||
const END_OF_DIRECTORY: &[u8] = b"PK\x05\x06";
|
||||
/// A central directory entry's signature.
|
||||
const ENTRY: &[u8] = b"PK\x01\x02";
|
||||
/// A local file header, which is what a zip starts with.
|
||||
pub const LOCAL_HEADER: &[u8] = b"PK\x03\x04";
|
||||
|
||||
/// How far back from the end the trailing record may be.
|
||||
///
|
||||
/// Its own size plus the largest comment a zip may carry.
|
||||
const SEARCH_BACK: u64 = 22 + 0xffff;
|
||||
|
||||
/// Most members this will list.
|
||||
///
|
||||
/// A book in an archive has tens of members; an archive with a hundred
|
||||
/// thousand is not a book, and reading all of its names would be the only
|
||||
/// expensive thing this program does.
|
||||
const MOST_MEMBERS: usize = 100_000;
|
||||
|
||||
/// The names of an archive's members, or `None` if it is not a zip this can
|
||||
/// read.
|
||||
pub fn members(source: &impl Bytes) -> Option<Vec<String>> {
|
||||
if !source.magic_at(0, LOCAL_HEADER) {
|
||||
return None;
|
||||
}
|
||||
let (count, mut at) = directory_start(source)?;
|
||||
|
||||
let mut names = Vec::with_capacity(count.min(1024));
|
||||
for _ in 0..count.min(MOST_MEMBERS) {
|
||||
let header = source.slice(at, 46).ok()?;
|
||||
if header.len() < 46 || header[..4] != *ENTRY {
|
||||
break;
|
||||
}
|
||||
let name_len = u16::from_le_bytes([header[28], header[29]]) as u64;
|
||||
let extra_len = u16::from_le_bytes([header[30], header[31]]) as u64;
|
||||
let comment_len = u16::from_le_bytes([header[32], header[33]]) as u64;
|
||||
|
||||
let name = source.slice(at + 46, name_len as usize).ok()?;
|
||||
// Member names are bytes, and an archive written on another machine
|
||||
// may not have written UTF-8. What is wanted here is a comparison
|
||||
// against ASCII, so anything else may as well be a replacement
|
||||
// character.
|
||||
names.push(String::from_utf8_lossy(&name).into_owned());
|
||||
at += 46 + name_len + extra_len + comment_len;
|
||||
}
|
||||
Some(names)
|
||||
}
|
||||
|
||||
/// How many entries the central directory has and where it starts.
|
||||
fn directory_start(source: &impl Bytes) -> Option<(usize, u64)> {
|
||||
let len = source.len();
|
||||
let window = SEARCH_BACK.min(len);
|
||||
let from = len - window;
|
||||
let tail = source.slice(from, window as usize).ok()?;
|
||||
|
||||
// The record is at the end, after a comment of unknown length, so it is
|
||||
// found by searching backwards for its signature.
|
||||
let at = tail
|
||||
.windows(4)
|
||||
.rposition(|window| window == END_OF_DIRECTORY)?;
|
||||
let record = &tail[at..];
|
||||
if record.len() < 22 {
|
||||
return None;
|
||||
}
|
||||
let count = u16::from_le_bytes([record[10], record[11]]) as usize;
|
||||
let offset = u32::from_le_bytes([record[16], record[17], record[18], record[19]]) as u64;
|
||||
(offset < len).then_some((count, offset))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a zip holding `names`, each with empty stored content.
|
||||
fn zip(names: &[&str]) -> Vec<u8> {
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
let mut offsets = Vec::new();
|
||||
for name in names {
|
||||
offsets.push(out.len() as u32);
|
||||
out.extend_from_slice(LOCAL_HEADER);
|
||||
out.extend_from_slice(&[0; 22]); // version .. uncompressed size
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes());
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
}
|
||||
let directory_at = out.len() as u32;
|
||||
for (name, offset) in names.iter().zip(&offsets) {
|
||||
out.extend_from_slice(ENTRY);
|
||||
out.extend_from_slice(&[0; 24]); // version .. uncompressed size
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes()); // extra
|
||||
out.extend_from_slice(&0u16.to_le_bytes()); // comment
|
||||
out.extend_from_slice(&[0; 8]); // disk, attributes
|
||||
out.extend_from_slice(&offset.to_le_bytes());
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
}
|
||||
out.extend_from_slice(END_OF_DIRECTORY);
|
||||
out.extend_from_slice(&[0; 4]); // disk numbers
|
||||
out.extend_from_slice(&(names.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(names.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&((out.len() as u32).saturating_sub(directory_at)).to_le_bytes());
|
||||
out.extend_from_slice(&directory_at.to_le_bytes());
|
||||
out.extend_from_slice(&0u16.to_le_bytes()); // comment length
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_member_names_come_back() {
|
||||
let archive = zip(&["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]);
|
||||
let names = members(&archive.as_slice()).expect("a zip");
|
||||
assert_eq!(
|
||||
names,
|
||||
["mimetype", "META-INF/container.xml", "OEBPS/book.opf"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_archive_with_no_members_is_not_read_as_one() {
|
||||
// It has no local header to start with, so nothing identifies it as
|
||||
// an archive in the first place — and an empty archive is not a book
|
||||
// under any reading.
|
||||
assert_eq!(members(&zip(&[]).as_slice()), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn something_that_is_not_a_zip_is_not_read_as_one() {
|
||||
assert_eq!(members(&b"%PDF-1.7".as_slice()), None);
|
||||
assert_eq!(members(&b"".as_slice()), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_archive_does_not_panic() {
|
||||
let archive = zip(&["a.pdf", "b.pdf"]);
|
||||
for cut in 0..archive.len() {
|
||||
let _ = members(&&archive[..cut]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_count_that_lies_stops_at_the_first_bad_entry() {
|
||||
let mut archive = zip(&["a.pdf"]);
|
||||
let at = archive.len() - 12;
|
||||
// Claim a thousand entries where there is one.
|
||||
archive[at - 6..at - 4].copy_from_slice(&1000u16.to_le_bytes());
|
||||
let names = members(&archive.as_slice()).expect("a zip");
|
||||
assert_eq!(names, ["a.pdf"]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//! The tool as a caller meets it: files in, records and an exit code out.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Command, Output};
|
||||
|
||||
fn run(args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_booksniff"))
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("the binary should run")
|
||||
}
|
||||
|
||||
fn written(name: &str, bytes: &[u8]) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join("booksniff-tests");
|
||||
std::fs::create_dir_all(&dir).expect("a writable temp directory");
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, bytes).expect("a writable path");
|
||||
path
|
||||
}
|
||||
|
||||
fn padded(head: &[u8]) -> Vec<u8> {
|
||||
let mut out = head.to_vec();
|
||||
out.resize(8192, b'x');
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_it_recognises_comes_back_as_a_record() {
|
||||
let path = written("book.pdf", &padded(b"%PDF-1.7\n"));
|
||||
let out = run(&[path.to_str().expect("utf-8 path")]);
|
||||
assert_eq!(out.status.code(), Some(0));
|
||||
|
||||
let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
|
||||
assert_eq!(doc["kind"], "pdf");
|
||||
assert_eq!(doc["declared_ext"], "pdf");
|
||||
assert_eq!(doc["bytes"], 8192);
|
||||
assert!(out.stderr.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_it_cannot_place_answers_no() {
|
||||
let path = written("mystery.bin", &padded(&[0x00, 0x01, 0x02]));
|
||||
let out = run(&[path.to_str().expect("utf-8 path")]);
|
||||
assert_eq!(out.status.code(), Some(1), "unknown must answer no");
|
||||
let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
|
||||
assert_eq!(doc["kind"], "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn several_files_come_back_as_several_records() {
|
||||
let one = written("one.pdf", &padded(b"%PDF-1.7"));
|
||||
let two = written("two.djvu", &padded(b"AT&TFORM"));
|
||||
let out = run(&[
|
||||
one.to_str().expect("utf-8 path"),
|
||||
two.to_str().expect("utf-8 path"),
|
||||
]);
|
||||
let kinds: Vec<String> = String::from_utf8(out.stdout)
|
||||
.expect("utf-8")
|
||||
.lines()
|
||||
.map(|line| {
|
||||
serde_json::from_str::<serde_json::Value>(line).expect("valid JSON")["kind"].to_string()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(kinds, ["\"pdf\"", "\"djvu\""]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_there_is_an_io_error_and_the_rest_still_run() {
|
||||
let path = written("kept.pdf", &padded(b"%PDF-1.7"));
|
||||
let out = run(&[
|
||||
"/definitely/not/here.pdf",
|
||||
path.to_str().expect("utf-8 path"),
|
||||
]);
|
||||
assert_eq!(out.status.code(), Some(74));
|
||||
assert!(!out.stderr.is_empty());
|
||||
// The readable file was still identified, on stdout, uncontaminated.
|
||||
let doc: serde_json::Value = serde_json::from_slice(&out.stdout).expect("one JSON record");
|
||||
assert_eq!(doc["kind"], "pdf");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_plain_form_is_for_a_person() {
|
||||
let path = written("lying.pdf", &padded(b"AT&TFORM"));
|
||||
let out = run(&["--plain", path.to_str().expect("utf-8 path")]);
|
||||
let said = String::from_utf8(out.stdout).expect("utf-8");
|
||||
assert!(said.starts_with("djvu\t"), "{said}");
|
||||
assert!(said.contains("named .pdf"), "{said}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_files_named_is_a_usage_error() {
|
||||
let out = run(&[]);
|
||||
assert_eq!(out.status.code(), Some(64), "usage errors exit 64");
|
||||
assert!(out.stdout.is_empty());
|
||||
}
|
||||
Reference in New Issue
Block a user