booksniff, and one place for the parts every tool shares

2026-10-22 18:57:14 +00:00 · 2026-09-19 12:40:20 -04:00
parent 36afe3350b
commit 1071f0bed3
32 changed files with 1426 additions and 237 deletions
@@ -14,6 +14,7 @@ publish = false
 python = []

 [dependencies]
+booksniff.workspace = true
 chunkmd.workspace = true
 cuekit.workspace = true
 cuepack.workspace = true
@@ -0,0 +1,20 @@
+{"name": "plain.pdf", "hex": "255044462d312e370a25e2e3cfd30a78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878"}
+{"name": "scan.pdf", "hex": "41542654464f524d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}
+{"name": "book.epub", "hex": "d0cf11e0a1b11ae100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}
+{"name": "notes.doc", "hex": "5468652050726f6a20706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f6320706c61696e2074657874206e616d6564202e646f63"}
+{"name": "empty.pdf", "hex": ""}
+{"name": "tiny.txt", "hex": "6869"}
+{"name": "prose.txt", "hex": "546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a546865206d696c6c2073746f6f6420617420746865206e6f72746865726e20656e642e0a"}
+{"name": "page.html", "hex": "3c21444f43545950452068746d6c3e0a3c68746d6c3e3c626f64793e746578743c2f626f64793e3c2f68746d6c3e"}
+{"name": "spine.xhtml", "hex": "3c3f786d6c2076657273696f6e3d22312e30223f3e0a3c68746d6c20786d6c6e733d2278223e3c626f64792f3e3c2f68746d6c3e"}
+{"name": "real.epub", "hex": "504b03041400000000009164335d6f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b03041400000000009164335d54995f930c0000000c000000160000004d4554412d494e462f636f6e7461696e65722e786d6c3c636f6e7461696e65722f3e504b03041400000000009164335d9b7d083a0a0000000a0000000e0000004f454250532f626f6f6b2e6f70663c7061636b6167652f3e504b010214031400000000009164335d6f61ab2c14000000140000000800000000000000000000008001000000006d696d6574797065504b010214031400000000009164335d54995f930c0000000c00000016000000000000000000000080013a0000004d4554412d494e462f636f6e7461696e65722e786d6c504b010214031400000000009164335d9b7d083a0a0000000a0000000e000000000000000000000080017a0000004f454250532f626f6f6b2e6f7066504b05060000000003000300b6000000b00000000000"}
+{"name": "saved.zip", "hex": "504b03041400000000009164335d6f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b03041400000000009164335d54995f930c0000000c000000160000004d4554412d494e462f636f6e7461696e65722e786d6c3c636f6e7461696e65722f3e504b010214031400000000009164335d6f61ab2c14000000140000000800000000000000000000008001000000006d696d6574797065504b010214031400000000009164335d54995f930c0000000c00000016000000000000000000000080013a0000004d4554412d494e462f636f6e7461696e65722e786d6c504b050600000000020002007a0000007a0000000000"}
+{"name": "wrapped.pdf", "hex": "504b03041400000000009164335da36258a4100000001000000008000000626f6f6b2e706466255044462d312e3720636f6e74656e74504b010214031400000000009164335da36258a41000000010000000080000000000000000000000800100000000626f6f6b2e706466504b0506000000000100010036000000360000000000"}
+{"name": "wrapped-noisy.zip", "hex": "504b03041400000000009164335d399cfb060400000004000000130000005f5f4d41434f53582f2e5f626f6f6b2e7064666a756e6b504b03041400000000009164335d399cfb060400000004000000090000002e44535f53746f72656a756e6b504b03041400000000009164335da36258a4100000001000000008000000626f6f6b2e706466255044462d312e3720636f6e74656e74504b010214031400000000009164335d399cfb0604000000040000001300000000000000000000008001000000005f5f4d41434f53582f2e5f626f6f6b2e706466504b010214031400000000009164335d399cfb0604000000040000000900000000000000000000008001350000002e44535f53746f7265504b010214031400000000009164335da36258a41000000010000000080000000000000000000000800160000000626f6f6b2e706466504b05060000000003000300ae000000960000000000"}
+{"name": "collection.zip", "hex": "504b03041400000000009164335d43beb7e80100000001000000070000006f6e652e70646661504b03041400000000009164335df9efbe7101000000010000000700000074776f2e70646662504b010214031400000000009164335d43beb7e801000000010000000700000000000000000000008001000000006f6e652e706466504b010214031400000000009164335df9efbe71010000000100000007000000000000000000000080012600000074776f2e706466504b050600000000020002006a0000004c0000000000"}
+{"name": "pages.zip", "hex": "504b03041400000000009164335d3ce3a64107000000070000000a000000696e6465782e68746d6c3c68746d6c2f3e504b03041400000000009164335d8316dc8c0100000001000000090000007374796c652e63737378504b010214031400000000009164335d3ce3a64107000000070000000a0000000000000000000000800100000000696e6465782e68746d6c504b010214031400000000009164335d8316dc8c010000000100000009000000000000000000000080012f0000007374796c652e637373504b050600000000020002006f000000570000000000"}
+{"name": "nothing.zip", "hex": "504b03041400000000009164335d8316dc8c01000000010000000a000000726561646d652e74787478504b010214031400000000009164335d8316dc8c01000000010000000a0000000000000000000000800100000000726561646d652e747874504b0506000000000100010038000000290000000000"}
+{"name": "uppercase.zip", "hex": "504b03041400000000009164335d18eb5070080000000800000008000000424f4f4b2e504446255044462d312e37504b010214031400000000009164335d18eb50700800000008000000080000000000000000000000800100000000424f4f4b2e504446504b05060000000001000100360000002e0000000000"}
+{"name": "subdir.zip", "hex": "504b03041400000000009164335d18eb507008000000080000000c0000006469722f626f6f6b2e706466255044462d312e37504b010214031400000000009164335d18eb507008000000080000000c00000000000000000000008001000000006469722f626f6f6b2e706466504b050600000000010001003a000000320000000000"}
+{"name": "truncated.zip", "hex": "504b03041400000000009164335d18eb5070080000000800000008000000626f6f6b2e7064662550"}
+{"name": "Book - Sample Chapter.pdf", "hex": "255044462d312e370a25e2e3cfd30a78787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878"}
@@ -0,0 +1,20 @@
+{"djvu": false, "name": "plain.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": true, "name": "scan.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "book.epub", "ole2": true, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "notes.doc", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "empty.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "tiny.txt", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "prose.txt", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "page.html", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "spine.xhtml", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
+{"djvu": false, "name": "real.epub", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "epub"}
+{"djvu": false, "name": "saved.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "epub"}
+{"djvu": false, "name": "wrapped.pdf", "ole2": false, "sole_pdf": "book.pdf", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "wrapped-noisy.zip", "ole2": false, "sole_pdf": "book.pdf", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "collection.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"}
+{"djvu": false, "name": "pages.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "htmlz"}
+{"djvu": false, "name": "nothing.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"}
+{"djvu": false, "name": "uppercase.zip", "ole2": false, "sole_pdf": "BOOK.PDF", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "subdir.zip", "ole2": false, "sole_pdf": "dir/book.pdf", "zip": true, "zip_kind": "pdf"}
+{"djvu": false, "name": "truncated.zip", "ole2": false, "sole_pdf": null, "zip": true, "zip_kind": "other"}
+{"djvu": false, "name": "Book - Sample Chapter.pdf", "ole2": false, "sole_pdf": null, "zip": false, "zip_kind": null}
@@ -0,0 +1,57 @@
+"""Identify containers the way the Python pipeline identifies them.
+
+Reads one `{"name", "hex"}` object per line, writes the bytes to a temporary
+file under that name, and reports what each of the Python's container tests
+makes of it.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+        python3 oracle/booksniff.py < cases/booksniff/files.ndjson
+"""
+
+import json
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from book_pipeline import convert  # noqa: E402
+
+
+def main() -> int:
+    out = sys.stdout
+    with tempfile.TemporaryDirectory() as tmp:
+        for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+            if not line.strip():
+                continue
+            case = json.loads(line)
+            path = Path(tmp) / case["name"]
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(bytes.fromhex(case["hex"]))
+
+            record = {
+                "name": case["name"],
+                "djvu": convert._is_djvu_document(path),
+                "zip": convert._is_zip_container(path),
+                "ole2": convert._is_ole2_document(path),
+                "zip_kind": None,
+                "sole_pdf": None,
+            }
+            if record["zip"]:
+                record["zip_kind"] = convert._classify_zip(path)
+                try:
+                    import zipfile
+
+                    with zipfile.ZipFile(path) as z:
+                        record["sole_pdf"] = convert._sole_pdf_member(z.namelist())
+                except Exception:
+                    record["sole_pdf"] = None
+            out.write(json.dumps(record, sort_keys=True) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);

 fn main() -> ExitCode {
-    let jobs: [Job; 9] = [
+    let jobs: [Job; 10] = [
        ("tokens", tokens),
        ("unescape", unescape),
        ("chunkmd", chunkmd),
@@ -29,6 +29,7 @@ fn main() -> ExitCode {
        ("cueseg", cueseg),
        ("rrf", rrf),
        ("cuepack", cuepack),
+        ("booksniff", booksniff),
    ];

    for (name, job) in jobs {
@@ -72,6 +73,18 @@ fn unescape() -> Written {
    Ok(vec![path.display().to_string()])
 }

+fn booksniff() -> Written {
+    let dir = parity::golden("booksniff");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/booksniff/files.ndjson");
+    let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let verdicts = parity::run_oracle("booksniff.py", &cases)?;
+    let path = dir.join("files.ndjson");
+    std::fs::write(&path, verdicts).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn cuepack() -> Written {
    let dir = parity::golden("cuepack");
    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
@@ -0,0 +1,93 @@
+//! Container identification must agree with the Python's.
+//!
+//! The consequence of getting this wrong is not a wrong record: it is a
+//! converter handed a file it cannot read, and a book that never enters the
+//! corpus with nothing but an unhelpful error to say why.
+
+use booksniff::bytes::Bytes;
+use corpus_core::Kind;
+use serde_json::Value;
+
+fn cases() -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join("cases/booksniff/files.ndjson"))
+        .expect("a readable case file");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn bytes_of(case: &Value) -> Vec<u8> {
+    let hex = case["hex"].as_str().expect("hex bytes");
+    (0..hex.len())
+        .step_by(2)
+        .map(|at| u8::from_str_radix(&hex[at..at + 2], 16).expect("valid hex"))
+        .collect()
+}
+
+#[test]
+fn every_container_is_read_the_same_as_python() {
+    let golden: Vec<Value> = parity::read_golden_text("booksniff/files.ndjson")
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect();
+    let cases = cases();
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    for (case, theirs) in cases.iter().zip(&golden) {
+        let name = case["name"].as_str().expect("a name");
+        assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
+        let bytes = bytes_of(case);
+        let source = bytes.as_slice();
+        let doc = booksniff::sniff(&source, name);
+
+        // The three magic-byte tests, each of which the Python answers
+        // directly.
+        assert_eq!(
+            doc.kind == Kind::Djvu,
+            theirs["djvu"].as_bool().expect("a verdict"),
+            "{name}: DjVu"
+        );
+        assert_eq!(
+            doc.kind == Kind::Ole2,
+            theirs["ole2"].as_bool().expect("a verdict"),
+            "{name}: OLE2"
+        );
+        assert_eq!(
+            source.magic_at(0, booksniff::zip::LOCAL_HEADER),
+            theirs["zip"].as_bool().expect("a verdict"),
+            "{name}: zip"
+        );
+
+        // What the archive holds, where the Python classifies one.
+        if let Some(kind) = theirs["zip_kind"].as_str() {
+            let ours = match doc.kind {
+                Kind::Epub => "epub",
+                Kind::ZipWrappedPdf => "pdf",
+                _ if doc.notes.iter().any(|note| note.contains("HTML")) => "htmlz",
+                _ => "other",
+            };
+            assert_eq!(ours, kind, "{name}: what the zip holds");
+        }
+
+        // And which member it would unwrap.
+        if let Some(names) = booksniff::zip::members(&source) {
+            assert_eq!(
+                booksniff::sole_pdf(&names),
+                theirs["sole_pdf"].as_str(),
+                "{name}: the sole PDF"
+            );
+        }
+    }
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/booksniff/files.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("booksniff.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("booksniff/files.ndjson"));
+}