mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
pdfgutter: two facing pages photographed as one, found and cut
This commit is contained in:
funkpower
Generated
+371
-12
@@ -8,6 +8,17 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher",
|
||||
"cpufeatures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
@@ -130,6 +141,30 @@ version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-padding"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.1.0"
|
||||
@@ -152,6 +187,12 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytecount"
|
||||
version = "0.6.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
@@ -167,6 +208,15 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbc"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
|
||||
dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.7"
|
||||
@@ -195,6 +245,16 @@ dependencies = [
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
@@ -226,7 +286,7 @@ dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -280,6 +340,15 @@ dependencies = [
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.2"
|
||||
@@ -289,6 +358,16 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.1.0"
|
||||
@@ -353,6 +432,16 @@ version = "1.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.7"
|
||||
@@ -361,7 +450,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -379,6 +468,15 @@ version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "ecb"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7"
|
||||
dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.1.0"
|
||||
@@ -393,6 +491,15 @@ dependencies = [
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
@@ -447,6 +554,16 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -458,6 +575,18 @@ dependencies = [
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasip2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
@@ -598,6 +727,16 @@ dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
@@ -640,6 +779,44 @@ version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "lopdf"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"bitflags",
|
||||
"cbc",
|
||||
"ecb",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"getrandom 0.3.4",
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"log",
|
||||
"md-5",
|
||||
"nom",
|
||||
"nom_locate",
|
||||
"rand",
|
||||
"rangemap",
|
||||
"sha2",
|
||||
"stringprep",
|
||||
"thiserror",
|
||||
"ttf-parser",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.1.0"
|
||||
@@ -684,6 +861,26 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "8.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom_locate"
|
||||
version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d"
|
||||
dependencies = [
|
||||
"bytecount",
|
||||
"memchr",
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.1.0"
|
||||
@@ -716,12 +913,26 @@ dependencies = [
|
||||
"cueseg",
|
||||
"mdscrub",
|
||||
"mdsplit",
|
||||
"pdfgutter",
|
||||
"pdftriage",
|
||||
"rrf",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdfgutter"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"lopdf",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdftriage"
|
||||
version = "0.1.0"
|
||||
@@ -756,6 +967,15 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
@@ -774,6 +994,47 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rangemap"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d"
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
@@ -791,7 +1052,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -831,7 +1092,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"getrandom 0.2.17",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys 0.52.0",
|
||||
@@ -910,7 +1171,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -940,7 +1201,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -951,7 +1212,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -967,6 +1228,17 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
@@ -991,6 +1263,17 @@ version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "stringprep"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
"unicode-properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
@@ -1003,6 +1286,17 @@ version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
@@ -1022,7 +1316,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1050,7 +1344,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1115,6 +1409,24 @@ version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "ttf-parser"
|
||||
version = "0.25.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-general-category"
|
||||
version = "1.1.0"
|
||||
@@ -1136,6 +1448,12 @@ dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-properties"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
@@ -1216,6 +1534,15 @@ version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "wasip2"
|
||||
version = "1.0.1+wasi-0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
|
||||
dependencies = [
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
@@ -1225,6 +1552,12 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "weezl"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
@@ -1313,6 +1646,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.46.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.4"
|
||||
@@ -1338,10 +1677,30 @@ checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.8"
|
||||
@@ -1359,7 +1718,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -1399,7 +1758,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -24,12 +24,16 @@ rrf = { path = "crates/rrf", version = "0.1.0" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.1.0" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.1.0" }
|
||||
avplan = { path = "crates/avplan", version = "0.1.0" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.1.0" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.1.0" }
|
||||
emb = { path = "crates/emb", version = "0.1.0" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
deunicode = "1"
|
||||
fancy-regex = "0.16"
|
||||
# 0.38 is the last release that builds on the pinned toolchain; the ones
|
||||
# after it need a newer compiler than the rest of this asks for.
|
||||
lopdf = { version = "0.38", default-features = false }
|
||||
regex = "1"
|
||||
schemars = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
|
||||
@@ -24,6 +24,7 @@ Each tool installs by name, so you can take one without taking the rest.
|
||||
| | |
|
||||
|---|---|
|
||||
| [`booksniff`](crates/booksniff/README.md) | What a file actually is, whatever its name claims |
|
||||
| [`pdfgutter`](crates/pdfgutter/README.md) | Books scanned two pages to a sheet, split back into pages |
|
||||
| [`pdftriage`](crates/pdftriage/README.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed |
|
||||
| [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change |
|
||||
| [`vocabmerge`](crates/bibclean/README.md#vocabmerge--the-other-command) | The near-duplicate headings in a catalogue's vocabulary |
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "pdfgutter"
|
||||
description = "Finds books scanned two pages to a sheet, and splits them back into pages."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
lopdf.workspace = true
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,164 @@
|
||||
# pdfgutter
|
||||
|
||||
Finds books scanned two pages to a sheet, and splits them back into pages.
|
||||
|
||||
Part of [catena](../../README.md).
|
||||
|
||||
## What it does
|
||||
|
||||
A book held open under a camera gives one wide image per shot, with two pages
|
||||
on it. Everything that reads it afterwards runs the two columns together: the
|
||||
lines interleave, the reading order comes out scrambled, and what lands in the
|
||||
index looks like prose and is not.
|
||||
|
||||
`pdfgutter` says whether a PDF is that, and splits it when it is. The split
|
||||
changes the page boxes and nothing else — no re-rendering, no loss, and a
|
||||
250-sheet book becomes a 500-page book at almost no cost in size.
|
||||
|
||||
## When you would use it
|
||||
|
||||
- **Before OCR.** Layout models read a spread as one page and produce jumbled
|
||||
text, or none.
|
||||
- **Before chunking.** A passage that alternates lines between two pages is
|
||||
unusable and does not look it.
|
||||
- **Sorting a scanned library.** Most books are fine; the answer tells you
|
||||
which ones are not.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
cargo install --git https://git.pumpin.online/funkpower/catena pdfgutter
|
||||
```
|
||||
|
||||
Detection needs `pdftoppm`, from poppler. Splitting needs nothing.
|
||||
|
||||
## Example
|
||||
|
||||
A book photographed two pages at a time:
|
||||
|
||||
```
|
||||
$ pdfgutter scan.pdf
|
||||
two pages to a sheet
|
||||
12 pages, median ratio 1.40, gutter on 100% of the wide ones
|
||||
```
|
||||
|
||||
An ordinary one:
|
||||
|
||||
```
|
||||
$ pdfgutter book.pdf
|
||||
one page to a sheet
|
||||
12 pages, median ratio 0.68, gutter on 0% of the wide ones
|
||||
$ echo $?
|
||||
1
|
||||
```
|
||||
|
||||
Splitting it:
|
||||
|
||||
```
|
||||
$ pdfgutter --into split.pdf scan.pdf
|
||||
two pages to a sheet
|
||||
12 pages, median ratio 1.40, gutter on 100% of the wide ones
|
||||
20 pages written to split.pdf
|
||||
```
|
||||
|
||||
Twelve sheets became twenty pages: eight spreads split in two, and the four
|
||||
covers and end leaves, which were single pages already, left alone.
|
||||
|
||||
What it looked at:
|
||||
|
||||
```
|
||||
$ pdfgutter --json scan.pdf | jq -c '.sampled[:3][]'
|
||||
{"number":3,"ratio":1.4,"gutter":true}
|
||||
{"number":4,"ratio":1.4,"gutter":true}
|
||||
{"number":5,"ratio":1.4,"gutter":true}
|
||||
```
|
||||
|
||||
Over a directory, splitting only what needs it:
|
||||
|
||||
```sh
|
||||
for f in *.pdf; do
|
||||
pdfgutter --quiet "$f" && pdfgutter --quiet --into "split/$f" "$f" || cp "$f" "split/$f"
|
||||
done
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Effect |
|
||||
|---|---|
|
||||
| `--into <FILE>` | Write the split document here |
|
||||
| `--json` | Emit a record — every page looked at, and what it measured |
|
||||
| `--quiet`, `-q` | Say nothing; the exit code is the answer |
|
||||
| `--min-ratio <RATIO>` | How wide a sheet must be to hold two pages. Default `1.15` |
|
||||
| `--gutter-fraction <FRACTION>` | What share of wide sheets must show a gutter. Default `0.6` |
|
||||
|
||||
`--into` writes whatever the verdict was: a document that is not two-up is
|
||||
copied through unchanged, so a loop over a directory does not have to branch.
|
||||
|
||||
## Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| `0` | Two pages to a sheet |
|
||||
| `1` | One page to a sheet |
|
||||
| `2` | It could not be judged — encrypted, or no pages |
|
||||
| `64` | The command line was wrong |
|
||||
| `69` | `pdftoppm` is not installed |
|
||||
| `74` | The file could not be read or written |
|
||||
|
||||
## How it decides
|
||||
|
||||
**Shape is not enough.** A foldout, a plate, a page set in one narrow centred
|
||||
column are all genuinely wide, and splitting one down the middle destroys it.
|
||||
So the shape of the ink is checked as well.
|
||||
|
||||
Up to eight pages are sampled, evenly spaced and away from the ends — covers
|
||||
and end matter are single pages even in a book otherwise shot two at a time,
|
||||
and sampling them would argue against a document that plainly is a spread.
|
||||
|
||||
Each sampled page that is wide enough to hold two pages gets rendered small
|
||||
and read as a column-by-column measure of how much ink is on it. A gutter is:
|
||||
|
||||
- a **blank column** somewhere in the middle eight per cent of the width,
|
||||
- with **real text** in the third of the page to its left,
|
||||
- and **real text** in the third to its right.
|
||||
|
||||
The outer tenth of the sheet is ignored on both sides, because that is where
|
||||
a scan carries the shadow of the binding and of the table the book sat on.
|
||||
|
||||
A document is two-up when the **middle** sampled page is wide enough and
|
||||
**at least three in five** of the wide ones showed a gutter. The middle rather
|
||||
than the average throughout, so that one plate or one blank leaf does not move
|
||||
the answer for a whole book.
|
||||
|
||||
### The split
|
||||
|
||||
Each wide sheet becomes two pages pointing at the same content, differing only
|
||||
in how much of it they show. A sheet the file asks to be turned on its side is
|
||||
cut across rather than down, and the halves come out in the order a reader
|
||||
would read them.
|
||||
|
||||
Splitting is safe to repeat: the halves are portrait, so a second pass finds
|
||||
nothing to do.
|
||||
|
||||
## As a library
|
||||
|
||||
```rust
|
||||
let mut pdf = pdfgutter::pdf::Pdf::open(std::path::Path::new("scan.pdf"))?;
|
||||
for index in pdfgutter::sample_indices(pdf.len()) {
|
||||
println!("page {} ratio {:.2}", index + 1, pdf.geometry(index).ratio());
|
||||
}
|
||||
let pages = pdf.split(pdfgutter::MIN_RATIO);
|
||||
pdf.save(std::path::Path::new("split.pdf"))?;
|
||||
println!("{pages} pages");
|
||||
# Ok::<(), pdfgutter::pdf::Error>(())
|
||||
```
|
||||
|
||||
`pdfgutter::has_gutter` takes a column ink profile and answers on its own, for
|
||||
anything that rasterises its own pages. `pdfgutter::ink::profile` turns a
|
||||
greymap into one.
|
||||
|
||||
## See also
|
||||
|
||||
- [`pdftriage`](../pdftriage/README.md) — whether the text layer is worth
|
||||
keeping, which is the next question after this one.
|
||||
- [`booksniff`](../booksniff/README.md) — whether the file is a PDF at all.
|
||||
@@ -0,0 +1,210 @@
|
||||
//! Reading a rendered page as a column-by-column measure of how much ink is
|
||||
//! on it.
|
||||
//!
|
||||
//! The page comes in as a greymap — the format `pdftoppm` writes and the
|
||||
//! simplest image format there is: a header, then one byte per pixel. Only
|
||||
//! the column sums are wanted, so nothing is kept but the profile.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
/// What can be wrong with a greymap.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Error {
|
||||
/// It does not begin `P5`.
|
||||
NotAGreymap,
|
||||
/// The header ended before the three numbers did.
|
||||
Truncated,
|
||||
/// A number in the header was not one.
|
||||
BadHeader,
|
||||
/// Sixteen-bit samples, which nothing here produces.
|
||||
SampleTooWide(u32),
|
||||
/// The header promises more pixels than the file holds.
|
||||
ShortImage {
|
||||
/// How many bytes the header calls for.
|
||||
expected: usize,
|
||||
/// How many are there.
|
||||
found: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Error::NotAGreymap => write!(f, "not a binary greymap"),
|
||||
Error::Truncated => write!(f, "the greymap header is cut short"),
|
||||
Error::BadHeader => write!(f, "the greymap header is not three numbers"),
|
||||
Error::SampleTooWide(max) => write!(f, "{max} levels of grey, which is more than 255"),
|
||||
Error::ShortImage { expected, found } => {
|
||||
write!(f, "the greymap promises {expected} bytes and holds {found}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
/// The fraction of each column that is ink.
|
||||
///
|
||||
/// One number per pixel column, left to right, each between zero and one.
|
||||
pub fn profile(greymap: &[u8], threshold: u8) -> Result<Vec<f64>, Error> {
|
||||
let (width, height, levels, pixels) = header(greymap)?;
|
||||
if levels > 255 {
|
||||
return Err(Error::SampleTooWide(levels));
|
||||
}
|
||||
let wanted = width.checked_mul(height).ok_or(Error::BadHeader)?;
|
||||
if pixels.len() < wanted {
|
||||
return Err(Error::ShortImage {
|
||||
expected: wanted,
|
||||
found: pixels.len(),
|
||||
});
|
||||
}
|
||||
if width == 0 || height == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// A greymap need not go all the way to white, and the threshold is on the
|
||||
// usual nought-to-255 scale, so compare in that scale rather than the
|
||||
// file's.
|
||||
let dark = |value: u8| (u32::from(value) * 255 / levels.max(1)) < u32::from(threshold);
|
||||
|
||||
let mut counts = vec![0usize; width];
|
||||
for row in pixels[..wanted].chunks_exact(width) {
|
||||
for (column, value) in row.iter().enumerate() {
|
||||
if dark(*value) {
|
||||
counts[column] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(counts
|
||||
.into_iter()
|
||||
.map(|count| count as f64 / height as f64)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Width, height, levels of grey, and where the pixels start.
|
||||
fn header(greymap: &[u8]) -> Result<(usize, usize, u32, &[u8]), Error> {
|
||||
let rest = greymap.strip_prefix(b"P5").ok_or(Error::NotAGreymap)?;
|
||||
|
||||
let mut at = 0;
|
||||
let mut numbers = [0u32; 3];
|
||||
for number in &mut numbers {
|
||||
at += skip(&rest[at..]);
|
||||
let digits = rest[at..]
|
||||
.iter()
|
||||
.position(|byte| !byte.is_ascii_digit())
|
||||
.ok_or(Error::Truncated)?;
|
||||
if digits == 0 {
|
||||
return Err(Error::BadHeader);
|
||||
}
|
||||
*number = std::str::from_utf8(&rest[at..at + digits])
|
||||
.map_err(|_| Error::BadHeader)?
|
||||
.parse()
|
||||
.map_err(|_| Error::BadHeader)?;
|
||||
at += digits;
|
||||
}
|
||||
|
||||
// Exactly one byte of whitespace separates the header from the pixels,
|
||||
// and a pixel may itself be a whitespace byte.
|
||||
let pixels = rest.get(at + 1..).ok_or(Error::Truncated)?;
|
||||
Ok((numbers[0] as usize, numbers[1] as usize, numbers[2], pixels))
|
||||
}
|
||||
|
||||
/// Whitespace and comments, which a greymap header may carry anywhere.
|
||||
fn skip(bytes: &[u8]) -> usize {
|
||||
let mut at = 0;
|
||||
loop {
|
||||
let was = at;
|
||||
while bytes.get(at).is_some_and(u8::is_ascii_whitespace) {
|
||||
at += 1;
|
||||
}
|
||||
if bytes.get(at) == Some(&b'#') {
|
||||
at += bytes[at..]
|
||||
.iter()
|
||||
.position(|byte| *byte == b'\n')
|
||||
.map_or(bytes.len() - at, |end| end + 1);
|
||||
}
|
||||
if at == was {
|
||||
return at;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Four columns, two rows: black, white, black, white.
|
||||
const STRIPES: &[u8] = b"P5\n4 2\n255\n\x00\xff\x00\xff\x00\xff\x00\xff";
|
||||
|
||||
#[test]
|
||||
fn a_column_of_black_is_all_ink_and_a_column_of_white_is_none() {
|
||||
assert_eq!(
|
||||
profile(STRIPES, 160).expect("a greymap"),
|
||||
[1.0, 0.0, 1.0, 0.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grey_falls_on_the_side_of_the_threshold_it_belongs_to() {
|
||||
let greys = b"P5\n3 1\n255\n\x9e\x9f\xa0"; // 158, 159, 160
|
||||
assert_eq!(profile(greys, 160).expect("a greymap"), [1.0, 1.0, 0.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_and_extra_whitespace_in_the_header_are_skipped() {
|
||||
let commented = b"P5\n# written by something\n4 2\n255\n\x00\xff\x00\xff\x00\xff\x00\xff";
|
||||
assert_eq!(
|
||||
profile(commented, 160).expect("a greymap"),
|
||||
[1.0, 0.0, 1.0, 0.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pixel_that_is_a_whitespace_byte_is_still_a_pixel() {
|
||||
// 0x20 is a space and also a very dark grey. Eating it as header
|
||||
// whitespace would shift every row by one and smear the profile.
|
||||
let spaces = b"P5\n2 1\n255\n\x20\xff";
|
||||
assert_eq!(profile(spaces, 160).expect("a greymap"), [1.0, 0.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fewer_levels_of_grey_are_scaled_to_the_usual_range() {
|
||||
// Maximum 1, so 0 is black and 1 is white however it is written.
|
||||
let bilevel = b"P5\n2 1\n1\n\x00\x01";
|
||||
assert_eq!(profile(bilevel, 160).expect("a greymap"), [1.0, 0.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_greymap_says_so() {
|
||||
assert_eq!(
|
||||
profile(b"P6\n1 1\n255\n\x00\x00\x00", 160),
|
||||
Err(Error::NotAGreymap)
|
||||
);
|
||||
assert_eq!(profile(b"", 160), Err(Error::NotAGreymap));
|
||||
assert_eq!(profile(b"P5\n4 2\n", 160), Err(Error::Truncated));
|
||||
assert_eq!(profile(b"P5\nwide 2\n255\n", 160), Err(Error::BadHeader));
|
||||
assert_eq!(
|
||||
profile(b"P5\n1 1\n65535\n\x00\x00", 160),
|
||||
Err(Error::SampleTooWide(65535))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_greymap_cut_short_is_not_padded_with_white() {
|
||||
assert_eq!(
|
||||
profile(b"P5\n4 2\n255\n\x00\xff\x00", 160),
|
||||
Err(Error::ShortImage {
|
||||
expected: 8,
|
||||
found: 3
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_greymap_with_no_pixels_has_no_profile() {
|
||||
assert_eq!(
|
||||
profile(b"P5\n0 0\n255\n", 160).expect("a greymap"),
|
||||
Vec::<f64>::new()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
//! Two facing pages photographed as one landscape sheet, found and split.
|
||||
//!
|
||||
//! A book held open under a camera gives one wide image per shot, with two
|
||||
//! pages on it. Everything downstream reads it as a single page: the columns
|
||||
//! interleave, the reading order comes out scrambled, and the text that
|
||||
//! results is worse than useless because it looks like text.
|
||||
//!
|
||||
//! Aspect ratio alone is not enough to say so — a foldout, a plate, a page
|
||||
//! set in one narrow centred column are all genuinely wide. So the shape of
|
||||
//! the ink is checked too: two blocks of text with a blank band down the
|
||||
//! middle, on most of the pages sampled.
|
||||
//!
|
||||
//! Splitting is a change to the page boxes and nothing else. Nothing is
|
||||
//! re-rendered, so a 250-sheet book becomes a 500-page book at no cost in
|
||||
//! quality and almost none in size.
|
||||
|
||||
pub mod ink;
|
||||
pub mod pdf;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Above this width-to-height ratio a sheet is wide enough to hold two pages.
|
||||
///
|
||||
/// Two ordinary portrait pages side by side land near 1.36. A lone wide page
|
||||
/// can sit anywhere, which is why the gutter is the real test.
|
||||
pub const MIN_RATIO: f64 = 1.15;
|
||||
|
||||
/// How many of the sampled wide sheets must show a gutter before the whole
|
||||
/// document is called two-up.
|
||||
pub const MIN_GUTTER_FRACTION: f64 = 0.6;
|
||||
|
||||
/// How dark a pixel has to be to count as ink, from 0 black to 255 white.
|
||||
pub const INK_THRESHOLD: u8 = 160;
|
||||
|
||||
/// What fraction of a column has to be ink before the column counts as text.
|
||||
pub const TEXT_COLUMN_INK: f64 = 0.02;
|
||||
|
||||
/// How many pages are looked at, however long the document is.
|
||||
pub const SAMPLE_SIZE: usize = 8;
|
||||
|
||||
/// What one page looks like.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Page {
|
||||
/// Which page it is, counting from one.
|
||||
pub number: usize,
|
||||
/// Its width divided by its height.
|
||||
#[serde(with = "corpus_core::finite")]
|
||||
#[schemars(with = "f64")]
|
||||
pub ratio: f64,
|
||||
/// Whether a gutter was found. `None` when the page was too narrow to be
|
||||
/// worth looking at.
|
||||
pub gutter: Option<bool>,
|
||||
}
|
||||
|
||||
/// What a document is.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Verdict {
|
||||
/// Record version.
|
||||
#[serde(default = "version")]
|
||||
pub v: u32,
|
||||
/// The file the verdict is about.
|
||||
pub path: String,
|
||||
/// Whether it is two pages to a sheet.
|
||||
pub two_up: bool,
|
||||
/// How many pages it has.
|
||||
pub pages: usize,
|
||||
/// The middle ratio of the sampled pages.
|
||||
#[serde(with = "corpus_core::finite")]
|
||||
#[schemars(with = "f64")]
|
||||
pub median_ratio: f64,
|
||||
/// What share of the sampled wide sheets showed a gutter.
|
||||
#[serde(with = "corpus_core::finite")]
|
||||
#[schemars(with = "f64")]
|
||||
pub gutter_fraction: f64,
|
||||
/// The pages that were looked at.
|
||||
pub sampled: Vec<Page>,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
corpus_core::V
|
||||
}
|
||||
|
||||
impl corpus_core::Record for Verdict {
|
||||
fn version(&self) -> u32 {
|
||||
self.v
|
||||
}
|
||||
}
|
||||
|
||||
/// A page's width divided by its height, or zero for a page with no height.
|
||||
pub fn aspect_ratio(width: f64, height: f64) -> f64 {
|
||||
match height {
|
||||
0.0 => 0.0,
|
||||
height => width / height,
|
||||
}
|
||||
}
|
||||
|
||||
/// Which pages to look at: evenly spaced, and away from the ends.
|
||||
///
|
||||
/// Covers and end matter are single pages even in a book that was otherwise
|
||||
/// shot two at a time, so sampling them would argue against a document that
|
||||
/// plainly is two-up.
|
||||
pub fn sample_indices(pages: usize) -> Vec<usize> {
|
||||
if pages <= 6 {
|
||||
return (0..pages).collect();
|
||||
}
|
||||
let (low, high) = (2, pages - 3);
|
||||
let step = ((high - low) / SAMPLE_SIZE).max(1);
|
||||
(low..=high).step_by(step).take(SAMPLE_SIZE).collect()
|
||||
}
|
||||
|
||||
/// Whether a column ink profile has a blank band down the middle with text on
|
||||
/// both sides of it.
|
||||
///
|
||||
/// The three bands are deliberately narrow and deliberately apart: the middle
|
||||
/// eight per cent for the gutter, and a third of the width either side of it
|
||||
/// for the text, leaving the outer tenth alone because a scan's edges carry
|
||||
/// the shadow of the book's binding and the table it sat on.
|
||||
pub fn has_gutter(ink: &[f64]) -> bool {
|
||||
let width = ink.len() as f64;
|
||||
let band = |from: f64, to: f64| {
|
||||
let (from, to) = ((from * width) as usize, (to * width) as usize);
|
||||
ink.get(from..to.min(ink.len())).unwrap_or(&[])
|
||||
};
|
||||
|
||||
let centre = band(0.46, 0.54);
|
||||
let left = band(0.10, 0.44);
|
||||
let right = band(0.56, 0.90);
|
||||
if centre.is_empty() || left.is_empty() || right.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let blank = centre.iter().copied().fold(f64::INFINITY, f64::min);
|
||||
blank < TEXT_COLUMN_INK && median(left) > TEXT_COLUMN_INK && median(right) > TEXT_COLUMN_INK
|
||||
}
|
||||
|
||||
/// The middle value, averaging the two middle values of an even-length run.
|
||||
///
|
||||
/// The middle rather than the mean throughout, because one page of solid
|
||||
/// plate or one blank leaf should not move the answer for a whole book.
|
||||
pub fn median(values: &[f64]) -> f64 {
|
||||
if values.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut sorted = values.to_vec();
|
||||
sorted.sort_by(f64::total_cmp);
|
||||
let middle = sorted.len() / 2;
|
||||
match sorted.len() % 2 {
|
||||
0 => (sorted[middle - 1] + sorted[middle]) / 2.0,
|
||||
_ => sorted[middle],
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the sampled pages add up to a two-up document.
|
||||
///
|
||||
/// `gutters` holds one answer per sampled page that was wide enough to look
|
||||
/// at, which is why it is usually shorter than `ratios`.
|
||||
pub fn is_two_up(ratios: &[f64], gutters: &[bool]) -> bool {
|
||||
let median_ratio = median(ratios);
|
||||
median_ratio > MIN_RATIO && gutter_fraction(gutters) >= MIN_GUTTER_FRACTION
|
||||
}
|
||||
|
||||
/// What share of the pages looked at showed a gutter.
|
||||
pub fn gutter_fraction(gutters: &[bool]) -> f64 {
|
||||
match gutters.len() {
|
||||
0 => 0.0,
|
||||
len => gutters.iter().filter(|found| **found).count() as f64 / len as f64,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A profile with text either side of a blank middle.
|
||||
fn two_pages() -> Vec<f64> {
|
||||
(0..1000)
|
||||
.map(|x| match x {
|
||||
x if (100..440).contains(&x) => 0.3,
|
||||
x if (560..900).contains(&x) => 0.3,
|
||||
_ => 0.0,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_columns_either_side_of_a_blank_band_is_a_gutter() {
|
||||
assert!(has_gutter(&two_pages()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_wide_block_of_text_is_not() {
|
||||
let solid: Vec<f64> = (0..1000).map(|_| 0.3).collect();
|
||||
assert!(!has_gutter(&solid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_narrow_centred_column_is_not_two_pages() {
|
||||
// The shape that makes aspect ratio alone useless: a wide sheet with
|
||||
// one column of text down the middle of it.
|
||||
let centred: Vec<f64> = (0..1000)
|
||||
.map(|x| match (400..600).contains(&x) {
|
||||
true => 0.3,
|
||||
false => 0.0,
|
||||
})
|
||||
.collect();
|
||||
assert!(!has_gutter(¢red));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_page_has_no_gutter_because_it_has_no_text() {
|
||||
assert!(!has_gutter(&[0.0; 1000]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_on_only_one_side_is_one_page_that_happens_to_be_wide() {
|
||||
let half: Vec<f64> = (0..1000)
|
||||
.map(|x| match (100..440).contains(&x) {
|
||||
true => 0.3,
|
||||
false => 0.0,
|
||||
})
|
||||
.collect();
|
||||
assert!(!has_gutter(&half));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gutter_narrower_than_the_band_it_is_looked_for_in_still_counts() {
|
||||
// Only one blank column is needed, not a blank band: scans are rarely
|
||||
// square to the camera and the gutter wanders.
|
||||
let mut ink = vec![0.3; 1000];
|
||||
ink[500] = 0.0;
|
||||
assert!(has_gutter(&ink));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_profile_too_short_to_have_bands_is_not_a_gutter() {
|
||||
assert!(!has_gutter(&[]));
|
||||
assert!(!has_gutter(&[0.0, 1.0]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_documents_are_sampled_whole() {
|
||||
assert_eq!(sample_indices(0), Vec::<usize>::new());
|
||||
assert_eq!(sample_indices(1), [0]);
|
||||
assert_eq!(sample_indices(6), [0, 1, 2, 3, 4, 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longer_ones_skip_the_covers() {
|
||||
assert_eq!(sample_indices(7), [2, 3, 4]);
|
||||
assert_eq!(sample_indices(12), [2, 3, 4, 5, 6, 7, 8, 9]);
|
||||
assert_eq!(sample_indices(100), [2, 13, 24, 35, 46, 57, 68, 79]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_more_than_eight_pages_are_ever_looked_at() {
|
||||
for pages in 7..500 {
|
||||
assert!(sample_indices(pages).len() <= SAMPLE_SIZE, "{pages}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_sampled_page_exists() {
|
||||
for pages in 0..500 {
|
||||
for index in sample_indices(pages) {
|
||||
assert!(index < pages, "{pages} has no page {index}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_with_no_height_has_no_ratio_rather_than_an_infinite_one() {
|
||||
assert_eq!(aspect_ratio(600.0, 0.0), 0.0);
|
||||
assert_eq!(aspect_ratio(1008.0, 720.0), 1.4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_middle_of_an_even_run_is_the_mean_of_the_two_middle_values() {
|
||||
assert_eq!(median(&[1.0, 2.0, 3.0, 4.0]), 2.5);
|
||||
assert_eq!(median(&[1.0, 2.0, 3.0]), 2.0);
|
||||
assert_eq!(median(&[]), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_needs_both_the_shape_and_the_gutter() {
|
||||
let wide = [1.4; 8];
|
||||
let narrow = [0.7; 8];
|
||||
assert!(is_two_up(&wide, &[true; 8]));
|
||||
assert!(!is_two_up(&narrow, &[true; 8]));
|
||||
assert!(!is_two_up(&wide, &[]));
|
||||
// Three of eight is not most of them.
|
||||
let mostly_not = [true, true, true, false, false, false, false, false];
|
||||
assert!(!is_two_up(&wide, &mostly_not));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_only_just_over_the_line_still_counts() {
|
||||
// Six of ten is exactly the threshold, and the threshold is inclusive.
|
||||
let gutters = [
|
||||
true, true, true, true, true, true, false, false, false, false,
|
||||
];
|
||||
assert_eq!(gutter_fraction(&gutters), 0.6);
|
||||
assert!(is_two_up(&[1.4; 10], &gutters));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
//! Finds books scanned two pages to a sheet, and splits them back into pages.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Command, ExitCode};
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
use pdfgutter::{Page, Verdict};
|
||||
|
||||
/// Say whether a PDF is two facing pages photographed as one sheet, and split
|
||||
/// it back into pages when it is.
|
||||
///
|
||||
/// A book held open under a camera gives one wide image per shot. Anything
|
||||
/// that reads it afterwards runs the two columns together, and the text that
|
||||
/// comes out looks like text while being unusable.
|
||||
///
|
||||
/// Shells out to `pdftoppm` to look at the pages. Splitting does not: it
|
||||
/// changes the page boxes and leaves the content alone, so nothing is
|
||||
/// re-rendered and nothing is lost.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 two pages to a sheet, 1 one page to a sheet, 2 could not tell,
|
||||
69 pdftoppm is not installed, 74 the file could not be read or written.
|
||||
|
||||
Example:
|
||||
|
||||
pdfgutter scan.pdf && pdfgutter --into split.pdf scan.pdf
|
||||
for f in *.pdf; do pdfgutter --quiet \"$f\" && pdfgutter --into \"split/$f\" \"$f\"; done"
|
||||
)]
|
||||
struct Args {
|
||||
/// The PDF to look at.
|
||||
path: PathBuf,
|
||||
|
||||
/// Write the split document here. Implies nothing about the verdict: a
|
||||
/// document that is not two-up is copied through unchanged.
|
||||
#[arg(long, value_name = "FILE")]
|
||||
into: Option<PathBuf>,
|
||||
|
||||
/// Emit a record instead of a sentence.
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
|
||||
/// Say nothing; the exit code is the answer.
|
||||
#[arg(long, short)]
|
||||
quiet: bool,
|
||||
|
||||
/// How wide a sheet has to be, as a multiple of its height, before it
|
||||
/// could hold two pages.
|
||||
#[arg(long, default_value_t = pdfgutter::MIN_RATIO, value_name = "RATIO")]
|
||||
min_ratio: f64,
|
||||
|
||||
/// What share of the wide sheets looked at must show a gutter.
|
||||
#[arg(long, default_value_t = pdfgutter::MIN_GUTTER_FRACTION, value_name = "FRACTION")]
|
||||
gutter_fraction: f64,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
if !(args.min_ratio.is_finite() && args.gutter_fraction.is_finite()) {
|
||||
eprintln!("pdfgutter: a threshold has to be a number");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
|
||||
let mut pdf = match pdfgutter::pdf::Pdf::open(&args.path) {
|
||||
Ok(pdf) => pdf,
|
||||
Err(why) => {
|
||||
eprintln!("pdfgutter: {}: {why}", args.path.display());
|
||||
return ExitCode::from(match why {
|
||||
pdfgutter::pdf::Error::Unreadable(_) => exit::IO,
|
||||
_ => exit::INDETERMINATE,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let verdict = match measure(&args, &pdf) {
|
||||
Ok(verdict) => verdict,
|
||||
Err(code) => return code,
|
||||
};
|
||||
|
||||
// The verdict first, because it is the answer; what was written to disk
|
||||
// after it, because that is what happened.
|
||||
let said = say(&args, &verdict);
|
||||
|
||||
if let Some(destination) = args.into.as_deref() {
|
||||
let pages = pdf.split(args.min_ratio);
|
||||
if let Err(why) = pdf.save(destination) {
|
||||
eprintln!("pdfgutter: {}: {why}", destination.display());
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
if !args.quiet && !args.json {
|
||||
println!("{pages} pages written to {}", destination.display());
|
||||
}
|
||||
}
|
||||
|
||||
said
|
||||
}
|
||||
|
||||
/// Sample the document and decide.
|
||||
fn measure(args: &Args, pdf: &pdfgutter::pdf::Pdf) -> Result<Verdict, ExitCode> {
|
||||
let mut rasteriser = None;
|
||||
let mut sampled = Vec::new();
|
||||
let mut ratios = Vec::new();
|
||||
let mut gutters = Vec::new();
|
||||
|
||||
for index in pdfgutter::sample_indices(pdf.len()) {
|
||||
let ratio = pdf.geometry(index).ratio();
|
||||
ratios.push(ratio);
|
||||
|
||||
// Rendering is the expensive part, and a sheet too narrow to hold two
|
||||
// pages cannot be two pages whatever its ink looks like.
|
||||
let gutter = match ratio > args.min_ratio {
|
||||
false => None,
|
||||
true => {
|
||||
if rasteriser.is_none() {
|
||||
match which("pdftoppm") {
|
||||
None => {
|
||||
eprintln!("pdfgutter: pdftoppm is not on PATH");
|
||||
return Err(ExitCode::from(exit::UNAVAILABLE));
|
||||
}
|
||||
Some(found) => rasteriser = Some(found),
|
||||
}
|
||||
}
|
||||
Some(gutter_on(&args.path, index))
|
||||
}
|
||||
};
|
||||
if let Some(found) = gutter {
|
||||
gutters.push(found);
|
||||
}
|
||||
sampled.push(Page {
|
||||
number: index + 1,
|
||||
ratio,
|
||||
gutter,
|
||||
});
|
||||
}
|
||||
|
||||
let median_ratio = pdfgutter::median(&ratios);
|
||||
let gutter_fraction = pdfgutter::gutter_fraction(&gutters);
|
||||
Ok(Verdict {
|
||||
v: corpus_core::V,
|
||||
path: args.path.display().to_string(),
|
||||
two_up: median_ratio > args.min_ratio && gutter_fraction >= args.gutter_fraction,
|
||||
pages: pdf.len(),
|
||||
median_ratio,
|
||||
gutter_fraction,
|
||||
sampled,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether one page has a blank band down the middle with text either side.
|
||||
///
|
||||
/// A page that will not render is a page with no gutter rather than a reason
|
||||
/// to give up on the document: one bad page in a book is ordinary.
|
||||
fn gutter_on(path: &std::path::Path, index: usize) -> bool {
|
||||
let page = (index + 1).to_string();
|
||||
let rendered = Command::new("pdftoppm")
|
||||
.args([
|
||||
"-gray",
|
||||
"-scale-to-x",
|
||||
"1000",
|
||||
"-scale-to-y",
|
||||
"-1",
|
||||
"-singlefile",
|
||||
])
|
||||
.args(["-f", &page, "-l", &page])
|
||||
.arg(path)
|
||||
.output();
|
||||
|
||||
let Ok(rendered) = rendered else {
|
||||
return false;
|
||||
};
|
||||
if !rendered.status.success() {
|
||||
return false;
|
||||
}
|
||||
match pdfgutter::ink::profile(&rendered.stdout, pdfgutter::INK_THRESHOLD) {
|
||||
Ok(profile) => pdfgutter::has_gutter(&profile),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the verdict, and leave with it.
|
||||
fn say(args: &Args, verdict: &Verdict) -> ExitCode {
|
||||
let answer = match verdict.two_up {
|
||||
true => exit::OK,
|
||||
false => exit::NO,
|
||||
};
|
||||
if args.quiet {
|
||||
return ExitCode::from(answer);
|
||||
}
|
||||
|
||||
let text = match args.json {
|
||||
true => format!(
|
||||
"{}\n",
|
||||
serde_json::to_string(verdict).expect("a verdict serialises")
|
||||
),
|
||||
false => format!(
|
||||
"{}\n {} pages, median ratio {:.2}, gutter on {:.0}% of the wide ones\n",
|
||||
match verdict.two_up {
|
||||
true => "two pages to a sheet",
|
||||
false => "one page to a sheet",
|
||||
},
|
||||
verdict.pages,
|
||||
verdict.median_ratio,
|
||||
verdict.gutter_fraction * 100.0
|
||||
),
|
||||
};
|
||||
let wrote = catena_cli::text("pdfgutter", &text);
|
||||
match format!("{wrote:?}") == format!("{:?}", ExitCode::from(exit::OK)) {
|
||||
true => ExitCode::from(answer),
|
||||
false => wrote,
|
||||
}
|
||||
}
|
||||
|
||||
fn which(program: &str) -> Option<PathBuf> {
|
||||
let path = std::env::var_os("PATH")?;
|
||||
std::env::split_paths(&path)
|
||||
.map(|dir| dir.join(program))
|
||||
.find(|candidate| is_program(candidate))
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn is_program(path: &std::path::Path) -> bool {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
std::fs::metadata(path)
|
||||
.is_ok_and(|meta| meta.is_file() && meta.permissions().mode() & 0o111 != 0)
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn is_program(path: &std::path::Path) -> bool {
|
||||
path.is_file()
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
//! The page geometry, and the split itself.
|
||||
//!
|
||||
//! Splitting is a change to two rectangles per page. The content stream is
|
||||
//! not touched, not re-rendered and not even read: both halves of a sheet
|
||||
//! point at the same stream and differ only in how much of it they show.
|
||||
|
||||
use std::fmt;
|
||||
use std::path::Path;
|
||||
|
||||
use lopdf::{Document, Object, ObjectId};
|
||||
|
||||
/// The four page attributes a page may inherit from the tree above it.
|
||||
const INHERITED: [&[u8]; 4] = [b"Resources", b"MediaBox", b"CropBox", b"Rotate"];
|
||||
|
||||
/// The boxes that describe a region smaller than the page, which a half-page
|
||||
/// would leave pointing at the wrong place.
|
||||
const NARROWING: [&[u8]; 3] = [b"BleedBox", b"TrimBox", b"ArtBox"];
|
||||
|
||||
/// How far up a page tree to look for an inherited attribute before deciding
|
||||
/// the file is looping.
|
||||
const MAX_DEPTH: usize = 32;
|
||||
|
||||
/// The default page, for a file whose pages carry no box at all: US Letter,
|
||||
/// which is what every reader falls back to.
|
||||
const DEFAULT_BOX: [f64; 4] = [0.0, 0.0, 612.0, 792.0];
|
||||
|
||||
/// What can be wrong with a PDF.
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
/// It could not be read at all.
|
||||
Unreadable(String),
|
||||
/// It is encrypted, and no password was offered.
|
||||
Encrypted,
|
||||
/// It has no pages.
|
||||
Empty,
|
||||
/// It could not be written.
|
||||
Unwritable(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Error::Unreadable(why) => write!(f, "{why}"),
|
||||
Error::Encrypted => write!(f, "it is encrypted"),
|
||||
Error::Empty => write!(f, "it has no pages"),
|
||||
Error::Unwritable(why) => write!(f, "{why}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
/// A page's shape as a reader sees it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct Geometry {
|
||||
/// The media box, left, bottom, right, top, with the corners in order.
|
||||
pub media: [f64; 4],
|
||||
/// A quarter turn clockwise per unit, as the file asks for it.
|
||||
pub rotate: i64,
|
||||
}
|
||||
|
||||
impl Geometry {
|
||||
/// Width and height as displayed, which is the media box turned by
|
||||
/// whatever the page's rotation asks for.
|
||||
pub fn displayed(self) -> (f64, f64) {
|
||||
let width = self.media[2] - self.media[0];
|
||||
let height = self.media[3] - self.media[1];
|
||||
match self.quarter_turns() % 2 {
|
||||
0 => (width, height),
|
||||
_ => (height, width),
|
||||
}
|
||||
}
|
||||
|
||||
/// Width over height, as displayed.
|
||||
pub fn ratio(self) -> f64 {
|
||||
let (width, height) = self.displayed();
|
||||
crate::aspect_ratio(width, height)
|
||||
}
|
||||
|
||||
/// The rotation as a number of quarter turns, from zero to three.
|
||||
///
|
||||
/// A file may write any multiple of ninety, negative ones included.
|
||||
fn quarter_turns(self) -> i64 {
|
||||
(self.rotate / 90).rem_euclid(4)
|
||||
}
|
||||
|
||||
/// The two halves of the sheet, in the order a reader would read them.
|
||||
///
|
||||
/// Which way to cut depends on the rotation: a sheet turned on its side
|
||||
/// has its two pages one above the other in the file, however they look.
|
||||
pub fn halves(self) -> [[f64; 4]; 2] {
|
||||
let [left, bottom, right, top] = self.media;
|
||||
let middle_x = (left + right) / 2.0;
|
||||
let middle_y = (bottom + top) / 2.0;
|
||||
|
||||
let west = [left, bottom, middle_x, top];
|
||||
let east = [middle_x, bottom, right, top];
|
||||
let south = [left, bottom, right, middle_y];
|
||||
let north = [left, middle_y, right, top];
|
||||
|
||||
match self.quarter_turns() {
|
||||
0 => [west, east],
|
||||
1 => [south, north],
|
||||
2 => [east, west],
|
||||
_ => [north, south],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A PDF, open and ready to be measured or cut.
|
||||
pub struct Pdf {
|
||||
document: Document,
|
||||
pages: Vec<ObjectId>,
|
||||
}
|
||||
|
||||
impl Pdf {
|
||||
/// Open a file.
|
||||
pub fn open(path: &Path) -> Result<Self, Error> {
|
||||
let document = Document::load(path).map_err(|e| Error::Unreadable(e.to_string()))?;
|
||||
if document.is_encrypted() {
|
||||
return Err(Error::Encrypted);
|
||||
}
|
||||
let pages: Vec<ObjectId> = document.get_pages().into_values().collect();
|
||||
match pages.is_empty() {
|
||||
true => Err(Error::Empty),
|
||||
false => Ok(Pdf { document, pages }),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many pages it has.
|
||||
pub fn len(&self) -> usize {
|
||||
self.pages.len()
|
||||
}
|
||||
|
||||
/// Whether it has no pages. It never does; `open` refuses one.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.pages.is_empty()
|
||||
}
|
||||
|
||||
/// The shape of one page, counting from zero.
|
||||
pub fn geometry(&self, index: usize) -> Geometry {
|
||||
let id = self.pages[index];
|
||||
let media = self
|
||||
.inherited(id, b"MediaBox")
|
||||
.and_then(|object| rectangle(&self.document, object))
|
||||
.unwrap_or(DEFAULT_BOX);
|
||||
let rotate = self
|
||||
.inherited(id, b"Rotate")
|
||||
.and_then(|object| self.document.dereference(object).ok())
|
||||
.and_then(|(_, object)| object.as_i64().ok())
|
||||
.unwrap_or(0);
|
||||
Geometry { media, rotate }
|
||||
}
|
||||
|
||||
/// An attribute of a page, from the page itself or from the tree above it.
|
||||
fn inherited(&self, page: ObjectId, key: &[u8]) -> Option<&Object> {
|
||||
let mut at = page;
|
||||
for _ in 0..MAX_DEPTH {
|
||||
let dictionary = self.document.get_dictionary(at).ok()?;
|
||||
if let Ok(object) = dictionary.get(key) {
|
||||
return Some(object);
|
||||
}
|
||||
at = dictionary.get(b"Parent").ok()?.as_reference().ok()?;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Cut every sheet wide enough to hold two pages into two, and leave the
|
||||
/// rest alone.
|
||||
///
|
||||
/// Returns how many pages the document now has.
|
||||
pub fn split(&mut self, min_ratio: f64) -> usize {
|
||||
// Whatever a page would have inherited has to be written onto it
|
||||
// first, because the tree it would have inherited from is about to be
|
||||
// replaced by a flat one.
|
||||
self.settle_inherited();
|
||||
|
||||
let mut order: Vec<ObjectId> = Vec::with_capacity(self.pages.len());
|
||||
for index in 0..self.pages.len() {
|
||||
let id = self.pages[index];
|
||||
let geometry = self.geometry(index);
|
||||
if geometry.ratio() <= min_ratio {
|
||||
order.push(id);
|
||||
continue;
|
||||
}
|
||||
|
||||
let [first, second] = geometry.halves();
|
||||
let twin = match self.document.get_dictionary(id) {
|
||||
Ok(dictionary) => dictionary.clone(),
|
||||
Err(_) => {
|
||||
order.push(id);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let twin = self.document.add_object(Object::Dictionary(twin));
|
||||
|
||||
self.set_box(id, first);
|
||||
self.set_box(twin, second);
|
||||
order.push(id);
|
||||
order.push(twin);
|
||||
}
|
||||
|
||||
self.reparent(&order);
|
||||
self.pages = order;
|
||||
self.pages.len()
|
||||
}
|
||||
|
||||
/// Write the result.
|
||||
pub fn save(&mut self, path: &Path) -> Result<(), Error> {
|
||||
self.document
|
||||
.save(path)
|
||||
.map(|_| ())
|
||||
.map_err(|e| Error::Unwritable(e.to_string()))
|
||||
}
|
||||
|
||||
/// Copy every inheritable attribute down onto the pages that rely on it.
|
||||
fn settle_inherited(&mut self) {
|
||||
for page in self.pages.clone() {
|
||||
for key in INHERITED {
|
||||
let Some(value) = self.inherited(page, key).cloned() else {
|
||||
continue;
|
||||
};
|
||||
if let Ok(dictionary) = self.document.get_dictionary_mut(page) {
|
||||
if !dictionary.has(key) {
|
||||
dictionary.set(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Point a page at half the sheet.
|
||||
fn set_box(&mut self, page: ObjectId, rectangle: [f64; 4]) {
|
||||
let Ok(dictionary) = self.document.get_dictionary_mut(page) else {
|
||||
return;
|
||||
};
|
||||
let value: Vec<Object> = rectangle
|
||||
.iter()
|
||||
.map(|edge| Object::Real(*edge as f32))
|
||||
.collect();
|
||||
dictionary.set("MediaBox", Object::Array(value.clone()));
|
||||
dictionary.set("CropBox", Object::Array(value));
|
||||
// These describe a region inside the old page, and half of them now
|
||||
// lie outside it entirely.
|
||||
for key in NARROWING {
|
||||
dictionary.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the page tree with one flat node holding the pages in order.
|
||||
fn reparent(&mut self, order: &[ObjectId]) {
|
||||
let Ok(root) = self
|
||||
.document
|
||||
.catalog()
|
||||
.and_then(|catalog| catalog.get(b"Pages"))
|
||||
.and_then(Object::as_reference)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
|
||||
for page in order {
|
||||
if let Ok(dictionary) = self.document.get_dictionary_mut(*page) {
|
||||
dictionary.set("Parent", Object::Reference(root));
|
||||
}
|
||||
}
|
||||
|
||||
let kids: Vec<Object> = order.iter().map(|id| Object::Reference(*id)).collect();
|
||||
if let Ok(pages) = self.document.get_dictionary_mut(root) {
|
||||
pages.set("Kids", Object::Array(kids));
|
||||
pages.set("Count", Object::Integer(order.len() as i64));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Four numbers, in the order a PDF rectangle gives them, with the corners
|
||||
/// put back in order.
|
||||
///
|
||||
/// A file may write a rectangle from any corner to any other, and readers are
|
||||
/// required to cope, so a box that runs right to left is not a broken one.
|
||||
fn rectangle(document: &Document, object: &Object) -> Option<[f64; 4]> {
|
||||
let (_, object) = document.dereference(object).ok()?;
|
||||
let array = object.as_array().ok()?;
|
||||
if array.len() != 4 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut edges = [0.0f64; 4];
|
||||
for (edge, object) in edges.iter_mut().zip(array) {
|
||||
let (_, object) = document.dereference(object).ok()?;
|
||||
*edge = f64::from(object.as_float().ok()?);
|
||||
if !edge.is_finite() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some([
|
||||
edges[0].min(edges[2]),
|
||||
edges[1].min(edges[3]),
|
||||
edges[0].max(edges[2]),
|
||||
edges[1].max(edges[3]),
|
||||
])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sheet(rotate: i64) -> Geometry {
|
||||
Geometry {
|
||||
media: [0.0, 0.0, 1000.0, 700.0],
|
||||
rotate,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_upright_sheet_is_cut_left_then_right() {
|
||||
assert_eq!(
|
||||
sheet(0).halves(),
|
||||
[[0.0, 0.0, 500.0, 700.0], [500.0, 0.0, 1000.0, 700.0]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sheet_turned_upside_down_is_cut_right_then_left() {
|
||||
assert_eq!(
|
||||
sheet(180).halves(),
|
||||
[[500.0, 0.0, 1000.0, 700.0], [0.0, 0.0, 500.0, 700.0]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sheet_on_its_side_is_cut_across_rather_than_down() {
|
||||
// Turned a quarter clockwise, the file's bottom half is what the
|
||||
// reader sees on the left.
|
||||
assert_eq!(
|
||||
sheet(90).halves(),
|
||||
[[0.0, 0.0, 1000.0, 350.0], [0.0, 350.0, 1000.0, 700.0]]
|
||||
);
|
||||
assert_eq!(
|
||||
sheet(270).halves(),
|
||||
[[0.0, 350.0, 1000.0, 700.0], [0.0, 0.0, 1000.0, 350.0]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rotation_the_file_wrote_oddly_still_means_a_quarter_turn() {
|
||||
assert_eq!(sheet(-90).halves(), sheet(270).halves());
|
||||
assert_eq!(sheet(450).halves(), sheet(90).halves());
|
||||
assert_eq!(sheet(360).halves(), sheet(0).halves());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turning_a_sheet_on_its_side_turns_its_shape_with_it() {
|
||||
assert_eq!(sheet(0).displayed(), (1000.0, 700.0));
|
||||
assert_eq!(sheet(90).displayed(), (700.0, 1000.0));
|
||||
assert!(sheet(0).ratio() > crate::MIN_RATIO);
|
||||
assert!(sheet(90).ratio() < crate::MIN_RATIO);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_two_halves_add_back_up_to_the_sheet() {
|
||||
for rotate in [0, 90, 180, 270] {
|
||||
let [first, second] = sheet(rotate).halves();
|
||||
let area = |b: [f64; 4]| (b[2] - b[0]) * (b[3] - b[1]);
|
||||
assert_eq!(area(first) + area(second), 1000.0 * 700.0, "{rotate}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
//! Splitting a real file, checked by reading the result back.
|
||||
//!
|
||||
//! The PDFs are built here rather than kept as fixtures, so that what each
|
||||
//! test is about — an inherited box, a rotated sheet, a mixed document — is
|
||||
//! visible in the test instead of hidden in a binary.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use pdfgutter::pdf::Pdf;
|
||||
|
||||
/// One page of a document under construction.
|
||||
struct Sheet {
|
||||
width: f64,
|
||||
height: f64,
|
||||
rotate: i64,
|
||||
}
|
||||
|
||||
/// Write a PDF whose pages carry their own boxes.
|
||||
fn write(path: &Path, sheets: &[Sheet]) {
|
||||
write_with(path, sheets, false);
|
||||
}
|
||||
|
||||
/// Write a PDF, optionally hoisting the box onto the page tree so that every
|
||||
/// page has to inherit it.
|
||||
fn write_with(path: &Path, sheets: &[Sheet], inherit: bool) {
|
||||
let first = 4;
|
||||
let kids: Vec<String> = (0..sheets.len())
|
||||
.map(|i| format!("{} 0 R", first + 2 * i))
|
||||
.collect();
|
||||
|
||||
let mut objects: Vec<(usize, Vec<u8>)> = Vec::new();
|
||||
objects.push((1, b"<< /Type /Catalog /Pages 2 0 R >>".to_vec()));
|
||||
|
||||
let shared = match (inherit, sheets.first()) {
|
||||
(true, Some(sheet)) => format!(" /MediaBox [0 0 {} {}]", sheet.width, sheet.height),
|
||||
_ => String::new(),
|
||||
};
|
||||
objects.push((
|
||||
2,
|
||||
format!(
|
||||
"<< /Type /Pages /Count {} /Kids [{}]{shared} >>",
|
||||
sheets.len(),
|
||||
kids.join(" ")
|
||||
)
|
||||
.into_bytes(),
|
||||
));
|
||||
objects.push((
|
||||
3,
|
||||
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(),
|
||||
));
|
||||
|
||||
for (i, sheet) in sheets.iter().enumerate() {
|
||||
let content = format!("BT /F1 12 Tf 40 40 Td (page {}) Tj ET", i + 1);
|
||||
let own = match inherit {
|
||||
true => String::new(),
|
||||
false => format!(" /MediaBox [0 0 {} {}]", sheet.width, sheet.height),
|
||||
};
|
||||
let rotate = match sheet.rotate {
|
||||
0 => String::new(),
|
||||
turn => format!(" /Rotate {turn}"),
|
||||
};
|
||||
objects.push((
|
||||
first + 2 * i,
|
||||
format!(
|
||||
"<< /Type /Page /Parent 2 0 R{own}{rotate} /Resources << /Font << /F1 3 0 R >> >> /Contents {} 0 R >>",
|
||||
first + 2 * i + 1
|
||||
)
|
||||
.into_bytes(),
|
||||
));
|
||||
let mut stream = format!("<< /Length {} >>\nstream\n", content.len()).into_bytes();
|
||||
stream.extend_from_slice(content.as_bytes());
|
||||
stream.extend_from_slice(b"\nendstream");
|
||||
objects.push((first + 2 * i + 1, stream));
|
||||
}
|
||||
|
||||
let mut out: Vec<u8> = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = vec![0usize; objects.len() + 5];
|
||||
for (number, body) in &objects {
|
||||
offsets[*number] = out.len();
|
||||
out.extend_from_slice(format!("{number} 0 obj\n").as_bytes());
|
||||
out.extend_from_slice(body);
|
||||
out.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
let start = out.len();
|
||||
let top = objects.iter().map(|(n, _)| *n).max().unwrap_or(0) + 1;
|
||||
out.extend_from_slice(format!("xref\n0 {top}\n0000000000 65535 f \n").as_bytes());
|
||||
for offset in offsets.iter().take(top).skip(1) {
|
||||
out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
out.extend_from_slice(
|
||||
format!("trailer\n<< /Size {top} /Root 1 0 R >>\nstartxref\n{start}\n%%EOF\n").as_bytes(),
|
||||
);
|
||||
std::fs::write(path, out).expect("a writable file");
|
||||
}
|
||||
|
||||
/// A directory this test can write into, named for the test.
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("pdfgutter-{name}"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).expect("a writable directory");
|
||||
dir
|
||||
}
|
||||
|
||||
fn wide() -> Sheet {
|
||||
Sheet {
|
||||
width: 1008.0,
|
||||
height: 720.0,
|
||||
rotate: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn tall() -> Sheet {
|
||||
Sheet {
|
||||
width: 480.0,
|
||||
height: 720.0,
|
||||
rotate: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wide_sheet_becomes_two_pages_and_a_tall_one_is_left_alone() {
|
||||
let dir = scratch("mixed");
|
||||
let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
|
||||
write(&source, &[tall(), wide(), wide(), tall()]);
|
||||
|
||||
let mut pdf = Pdf::open(&source).expect("a readable PDF");
|
||||
assert_eq!(pdf.len(), 4);
|
||||
assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 6);
|
||||
pdf.save(&destination).expect("a written PDF");
|
||||
|
||||
let split = Pdf::open(&destination).expect("a readable PDF");
|
||||
assert_eq!(split.len(), 6);
|
||||
|
||||
// The covers keep their shape; the halves are each half a sheet wide.
|
||||
let widths: Vec<f64> = (0..6).map(|i| split.geometry(i).displayed().0).collect();
|
||||
assert_eq!(widths, [480.0, 504.0, 504.0, 504.0, 504.0, 480.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_halves_come_out_in_reading_order() {
|
||||
let dir = scratch("order");
|
||||
let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
|
||||
write(&source, &[wide()]);
|
||||
|
||||
let mut pdf = Pdf::open(&source).expect("a readable PDF");
|
||||
pdf.split(pdfgutter::MIN_RATIO);
|
||||
pdf.save(&destination).expect("a written PDF");
|
||||
|
||||
let split = Pdf::open(&destination).expect("a readable PDF");
|
||||
assert_eq!(split.geometry(0).media, [0.0, 0.0, 504.0, 720.0]);
|
||||
assert_eq!(split.geometry(1).media, [504.0, 0.0, 1008.0, 720.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_box_the_pages_inherit_is_found_and_kept() {
|
||||
// A page tree that puts one MediaBox at the top and none on any page is
|
||||
// ordinary, and reading the pages alone would see no box at all.
|
||||
let dir = scratch("inherited");
|
||||
let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
|
||||
write_with(&source, &[wide(), wide()], true);
|
||||
|
||||
let pdf = Pdf::open(&source).expect("a readable PDF");
|
||||
assert_eq!(pdf.geometry(0).media, [0.0, 0.0, 1008.0, 720.0]);
|
||||
|
||||
let mut pdf = pdf;
|
||||
assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 4);
|
||||
pdf.save(&destination).expect("a written PDF");
|
||||
|
||||
let split = Pdf::open(&destination).expect("a readable PDF");
|
||||
assert_eq!(split.geometry(0).media, [0.0, 0.0, 504.0, 720.0]);
|
||||
assert_eq!(split.geometry(3).media, [504.0, 0.0, 1008.0, 720.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sheet_on_its_side_is_cut_the_way_it_is_read_and_not_the_way_it_is_stored() {
|
||||
let dir = scratch("rotated");
|
||||
let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
|
||||
// Stored tall, displayed wide: a quarter turn makes this a spread.
|
||||
write(
|
||||
&source,
|
||||
&[Sheet {
|
||||
width: 720.0,
|
||||
height: 1008.0,
|
||||
rotate: 90,
|
||||
}],
|
||||
);
|
||||
|
||||
let pdf = Pdf::open(&source).expect("a readable PDF");
|
||||
assert_eq!(pdf.geometry(0).displayed(), (1008.0, 720.0));
|
||||
|
||||
let mut pdf = pdf;
|
||||
assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 2);
|
||||
pdf.save(&destination).expect("a written PDF");
|
||||
|
||||
let split = Pdf::open(&destination).expect("a readable PDF");
|
||||
// Cut across rather than down, and each half still displays upright.
|
||||
assert_eq!(split.geometry(0).media, [0.0, 0.0, 720.0, 504.0]);
|
||||
assert_eq!(split.geometry(1).media, [0.0, 504.0, 720.0, 1008.0]);
|
||||
assert_eq!(split.geometry(0).displayed(), (504.0, 720.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_of_ordinary_pages_comes_through_unchanged() {
|
||||
let dir = scratch("untouched");
|
||||
let (source, destination) = (dir.join("in.pdf"), dir.join("out.pdf"));
|
||||
write(&source, &[tall(), tall(), tall()]);
|
||||
|
||||
let mut pdf = Pdf::open(&source).expect("a readable PDF");
|
||||
assert_eq!(pdf.split(pdfgutter::MIN_RATIO), 3);
|
||||
pdf.save(&destination).expect("a written PDF");
|
||||
|
||||
let split = Pdf::open(&destination).expect("a readable PDF");
|
||||
assert_eq!(split.len(), 3);
|
||||
for index in 0..3 {
|
||||
assert_eq!(split.geometry(index).media, [0.0, 0.0, 480.0, 720.0]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splitting_twice_does_not_split_twice() {
|
||||
// Halves are portrait, so a second pass has nothing to do. Anything that
|
||||
// runs over a directory will eventually run over its own output.
|
||||
let dir = scratch("idempotent");
|
||||
let (source, once, twice) = (dir.join("in.pdf"), dir.join("1.pdf"), dir.join("2.pdf"));
|
||||
write(&source, &[wide(), wide(), wide()]);
|
||||
|
||||
let mut pdf = Pdf::open(&source).expect("a readable PDF");
|
||||
pdf.split(pdfgutter::MIN_RATIO);
|
||||
pdf.save(&once).expect("a written PDF");
|
||||
|
||||
let mut again = Pdf::open(&once).expect("a readable PDF");
|
||||
assert_eq!(again.split(pdfgutter::MIN_RATIO), 6);
|
||||
again.save(&twice).expect("a written PDF");
|
||||
|
||||
let split = Pdf::open(&twice).expect("a readable PDF");
|
||||
assert_eq!(split.len(), 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_pdf_is_refused_rather_than_guessed_at() {
|
||||
let dir = scratch("refused");
|
||||
let path = dir.join("not.pdf");
|
||||
std::fs::write(&path, b"this is not a PDF at all").expect("a writable file");
|
||||
assert!(Pdf::open(&path).is_err());
|
||||
}
|
||||
@@ -23,6 +23,7 @@ cueseg.workspace = true
|
||||
mdscrub.workspace = true
|
||||
mdsplit.workspace = true
|
||||
avplan.workspace = true
|
||||
pdfgutter.workspace = true
|
||||
pdftriage.workspace = true
|
||||
rrf.workspace = true
|
||||
corpus-core.workspace = true
|
||||
|
||||
@@ -201,3 +201,70 @@ metadata and is wrong often enough that the comment above it says so.
|
||||
**Why this is acceptable:** there is no database, and inventing a codec is
|
||||
how a compatible file gets re-encoded for forty minutes. Saying so is the
|
||||
honest answer and the cheap one.
|
||||
|
||||
## pdfgutter — a different rasteriser, so the profiles are compared rather than the pixels
|
||||
|
||||
**Python:** renders each sampled page with pdfium at a scale that makes it a
|
||||
thousand pixels wide, and counts dark pixels per column.
|
||||
|
||||
**Rust:** asks `pdftoppm` for the same thousand-pixel-wide greymap and counts
|
||||
the same way.
|
||||
|
||||
**Why this is acceptable:** two rasterisers antialias differently, so a pixel
|
||||
on the edge of a letterform can land either side of the ink threshold in one
|
||||
and not the other. No amount of care makes two renderers agree pixel for
|
||||
pixel, and a parity test that demanded it would be a test of pdfium rather
|
||||
than of this.
|
||||
|
||||
What is compared instead is everything that decides: the aspect ratio, which
|
||||
pages get sampled, whether a given ink profile has a gutter, and the verdict
|
||||
a set of ratios and gutters adds up to. Both sides are handed the same
|
||||
profiles and have to agree exactly.
|
||||
|
||||
**Measured:** 72 cases, including profiles either side of the ink threshold by
|
||||
one ten-thousandth and profiles too narrow to hold the three bands the test
|
||||
looks at.
|
||||
|
||||
## pdfgutter — the thresholds are arguments, not environment variables
|
||||
|
||||
**Python:** reads `BOOK_PIPELINE_2UP_MIN_RATIO` and
|
||||
`BOOK_PIPELINE_2UP_GUTTER_FRAC` at import.
|
||||
|
||||
**Rust:** `--min-ratio` and `--gutter-fraction`, with the same defaults.
|
||||
|
||||
**Why this is acceptable:** an environment variable read at import makes the
|
||||
answer depend on the shell that happened to launch the process, which is the
|
||||
one thing a filter must not do. A flag says the same thing where a reader can
|
||||
see it, and shows up in the record `--json` writes.
|
||||
|
||||
## pdfgutter — a sheet on its side is cut the way it is read
|
||||
|
||||
**Python:** takes the media box and ignores `/Rotate` entirely. A sheet stored
|
||||
tall and displayed wide is never a split candidate; one stored wide and
|
||||
displayed tall is a candidate, and would be cut down the middle of an axis
|
||||
that has nothing down the middle of it.
|
||||
|
||||
**Rust:** turns the media box by whatever `/Rotate` asks for before measuring
|
||||
it, and cuts across rather than down for a quarter turn, in the order a
|
||||
reader would read the halves.
|
||||
|
||||
**Why this is acceptable:** the wrong half of a rotated sheet is a page with
|
||||
the top of one page and the bottom of another on it, and no pipeline
|
||||
downstream can recover from that. The rotation is in the file precisely so
|
||||
that readers honour it. Documents with no rotation — nearly all of them —
|
||||
behave identically.
|
||||
|
||||
## pdfgutter — an unreadable file is an answer, not a false
|
||||
|
||||
**Python:** `is_double_page` catches everything and returns `False`, so a
|
||||
corrupt file, an encrypted one and an ordinary single-page book are the same
|
||||
answer.
|
||||
|
||||
**Rust:** exit 2 for a file it cannot judge, exit 74 for one it cannot read,
|
||||
and exit 1 only for a document it looked at and found to be one page per
|
||||
sheet.
|
||||
|
||||
**Why this is acceptable:** "no" and "could not tell" are different answers
|
||||
and the contract gives them different numbers. A caller that wants the
|
||||
Python's behaviour treats both as no; a caller that wants to know which files
|
||||
it failed on can now find out.
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,72 @@
|
||||
{"kind": "ratio", "name": "two-portraits-side-by-side", "ratio": 1.4}
|
||||
{"kind": "ratio", "name": "a4-landscape", "ratio": 1.4142756349952963}
|
||||
{"kind": "ratio", "name": "a4-portrait", "ratio": 0.7070757462376319}
|
||||
{"kind": "ratio", "name": "us-letter", "ratio": 0.7727272727272727}
|
||||
{"kind": "ratio", "name": "square", "ratio": 1.0}
|
||||
{"kind": "ratio", "name": "exactly-at-the-threshold", "ratio": 1.15}
|
||||
{"kind": "ratio", "name": "a-hair-over", "ratio": 1.1501388888888888}
|
||||
{"kind": "ratio", "name": "no-height", "ratio": 0.0}
|
||||
{"kind": "ratio", "name": "no-width", "ratio": 0.0}
|
||||
{"kind": "ratio", "name": "a-very-long-foldout", "ratio": 5.714285714285714}
|
||||
{"gutter": true, "kind": "gutter", "name": "two-columns"}
|
||||
{"gutter": false, "kind": "gutter", "name": "one-wide-block"}
|
||||
{"gutter": false, "kind": "gutter", "name": "a-narrow-centred-column"}
|
||||
{"gutter": false, "kind": "gutter", "name": "blank"}
|
||||
{"gutter": false, "kind": "gutter", "name": "text-on-the-left-only"}
|
||||
{"gutter": false, "kind": "gutter", "name": "text-on-the-right-only"}
|
||||
{"gutter": true, "kind": "gutter", "name": "a-single-blank-column"}
|
||||
{"gutter": true, "kind": "gutter", "name": "a-gutter-off-centre"}
|
||||
{"gutter": false, "kind": "gutter", "name": "text-just-at-the-ink-threshold"}
|
||||
{"gutter": true, "kind": "gutter", "name": "text-just-over-it"}
|
||||
{"gutter": false, "kind": "gutter", "name": "a-gutter-that-is-not-quite-blank"}
|
||||
{"gutter": false, "kind": "gutter", "name": "empty"}
|
||||
{"gutter": false, "kind": "gutter", "name": "two-columns-too-few"}
|
||||
{"gutter": true, "kind": "gutter", "name": "odd-width"}
|
||||
{"gutter": true, "kind": "gutter", "name": "a-shadow-at-the-edges"}
|
||||
{"gutter": false, "kind": "gutter", "name": "tiny-7"}
|
||||
{"gutter": true, "kind": "gutter", "name": "tiny-13"}
|
||||
{"gutter": true, "kind": "gutter", "name": "tiny-25"}
|
||||
{"gutter": true, "kind": "gutter", "name": "tiny-101"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-0"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-1"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-2"}
|
||||
{"gutter": false, "kind": "gutter", "name": "random-3"}
|
||||
{"gutter": false, "kind": "gutter", "name": "random-4"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-5"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-6"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-7"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-8"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-9"}
|
||||
{"gutter": false, "kind": "gutter", "name": "random-10"}
|
||||
{"gutter": true, "kind": "gutter", "name": "random-11"}
|
||||
{"indices": [], "kind": "sample", "name": "pages-0"}
|
||||
{"indices": [0], "kind": "sample", "name": "pages-1"}
|
||||
{"indices": [0, 1], "kind": "sample", "name": "pages-2"}
|
||||
{"indices": [0, 1, 2, 3, 4], "kind": "sample", "name": "pages-5"}
|
||||
{"indices": [0, 1, 2, 3, 4, 5], "kind": "sample", "name": "pages-6"}
|
||||
{"indices": [2, 3, 4], "kind": "sample", "name": "pages-7"}
|
||||
{"indices": [2, 3, 4, 5], "kind": "sample", "name": "pages-8"}
|
||||
{"indices": [2, 3, 4, 5, 6], "kind": "sample", "name": "pages-9"}
|
||||
{"indices": [2, 3, 4, 5, 6, 7, 8], "kind": "sample", "name": "pages-11"}
|
||||
{"indices": [2, 3, 4, 5, 6, 7, 8, 9], "kind": "sample", "name": "pages-12"}
|
||||
{"indices": [2, 3, 4, 5, 6, 7, 8, 9], "kind": "sample", "name": "pages-16"}
|
||||
{"indices": [2, 3, 4, 5, 6, 7, 8, 9], "kind": "sample", "name": "pages-17"}
|
||||
{"indices": [2, 3, 4, 5, 6, 7, 8, 9], "kind": "sample", "name": "pages-20"}
|
||||
{"indices": [2, 7, 12, 17, 22, 27, 32, 37], "kind": "sample", "name": "pages-50"}
|
||||
{"indices": [2, 13, 24, 35, 46, 57, 68, 79], "kind": "sample", "name": "pages-99"}
|
||||
{"indices": [2, 13, 24, 35, 46, 57, 68, 79], "kind": "sample", "name": "pages-100"}
|
||||
{"indices": [2, 32, 62, 92, 122, 152, 182, 212], "kind": "sample", "name": "pages-250"}
|
||||
{"indices": [2, 64, 126, 188, 250, 312, 374, 436], "kind": "sample", "name": "pages-501"}
|
||||
{"indices": [2, 126, 250, 374, 498, 622, 746, 870], "kind": "sample", "name": "pages-1000"}
|
||||
{"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.4, "name": "a-clear-two-up", "two_up": true}
|
||||
{"gutter_fraction": 0.0, "kind": "verdict", "median_ratio": 0.68, "name": "a-clear-single", "two_up": false}
|
||||
{"gutter_fraction": 0.0, "kind": "verdict", "median_ratio": 1.4, "name": "wide-but-no-gutters", "two_up": false}
|
||||
{"gutter_fraction": 0.6, "kind": "verdict", "median_ratio": 1.4, "name": "exactly-six-in-ten", "two_up": true}
|
||||
{"gutter_fraction": 0.5, "kind": "verdict", "median_ratio": 1.4, "name": "five-in-ten", "two_up": false}
|
||||
{"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.4, "name": "mixed-shapes", "two_up": true}
|
||||
{"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.04, "name": "an-even-split-of-shapes", "two_up": false}
|
||||
{"gutter_fraction": 0.0, "kind": "verdict", "median_ratio": 0.0, "name": "nothing-sampled", "two_up": false}
|
||||
{"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.4, "name": "one-page", "two_up": true}
|
||||
{"gutter_fraction": 0.6666666666666666, "kind": "verdict", "median_ratio": 1.4, "name": "two-of-three", "two_up": true}
|
||||
{"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.1501, "name": "just-over-the-ratio", "two_up": true}
|
||||
{"gutter_fraction": 1.0, "kind": "verdict", "median_ratio": 1.15, "name": "exactly-the-ratio", "two_up": false}
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Decide 2-up scans the way the book pipeline did, for comparison.
|
||||
|
||||
The four functions below are the originals, copied out of the preprocessing
|
||||
module verbatim — importing it would pull in a PDF renderer and a page tree,
|
||||
and none of that decides anything. What decides is the arithmetic, which is
|
||||
what this compares.
|
||||
|
||||
python3 oracle/pdfgutter.py < cases.ndjson
|
||||
|
||||
One case per line, tagged with what it is asking about. Writes one JSON
|
||||
answer per line.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
_MIN_RATIO = 1.15
|
||||
_MIN_GUTTER_FRAC = 0.6
|
||||
_TEXT_COL_INK = 0.02
|
||||
|
||||
|
||||
def _ratio(width: float, height: float) -> float:
|
||||
return width / height if height else 0.0
|
||||
|
||||
|
||||
def _has_gutter(ink: np.ndarray) -> bool:
|
||||
w = len(ink)
|
||||
center = ink[int(0.46 * w):int(0.54 * w)]
|
||||
left = ink[int(0.10 * w):int(0.44 * w)]
|
||||
right = ink[int(0.56 * w):int(0.90 * w)]
|
||||
if center.size == 0 or left.size == 0 or right.size == 0:
|
||||
return False
|
||||
return bool(
|
||||
center.min() < _TEXT_COL_INK
|
||||
and np.median(left) > _TEXT_COL_INK
|
||||
and np.median(right) > _TEXT_COL_INK
|
||||
)
|
||||
|
||||
|
||||
def _sample_indices(n: int, k: int = 8) -> list[int]:
|
||||
if n <= 6:
|
||||
return list(range(n))
|
||||
lo, hi = 2, n - 3
|
||||
step = max(1, (hi - lo) // k)
|
||||
return list(range(lo, hi + 1, step))[:k]
|
||||
|
||||
|
||||
def _verdict(ratios: list[float], gutters: list[bool]) -> dict:
|
||||
median_ratio = float(np.median(ratios)) if ratios else 0.0
|
||||
gutter_frac = (sum(gutters) / len(gutters)) if gutters else 0.0
|
||||
return {
|
||||
"median_ratio": median_ratio,
|
||||
"gutter_fraction": gutter_frac,
|
||||
"two_up": bool(median_ratio > _MIN_RATIO and gutter_frac >= _MIN_GUTTER_FRAC),
|
||||
}
|
||||
|
||||
|
||||
def answer(case: dict) -> dict:
|
||||
kind = case["kind"]
|
||||
if kind == "ratio":
|
||||
return {"ratio": _ratio(case["width"], case["height"])}
|
||||
if kind == "gutter":
|
||||
return {"gutter": _has_gutter(np.asarray(case["ink"], dtype=float))}
|
||||
if kind == "sample":
|
||||
return {"indices": _sample_indices(case["pages"])}
|
||||
if kind == "verdict":
|
||||
return _verdict(case["ratios"], case["gutters"])
|
||||
raise SystemExit(f"no such case: {kind}")
|
||||
|
||||
|
||||
for line in sys.stdin:
|
||||
if not line.strip():
|
||||
continue
|
||||
case = json.loads(line)
|
||||
out = {"name": case["name"], "kind": case["kind"]}
|
||||
out.update(answer(case))
|
||||
sys.stdout.write(json.dumps(out, sort_keys=True) + "\n")
|
||||
+14
-1
@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
|
||||
type Job = (&'static str, fn() -> Written);
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let jobs: [Job; 16] = [
|
||||
let jobs: [Job; 17] = [
|
||||
("tokens", tokens),
|
||||
("unescape", unescape),
|
||||
("chunkmd", chunkmd),
|
||||
@@ -36,6 +36,7 @@ fn main() -> ExitCode {
|
||||
("vocab", vocab),
|
||||
("mdsplit", mdsplit),
|
||||
("avplan", avplan),
|
||||
("pdfgutter", pdfgutter),
|
||||
];
|
||||
|
||||
for (name, job) in jobs {
|
||||
@@ -218,6 +219,18 @@ fn cueseg() -> Written {
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
fn pdfgutter() -> Written {
|
||||
let dir = parity::golden("pdfgutter");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
let cases = parity::dir().join("cases/pdfgutter/cases.ndjson");
|
||||
let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
|
||||
let answers = parity::run_oracle("pdfgutter.py", &text)?;
|
||||
let path = dir.join("cases.ndjson");
|
||||
std::fs::write(&path, answers).map_err(|e| e.to_string())?;
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn avplan() -> Written {
|
||||
let dir = parity::golden("avplan");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
//! The arithmetic that decides a two-up scan must match the Python exactly.
|
||||
//!
|
||||
//! The consequence of getting it wrong is not a wrong page count — it is a
|
||||
//! book whose text comes out interleaved line by line between two pages,
|
||||
//! which reads as prose and is not. Nobody notices that in a corpus until
|
||||
//! they read a retrieved passage and find it is nonsense.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
fn records(text: &str) -> Vec<Value> {
|
||||
text.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| serde_json::from_str(line).expect("valid JSON"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn cases() -> Vec<Value> {
|
||||
let text = std::fs::read_to_string(parity::dir().join("cases/pdfgutter/cases.ndjson"))
|
||||
.expect("a readable case file");
|
||||
records(&text)
|
||||
}
|
||||
|
||||
fn floats(value: &Value) -> Vec<f64> {
|
||||
value
|
||||
.as_array()
|
||||
.expect("an array")
|
||||
.iter()
|
||||
.map(|number| number.as_f64().expect("a number"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_answer_matches_python() {
|
||||
let golden = records(&parity::read_golden_text("pdfgutter/cases.ndjson"));
|
||||
let cases = cases();
|
||||
assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
|
||||
|
||||
let mut wrong: Vec<String> = Vec::new();
|
||||
for (case, theirs) in cases.iter().zip(&golden) {
|
||||
let name = case["name"].as_str().expect("a name");
|
||||
let kind = case["kind"].as_str().expect("a kind");
|
||||
assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
|
||||
assert_eq!(theirs["kind"].as_str(), Some(kind), "cases out of order");
|
||||
|
||||
let mut note = |field: &str, ours: String, theirs: &Value| {
|
||||
wrong.push(format!("{name}.{field}: python {theirs}, rust {ours}"));
|
||||
};
|
||||
|
||||
match kind {
|
||||
"ratio" => {
|
||||
let ours = pdfgutter::aspect_ratio(
|
||||
case["width"].as_f64().expect("a width"),
|
||||
case["height"].as_f64().expect("a height"),
|
||||
);
|
||||
if Some(ours) != theirs["ratio"].as_f64() {
|
||||
note("ratio", ours.to_string(), &theirs["ratio"]);
|
||||
}
|
||||
}
|
||||
"gutter" => {
|
||||
let ours = pdfgutter::has_gutter(&floats(&case["ink"]));
|
||||
if Some(ours) != theirs["gutter"].as_bool() {
|
||||
note("gutter", ours.to_string(), &theirs["gutter"]);
|
||||
}
|
||||
}
|
||||
"sample" => {
|
||||
let ours = pdfgutter::sample_indices(
|
||||
case["pages"].as_u64().expect("a page count") as usize
|
||||
);
|
||||
let theirs_indices: Vec<usize> = theirs["indices"]
|
||||
.as_array()
|
||||
.expect("an array")
|
||||
.iter()
|
||||
.map(|index| index.as_u64().expect("a number") as usize)
|
||||
.collect();
|
||||
if ours != theirs_indices {
|
||||
note("indices", format!("{ours:?}"), &theirs["indices"]);
|
||||
}
|
||||
}
|
||||
"verdict" => {
|
||||
let ratios = floats(&case["ratios"]);
|
||||
let gutters: Vec<bool> = case["gutters"]
|
||||
.as_array()
|
||||
.expect("an array")
|
||||
.iter()
|
||||
.map(|found| found.as_bool().expect("a boolean"))
|
||||
.collect();
|
||||
|
||||
let median = pdfgutter::median(&ratios);
|
||||
if Some(median) != theirs["median_ratio"].as_f64() {
|
||||
note("median_ratio", median.to_string(), &theirs["median_ratio"]);
|
||||
}
|
||||
let fraction = pdfgutter::gutter_fraction(&gutters);
|
||||
if Some(fraction) != theirs["gutter_fraction"].as_f64() {
|
||||
note(
|
||||
"gutter_fraction",
|
||||
fraction.to_string(),
|
||||
&theirs["gutter_fraction"],
|
||||
);
|
||||
}
|
||||
let two_up = pdfgutter::is_two_up(&ratios, &gutters);
|
||||
if Some(two_up) != theirs["two_up"].as_bool() {
|
||||
note("two_up", two_up.to_string(), &theirs["two_up"]);
|
||||
}
|
||||
}
|
||||
other => panic!("nothing answers a {other} case"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
wrong.is_empty(),
|
||||
"{} answers differ:\n{}",
|
||||
wrong.len(),
|
||||
wrong
|
||||
.iter()
|
||||
.take(20)
|
||||
.map(|line| line.chars().take(200).collect::<String>())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// All four questions have to be asked, or the file proves less than it looks
|
||||
/// like it does.
|
||||
#[test]
|
||||
fn every_kind_of_case_is_present() {
|
||||
let cases = cases();
|
||||
for kind in ["ratio", "gutter", "sample", "verdict"] {
|
||||
assert!(
|
||||
cases.iter().any(|case| case["kind"] == kind),
|
||||
"no {kind} case"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Catches a golden file drifting away from a Python that has moved on.
|
||||
#[test]
|
||||
#[cfg(feature = "python")]
|
||||
fn the_golden_file_still_matches_live_python() {
|
||||
let cases = std::fs::read_to_string(parity::dir().join("cases/pdfgutter/cases.ndjson"))
|
||||
.expect("a readable case file");
|
||||
let live = parity::run_oracle("pdfgutter.py", &cases).expect("the oracle runs");
|
||||
assert_eq!(live, parity::read_golden_text("pdfgutter/cases.ndjson"));
|
||||
}
|
||||
Reference in New Issue
Block a user