mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
colophon: six more things the front matter of a book is not
Read out of the review queue a real run produced, where each of these was sitting as a proposed title waiting for a person to say no: 1. General Remarks a numbered heading 2 James W. McAllister a numbered heading 5222 What Is This a folio that ran into the line 827 North Fourth Street an address Page xxxvi / Chapter I a marker title a form label, printed literally Sidebottom, Harry. Ancient a bibliography entry To Mackenzie, a dedication The numbers are the delicate part. A leading number that could be a year is left alone, because 1984 Revisited is a book and 5222 What Is This is a folio, and nothing else distinguishes them. A dedication has to end in a comma, so To Kill a Mockingbird survives. The copyright notices in the same pile are already fixed: those proposals were written before 0.2.2 and are stale rather than wrong.
This commit is contained in:
funkpower
Generated
+26
-26
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "avplan"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bibclean"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -143,7 +143,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bibfetch"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -196,7 +196,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bookmeta"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -210,7 +210,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"bibfetch",
|
||||
@@ -277,7 +277,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_mangen",
|
||||
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -334,7 +334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "citefind"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"chunkmd",
|
||||
@@ -399,7 +399,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "colophon"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -448,7 +448,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
@@ -492,7 +492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"corpus-core",
|
||||
"regex",
|
||||
@@ -504,7 +504,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuenorm"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -514,7 +514,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cuepack"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -525,7 +525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cueseg"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -612,7 +612,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -876,7 +876,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
@@ -942,7 +942,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "langid"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1022,7 +1022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1034,7 +1034,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mdsplit"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdfgutter"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1126,7 +1126,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdftriage"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
||||
|
||||
[[package]]
|
||||
name = "rrf"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1517,7 +1517,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "seriesof"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"tiktoken-rs",
|
||||
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "tocfind"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
|
||||
+27
-27
@@ -3,39 +3,39 @@ resolver = "3"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MPL-2.0"
|
||||
repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[workspace.dependencies]
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.2.2" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.2.2" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.2.2" }
|
||||
textkit = { path = "crates/textkit", version = "0.2.2" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.2.2" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.2.2" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.2.2" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.2.2" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.2.2" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.2.2" }
|
||||
rrf = { path = "crates/rrf", version = "0.2.2" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.2.2" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.2.2" }
|
||||
avplan = { path = "crates/avplan", version = "0.2.2" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.2.2" }
|
||||
catena = { path = "crates/bindings", version = "0.2.2" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.2.2" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.2.2" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.2.2" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.2.2" }
|
||||
langid = { path = "crates/langid", version = "0.2.2" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.2.2" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.2.2" }
|
||||
citefind = { path = "crates/citefind", version = "0.2.2" }
|
||||
colophon = { path = "crates/colophon", version = "0.2.2" }
|
||||
emb = { path = "crates/emb", version = "0.2.2" }
|
||||
catena-cli = { path = "crates/catena-cli", version = "0.2.3" }
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.2.3" }
|
||||
cuekit = { path = "crates/cuekit", version = "0.2.3" }
|
||||
textkit = { path = "crates/textkit", version = "0.2.3" }
|
||||
chunkmd = { path = "crates/chunkmd", version = "0.2.3" }
|
||||
mdscrub = { path = "crates/mdscrub", version = "0.2.3" }
|
||||
mdsplit = { path = "crates/mdsplit", version = "0.2.3" }
|
||||
pdftriage = { path = "crates/pdftriage", version = "0.2.3" }
|
||||
cuenorm = { path = "crates/cuenorm", version = "0.2.3" }
|
||||
cueseg = { path = "crates/cueseg", version = "0.2.3" }
|
||||
rrf = { path = "crates/rrf", version = "0.2.3" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.2.3" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.2.3" }
|
||||
avplan = { path = "crates/avplan", version = "0.2.3" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.2.3" }
|
||||
catena = { path = "crates/bindings", version = "0.2.3" }
|
||||
seriesof = { path = "crates/seriesof", version = "0.2.3" }
|
||||
idnorm = { path = "crates/idnorm", version = "0.2.3" }
|
||||
bibfetch = { path = "crates/bibfetch", version = "0.2.3" }
|
||||
bookmeta = { path = "crates/bookmeta", version = "0.2.3" }
|
||||
langid = { path = "crates/langid", version = "0.2.3" }
|
||||
tocfind = { path = "crates/tocfind", version = "0.2.3" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.2.3" }
|
||||
citefind = { path = "crates/citefind", version = "0.2.3" }
|
||||
colophon = { path = "crates/colophon", version = "0.2.3" }
|
||||
emb = { path = "crates/emb", version = "0.2.3" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
|
||||
@@ -35,6 +35,29 @@ static LIFESPAN: LazyLock<Regex> = LazyLock::new(|| {
|
||||
static BARE_DATE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)^\d{1,4}\s*(bc|ad|bce|ce)?\.?$").expect("a valid pattern"));
|
||||
/// `Copyright © 2013 by`, `\u{a9} 2013 Richard Swinburne`, `Copyright 1971 by`.
|
||||
/// `1. General Remarks`, `2 James W. McAllister` — a numbered heading.
|
||||
static NUMBERED_HEADING: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*\d+\s*[.)]\s+\S|^\s*\d+\s{2,}\S").expect("a valid pattern"));
|
||||
/// `5222 What Is This`, `827 North Fourth Street` — a number that is not a
|
||||
/// year, leading a line.
|
||||
static LEADING_NUMBER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*(\d{1,5})\s+\S").expect("a valid pattern"));
|
||||
/// `Page xxxvi`, `Chapter I`, `Fig. 3` — a marker, not a name.
|
||||
static MARKER: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)^\s*(page|pages|p\.|pp\.|fig\.|figure|plate|chapter|ch\.|section|part)\s+[\divxlcm]",
|
||||
)
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
/// `Sidebottom, Harry. Ancient warfare` — a bibliography entry, which front
|
||||
/// matter prints in its cataloguing block and in its further reading.
|
||||
static CATALOGUE_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^[A-Z][A-Za-z'\-]+,\s+[A-Z][A-Za-z'.\-]*\.\s+\S").expect("a valid pattern")
|
||||
});
|
||||
/// `To Mackenzie,` — a dedication, set alone on its own page.
|
||||
static DEDICATION: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)^\s*(to|for|in memory of)\s+\S.*,\s*$").expect("a valid pattern")
|
||||
});
|
||||
static COPYRIGHT_NOTICE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
// The year need not follow the symbol directly: "(c) Alfredo Ferrarin
|
||||
// 2004" puts the holder between them, and was read as a title.
|
||||
@@ -79,10 +102,13 @@ copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*",
|
||||
static STRUCTURAL: &[&str] = &[
|
||||
"abbreviations",
|
||||
"about the author",
|
||||
"abstract",
|
||||
"acknowledgements",
|
||||
"acknowledgments",
|
||||
"afterword",
|
||||
"appendix",
|
||||
"author",
|
||||
"authors",
|
||||
"bibliography",
|
||||
"contents",
|
||||
"dedication",
|
||||
@@ -95,6 +121,7 @@ static STRUCTURAL: &[&str] = &[
|
||||
"illustrations",
|
||||
"index",
|
||||
"introduction",
|
||||
"keywords",
|
||||
"list of contributors",
|
||||
"list of illustrations",
|
||||
"maps",
|
||||
@@ -102,7 +129,9 @@ static STRUCTURAL: &[&str] = &[
|
||||
"postscript",
|
||||
"preface",
|
||||
"prologue",
|
||||
"subtitle",
|
||||
"table of contents",
|
||||
"title",
|
||||
"title page",
|
||||
"translator's note",
|
||||
];
|
||||
@@ -182,6 +211,22 @@ fn is_furniture(line: &str) -> bool {
|
||||
}
|
||||
// Sentences a copyright page sets, which are not notices and so not caught
|
||||
// by the notice pattern.
|
||||
if NUMBERED_HEADING.is_match(t) || MARKER.is_match(t) || CATALOGUE_ENTRY.is_match(t) {
|
||||
return true;
|
||||
}
|
||||
// A line opening with a number that is not a year. "5222 What Is This" and
|
||||
// "827 North Fourth Street" are a stray folio and an address; "1984
|
||||
// Revisited" is a book, and the difference is whether the number could be
|
||||
// the year of one.
|
||||
if let Some(c) = LEADING_NUMBER.captures(t) {
|
||||
let n: u32 = c[1].parse().unwrap_or(0);
|
||||
if !(1000..=2099).contains(&n) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if DEDICATION.is_match(t) {
|
||||
return true;
|
||||
}
|
||||
let lower = t.to_lowercase();
|
||||
for phrase in [
|
||||
"moral rights of the author",
|
||||
@@ -496,3 +541,54 @@ mod conversion_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod review_pile_tests {
|
||||
use super::*;
|
||||
|
||||
/// Everything here was proposed as a title by a real run over this
|
||||
/// library, and sat in the review queue waiting for a person to say no.
|
||||
#[test]
|
||||
fn what_the_review_pile_was_full_of() {
|
||||
for line in [
|
||||
"1. General Remarks",
|
||||
"2 James W. McAllister",
|
||||
"5222 What Is This",
|
||||
"827 North Fourth Street",
|
||||
"Page xxxvi",
|
||||
"Page xxxviii",
|
||||
"Chapter I",
|
||||
"title",
|
||||
"Sidebottom, Harry. Ancient warfare",
|
||||
"To Mackenzie,",
|
||||
] {
|
||||
assert!(
|
||||
is_structural(line) || is_furniture(line),
|
||||
"still read as a title: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// And none of it catches a book. The numbers are the delicate part: a
|
||||
/// year can open a title, and a folio cannot.
|
||||
#[test]
|
||||
fn a_book_is_still_a_book() {
|
||||
for line in [
|
||||
"1984 Revisited",
|
||||
"1914: The Year the World Ended",
|
||||
"Chapter and Verse",
|
||||
"Part of the Furniture",
|
||||
"To Kill a Mockingbird",
|
||||
"For Whom the Bell Tolls",
|
||||
"The Title of This Book",
|
||||
"Pages from a Cold Island",
|
||||
"Sections of a Broken Whole",
|
||||
"In Memory of Bread",
|
||||
] {
|
||||
assert!(
|
||||
!(is_structural(line) || is_furniture(line)),
|
||||
"rejected a real title: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user