3 Commits
Author SHA1 Message Date
funkpower 90fb977a10 colophon: six more things the front matter of a book is not
Read out of the review queue a real run produced, where each of these was
sitting as a proposed title waiting for a person to say no:

  1. General Remarks            a numbered heading
  2   James W. McAllister       a numbered heading
  5222 What Is This             a folio that ran into the line
  827 North Fourth Street       an address
  Page xxxvi / Chapter I        a marker
  title                         a form label, printed literally
  Sidebottom, Harry. Ancient    a bibliography entry
  To Mackenzie,                 a dedication

The numbers are the delicate part. A leading number that could be a year
is left alone, because 1984 Revisited is a book and 5222 What Is This is
a folio, and nothing else distinguishes them. A dedication has to end in
a comma, so To Kill a Mockingbird survives.

The copyright notices in the same pile are already fixed: those
proposals were written before 0.2.2 and are stale rather than wrong.
2026-09-21 15:47:24 -04:00
funkpower 05b712d084 0.2.2 — a subtitle begins with a capital
title_case counted small words from the start of the whole string, so
every subtitle that opened with one was set in lower case: 'Aesthetic in
Kant: a Critique', 'Born Fighting: the Scots-Irish'. A colon starts a
title of its own and the first word of one is capitalised however small
it is.

Found by book-pipeline, which had been proposing a title page's capitals
verbatim into a catalogue and needed this to stop.
2026-09-21 11:30:26 -04:00
funkpower 60943420b4 0.2.1 — the holder goes between the symbol and the year
'(c) Alfredo Ferrarin 2004' was read as a title. The notice pattern
wanted the year to follow the symbol directly, and a copyright line puts
the holder's name in between, which is the ordinary way to set one.
2026-09-21 11:19:54 -04:00
5 changed files with 205 additions and 56 deletions
Generated
+26 -26
View File
@@ -101,7 +101,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "avplan"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -126,7 +126,7 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "bibclean"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -143,7 +143,7 @@ dependencies = [
[[package]]
name = "bibfetch"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -196,7 +196,7 @@ dependencies = [
[[package]]
name = "bookmeta"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -210,7 +210,7 @@ dependencies = [
[[package]]
name = "booksniff"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -250,7 +250,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "catena"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"bibclean",
"bibfetch",
@@ -277,7 +277,7 @@ dependencies = [
[[package]]
name = "catena-cli"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"clap",
"clap_mangen",
@@ -312,7 +312,7 @@ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
[[package]]
name = "chunkmd"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -334,7 +334,7 @@ dependencies = [
[[package]]
name = "citefind"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"chunkmd",
@@ -399,7 +399,7 @@ dependencies = [
[[package]]
name = "colophon"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"bibclean",
"catena-cli",
@@ -448,7 +448,7 @@ dependencies = [
[[package]]
name = "corpus-core"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"schemars",
"serde",
@@ -492,7 +492,7 @@ dependencies = [
[[package]]
name = "cuekit"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"corpus-core",
"regex",
@@ -504,7 +504,7 @@ dependencies = [
[[package]]
name = "cuenorm"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -514,7 +514,7 @@ dependencies = [
[[package]]
name = "cuepack"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -525,7 +525,7 @@ dependencies = [
[[package]]
name = "cueseg"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -612,7 +612,7 @@ dependencies = [
[[package]]
name = "emb"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -876,7 +876,7 @@ dependencies = [
[[package]]
name = "idnorm"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"bibclean",
"catena-cli",
@@ -942,7 +942,7 @@ dependencies = [
[[package]]
name = "langid"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1022,7 +1022,7 @@ dependencies = [
[[package]]
name = "mdscrub"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1034,7 +1034,7 @@ dependencies = [
[[package]]
name = "mdsplit"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1113,7 +1113,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "pdfgutter"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1126,7 +1126,7 @@ dependencies = [
[[package]]
name = "pdftriage"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1370,7 +1370,7 @@ checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
[[package]]
name = "rrf"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1517,7 +1517,7 @@ dependencies = [
[[package]]
name = "seriesof"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
@@ -1634,7 +1634,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "textkit"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"regex",
"tiktoken-rs",
@@ -1724,7 +1724,7 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
[[package]]
name = "tocfind"
version = "0.2.0"
version = "0.2.3"
dependencies = [
"catena-cli",
"clap",
+27 -27
View File
@@ -3,39 +3,39 @@ resolver = "3"
members = ["crates/*"]
[workspace.package]
version = "0.2.0"
version = "0.2.3"
edition = "2024"
rust-version = "1.85"
license = "MPL-2.0"
repository = "https://git.pumpin.online/funkpower/catena"
[workspace.dependencies]
catena-cli = { path = "crates/catena-cli", version = "0.2.0" }
corpus-core = { path = "crates/corpus-core", version = "0.2.0" }
cuekit = { path = "crates/cuekit", version = "0.2.0" }
textkit = { path = "crates/textkit", version = "0.2.0" }
chunkmd = { path = "crates/chunkmd", version = "0.2.0" }
mdscrub = { path = "crates/mdscrub", version = "0.2.0" }
mdsplit = { path = "crates/mdsplit", version = "0.2.0" }
pdftriage = { path = "crates/pdftriage", version = "0.2.0" }
cuenorm = { path = "crates/cuenorm", version = "0.2.0" }
cueseg = { path = "crates/cueseg", version = "0.2.0" }
rrf = { path = "crates/rrf", version = "0.2.0" }
cuepack = { path = "crates/cuepack", version = "0.2.0" }
bibclean = { path = "crates/bibclean", version = "0.2.0" }
avplan = { path = "crates/avplan", version = "0.2.0" }
pdfgutter = { path = "crates/pdfgutter", version = "0.2.0" }
catena = { path = "crates/bindings", version = "0.2.0" }
seriesof = { path = "crates/seriesof", version = "0.2.0" }
idnorm = { path = "crates/idnorm", version = "0.2.0" }
bibfetch = { path = "crates/bibfetch", version = "0.2.0" }
bookmeta = { path = "crates/bookmeta", version = "0.2.0" }
langid = { path = "crates/langid", version = "0.2.0" }
tocfind = { path = "crates/tocfind", version = "0.2.0" }
booksniff = { path = "crates/booksniff", version = "0.2.0" }
citefind = { path = "crates/citefind", version = "0.2.0" }
colophon = { path = "crates/colophon", version = "0.2.0" }
emb = { path = "crates/emb", version = "0.2.0" }
catena-cli = { path = "crates/catena-cli", version = "0.2.3" }
corpus-core = { path = "crates/corpus-core", version = "0.2.3" }
cuekit = { path = "crates/cuekit", version = "0.2.3" }
textkit = { path = "crates/textkit", version = "0.2.3" }
chunkmd = { path = "crates/chunkmd", version = "0.2.3" }
mdscrub = { path = "crates/mdscrub", version = "0.2.3" }
mdsplit = { path = "crates/mdsplit", version = "0.2.3" }
pdftriage = { path = "crates/pdftriage", version = "0.2.3" }
cuenorm = { path = "crates/cuenorm", version = "0.2.3" }
cueseg = { path = "crates/cueseg", version = "0.2.3" }
rrf = { path = "crates/rrf", version = "0.2.3" }
cuepack = { path = "crates/cuepack", version = "0.2.3" }
bibclean = { path = "crates/bibclean", version = "0.2.3" }
avplan = { path = "crates/avplan", version = "0.2.3" }
pdfgutter = { path = "crates/pdfgutter", version = "0.2.3" }
catena = { path = "crates/bindings", version = "0.2.3" }
seriesof = { path = "crates/seriesof", version = "0.2.3" }
idnorm = { path = "crates/idnorm", version = "0.2.3" }
bibfetch = { path = "crates/bibfetch", version = "0.2.3" }
bookmeta = { path = "crates/bookmeta", version = "0.2.3" }
langid = { path = "crates/langid", version = "0.2.3" }
tocfind = { path = "crates/tocfind", version = "0.2.3" }
booksniff = { path = "crates/booksniff", version = "0.2.3" }
citefind = { path = "crates/citefind", version = "0.2.3" }
colophon = { path = "crates/colophon", version = "0.2.3" }
emb = { path = "crates/emb", version = "0.2.3" }
clap = { version = "4", features = ["derive"] }
clap_mangen = "0.3"
+45 -1
View File
@@ -67,7 +67,19 @@ pub fn title_case(title: &str) -> String {
.to_lowercase()
.trim_end_matches(['.', ',', ':', ';', '!', '?', ')', ']', '"', '\''])
.to_string();
if SMALL_WORDS.binary_search(&core.as_str()).is_ok() && at > 0 && at < last {
// A small word stays small in the middle of a title, but not
// where a title actually starts. A colon starts a subtitle, and
// "Aesthetic in Kant: a Critique" is the giveaway that the rule
// was only ever counting from the beginning of the whole string.
let opens_a_subtitle = at > 0
&& words[at - 1]
.trim_end_matches(['"', '\'', ')', ']'])
.ends_with([':', ';', '?', '!', '.']);
if SMALL_WORDS.binary_search(&core.as_str()).is_ok()
&& at > 0
&& at < last
&& !opens_a_subtitle
{
return word.to_lowercase();
}
let lowered = word.to_lowercase();
@@ -218,3 +230,35 @@ mod tests {
);
}
}
#[cfg(test)]
mod subtitle_case_tests {
use super::title_case;
#[test]
fn a_subtitle_starts_with_a_capital() {
// The rule counted small words from the start of the string, so every
// subtitle beginning "A", "The" or "Of" was set lower case.
assert_eq!(
title_case("AESTHETIC IN KANT: A CRITIQUE"),
"Aesthetic in Kant: A Critique"
);
assert_eq!(
title_case("BORN FIGHTING: THE SCOTS-IRISH"),
"Born Fighting: The Scots-Irish"
);
assert_eq!(
title_case("A HISTORY: AN INTRODUCTION TO IT"),
"A History: An Introduction to It"
);
}
#[test]
fn a_small_word_in_the_middle_is_still_small() {
assert_eq!(
title_case("THE RISE AND FALL OF THE THIRD REICH"),
"The Rise and Fall of the Third Reich"
);
assert_eq!(title_case("WAR OF THE WORLDS"), "War of the Worlds");
}
}
+6 -1
View File
@@ -260,9 +260,14 @@ mod tests {
#[test]
fn an_underscored_title_is_cleaned_before_it_is_cased() {
let cleaned = clean(&book("THE FALL_ AND THE RISE_txt", "Arendt, Hannah"));
// "And" is capitalised because the colon starts a subtitle, and the
// first word of one is capitalised however small it is. This asserted
// "and" until the rule was found to be counting small words from the
// start of the whole string, which set every subtitle opening "A",
// "The" or "An" in lower case.
assert_eq!(
cleaned.book.title.as_deref(),
Some("The Fall: and the Rise")
Some("The Fall: And the Rise")
);
assert_eq!(cleaned.changes.len(), 2);
}
+101 -1
View File
@@ -35,8 +35,33 @@ static LIFESPAN: LazyLock<Regex> = LazyLock::new(|| {
static BARE_DATE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)^\d{1,4}\s*(bc|ad|bce|ce)?\.?$").expect("a valid pattern"));
/// `Copyright © 2013 by`, `\u{a9} 2013 Richard Swinburne`, `Copyright 1971 by`.
/// `1. General Remarks`, `2 James W. McAllister` — a numbered heading.
static NUMBERED_HEADING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^\s*\d+\s*[.)]\s+\S|^\s*\d+\s{2,}\S").expect("a valid pattern"));
/// `5222 What Is This`, `827 North Fourth Street` — a number that is not a
/// year, leading a line.
static LEADING_NUMBER: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^\s*(\d{1,5})\s+\S").expect("a valid pattern"));
/// `Page xxxvi`, `Chapter I`, `Fig. 3` — a marker, not a name.
static MARKER: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(
r"(?i)^\s*(page|pages|p\.|pp\.|fig\.|figure|plate|chapter|ch\.|section|part)\s+[\divxlcm]",
)
.expect("a valid pattern")
});
/// `Sidebottom, Harry. Ancient warfare` — a bibliography entry, which front
/// matter prints in its cataloguing block and in its further reading.
static CATALOGUE_ENTRY: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^[A-Z][A-Za-z'\-]+,\s+[A-Z][A-Za-z'.\-]*\.\s+\S").expect("a valid pattern")
});
/// `To Mackenzie,` — a dedication, set alone on its own page.
static DEDICATION: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"(?i)^\s*(to|for|in memory of)\s+\S.*,\s*$").expect("a valid pattern")
});
static COPYRIGHT_NOTICE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"(?i)(\u{a9}|\(c\)|copyright)\s*(\u{a9}|\(c\))?\s*(1[0-9]|20)\d{2}")
// The year need not follow the symbol directly: "(c) Alfredo Ferrarin
// 2004" puts the holder between them, and was read as a title.
Regex::new(r"(?i)(\u{a9}|\(c\)|copyright)[^\n]{0,60}?(1[0-9]|20)\d{2}")
.expect("a valid pattern")
});
/// `Edited by`, with whoever it was set on the line beneath.
@@ -77,10 +102,13 @@ copenhagen|stockholm|moscow|beijing|tokyo|delhi|nairobi)\b[\s,·•|and]*",
static STRUCTURAL: &[&str] = &[
"abbreviations",
"about the author",
"abstract",
"acknowledgements",
"acknowledgments",
"afterword",
"appendix",
"author",
"authors",
"bibliography",
"contents",
"dedication",
@@ -93,6 +121,7 @@ static STRUCTURAL: &[&str] = &[
"illustrations",
"index",
"introduction",
"keywords",
"list of contributors",
"list of illustrations",
"maps",
@@ -100,7 +129,9 @@ static STRUCTURAL: &[&str] = &[
"postscript",
"preface",
"prologue",
"subtitle",
"table of contents",
"title",
"title page",
"translator's note",
];
@@ -180,6 +211,22 @@ fn is_furniture(line: &str) -> bool {
}
// Sentences a copyright page sets, which are not notices and so not caught
// by the notice pattern.
if NUMBERED_HEADING.is_match(t) || MARKER.is_match(t) || CATALOGUE_ENTRY.is_match(t) {
return true;
}
// A line opening with a number that is not a year. "5222 What Is This" and
// "827 North Fourth Street" are a stray folio and an address; "1984
// Revisited" is a book, and the difference is whether the number could be
// the year of one.
if let Some(c) = LEADING_NUMBER.captures(t) {
let n: u32 = c[1].parse().unwrap_or(0);
if !(1000..=2099).contains(&n) {
return true;
}
}
if DEDICATION.is_match(t) {
return true;
}
let lower = t.to_lowercase();
for phrase in [
"moral rights of the author",
@@ -413,6 +460,8 @@ mod furniture_tests {
"12 INDEX",
"Translated from the French by Willard R. Trask",
"Copyright © 2013 by Barry Davies, BEM",
"© Alfredo Ferrarin 2004",
"\u{a9} Cambridge University Press 2011",
"[PART ONE:",
"Piper Verlag GmbH, Munich<span style=\"font-size\">",
"Vol. 11. HENRY JAMES, JR.",
@@ -492,3 +541,54 @@ mod conversion_tests {
}
}
}
#[cfg(test)]
mod review_pile_tests {
use super::*;
/// Everything here was proposed as a title by a real run over this
/// library, and sat in the review queue waiting for a person to say no.
#[test]
fn what_the_review_pile_was_full_of() {
for line in [
"1. General Remarks",
"2 James W. McAllister",
"5222 What Is This",
"827 North Fourth Street",
"Page xxxvi",
"Page xxxviii",
"Chapter I",
"title",
"Sidebottom, Harry. Ancient warfare",
"To Mackenzie,",
] {
assert!(
is_structural(line) || is_furniture(line),
"still read as a title: {line:?}"
);
}
}
/// And none of it catches a book. The numbers are the delicate part: a
/// year can open a title, and a folio cannot.
#[test]
fn a_book_is_still_a_book() {
for line in [
"1984 Revisited",
"1914: The Year the World Ended",
"Chapter and Verse",
"Part of the Furniture",
"To Kill a Mockingbird",
"For Whom the Bell Tolls",
"The Title of This Book",
"Pages from a Cold Island",
"Sections of a Broken Whole",
"In Memory of Bread",
] {
assert!(
!(is_structural(line) || is_furniture(line)),
"rejected a real title: {line:?}"
);
}
}
}