chunkmd, mdscrub: the chapter a converted book is actually divided by

Two things between a converted EPUB and its chapter titles.

The heading level taken as the chapter map was the shallowest present. Pandoc
writes the book's title as the only `#` and every real chapter as a `##`, so
the shallowest level had one heading in it and the whole book came back as a
single chapter. The dividing level is now the shallowest one used more than
once — a level used once is a title.

And an empty bracketed span survived the heading cleaner, because the pattern
wanted at least one character between the brackets. Pandoc writes an anchor
as exactly that: `## []{#link2H_INTR} Introduction`.

Gibbon's first volume, converted from the Project Gutenberg EPUB: 2 distinct
chapter titles before, 61 after, and none of them carrying an element id.
This commit is contained in:
funkpower
2026-09-21 07:30:55 -04:00
parent 8b3c1a0236
commit cfa86f778f
6 changed files with 778 additions and 6 deletions
+31 -4
View File
@@ -4,6 +4,7 @@
//! and chapter boundaries as headings. Both are needed on every chunk: a
//! passage a reader cannot locate in the book is a passage they cannot check.
use std::collections::BTreeMap;
use std::sync::LazyLock;
use regex::Regex;
@@ -117,12 +118,22 @@ pub fn chapters(markdown: &str) -> Vec<Chapter> {
return Vec::new();
}
let shallowest = headings
// The shallowest level that divides the document, which is not always the
// shallowest level in it. A converted EPUB carries the book's title as a
// lone `#` above sixty-two `##` chapters, and taking the lone one makes
// the whole book a single chapter — so a level used once is a title and a
// level used twice is a chapter map.
let mut counts: BTreeMap<usize, usize> = BTreeMap::new();
for (level, _, _) in &headings {
*counts.entry(*level).or_default() += 1;
}
let dividing = counts
.iter()
.map(|(level, _, _)| *level)
.min()
.find(|(_, used)| **used > 1)
.map(|(level, _)| *level)
.or_else(|| counts.keys().next().copied())
.expect("a heading");
headings.retain(|(level, _, _)| *level == shallowest);
headings.retain(|(level, _, _)| *level == dividing);
let page_at = PageIndex::new(markdown);
let mut chapters = Vec::with_capacity(headings.len());
@@ -245,6 +256,22 @@ mod tests {
assert_eq!(pages[0].number, Some(7));
}
#[test]
fn a_lone_shallow_heading_is_a_title_rather_than_a_chapter_map() {
// What a converted EPUB looks like: the book's title as the only
// `#`, and every actual chapter a `##` under it. Taking the lone
// one makes the whole book a single chapter.
let markdown = "# The Whole Book\n\n## Chapter One\n\ntext\n\n## Chapter Two\n\ntext";
let found = chapters(markdown);
assert_eq!(
found
.iter()
.map(|one| one.title.as_str())
.collect::<Vec<_>>(),
["Chapter One", "Chapter Two"]
);
}
#[test]
fn chapters_come_from_the_shallowest_heading_level_present() {
let markdown = "## Alpha\n\ntext\n\n### Sub\n\ntext\n\n## Beta\n\ntext";
+4 -2
View File
@@ -18,8 +18,10 @@ fn pattern(source: &str) -> Regex {
/// `[text](url)` — a heading wrapped in a link to its own anchor.
static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"));
/// `[text]{.cls}` — pandoc's bracketed inline span.
static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
/// `[text]{.cls}` — pandoc's bracketed inline span. The text may be empty:
/// `[]{#chapter-3}` is how an anchor with nothing in it arrives, and a
/// heading that keeps one is a chapter title with an element id in it.
static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\{[^}]*\}"));
/// A trailing `{.attr}` block.
static ATTRIBUTE_TAIL: LazyLock<Regex> =