mdscrub: residual html, control characters, headings, page markers

crates/mdscrub/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "mdscrub"
+description = "Cleans converted markdown: residual HTML, control characters, headings, page markers."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+regex.workspace = true
+textkit.workspace = true
+
+[lints]
+workspace = true

crates/mdscrub/src/heading.rs

@@ -0,0 +1,149 @@
+//! Making a heading line readable again.
+//!
+//! A converted heading arrives carrying whatever the source used to style it:
+//! link wrappers around the whole title, attribute blocks, anchor spans, bold
+//! markers around a title that was set in capitals. The heading text is what
+//! every chunk in the chapter will be labelled with, so it is worth cleaning
+//! properly rather than passing through.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+fn pattern(source: &str) -> Regex {
+    Regex::new(source).expect("a valid pattern")
+}
+
+/// `[text](url)` — a heading wrapped in a link to its own anchor.
+static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"));
+
+/// `[text]{.cls}` — pandoc's bracketed inline span.
+static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
+
+/// A trailing `{.attr}` block.
+static ATTRIBUTE_TAIL: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s*\{[^}]*\}\s*$"));
+
+/// Any inline tag. Broader than the one used on body text, because a heading
+/// has no code fences to protect.
+static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"<[^>]+>"));
+
+static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s+"));
+
+/// A heading line, and the text after its hashes.
+static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(r"^(#{1,6})\s+(.*)$"));
+
+/// A page anchor: `<span id="…page42">`, `<a id="page-42">`, and the rest of
+/// the shapes an EPUB uses to say where a printed page began.
+pub static PAGE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(
+        r#"(?i)<(?:span|a|div)\b[^>]*\bid\s*=\s*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#,
+    )
+});
+
+/// Clean one heading's text.
+pub fn clean(raw: &str) -> String {
+    let text = LINK.replace_all(raw, "$1");
+    let text = BRACKETED.replace_all(&text, "$1");
+    let text = ATTRIBUTE_TAIL.replace_all(&text, "");
+    let text = TAG.replace_all(&text, "");
+    let text = SPACE_RUN.replace_all(&text, " ");
+    let text = text.trim_matches(textkit::is_space);
+    // Emphasis markers around a title that was set in capitals say nothing
+    // once the title is a label on a search result.
+    text.replace("**", "").replace("__", "")
+}
+
+/// Clean every heading in a document, moving any page anchor found in a
+/// heading to a marker on its own line just after it.
+///
+/// After the heading rather than before, so that the heading keeps its offset
+/// relative to the body text that follows, and the chapter title stays whole.
+pub fn clean_all(markdown: &str) -> String {
+    let mut out: Vec<String> = Vec::new();
+    for line in markdown.split('\n') {
+        let Some(found) = HEADING.captures(line) else {
+            out.push(line.to_string());
+            continue;
+        };
+        let hashes = &found[1];
+        let raw = &found[2];
+        let pages: Vec<u32> = PAGE_ANCHOR
+            .captures_iter(raw)
+            .map(|anchor| textkit::decimal(&anchor[1]))
+            .collect();
+        let cleaned = clean(&PAGE_ANCHOR.replace_all(raw, ""));
+        if !cleaned.is_empty() {
+            out.push(format!("{hashes} {cleaned}"));
+        }
+        for page in pages {
+            out.push(String::new());
+            out.push(format!("<!-- Page {page} -->"));
+        }
+    }
+    out.join("\n")
+}
+
+/// Turn the page anchors left in the body into page markers.
+pub fn anchors_to_markers(markdown: &str) -> String {
+    PAGE_ANCHOR
+        .replace_all(markdown, |found: &regex::Captures| {
+            format!("\n\n<!-- Page {} -->\n\n", textkit::decimal(&found[1]))
+        })
+        .into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_link_wrapper_comes_off() {
+        assert_eq!(clean("[I. The Setting](#ch1)"), "I. The Setting");
+    }
+
+    #[test]
+    fn a_bracketed_span_comes_off() {
+        assert_eq!(clean("[The Quarrel]{.chapter-title}"), "The Quarrel");
+    }
+
+    #[test]
+    fn a_trailing_attribute_block_comes_off() {
+        assert_eq!(clean("The Break {#ch3 .unnumbered}"), "The Break");
+    }
+
+    #[test]
+    fn tags_come_off_and_whitespace_collapses() {
+        assert_eq!(clean("<span id=\"x\">The   Break</span>  "), "The Break");
+    }
+
+    #[test]
+    fn emphasis_markers_come_off() {
+        assert_eq!(clean("**THE SETTLEMENT**"), "THE SETTLEMENT");
+    }
+
+    #[test]
+    fn a_page_anchor_in_a_heading_becomes_a_marker_after_it() {
+        let cleaned = clean_all("# <span id=\"page42\"></span>The Quarrel");
+        assert_eq!(cleaned, "# The Quarrel\n\n<!-- Page 42 -->");
+    }
+
+    #[test]
+    fn a_heading_that_was_only_an_anchor_leaves_only_the_marker() {
+        let cleaned = clean_all("## <a id=\"page-7\"></a>");
+        assert_eq!(cleaned, "\n<!-- Page 7 -->");
+    }
+
+    #[test]
+    fn body_anchors_become_markers() {
+        let cleaned = anchors_to_markers("text <span id=\"pg_page19\"></span> more");
+        assert!(cleaned.contains("<!-- Page 19 -->"));
+    }
+
+    #[test]
+    fn a_line_that_is_not_a_heading_is_untouched() {
+        assert_eq!(
+            clean_all("#no space is not a heading"),
+            "#no space is not a heading"
+        );
+    }
+}

crates/mdscrub/src/html.rs

@@ -0,0 +1,122 @@
+//! Removing the HTML that survives conversion to markdown.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// An opening or closing inline tag.
+///
+/// Converted markdown is full of them: pandoc's commonmark output keeps an
+/// EPUB's styling spans and anchors, and a PDF converter emits anchor spans
+/// and superscript note references. None of it means anything once the page
+/// anchors have been pulled out, and all of it pollutes an embedding, a
+/// full-text index, and every result a reader sees.
+///
+/// Comments are deliberately not matched: `<!-- Page 41 -->` has to survive.
+static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?[A-Za-z][^>]*>").expect("valid"));
+
+/// A tag that stands for a line break rather than for styling.
+static BREAK: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<br\s*/?>").expect("valid"));
+
+/// The start of a fenced code block.
+static FENCE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\s*(```|~~~)").expect("valid"));
+
+/// Control characters that must never reach a text column.
+///
+/// A database will reject `0x00` outright. The other C0 controls — tab,
+/// newline and carriage return excepted — are storable and are extraction
+/// garbage: bad scans and malformed PDFs emit them routinely.
+static CONTROL: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[\x00-\x08\x0b\x0c\x0e-\x1f]").expect("valid"));
+
+/// Three or more newlines, left behind wherever a tag-only line was removed.
+static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").expect("valid"));
+
+/// Strip inline HTML, keeping the text inside it and the HTML comments.
+///
+/// Fenced code blocks are left alone, so a book that quotes HTML keeps its
+/// examples. `<br>` becomes a line break rather than nothing, because
+/// deleting it welds the words on either side into one — which is a spelling
+/// mistake in the index that no reader can search for.
+pub fn strip_tags(markdown: &str) -> String {
+    let mut out = Vec::new();
+    let mut in_fence = false;
+    for line in markdown.split('\n') {
+        if FENCE.is_match(line) {
+            in_fence = !in_fence;
+            out.push(line.to_string());
+            continue;
+        }
+        if in_fence {
+            out.push(line.to_string());
+        } else {
+            let line = BREAK.replace_all(line, "\n");
+            out.push(TAG.replace_all(&line, "").into_owned());
+        }
+    }
+    out.join("\n")
+}
+
+/// Remove the control characters a text column cannot hold.
+pub fn scrub_control(text: &str) -> String {
+    CONTROL.replace_all(text, "").into_owned()
+}
+
+/// Collapse runs of blank lines to a single blank line.
+pub fn collapse_blank_runs(text: &str) -> String {
+    BLANK_RUN.replace_all(text, "\n\n").into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn styling_tags_go_and_their_text_stays() {
+        assert_eq!(
+            strip_tags("<span class=\"c1\">The</span> second <em>estate</em>"),
+            "The second estate"
+        );
+    }
+
+    #[test]
+    fn html_comments_survive() {
+        assert_eq!(strip_tags("<!-- Page 41 -->"), "<!-- Page 41 -->");
+    }
+
+    #[test]
+    fn a_break_tag_does_not_weld_the_words_either_side_of_it() {
+        assert_eq!(strip_tags("first<br>second"), "first\nsecond");
+        assert_eq!(strip_tags("first<br />second"), "first\nsecond");
+        assert_eq!(strip_tags("first<BR/>second"), "first\nsecond");
+    }
+
+    #[test]
+    fn fenced_code_keeps_its_tags() {
+        let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n```\n\nafter <b>y</b>";
+        let cleaned = strip_tags(markdown);
+        assert!(cleaned.contains("<span>kept</span>"));
+        assert!(cleaned.contains("before x"));
+        assert!(cleaned.contains("after y"));
+    }
+
+    #[test]
+    fn a_tag_split_across_two_lines_is_left_alone() {
+        // Matching per line is what keeps a stray `<` in prose from eating
+        // the rest of the document.
+        let markdown = "a < b and\nc > d";
+        assert_eq!(strip_tags(markdown), markdown);
+    }
+
+    #[test]
+    fn control_characters_go_but_tabs_and_newlines_stay() {
+        assert_eq!(scrub_control("a\u{0}b\u{1}c"), "abc");
+        assert_eq!(scrub_control("a\tb\nc\r\n"), "a\tb\nc\r\n");
+    }
+
+    #[test]
+    fn blank_line_runs_collapse_to_one() {
+        assert_eq!(collapse_blank_runs("a\n\n\n\n\nb"), "a\n\nb");
+        assert_eq!(collapse_blank_runs("a\n\nb"), "a\n\nb");
+    }
+}

crates/mdscrub/src/lib.rs

@@ -0,0 +1,69 @@
+//! Cleans converted markdown.
+//!
+//! Conversion from a PDF or an EPUB leaves markdown that is readable but not
+//! clean: styling tags that survived, control characters a text column will
+//! reject, headings wrapped in links to their own anchors, and page
+//! boundaries spelled three different ways. None of it means anything
+//! downstream, and all of it ends up in an index.
+//!
+//! The order of the passes matters. Page boundaries are normalised first,
+//! because one of the things that marks a page boundary — a form feed — is
+//! also a control character that the later pass deletes.
+
+pub mod heading;
+pub mod html;
+pub mod pages;
+
+/// Clean a converted markdown document.
+///
+/// Safe to run twice: the second run changes nothing.
+pub fn scrub(markdown: &str) -> String {
+    let text = pages::normalize(markdown);
+    let text = heading::clean_all(&text);
+    let text = heading::anchors_to_markers(&text);
+    let text = html::strip_tags(&text);
+    let text = html::scrub_control(&text);
+    let text = html::collapse_blank_runs(&text);
+    format!("{}\n", text.trim_matches(textkit::is_space))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_converted_heading_comes_out_readable() {
+        let markdown = "# [<span id=\"page12\"></span>I. The Setting](#ch1) {.unnumbered}";
+        assert_eq!(scrub(markdown), "# I. The Setting\n\n<!-- Page 12 -->\n");
+    }
+
+    #[test]
+    fn running_it_twice_changes_nothing() {
+        let markdown = "<!-- Page 3 -->\n\n# The <em>Quarrel</em>\n\n\n\nA <b>mill</b>\u{0} \
+                        stood here.<br>It still does.\n\n<span id=\"page4\"></span>\n\nOn.";
+        let once = scrub(markdown);
+        assert_eq!(scrub(&once), once);
+    }
+
+    #[test]
+    fn a_page_marker_survives_every_pass() {
+        assert!(scrub("<!-- Page 41 -->\n\ntext").contains("<!-- Page 41 -->"));
+    }
+
+    #[test]
+    fn the_result_always_ends_in_exactly_one_newline() {
+        for markdown in ["", "text", "text\n\n\n", "  \n"] {
+            let cleaned = scrub(markdown);
+            assert!(cleaned.ends_with('\n'), "{markdown:?}");
+            assert!(!cleaned.ends_with("\n\n"), "{markdown:?}");
+        }
+    }
+
+    #[test]
+    fn form_feeds_become_pages_rather_than_disappearing() {
+        // The control-character pass would delete a form feed, so page
+        // normalisation has to happen first or the boundary is lost.
+        let cleaned = scrub("one\u{c}two");
+        assert!(cleaned.contains("<!-- Page 2 -->"), "{cleaned:?}");
+    }
+}

crates/mdscrub/src/main.rs

@@ -0,0 +1,62 @@
+//! Cleans converted markdown.
+
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::exit;
+
+/// Clean converted markdown: drop residual HTML and control characters,
+/// make headings readable, and spell every page boundary the same way.
+///
+/// Reads markdown on stdin, or from a path, and writes it to stdout. Safe to
+/// run twice.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    long_about = None,
+    after_help = "\
+Example:
+
+    pandoc -t commonmark book.epub | mdscrub | chunkmd > chunks.ndjson"
+)]
+struct Args {
+    /// Markdown file to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+
+    let markdown = match read(args.path.as_deref()) {
+        Ok(markdown) => markdown,
+        Err(e) => {
+            eprintln!("mdscrub: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    let cleaned = mdscrub::scrub(&markdown);
+    match std::io::stdout().write_all(cleaned.as_bytes()) {
+        Ok(()) => ExitCode::from(exit::OK),
+        // A consumer that stopped reading is not a failure.
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("mdscrub: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+fn read(path: Option<&Path>) -> std::io::Result<String> {
+    match path {
+        Some(path) => std::fs::read_to_string(path),
+        None => {
+            let mut markdown = String::new();
+            std::io::stdin().read_to_string(&mut markdown)?;
+            Ok(markdown)
+        }
+    }
+}

crates/mdscrub/src/pages.rs

@@ -0,0 +1,112 @@
+//! Turning whatever a converter used for page boundaries into one marker.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// The separator a paginating PDF converter writes between pages, like
+/// `{0}------------------------------`. Its numbering is zero-based.
+static SEPARATOR: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[^\S\n]*\n").expect("a valid pattern"));
+
+/// A form feed, which is what a plain text extractor leaves between pages.
+const FORM_FEED: char = '\u{c}';
+
+/// Rewrite page boundaries as `<!-- Page N -->` markers.
+///
+/// Separators first, then form feeds, and text with neither is returned
+/// unchanged — a cleanup pass has to be safe to run twice, and inventing a
+/// page marker for a document that has none would not be.
+pub fn normalize(text: &str) -> String {
+    if SEPARATOR.is_match(text) {
+        return from_separators(text);
+    }
+    if text.contains(FORM_FEED) {
+        return from_form_feeds(text);
+    }
+    text.to_string()
+}
+
+fn from_separators(text: &str) -> String {
+    let mut pages: Vec<String> = Vec::new();
+    let mut at = 0;
+    let mut number = 1;
+
+    for found in SEPARATOR.captures_iter(text) {
+        let whole = found.get(0).expect("group 0");
+        let before = trim(&text[at..whole.start()]);
+        if !before.is_empty() {
+            pages.push(format!("<!-- Page {number} -->\n\n{before}"));
+        }
+        // The separator carries the number of the page it *ends*, counted
+        // from zero, so the page that follows it is that number plus one.
+        number = textkit::decimal(&found[1]).saturating_add(1);
+        at = whole.end();
+    }
+
+    let last = trim(&text[at..]);
+    if !last.is_empty() {
+        pages.push(format!("<!-- Page {number} -->\n\n{last}"));
+    }
+    pages.join("\n\n")
+}
+
+fn from_form_feeds(text: &str) -> String {
+    text.split(FORM_FEED)
+        .map(trim)
+        .enumerate()
+        .filter(|(_, page)| !page.is_empty())
+        .map(|(at, page)| format!("<!-- Page {} -->\n\n{page}", at + 1))
+        .collect::<Vec<_>>()
+        .join("\n\n")
+}
+
+fn trim(text: &str) -> &str {
+    text.trim_matches(textkit::is_space)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn separators_become_markers_numbered_from_one() {
+        let text = "first page\n{0}------------------------------\nsecond page\n\
+                    {1}------------------------------\nthird page";
+        assert_eq!(
+            normalize(text),
+            "<!-- Page 1 -->\n\nfirst page\n\n\
+             <!-- Page 1 -->\n\nsecond page\n\n\
+             <!-- Page 2 -->\n\nthird page"
+        );
+    }
+
+    #[test]
+    fn form_feeds_become_markers() {
+        let text = "one\u{c}two\u{c}three";
+        assert_eq!(
+            normalize(text),
+            "<!-- Page 1 -->\n\none\n\n<!-- Page 2 -->\n\ntwo\n\n<!-- Page 3 -->\n\nthree"
+        );
+    }
+
+    #[test]
+    fn an_empty_page_is_dropped_and_the_numbering_carries_on() {
+        assert_eq!(
+            normalize("one\u{c}\u{c}three"),
+            "<!-- Page 1 -->\n\none\n\n<!-- Page 3 -->\n\nthree"
+        );
+    }
+
+    #[test]
+    fn text_with_no_page_boundaries_is_left_exactly_as_it_was() {
+        let text = "# A Heading\n\nSome prose, with no pagination at all.\n";
+        assert_eq!(normalize(text), text);
+    }
+
+    #[test]
+    fn a_short_dash_run_is_not_a_separator() {
+        let text = "before\n{0}-----\nafter";
+        assert_eq!(normalize(text), text);
+    }
+}