mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
mdscrub: residual html, control characters, headings, page markers
This commit is contained in:
funkpower
@@ -0,0 +1,149 @@
|
||||
//! Making a heading line readable again.
|
||||
//!
|
||||
//! A converted heading arrives carrying whatever the source used to style it:
|
||||
//! link wrappers around the whole title, attribute blocks, anchor spans, bold
|
||||
//! markers around a title that was set in capitals. The heading text is what
|
||||
//! every chunk in the chapter will be labelled with, so it is worth cleaning
|
||||
//! properly rather than passing through.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
fn pattern(source: &str) -> Regex {
|
||||
Regex::new(source).expect("a valid pattern")
|
||||
}
|
||||
|
||||
/// `[text](url)` — a heading wrapped in a link to its own anchor.
|
||||
static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"));
|
||||
|
||||
/// `[text]{.cls}` — pandoc's bracketed inline span.
|
||||
static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
|
||||
|
||||
/// A trailing `{.attr}` block.
|
||||
static ATTRIBUTE_TAIL: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s*\{[^}]*\}\s*$"));
|
||||
|
||||
/// Any inline tag. Broader than the one used on body text, because a heading
|
||||
/// has no code fences to protect.
|
||||
static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"<[^>]+>"));
|
||||
|
||||
static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s+"));
|
||||
|
||||
/// A heading line, and the text after its hashes.
|
||||
static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(r"^(#{1,6})\s+(.*)$"));
|
||||
|
||||
/// A page anchor: `<span id="…page42">`, `<a id="page-42">`, and the rest of
|
||||
/// the shapes an EPUB uses to say where a printed page began.
|
||||
pub static PAGE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
|
||||
pattern(
|
||||
r#"(?i)<(?:span|a|div)\b[^>]*\bid\s*=\s*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#,
|
||||
)
|
||||
});
|
||||
|
||||
/// Clean one heading's text.
|
||||
pub fn clean(raw: &str) -> String {
|
||||
let text = LINK.replace_all(raw, "$1");
|
||||
let text = BRACKETED.replace_all(&text, "$1");
|
||||
let text = ATTRIBUTE_TAIL.replace_all(&text, "");
|
||||
let text = TAG.replace_all(&text, "");
|
||||
let text = SPACE_RUN.replace_all(&text, " ");
|
||||
let text = text.trim_matches(textkit::is_space);
|
||||
// Emphasis markers around a title that was set in capitals say nothing
|
||||
// once the title is a label on a search result.
|
||||
text.replace("**", "").replace("__", "")
|
||||
}
|
||||
|
||||
/// Clean every heading in a document, moving any page anchor found in a
|
||||
/// heading to a marker on its own line just after it.
|
||||
///
|
||||
/// After the heading rather than before, so that the heading keeps its offset
|
||||
/// relative to the body text that follows, and the chapter title stays whole.
|
||||
pub fn clean_all(markdown: &str) -> String {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for line in markdown.split('\n') {
|
||||
let Some(found) = HEADING.captures(line) else {
|
||||
out.push(line.to_string());
|
||||
continue;
|
||||
};
|
||||
let hashes = &found[1];
|
||||
let raw = &found[2];
|
||||
let pages: Vec<u32> = PAGE_ANCHOR
|
||||
.captures_iter(raw)
|
||||
.map(|anchor| textkit::decimal(&anchor[1]))
|
||||
.collect();
|
||||
let cleaned = clean(&PAGE_ANCHOR.replace_all(raw, ""));
|
||||
if !cleaned.is_empty() {
|
||||
out.push(format!("{hashes} {cleaned}"));
|
||||
}
|
||||
for page in pages {
|
||||
out.push(String::new());
|
||||
out.push(format!("<!-- Page {page} -->"));
|
||||
}
|
||||
}
|
||||
out.join("\n")
|
||||
}
|
||||
|
||||
/// Turn the page anchors left in the body into page markers.
|
||||
pub fn anchors_to_markers(markdown: &str) -> String {
|
||||
PAGE_ANCHOR
|
||||
.replace_all(markdown, |found: ®ex::Captures| {
|
||||
format!("\n\n<!-- Page {} -->\n\n", textkit::decimal(&found[1]))
|
||||
})
|
||||
.into_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_link_wrapper_comes_off() {
|
||||
assert_eq!(clean("[I. The Setting](#ch1)"), "I. The Setting");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bracketed_span_comes_off() {
|
||||
assert_eq!(clean("[The Quarrel]{.chapter-title}"), "The Quarrel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_trailing_attribute_block_comes_off() {
|
||||
assert_eq!(clean("The Break {#ch3 .unnumbered}"), "The Break");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_come_off_and_whitespace_collapses() {
|
||||
assert_eq!(clean("<span id=\"x\">The Break</span> "), "The Break");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emphasis_markers_come_off() {
|
||||
assert_eq!(clean("**THE SETTLEMENT**"), "THE SETTLEMENT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_anchor_in_a_heading_becomes_a_marker_after_it() {
|
||||
let cleaned = clean_all("# <span id=\"page42\"></span>The Quarrel");
|
||||
assert_eq!(cleaned, "# The Quarrel\n\n<!-- Page 42 -->");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_heading_that_was_only_an_anchor_leaves_only_the_marker() {
|
||||
let cleaned = clean_all("## <a id=\"page-7\"></a>");
|
||||
assert_eq!(cleaned, "\n<!-- Page 7 -->");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_anchors_become_markers() {
|
||||
let cleaned = anchors_to_markers("text <span id=\"pg_page19\"></span> more");
|
||||
assert!(cleaned.contains("<!-- Page 19 -->"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_line_that_is_not_a_heading_is_untouched() {
|
||||
assert_eq!(
|
||||
clean_all("#no space is not a heading"),
|
||||
"#no space is not a heading"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
//! Removing the HTML that survives conversion to markdown.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
/// An opening or closing inline tag.
|
||||
///
|
||||
/// Converted markdown is full of them: pandoc's commonmark output keeps an
|
||||
/// EPUB's styling spans and anchors, and a PDF converter emits anchor spans
|
||||
/// and superscript note references. None of it means anything once the page
|
||||
/// anchors have been pulled out, and all of it pollutes an embedding, a
|
||||
/// full-text index, and every result a reader sees.
|
||||
///
|
||||
/// Comments are deliberately not matched: `<!-- Page 41 -->` has to survive.
|
||||
static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?[A-Za-z][^>]*>").expect("valid"));
|
||||
|
||||
/// A tag that stands for a line break rather than for styling.
|
||||
static BREAK: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<br\s*/?>").expect("valid"));
|
||||
|
||||
/// The start of a fenced code block.
|
||||
static FENCE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\s*(```|~~~)").expect("valid"));
|
||||
|
||||
/// Control characters that must never reach a text column.
|
||||
///
|
||||
/// A database will reject `0x00` outright. The other C0 controls — tab,
|
||||
/// newline and carriage return excepted — are storable and are extraction
|
||||
/// garbage: bad scans and malformed PDFs emit them routinely.
|
||||
static CONTROL: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[\x00-\x08\x0b\x0c\x0e-\x1f]").expect("valid"));
|
||||
|
||||
/// Three or more newlines, left behind wherever a tag-only line was removed.
|
||||
static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").expect("valid"));
|
||||
|
||||
/// Strip inline HTML, keeping the text inside it and the HTML comments.
|
||||
///
|
||||
/// Fenced code blocks are left alone, so a book that quotes HTML keeps its
|
||||
/// examples. `<br>` becomes a line break rather than nothing, because
|
||||
/// deleting it welds the words on either side into one — which is a spelling
|
||||
/// mistake in the index that no reader can search for.
|
||||
pub fn strip_tags(markdown: &str) -> String {
|
||||
let mut out = Vec::new();
|
||||
let mut in_fence = false;
|
||||
for line in markdown.split('\n') {
|
||||
if FENCE.is_match(line) {
|
||||
in_fence = !in_fence;
|
||||
out.push(line.to_string());
|
||||
continue;
|
||||
}
|
||||
if in_fence {
|
||||
out.push(line.to_string());
|
||||
} else {
|
||||
let line = BREAK.replace_all(line, "\n");
|
||||
out.push(TAG.replace_all(&line, "").into_owned());
|
||||
}
|
||||
}
|
||||
out.join("\n")
|
||||
}
|
||||
|
||||
/// Remove the control characters a text column cannot hold.
|
||||
pub fn scrub_control(text: &str) -> String {
|
||||
CONTROL.replace_all(text, "").into_owned()
|
||||
}
|
||||
|
||||
/// Collapse runs of blank lines to a single blank line.
|
||||
pub fn collapse_blank_runs(text: &str) -> String {
|
||||
BLANK_RUN.replace_all(text, "\n\n").into_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn styling_tags_go_and_their_text_stays() {
|
||||
assert_eq!(
|
||||
strip_tags("<span class=\"c1\">The</span> second <em>estate</em>"),
|
||||
"The second estate"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_comments_survive() {
|
||||
assert_eq!(strip_tags("<!-- Page 41 -->"), "<!-- Page 41 -->");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_break_tag_does_not_weld_the_words_either_side_of_it() {
|
||||
assert_eq!(strip_tags("first<br>second"), "first\nsecond");
|
||||
assert_eq!(strip_tags("first<br />second"), "first\nsecond");
|
||||
assert_eq!(strip_tags("first<BR/>second"), "first\nsecond");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fenced_code_keeps_its_tags() {
|
||||
let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n```\n\nafter <b>y</b>";
|
||||
let cleaned = strip_tags(markdown);
|
||||
assert!(cleaned.contains("<span>kept</span>"));
|
||||
assert!(cleaned.contains("before x"));
|
||||
assert!(cleaned.contains("after y"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tag_split_across_two_lines_is_left_alone() {
|
||||
// Matching per line is what keeps a stray `<` in prose from eating
|
||||
// the rest of the document.
|
||||
let markdown = "a < b and\nc > d";
|
||||
assert_eq!(strip_tags(markdown), markdown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_characters_go_but_tabs_and_newlines_stay() {
|
||||
assert_eq!(scrub_control("a\u{0}b\u{1}c"), "abc");
|
||||
assert_eq!(scrub_control("a\tb\nc\r\n"), "a\tb\nc\r\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_line_runs_collapse_to_one() {
|
||||
assert_eq!(collapse_blank_runs("a\n\n\n\n\nb"), "a\n\nb");
|
||||
assert_eq!(collapse_blank_runs("a\n\nb"), "a\n\nb");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//! Cleans converted markdown.
|
||||
//!
|
||||
//! Conversion from a PDF or an EPUB leaves markdown that is readable but not
|
||||
//! clean: styling tags that survived, control characters a text column will
|
||||
//! reject, headings wrapped in links to their own anchors, and page
|
||||
//! boundaries spelled three different ways. None of it means anything
|
||||
//! downstream, and all of it ends up in an index.
|
||||
//!
|
||||
//! The order of the passes matters. Page boundaries are normalised first,
|
||||
//! because one of the things that marks a page boundary — a form feed — is
|
||||
//! also a control character that the later pass deletes.
|
||||
|
||||
pub mod heading;
|
||||
pub mod html;
|
||||
pub mod pages;
|
||||
|
||||
/// Clean a converted markdown document.
|
||||
///
|
||||
/// Safe to run twice: the second run changes nothing.
|
||||
pub fn scrub(markdown: &str) -> String {
|
||||
let text = pages::normalize(markdown);
|
||||
let text = heading::clean_all(&text);
|
||||
let text = heading::anchors_to_markers(&text);
|
||||
let text = html::strip_tags(&text);
|
||||
let text = html::scrub_control(&text);
|
||||
let text = html::collapse_blank_runs(&text);
|
||||
format!("{}\n", text.trim_matches(textkit::is_space))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_converted_heading_comes_out_readable() {
|
||||
let markdown = "# [<span id=\"page12\"></span>I. The Setting](#ch1) {.unnumbered}";
|
||||
assert_eq!(scrub(markdown), "# I. The Setting\n\n<!-- Page 12 -->\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_it_twice_changes_nothing() {
|
||||
let markdown = "<!-- Page 3 -->\n\n# The <em>Quarrel</em>\n\n\n\nA <b>mill</b>\u{0} \
|
||||
stood here.<br>It still does.\n\n<span id=\"page4\"></span>\n\nOn.";
|
||||
let once = scrub(markdown);
|
||||
assert_eq!(scrub(&once), once);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_marker_survives_every_pass() {
|
||||
assert!(scrub("<!-- Page 41 -->\n\ntext").contains("<!-- Page 41 -->"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_result_always_ends_in_exactly_one_newline() {
|
||||
for markdown in ["", "text", "text\n\n\n", " \n"] {
|
||||
let cleaned = scrub(markdown);
|
||||
assert!(cleaned.ends_with('\n'), "{markdown:?}");
|
||||
assert!(!cleaned.ends_with("\n\n"), "{markdown:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn form_feeds_become_pages_rather_than_disappearing() {
|
||||
// The control-character pass would delete a form feed, so page
|
||||
// normalisation has to happen first or the boundary is lost.
|
||||
let cleaned = scrub("one\u{c}two");
|
||||
assert!(cleaned.contains("<!-- Page 2 -->"), "{cleaned:?}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//! Cleans converted markdown.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
|
||||
/// Clean converted markdown: drop residual HTML and control characters,
|
||||
/// make headings readable, and spell every page boundary the same way.
|
||||
///
|
||||
/// Reads markdown on stdin, or from a path, and writes it to stdout. Safe to
|
||||
/// run twice.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Example:
|
||||
|
||||
pandoc -t commonmark book.epub | mdscrub | chunkmd > chunks.ndjson"
|
||||
)]
|
||||
struct Args {
|
||||
/// Markdown file to read. Reads stdin when absent.
|
||||
path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args = Args::parse();
|
||||
|
||||
let markdown = match read(args.path.as_deref()) {
|
||||
Ok(markdown) => markdown,
|
||||
Err(e) => {
|
||||
eprintln!("mdscrub: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
};
|
||||
|
||||
let cleaned = mdscrub::scrub(&markdown);
|
||||
match std::io::stdout().write_all(cleaned.as_bytes()) {
|
||||
Ok(()) => ExitCode::from(exit::OK),
|
||||
// A consumer that stopped reading is not a failure.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
|
||||
Err(e) => {
|
||||
eprintln!("mdscrub: {e}");
|
||||
ExitCode::from(exit::IO)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read(path: Option<&Path>) -> std::io::Result<String> {
|
||||
match path {
|
||||
Some(path) => std::fs::read_to_string(path),
|
||||
None => {
|
||||
let mut markdown = String::new();
|
||||
std::io::stdin().read_to_string(&mut markdown)?;
|
||||
Ok(markdown)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//! Turning whatever a converter used for page boundaries into one marker.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
/// The separator a paginating PDF converter writes between pages, like
|
||||
/// `{0}------------------------------`. Its numbering is zero-based.
|
||||
static SEPARATOR: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[^\S\n]*\n").expect("a valid pattern"));
|
||||
|
||||
/// A form feed, which is what a plain text extractor leaves between pages.
|
||||
const FORM_FEED: char = '\u{c}';
|
||||
|
||||
/// Rewrite page boundaries as `<!-- Page N -->` markers.
|
||||
///
|
||||
/// Separators first, then form feeds, and text with neither is returned
|
||||
/// unchanged — a cleanup pass has to be safe to run twice, and inventing a
|
||||
/// page marker for a document that has none would not be.
|
||||
pub fn normalize(text: &str) -> String {
|
||||
if SEPARATOR.is_match(text) {
|
||||
return from_separators(text);
|
||||
}
|
||||
if text.contains(FORM_FEED) {
|
||||
return from_form_feeds(text);
|
||||
}
|
||||
text.to_string()
|
||||
}
|
||||
|
||||
fn from_separators(text: &str) -> String {
|
||||
let mut pages: Vec<String> = Vec::new();
|
||||
let mut at = 0;
|
||||
let mut number = 1;
|
||||
|
||||
for found in SEPARATOR.captures_iter(text) {
|
||||
let whole = found.get(0).expect("group 0");
|
||||
let before = trim(&text[at..whole.start()]);
|
||||
if !before.is_empty() {
|
||||
pages.push(format!("<!-- Page {number} -->\n\n{before}"));
|
||||
}
|
||||
// The separator carries the number of the page it *ends*, counted
|
||||
// from zero, so the page that follows it is that number plus one.
|
||||
number = textkit::decimal(&found[1]).saturating_add(1);
|
||||
at = whole.end();
|
||||
}
|
||||
|
||||
let last = trim(&text[at..]);
|
||||
if !last.is_empty() {
|
||||
pages.push(format!("<!-- Page {number} -->\n\n{last}"));
|
||||
}
|
||||
pages.join("\n\n")
|
||||
}
|
||||
|
||||
fn from_form_feeds(text: &str) -> String {
|
||||
text.split(FORM_FEED)
|
||||
.map(trim)
|
||||
.enumerate()
|
||||
.filter(|(_, page)| !page.is_empty())
|
||||
.map(|(at, page)| format!("<!-- Page {} -->\n\n{page}", at + 1))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n")
|
||||
}
|
||||
|
||||
fn trim(text: &str) -> &str {
|
||||
text.trim_matches(textkit::is_space)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn separators_become_markers_numbered_from_one() {
|
||||
let text = "first page\n{0}------------------------------\nsecond page\n\
|
||||
{1}------------------------------\nthird page";
|
||||
assert_eq!(
|
||||
normalize(text),
|
||||
"<!-- Page 1 -->\n\nfirst page\n\n\
|
||||
<!-- Page 1 -->\n\nsecond page\n\n\
|
||||
<!-- Page 2 -->\n\nthird page"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn form_feeds_become_markers() {
|
||||
let text = "one\u{c}two\u{c}three";
|
||||
assert_eq!(
|
||||
normalize(text),
|
||||
"<!-- Page 1 -->\n\none\n\n<!-- Page 2 -->\n\ntwo\n\n<!-- Page 3 -->\n\nthree"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_page_is_dropped_and_the_numbering_carries_on() {
|
||||
assert_eq!(
|
||||
normalize("one\u{c}\u{c}three"),
|
||||
"<!-- Page 1 -->\n\none\n\n<!-- Page 3 -->\n\nthree"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_with_no_page_boundaries_is_left_exactly_as_it_was() {
|
||||
let text = "# A Heading\n\nSome prose, with no pagination at all.\n";
|
||||
assert_eq!(normalize(text), text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_short_dash_run_is_not_a_separator() {
|
||||
let text = "before\n{0}-----\nafter";
|
||||
assert_eq!(normalize(text), text);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user