funkpower/catena
1
0
Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-10-22 18:57:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

mdscrub: decode character references, keep block boundaries, narrow the fence

Browse Source
This commit is contained in:
funkpower
2026-09-19 11:36:31 -04:00
parent 1b3c002b3d
commit bd0ae11d01
17 changed files with 7274 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/mdscrub/src/entities.rs
+2296
View File
File diff suppressed because it is too large Load Diff
crates/mdscrub/src/html.rs
+154 -30
View File
@@ -4,7 +4,16 @@ use std::sync::LazyLock;
use regex::Regex;
/// An opening or closing inline tag.
/// Python's `\s`, which is the Unicode whitespace property plus the four
/// ASCII information separators. Spelled out wherever a ported pattern uses
/// `\s`, because the difference decides where a page boundary is.
const SPACE: &str = r"[\s\x{1c}-\x{1f}]";
fn pattern(source: &str) -> Regex {
Regex::new(source).expect("a valid pattern")
}
/// An opening or closing inline tag, and its name.
///
/// Converted markdown is full of them: pandoc's commonmark output keeps an
/// EPUB's styling spans and anchors, and a PDF converter emits anchor spans
@@ -13,50 +22,133 @@ use regex::Regex;
/// full-text index, and every result a reader sees.
///
/// Comments are deliberately not matched: `<!-- Page 41 -->` has to survive.
static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?[A-Za-z][^>]*>").expect("valid"));
static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"</?([A-Za-z][A-Za-z0-9]*)[^>]*>"));
/// A tag that stands for a line break rather than for styling.
static BREAK: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<br\s*/?>").expect("valid"));
/// The start of a fenced code block.
static FENCE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\s*(```|~~~)").expect("valid"));
/// A markdown image carrying its payload inline.
///
/// A generated EPUB cover arrives this way — a few hundred characters of
/// base64 in a wrapper — and because the cover is the spine's first document
/// it lands in the first chunk of the book, which is the chunk a "what is
/// this book about" query matches. It holds no text, so it can only dilute
/// the chunk it sits in.
static DATA_URI_IMAGE: LazyLock<Regex> =
LazyLock::new(|| pattern(&format!(r"!\[[^\]]*\]\({SPACE}*data:[^)]*\)")));
/// Control characters that must never reach a text column.
///
/// A database will reject `0x00` outright. The other C0 controls — tab,
/// newline and carriage return excepted — are storable and are extraction
/// garbage: bad scans and malformed PDFs emit them routinely.
static CONTROL: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[\x00-\x08\x0b\x0c\x0e-\x1f]").expect("valid"));
static CONTROL: LazyLock<Regex> = LazyLock::new(|| pattern(r"[\x00-\x08\x0b\x0c\x0e-\x1f]"));
/// Three or more newlines, left behind wherever a tag-only line was removed.
static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").expect("valid"));
static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\n{3,}"));
/// Tags that stand for a line or block boundary rather than a span of
/// styling.
///
/// Deleting one outright welds the words either side of it together: a table
/// of contents reading `APPENDICES<br>OF<br>LIST<br>xii` comes out as
/// `APPENDICESOFLISTxii`, which is a word no reader can search for and no
/// tokeniser can split. These become a space instead. Genuinely inline tags
/// still go to nothing, because `claim<sup>16</sup>` has to stay `claim16`.
const BREAK_TAGS: &[&str] = &[
"address",
"article",
"aside",
"blockquote",
"br",
"caption",
"dd",
"div",
"dl",
"dt",
"figcaption",
"figure",
"footer",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"li",
"main",
"nav",
"ol",
"p",
"pre",
"section",
"table",
"tbody",
"td",
"tfoot",
"th",
"thead",
"tr",
"ul",
];
/// Strip inline HTML, keeping the text inside it and the HTML comments.
///
/// Fenced code blocks are left alone, so a book that quotes HTML keeps its
/// examples. `<br>` becomes a line break rather than nothing, because
/// deleting it welds the words on either side into one — which is a spelling
/// mistake in the index that no reader can search for.
/// Character references are decoded first: a tag that was escaped twice
/// arrives as `&lt;sup>107` and no tag pattern can see it, so the markup
/// would otherwise survive into the text. Fenced code blocks are left alone,
/// so a book that quotes HTML keeps its examples.
pub fn strip_tags(markdown: &str) -> String {
let mut out = Vec::new();
let mut out: Vec<String> = Vec::new();
let mut in_fence = false;
for line in markdown.split('\n') {
if FENCE.is_match(line) {
if opens_a_fence(line) {
in_fence = !in_fence;
out.push(line.to_string());
continue;
}
if in_fence {
out.push(line.to_string());
continue;
}
let text = crate::unescape::unescape(line);
let text = DATA_URI_IMAGE.replace_all(&text, "");
let text = TAG.replace_all(
&text,
|found: &regex::Captures| {
if is_break_tag(&found[1]) { " " } else { "" }
},
);
// A line that held nothing but tags becomes empty rather than a lone
// space, so the blank-line collapse below still sees it as blank.
if text.trim_matches(textkit::is_space).is_empty() {
out.push(String::new());
} else {
let line = BREAK.replace_all(line, "\n");
out.push(TAG.replace_all(&line, "").into_owned());
out.push(text.into_owned());
}
}
out.join("\n")
}
/// Whether a line opens or closes a fenced code block.
///
/// A run of at least three backticks or tildes whose info string does not
/// itself contain the fence character. The looser reading — any three of
/// them — treats a decorative rule as a fence: `~~~~~~~~~END~~~~~~~~~` in a
/// scanned book switched cleaning off for the rest of the document and left
/// the markup behind.
fn opens_a_fence(line: &str) -> bool {
let line = line.trim_start_matches(textkit::is_space);
['`', '~'].into_iter().any(|fence| {
let rest = line.trim_start_matches(fence);
line.len() - rest.len() >= 3 && !rest.contains(fence)
})
}
fn is_break_tag(name: &str) -> bool {
let name = name.to_lowercase();
BREAK_TAGS.binary_search(&name.as_str()).is_ok()
}
/// Remove the control characters a text column cannot hold.
pub fn scrub_control(text: &str) -> String {
CONTROL.replace_all(text, "").into_owned()
@@ -71,6 +163,11 @@ pub fn collapse_blank_runs(text: &str) -> String {
mod tests {
use super::*;
#[test]
fn the_break_tag_list_is_sorted_so_it_can_be_searched() {
assert!(BREAK_TAGS.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn styling_tags_go_and_their_text_stays() {
assert_eq!(
@@ -79,33 +176,60 @@ mod tests {
);
}
#[test]
fn an_inline_tag_leaves_nothing_behind() {
assert_eq!(strip_tags("claim<sup>16</sup>"), "claim16");
}
#[test]
fn a_block_tag_leaves_a_space_so_words_do_not_weld() {
assert_eq!(strip_tags("APPENDICES<br>OF<br>LIST"), "APPENDICES OF LIST");
assert_eq!(strip_tags("<p>one</p><p>two</p>"), " one two ");
assert_eq!(strip_tags("a<BR/>b"), "a b");
}
#[test]
fn html_comments_survive() {
assert_eq!(strip_tags("<!-- Page 41 -->"), "<!-- Page 41 -->");
}
#[test]
fn a_break_tag_does_not_weld_the_words_either_side_of_it() {
assert_eq!(strip_tags("first<br>second"), "first\nsecond");
assert_eq!(strip_tags("first<br />second"), "first\nsecond");
assert_eq!(strip_tags("first<BR/>second"), "first\nsecond");
fn a_doubly_escaped_tag_is_decoded_and_then_removed() {
assert_eq!(strip_tags("&lt;sup>107Ibid."), "107Ibid.");
}
#[test]
fn fenced_code_keeps_its_tags() {
let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n```\n\nafter <b>y</b>";
fn an_inline_image_payload_goes() {
let line = "![cover](data:image/svg+xml;base64,PHN2ZyB4bWxucz0i) and text";
assert_eq!(strip_tags(line), " and text");
}
#[test]
fn fenced_code_keeps_its_tags_and_its_entities() {
let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n&amp; kept\n```\n\nafter";
let cleaned = strip_tags(markdown);
assert!(cleaned.contains("<span>kept</span>"));
assert!(cleaned.contains("&amp; kept"));
assert!(cleaned.contains("before x"));
assert!(cleaned.contains("after y"));
}
#[test]
fn a_tag_split_across_two_lines_is_left_alone() {
// Matching per line is what keeps a stray `<` in prose from eating
// the rest of the document.
let markdown = "a < b and\nc > d";
assert_eq!(strip_tags(markdown), markdown);
fn a_decorative_rule_is_not_a_fence() {
let markdown = "~~~~~~~~~END~~~~~~~~~\n<span class=\"calibre_10\">after</span>";
assert_eq!(strip_tags(markdown), "~~~~~~~~~END~~~~~~~~~\nafter");
}
#[test]
fn a_real_fence_with_an_info_string_still_opens() {
assert!(opens_a_fence("```html"));
assert!(opens_a_fence(" ~~~"));
assert!(!opens_a_fence("``"));
assert!(!opens_a_fence("text ```"));
}
#[test]
fn a_line_of_nothing_but_tags_becomes_blank() {
assert_eq!(strip_tags("a\n<span></span>\nb"), "a\n\nb");
}
#[test]
crates/mdscrub/src/lib.rs
+2
View File
@@ -10,9 +10,11 @@
//! because one of the things that marks a page boundary — a form feed — is
//! also a control character that the later pass deletes.
mod entities;
pub mod heading;
pub mod html;
pub mod pages;
pub mod unescape;
/// Clean a converted markdown document.
///
crates/mdscrub/src/pages.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use regex::Regex;
/// The separator a paginating PDF converter writes between pages, like
/// `{0}------------------------------`. Its numbering is zero-based.
static SEPARATOR: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[^\S\n]*\n").expect("a valid pattern"));
LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[\s\x{1c}-\x{1f}]*\n").expect("a valid pattern"));
/// A form feed, which is what a plain text extractor leaves between pages.
const FORM_FEED: char = '\u{c}';
crates/mdscrub/src/unescape.rs
+150
View File
@@ -0,0 +1,150 @@
//! Turning HTML character references back into the characters they name.
//!
//! Converted markdown carries doubly-escaped markup: a tag that was escaped
//! once by the source and once by the converter arrives as `&lt;sup>107`,
//! which no tag pattern can see. Decoding first turns it into a real tag that
//! the stripping pass then removes, and fixes `&amp;` and the numeric forms
//! in ordinary prose on the way past.
//!
//! The rules are the specification's, including the parts that exist only
//! because browsers had to accept what was already on the web: a reference
//! with no closing semicolon, a name that is the prefix of a longer name, and
//! numbers that mean Windows-1252 positions rather than code points.
use std::sync::LazyLock;
use regex::Regex;
use crate::entities::{INVALID_CHARREFS, INVALID_CODEPOINTS, NAMED};
/// A character reference: a number, a hexadecimal number, or a name of up to
/// thirty-two characters, each with an optional closing semicolon.
static CHARREF: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\x0c <&#;]{1,32};?)").expect("valid")
});
/// Decode every character reference in `text`.
pub fn unescape(text: &str) -> String {
if !text.contains('&') {
return text.to_string();
}
CHARREF
.replace_all(text, |found: &regex::Captures| replace(&found[1]))
.into_owned()
}
fn replace(reference: &str) -> String {
match reference.strip_prefix('#') {
Some(number) => numeric(number),
None => named(reference),
}
}
fn numeric(number: &str) -> String {
let digits = number.trim_end_matches(';');
let value = match digits.strip_prefix(['x', 'X']) {
Some(hex) => u32::from_str_radix(hex, 16),
None => digits.parse::<u32>(),
};
// A number too long to be a code point at all is out of range, which the
// specification answers the same way as any other out-of-range number.
let Ok(value) = value else {
return '\u{fffd}'.to_string();
};
if let Ok(at) = INVALID_CHARREFS.binary_search_by_key(&value, |(number, _)| *number) {
return INVALID_CHARREFS[at].1.to_string();
}
if (0xd800..=0xdfff).contains(&value) || value > 0x10ffff {
return '\u{fffd}'.to_string();
}
if INVALID_CODEPOINTS.binary_search(&value).is_ok() {
return String::new();
}
char::from_u32(value).map(String::from).unwrap_or_default()
}
fn named(name: &str) -> String {
if let Some(value) = lookup(name) {
return value.to_string();
}
// The longest name that is a prefix of what was written wins, and the
// rest is left as text: `&notit;` is `¬it;`, not an unknown reference.
for end in (2..name.len()).rev() {
if !name.is_char_boundary(end) {
continue;
}
if let Some(value) = lookup(&name[..end]) {
return format!("{value}{}", &name[end..]);
}
}
format!("&{name}")
}
fn lookup(name: &str) -> Option<&'static str> {
NAMED
.binary_search_by_key(&name, |(named, _)| named)
.ok()
.map(|at| NAMED[at].1)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_table_is_sorted_so_it_can_be_searched() {
assert!(NAMED.windows(2).all(|pair| pair[0].0 < pair[1].0));
assert!(
INVALID_CHARREFS
.windows(2)
.all(|pair| pair[0].0 < pair[1].0)
);
assert!(INVALID_CODEPOINTS.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn ordinary_references_decode() {
assert_eq!(unescape("a &amp; b"), "a & b");
assert_eq!(unescape("&lt;sup&gt;107"), "<sup>107");
assert_eq!(unescape("&#65;&#x42;"), "AB");
}
#[test]
fn a_missing_semicolon_is_still_a_reference() {
assert_eq!(unescape("&amp"), "&");
assert_eq!(unescape("&#65"), "A");
assert_eq!(unescape("&AElig"), "Æ");
}
#[test]
fn the_longest_name_wins_and_the_rest_stays_text() {
assert_eq!(unescape("&notit;"), "¬it;");
assert_eq!(unescape("&notin;"), "∉");
}
#[test]
fn something_that_is_not_a_reference_is_left_alone() {
assert_eq!(unescape("&unknown;"), "&unknown;");
assert_eq!(unescape("a & b"), "a & b");
assert_eq!(unescape("no ampersand here"), "no ampersand here");
}
#[test]
fn windows_1252_numbers_decode_as_the_specification_says() {
assert_eq!(unescape("&#128;"), "€");
assert_eq!(unescape("&#0;"), "\u{fffd}");
}
#[test]
fn a_number_that_is_not_a_code_point_is_refused() {
assert_eq!(unescape("&#xD800;"), "\u{fffd}");
assert_eq!(unescape("&#1114112;"), "\u{fffd}");
assert_eq!(unescape("&#99999999999999999999;"), "\u{fffd}");
}
#[test]
fn a_code_point_the_specification_drops_comes_out_as_nothing() {
assert_eq!(unescape("a&#1;b"), "ab");
}
}