mdscrub: decode character references, keep block boundaries, narrow the fence

crates/mdscrub/src/html.rs

@@ -4,7 +4,16 @@ use std::sync::LazyLock;
 
 use regex::Regex;
 
-/// An opening or closing inline tag.
+/// Python's `\s`, which is the Unicode whitespace property plus the four
+/// ASCII information separators. Spelled out wherever a ported pattern uses
+/// `\s`, because the difference decides where a page boundary is.
+const SPACE: &str = r"[\s\x{1c}-\x{1f}]";
+
+fn pattern(source: &str) -> Regex {
+    Regex::new(source).expect("a valid pattern")
+}
+
+/// An opening or closing inline tag, and its name.
 ///
 /// Converted markdown is full of them: pandoc's commonmark output keeps an
 /// EPUB's styling spans and anchors, and a PDF converter emits anchor spans
@@ -13,50 +22,133 @@ use regex::Regex;
 /// full-text index, and every result a reader sees.
 ///
 /// Comments are deliberately not matched: `<!-- Page 41 -->` has to survive.
-static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?[A-Za-z][^>]*>").expect("valid"));
+static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"</?([A-Za-z][A-Za-z0-9]*)[^>]*>"));
 
-/// A tag that stands for a line break rather than for styling.
-static BREAK: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<br\s*/?>").expect("valid"));
-
-/// The start of a fenced code block.
-static FENCE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\s*(```|~~~)").expect("valid"));
+/// A markdown image carrying its payload inline.
+///
+/// A generated EPUB cover arrives this way — a few hundred characters of
+/// base64 in a wrapper — and because the cover is the spine's first document
+/// it lands in the first chunk of the book, which is the chunk a "what is
+/// this book about" query matches. It holds no text, so it can only dilute
+/// the chunk it sits in.
+static DATA_URI_IMAGE: LazyLock<Regex> =
+    LazyLock::new(|| pattern(&format!(r"!\[[^\]]*\]\({SPACE}*data:[^)]*\)")));
 
 /// Control characters that must never reach a text column.
 ///
 /// A database will reject `0x00` outright. The other C0 controls — tab,
 /// newline and carriage return excepted — are storable and are extraction
 /// garbage: bad scans and malformed PDFs emit them routinely.
-static CONTROL: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"[\x00-\x08\x0b\x0c\x0e-\x1f]").expect("valid"));
+static CONTROL: LazyLock<Regex> = LazyLock::new(|| pattern(r"[\x00-\x08\x0b\x0c\x0e-\x1f]"));
 
 /// Three or more newlines, left behind wherever a tag-only line was removed.
-static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").expect("valid"));
+static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\n{3,}"));
+
+/// Tags that stand for a line or block boundary rather than a span of
+/// styling.
+///
+/// Deleting one outright welds the words either side of it together: a table
+/// of contents reading `APPENDICES<br>OF<br>LIST<br>xii` comes out as
+/// `APPENDICESOFLISTxii`, which is a word no reader can search for and no
+/// tokeniser can split. These become a space instead. Genuinely inline tags
+/// still go to nothing, because `claim<sup>16</sup>` has to stay `claim16`.
+const BREAK_TAGS: &[&str] = &[
+    "address",
+    "article",
+    "aside",
+    "blockquote",
+    "br",
+    "caption",
+    "dd",
+    "div",
+    "dl",
+    "dt",
+    "figcaption",
+    "figure",
+    "footer",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "header",
+    "hr",
+    "li",
+    "main",
+    "nav",
+    "ol",
+    "p",
+    "pre",
+    "section",
+    "table",
+    "tbody",
+    "td",
+    "tfoot",
+    "th",
+    "thead",
+    "tr",
+    "ul",
+];
 
 /// Strip inline HTML, keeping the text inside it and the HTML comments.
 ///
-/// Fenced code blocks are left alone, so a book that quotes HTML keeps its
-/// examples. `<br>` becomes a line break rather than nothing, because
-/// deleting it welds the words on either side into one — which is a spelling
-/// mistake in the index that no reader can search for.
+/// Character references are decoded first: a tag that was escaped twice
+/// arrives as `&lt;sup>107` and no tag pattern can see it, so the markup
+/// would otherwise survive into the text. Fenced code blocks are left alone,
+/// so a book that quotes HTML keeps its examples.
 pub fn strip_tags(markdown: &str) -> String {
-    let mut out = Vec::new();
+    let mut out: Vec<String> = Vec::new();
     let mut in_fence = false;
     for line in markdown.split('\n') {
-        if FENCE.is_match(line) {
+        if opens_a_fence(line) {
             in_fence = !in_fence;
             out.push(line.to_string());
             continue;
         }
         if in_fence {
             out.push(line.to_string());
+            continue;
+        }
+        let text = crate::unescape::unescape(line);
+        let text = DATA_URI_IMAGE.replace_all(&text, "");
+        let text = TAG.replace_all(
+            &text,
+            |found: &regex::Captures| {
+                if is_break_tag(&found[1]) { " " } else { "" }
+            },
+        );
+        // A line that held nothing but tags becomes empty rather than a lone
+        // space, so the blank-line collapse below still sees it as blank.
+        if text.trim_matches(textkit::is_space).is_empty() {
+            out.push(String::new());
         } else {
-            let line = BREAK.replace_all(line, "\n");
-            out.push(TAG.replace_all(&line, "").into_owned());
+            out.push(text.into_owned());
         }
     }
     out.join("\n")
 }
 
+/// Whether a line opens or closes a fenced code block.
+///
+/// A run of at least three backticks or tildes whose info string does not
+/// itself contain the fence character. The looser reading — any three of
+/// them — treats a decorative rule as a fence: `~~~~~~~~~END~~~~~~~~~` in a
+/// scanned book switched cleaning off for the rest of the document and left
+/// the markup behind.
+fn opens_a_fence(line: &str) -> bool {
+    let line = line.trim_start_matches(textkit::is_space);
+    ['`', '~'].into_iter().any(|fence| {
+        let rest = line.trim_start_matches(fence);
+        line.len() - rest.len() >= 3 && !rest.contains(fence)
+    })
+}
+
+fn is_break_tag(name: &str) -> bool {
+    let name = name.to_lowercase();
+    BREAK_TAGS.binary_search(&name.as_str()).is_ok()
+}
+
 /// Remove the control characters a text column cannot hold.
 pub fn scrub_control(text: &str) -> String {
     CONTROL.replace_all(text, "").into_owned()
@@ -71,6 +163,11 @@ pub fn collapse_blank_runs(text: &str) -> String {
 mod tests {
     use super::*;
 
+    #[test]
+    fn the_break_tag_list_is_sorted_so_it_can_be_searched() {
+        assert!(BREAK_TAGS.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
     #[test]
     fn styling_tags_go_and_their_text_stays() {
         assert_eq!(
@@ -79,33 +176,60 @@ mod tests {
         );
     }
 
+    #[test]
+    fn an_inline_tag_leaves_nothing_behind() {
+        assert_eq!(strip_tags("claim<sup>16</sup>"), "claim16");
+    }
+
+    #[test]
+    fn a_block_tag_leaves_a_space_so_words_do_not_weld() {
+        assert_eq!(strip_tags("APPENDICES<br>OF<br>LIST"), "APPENDICES OF LIST");
+        assert_eq!(strip_tags("<p>one</p><p>two</p>"), " one  two ");
+        assert_eq!(strip_tags("a<BR/>b"), "a b");
+    }
+
     #[test]
     fn html_comments_survive() {
         assert_eq!(strip_tags("<!-- Page 41 -->"), "<!-- Page 41 -->");
     }
 
     #[test]
-    fn a_break_tag_does_not_weld_the_words_either_side_of_it() {
-        assert_eq!(strip_tags("first<br>second"), "first\nsecond");
-        assert_eq!(strip_tags("first<br />second"), "first\nsecond");
-        assert_eq!(strip_tags("first<BR/>second"), "first\nsecond");
+    fn a_doubly_escaped_tag_is_decoded_and_then_removed() {
+        assert_eq!(strip_tags("&lt;sup>107Ibid."), "107Ibid.");
     }
 
     #[test]
-    fn fenced_code_keeps_its_tags() {
-        let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n```\n\nafter <b>y</b>";
+    fn an_inline_image_payload_goes() {
+        let line = "![cover](data:image/svg+xml;base64,PHN2ZyB4bWxucz0i) and text";
+        assert_eq!(strip_tags(line), " and text");
+    }
+
+    #[test]
+    fn fenced_code_keeps_its_tags_and_its_entities() {
+        let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n&amp; kept\n```\n\nafter";
         let cleaned = strip_tags(markdown);
         assert!(cleaned.contains("<span>kept</span>"));
+        assert!(cleaned.contains("&amp; kept"));
         assert!(cleaned.contains("before x"));
-        assert!(cleaned.contains("after y"));
     }
 
     #[test]
-    fn a_tag_split_across_two_lines_is_left_alone() {
-        // Matching per line is what keeps a stray `<` in prose from eating
-        // the rest of the document.
-        let markdown = "a < b and\nc > d";
-        assert_eq!(strip_tags(markdown), markdown);
+    fn a_decorative_rule_is_not_a_fence() {
+        let markdown = "~~~~~~~~~END~~~~~~~~~\n<span class=\"calibre_10\">after</span>";
+        assert_eq!(strip_tags(markdown), "~~~~~~~~~END~~~~~~~~~\nafter");
+    }
+
+    #[test]
+    fn a_real_fence_with_an_info_string_still_opens() {
+        assert!(opens_a_fence("```html"));
+        assert!(opens_a_fence("   ~~~"));
+        assert!(!opens_a_fence("``"));
+        assert!(!opens_a_fence("text ```"));
+    }
+
+    #[test]
+    fn a_line_of_nothing_but_tags_becomes_blank() {
+        assert_eq!(strip_tags("a\n<span></span>\nb"), "a\n\nb");
     }
 
     #[test]

crates/mdscrub/src/lib.rs

@@ -10,9 +10,11 @@
 //! because one of the things that marks a page boundary — a form feed — is
 //! also a control character that the later pass deletes.
 
+mod entities;
 pub mod heading;
 pub mod html;
 pub mod pages;
+pub mod unescape;
 
 /// Clean a converted markdown document.
 ///

crates/mdscrub/src/pages.rs

@@ -7,7 +7,7 @@ use regex::Regex;
 /// The separator a paginating PDF converter writes between pages, like
 /// `{0}------------------------------`. Its numbering is zero-based.
 static SEPARATOR: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[^\S\n]*\n").expect("a valid pattern"));
+    LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[\s\x{1c}-\x{1f}]*\n").expect("a valid pattern"));
 
 /// A form feed, which is what a plain text extractor leaves between pages.
 const FORM_FEED: char = '\u{c}';

crates/mdscrub/src/unescape.rs

@@ -0,0 +1,150 @@
+//! Turning HTML character references back into the characters they name.
+//!
+//! Converted markdown carries doubly-escaped markup: a tag that was escaped
+//! once by the source and once by the converter arrives as `<sup>107`,
+//! which no tag pattern can see. Decoding first turns it into a real tag that
+//! the stripping pass then removes, and fixes `&` and the numeric forms
+//! in ordinary prose on the way past.
+//!
+//! The rules are the specification's, including the parts that exist only
+//! because browsers had to accept what was already on the web: a reference
+//! with no closing semicolon, a name that is the prefix of a longer name, and
+//! numbers that mean Windows-1252 positions rather than code points.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+use crate::entities::{INVALID_CHARREFS, INVALID_CODEPOINTS, NAMED};
+
+/// A character reference: a number, a hexadecimal number, or a name of up to
+/// thirty-two characters, each with an optional closing semicolon.
+static CHARREF: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r"&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\x0c <&#;]{1,32};?)").expect("valid")
+});
+
+/// Decode every character reference in `text`.
+pub fn unescape(text: &str) -> String {
+    if !text.contains('&') {
+        return text.to_string();
+    }
+    CHARREF
+        .replace_all(text, |found: &regex::Captures| replace(&found[1]))
+        .into_owned()
+}
+
+fn replace(reference: &str) -> String {
+    match reference.strip_prefix('#') {
+        Some(number) => numeric(number),
+        None => named(reference),
+    }
+}
+
+fn numeric(number: &str) -> String {
+    let digits = number.trim_end_matches(';');
+    let value = match digits.strip_prefix(['x', 'X']) {
+        Some(hex) => u32::from_str_radix(hex, 16),
+        None => digits.parse::<u32>(),
+    };
+    // A number too long to be a code point at all is out of range, which the
+    // specification answers the same way as any other out-of-range number.
+    let Ok(value) = value else {
+        return '\u{fffd}'.to_string();
+    };
+
+    if let Ok(at) = INVALID_CHARREFS.binary_search_by_key(&value, |(number, _)| *number) {
+        return INVALID_CHARREFS[at].1.to_string();
+    }
+    if (0xd800..=0xdfff).contains(&value) || value > 0x10ffff {
+        return '\u{fffd}'.to_string();
+    }
+    if INVALID_CODEPOINTS.binary_search(&value).is_ok() {
+        return String::new();
+    }
+    char::from_u32(value).map(String::from).unwrap_or_default()
+}
+
+fn named(name: &str) -> String {
+    if let Some(value) = lookup(name) {
+        return value.to_string();
+    }
+    // The longest name that is a prefix of what was written wins, and the
+    // rest is left as text: `&notit;` is `¬it;`, not an unknown reference.
+    for end in (2..name.len()).rev() {
+        if !name.is_char_boundary(end) {
+            continue;
+        }
+        if let Some(value) = lookup(&name[..end]) {
+            return format!("{value}{}", &name[end..]);
+        }
+    }
+    format!("&{name}")
+}
+
+fn lookup(name: &str) -> Option<&'static str> {
+    NAMED
+        .binary_search_by_key(&name, |(named, _)| named)
+        .ok()
+        .map(|at| NAMED[at].1)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_table_is_sorted_so_it_can_be_searched() {
+        assert!(NAMED.windows(2).all(|pair| pair[0].0 < pair[1].0));
+        assert!(
+            INVALID_CHARREFS
+                .windows(2)
+                .all(|pair| pair[0].0 < pair[1].0)
+        );
+        assert!(INVALID_CODEPOINTS.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
+    #[test]
+    fn ordinary_references_decode() {
+        assert_eq!(unescape("a &amp; b"), "a & b");
+        assert_eq!(unescape("&lt;sup&gt;107"), "<sup>107");
+        assert_eq!(unescape("&#65;&#x42;"), "AB");
+    }
+
+    #[test]
+    fn a_missing_semicolon_is_still_a_reference() {
+        assert_eq!(unescape("&amp"), "&");
+        assert_eq!(unescape("&#65"), "A");
+        assert_eq!(unescape("&AElig"), "Æ");
+    }
+
+    #[test]
+    fn the_longest_name_wins_and_the_rest_stays_text() {
+        assert_eq!(unescape("&notit;"), "¬it;");
+        assert_eq!(unescape("&notin;"), "∉");
+    }
+
+    #[test]
+    fn something_that_is_not_a_reference_is_left_alone() {
+        assert_eq!(unescape("&unknown;"), "&unknown;");
+        assert_eq!(unescape("a & b"), "a & b");
+        assert_eq!(unescape("no ampersand here"), "no ampersand here");
+    }
+
+    #[test]
+    fn windows_1252_numbers_decode_as_the_specification_says() {
+        assert_eq!(unescape("&#128;"), "€");
+        assert_eq!(unescape("&#0;"), "\u{fffd}");
+    }
+
+    #[test]
+    fn a_number_that_is_not_a_code_point_is_refused() {
+        assert_eq!(unescape("&#xD800;"), "\u{fffd}");
+        assert_eq!(unescape("&#1114112;"), "\u{fffd}");
+        assert_eq!(unescape("&#99999999999999999999;"), "\u{fffd}");
+    }
+
+    #[test]
+    fn a_code_point_the_specification_drops_comes_out_as_nothing() {
+        assert_eq!(unescape("a&#1;b"), "ab");
+    }
+}

parity/KNOWN.md

@@ -18,25 +18,6 @@ affected and how often it occurs in practice.
 
 ---
 
-## mdscrub — a line-break tag becomes a line break, not nothing
-
-**Python:** `strip_residual_html` removes every inline tag, `<br>` included,
-so `the<br>northern end` becomes `thenorthern end`.
-
-**Rust:** `<br>`, `<br/>` and `<br />` become a newline; every other tag is
-still removed.
-
-**Why this is acceptable:** the welded word is not a formatting difference,
-it is a word that does not exist. It cannot be searched for, it embeds as a
-token sequence no query will produce, and it is invisible in a result because
-the surrounding sentence reads normally. The divergence only affects
-documents that use `<br>` at all — mostly verse, addresses and tables of
-contents converted out of HTML — and within those, only the joins.
-
-**Measured:** `parity/cases/mdscrub/breaks.md`; both answers are committed as
-`golden/mdscrub/breaks.md` and `golden/mdscrub/breaks.rust.md`. Three welds
-in eight lines.
-
 ## mdscrub — un-paginated text is not given a page 1 marker
 
 **Python:** `_normalize_marker_pages` ends by wrapping any text it found no

parity/README.md

@@ -23,9 +23,14 @@ can be re-run live, which catches a golden file drifting away from a Python
 implementation that has moved on:
 
 ```sh
-CATENA_PARITY_PYTHON=/path/to/venv/bin/python cargo test -p parity --features python
+CATENA_PARITY_PYTHON=/path/to/venv/bin/python \
+CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+    cargo test -p parity --features python
 ```
 
+`SOURCE.md` records which commit of that source the golden files came from.
+Pointing the oracles at an older checkout will fail, and should.
+
 ## Regenerating
 
 Golden files are never regenerated by a test — a golden file that rewrites
@@ -33,7 +38,9 @@ itself when it fails is a test that cannot fail. Regenerate deliberately,
 read the diff, and commit it with the reason:
 
 ```sh
-CATENA_PARITY_PYTHON=/path/to/venv/bin/python cargo run -p parity --bin regen
+CATENA_PARITY_PYTHON=/path/to/venv/bin/python \
+CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+    cargo run -p parity --bin regen
 ```
 
 ## Fixtures

parity/SOURCE.md

@@ -0,0 +1,19 @@
+# What the golden files were generated from
+
+The oracles import the Python these tools were ported from. The golden files
+in this directory are the answers that Python gave, at one commit:
+
+| Source | Commit |
+|---|---|
+| `book-pipeline` | `7499e3257572968b437ecfdc79feccc91b96585b` |
+
+Regenerating against a different commit will move the goldens. That is the
+point — the diff is the review. Record the new commit here in the same change.
+
+The oracles find the source through two environment variables, because where
+a checkout lives is a fact about one machine and must not be compiled in:
+
+```sh
+CATENA_PARITY_PYTHON=/path/to/venv/bin/python
+CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src
+```

parity/golden/mdscrub/breaks.md

@@ -1,5 +1,5 @@
-The address was set with explicit line breaks:Twelve Mill LaneThe
-Valleyand the county beyond.
+The address was set with explicit line breaks: Twelve Mill Lane The
+Valley and the county beyond.
 
 A second paragraph, where the break sits mid sentence — the mill stood at
-thenorthern end of the valley, where the fall was steepest.
+the northern end of the valley, where the fall was steepest.

parity/golden/mdscrub/breaks.rust.md

@@ -1,9 +0,0 @@
-The address was set with explicit line breaks:
-Twelve Mill Lane
-The
-Valley
-and the county beyond.
-
-A second paragraph, where the break sits mid sentence — the mill stood at
-the
-northern end of the valley, where the fall was steepest.

parity/oracle/unescape.py

@@ -0,0 +1,23 @@
+"""Decode HTML character references the way Python does, one line at a time.
+
+Each line of stdin is one case; each line of stdout is that case decoded and
+JSON-encoded, because a decoded reference can itself be a newline.
+
+    python3 oracle/unescape.py < cases/unescape/entities.txt
+"""
+
+import html
+import json
+import sys
+
+
+def main() -> int:
+    text = sys.stdin.buffer.read().decode("utf-8")
+    out = sys.stdout
+    for line in text.split("\n"):
+        out.write(json.dumps(html.unescape(line), ensure_ascii=False) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,8 +19,9 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 3] = [
+    let jobs: [Job; 4] = [
         ("tokens", tokens),
+        ("unescape", unescape),
         ("chunkmd", chunkmd),
         ("mdscrub", mdscrub),
     ];
@@ -54,6 +55,18 @@ fn tokens() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn unescape() -> Written {
+    let dir = parity::golden("unescape");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/unescape/entities.txt");
+    let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let decoded = parity::run_oracle("unescape.py", &text)?;
+    let path = dir.join("entities.jsonl");
+    std::fs::write(&path, decoded).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn mdscrub() -> Written {
     let dir = parity::golden("mdscrub");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
@@ -66,15 +79,6 @@ fn mdscrub() -> Written {
         let path = dir.join(format!("{name}.md"));
         std::fs::write(&path, cleaned).map_err(|e| e.to_string())?;
         written.push(path.display().to_string());
-
-        // Where the Rust deliberately differs, its own answer is committed
-        // too, so the difference is visible in a review rather than only in
-        // a test failure. See parity/KNOWN.md.
-        if parity::DIVERGENT_MDSCRUB.contains(&name.as_str()) {
-            let path = dir.join(format!("{name}.rust.md"));
-            std::fs::write(&path, ::mdscrub::scrub(&markdown)).map_err(|e| e.to_string())?;
-            written.push(path.display().to_string());
-        }
     }
     Ok(written)
 }

parity/src/lib.rs

@@ -83,10 +83,6 @@ pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<Str
         .map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
 }
 
-/// Cases where `mdscrub` is meant to differ from the Python, each one
-/// argued for in `KNOWN.md`.
-pub const DIVERGENT_MDSCRUB: &[&str] = &["breaks"];
-
 /// Every case file for a tool, in a fixed order.
 pub fn cases(tool: &str) -> Vec<PathBuf> {
     let dir = dir().join("cases").join(tool);

parity/tests/mdscrub.rs

@@ -13,37 +13,18 @@ fn every_case_cleans_the_same_as_python() {
 
     for case in cases {
         let name = parity::case_name(&case);
-        if parity::DIVERGENT_MDSCRUB.contains(&name.as_str()) {
-            continue;
-        }
         let ours = mdscrub::scrub(&case_text(&case));
         let theirs = parity::read_golden_text(&format!("mdscrub/{name}.md"));
         assert_eq!(ours, theirs, "{name} differs from the python");
     }
 }
 
-#[test]
-fn the_documented_divergences_are_exactly_what_was_argued_for() {
-    for name in parity::DIVERGENT_MDSCRUB {
-        let case = parity::dir()
-            .join("cases/mdscrub")
-            .join(format!("{name}.md"));
-        let ours = mdscrub::scrub(&case_text(&case));
-        let theirs = parity::read_golden_text(&format!("mdscrub/{name}.md"));
-        assert_ne!(ours, theirs, "{name} no longer diverges — update KNOWN.md");
-        assert_eq!(
-            ours,
-            parity::read_golden_text(&format!("mdscrub/{name}.rust.md")),
-            "{name} diverges differently than recorded"
-        );
-    }
-}
-
-/// The one thing the divergence exists to prevent.
+/// The thing the break-tag rule exists to prevent.
 #[test]
 fn a_line_break_tag_never_welds_two_words_together() {
     let cleaned = mdscrub::scrub("the mill stood at the<br>northern end");
     assert!(!cleaned.contains("thenorthern"), "{cleaned:?}");
+    assert!(cleaned.contains("the northern"), "{cleaned:?}");
 }
 
 /// Catches a golden file drifting away from a Python that has moved on.

parity/tests/unescape.rs

@@ -0,0 +1,57 @@
+//! Character references must decode exactly as Python decodes them.
+//!
+//! The table is two thousand entries of specification data and the rules
+//! around it exist only because browsers had to accept what was already on
+//! the web. Getting one of them wrong leaves markup in the text of a book,
+//! which is how `<sup>107` reached thousands of passages before anyone
+//! noticed.
+
+#[test]
+fn every_reference_decodes_the_same_as_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/unescape/entities.txt"))
+        .expect("a readable case file");
+    let cases: Vec<&str> = cases.split('\n').collect();
+    let golden = parity::read_golden_text("unescape/entities.jsonl");
+    let golden: Vec<String> = golden
+        .lines()
+        .map(|line| serde_json::from_str(line).expect("the oracle writes JSON strings"))
+        .collect();
+
+    assert_eq!(
+        cases.len(),
+        golden.len(),
+        "{} cases against {} golden answers — regenerate",
+        cases.len(),
+        golden.len()
+    );
+
+    let mut wrong = Vec::new();
+    for (case, expected) in cases.iter().zip(&golden) {
+        let got = mdscrub::unescape::unescape(case);
+        if &got != expected {
+            wrong.push(format!("{case:?}: python {expected:?}, rust {got:?}"));
+        }
+    }
+    assert!(
+        wrong.is_empty(),
+        "{} of {} references differ:\n{}",
+        wrong.len(),
+        cases.len(),
+        wrong
+            .iter()
+            .take(20)
+            .cloned()
+            .collect::<Vec<_>>()
+            .join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/unescape/entities.txt"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("unescape.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("unescape/entities.jsonl"));
+}