mdscrub: decode character references, keep block boundaries, narrow the fence

parity/tests/mdscrub.rs

@@ -13,37 +13,18 @@ fn every_case_cleans_the_same_as_python() {
 
     for case in cases {
         let name = parity::case_name(&case);
-        if parity::DIVERGENT_MDSCRUB.contains(&name.as_str()) {
-            continue;
-        }
         let ours = mdscrub::scrub(&case_text(&case));
         let theirs = parity::read_golden_text(&format!("mdscrub/{name}.md"));
         assert_eq!(ours, theirs, "{name} differs from the python");
     }
 }
 
-#[test]
-fn the_documented_divergences_are_exactly_what_was_argued_for() {
-    for name in parity::DIVERGENT_MDSCRUB {
-        let case = parity::dir()
-            .join("cases/mdscrub")
-            .join(format!("{name}.md"));
-        let ours = mdscrub::scrub(&case_text(&case));
-        let theirs = parity::read_golden_text(&format!("mdscrub/{name}.md"));
-        assert_ne!(ours, theirs, "{name} no longer diverges — update KNOWN.md");
-        assert_eq!(
-            ours,
-            parity::read_golden_text(&format!("mdscrub/{name}.rust.md")),
-            "{name} diverges differently than recorded"
-        );
-    }
-}
-
-/// The one thing the divergence exists to prevent.
+/// The thing the break-tag rule exists to prevent.
 #[test]
 fn a_line_break_tag_never_welds_two_words_together() {
     let cleaned = mdscrub::scrub("the mill stood at the<br>northern end");
     assert!(!cleaned.contains("thenorthern"), "{cleaned:?}");
+    assert!(cleaned.contains("the northern"), "{cleaned:?}");
 }
 
 /// Catches a golden file drifting away from a Python that has moved on.

parity/tests/unescape.rs

@@ -0,0 +1,57 @@
+//! Character references must decode exactly as Python decodes them.
+//!
+//! The table is two thousand entries of specification data and the rules
+//! around it exist only because browsers had to accept what was already on
+//! the web. Getting one of them wrong leaves markup in the text of a book,
+//! which is how `<sup>107` reached thousands of passages before anyone
+//! noticed.
+
+#[test]
+fn every_reference_decodes_the_same_as_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/unescape/entities.txt"))
+        .expect("a readable case file");
+    let cases: Vec<&str> = cases.split('\n').collect();
+    let golden = parity::read_golden_text("unescape/entities.jsonl");
+    let golden: Vec<String> = golden
+        .lines()
+        .map(|line| serde_json::from_str(line).expect("the oracle writes JSON strings"))
+        .collect();
+
+    assert_eq!(
+        cases.len(),
+        golden.len(),
+        "{} cases against {} golden answers — regenerate",
+        cases.len(),
+        golden.len()
+    );
+
+    let mut wrong = Vec::new();
+    for (case, expected) in cases.iter().zip(&golden) {
+        let got = mdscrub::unescape::unescape(case);
+        if &got != expected {
+            wrong.push(format!("{case:?}: python {expected:?}, rust {got:?}"));
+        }
+    }
+    assert!(
+        wrong.is_empty(),
+        "{} of {} references differ:\n{}",
+        wrong.len(),
+        cases.len(),
+        wrong
+            .iter()
+            .take(20)
+            .cloned()
+            .collect::<Vec<_>>()
+            .join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/unescape/entities.txt"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("unescape.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("unescape/entities.jsonl"));
+}