mdscrub: residual html, control characters, headings, page markers

parity/src/bin/regen.rs

@@ -19,7 +19,11 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 2] = [("tokens", tokens), ("chunkmd", chunkmd)];
+    let jobs: [Job; 3] = [
+        ("tokens", tokens),
+        ("chunkmd", chunkmd),
+        ("mdscrub", mdscrub),
+    ];
 
     for (name, job) in jobs {
         match job() {
@@ -50,6 +54,31 @@ fn tokens() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn mdscrub() -> Written {
+    let dir = parity::golden("mdscrub");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let mut written = Vec::new();
+    for case in parity::cases("mdscrub") {
+        let markdown = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
+        let name = parity::case_name(&case);
+        let cleaned = parity::run_oracle("mdscrub.py", &markdown)?;
+        let path = dir.join(format!("{name}.md"));
+        std::fs::write(&path, cleaned).map_err(|e| e.to_string())?;
+        written.push(path.display().to_string());
+
+        // Where the Rust deliberately differs, its own answer is committed
+        // too, so the difference is visible in a review rather than only in
+        // a test failure. See parity/KNOWN.md.
+        if parity::DIVERGENT_MDSCRUB.contains(&name.as_str()) {
+            let path = dir.join(format!("{name}.rust.md"));
+            std::fs::write(&path, ::mdscrub::scrub(&markdown)).map_err(|e| e.to_string())?;
+            written.push(path.display().to_string());
+        }
+    }
+    Ok(written)
+}
+
 fn chunkmd() -> Written {
     let dir = parity::golden("chunkmd");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/src/lib.rs

@@ -83,6 +83,10 @@ pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<Str
         .map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
 }
 
+/// Cases where `mdscrub` is meant to differ from the Python, each one
+/// argued for in `KNOWN.md`.
+pub const DIVERGENT_MDSCRUB: &[&str] = &["breaks"];
+
 /// Every case file for a tool, in a fixed order.
 pub fn cases(tool: &str) -> Vec<PathBuf> {
     let dir = dir().join("cases").join(tool);