cuekit and cuenorm: three subtitle formats, one cue stream

parity/Cargo.toml

@@ -15,6 +15,7 @@ python = []
 
 [dependencies]
 chunkmd.workspace = true
+cuekit.workspace = true
 mdscrub.workspace = true
 pdftriage.workspace = true
 corpus-core.workspace = true

parity/KNOWN.md

@@ -18,6 +18,26 @@ affected and how often it occurs in practice.
 
 ---
 
+## cuenorm — a cue line with trailing settings still reads
+
+**Original:** the timestamp line is split on the arrow and each half is read
+whole, so `00:00:01,000 --> 00:00:03,000  X1:040 X2:560` gives an end time
+that is not a number and the cue is dropped entirely.
+
+**Rust:** each half is reduced to the one token that could be a timestamp, so
+the trailing text is ignored and the cue reads.
+
+**Why this is acceptable:** it is what makes the same reader take WebVTT,
+where positioning settings after the end time are not a malformation but the
+format — `00:00:01.000 --> 00:00:03.000 line:0 position:20%` is an ordinary
+cue. Without it the tool would need a second parser that differed in one
+line. The divergence can only ever recover a cue that the original discarded;
+there is no input where it changes a cue the original read.
+
+**Measured:** `parity/cases/cuenorm/coordinates.srt` — one cue of two
+recovered. Across the other eight cases, including deliberately malformed
+timestamps, the two agree cue for cue.
+
 ## mdscrub — un-paginated text is not given a page 1 marker
 
 **Python:** `_normalize_marker_pages` ends by wrapping any text it found no

parity/SOURCE.md

@@ -6,6 +6,7 @@ in this directory are the answers that Python gave, at one commit:
 | Source | Commit |
 |---|---|
 | `book-pipeline` | `7499e3257572968b437ecfdc79feccc91b96585b` |
+| `podcast-download-systemd` | `ab20139cd0defb400a0dd9ed9034e03c1d1e356d` |
 
 Regenerating against a different commit will move the goldens. That is the
 point — the diff is the review. Record the new commit here in the same change.
@@ -16,4 +17,9 @@ a checkout lives is a fact about one machine and must not be compiled in:
 ```sh
 CATENA_PARITY_PYTHON=/path/to/venv/bin/python
 CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src
+CATENA_PARITY_NODE=node   # only if it is not already on PATH
 ```
+
+One oracle is JavaScript rather than Python, because the code it compares
+against is. Its three functions are copied into `oracle/cuenorm.mjs` verbatim,
+since the module they live in opens a database connection when it loads.

parity/cases/cuenorm/brackets.srt

@@ -0,0 +1,11 @@
+1
+00:00:01,000 --> 00:00:03,000
+[Music]
+
+2
+00:00:03,000 --> 00:00:05,000
+[Applause] The mill stood. [inaudible]
+
+3
+00:00:05,000 --> 00:00:07,000
+[a] kept [b]

parity/cases/cuenorm/coordinates.srt

@@ -0,0 +1,7 @@
+1
+00:00:01,000 --> 00:00:03,000  X1:040 X2:560 Y1:050 Y2:100
+The mill stood at the northern end.
+
+2
+00:00:03,000 --> 00:00:05,000
+Its wheel was undershot.

parity/cases/cuenorm/crlf.srt

@@ -0,0 +1,8 @@
+1
+00:00:01,000 --> 00:00:03,000
+The mill stood at the northern end,
+where the fall was steepest.
+
+2
+00:00:03,000 --> 00:00:05,000
+Its wheel was undershot.

parity/cases/cuenorm/malformed.srt

@@ -0,0 +1,20 @@
+1
+nonsense --> also nonsense
+this block has unreadable times
+
+2
+00:00:03,000 --> 00:00:05,000
+this one is fine
+
+a block with no arrow at all
+
+3
+00:00:05,000 --> 
+this one has no end time
+
+4
+ --> 00:00:09,000
+this one has no start time
+
+5
+00:00:09,000 --> 00:00:11,000

parity/cases/cuenorm/multiline.srt

@@ -0,0 +1,9 @@
+1
+00:00:01,000 --> 00:00:03,000
+The mill stood at the northern end,
+where the fall of the river was steepest
+and the ice came latest.
+
+2
+00:00:03,000 --> 00:00:05,000
+Its wheel was undershot.

parity/cases/cuenorm/oddtimes.srt

@@ -0,0 +1,19 @@
+1
+02:03,500 --> 02:05,000
+a two part timestamp
+
+2
+00:00:05,000 --> 00:00:07,000,000
+an extra comma in the end time
+
+3
+0:0:8 --> 0:0:9
+unpadded fields
+
+4
+ 00:00:09,000  -->  00:00:11,000 
+padded with spaces
+
+5
+1:2:3.456 --> 1:2:4.5
+a point instead of a comma

parity/cases/cuenorm/plain.srt

@@ -0,0 +1,11 @@
+1
+00:00:01,000 --> 00:00:03,000
+The mill stood at the northern end of the valley.
+
+2
+00:00:03,000 --> 00:00:05,500
+Its wheel was undershot, which was wasteful.
+
+3
+00:00:05,500 --> 00:00:09,250
+Nobody replaced it for two hundred years.

parity/cases/cuenorm/repeats.srt

@@ -0,0 +1,19 @@
+1
+00:00:00,000 --> 00:00:02,000
+the mill stood at the
+
+2
+00:00:01,500 --> 00:00:03,500
+the mill stood at the
+
+3
+00:00:03,000 --> 00:00:05,000
+the mill stood at the
+
+4
+00:00:05,000 --> 00:00:07,000
+northern end of the valley
+
+5
+00:00:07,000 --> 00:00:09,000
+northern end of the valley

parity/cases/cuenorm/tags.srt

@@ -0,0 +1,11 @@
+1
+00:00:01,000 --> 00:00:03,000
+<i>The mill</i> stood at the <b>northern</b> end.
+
+2
+00:00:03,000 --> 00:00:05,000
+<font color="#ffffff">Its wheel was undershot.</font>
+
+3
+00:00:05,000 --> 00:00:07,000
+a < b and c > d

parity/golden/cuenorm/brackets.ndjson

@@ -0,0 +1,2 @@
+{"start":3,"end":5,"text":"The mill stood."}
+{"start":5,"end":7,"text":"kept"}

parity/golden/cuenorm/coordinates.ndjson

@@ -0,0 +1 @@
+{"start":3,"end":5,"text":"Its wheel was undershot."}

parity/golden/cuenorm/crlf.ndjson

@@ -0,0 +1 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end,\r where the fall was steepest.\r \r 2\r 00:00:03,000 --> 00:00:05,000\r Its wheel was undershot."}

parity/golden/cuenorm/malformed.ndjson

@@ -0,0 +1 @@
+{"start":3,"end":5,"text":"this one is fine"}

parity/golden/cuenorm/multiline.ndjson

@@ -0,0 +1,2 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end, where the fall of the river was steepest and the ice came latest."}
+{"start":3,"end":5,"text":"Its wheel was undershot."}

parity/golden/cuenorm/oddtimes.ndjson

@@ -0,0 +1,5 @@
+{"start":123.5,"end":125,"text":"a two part timestamp"}
+{"start":5,"end":7,"text":"an extra comma in the end time"}
+{"start":8,"end":9,"text":"unpadded fields"}
+{"start":9,"end":11,"text":"padded with spaces"}
+{"start":3723.456,"end":3724.5,"text":"a point instead of a comma"}

parity/golden/cuenorm/plain.ndjson

@@ -0,0 +1,3 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end of the valley."}
+{"start":3,"end":5.5,"text":"Its wheel was undershot, which was wasteful."}
+{"start":5.5,"end":9.25,"text":"Nobody replaced it for two hundred years."}

parity/golden/cuenorm/repeats.ndjson

@@ -0,0 +1,2 @@
+{"start":0,"end":5,"text":"the mill stood at the"}
+{"start":5,"end":9,"text":"northern end of the valley"}

parity/golden/cuenorm/tags.ndjson

@@ -0,0 +1,3 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end."}
+{"start":3,"end":5,"text":"Its wheel was undershot."}
+{"start":5,"end":7,"text":"a  d"}

parity/oracle/cuenorm.mjs

@@ -0,0 +1,86 @@
+/**
+ * Parse SubRip the way the podcast archive was indexed, for comparison.
+ *
+ * The three functions below are the originals, copied out of the indexer
+ * verbatim and with their type annotations dropped — the module they live in
+ * opens a database connection when it loads, which an oracle must not do.
+ *
+ *     node oracle/cuenorm.mjs < case.srt
+ *
+ * Writes one JSON cue per line.
+ */
+
+/** Parse an SRT timestamp (00:01:23,456) into seconds. */
+function parseSrtTimestamp(ts) {
+  // SRT format: HH:MM:SS,mmm  (comma separates ms)
+  const cleaned = ts.trim().replace(",", ".");
+  const parts = cleaned.split(":");
+  if (parts.length === 3) {
+    return Number(parts[0]) * 3600 + Number(parts[1]) * 60 + parseFloat(parts[2]);
+  }
+  if (parts.length === 2) {
+    return Number(parts[0]) * 60 + parseFloat(parts[1]);
+  }
+  return NaN;
+}
+
+/** Parse an SRT file content into cues. */
+function parseSrt(content) {
+  const cues = [];
+  const blocks = content.split(/\n\n+/);
+
+  for (const block of blocks) {
+    const lines = block.trim().split("\n");
+    const tsLine = lines.find((l) => l.includes("-->"));
+    if (!tsLine) continue;
+
+    const [startStr, endStr] = tsLine.split("-->").map((s) => s.trim());
+    const startTime = parseSrtTimestamp(startStr);
+    const endTime = parseSrtTimestamp(endStr);
+    if (isNaN(startTime) || isNaN(endTime)) continue;
+
+    const tsIdx = lines.indexOf(tsLine);
+    const text = lines
+      .slice(tsIdx + 1)
+      .join(" ")
+      .replace(/<[^>]+>/g, "")
+      .replace(/\[.*?\]/g, "")
+      .trim();
+
+    if (text) {
+      cues.push({ startTime, endTime, text });
+    }
+  }
+
+  return cues;
+}
+
+/** Merge consecutive cues with identical text. */
+function deduplicateCues(cues) {
+  if (cues.length === 0) return [];
+
+  const result = [{ ...cues[0] }];
+
+  for (let i = 1; i < cues.length; i++) {
+    const prev = result[result.length - 1];
+    const curr = cues[i];
+
+    if (curr.text === prev.text) {
+      prev.endTime = Math.max(prev.endTime, curr.endTime);
+    } else {
+      result.push({ ...curr });
+    }
+  }
+
+  return result;
+}
+
+const chunks = [];
+for await (const chunk of process.stdin) chunks.push(chunk);
+const content = Buffer.concat(chunks).toString("utf-8");
+
+for (const cue of deduplicateCues(parseSrt(content))) {
+  process.stdout.write(
+    JSON.stringify({ start: cue.startTime, end: cue.endTime, text: cue.text }) + "\n",
+  );
+}

parity/src/bin/regen.rs

@@ -19,12 +19,13 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 5] = [
+    let jobs: [Job; 6] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
         ("mdscrub", mdscrub),
         ("pdftriage", pdftriage),
+        ("cuenorm", cuenorm),
     ];
 
     for (name, job) in jobs {
@@ -68,6 +69,21 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn cuenorm() -> Written {
+    let dir = parity::golden("cuenorm");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let mut written = Vec::new();
+    for case in parity::cases_with_extension("cuenorm", "srt") {
+        let content = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
+        let cues = parity::run_node_oracle("cuenorm.mjs", &content)?;
+        let path = dir.join(format!("{}.ndjson", parity::case_name(&case)));
+        std::fs::write(&path, cues).map_err(|e| e.to_string())?;
+        written.push(path.display().to_string());
+    }
+    Ok(written)
+}
+
 fn pdftriage() -> Written {
     let dir = parity::golden("pdftriage");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/src/lib.rs

@@ -35,6 +35,20 @@ pub fn oracle(name: &str) -> PathBuf {
     dir().join("oracle").join(name)
 }
 
+/// Cases where `cuenorm` is meant to differ from the JavaScript it was
+/// ported from, each one argued for in `KNOWN.md`.
+pub const DIVERGENT_CUENORM: &[&str] = &["coordinates"];
+
+/// The JavaScript runtime to run an oracle with.
+pub fn node() -> String {
+    std::env::var("CATENA_PARITY_NODE").unwrap_or_else(|_| "node".to_string())
+}
+
+/// Run a JavaScript oracle over `input` and return its stdout.
+pub fn run_node_oracle(script: &str, input: &str) -> Result<String, String> {
+    run_oracle_in(node(), script, &[], input)
+}
+
 /// The Python interpreter to run an oracle with.
 ///
 /// Taken from `CATENA_PARITY_PYTHON`, because the interpreter that has the
@@ -55,18 +69,28 @@ pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
 
 /// Run an oracle script with arguments.
 pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<String, String> {
+    run_oracle_in(python(), script, args, input)
+}
+
+/// Run an oracle under a named runtime.
+pub fn run_oracle_in(
+    runtime: String,
+    script: &str,
+    args: &[String],
+    input: &str,
+) -> Result<String, String> {
     use std::io::Write as _;
     use std::process::{Command, Stdio};
 
     let script = oracle(script);
-    let mut child = Command::new(python())
+    let mut child = Command::new(&runtime)
         .arg(&script)
         .args(args)
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::inherit())
         .spawn()
-        .map_err(|e| format!("running {} with {}: {e}", script.display(), python()))?;
+        .map_err(|e| format!("running {} with {runtime}: {e}", script.display()))?;
     child
         .stdin
         .take()
@@ -85,12 +109,17 @@ pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<Str
 
 /// Every case file for a tool, in a fixed order.
 pub fn cases(tool: &str) -> Vec<PathBuf> {
+    cases_with_extension(tool, "md")
+}
+
+/// Every case file for a tool with a given extension, in a fixed order.
+pub fn cases_with_extension(tool: &str, extension: &str) -> Vec<PathBuf> {
     let dir = dir().join("cases").join(tool);
     let mut found: Vec<PathBuf> = std::fs::read_dir(&dir)
         .unwrap_or_else(|e| panic!("reading {}: {e}", dir.display()))
         .filter_map(Result::ok)
         .map(|entry| entry.path())
-        .filter(|path| path.extension().is_some_and(|ext| ext == "md"))
+        .filter(|path| path.extension().is_some_and(|ext| ext == extension))
         .collect();
     found.sort();
     found

parity/tests/cuenorm.rs

@@ -0,0 +1,99 @@
+//! Cue streams must match the JavaScript the archive was indexed with.
+//!
+//! A cue that shifts by a tenth of a second is a search result that lands in
+//! the wrong sentence; a cue that is dropped is a sentence that cannot be
+//! found at all. Both are invisible until somebody searches for exactly that
+//! line.
+
+use serde_json::Value;
+
+/// A cue, as the two sides can be compared.
+///
+/// Not as JSON values: JavaScript writes a whole number as `5` where Rust
+/// writes `5.0`, and the two parse to different kinds of JSON number for the
+/// same quantity.
+type Cue = (f64, f64, String);
+
+fn cues(text: &str) -> Vec<Cue> {
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let cue: Value = serde_json::from_str(line).expect("valid JSON");
+            (
+                cue["start"].as_f64().expect("a start time"),
+                cue["end"].as_f64().expect("an end time"),
+                cue["text"].as_str().expect("text").to_string(),
+            )
+        })
+        .collect()
+}
+
+fn ours(content: &str) -> Vec<Cue> {
+    cuekit::merge_repeats(&cuekit::parse(content))
+        .iter()
+        .map(|cue| (cue.start, cue.end, cue.text.clone()))
+        .collect()
+}
+
+#[test]
+fn every_case_parses_the_same_as_the_original() {
+    let cases = parity::cases_with_extension("cuenorm", "srt");
+    assert!(!cases.is_empty(), "no cases found");
+
+    for case in cases {
+        let name = parity::case_name(&case);
+        if parity::DIVERGENT_CUENORM.contains(&name.as_str()) {
+            continue;
+        }
+        let content = std::fs::read_to_string(&case).expect("a readable case");
+        let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson")));
+        let ours = ours(&content);
+
+        for (at, (ours, theirs)) in ours.iter().zip(&theirs).enumerate() {
+            assert_eq!(ours, theirs, "{name} cue {at} differs");
+        }
+
+        assert_eq!(
+            ours.len(),
+            theirs.len(),
+            "{name}: the original found {} cues, this found {}",
+            theirs.len(),
+            ours.len()
+        );
+    }
+}
+
+/// The divergence is that a cue line carrying screen coordinates after the
+/// end time still reads. The original drops such a cue whole.
+#[test]
+fn the_documented_divergence_is_exactly_what_was_argued_for() {
+    for name in parity::DIVERGENT_CUENORM {
+        let case = parity::dir()
+            .join("cases/cuenorm")
+            .join(format!("{name}.srt"));
+        let content = std::fs::read_to_string(&case).expect("a readable case");
+        let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson")));
+        let ours = ours(&content);
+        assert!(
+            ours.len() > theirs.len(),
+            "{name} no longer recovers more than the original — update KNOWN.md"
+        );
+        // Everything the original did find, this finds too, unchanged.
+        for cue in &theirs {
+            assert!(ours.contains(cue), "{name} lost a cue the original kept");
+        }
+    }
+}
+
+/// Catches a golden file drifting away from an original that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_files_still_match_the_live_original() {
+    for case in parity::cases_with_extension("cuenorm", "srt") {
+        let name = parity::case_name(&case);
+        let content = std::fs::read_to_string(&case).expect("a readable case");
+        let live = parity::run_node_oracle("cuenorm.mjs", &content).expect("the oracle runs");
+        let golden = parity::read_golden_text(&format!("cuenorm/{name}.ndjson"));
+        assert_eq!(live, golden, "golden file for {name} is stale");
+    }
+}