cuekit and cuenorm: three subtitle formats, one cue stream

Cargo.lock

@@ -166,6 +166,27 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "cuekit"
+version = "0.1.0"
+dependencies = [
+ "corpus-core",
+ "regex",
+ "schemars",
+ "serde",
+ "serde_json",
+ "textkit",
+]
+
+[[package]]
+name = "cuenorm"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "corpus-core",
+ "cuekit",
+]
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.20"
@@ -246,6 +267,7 @@ version = "0.1.0"
 dependencies = [
  "chunkmd",
  "corpus-core",
+ "cuekit",
  "mdscrub",
  "pdftriage",
  "serde_json",

Cargo.toml

@@ -11,10 +11,12 @@ repository = "https://git.pumpin.online/funkpower/catena"
 
 [workspace.dependencies]
 corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
+cuekit = { path = "crates/cuekit", version = "0.1.0" }
 textkit = { path = "crates/textkit", version = "0.1.0" }
 chunkmd = { path = "crates/chunkmd", version = "0.1.0" }
 mdscrub = { path = "crates/mdscrub", version = "0.1.0" }
 pdftriage = { path = "crates/pdftriage", version = "0.1.0" }
+cuenorm = { path = "crates/cuenorm", version = "0.1.0" }
 
 clap = { version = "4", features = ["derive"] }
 fancy-regex = "0.16"

crates/corpus-core/src/chunk.rs

@@ -26,18 +26,12 @@ pub struct Chunk {
     /// What part of the work the passage belongs to.
     pub role: Role,
     /// Seconds from the start of the recording, for timed sources.
-    #[serde(
+    #[serde(default, with = "crate::finite::option")]
-        default,
+    #[schemars(with = "Option<f64>")]
-        serialize_with = "crate::float::serialize_option",
-        deserialize_with = "crate::float::deserialize_option"
-    )]
     pub start_time: Option<f64>,
     /// Seconds from the start of the recording, for timed sources.
-    #[serde(
+    #[serde(default, with = "crate::finite::option")]
-        default,
+    #[schemars(with = "Option<f64>")]
-        serialize_with = "crate::float::serialize_option",
-        deserialize_with = "crate::float::deserialize_option"
-    )]
     pub end_time: Option<f64>,
     /// Set tags; see [`flags`]. Sorted and deduplicated on the way in and on
     /// the way out, so the same set of flags is always the same bytes

crates/corpus-core/src/finite.rs

@@ -0,0 +1,57 @@
+//! Serde plumbing for floats that JSON can actually carry.
+//!
+//! Use it as `#[serde(with = "corpus_core::finite")]` on an `f64`, or
+//! `#[serde(default, with = "corpus_core::finite::option")]` on an
+//! `Option<f64>`.
+//!
+//! JSON has no spelling for NaN or infinity, and `serde_json` writes them as
+//! `null`. That is a silent corruption in both directions: a score written as
+//! `null` fails to read back and kills the stream at the consumer, and a
+//! timestamp written as `null` reads back as "no timestamp" with nothing to
+//! say it was ever anything else. So a non-finite number is refused where it
+//! is produced, which is the only place anyone can still fix it.
+
+use serde::{Deserialize, Deserializer, Serializer, de::Error as _, ser::Error as _};
+
+/// Write a float, refusing one JSON cannot carry.
+pub fn serialize<S: Serializer>(value: &f64, out: S) -> Result<S::Ok, S::Error> {
+    if !value.is_finite() {
+        return Err(S::Error::custom(format!(
+            "{value} cannot be written as JSON"
+        )));
+    }
+    out.serialize_f64(*value)
+}
+
+/// Read a float, refusing one that is not finite.
+pub fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result<f64, D::Error> {
+    let value = f64::deserialize(input)?;
+    if !value.is_finite() {
+        return Err(D::Error::custom(format!("{value} is not a finite number")));
+    }
+    Ok(value)
+}
+
+/// The same, for a field that may be absent.
+pub mod option {
+    use serde::{Deserialize, Deserializer, Serializer, de::Error as _};
+
+    /// Write an optional float, refusing one JSON cannot carry.
+    pub fn serialize<S: Serializer>(value: &Option<f64>, out: S) -> Result<S::Ok, S::Error> {
+        match value {
+            Some(value) => super::serialize(value, out),
+            None => out.serialize_none(),
+        }
+    }
+
+    /// Read an optional float, refusing one that is not finite.
+    pub fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result<Option<f64>, D::Error> {
+        let value = Option::<f64>::deserialize(input)?;
+        match value {
+            Some(value) if !value.is_finite() => {
+                Err(D::Error::custom(format!("{value} is not a finite number")))
+            }
+            value => Ok(value),
+        }
+    }
+}

crates/corpus-core/src/float.rs

@@ -1,49 +0,0 @@
-//! Serde plumbing for floats that JSON can actually carry.
-//!
-//! JSON has no spelling for NaN or infinity, and `serde_json` writes them as
-//! `null`. That is a silent corruption in both directions: a score written as
-//! `null` fails to read back and kills the stream at the consumer, and a
-//! timestamp written as `null` reads back as "no timestamp" with nothing to
-//! say it was ever anything else. So a non-finite number is refused where it
-//! is produced, which is the only place anyone can still fix it.
-
-use serde::{Deserialize, Deserializer, Serializer, de::Error as _, ser::Error as _};
-
-pub(crate) fn serialize<S: Serializer>(value: &f64, out: S) -> Result<S::Ok, S::Error> {
-    if !value.is_finite() {
-        return Err(S::Error::custom(format!(
-            "{value} cannot be written as JSON"
-        )));
-    }
-    out.serialize_f64(*value)
-}
-
-pub(crate) fn serialize_option<S: Serializer>(
-    value: &Option<f64>,
-    out: S,
-) -> Result<S::Ok, S::Error> {
-    match value {
-        Some(value) => serialize(value, out),
-        None => out.serialize_none(),
-    }
-}
-
-pub(crate) fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result<f64, D::Error> {
-    let value = f64::deserialize(input)?;
-    if !value.is_finite() {
-        return Err(D::Error::custom(format!("{value} is not a finite number")));
-    }
-    Ok(value)
-}
-
-pub(crate) fn deserialize_option<'de, D: Deserializer<'de>>(
-    input: D,
-) -> Result<Option<f64>, D::Error> {
-    let value = Option::<f64>::deserialize(input)?;
-    match value {
-        Some(value) if !value.is_finite() => {
-            Err(D::Error::custom(format!("{value} is not a finite number")))
-        }
-        value => Ok(value),
-    }
-}

crates/corpus-core/src/hit.rs

@@ -22,10 +22,8 @@ pub struct Hit {
     /// Position within the channel's own ranking, from one.
     pub rank: u32,
     /// The channel's raw score, on whatever scale that channel uses.
-    #[serde(
+    #[serde(with = "crate::finite")]
-        serialize_with = "crate::float::serialize",
+    #[schemars(with = "f64")]
-        deserialize_with = "crate::float::deserialize"
-    )]
     pub score: f64,
     /// Anything the producer wants to carry through fusion untouched.
     ///

crates/corpus-core/src/lib.rs

@@ -11,7 +11,7 @@
 pub mod chunk;
 pub mod doc;
 pub mod exit;
-mod float;
+pub mod finite;
 pub mod hit;
 pub mod ndjson;
 

crates/cuekit/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "cuekit"
+description = "The cue model everything timed shares, and the subtitle formats it is read from."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+corpus-core.workspace = true
+regex.workspace = true
+schemars.workspace = true
+serde.workspace = true
+textkit.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
+
+[lints]
+workspace = true

crates/cuekit/src/ass.rs

@@ -0,0 +1,179 @@
+//! Advanced SubStation Alpha, which is a spreadsheet pretending to be a
+//! subtitle file.
+//!
+//! Events are rows under a `Format:` line that names the columns, so the
+//! column order has to be read rather than assumed — a file written by one
+//! tool puts the text last after nine fields, and one written by another puts
+//! it last after eight.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+use crate::cue::Cue;
+use crate::text;
+
+/// An override block: `{\an8}`, `{\pos(120,400)}`, `{\i1}`.
+static OVERRIDE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\{[^}]*\}").expect("valid"));
+
+/// A drawing command block, which is a shape rather than speech.
+static DRAWING: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)\{[^}]*\\p[1-9]").expect("valid"));
+
+/// Parse an ASS or SSA file.
+pub fn parse(content: &str) -> Vec<Cue> {
+    let mut columns: Option<Vec<String>> = None;
+    let mut in_events = false;
+    let mut cues = Vec::new();
+
+    for line in content.split('\n') {
+        let line = line.trim_matches(textkit::is_space);
+        if line.starts_with('[') {
+            in_events = line.eq_ignore_ascii_case("[events]");
+            continue;
+        }
+        if !in_events {
+            continue;
+        }
+        if let Some(format) = line.strip_prefix("Format:") {
+            columns = Some(
+                format
+                    .split(',')
+                    .map(|name| name.trim_matches(textkit::is_space).to_lowercase())
+                    .collect(),
+            );
+            continue;
+        }
+        let Some(event) = line.strip_prefix("Dialogue:") else {
+            continue;
+        };
+        let Some(columns) = columns.as_ref() else {
+            continue;
+        };
+        if let Some(cue) = dialogue(event, columns) {
+            cues.push(cue);
+        }
+    }
+    cues
+}
+
+fn dialogue(event: &str, columns: &[String]) -> Option<Cue> {
+    let start_at = columns.iter().position(|name| name == "start")?;
+    let end_at = columns.iter().position(|name| name == "end")?;
+    let text_at = columns.iter().position(|name| name == "text")?;
+
+    // The text field is last and may itself contain commas, so the row is
+    // split only as many times as there are columns before it.
+    let mut fields: Vec<&str> = event.splitn(columns.len(), ',').collect();
+    if fields.len() < columns.len() {
+        return None;
+    }
+    let text = if text_at == columns.len() - 1 {
+        fields.pop().unwrap_or("")
+    } else {
+        fields.get(text_at).copied().unwrap_or("")
+    };
+
+    let start = timestamp(fields.get(start_at)?.trim_matches(textkit::is_space));
+    let end = timestamp(fields.get(end_at)?.trim_matches(textkit::is_space));
+    if !start.is_finite() || !end.is_finite() {
+        return None;
+    }
+
+    // A drawing command is a shape, not speech, and reads as a string of
+    // coordinates if it is kept.
+    if DRAWING.is_match(text) {
+        return None;
+    }
+    let text = OVERRIDE.replace_all(text, "");
+    let text = text
+        .replace("\\N", " ")
+        .replace("\\n", " ")
+        .replace("\\h", " ");
+    let text = text::clean(&text);
+    (!text.is_empty()).then(|| Cue::new(start, end, text))
+}
+
+/// Seconds from an `H:MM:SS.cc` timestamp.
+pub fn timestamp(text: &str) -> f64 {
+    let parts: Vec<&str> = text.split(':').collect();
+    match parts.as_slice() {
+        [hours, minutes, seconds] => {
+            crate::jsnum::number(hours) * 3600.0
+                + crate::jsnum::number(minutes) * 60.0
+                + crate::jsnum::parse_float(seconds)
+        }
+        [minutes, seconds] => {
+            crate::jsnum::number(minutes) * 60.0 + crate::jsnum::parse_float(seconds)
+        }
+        _ => f64::NAN,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FILE: &str = "[Script Info]\n\
+        Title: A Recording\n\
+        \n\
+        [V4+ Styles]\n\
+        Format: Name, Fontname\n\
+        Style: Default,Arial\n\
+        \n\
+        [Events]\n\
+        Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n\
+        Dialogue: 0,0:00:01.00,0:00:03.00,Default,,0,0,0,,The mill stood at the northern end.\n\
+        Dialogue: 0,0:00:03.00,0:00:05.50,Default,,0,0,0,,{\\an8}Its wheel was undershot,\\Nwhich was wasteful.\n";
+
+    #[test]
+    fn events_become_cues() {
+        let cues = parse(FILE);
+        assert_eq!(cues.len(), 2);
+        assert_eq!(cues[0].start, 1.0);
+        assert_eq!(cues[0].end, 3.0);
+        assert_eq!(cues[0].text, "The mill stood at the northern end.");
+    }
+
+    #[test]
+    fn override_blocks_and_line_breaks_come_out() {
+        let cues = parse(FILE);
+        assert_eq!(cues[1].text, "Its wheel was undershot, which was wasteful.");
+    }
+
+    #[test]
+    fn a_comma_in_the_text_is_not_a_column_break() {
+        let cues = parse(FILE);
+        assert!(cues[1].text.contains(','), "{:?}", cues[1].text);
+    }
+
+    #[test]
+    fn a_drawing_command_is_not_speech() {
+        let file = FILE.to_string()
+            + "Dialogue: 0,0:00:06.00,0:00:07.00,Default,,0,0,0,,{\\p1}m 0 0 l 100 0 100 100\n";
+        assert_eq!(parse(&file).len(), 2);
+    }
+
+    #[test]
+    fn a_row_outside_the_events_section_is_ignored() {
+        let file = "[Script Info]\nDialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,no\n";
+        assert!(parse(file).is_empty());
+    }
+
+    #[test]
+    fn a_column_order_of_its_own_is_honoured() {
+        let file = "[Events]\n\
+            Format: Start, End, Text\n\
+            Dialogue: 0:00:01.00,0:00:02.00,Reordered, with a comma\n";
+        let cues = parse(file);
+        assert_eq!(cues.len(), 1);
+        assert_eq!(cues[0].text, "Reordered, with a comma");
+    }
+
+    #[test]
+    fn a_timestamp_is_read_in_seconds() {
+        assert_eq!(timestamp("0:00:01.50"), 1.5);
+        assert_eq!(timestamp("1:02:03.04"), 3723.04);
+        assert!(timestamp("rubbish").is_nan());
+    }
+}

crates/cuekit/src/cue.rs

@@ -0,0 +1,82 @@
+//! One line of a transcript, and when it was said.
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+/// A subtitle cue: a span of time and what was said in it.
+///
+/// Deliberately smaller than a chunk. A cue is what a subtitle file holds —
+/// a few seconds, often half a sentence — and turning cues into passages
+/// worth retrieving is a later stage's job.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Cue {
+    /// Record version.
+    #[serde(default = "version")]
+    pub v: u32,
+    /// Seconds from the start of the recording.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub start: f64,
+    /// Seconds from the start of the recording.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub end: f64,
+    /// What was said, with the markup taken out.
+    pub text: String,
+}
+
+fn version() -> u32 {
+    corpus_core::V
+}
+
+impl Cue {
+    /// A cue.
+    pub fn new(start: f64, end: f64, text: impl Into<String>) -> Self {
+        Self {
+            v: corpus_core::V,
+            start,
+            end,
+            text: text.into(),
+        }
+    }
+
+    /// How long the cue lasts. Negative if the file says it ends before it
+    /// starts, which some do.
+    pub fn seconds(&self) -> f64 {
+        self.end - self.start
+    }
+}
+
+impl corpus_core::Record for Cue {
+    fn version(&self) -> u32 {
+        self.v
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn every_field_is_emitted() {
+        let cue = Cue::new(12.5, 15.0, "the mill stood at the northern end");
+        assert_eq!(
+            serde_json::to_string(&cue).unwrap(),
+            r#"{"v":1,"start":12.5,"end":15.0,"text":"the mill stood at the northern end"}"#
+        );
+    }
+
+    #[test]
+    fn a_cue_reads_back_identical() {
+        let cue = Cue::new(0.0, 1.0 / 3.0, "x");
+        let text = serde_json::to_string(&cue).unwrap();
+        let back: Cue = serde_json::from_str(&text).unwrap();
+        assert_eq!(back.end.to_bits(), cue.end.to_bits());
+    }
+
+    #[test]
+    fn a_time_that_json_cannot_carry_is_refused() {
+        let cue = Cue::new(f64::NAN, 1.0, "x");
+        assert!(serde_json::to_string(&cue).is_err());
+    }
+}

crates/cuekit/src/dedup.rs

@@ -0,0 +1,66 @@
+//! Collapsing the repetition automatic captioning produces.
+
+use crate::cue::Cue;
+
+/// Merge consecutive cues whose text is identical, keeping the earliest start
+/// and the latest end.
+///
+/// Automatic captioning emits a rolling window: the same line appears in cue
+/// after cue as the window slides, so a ten-minute stretch of speech can
+/// arrive as three times as many cues as there are sentences. Indexed as they
+/// come, every one of those is a separate hit for the same words at almost
+/// the same timestamp, which buries everything else in a result list.
+pub fn merge_repeats(cues: &[Cue]) -> Vec<Cue> {
+    let mut merged: Vec<Cue> = Vec::with_capacity(cues.len());
+    for cue in cues {
+        match merged.last_mut() {
+            Some(previous) if previous.text == cue.text => {
+                previous.end = previous.end.max(cue.end);
+            }
+            _ => merged.push(cue.clone()),
+        }
+    }
+    merged
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_repeated_line_becomes_one_cue_spanning_both() {
+        let cues = [
+            Cue::new(0.0, 2.0, "the mill stood"),
+            Cue::new(1.0, 3.0, "the mill stood"),
+            Cue::new(3.0, 4.0, "at the northern end"),
+        ];
+        let merged = merge_repeats(&cues);
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].start, 0.0);
+        assert_eq!(merged[0].end, 3.0);
+        assert_eq!(merged[1].text, "at the northern end");
+    }
+
+    #[test]
+    fn a_line_that_comes_back_later_is_kept_separately() {
+        let cues = [
+            Cue::new(0.0, 1.0, "yes"),
+            Cue::new(1.0, 2.0, "no"),
+            Cue::new(2.0, 3.0, "yes"),
+        ];
+        assert_eq!(merge_repeats(&cues).len(), 3);
+    }
+
+    #[test]
+    fn an_end_that_goes_backwards_does_not_shorten_the_merged_cue() {
+        let cues = [Cue::new(0.0, 9.0, "x"), Cue::new(1.0, 2.0, "x")];
+        let merged = merge_repeats(&cues);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].end, 9.0);
+    }
+
+    #[test]
+    fn nothing_in_nothing_out() {
+        assert!(merge_repeats(&[]).is_empty());
+    }
+}

crates/cuekit/src/jsnum.rs

@@ -0,0 +1,176 @@
+//! Reading numbers the way the JavaScript this was ported from reads them.
+//!
+//! The timestamp parser it came from calls `Number()` on the hours and the
+//! minutes and `parseFloat()` on the seconds. The two disagree about almost
+//! everything an ill-formed timestamp can contain — `Number("")` is zero
+//! where `parseFloat("")` is not a number, `Number("1a")` is not a number
+//! where `parseFloat("1a")` is one — and a subtitle file that has been cut,
+//! concatenated or hand-edited produces exactly those. Reproducing both is
+//! what makes a malformed file drop the same cues here as there.
+
+/// What JavaScript's `Number(text)` gives.
+///
+/// Whitespace is trimmed, an empty string is zero, the radix prefixes are
+/// honoured, and anything else that is not a complete numeric literal is not
+/// a number. Note that `Number` accepts `Infinity` but not `NaN`, and neither
+/// of the spellings Rust's own parser accepts for them.
+pub fn number(text: &str) -> f64 {
+    let text = text.trim_matches(is_js_space);
+    if text.is_empty() {
+        return 0.0;
+    }
+    if let Some(value) = radix_literal(text) {
+        return value;
+    }
+    let body = text.strip_prefix(['+', '-']).unwrap_or(text);
+    // Rust's parser accepts spellings JavaScript's does not.
+    if body.eq_ignore_ascii_case("inf")
+        || body.eq_ignore_ascii_case("nan")
+        || body.eq_ignore_ascii_case("infinity")
+    {
+        return if body.eq_ignore_ascii_case("infinity") {
+            match text.starts_with('-') {
+                true => f64::NEG_INFINITY,
+                false => f64::INFINITY,
+            }
+        } else {
+            f64::NAN
+        };
+    }
+    text.parse().unwrap_or(f64::NAN)
+}
+
+/// What JavaScript's `parseFloat(text)` gives: the longest decimal number at
+/// the front, ignoring whatever follows, and not a number when there is none.
+pub fn parse_float(text: &str) -> f64 {
+    let text = text.trim_start_matches(is_js_space);
+    let prefix = decimal_prefix(text);
+    if prefix.is_empty() {
+        return f64::NAN;
+    }
+    prefix.parse().unwrap_or(f64::NAN)
+}
+
+/// The longest leading run that is a decimal number, by JavaScript's grammar.
+fn decimal_prefix(text: &str) -> &str {
+    let bytes = text.as_bytes();
+    let mut at = 0;
+    if matches!(bytes.first(), Some(b'+' | b'-')) {
+        at = 1;
+    }
+    if text[at..].starts_with("Infinity") {
+        return &text[..at + "Infinity".len()];
+    }
+
+    let mut end = at;
+    let digits = |from: usize| {
+        let mut to = from;
+        while to < bytes.len() && bytes[to].is_ascii_digit() {
+            to += 1;
+        }
+        to
+    };
+    end = digits(end);
+    if end < bytes.len() && bytes[end] == b'.' {
+        end = digits(end + 1);
+    }
+    // A run with no digits at all on either side of the point is not a
+    // number: `.` and `+.` are nothing, `.5` and `5.` are numbers.
+    if !text[at..end].chars().any(|c| c.is_ascii_digit()) {
+        return "";
+    }
+    if end < bytes.len() && (bytes[end] | 0x20) == b'e' {
+        let mut exponent = end + 1;
+        if matches!(bytes.get(exponent), Some(b'+' | b'-')) {
+            exponent += 1;
+        }
+        let after = digits(exponent);
+        if after > exponent {
+            end = after;
+        }
+    }
+    &text[..end]
+}
+
+fn radix_literal(text: &str) -> Option<f64> {
+    let (radix, digits) = match text.get(..2)?.to_ascii_lowercase().as_str() {
+        "0x" => (16, &text[2..]),
+        "0o" => (8, &text[2..]),
+        "0b" => (2, &text[2..]),
+        _ => return None,
+    };
+    if digits.is_empty() {
+        return Some(f64::NAN);
+    }
+    Some(match u64::from_str_radix(digits, radix) {
+        Ok(value) => value as f64,
+        Err(_) => f64::NAN,
+    })
+}
+
+/// JavaScript's whitespace for the purpose of trimming a numeric string:
+/// Unicode whitespace plus the byte-order mark.
+fn is_js_space(c: char) -> bool {
+    c.is_whitespace() || c == '\u{feff}'
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn number_reads_ordinary_numbers() {
+        assert_eq!(number("12"), 12.0);
+        assert_eq!(number("  07 "), 7.0);
+        assert_eq!(number("1.5"), 1.5);
+        assert_eq!(number("1e2"), 100.0);
+        assert_eq!(number("-3"), -3.0);
+    }
+
+    #[test]
+    fn number_of_nothing_is_zero() {
+        assert_eq!(number(""), 0.0);
+        assert_eq!(number("   "), 0.0);
+    }
+
+    #[test]
+    fn number_refuses_a_partial_literal() {
+        assert!(number("1a").is_nan());
+        assert!(number("12:").is_nan());
+        assert!(number("--1").is_nan());
+    }
+
+    #[test]
+    fn number_does_not_accept_the_rust_spellings() {
+        assert!(number("inf").is_nan());
+        assert!(number("NaN").is_nan());
+        assert_eq!(number("Infinity"), f64::INFINITY);
+        assert_eq!(number("-Infinity"), f64::NEG_INFINITY);
+    }
+
+    #[test]
+    fn number_honours_the_radix_prefixes() {
+        assert_eq!(number("0x10"), 16.0);
+        assert_eq!(number("0b101"), 5.0);
+        assert_eq!(number("0o17"), 15.0);
+    }
+
+    #[test]
+    fn parse_float_takes_the_longest_prefix() {
+        assert_eq!(parse_float("23.456"), 23.456);
+        assert_eq!(parse_float("23.456abc"), 23.456);
+        assert_eq!(parse_float("  1.5xyz"), 1.5);
+        assert_eq!(parse_float("5."), 5.0);
+        assert_eq!(parse_float(".5"), 0.5);
+        assert_eq!(parse_float("1e3x"), 1000.0);
+        assert_eq!(parse_float("1e"), 1.0);
+    }
+
+    #[test]
+    fn parse_float_of_nothing_is_not_a_number() {
+        assert!(parse_float("").is_nan());
+        assert!(parse_float("abc").is_nan());
+        assert!(parse_float(".").is_nan());
+        assert!(parse_float("+").is_nan());
+    }
+}

crates/cuekit/src/lib.rs

@@ -0,0 +1,114 @@
+//! The cue model everything timed shares, and the subtitle formats it is
+//! read from.
+//!
+//! Three formats go in — SubRip, WebVTT and SubStation Alpha — and one cue
+//! stream comes out. Everything downstream of this works on cues and never
+//! has to know which of the three a recording arrived with.
+//!
+//! The SubRip reader is a port, quirks kept, of the parser a podcast archive
+//! was indexed with; the other two are new, and are written to the same
+//! shape. Where the port deliberately differs, `parity/KNOWN.md` says so.
+
+pub mod ass;
+pub mod cue;
+pub mod dedup;
+pub mod jsnum;
+pub mod srt;
+pub mod text;
+
+pub use cue::Cue;
+pub use dedup::merge_repeats;
+
+/// Which subtitle format a file is written in.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Format {
+    /// SubRip.
+    SubRip,
+    /// WebVTT, which this reads with the SubRip reader.
+    WebVtt,
+    /// SubStation Alpha, advanced or otherwise.
+    SubStationAlpha,
+}
+
+impl Format {
+    /// What this format is called.
+    pub fn name(self) -> &'static str {
+        match self {
+            Format::SubRip => "srt",
+            Format::WebVtt => "vtt",
+            Format::SubStationAlpha => "ass",
+        }
+    }
+}
+
+/// Work out which format a file is, from its content rather than its name.
+///
+/// A subtitle file arrives named for the format someone expected it to be in.
+/// The content says what it actually is, and the three are easy to tell
+/// apart: WebVTT announces itself, SubStation Alpha has sections in square
+/// brackets, and SubRip has neither.
+pub fn detect(content: &str) -> Format {
+    let head = content.trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}');
+    if head.starts_with("WEBVTT") {
+        return Format::WebVtt;
+    }
+    for line in head.split('\n').take(40) {
+        let line = line.trim_matches(textkit::is_space);
+        if line.starts_with("[Script Info]") || line.eq_ignore_ascii_case("[events]") {
+            return Format::SubStationAlpha;
+        }
+        if line.starts_with("Dialogue:") || line.starts_with("ScriptType:") {
+            return Format::SubStationAlpha;
+        }
+    }
+    Format::SubRip
+}
+
+/// Read a subtitle file of any supported format into cues.
+pub fn parse(content: &str) -> Vec<Cue> {
+    parse_as(content, detect(content))
+}
+
+/// Read a subtitle file that is known to be in `format`.
+pub fn parse_as(content: &str, format: Format) -> Vec<Cue> {
+    match format {
+        Format::SubRip | Format::WebVtt => srt::parse(content),
+        Format::SubStationAlpha => ass::parse(content),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_file_is_recognised_by_what_is_in_it() {
+        assert_eq!(
+            detect("WEBVTT\n\n00:00.000 --> 00:01.000\nx"),
+            Format::WebVtt
+        );
+        assert_eq!(detect("\u{feff}WEBVTT\n"), Format::WebVtt);
+        assert_eq!(detect("[Script Info]\nTitle: x\n"), Format::SubStationAlpha);
+        assert_eq!(
+            detect("[Events]\nFormat: Start, End, Text\n"),
+            Format::SubStationAlpha
+        );
+        assert_eq!(
+            detect("1\n00:00:01,000 --> 00:00:02,000\nx\n"),
+            Format::SubRip
+        );
+        assert_eq!(detect(""), Format::SubRip);
+    }
+
+    #[test]
+    fn all_three_formats_give_the_same_cue() {
+        let srt = "1\n00:00:01,000 --> 00:00:03,000\nThe mill stood.\n";
+        let vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:03.000\nThe mill stood.\n";
+        let ass = "[Events]\nFormat: Start, End, Text\n\
+                   Dialogue: 0:00:01.00,0:00:03.00,The mill stood.\n";
+        let expected = vec![Cue::new(1.0, 3.0, "The mill stood.")];
+        assert_eq!(parse(srt), expected);
+        assert_eq!(parse(vtt), expected);
+        assert_eq!(parse(ass), expected);
+    }
+}

crates/cuekit/src/srt.rs

@@ -0,0 +1,138 @@
+//! SubRip, and the WebVTT that is close enough to share its reader.
+//!
+//! A port of the parser the podcast archive was indexed with, quirks kept:
+//! the timestamp line is found by looking for an arrow anywhere in the block
+//! rather than at a fixed position, only the first comma in a timestamp
+//! becomes a point, and a block whose times do not read as numbers is
+//! dropped whole rather than guessed at.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+use crate::cue::Cue;
+use crate::jsnum;
+use crate::text;
+
+/// Blocks are separated by one or more blank lines.
+static BLOCKS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n\n+").expect("a valid pattern"));
+
+/// Parse a SubRip file.
+pub fn parse(content: &str) -> Vec<Cue> {
+    let mut cues = Vec::new();
+    for block in BLOCKS.split(content) {
+        let lines: Vec<&str> = block.trim_matches(textkit::is_space).split('\n').collect();
+        let Some(at) = lines.iter().position(|line| line.contains("-->")) else {
+            continue;
+        };
+        let Some((start, end)) = times(lines[at]) else {
+            continue;
+        };
+        let text = text::clean(&lines[at + 1..].join(" "));
+        if !text.is_empty() {
+            cues.push(Cue::new(start, end, text));
+        }
+    }
+    cues
+}
+
+/// Read a timestamp line, or `None` if either end is not a number.
+///
+/// Each half is reduced to the one token that could be a timestamp, which is
+/// what lets the same reader take a WebVTT cue line — where the end time is
+/// followed by positioning settings — and an old SubRip file with screen
+/// coordinates after it. See `parity/KNOWN.md`: the original reads each half
+/// whole and drops such a cue entirely.
+fn times(line: &str) -> Option<(f64, f64)> {
+    let (before, after) = line.split_once("-->")?;
+    let start = timestamp(before.split_whitespace().next_back().unwrap_or(""));
+    let end = timestamp(after.split_whitespace().next().unwrap_or(""));
+    (start.is_finite() && end.is_finite()).then_some((start, end))
+}
+
+/// Seconds from a `HH:MM:SS,mmm` or `MM:SS.mmm` timestamp.
+///
+/// Only the first comma is replaced, exactly as the original does — a
+/// timestamp written with a comma for the thousands separator as well as for
+/// the decimal point therefore parses as far as the second comma and no
+/// further.
+pub fn timestamp(text: &str) -> f64 {
+    let text = text.trim_matches(textkit::is_space);
+    let cleaned = text.replacen(',', ".", 1);
+    let parts: Vec<&str> = cleaned.split(':').collect();
+    match parts.len() {
+        3 => {
+            jsnum::number(parts[0]) * 3600.0
+                + jsnum::number(parts[1]) * 60.0
+                + jsnum::parse_float(parts[2])
+        }
+        2 => jsnum::number(parts[0]) * 60.0 + jsnum::parse_float(parts[1]),
+        _ => f64::NAN,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_timestamp_is_read_in_seconds() {
+        assert_eq!(timestamp("00:01:23,456"), 83.456);
+        assert_eq!(timestamp("01:00:00,000"), 3600.0);
+        assert_eq!(timestamp("02:03.500"), 123.5);
+    }
+
+    #[test]
+    fn a_timestamp_that_is_not_one_is_not_a_number() {
+        assert!(timestamp("").is_nan());
+        assert!(timestamp("nonsense").is_nan());
+        assert!(timestamp("00:00:00:00").is_nan());
+    }
+
+    #[test]
+    fn a_file_parses_into_cues() {
+        let srt = "1\n00:00:01,000 --> 00:00:03,000\nThe mill stood at the northern end.\n\n\
+                   2\n00:00:03,000 --> 00:00:05,500\nIts wheel was undershot,\nwhich was wasteful.\n";
+        let cues = parse(srt);
+        assert_eq!(cues.len(), 2);
+        assert_eq!(cues[0].start, 1.0);
+        assert_eq!(cues[0].end, 3.0);
+        assert_eq!(cues[0].text, "The mill stood at the northern end.");
+        assert_eq!(cues[1].text, "Its wheel was undershot, which was wasteful.");
+    }
+
+    #[test]
+    fn a_block_with_no_arrow_is_skipped() {
+        assert!(parse("WEBVTT\n\nsome preamble\n").is_empty());
+    }
+
+    #[test]
+    fn a_block_with_unreadable_times_is_dropped_whole() {
+        let srt = "1\nnonsense --> also nonsense\ntext\n\n\
+                   2\n00:00:03,000 --> 00:00:05,000\nkept\n";
+        let cues = parse(srt);
+        assert_eq!(cues.len(), 1);
+        assert_eq!(cues[0].text, "kept");
+    }
+
+    #[test]
+    fn a_block_with_no_text_is_dropped() {
+        assert!(parse("1\n00:00:01,000 --> 00:00:03,000\n[Music]\n").is_empty());
+    }
+
+    #[test]
+    fn several_blank_lines_still_separate_one_block_from_the_next() {
+        let srt = "1\n00:00:01,000 --> 00:00:02,000\none\n\n\n\n\
+                   2\n00:00:02,000 --> 00:00:03,000\ntwo\n";
+        assert_eq!(parse(srt).len(), 2);
+    }
+
+    #[test]
+    fn webvtt_timing_and_settings_read_too() {
+        let vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:03.000 line:0 position:20%\nThe mill.\n";
+        let cues = parse(vtt);
+        assert_eq!(cues.len(), 1);
+        assert_eq!(cues[0].start, 1.0);
+        assert_eq!(cues[0].end, 3.0);
+    }
+}

crates/cuekit/src/text.rs

@@ -0,0 +1,63 @@
+//! Cleaning the markup out of a cue's text.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// Styling and speaker tags: `<i>`, `<v Speaker>`, `<c.colour>`.
+static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"<[^>]+>").expect("a valid pattern"));
+
+/// Bracketed sound cues: `[Music]`, `[Applause]`, `[inaudible]`.
+///
+/// Not speech, and a search that matches them matches every recording with a
+/// theme tune.
+static BRACKETED: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"\[.*?\]").expect("a valid pattern"));
+
+/// Strip markup from a cue's text.
+pub fn clean(text: &str) -> String {
+    let text = TAG.replace_all(text, "");
+    let text = BRACKETED.replace_all(&text, "");
+    text.trim_matches(is_js_space).to_string()
+}
+
+/// What JavaScript's `String.trim` removes: Unicode whitespace and the
+/// byte-order mark.
+fn is_js_space(c: char) -> bool {
+    c.is_whitespace() || c == '\u{feff}'
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn styling_tags_go() {
+        assert_eq!(clean("<i>The mill</i> stood"), "The mill stood");
+        assert_eq!(clean("<v Speaker>Hello"), "Hello");
+    }
+
+    #[test]
+    fn bracketed_sound_cues_go() {
+        assert_eq!(clean("[Music] The mill stood [Applause]"), "The mill stood");
+        assert_eq!(clean("[Music]"), "");
+    }
+
+    #[test]
+    fn the_bracket_match_is_not_greedy() {
+        assert_eq!(clean("[a] kept [b]"), "kept");
+    }
+
+    #[test]
+    fn a_stray_pair_of_angle_brackets_takes_what_is_between_them() {
+        // Faithful to the original, which uses the same pattern. A spoken
+        // line with a less-than and a greater-than in it is vanishingly rare
+        // next to the styling tags this exists to remove.
+        assert_eq!(clean("a < b and c > d"), "a  d");
+    }
+
+    #[test]
+    fn a_lone_angle_bracket_is_left_alone() {
+        assert_eq!(clean("a < b"), "a < b");
+    }
+}

crates/cuenorm/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "cuenorm"
+description = "Any subtitle format in, one normalised cue stream out."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+cuekit.workspace = true
+
+[lints]
+workspace = true

crates/cuenorm/src/main.rs

@@ -0,0 +1,94 @@
+//! Any subtitle format in, one normalised cue stream out.
+
+use std::io::{BufWriter, Read, Write};
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::{exit, ndjson};
+
+/// Read a subtitle file — SubRip, WebVTT or SubStation Alpha — and write one
+/// cue per line as JSON.
+///
+/// The format is worked out from the content, not from the name, because a
+/// subtitle file is named for the format somebody expected it to be in.
+/// Markup and bracketed sound cues are taken out, and consecutive cues with
+/// identical text are merged, which is what automatic captioning needs: it
+/// repeats each line across overlapping windows, so the same sentence arrives
+/// three or four times over.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    long_about = None,
+    after_help = "\
+Example:
+
+    cuenorm episode.vtt | cuepack --seconds 60 > passages.ndjson"
+)]
+struct Args {
+    /// Subtitle file to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+
+    /// Keep consecutive repeats instead of merging them.
+    #[arg(long)]
+    keep_repeats: bool,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+
+    let content = match read(args.path.as_deref()) {
+        Ok(content) => content,
+        Err(e) => {
+            eprintln!("cuenorm: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    let cues = cuekit::parse(&content);
+    let cues = if args.keep_repeats {
+        cues
+    } else {
+        cuekit::merge_repeats(&cues)
+    };
+
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    for cue in &cues {
+        if let Err(e) = ndjson::write_line(&mut out, cue) {
+            // A consumer that stopped reading is not a failure.
+            if e.is_broken_pipe() {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("cuenorm: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+    match out.flush() {
+        Ok(()) => ExitCode::from(exit::OK),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("cuenorm: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Read the file as text, tolerating the encodings a subtitle file arrives in.
+///
+/// Not UTF-8 is common: a subtitle file downloaded from anywhere is as likely
+/// to be Latin-1, and refusing it would make the tool useless for exactly the
+/// archives that need it. Undecodable bytes become the replacement character
+/// rather than an error, because a cue with one wrong character still has its
+/// timing and most of its words.
+fn read(path: Option<&Path>) -> std::io::Result<String> {
+    let bytes = match path {
+        Some(path) => std::fs::read(path)?,
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin().read_to_end(&mut bytes)?;
+            bytes
+        }
+    };
+    Ok(String::from_utf8_lossy(&bytes).into_owned())
+}

parity/Cargo.toml

@@ -15,6 +15,7 @@ python = []
 
 [dependencies]
 chunkmd.workspace = true
+cuekit.workspace = true
 mdscrub.workspace = true
 pdftriage.workspace = true
 corpus-core.workspace = true

parity/KNOWN.md

@@ -18,6 +18,26 @@ affected and how often it occurs in practice.
 
 ---
 
+## cuenorm — a cue line with trailing settings still reads
+
+**Original:** the timestamp line is split on the arrow and each half is read
+whole, so `00:00:01,000 --> 00:00:03,000  X1:040 X2:560` gives an end time
+that is not a number and the cue is dropped entirely.
+
+**Rust:** each half is reduced to the one token that could be a timestamp, so
+the trailing text is ignored and the cue reads.
+
+**Why this is acceptable:** it is what makes the same reader take WebVTT,
+where positioning settings after the end time are not a malformation but the
+format — `00:00:01.000 --> 00:00:03.000 line:0 position:20%` is an ordinary
+cue. Without it the tool would need a second parser that differed in one
+line. The divergence can only ever recover a cue that the original discarded;
+there is no input where it changes a cue the original read.
+
+**Measured:** `parity/cases/cuenorm/coordinates.srt` — one cue of two
+recovered. Across the other eight cases, including deliberately malformed
+timestamps, the two agree cue for cue.
+
 ## mdscrub — un-paginated text is not given a page 1 marker
 
 **Python:** `_normalize_marker_pages` ends by wrapping any text it found no

parity/SOURCE.md

@@ -6,6 +6,7 @@ in this directory are the answers that Python gave, at one commit:
 | Source | Commit |
 |---|---|
 | `book-pipeline` | `7499e3257572968b437ecfdc79feccc91b96585b` |
+| `podcast-download-systemd` | `ab20139cd0defb400a0dd9ed9034e03c1d1e356d` |
 
 Regenerating against a different commit will move the goldens. That is the
 point — the diff is the review. Record the new commit here in the same change.
@@ -16,4 +17,9 @@ a checkout lives is a fact about one machine and must not be compiled in:
 ```sh
 CATENA_PARITY_PYTHON=/path/to/venv/bin/python
 CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src
+CATENA_PARITY_NODE=node   # only if it is not already on PATH
 ```
+
+One oracle is JavaScript rather than Python, because the code it compares
+against is. Its three functions are copied into `oracle/cuenorm.mjs` verbatim,
+since the module they live in opens a database connection when it loads.

parity/cases/cuenorm/brackets.srt

@@ -0,0 +1,11 @@
+1
+00:00:01,000 --> 00:00:03,000
+[Music]
+
+2
+00:00:03,000 --> 00:00:05,000
+[Applause] The mill stood. [inaudible]
+
+3
+00:00:05,000 --> 00:00:07,000
+[a] kept [b]

parity/cases/cuenorm/coordinates.srt

@@ -0,0 +1,7 @@
+1
+00:00:01,000 --> 00:00:03,000  X1:040 X2:560 Y1:050 Y2:100
+The mill stood at the northern end.
+
+2
+00:00:03,000 --> 00:00:05,000
+Its wheel was undershot.

parity/cases/cuenorm/crlf.srt

@@ -0,0 +1,8 @@
+1
+00:00:01,000 --> 00:00:03,000
+The mill stood at the northern end,
+where the fall was steepest.
+
+2
+00:00:03,000 --> 00:00:05,000
+Its wheel was undershot.

parity/cases/cuenorm/malformed.srt

@@ -0,0 +1,20 @@
+1
+nonsense --> also nonsense
+this block has unreadable times
+
+2
+00:00:03,000 --> 00:00:05,000
+this one is fine
+
+a block with no arrow at all
+
+3
+00:00:05,000 --> 
+this one has no end time
+
+4
+ --> 00:00:09,000
+this one has no start time
+
+5
+00:00:09,000 --> 00:00:11,000

parity/cases/cuenorm/multiline.srt

@@ -0,0 +1,9 @@
+1
+00:00:01,000 --> 00:00:03,000
+The mill stood at the northern end,
+where the fall of the river was steepest
+and the ice came latest.
+
+2
+00:00:03,000 --> 00:00:05,000
+Its wheel was undershot.

parity/cases/cuenorm/oddtimes.srt

@@ -0,0 +1,19 @@
+1
+02:03,500 --> 02:05,000
+a two part timestamp
+
+2
+00:00:05,000 --> 00:00:07,000,000
+an extra comma in the end time
+
+3
+0:0:8 --> 0:0:9
+unpadded fields
+
+4
+ 00:00:09,000  -->  00:00:11,000 
+padded with spaces
+
+5
+1:2:3.456 --> 1:2:4.5
+a point instead of a comma

parity/cases/cuenorm/plain.srt

@@ -0,0 +1,11 @@
+1
+00:00:01,000 --> 00:00:03,000
+The mill stood at the northern end of the valley.
+
+2
+00:00:03,000 --> 00:00:05,500
+Its wheel was undershot, which was wasteful.
+
+3
+00:00:05,500 --> 00:00:09,250
+Nobody replaced it for two hundred years.

parity/cases/cuenorm/repeats.srt

@@ -0,0 +1,19 @@
+1
+00:00:00,000 --> 00:00:02,000
+the mill stood at the
+
+2
+00:00:01,500 --> 00:00:03,500
+the mill stood at the
+
+3
+00:00:03,000 --> 00:00:05,000
+the mill stood at the
+
+4
+00:00:05,000 --> 00:00:07,000
+northern end of the valley
+
+5
+00:00:07,000 --> 00:00:09,000
+northern end of the valley

parity/cases/cuenorm/tags.srt

@@ -0,0 +1,11 @@
+1
+00:00:01,000 --> 00:00:03,000
+<i>The mill</i> stood at the <b>northern</b> end.
+
+2
+00:00:03,000 --> 00:00:05,000
+<font color="#ffffff">Its wheel was undershot.</font>
+
+3
+00:00:05,000 --> 00:00:07,000
+a < b and c > d

parity/golden/cuenorm/brackets.ndjson

@@ -0,0 +1,2 @@
+{"start":3,"end":5,"text":"The mill stood."}
+{"start":5,"end":7,"text":"kept"}

parity/golden/cuenorm/coordinates.ndjson

@@ -0,0 +1 @@
+{"start":3,"end":5,"text":"Its wheel was undershot."}

parity/golden/cuenorm/crlf.ndjson

@@ -0,0 +1 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end,\r where the fall was steepest.\r \r 2\r 00:00:03,000 --> 00:00:05,000\r Its wheel was undershot."}

parity/golden/cuenorm/malformed.ndjson

@@ -0,0 +1 @@
+{"start":3,"end":5,"text":"this one is fine"}

parity/golden/cuenorm/multiline.ndjson

@@ -0,0 +1,2 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end, where the fall of the river was steepest and the ice came latest."}
+{"start":3,"end":5,"text":"Its wheel was undershot."}

parity/golden/cuenorm/oddtimes.ndjson

@@ -0,0 +1,5 @@
+{"start":123.5,"end":125,"text":"a two part timestamp"}
+{"start":5,"end":7,"text":"an extra comma in the end time"}
+{"start":8,"end":9,"text":"unpadded fields"}
+{"start":9,"end":11,"text":"padded with spaces"}
+{"start":3723.456,"end":3724.5,"text":"a point instead of a comma"}

parity/golden/cuenorm/plain.ndjson

@@ -0,0 +1,3 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end of the valley."}
+{"start":3,"end":5.5,"text":"Its wheel was undershot, which was wasteful."}
+{"start":5.5,"end":9.25,"text":"Nobody replaced it for two hundred years."}

parity/golden/cuenorm/repeats.ndjson

@@ -0,0 +1,2 @@
+{"start":0,"end":5,"text":"the mill stood at the"}
+{"start":5,"end":9,"text":"northern end of the valley"}

parity/golden/cuenorm/tags.ndjson

@@ -0,0 +1,3 @@
+{"start":1,"end":3,"text":"The mill stood at the northern end."}
+{"start":3,"end":5,"text":"Its wheel was undershot."}
+{"start":5,"end":7,"text":"a  d"}

parity/oracle/cuenorm.mjs

@@ -0,0 +1,86 @@
+/**
+ * Parse SubRip the way the podcast archive was indexed, for comparison.
+ *
+ * The three functions below are the originals, copied out of the indexer
+ * verbatim and with their type annotations dropped — the module they live in
+ * opens a database connection when it loads, which an oracle must not do.
+ *
+ *     node oracle/cuenorm.mjs < case.srt
+ *
+ * Writes one JSON cue per line.
+ */
+
+/** Parse an SRT timestamp (00:01:23,456) into seconds. */
+function parseSrtTimestamp(ts) {
+  // SRT format: HH:MM:SS,mmm  (comma separates ms)
+  const cleaned = ts.trim().replace(",", ".");
+  const parts = cleaned.split(":");
+  if (parts.length === 3) {
+    return Number(parts[0]) * 3600 + Number(parts[1]) * 60 + parseFloat(parts[2]);
+  }
+  if (parts.length === 2) {
+    return Number(parts[0]) * 60 + parseFloat(parts[1]);
+  }
+  return NaN;
+}
+
+/** Parse an SRT file content into cues. */
+function parseSrt(content) {
+  const cues = [];
+  const blocks = content.split(/\n\n+/);
+
+  for (const block of blocks) {
+    const lines = block.trim().split("\n");
+    const tsLine = lines.find((l) => l.includes("-->"));
+    if (!tsLine) continue;
+
+    const [startStr, endStr] = tsLine.split("-->").map((s) => s.trim());
+    const startTime = parseSrtTimestamp(startStr);
+    const endTime = parseSrtTimestamp(endStr);
+    if (isNaN(startTime) || isNaN(endTime)) continue;
+
+    const tsIdx = lines.indexOf(tsLine);
+    const text = lines
+      .slice(tsIdx + 1)
+      .join(" ")
+      .replace(/<[^>]+>/g, "")
+      .replace(/\[.*?\]/g, "")
+      .trim();
+
+    if (text) {
+      cues.push({ startTime, endTime, text });
+    }
+  }
+
+  return cues;
+}
+
+/** Merge consecutive cues with identical text. */
+function deduplicateCues(cues) {
+  if (cues.length === 0) return [];
+
+  const result = [{ ...cues[0] }];
+
+  for (let i = 1; i < cues.length; i++) {
+    const prev = result[result.length - 1];
+    const curr = cues[i];
+
+    if (curr.text === prev.text) {
+      prev.endTime = Math.max(prev.endTime, curr.endTime);
+    } else {
+      result.push({ ...curr });
+    }
+  }
+
+  return result;
+}
+
+const chunks = [];
+for await (const chunk of process.stdin) chunks.push(chunk);
+const content = Buffer.concat(chunks).toString("utf-8");
+
+for (const cue of deduplicateCues(parseSrt(content))) {
+  process.stdout.write(
+    JSON.stringify({ start: cue.startTime, end: cue.endTime, text: cue.text }) + "\n",
+  );
+}

parity/src/bin/regen.rs

@@ -19,12 +19,13 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 5] = [
+    let jobs: [Job; 6] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
         ("mdscrub", mdscrub),
         ("pdftriage", pdftriage),
+        ("cuenorm", cuenorm),
     ];
 
     for (name, job) in jobs {
@@ -68,6 +69,21 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn cuenorm() -> Written {
+    let dir = parity::golden("cuenorm");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let mut written = Vec::new();
+    for case in parity::cases_with_extension("cuenorm", "srt") {
+        let content = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
+        let cues = parity::run_node_oracle("cuenorm.mjs", &content)?;
+        let path = dir.join(format!("{}.ndjson", parity::case_name(&case)));
+        std::fs::write(&path, cues).map_err(|e| e.to_string())?;
+        written.push(path.display().to_string());
+    }
+    Ok(written)
+}
+
 fn pdftriage() -> Written {
     let dir = parity::golden("pdftriage");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/src/lib.rs

@@ -35,6 +35,20 @@ pub fn oracle(name: &str) -> PathBuf {
     dir().join("oracle").join(name)
 }
 
+/// Cases where `cuenorm` is meant to differ from the JavaScript it was
+/// ported from, each one argued for in `KNOWN.md`.
+pub const DIVERGENT_CUENORM: &[&str] = &["coordinates"];
+
+/// The JavaScript runtime to run an oracle with.
+pub fn node() -> String {
+    std::env::var("CATENA_PARITY_NODE").unwrap_or_else(|_| "node".to_string())
+}
+
+/// Run a JavaScript oracle over `input` and return its stdout.
+pub fn run_node_oracle(script: &str, input: &str) -> Result<String, String> {
+    run_oracle_in(node(), script, &[], input)
+}
+
 /// The Python interpreter to run an oracle with.
 ///
 /// Taken from `CATENA_PARITY_PYTHON`, because the interpreter that has the
@@ -55,18 +69,28 @@ pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
 
 /// Run an oracle script with arguments.
 pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<String, String> {
+    run_oracle_in(python(), script, args, input)
+}
+
+/// Run an oracle under a named runtime.
+pub fn run_oracle_in(
+    runtime: String,
+    script: &str,
+    args: &[String],
+    input: &str,
+) -> Result<String, String> {
     use std::io::Write as _;
     use std::process::{Command, Stdio};
 
     let script = oracle(script);
-    let mut child = Command::new(python())
+    let mut child = Command::new(&runtime)
         .arg(&script)
         .args(args)
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::inherit())
         .spawn()
-        .map_err(|e| format!("running {} with {}: {e}", script.display(), python()))?;
+        .map_err(|e| format!("running {} with {runtime}: {e}", script.display()))?;
     child
         .stdin
         .take()
@@ -85,12 +109,17 @@ pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<Str
 
 /// Every case file for a tool, in a fixed order.
 pub fn cases(tool: &str) -> Vec<PathBuf> {
+    cases_with_extension(tool, "md")
+}
+
+/// Every case file for a tool with a given extension, in a fixed order.
+pub fn cases_with_extension(tool: &str, extension: &str) -> Vec<PathBuf> {
     let dir = dir().join("cases").join(tool);
     let mut found: Vec<PathBuf> = std::fs::read_dir(&dir)
         .unwrap_or_else(|e| panic!("reading {}: {e}", dir.display()))
         .filter_map(Result::ok)
         .map(|entry| entry.path())
-        .filter(|path| path.extension().is_some_and(|ext| ext == "md"))
+        .filter(|path| path.extension().is_some_and(|ext| ext == extension))
         .collect();
     found.sort();
     found

parity/tests/cuenorm.rs

@@ -0,0 +1,99 @@
+//! Cue streams must match the JavaScript the archive was indexed with.
+//!
+//! A cue that shifts by a tenth of a second is a search result that lands in
+//! the wrong sentence; a cue that is dropped is a sentence that cannot be
+//! found at all. Both are invisible until somebody searches for exactly that
+//! line.
+
+use serde_json::Value;
+
+/// A cue, as the two sides can be compared.
+///
+/// Not as JSON values: JavaScript writes a whole number as `5` where Rust
+/// writes `5.0`, and the two parse to different kinds of JSON number for the
+/// same quantity.
+type Cue = (f64, f64, String);
+
+fn cues(text: &str) -> Vec<Cue> {
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let cue: Value = serde_json::from_str(line).expect("valid JSON");
+            (
+                cue["start"].as_f64().expect("a start time"),
+                cue["end"].as_f64().expect("an end time"),
+                cue["text"].as_str().expect("text").to_string(),
+            )
+        })
+        .collect()
+}
+
+fn ours(content: &str) -> Vec<Cue> {
+    cuekit::merge_repeats(&cuekit::parse(content))
+        .iter()
+        .map(|cue| (cue.start, cue.end, cue.text.clone()))
+        .collect()
+}
+
+#[test]
+fn every_case_parses_the_same_as_the_original() {
+    let cases = parity::cases_with_extension("cuenorm", "srt");
+    assert!(!cases.is_empty(), "no cases found");
+
+    for case in cases {
+        let name = parity::case_name(&case);
+        if parity::DIVERGENT_CUENORM.contains(&name.as_str()) {
+            continue;
+        }
+        let content = std::fs::read_to_string(&case).expect("a readable case");
+        let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson")));
+        let ours = ours(&content);
+
+        for (at, (ours, theirs)) in ours.iter().zip(&theirs).enumerate() {
+            assert_eq!(ours, theirs, "{name} cue {at} differs");
+        }
+
+        assert_eq!(
+            ours.len(),
+            theirs.len(),
+            "{name}: the original found {} cues, this found {}",
+            theirs.len(),
+            ours.len()
+        );
+    }
+}
+
+/// The divergence is that a cue line carrying screen coordinates after the
+/// end time still reads. The original drops such a cue whole.
+#[test]
+fn the_documented_divergence_is_exactly_what_was_argued_for() {
+    for name in parity::DIVERGENT_CUENORM {
+        let case = parity::dir()
+            .join("cases/cuenorm")
+            .join(format!("{name}.srt"));
+        let content = std::fs::read_to_string(&case).expect("a readable case");
+        let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson")));
+        let ours = ours(&content);
+        assert!(
+            ours.len() > theirs.len(),
+            "{name} no longer recovers more than the original — update KNOWN.md"
+        );
+        // Everything the original did find, this finds too, unchanged.
+        for cue in &theirs {
+            assert!(ours.contains(cue), "{name} lost a cue the original kept");
+        }
+    }
+}
+
+/// Catches a golden file drifting away from an original that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_files_still_match_the_live_original() {
+    for case in parity::cases_with_extension("cuenorm", "srt") {
+        let name = parity::case_name(&case);
+        let content = std::fs::read_to_string(&case).expect("a readable case");
+        let live = parity::run_node_oracle("cuenorm.mjs", &content).expect("the oracle runs");
+        let golden = parity::read_golden_text(&format!("cuenorm/{name}.ndjson"));
+        assert_eq!(live, golden, "golden file for {name} is stale");
+    }
+}