diff --git a/Cargo.lock b/Cargo.lock index be7052c..2b0d96a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,6 +166,27 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cuekit" +version = "0.1.0" +dependencies = [ + "corpus-core", + "regex", + "schemars", + "serde", + "serde_json", + "textkit", +] + +[[package]] +name = "cuenorm" +version = "0.1.0" +dependencies = [ + "clap", + "corpus-core", + "cuekit", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -246,6 +267,7 @@ version = "0.1.0" dependencies = [ "chunkmd", "corpus-core", + "cuekit", "mdscrub", "pdftriage", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index a3c50fc..a5b4e76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,10 +11,12 @@ repository = "https://git.pumpin.online/funkpower/catena" [workspace.dependencies] corpus-core = { path = "crates/corpus-core", version = "0.1.0" } +cuekit = { path = "crates/cuekit", version = "0.1.0" } textkit = { path = "crates/textkit", version = "0.1.0" } chunkmd = { path = "crates/chunkmd", version = "0.1.0" } mdscrub = { path = "crates/mdscrub", version = "0.1.0" } pdftriage = { path = "crates/pdftriage", version = "0.1.0" } +cuenorm = { path = "crates/cuenorm", version = "0.1.0" } clap = { version = "4", features = ["derive"] } fancy-regex = "0.16" diff --git a/crates/corpus-core/src/chunk.rs b/crates/corpus-core/src/chunk.rs index e01a942..6cf4789 100644 --- a/crates/corpus-core/src/chunk.rs +++ b/crates/corpus-core/src/chunk.rs @@ -26,18 +26,12 @@ pub struct Chunk { /// What part of the work the passage belongs to. pub role: Role, /// Seconds from the start of the recording, for timed sources. - #[serde( - default, - serialize_with = "crate::float::serialize_option", - deserialize_with = "crate::float::deserialize_option" - )] + #[serde(default, with = "crate::finite::option")] + #[schemars(with = "Option")] pub start_time: Option, /// Seconds from the start of the recording, for timed sources. - #[serde( - default, - serialize_with = "crate::float::serialize_option", - deserialize_with = "crate::float::deserialize_option" - )] + #[serde(default, with = "crate::finite::option")] + #[schemars(with = "Option")] pub end_time: Option, /// Set tags; see [`flags`]. Sorted and deduplicated on the way in and on /// the way out, so the same set of flags is always the same bytes diff --git a/crates/corpus-core/src/finite.rs b/crates/corpus-core/src/finite.rs new file mode 100644 index 0000000..9e5237a --- /dev/null +++ b/crates/corpus-core/src/finite.rs @@ -0,0 +1,57 @@ +//! Serde plumbing for floats that JSON can actually carry. +//! +//! Use it as `#[serde(with = "corpus_core::finite")]` on an `f64`, or +//! `#[serde(default, with = "corpus_core::finite::option")]` on an +//! `Option`. +//! +//! JSON has no spelling for NaN or infinity, and `serde_json` writes them as +//! `null`. That is a silent corruption in both directions: a score written as +//! `null` fails to read back and kills the stream at the consumer, and a +//! timestamp written as `null` reads back as "no timestamp" with nothing to +//! say it was ever anything else. So a non-finite number is refused where it +//! is produced, which is the only place anyone can still fix it. + +use serde::{Deserialize, Deserializer, Serializer, de::Error as _, ser::Error as _}; + +/// Write a float, refusing one JSON cannot carry. +pub fn serialize(value: &f64, out: S) -> Result { + if !value.is_finite() { + return Err(S::Error::custom(format!( + "{value} cannot be written as JSON" + ))); + } + out.serialize_f64(*value) +} + +/// Read a float, refusing one that is not finite. +pub fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result { + let value = f64::deserialize(input)?; + if !value.is_finite() { + return Err(D::Error::custom(format!("{value} is not a finite number"))); + } + Ok(value) +} + +/// The same, for a field that may be absent. +pub mod option { + use serde::{Deserialize, Deserializer, Serializer, de::Error as _}; + + /// Write an optional float, refusing one JSON cannot carry. + pub fn serialize(value: &Option, out: S) -> Result { + match value { + Some(value) => super::serialize(value, out), + None => out.serialize_none(), + } + } + + /// Read an optional float, refusing one that is not finite. + pub fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result, D::Error> { + let value = Option::::deserialize(input)?; + match value { + Some(value) if !value.is_finite() => { + Err(D::Error::custom(format!("{value} is not a finite number"))) + } + value => Ok(value), + } + } +} diff --git a/crates/corpus-core/src/float.rs b/crates/corpus-core/src/float.rs deleted file mode 100644 index c6fe4c6..0000000 --- a/crates/corpus-core/src/float.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Serde plumbing for floats that JSON can actually carry. -//! -//! JSON has no spelling for NaN or infinity, and `serde_json` writes them as -//! `null`. That is a silent corruption in both directions: a score written as -//! `null` fails to read back and kills the stream at the consumer, and a -//! timestamp written as `null` reads back as "no timestamp" with nothing to -//! say it was ever anything else. So a non-finite number is refused where it -//! is produced, which is the only place anyone can still fix it. - -use serde::{Deserialize, Deserializer, Serializer, de::Error as _, ser::Error as _}; - -pub(crate) fn serialize(value: &f64, out: S) -> Result { - if !value.is_finite() { - return Err(S::Error::custom(format!( - "{value} cannot be written as JSON" - ))); - } - out.serialize_f64(*value) -} - -pub(crate) fn serialize_option( - value: &Option, - out: S, -) -> Result { - match value { - Some(value) => serialize(value, out), - None => out.serialize_none(), - } -} - -pub(crate) fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result { - let value = f64::deserialize(input)?; - if !value.is_finite() { - return Err(D::Error::custom(format!("{value} is not a finite number"))); - } - Ok(value) -} - -pub(crate) fn deserialize_option<'de, D: Deserializer<'de>>( - input: D, -) -> Result, D::Error> { - let value = Option::::deserialize(input)?; - match value { - Some(value) if !value.is_finite() => { - Err(D::Error::custom(format!("{value} is not a finite number"))) - } - value => Ok(value), - } -} diff --git a/crates/corpus-core/src/hit.rs b/crates/corpus-core/src/hit.rs index c75ae4b..1396801 100644 --- a/crates/corpus-core/src/hit.rs +++ b/crates/corpus-core/src/hit.rs @@ -22,10 +22,8 @@ pub struct Hit { /// Position within the channel's own ranking, from one. pub rank: u32, /// The channel's raw score, on whatever scale that channel uses. - #[serde( - serialize_with = "crate::float::serialize", - deserialize_with = "crate::float::deserialize" - )] + #[serde(with = "crate::finite")] + #[schemars(with = "f64")] pub score: f64, /// Anything the producer wants to carry through fusion untouched. /// diff --git a/crates/corpus-core/src/lib.rs b/crates/corpus-core/src/lib.rs index a9caba1..cb231bb 100644 --- a/crates/corpus-core/src/lib.rs +++ b/crates/corpus-core/src/lib.rs @@ -11,7 +11,7 @@ pub mod chunk; pub mod doc; pub mod exit; -mod float; +pub mod finite; pub mod hit; pub mod ndjson; diff --git a/crates/cuekit/Cargo.toml b/crates/cuekit/Cargo.toml new file mode 100644 index 0000000..53b9c17 --- /dev/null +++ b/crates/cuekit/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cuekit" +description = "The cue model everything timed shares, and the subtitle formats it is read from." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +corpus-core.workspace = true +regex.workspace = true +schemars.workspace = true +serde.workspace = true +textkit.workspace = true + +[dev-dependencies] +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/cuekit/src/ass.rs b/crates/cuekit/src/ass.rs new file mode 100644 index 0000000..6a1a977 --- /dev/null +++ b/crates/cuekit/src/ass.rs @@ -0,0 +1,179 @@ +//! Advanced SubStation Alpha, which is a spreadsheet pretending to be a +//! subtitle file. +//! +//! Events are rows under a `Format:` line that names the columns, so the +//! column order has to be read rather than assumed — a file written by one +//! tool puts the text last after nine fields, and one written by another puts +//! it last after eight. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::cue::Cue; +use crate::text; + +/// An override block: `{\an8}`, `{\pos(120,400)}`, `{\i1}`. +static OVERRIDE: LazyLock = LazyLock::new(|| Regex::new(r"\{[^}]*\}").expect("valid")); + +/// A drawing command block, which is a shape rather than speech. +static DRAWING: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\{[^}]*\\p[1-9]").expect("valid")); + +/// Parse an ASS or SSA file. +pub fn parse(content: &str) -> Vec { + let mut columns: Option> = None; + let mut in_events = false; + let mut cues = Vec::new(); + + for line in content.split('\n') { + let line = line.trim_matches(textkit::is_space); + if line.starts_with('[') { + in_events = line.eq_ignore_ascii_case("[events]"); + continue; + } + if !in_events { + continue; + } + if let Some(format) = line.strip_prefix("Format:") { + columns = Some( + format + .split(',') + .map(|name| name.trim_matches(textkit::is_space).to_lowercase()) + .collect(), + ); + continue; + } + let Some(event) = line.strip_prefix("Dialogue:") else { + continue; + }; + let Some(columns) = columns.as_ref() else { + continue; + }; + if let Some(cue) = dialogue(event, columns) { + cues.push(cue); + } + } + cues +} + +fn dialogue(event: &str, columns: &[String]) -> Option { + let start_at = columns.iter().position(|name| name == "start")?; + let end_at = columns.iter().position(|name| name == "end")?; + let text_at = columns.iter().position(|name| name == "text")?; + + // The text field is last and may itself contain commas, so the row is + // split only as many times as there are columns before it. + let mut fields: Vec<&str> = event.splitn(columns.len(), ',').collect(); + if fields.len() < columns.len() { + return None; + } + let text = if text_at == columns.len() - 1 { + fields.pop().unwrap_or("") + } else { + fields.get(text_at).copied().unwrap_or("") + }; + + let start = timestamp(fields.get(start_at)?.trim_matches(textkit::is_space)); + let end = timestamp(fields.get(end_at)?.trim_matches(textkit::is_space)); + if !start.is_finite() || !end.is_finite() { + return None; + } + + // A drawing command is a shape, not speech, and reads as a string of + // coordinates if it is kept. + if DRAWING.is_match(text) { + return None; + } + let text = OVERRIDE.replace_all(text, ""); + let text = text + .replace("\\N", " ") + .replace("\\n", " ") + .replace("\\h", " "); + let text = text::clean(&text); + (!text.is_empty()).then(|| Cue::new(start, end, text)) +} + +/// Seconds from an `H:MM:SS.cc` timestamp. +pub fn timestamp(text: &str) -> f64 { + let parts: Vec<&str> = text.split(':').collect(); + match parts.as_slice() { + [hours, minutes, seconds] => { + crate::jsnum::number(hours) * 3600.0 + + crate::jsnum::number(minutes) * 60.0 + + crate::jsnum::parse_float(seconds) + } + [minutes, seconds] => { + crate::jsnum::number(minutes) * 60.0 + crate::jsnum::parse_float(seconds) + } + _ => f64::NAN, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const FILE: &str = "[Script Info]\n\ + Title: A Recording\n\ + \n\ + [V4+ Styles]\n\ + Format: Name, Fontname\n\ + Style: Default,Arial\n\ + \n\ + [Events]\n\ + Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n\ + Dialogue: 0,0:00:01.00,0:00:03.00,Default,,0,0,0,,The mill stood at the northern end.\n\ + Dialogue: 0,0:00:03.00,0:00:05.50,Default,,0,0,0,,{\\an8}Its wheel was undershot,\\Nwhich was wasteful.\n"; + + #[test] + fn events_become_cues() { + let cues = parse(FILE); + assert_eq!(cues.len(), 2); + assert_eq!(cues[0].start, 1.0); + assert_eq!(cues[0].end, 3.0); + assert_eq!(cues[0].text, "The mill stood at the northern end."); + } + + #[test] + fn override_blocks_and_line_breaks_come_out() { + let cues = parse(FILE); + assert_eq!(cues[1].text, "Its wheel was undershot, which was wasteful."); + } + + #[test] + fn a_comma_in_the_text_is_not_a_column_break() { + let cues = parse(FILE); + assert!(cues[1].text.contains(','), "{:?}", cues[1].text); + } + + #[test] + fn a_drawing_command_is_not_speech() { + let file = FILE.to_string() + + "Dialogue: 0,0:00:06.00,0:00:07.00,Default,,0,0,0,,{\\p1}m 0 0 l 100 0 100 100\n"; + assert_eq!(parse(&file).len(), 2); + } + + #[test] + fn a_row_outside_the_events_section_is_ignored() { + let file = "[Script Info]\nDialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,no\n"; + assert!(parse(file).is_empty()); + } + + #[test] + fn a_column_order_of_its_own_is_honoured() { + let file = "[Events]\n\ + Format: Start, End, Text\n\ + Dialogue: 0:00:01.00,0:00:02.00,Reordered, with a comma\n"; + let cues = parse(file); + assert_eq!(cues.len(), 1); + assert_eq!(cues[0].text, "Reordered, with a comma"); + } + + #[test] + fn a_timestamp_is_read_in_seconds() { + assert_eq!(timestamp("0:00:01.50"), 1.5); + assert_eq!(timestamp("1:02:03.04"), 3723.04); + assert!(timestamp("rubbish").is_nan()); + } +} diff --git a/crates/cuekit/src/cue.rs b/crates/cuekit/src/cue.rs new file mode 100644 index 0000000..7bee552 --- /dev/null +++ b/crates/cuekit/src/cue.rs @@ -0,0 +1,82 @@ +//! One line of a transcript, and when it was said. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// A subtitle cue: a span of time and what was said in it. +/// +/// Deliberately smaller than a chunk. A cue is what a subtitle file holds — +/// a few seconds, often half a sentence — and turning cues into passages +/// worth retrieving is a later stage's job. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Cue { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// Seconds from the start of the recording. + #[serde(with = "corpus_core::finite")] + #[schemars(with = "f64")] + pub start: f64, + /// Seconds from the start of the recording. + #[serde(with = "corpus_core::finite")] + #[schemars(with = "f64")] + pub end: f64, + /// What was said, with the markup taken out. + pub text: String, +} + +fn version() -> u32 { + corpus_core::V +} + +impl Cue { + /// A cue. + pub fn new(start: f64, end: f64, text: impl Into) -> Self { + Self { + v: corpus_core::V, + start, + end, + text: text.into(), + } + } + + /// How long the cue lasts. Negative if the file says it ends before it + /// starts, which some do. + pub fn seconds(&self) -> f64 { + self.end - self.start + } +} + +impl corpus_core::Record for Cue { + fn version(&self) -> u32 { + self.v + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_field_is_emitted() { + let cue = Cue::new(12.5, 15.0, "the mill stood at the northern end"); + assert_eq!( + serde_json::to_string(&cue).unwrap(), + r#"{"v":1,"start":12.5,"end":15.0,"text":"the mill stood at the northern end"}"# + ); + } + + #[test] + fn a_cue_reads_back_identical() { + let cue = Cue::new(0.0, 1.0 / 3.0, "x"); + let text = serde_json::to_string(&cue).unwrap(); + let back: Cue = serde_json::from_str(&text).unwrap(); + assert_eq!(back.end.to_bits(), cue.end.to_bits()); + } + + #[test] + fn a_time_that_json_cannot_carry_is_refused() { + let cue = Cue::new(f64::NAN, 1.0, "x"); + assert!(serde_json::to_string(&cue).is_err()); + } +} diff --git a/crates/cuekit/src/dedup.rs b/crates/cuekit/src/dedup.rs new file mode 100644 index 0000000..05ae570 --- /dev/null +++ b/crates/cuekit/src/dedup.rs @@ -0,0 +1,66 @@ +//! Collapsing the repetition automatic captioning produces. + +use crate::cue::Cue; + +/// Merge consecutive cues whose text is identical, keeping the earliest start +/// and the latest end. +/// +/// Automatic captioning emits a rolling window: the same line appears in cue +/// after cue as the window slides, so a ten-minute stretch of speech can +/// arrive as three times as many cues as there are sentences. Indexed as they +/// come, every one of those is a separate hit for the same words at almost +/// the same timestamp, which buries everything else in a result list. +pub fn merge_repeats(cues: &[Cue]) -> Vec { + let mut merged: Vec = Vec::with_capacity(cues.len()); + for cue in cues { + match merged.last_mut() { + Some(previous) if previous.text == cue.text => { + previous.end = previous.end.max(cue.end); + } + _ => merged.push(cue.clone()), + } + } + merged +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_repeated_line_becomes_one_cue_spanning_both() { + let cues = [ + Cue::new(0.0, 2.0, "the mill stood"), + Cue::new(1.0, 3.0, "the mill stood"), + Cue::new(3.0, 4.0, "at the northern end"), + ]; + let merged = merge_repeats(&cues); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].start, 0.0); + assert_eq!(merged[0].end, 3.0); + assert_eq!(merged[1].text, "at the northern end"); + } + + #[test] + fn a_line_that_comes_back_later_is_kept_separately() { + let cues = [ + Cue::new(0.0, 1.0, "yes"), + Cue::new(1.0, 2.0, "no"), + Cue::new(2.0, 3.0, "yes"), + ]; + assert_eq!(merge_repeats(&cues).len(), 3); + } + + #[test] + fn an_end_that_goes_backwards_does_not_shorten_the_merged_cue() { + let cues = [Cue::new(0.0, 9.0, "x"), Cue::new(1.0, 2.0, "x")]; + let merged = merge_repeats(&cues); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].end, 9.0); + } + + #[test] + fn nothing_in_nothing_out() { + assert!(merge_repeats(&[]).is_empty()); + } +} diff --git a/crates/cuekit/src/jsnum.rs b/crates/cuekit/src/jsnum.rs new file mode 100644 index 0000000..d083d6d --- /dev/null +++ b/crates/cuekit/src/jsnum.rs @@ -0,0 +1,176 @@ +//! Reading numbers the way the JavaScript this was ported from reads them. +//! +//! The timestamp parser it came from calls `Number()` on the hours and the +//! minutes and `parseFloat()` on the seconds. The two disagree about almost +//! everything an ill-formed timestamp can contain — `Number("")` is zero +//! where `parseFloat("")` is not a number, `Number("1a")` is not a number +//! where `parseFloat("1a")` is one — and a subtitle file that has been cut, +//! concatenated or hand-edited produces exactly those. Reproducing both is +//! what makes a malformed file drop the same cues here as there. + +/// What JavaScript's `Number(text)` gives. +/// +/// Whitespace is trimmed, an empty string is zero, the radix prefixes are +/// honoured, and anything else that is not a complete numeric literal is not +/// a number. Note that `Number` accepts `Infinity` but not `NaN`, and neither +/// of the spellings Rust's own parser accepts for them. +pub fn number(text: &str) -> f64 { + let text = text.trim_matches(is_js_space); + if text.is_empty() { + return 0.0; + } + if let Some(value) = radix_literal(text) { + return value; + } + let body = text.strip_prefix(['+', '-']).unwrap_or(text); + // Rust's parser accepts spellings JavaScript's does not. + if body.eq_ignore_ascii_case("inf") + || body.eq_ignore_ascii_case("nan") + || body.eq_ignore_ascii_case("infinity") + { + return if body.eq_ignore_ascii_case("infinity") { + match text.starts_with('-') { + true => f64::NEG_INFINITY, + false => f64::INFINITY, + } + } else { + f64::NAN + }; + } + text.parse().unwrap_or(f64::NAN) +} + +/// What JavaScript's `parseFloat(text)` gives: the longest decimal number at +/// the front, ignoring whatever follows, and not a number when there is none. +pub fn parse_float(text: &str) -> f64 { + let text = text.trim_start_matches(is_js_space); + let prefix = decimal_prefix(text); + if prefix.is_empty() { + return f64::NAN; + } + prefix.parse().unwrap_or(f64::NAN) +} + +/// The longest leading run that is a decimal number, by JavaScript's grammar. +fn decimal_prefix(text: &str) -> &str { + let bytes = text.as_bytes(); + let mut at = 0; + if matches!(bytes.first(), Some(b'+' | b'-')) { + at = 1; + } + if text[at..].starts_with("Infinity") { + return &text[..at + "Infinity".len()]; + } + + let mut end = at; + let digits = |from: usize| { + let mut to = from; + while to < bytes.len() && bytes[to].is_ascii_digit() { + to += 1; + } + to + }; + end = digits(end); + if end < bytes.len() && bytes[end] == b'.' { + end = digits(end + 1); + } + // A run with no digits at all on either side of the point is not a + // number: `.` and `+.` are nothing, `.5` and `5.` are numbers. + if !text[at..end].chars().any(|c| c.is_ascii_digit()) { + return ""; + } + if end < bytes.len() && (bytes[end] | 0x20) == b'e' { + let mut exponent = end + 1; + if matches!(bytes.get(exponent), Some(b'+' | b'-')) { + exponent += 1; + } + let after = digits(exponent); + if after > exponent { + end = after; + } + } + &text[..end] +} + +fn radix_literal(text: &str) -> Option { + let (radix, digits) = match text.get(..2)?.to_ascii_lowercase().as_str() { + "0x" => (16, &text[2..]), + "0o" => (8, &text[2..]), + "0b" => (2, &text[2..]), + _ => return None, + }; + if digits.is_empty() { + return Some(f64::NAN); + } + Some(match u64::from_str_radix(digits, radix) { + Ok(value) => value as f64, + Err(_) => f64::NAN, + }) +} + +/// JavaScript's whitespace for the purpose of trimming a numeric string: +/// Unicode whitespace plus the byte-order mark. +fn is_js_space(c: char) -> bool { + c.is_whitespace() || c == '\u{feff}' +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn number_reads_ordinary_numbers() { + assert_eq!(number("12"), 12.0); + assert_eq!(number(" 07 "), 7.0); + assert_eq!(number("1.5"), 1.5); + assert_eq!(number("1e2"), 100.0); + assert_eq!(number("-3"), -3.0); + } + + #[test] + fn number_of_nothing_is_zero() { + assert_eq!(number(""), 0.0); + assert_eq!(number(" "), 0.0); + } + + #[test] + fn number_refuses_a_partial_literal() { + assert!(number("1a").is_nan()); + assert!(number("12:").is_nan()); + assert!(number("--1").is_nan()); + } + + #[test] + fn number_does_not_accept_the_rust_spellings() { + assert!(number("inf").is_nan()); + assert!(number("NaN").is_nan()); + assert_eq!(number("Infinity"), f64::INFINITY); + assert_eq!(number("-Infinity"), f64::NEG_INFINITY); + } + + #[test] + fn number_honours_the_radix_prefixes() { + assert_eq!(number("0x10"), 16.0); + assert_eq!(number("0b101"), 5.0); + assert_eq!(number("0o17"), 15.0); + } + + #[test] + fn parse_float_takes_the_longest_prefix() { + assert_eq!(parse_float("23.456"), 23.456); + assert_eq!(parse_float("23.456abc"), 23.456); + assert_eq!(parse_float(" 1.5xyz"), 1.5); + assert_eq!(parse_float("5."), 5.0); + assert_eq!(parse_float(".5"), 0.5); + assert_eq!(parse_float("1e3x"), 1000.0); + assert_eq!(parse_float("1e"), 1.0); + } + + #[test] + fn parse_float_of_nothing_is_not_a_number() { + assert!(parse_float("").is_nan()); + assert!(parse_float("abc").is_nan()); + assert!(parse_float(".").is_nan()); + assert!(parse_float("+").is_nan()); + } +} diff --git a/crates/cuekit/src/lib.rs b/crates/cuekit/src/lib.rs new file mode 100644 index 0000000..434087b --- /dev/null +++ b/crates/cuekit/src/lib.rs @@ -0,0 +1,114 @@ +//! The cue model everything timed shares, and the subtitle formats it is +//! read from. +//! +//! Three formats go in — SubRip, WebVTT and SubStation Alpha — and one cue +//! stream comes out. Everything downstream of this works on cues and never +//! has to know which of the three a recording arrived with. +//! +//! The SubRip reader is a port, quirks kept, of the parser a podcast archive +//! was indexed with; the other two are new, and are written to the same +//! shape. Where the port deliberately differs, `parity/KNOWN.md` says so. + +pub mod ass; +pub mod cue; +pub mod dedup; +pub mod jsnum; +pub mod srt; +pub mod text; + +pub use cue::Cue; +pub use dedup::merge_repeats; + +/// Which subtitle format a file is written in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Format { + /// SubRip. + SubRip, + /// WebVTT, which this reads with the SubRip reader. + WebVtt, + /// SubStation Alpha, advanced or otherwise. + SubStationAlpha, +} + +impl Format { + /// What this format is called. + pub fn name(self) -> &'static str { + match self { + Format::SubRip => "srt", + Format::WebVtt => "vtt", + Format::SubStationAlpha => "ass", + } + } +} + +/// Work out which format a file is, from its content rather than its name. +/// +/// A subtitle file arrives named for the format someone expected it to be in. +/// The content says what it actually is, and the three are easy to tell +/// apart: WebVTT announces itself, SubStation Alpha has sections in square +/// brackets, and SubRip has neither. +pub fn detect(content: &str) -> Format { + let head = content.trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}'); + if head.starts_with("WEBVTT") { + return Format::WebVtt; + } + for line in head.split('\n').take(40) { + let line = line.trim_matches(textkit::is_space); + if line.starts_with("[Script Info]") || line.eq_ignore_ascii_case("[events]") { + return Format::SubStationAlpha; + } + if line.starts_with("Dialogue:") || line.starts_with("ScriptType:") { + return Format::SubStationAlpha; + } + } + Format::SubRip +} + +/// Read a subtitle file of any supported format into cues. +pub fn parse(content: &str) -> Vec { + parse_as(content, detect(content)) +} + +/// Read a subtitle file that is known to be in `format`. +pub fn parse_as(content: &str, format: Format) -> Vec { + match format { + Format::SubRip | Format::WebVtt => srt::parse(content), + Format::SubStationAlpha => ass::parse(content), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_file_is_recognised_by_what_is_in_it() { + assert_eq!( + detect("WEBVTT\n\n00:00.000 --> 00:01.000\nx"), + Format::WebVtt + ); + assert_eq!(detect("\u{feff}WEBVTT\n"), Format::WebVtt); + assert_eq!(detect("[Script Info]\nTitle: x\n"), Format::SubStationAlpha); + assert_eq!( + detect("[Events]\nFormat: Start, End, Text\n"), + Format::SubStationAlpha + ); + assert_eq!( + detect("1\n00:00:01,000 --> 00:00:02,000\nx\n"), + Format::SubRip + ); + assert_eq!(detect(""), Format::SubRip); + } + + #[test] + fn all_three_formats_give_the_same_cue() { + let srt = "1\n00:00:01,000 --> 00:00:03,000\nThe mill stood.\n"; + let vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:03.000\nThe mill stood.\n"; + let ass = "[Events]\nFormat: Start, End, Text\n\ + Dialogue: 0:00:01.00,0:00:03.00,The mill stood.\n"; + let expected = vec![Cue::new(1.0, 3.0, "The mill stood.")]; + assert_eq!(parse(srt), expected); + assert_eq!(parse(vtt), expected); + assert_eq!(parse(ass), expected); + } +} diff --git a/crates/cuekit/src/srt.rs b/crates/cuekit/src/srt.rs new file mode 100644 index 0000000..d2b104a --- /dev/null +++ b/crates/cuekit/src/srt.rs @@ -0,0 +1,138 @@ +//! SubRip, and the WebVTT that is close enough to share its reader. +//! +//! A port of the parser the podcast archive was indexed with, quirks kept: +//! the timestamp line is found by looking for an arrow anywhere in the block +//! rather than at a fixed position, only the first comma in a timestamp +//! becomes a point, and a block whose times do not read as numbers is +//! dropped whole rather than guessed at. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::cue::Cue; +use crate::jsnum; +use crate::text; + +/// Blocks are separated by one or more blank lines. +static BLOCKS: LazyLock = LazyLock::new(|| Regex::new(r"\n\n+").expect("a valid pattern")); + +/// Parse a SubRip file. +pub fn parse(content: &str) -> Vec { + let mut cues = Vec::new(); + for block in BLOCKS.split(content) { + let lines: Vec<&str> = block.trim_matches(textkit::is_space).split('\n').collect(); + let Some(at) = lines.iter().position(|line| line.contains("-->")) else { + continue; + }; + let Some((start, end)) = times(lines[at]) else { + continue; + }; + let text = text::clean(&lines[at + 1..].join(" ")); + if !text.is_empty() { + cues.push(Cue::new(start, end, text)); + } + } + cues +} + +/// Read a timestamp line, or `None` if either end is not a number. +/// +/// Each half is reduced to the one token that could be a timestamp, which is +/// what lets the same reader take a WebVTT cue line — where the end time is +/// followed by positioning settings — and an old SubRip file with screen +/// coordinates after it. See `parity/KNOWN.md`: the original reads each half +/// whole and drops such a cue entirely. +fn times(line: &str) -> Option<(f64, f64)> { + let (before, after) = line.split_once("-->")?; + let start = timestamp(before.split_whitespace().next_back().unwrap_or("")); + let end = timestamp(after.split_whitespace().next().unwrap_or("")); + (start.is_finite() && end.is_finite()).then_some((start, end)) +} + +/// Seconds from a `HH:MM:SS,mmm` or `MM:SS.mmm` timestamp. +/// +/// Only the first comma is replaced, exactly as the original does — a +/// timestamp written with a comma for the thousands separator as well as for +/// the decimal point therefore parses as far as the second comma and no +/// further. +pub fn timestamp(text: &str) -> f64 { + let text = text.trim_matches(textkit::is_space); + let cleaned = text.replacen(',', ".", 1); + let parts: Vec<&str> = cleaned.split(':').collect(); + match parts.len() { + 3 => { + jsnum::number(parts[0]) * 3600.0 + + jsnum::number(parts[1]) * 60.0 + + jsnum::parse_float(parts[2]) + } + 2 => jsnum::number(parts[0]) * 60.0 + jsnum::parse_float(parts[1]), + _ => f64::NAN, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_timestamp_is_read_in_seconds() { + assert_eq!(timestamp("00:01:23,456"), 83.456); + assert_eq!(timestamp("01:00:00,000"), 3600.0); + assert_eq!(timestamp("02:03.500"), 123.5); + } + + #[test] + fn a_timestamp_that_is_not_one_is_not_a_number() { + assert!(timestamp("").is_nan()); + assert!(timestamp("nonsense").is_nan()); + assert!(timestamp("00:00:00:00").is_nan()); + } + + #[test] + fn a_file_parses_into_cues() { + let srt = "1\n00:00:01,000 --> 00:00:03,000\nThe mill stood at the northern end.\n\n\ + 2\n00:00:03,000 --> 00:00:05,500\nIts wheel was undershot,\nwhich was wasteful.\n"; + let cues = parse(srt); + assert_eq!(cues.len(), 2); + assert_eq!(cues[0].start, 1.0); + assert_eq!(cues[0].end, 3.0); + assert_eq!(cues[0].text, "The mill stood at the northern end."); + assert_eq!(cues[1].text, "Its wheel was undershot, which was wasteful."); + } + + #[test] + fn a_block_with_no_arrow_is_skipped() { + assert!(parse("WEBVTT\n\nsome preamble\n").is_empty()); + } + + #[test] + fn a_block_with_unreadable_times_is_dropped_whole() { + let srt = "1\nnonsense --> also nonsense\ntext\n\n\ + 2\n00:00:03,000 --> 00:00:05,000\nkept\n"; + let cues = parse(srt); + assert_eq!(cues.len(), 1); + assert_eq!(cues[0].text, "kept"); + } + + #[test] + fn a_block_with_no_text_is_dropped() { + assert!(parse("1\n00:00:01,000 --> 00:00:03,000\n[Music]\n").is_empty()); + } + + #[test] + fn several_blank_lines_still_separate_one_block_from_the_next() { + let srt = "1\n00:00:01,000 --> 00:00:02,000\none\n\n\n\n\ + 2\n00:00:02,000 --> 00:00:03,000\ntwo\n"; + assert_eq!(parse(srt).len(), 2); + } + + #[test] + fn webvtt_timing_and_settings_read_too() { + let vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:03.000 line:0 position:20%\nThe mill.\n"; + let cues = parse(vtt); + assert_eq!(cues.len(), 1); + assert_eq!(cues[0].start, 1.0); + assert_eq!(cues[0].end, 3.0); + } +} diff --git a/crates/cuekit/src/text.rs b/crates/cuekit/src/text.rs new file mode 100644 index 0000000..664a838 --- /dev/null +++ b/crates/cuekit/src/text.rs @@ -0,0 +1,63 @@ +//! Cleaning the markup out of a cue's text. + +use std::sync::LazyLock; + +use regex::Regex; + +/// Styling and speaker tags: ``, ``, ``. +static TAG: LazyLock = LazyLock::new(|| Regex::new(r"<[^>]+>").expect("a valid pattern")); + +/// Bracketed sound cues: `[Music]`, `[Applause]`, `[inaudible]`. +/// +/// Not speech, and a search that matches them matches every recording with a +/// theme tune. +static BRACKETED: LazyLock = + LazyLock::new(|| Regex::new(r"\[.*?\]").expect("a valid pattern")); + +/// Strip markup from a cue's text. +pub fn clean(text: &str) -> String { + let text = TAG.replace_all(text, ""); + let text = BRACKETED.replace_all(&text, ""); + text.trim_matches(is_js_space).to_string() +} + +/// What JavaScript's `String.trim` removes: Unicode whitespace and the +/// byte-order mark. +fn is_js_space(c: char) -> bool { + c.is_whitespace() || c == '\u{feff}' +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn styling_tags_go() { + assert_eq!(clean("The mill stood"), "The mill stood"); + assert_eq!(clean("Hello"), "Hello"); + } + + #[test] + fn bracketed_sound_cues_go() { + assert_eq!(clean("[Music] The mill stood [Applause]"), "The mill stood"); + assert_eq!(clean("[Music]"), ""); + } + + #[test] + fn the_bracket_match_is_not_greedy() { + assert_eq!(clean("[a] kept [b]"), "kept"); + } + + #[test] + fn a_stray_pair_of_angle_brackets_takes_what_is_between_them() { + // Faithful to the original, which uses the same pattern. A spoken + // line with a less-than and a greater-than in it is vanishingly rare + // next to the styling tags this exists to remove. + assert_eq!(clean("a < b and c > d"), "a d"); + } + + #[test] + fn a_lone_angle_bracket_is_left_alone() { + assert_eq!(clean("a < b"), "a < b"); + } +} diff --git a/crates/cuenorm/Cargo.toml b/crates/cuenorm/Cargo.toml new file mode 100644 index 0000000..86c0c3e --- /dev/null +++ b/crates/cuenorm/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "cuenorm" +description = "Any subtitle format in, one normalised cue stream out." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +clap.workspace = true +corpus-core.workspace = true +cuekit.workspace = true + +[lints] +workspace = true diff --git a/crates/cuenorm/src/main.rs b/crates/cuenorm/src/main.rs new file mode 100644 index 0000000..a067f5d --- /dev/null +++ b/crates/cuenorm/src/main.rs @@ -0,0 +1,94 @@ +//! Any subtitle format in, one normalised cue stream out. + +use std::io::{BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::{exit, ndjson}; + +/// Read a subtitle file — SubRip, WebVTT or SubStation Alpha — and write one +/// cue per line as JSON. +/// +/// The format is worked out from the content, not from the name, because a +/// subtitle file is named for the format somebody expected it to be in. +/// Markup and bracketed sound cues are taken out, and consecutive cues with +/// identical text are merged, which is what automatic captioning needs: it +/// repeats each line across overlapping windows, so the same sentence arrives +/// three or four times over. +#[derive(Parser)] +#[command( + version, + about, + long_about = None, + after_help = "\ +Example: + + cuenorm episode.vtt | cuepack --seconds 60 > passages.ndjson" +)] +struct Args { + /// Subtitle file to read. Reads stdin when absent. + path: Option, + + /// Keep consecutive repeats instead of merging them. + #[arg(long)] + keep_repeats: bool, +} + +fn main() -> ExitCode { + let args = Args::parse(); + + let content = match read(args.path.as_deref()) { + Ok(content) => content, + Err(e) => { + eprintln!("cuenorm: {e}"); + return ExitCode::from(exit::IO); + } + }; + + let cues = cuekit::parse(&content); + let cues = if args.keep_repeats { + cues + } else { + cuekit::merge_repeats(&cues) + }; + + let mut out = BufWriter::new(std::io::stdout().lock()); + for cue in &cues { + if let Err(e) = ndjson::write_line(&mut out, cue) { + // A consumer that stopped reading is not a failure. + if e.is_broken_pipe() { + return ExitCode::from(exit::OK); + } + eprintln!("cuenorm: {e}"); + return ExitCode::from(exit::IO); + } + } + match out.flush() { + Ok(()) => ExitCode::from(exit::OK), + Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK), + Err(e) => { + eprintln!("cuenorm: {e}"); + ExitCode::from(exit::IO) + } + } +} + +/// Read the file as text, tolerating the encodings a subtitle file arrives in. +/// +/// Not UTF-8 is common: a subtitle file downloaded from anywhere is as likely +/// to be Latin-1, and refusing it would make the tool useless for exactly the +/// archives that need it. Undecodable bytes become the replacement character +/// rather than an error, because a cue with one wrong character still has its +/// timing and most of its words. +fn read(path: Option<&Path>) -> std::io::Result { + let bytes = match path { + Some(path) => std::fs::read(path)?, + None => { + let mut bytes = Vec::new(); + std::io::stdin().read_to_end(&mut bytes)?; + bytes + } + }; + Ok(String::from_utf8_lossy(&bytes).into_owned()) +} diff --git a/parity/Cargo.toml b/parity/Cargo.toml index c2f4d73..42a6504 100644 --- a/parity/Cargo.toml +++ b/parity/Cargo.toml @@ -15,6 +15,7 @@ python = [] [dependencies] chunkmd.workspace = true +cuekit.workspace = true mdscrub.workspace = true pdftriage.workspace = true corpus-core.workspace = true diff --git a/parity/KNOWN.md b/parity/KNOWN.md index 419c9fc..5f2ae8a 100644 --- a/parity/KNOWN.md +++ b/parity/KNOWN.md @@ -18,6 +18,26 @@ affected and how often it occurs in practice. --- +## cuenorm — a cue line with trailing settings still reads + +**Original:** the timestamp line is split on the arrow and each half is read +whole, so `00:00:01,000 --> 00:00:03,000 X1:040 X2:560` gives an end time +that is not a number and the cue is dropped entirely. + +**Rust:** each half is reduced to the one token that could be a timestamp, so +the trailing text is ignored and the cue reads. + +**Why this is acceptable:** it is what makes the same reader take WebVTT, +where positioning settings after the end time are not a malformation but the +format — `00:00:01.000 --> 00:00:03.000 line:0 position:20%` is an ordinary +cue. Without it the tool would need a second parser that differed in one +line. The divergence can only ever recover a cue that the original discarded; +there is no input where it changes a cue the original read. + +**Measured:** `parity/cases/cuenorm/coordinates.srt` — one cue of two +recovered. Across the other eight cases, including deliberately malformed +timestamps, the two agree cue for cue. + ## mdscrub — un-paginated text is not given a page 1 marker **Python:** `_normalize_marker_pages` ends by wrapping any text it found no diff --git a/parity/SOURCE.md b/parity/SOURCE.md index d95a22e..506fe7d 100644 --- a/parity/SOURCE.md +++ b/parity/SOURCE.md @@ -6,6 +6,7 @@ in this directory are the answers that Python gave, at one commit: | Source | Commit | |---|---| | `book-pipeline` | `7499e3257572968b437ecfdc79feccc91b96585b` | +| `podcast-download-systemd` | `ab20139cd0defb400a0dd9ed9034e03c1d1e356d` | Regenerating against a different commit will move the goldens. That is the point — the diff is the review. Record the new commit here in the same change. @@ -16,4 +17,9 @@ a checkout lives is a fact about one machine and must not be compiled in: ```sh CATENA_PARITY_PYTHON=/path/to/venv/bin/python CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src +CATENA_PARITY_NODE=node # only if it is not already on PATH ``` + +One oracle is JavaScript rather than Python, because the code it compares +against is. Its three functions are copied into `oracle/cuenorm.mjs` verbatim, +since the module they live in opens a database connection when it loads. diff --git a/parity/cases/cuenorm/brackets.srt b/parity/cases/cuenorm/brackets.srt new file mode 100644 index 0000000..193a6d2 --- /dev/null +++ b/parity/cases/cuenorm/brackets.srt @@ -0,0 +1,11 @@ +1 +00:00:01,000 --> 00:00:03,000 +[Music] + +2 +00:00:03,000 --> 00:00:05,000 +[Applause] The mill stood. [inaudible] + +3 +00:00:05,000 --> 00:00:07,000 +[a] kept [b] diff --git a/parity/cases/cuenorm/coordinates.srt b/parity/cases/cuenorm/coordinates.srt new file mode 100644 index 0000000..846d817 --- /dev/null +++ b/parity/cases/cuenorm/coordinates.srt @@ -0,0 +1,7 @@ +1 +00:00:01,000 --> 00:00:03,000 X1:040 X2:560 Y1:050 Y2:100 +The mill stood at the northern end. + +2 +00:00:03,000 --> 00:00:05,000 +Its wheel was undershot. diff --git a/parity/cases/cuenorm/crlf.srt b/parity/cases/cuenorm/crlf.srt new file mode 100644 index 0000000..2f1e746 --- /dev/null +++ b/parity/cases/cuenorm/crlf.srt @@ -0,0 +1,8 @@ +1 +00:00:01,000 --> 00:00:03,000 +The mill stood at the northern end, +where the fall was steepest. + +2 +00:00:03,000 --> 00:00:05,000 +Its wheel was undershot. diff --git a/parity/cases/cuenorm/malformed.srt b/parity/cases/cuenorm/malformed.srt new file mode 100644 index 0000000..4205fc4 --- /dev/null +++ b/parity/cases/cuenorm/malformed.srt @@ -0,0 +1,20 @@ +1 +nonsense --> also nonsense +this block has unreadable times + +2 +00:00:03,000 --> 00:00:05,000 +this one is fine + +a block with no arrow at all + +3 +00:00:05,000 --> +this one has no end time + +4 + --> 00:00:09,000 +this one has no start time + +5 +00:00:09,000 --> 00:00:11,000 diff --git a/parity/cases/cuenorm/multiline.srt b/parity/cases/cuenorm/multiline.srt new file mode 100644 index 0000000..9dfb2ba --- /dev/null +++ b/parity/cases/cuenorm/multiline.srt @@ -0,0 +1,9 @@ +1 +00:00:01,000 --> 00:00:03,000 +The mill stood at the northern end, +where the fall of the river was steepest +and the ice came latest. + +2 +00:00:03,000 --> 00:00:05,000 +Its wheel was undershot. diff --git a/parity/cases/cuenorm/oddtimes.srt b/parity/cases/cuenorm/oddtimes.srt new file mode 100644 index 0000000..7588f82 --- /dev/null +++ b/parity/cases/cuenorm/oddtimes.srt @@ -0,0 +1,19 @@ +1 +02:03,500 --> 02:05,000 +a two part timestamp + +2 +00:00:05,000 --> 00:00:07,000,000 +an extra comma in the end time + +3 +0:0:8 --> 0:0:9 +unpadded fields + +4 + 00:00:09,000 --> 00:00:11,000 +padded with spaces + +5 +1:2:3.456 --> 1:2:4.5 +a point instead of a comma diff --git a/parity/cases/cuenorm/plain.srt b/parity/cases/cuenorm/plain.srt new file mode 100644 index 0000000..47feaf9 --- /dev/null +++ b/parity/cases/cuenorm/plain.srt @@ -0,0 +1,11 @@ +1 +00:00:01,000 --> 00:00:03,000 +The mill stood at the northern end of the valley. + +2 +00:00:03,000 --> 00:00:05,500 +Its wheel was undershot, which was wasteful. + +3 +00:00:05,500 --> 00:00:09,250 +Nobody replaced it for two hundred years. diff --git a/parity/cases/cuenorm/repeats.srt b/parity/cases/cuenorm/repeats.srt new file mode 100644 index 0000000..49d4529 --- /dev/null +++ b/parity/cases/cuenorm/repeats.srt @@ -0,0 +1,19 @@ +1 +00:00:00,000 --> 00:00:02,000 +the mill stood at the + +2 +00:00:01,500 --> 00:00:03,500 +the mill stood at the + +3 +00:00:03,000 --> 00:00:05,000 +the mill stood at the + +4 +00:00:05,000 --> 00:00:07,000 +northern end of the valley + +5 +00:00:07,000 --> 00:00:09,000 +northern end of the valley diff --git a/parity/cases/cuenorm/tags.srt b/parity/cases/cuenorm/tags.srt new file mode 100644 index 0000000..3ceb251 --- /dev/null +++ b/parity/cases/cuenorm/tags.srt @@ -0,0 +1,11 @@ +1 +00:00:01,000 --> 00:00:03,000 +The mill stood at the northern end. + +2 +00:00:03,000 --> 00:00:05,000 +Its wheel was undershot. + +3 +00:00:05,000 --> 00:00:07,000 +a < b and c > d diff --git a/parity/golden/cuenorm/brackets.ndjson b/parity/golden/cuenorm/brackets.ndjson new file mode 100644 index 0000000..a73b503 --- /dev/null +++ b/parity/golden/cuenorm/brackets.ndjson @@ -0,0 +1,2 @@ +{"start":3,"end":5,"text":"The mill stood."} +{"start":5,"end":7,"text":"kept"} diff --git a/parity/golden/cuenorm/coordinates.ndjson b/parity/golden/cuenorm/coordinates.ndjson new file mode 100644 index 0000000..032eb7f --- /dev/null +++ b/parity/golden/cuenorm/coordinates.ndjson @@ -0,0 +1 @@ +{"start":3,"end":5,"text":"Its wheel was undershot."} diff --git a/parity/golden/cuenorm/crlf.ndjson b/parity/golden/cuenorm/crlf.ndjson new file mode 100644 index 0000000..ae287dd --- /dev/null +++ b/parity/golden/cuenorm/crlf.ndjson @@ -0,0 +1 @@ +{"start":1,"end":3,"text":"The mill stood at the northern end,\r where the fall was steepest.\r \r 2\r 00:00:03,000 --> 00:00:05,000\r Its wheel was undershot."} diff --git a/parity/golden/cuenorm/malformed.ndjson b/parity/golden/cuenorm/malformed.ndjson new file mode 100644 index 0000000..ae7b735 --- /dev/null +++ b/parity/golden/cuenorm/malformed.ndjson @@ -0,0 +1 @@ +{"start":3,"end":5,"text":"this one is fine"} diff --git a/parity/golden/cuenorm/multiline.ndjson b/parity/golden/cuenorm/multiline.ndjson new file mode 100644 index 0000000..86bbe26 --- /dev/null +++ b/parity/golden/cuenorm/multiline.ndjson @@ -0,0 +1,2 @@ +{"start":1,"end":3,"text":"The mill stood at the northern end, where the fall of the river was steepest and the ice came latest."} +{"start":3,"end":5,"text":"Its wheel was undershot."} diff --git a/parity/golden/cuenorm/oddtimes.ndjson b/parity/golden/cuenorm/oddtimes.ndjson new file mode 100644 index 0000000..6752a67 --- /dev/null +++ b/parity/golden/cuenorm/oddtimes.ndjson @@ -0,0 +1,5 @@ +{"start":123.5,"end":125,"text":"a two part timestamp"} +{"start":5,"end":7,"text":"an extra comma in the end time"} +{"start":8,"end":9,"text":"unpadded fields"} +{"start":9,"end":11,"text":"padded with spaces"} +{"start":3723.456,"end":3724.5,"text":"a point instead of a comma"} diff --git a/parity/golden/cuenorm/plain.ndjson b/parity/golden/cuenorm/plain.ndjson new file mode 100644 index 0000000..4b08814 --- /dev/null +++ b/parity/golden/cuenorm/plain.ndjson @@ -0,0 +1,3 @@ +{"start":1,"end":3,"text":"The mill stood at the northern end of the valley."} +{"start":3,"end":5.5,"text":"Its wheel was undershot, which was wasteful."} +{"start":5.5,"end":9.25,"text":"Nobody replaced it for two hundred years."} diff --git a/parity/golden/cuenorm/repeats.ndjson b/parity/golden/cuenorm/repeats.ndjson new file mode 100644 index 0000000..169ba9f --- /dev/null +++ b/parity/golden/cuenorm/repeats.ndjson @@ -0,0 +1,2 @@ +{"start":0,"end":5,"text":"the mill stood at the"} +{"start":5,"end":9,"text":"northern end of the valley"} diff --git a/parity/golden/cuenorm/tags.ndjson b/parity/golden/cuenorm/tags.ndjson new file mode 100644 index 0000000..c0809a8 --- /dev/null +++ b/parity/golden/cuenorm/tags.ndjson @@ -0,0 +1,3 @@ +{"start":1,"end":3,"text":"The mill stood at the northern end."} +{"start":3,"end":5,"text":"Its wheel was undershot."} +{"start":5,"end":7,"text":"a d"} diff --git a/parity/oracle/cuenorm.mjs b/parity/oracle/cuenorm.mjs new file mode 100644 index 0000000..27499eb --- /dev/null +++ b/parity/oracle/cuenorm.mjs @@ -0,0 +1,86 @@ +/** + * Parse SubRip the way the podcast archive was indexed, for comparison. + * + * The three functions below are the originals, copied out of the indexer + * verbatim and with their type annotations dropped — the module they live in + * opens a database connection when it loads, which an oracle must not do. + * + * node oracle/cuenorm.mjs < case.srt + * + * Writes one JSON cue per line. + */ + +/** Parse an SRT timestamp (00:01:23,456) into seconds. */ +function parseSrtTimestamp(ts) { + // SRT format: HH:MM:SS,mmm (comma separates ms) + const cleaned = ts.trim().replace(",", "."); + const parts = cleaned.split(":"); + if (parts.length === 3) { + return Number(parts[0]) * 3600 + Number(parts[1]) * 60 + parseFloat(parts[2]); + } + if (parts.length === 2) { + return Number(parts[0]) * 60 + parseFloat(parts[1]); + } + return NaN; +} + +/** Parse an SRT file content into cues. */ +function parseSrt(content) { + const cues = []; + const blocks = content.split(/\n\n+/); + + for (const block of blocks) { + const lines = block.trim().split("\n"); + const tsLine = lines.find((l) => l.includes("-->")); + if (!tsLine) continue; + + const [startStr, endStr] = tsLine.split("-->").map((s) => s.trim()); + const startTime = parseSrtTimestamp(startStr); + const endTime = parseSrtTimestamp(endStr); + if (isNaN(startTime) || isNaN(endTime)) continue; + + const tsIdx = lines.indexOf(tsLine); + const text = lines + .slice(tsIdx + 1) + .join(" ") + .replace(/<[^>]+>/g, "") + .replace(/\[.*?\]/g, "") + .trim(); + + if (text) { + cues.push({ startTime, endTime, text }); + } + } + + return cues; +} + +/** Merge consecutive cues with identical text. */ +function deduplicateCues(cues) { + if (cues.length === 0) return []; + + const result = [{ ...cues[0] }]; + + for (let i = 1; i < cues.length; i++) { + const prev = result[result.length - 1]; + const curr = cues[i]; + + if (curr.text === prev.text) { + prev.endTime = Math.max(prev.endTime, curr.endTime); + } else { + result.push({ ...curr }); + } + } + + return result; +} + +const chunks = []; +for await (const chunk of process.stdin) chunks.push(chunk); +const content = Buffer.concat(chunks).toString("utf-8"); + +for (const cue of deduplicateCues(parseSrt(content))) { + process.stdout.write( + JSON.stringify({ start: cue.startTime, end: cue.endTime, text: cue.text }) + "\n", + ); +} diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index 1dd766c..27d6874 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,12 +19,13 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 5] = [ + let jobs: [Job; 6] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), ("mdscrub", mdscrub), ("pdftriage", pdftriage), + ("cuenorm", cuenorm), ]; for (name, job) in jobs { @@ -68,6 +69,21 @@ fn unescape() -> Written { Ok(vec![path.display().to_string()]) } +fn cuenorm() -> Written { + let dir = parity::golden("cuenorm"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let mut written = Vec::new(); + for case in parity::cases_with_extension("cuenorm", "srt") { + let content = std::fs::read_to_string(&case).map_err(|e| e.to_string())?; + let cues = parity::run_node_oracle("cuenorm.mjs", &content)?; + let path = dir.join(format!("{}.ndjson", parity::case_name(&case))); + std::fs::write(&path, cues).map_err(|e| e.to_string())?; + written.push(path.display().to_string()); + } + Ok(written) +} + fn pdftriage() -> Written { let dir = parity::golden("pdftriage"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/src/lib.rs b/parity/src/lib.rs index a7bace8..8b35742 100644 --- a/parity/src/lib.rs +++ b/parity/src/lib.rs @@ -35,6 +35,20 @@ pub fn oracle(name: &str) -> PathBuf { dir().join("oracle").join(name) } +/// Cases where `cuenorm` is meant to differ from the JavaScript it was +/// ported from, each one argued for in `KNOWN.md`. +pub const DIVERGENT_CUENORM: &[&str] = &["coordinates"]; + +/// The JavaScript runtime to run an oracle with. +pub fn node() -> String { + std::env::var("CATENA_PARITY_NODE").unwrap_or_else(|_| "node".to_string()) +} + +/// Run a JavaScript oracle over `input` and return its stdout. +pub fn run_node_oracle(script: &str, input: &str) -> Result { + run_oracle_in(node(), script, &[], input) +} + /// The Python interpreter to run an oracle with. /// /// Taken from `CATENA_PARITY_PYTHON`, because the interpreter that has the @@ -55,18 +69,28 @@ pub fn run_oracle(script: &str, input: &str) -> Result { /// Run an oracle script with arguments. pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result { + run_oracle_in(python(), script, args, input) +} + +/// Run an oracle under a named runtime. +pub fn run_oracle_in( + runtime: String, + script: &str, + args: &[String], + input: &str, +) -> Result { use std::io::Write as _; use std::process::{Command, Stdio}; let script = oracle(script); - let mut child = Command::new(python()) + let mut child = Command::new(&runtime) .arg(&script) .args(args) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .spawn() - .map_err(|e| format!("running {} with {}: {e}", script.display(), python()))?; + .map_err(|e| format!("running {} with {runtime}: {e}", script.display()))?; child .stdin .take() @@ -85,12 +109,17 @@ pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result Vec { + cases_with_extension(tool, "md") +} + +/// Every case file for a tool with a given extension, in a fixed order. +pub fn cases_with_extension(tool: &str, extension: &str) -> Vec { let dir = dir().join("cases").join(tool); let mut found: Vec = std::fs::read_dir(&dir) .unwrap_or_else(|e| panic!("reading {}: {e}", dir.display())) .filter_map(Result::ok) .map(|entry| entry.path()) - .filter(|path| path.extension().is_some_and(|ext| ext == "md")) + .filter(|path| path.extension().is_some_and(|ext| ext == extension)) .collect(); found.sort(); found diff --git a/parity/tests/cuenorm.rs b/parity/tests/cuenorm.rs new file mode 100644 index 0000000..d5a7705 --- /dev/null +++ b/parity/tests/cuenorm.rs @@ -0,0 +1,99 @@ +//! Cue streams must match the JavaScript the archive was indexed with. +//! +//! A cue that shifts by a tenth of a second is a search result that lands in +//! the wrong sentence; a cue that is dropped is a sentence that cannot be +//! found at all. Both are invisible until somebody searches for exactly that +//! line. + +use serde_json::Value; + +/// A cue, as the two sides can be compared. +/// +/// Not as JSON values: JavaScript writes a whole number as `5` where Rust +/// writes `5.0`, and the two parse to different kinds of JSON number for the +/// same quantity. +type Cue = (f64, f64, String); + +fn cues(text: &str) -> Vec { + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + let cue: Value = serde_json::from_str(line).expect("valid JSON"); + ( + cue["start"].as_f64().expect("a start time"), + cue["end"].as_f64().expect("an end time"), + cue["text"].as_str().expect("text").to_string(), + ) + }) + .collect() +} + +fn ours(content: &str) -> Vec { + cuekit::merge_repeats(&cuekit::parse(content)) + .iter() + .map(|cue| (cue.start, cue.end, cue.text.clone())) + .collect() +} + +#[test] +fn every_case_parses_the_same_as_the_original() { + let cases = parity::cases_with_extension("cuenorm", "srt"); + assert!(!cases.is_empty(), "no cases found"); + + for case in cases { + let name = parity::case_name(&case); + if parity::DIVERGENT_CUENORM.contains(&name.as_str()) { + continue; + } + let content = std::fs::read_to_string(&case).expect("a readable case"); + let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson"))); + let ours = ours(&content); + + for (at, (ours, theirs)) in ours.iter().zip(&theirs).enumerate() { + assert_eq!(ours, theirs, "{name} cue {at} differs"); + } + + assert_eq!( + ours.len(), + theirs.len(), + "{name}: the original found {} cues, this found {}", + theirs.len(), + ours.len() + ); + } +} + +/// The divergence is that a cue line carrying screen coordinates after the +/// end time still reads. The original drops such a cue whole. +#[test] +fn the_documented_divergence_is_exactly_what_was_argued_for() { + for name in parity::DIVERGENT_CUENORM { + let case = parity::dir() + .join("cases/cuenorm") + .join(format!("{name}.srt")); + let content = std::fs::read_to_string(&case).expect("a readable case"); + let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson"))); + let ours = ours(&content); + assert!( + ours.len() > theirs.len(), + "{name} no longer recovers more than the original — update KNOWN.md" + ); + // Everything the original did find, this finds too, unchanged. + for cue in &theirs { + assert!(ours.contains(cue), "{name} lost a cue the original kept"); + } + } +} + +/// Catches a golden file drifting away from an original that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_files_still_match_the_live_original() { + for case in parity::cases_with_extension("cuenorm", "srt") { + let name = parity::case_name(&case); + let content = std::fs::read_to_string(&case).expect("a readable case"); + let live = parity::run_node_oracle("cuenorm.mjs", &content).expect("the oracle runs"); + let golden = parity::read_golden_text(&format!("cuenorm/{name}.ndjson")); + assert_eq!(live, golden, "golden file for {name} is stale"); + } +}