funkpower/catena
1
0
Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-10-23 03:07:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

cuekit and cuenorm: three subtitle formats, one cue stream

Browse Source
This commit is contained in:
funkpower
2026-09-19 11:54:36 -04:00
parent 33c2e40a1a
commit c7d0767376
42 changed files with 1433 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+22
View File
@@ -166,6 +166,27 @@ dependencies = [
"thiserror",
]
[[package]]
name = "cuekit"
version = "0.1.0"
dependencies = [
"corpus-core",
"regex",
"schemars",
"serde",
"serde_json",
"textkit",
]
[[package]]
name = "cuenorm"
version = "0.1.0"
dependencies = [
"clap",
"corpus-core",
"cuekit",
]
[[package]]
name = "dyn-clone"
version = "1.0.20"
@@ -246,6 +267,7 @@ version = "0.1.0"
dependencies = [
"chunkmd",
"corpus-core",
"cuekit",
"mdscrub",
"pdftriage",
"serde_json",
Cargo.toml
+2
View File
@@ -11,10 +11,12 @@ repository = "https://git.pumpin.online/funkpower/catena"
[workspace.dependencies]
corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
cuekit = { path = "crates/cuekit", version = "0.1.0" }
textkit = { path = "crates/textkit", version = "0.1.0" }
chunkmd = { path = "crates/chunkmd", version = "0.1.0" }
mdscrub = { path = "crates/mdscrub", version = "0.1.0" }
pdftriage = { path = "crates/pdftriage", version = "0.1.0" }
cuenorm = { path = "crates/cuenorm", version = "0.1.0" }
clap = { version = "4", features = ["derive"] }
fancy-regex = "0.16"
crates/corpus-core/src/chunk.rs
+4 -10
View File
@@ -26,18 +26,12 @@ pub struct Chunk {
/// What part of the work the passage belongs to.
pub role: Role,
/// Seconds from the start of the recording, for timed sources.
#[serde(
default,
serialize_with = "crate::float::serialize_option",
deserialize_with = "crate::float::deserialize_option"
)]
#[serde(default, with = "crate::finite::option")]
#[schemars(with = "Option<f64>")]
pub start_time: Option<f64>,
/// Seconds from the start of the recording, for timed sources.
#[serde(
default,
serialize_with = "crate::float::serialize_option",
deserialize_with = "crate::float::deserialize_option"
)]
#[serde(default, with = "crate::finite::option")]
#[schemars(with = "Option<f64>")]
pub end_time: Option<f64>,
/// Set tags; see [`flags`]. Sorted and deduplicated on the way in and on
/// the way out, so the same set of flags is always the same bytes
crates/corpus-core/src/finite.rs
+57
View File
@@ -0,0 +1,57 @@
//! Serde plumbing for floats that JSON can actually carry.
//!
//! Use it as `#[serde(with = "corpus_core::finite")]` on an `f64`, or
//! `#[serde(default, with = "corpus_core::finite::option")]` on an
//! `Option<f64>`.
//!
//! JSON has no spelling for NaN or infinity, and `serde_json` writes them as
//! `null`. That is a silent corruption in both directions: a score written as
//! `null` fails to read back and kills the stream at the consumer, and a
//! timestamp written as `null` reads back as "no timestamp" with nothing to
//! say it was ever anything else. So a non-finite number is refused where it
//! is produced, which is the only place anyone can still fix it.
use serde::{Deserialize, Deserializer, Serializer, de::Error as _, ser::Error as _};
/// Write a float, refusing one JSON cannot carry.
pub fn serialize<S: Serializer>(value: &f64, out: S) -> Result<S::Ok, S::Error> {
if !value.is_finite() {
return Err(S::Error::custom(format!(
"{value} cannot be written as JSON"
)));
}
out.serialize_f64(*value)
}
/// Read a float, refusing one that is not finite.
pub fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result<f64, D::Error> {
let value = f64::deserialize(input)?;
if !value.is_finite() {
return Err(D::Error::custom(format!("{value} is not a finite number")));
}
Ok(value)
}
/// The same, for a field that may be absent.
pub mod option {
use serde::{Deserialize, Deserializer, Serializer, de::Error as _};
/// Write an optional float, refusing one JSON cannot carry.
pub fn serialize<S: Serializer>(value: &Option<f64>, out: S) -> Result<S::Ok, S::Error> {
match value {
Some(value) => super::serialize(value, out),
None => out.serialize_none(),
}
}
/// Read an optional float, refusing one that is not finite.
pub fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result<Option<f64>, D::Error> {
let value = Option::<f64>::deserialize(input)?;
match value {
Some(value) if !value.is_finite() => {
Err(D::Error::custom(format!("{value} is not a finite number")))
}
value => Ok(value),
}
}
}
crates/corpus-core/src/float.rs
-49
View File
@@ -1,49 +0,0 @@
//! Serde plumbing for floats that JSON can actually carry.
//!
//! JSON has no spelling for NaN or infinity, and `serde_json` writes them as
//! `null`. That is a silent corruption in both directions: a score written as
//! `null` fails to read back and kills the stream at the consumer, and a
//! timestamp written as `null` reads back as "no timestamp" with nothing to
//! say it was ever anything else. So a non-finite number is refused where it
//! is produced, which is the only place anyone can still fix it.
use serde::{Deserialize, Deserializer, Serializer, de::Error as _, ser::Error as _};
pub(crate) fn serialize<S: Serializer>(value: &f64, out: S) -> Result<S::Ok, S::Error> {
if !value.is_finite() {
return Err(S::Error::custom(format!(
"{value} cannot be written as JSON"
)));
}
out.serialize_f64(*value)
}
pub(crate) fn serialize_option<S: Serializer>(
value: &Option<f64>,
out: S,
) -> Result<S::Ok, S::Error> {
match value {
Some(value) => serialize(value, out),
None => out.serialize_none(),
}
}
pub(crate) fn deserialize<'de, D: Deserializer<'de>>(input: D) -> Result<f64, D::Error> {
let value = f64::deserialize(input)?;
if !value.is_finite() {
return Err(D::Error::custom(format!("{value} is not a finite number")));
}
Ok(value)
}
pub(crate) fn deserialize_option<'de, D: Deserializer<'de>>(
input: D,
) -> Result<Option<f64>, D::Error> {
let value = Option::<f64>::deserialize(input)?;
match value {
Some(value) if !value.is_finite() => {
Err(D::Error::custom(format!("{value} is not a finite number")))
}
value => Ok(value),
}
}
crates/corpus-core/src/hit.rs
+2 -4
View File
@@ -22,10 +22,8 @@ pub struct Hit {
/// Position within the channel's own ranking, from one.
pub rank: u32,
/// The channel's raw score, on whatever scale that channel uses.
#[serde(
serialize_with = "crate::float::serialize",
deserialize_with = "crate::float::deserialize"
)]
#[serde(with = "crate::finite")]
#[schemars(with = "f64")]
pub score: f64,
/// Anything the producer wants to carry through fusion untouched.
///
crates/corpus-core/src/lib.rs
+1 -1
View File
@@ -11,7 +11,7 @@
pub mod chunk;
pub mod doc;
pub mod exit;
mod float;
pub mod finite;
pub mod hit;
pub mod ndjson;
crates/cuekit/Cargo.toml
+21
View File
@@ -0,0 +1,21 @@
[package]
name = "cuekit"
description = "The cue model everything timed shares, and the subtitle formats it is read from."
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
corpus-core.workspace = true
regex.workspace = true
schemars.workspace = true
serde.workspace = true
textkit.workspace = true
[dev-dependencies]
serde_json.workspace = true
[lints]
workspace = true
crates/cuekit/src/ass.rs
+179
View File
@@ -0,0 +1,179 @@
//! Advanced SubStation Alpha, which is a spreadsheet pretending to be a
//! subtitle file.
//!
//! Events are rows under a `Format:` line that names the columns, so the
//! column order has to be read rather than assumed — a file written by one
//! tool puts the text last after nine fields, and one written by another puts
//! it last after eight.
use std::sync::LazyLock;
use regex::Regex;
use crate::cue::Cue;
use crate::text;
/// An override block: `{\an8}`, `{\pos(120,400)}`, `{\i1}`.
static OVERRIDE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\{[^}]*\}").expect("valid"));
/// A drawing command block, which is a shape rather than speech.
static DRAWING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\{[^}]*\\p[1-9]").expect("valid"));
/// Parse an ASS or SSA file.
pub fn parse(content: &str) -> Vec<Cue> {
let mut columns: Option<Vec<String>> = None;
let mut in_events = false;
let mut cues = Vec::new();
for line in content.split('\n') {
let line = line.trim_matches(textkit::is_space);
if line.starts_with('[') {
in_events = line.eq_ignore_ascii_case("[events]");
continue;
}
if !in_events {
continue;
}
if let Some(format) = line.strip_prefix("Format:") {
columns = Some(
format
.split(',')
.map(|name| name.trim_matches(textkit::is_space).to_lowercase())
.collect(),
);
continue;
}
let Some(event) = line.strip_prefix("Dialogue:") else {
continue;
};
let Some(columns) = columns.as_ref() else {
continue;
};
if let Some(cue) = dialogue(event, columns) {
cues.push(cue);
}
}
cues
}
fn dialogue(event: &str, columns: &[String]) -> Option<Cue> {
let start_at = columns.iter().position(|name| name == "start")?;
let end_at = columns.iter().position(|name| name == "end")?;
let text_at = columns.iter().position(|name| name == "text")?;
// The text field is last and may itself contain commas, so the row is
// split only as many times as there are columns before it.
let mut fields: Vec<&str> = event.splitn(columns.len(), ',').collect();
if fields.len() < columns.len() {
return None;
}
let text = if text_at == columns.len() - 1 {
fields.pop().unwrap_or("")
} else {
fields.get(text_at).copied().unwrap_or("")
};
let start = timestamp(fields.get(start_at)?.trim_matches(textkit::is_space));
let end = timestamp(fields.get(end_at)?.trim_matches(textkit::is_space));
if !start.is_finite() || !end.is_finite() {
return None;
}
// A drawing command is a shape, not speech, and reads as a string of
// coordinates if it is kept.
if DRAWING.is_match(text) {
return None;
}
let text = OVERRIDE.replace_all(text, "");
let text = text
.replace("\\N", " ")
.replace("\\n", " ")
.replace("\\h", " ");
let text = text::clean(&text);
(!text.is_empty()).then(|| Cue::new(start, end, text))
}
/// Seconds from an `H:MM:SS.cc` timestamp.
pub fn timestamp(text: &str) -> f64 {
let parts: Vec<&str> = text.split(':').collect();
match parts.as_slice() {
[hours, minutes, seconds] => {
crate::jsnum::number(hours) * 3600.0
+ crate::jsnum::number(minutes) * 60.0
+ crate::jsnum::parse_float(seconds)
}
[minutes, seconds] => {
crate::jsnum::number(minutes) * 60.0 + crate::jsnum::parse_float(seconds)
}
_ => f64::NAN,
}
}
#[cfg(test)]
mod tests {
use super::*;
const FILE: &str = "[Script Info]\n\
Title: A Recording\n\
\n\
[V4+ Styles]\n\
Format: Name, Fontname\n\
Style: Default,Arial\n\
\n\
[Events]\n\
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n\
Dialogue: 0,0:00:01.00,0:00:03.00,Default,,0,0,0,,The mill stood at the northern end.\n\
Dialogue: 0,0:00:03.00,0:00:05.50,Default,,0,0,0,,{\\an8}Its wheel was undershot,\\Nwhich was wasteful.\n";
#[test]
fn events_become_cues() {
let cues = parse(FILE);
assert_eq!(cues.len(), 2);
assert_eq!(cues[0].start, 1.0);
assert_eq!(cues[0].end, 3.0);
assert_eq!(cues[0].text, "The mill stood at the northern end.");
}
#[test]
fn override_blocks_and_line_breaks_come_out() {
let cues = parse(FILE);
assert_eq!(cues[1].text, "Its wheel was undershot, which was wasteful.");
}
#[test]
fn a_comma_in_the_text_is_not_a_column_break() {
let cues = parse(FILE);
assert!(cues[1].text.contains(','), "{:?}", cues[1].text);
}
#[test]
fn a_drawing_command_is_not_speech() {
let file = FILE.to_string()
+ "Dialogue: 0,0:00:06.00,0:00:07.00,Default,,0,0,0,,{\\p1}m 0 0 l 100 0 100 100\n";
assert_eq!(parse(&file).len(), 2);
}
#[test]
fn a_row_outside_the_events_section_is_ignored() {
let file = "[Script Info]\nDialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,no\n";
assert!(parse(file).is_empty());
}
#[test]
fn a_column_order_of_its_own_is_honoured() {
let file = "[Events]\n\
Format: Start, End, Text\n\
Dialogue: 0:00:01.00,0:00:02.00,Reordered, with a comma\n";
let cues = parse(file);
assert_eq!(cues.len(), 1);
assert_eq!(cues[0].text, "Reordered, with a comma");
}
#[test]
fn a_timestamp_is_read_in_seconds() {
assert_eq!(timestamp("0:00:01.50"), 1.5);
assert_eq!(timestamp("1:02:03.04"), 3723.04);
assert!(timestamp("rubbish").is_nan());
}
}
crates/cuekit/src/cue.rs
+82
View File
@@ -0,0 +1,82 @@
//! One line of a transcript, and when it was said.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
/// A subtitle cue: a span of time and what was said in it.
///
/// Deliberately smaller than a chunk. A cue is what a subtitle file holds —
/// a few seconds, often half a sentence — and turning cues into passages
/// worth retrieving is a later stage's job.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct Cue {
/// Record version.
#[serde(default = "version")]
pub v: u32,
/// Seconds from the start of the recording.
#[serde(with = "corpus_core::finite")]
#[schemars(with = "f64")]
pub start: f64,
/// Seconds from the start of the recording.
#[serde(with = "corpus_core::finite")]
#[schemars(with = "f64")]
pub end: f64,
/// What was said, with the markup taken out.
pub text: String,
}
fn version() -> u32 {
corpus_core::V
}
impl Cue {
/// A cue.
pub fn new(start: f64, end: f64, text: impl Into<String>) -> Self {
Self {
v: corpus_core::V,
start,
end,
text: text.into(),
}
}
/// How long the cue lasts. Negative if the file says it ends before it
/// starts, which some do.
pub fn seconds(&self) -> f64 {
self.end - self.start
}
}
impl corpus_core::Record for Cue {
fn version(&self) -> u32 {
self.v
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_field_is_emitted() {
let cue = Cue::new(12.5, 15.0, "the mill stood at the northern end");
assert_eq!(
serde_json::to_string(&cue).unwrap(),
r#"{"v":1,"start":12.5,"end":15.0,"text":"the mill stood at the northern end"}"#
);
}
#[test]
fn a_cue_reads_back_identical() {
let cue = Cue::new(0.0, 1.0 / 3.0, "x");
let text = serde_json::to_string(&cue).unwrap();
let back: Cue = serde_json::from_str(&text).unwrap();
assert_eq!(back.end.to_bits(), cue.end.to_bits());
}
#[test]
fn a_time_that_json_cannot_carry_is_refused() {
let cue = Cue::new(f64::NAN, 1.0, "x");
assert!(serde_json::to_string(&cue).is_err());
}
}
crates/cuekit/src/dedup.rs
+66
View File
@@ -0,0 +1,66 @@
//! Collapsing the repetition automatic captioning produces.
use crate::cue::Cue;
/// Merge consecutive cues whose text is identical, keeping the earliest start
/// and the latest end.
///
/// Automatic captioning emits a rolling window: the same line appears in cue
/// after cue as the window slides, so a ten-minute stretch of speech can
/// arrive as three times as many cues as there are sentences. Indexed as they
/// come, every one of those is a separate hit for the same words at almost
/// the same timestamp, which buries everything else in a result list.
pub fn merge_repeats(cues: &[Cue]) -> Vec<Cue> {
let mut merged: Vec<Cue> = Vec::with_capacity(cues.len());
for cue in cues {
match merged.last_mut() {
Some(previous) if previous.text == cue.text => {
previous.end = previous.end.max(cue.end);
}
_ => merged.push(cue.clone()),
}
}
merged
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_repeated_line_becomes_one_cue_spanning_both() {
let cues = [
Cue::new(0.0, 2.0, "the mill stood"),
Cue::new(1.0, 3.0, "the mill stood"),
Cue::new(3.0, 4.0, "at the northern end"),
];
let merged = merge_repeats(&cues);
assert_eq!(merged.len(), 2);
assert_eq!(merged[0].start, 0.0);
assert_eq!(merged[0].end, 3.0);
assert_eq!(merged[1].text, "at the northern end");
}
#[test]
fn a_line_that_comes_back_later_is_kept_separately() {
let cues = [
Cue::new(0.0, 1.0, "yes"),
Cue::new(1.0, 2.0, "no"),
Cue::new(2.0, 3.0, "yes"),
];
assert_eq!(merge_repeats(&cues).len(), 3);
}
#[test]
fn an_end_that_goes_backwards_does_not_shorten_the_merged_cue() {
let cues = [Cue::new(0.0, 9.0, "x"), Cue::new(1.0, 2.0, "x")];
let merged = merge_repeats(&cues);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].end, 9.0);
}
#[test]
fn nothing_in_nothing_out() {
assert!(merge_repeats(&[]).is_empty());
}
}
crates/cuekit/src/jsnum.rs
+176
View File
@@ -0,0 +1,176 @@
//! Reading numbers the way the JavaScript this was ported from reads them.
//!
//! The timestamp parser it came from calls `Number()` on the hours and the
//! minutes and `parseFloat()` on the seconds. The two disagree about almost
//! everything an ill-formed timestamp can contain — `Number("")` is zero
//! where `parseFloat("")` is not a number, `Number("1a")` is not a number
//! where `parseFloat("1a")` is one — and a subtitle file that has been cut,
//! concatenated or hand-edited produces exactly those. Reproducing both is
//! what makes a malformed file drop the same cues here as there.
/// What JavaScript's `Number(text)` gives.
///
/// Whitespace is trimmed, an empty string is zero, the radix prefixes are
/// honoured, and anything else that is not a complete numeric literal is not
/// a number. Note that `Number` accepts `Infinity` but not `NaN`, and neither
/// of the spellings Rust's own parser accepts for them.
pub fn number(text: &str) -> f64 {
let text = text.trim_matches(is_js_space);
if text.is_empty() {
return 0.0;
}
if let Some(value) = radix_literal(text) {
return value;
}
let body = text.strip_prefix(['+', '-']).unwrap_or(text);
// Rust's parser accepts spellings JavaScript's does not.
if body.eq_ignore_ascii_case("inf")
|| body.eq_ignore_ascii_case("nan")
|| body.eq_ignore_ascii_case("infinity")
{
return if body.eq_ignore_ascii_case("infinity") {
match text.starts_with('-') {
true => f64::NEG_INFINITY,
false => f64::INFINITY,
}
} else {
f64::NAN
};
}
text.parse().unwrap_or(f64::NAN)
}
/// What JavaScript's `parseFloat(text)` gives: the longest decimal number at
/// the front, ignoring whatever follows, and not a number when there is none.
pub fn parse_float(text: &str) -> f64 {
let text = text.trim_start_matches(is_js_space);
let prefix = decimal_prefix(text);
if prefix.is_empty() {
return f64::NAN;
}
prefix.parse().unwrap_or(f64::NAN)
}
/// The longest leading run that is a decimal number, by JavaScript's grammar.
fn decimal_prefix(text: &str) -> &str {
let bytes = text.as_bytes();
let mut at = 0;
if matches!(bytes.first(), Some(b'+' | b'-')) {
at = 1;
}
if text[at..].starts_with("Infinity") {
return &text[..at + "Infinity".len()];
}
let mut end = at;
let digits = |from: usize| {
let mut to = from;
while to < bytes.len() && bytes[to].is_ascii_digit() {
to += 1;
}
to
};
end = digits(end);
if end < bytes.len() && bytes[end] == b'.' {
end = digits(end + 1);
}
// A run with no digits at all on either side of the point is not a
// number: `.` and `+.` are nothing, `.5` and `5.` are numbers.
if !text[at..end].chars().any(|c| c.is_ascii_digit()) {
return "";
}
if end < bytes.len() && (bytes[end] | 0x20) == b'e' {
let mut exponent = end + 1;
if matches!(bytes.get(exponent), Some(b'+' | b'-')) {
exponent += 1;
}
let after = digits(exponent);
if after > exponent {
end = after;
}
}
&text[..end]
}
fn radix_literal(text: &str) -> Option<f64> {
let (radix, digits) = match text.get(..2)?.to_ascii_lowercase().as_str() {
"0x" => (16, &text[2..]),
"0o" => (8, &text[2..]),
"0b" => (2, &text[2..]),
_ => return None,
};
if digits.is_empty() {
return Some(f64::NAN);
}
Some(match u64::from_str_radix(digits, radix) {
Ok(value) => value as f64,
Err(_) => f64::NAN,
})
}
/// JavaScript's whitespace for the purpose of trimming a numeric string:
/// Unicode whitespace plus the byte-order mark.
fn is_js_space(c: char) -> bool {
c.is_whitespace() || c == '\u{feff}'
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn number_reads_ordinary_numbers() {
assert_eq!(number("12"), 12.0);
assert_eq!(number(" 07 "), 7.0);
assert_eq!(number("1.5"), 1.5);
assert_eq!(number("1e2"), 100.0);
assert_eq!(number("-3"), -3.0);
}
#[test]
fn number_of_nothing_is_zero() {
assert_eq!(number(""), 0.0);
assert_eq!(number(" "), 0.0);
}
#[test]
fn number_refuses_a_partial_literal() {
assert!(number("1a").is_nan());
assert!(number("12:").is_nan());
assert!(number("--1").is_nan());
}
#[test]
fn number_does_not_accept_the_rust_spellings() {
assert!(number("inf").is_nan());
assert!(number("NaN").is_nan());
assert_eq!(number("Infinity"), f64::INFINITY);
assert_eq!(number("-Infinity"), f64::NEG_INFINITY);
}
#[test]
fn number_honours_the_radix_prefixes() {
assert_eq!(number("0x10"), 16.0);
assert_eq!(number("0b101"), 5.0);
assert_eq!(number("0o17"), 15.0);
}
#[test]
fn parse_float_takes_the_longest_prefix() {
assert_eq!(parse_float("23.456"), 23.456);
assert_eq!(parse_float("23.456abc"), 23.456);
assert_eq!(parse_float(" 1.5xyz"), 1.5);
assert_eq!(parse_float("5."), 5.0);
assert_eq!(parse_float(".5"), 0.5);
assert_eq!(parse_float("1e3x"), 1000.0);
assert_eq!(parse_float("1e"), 1.0);
}
#[test]
fn parse_float_of_nothing_is_not_a_number() {
assert!(parse_float("").is_nan());
assert!(parse_float("abc").is_nan());
assert!(parse_float(".").is_nan());
assert!(parse_float("+").is_nan());
}
}
crates/cuekit/src/lib.rs
+114
View File
@@ -0,0 +1,114 @@
//! The cue model everything timed shares, and the subtitle formats it is
//! read from.
//!
//! Three formats go in — SubRip, WebVTT and SubStation Alpha — and one cue
//! stream comes out. Everything downstream of this works on cues and never
//! has to know which of the three a recording arrived with.
//!
//! The SubRip reader is a port, quirks kept, of the parser a podcast archive
//! was indexed with; the other two are new, and are written to the same
//! shape. Where the port deliberately differs, `parity/KNOWN.md` says so.
pub mod ass;
pub mod cue;
pub mod dedup;
pub mod jsnum;
pub mod srt;
pub mod text;
pub use cue::Cue;
pub use dedup::merge_repeats;
/// Which subtitle format a file is written in.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Format {
/// SubRip.
SubRip,
/// WebVTT, which this reads with the SubRip reader.
WebVtt,
/// SubStation Alpha, advanced or otherwise.
SubStationAlpha,
}
impl Format {
/// What this format is called.
pub fn name(self) -> &'static str {
match self {
Format::SubRip => "srt",
Format::WebVtt => "vtt",
Format::SubStationAlpha => "ass",
}
}
}
/// Work out which format a file is, from its content rather than its name.
///
/// A subtitle file arrives named for the format someone expected it to be in.
/// The content says what it actually is, and the three are easy to tell
/// apart: WebVTT announces itself, SubStation Alpha has sections in square
/// brackets, and SubRip has neither.
pub fn detect(content: &str) -> Format {
let head = content.trim_start_matches(|c: char| textkit::is_space(c) || c == '\u{feff}');
if head.starts_with("WEBVTT") {
return Format::WebVtt;
}
for line in head.split('\n').take(40) {
let line = line.trim_matches(textkit::is_space);
if line.starts_with("[Script Info]") || line.eq_ignore_ascii_case("[events]") {
return Format::SubStationAlpha;
}
if line.starts_with("Dialogue:") || line.starts_with("ScriptType:") {
return Format::SubStationAlpha;
}
}
Format::SubRip
}
/// Read a subtitle file of any supported format into cues.
pub fn parse(content: &str) -> Vec<Cue> {
parse_as(content, detect(content))
}
/// Read a subtitle file that is known to be in `format`.
pub fn parse_as(content: &str, format: Format) -> Vec<Cue> {
match format {
Format::SubRip | Format::WebVtt => srt::parse(content),
Format::SubStationAlpha => ass::parse(content),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_file_is_recognised_by_what_is_in_it() {
assert_eq!(
detect("WEBVTT\n\n00:00.000 --> 00:01.000\nx"),
Format::WebVtt
);
assert_eq!(detect("\u{feff}WEBVTT\n"), Format::WebVtt);
assert_eq!(detect("[Script Info]\nTitle: x\n"), Format::SubStationAlpha);
assert_eq!(
detect("[Events]\nFormat: Start, End, Text\n"),
Format::SubStationAlpha
);
assert_eq!(
detect("1\n00:00:01,000 --> 00:00:02,000\nx\n"),
Format::SubRip
);
assert_eq!(detect(""), Format::SubRip);
}
#[test]
fn all_three_formats_give_the_same_cue() {
let srt = "1\n00:00:01,000 --> 00:00:03,000\nThe mill stood.\n";
let vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:03.000\nThe mill stood.\n";
let ass = "[Events]\nFormat: Start, End, Text\n\
Dialogue: 0:00:01.00,0:00:03.00,The mill stood.\n";
let expected = vec![Cue::new(1.0, 3.0, "The mill stood.")];
assert_eq!(parse(srt), expected);
assert_eq!(parse(vtt), expected);
assert_eq!(parse(ass), expected);
}
}
crates/cuekit/src/srt.rs
+138
View File
@@ -0,0 +1,138 @@
//! SubRip, and the WebVTT that is close enough to share its reader.
//!
//! A port of the parser the podcast archive was indexed with, quirks kept:
//! the timestamp line is found by looking for an arrow anywhere in the block
//! rather than at a fixed position, only the first comma in a timestamp
//! becomes a point, and a block whose times do not read as numbers is
//! dropped whole rather than guessed at.
use std::sync::LazyLock;
use regex::Regex;
use crate::cue::Cue;
use crate::jsnum;
use crate::text;
/// Blocks are separated by one or more blank lines.
static BLOCKS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n\n+").expect("a valid pattern"));
/// Parse a SubRip file.
pub fn parse(content: &str) -> Vec<Cue> {
let mut cues = Vec::new();
for block in BLOCKS.split(content) {
let lines: Vec<&str> = block.trim_matches(textkit::is_space).split('\n').collect();
let Some(at) = lines.iter().position(|line| line.contains("-->")) else {
continue;
};
let Some((start, end)) = times(lines[at]) else {
continue;
};
let text = text::clean(&lines[at + 1..].join(" "));
if !text.is_empty() {
cues.push(Cue::new(start, end, text));
}
}
cues
}
/// Read a timestamp line, or `None` if either end is not a number.
///
/// Each half is reduced to the one token that could be a timestamp, which is
/// what lets the same reader take a WebVTT cue line — where the end time is
/// followed by positioning settings — and an old SubRip file with screen
/// coordinates after it. See `parity/KNOWN.md`: the original reads each half
/// whole and drops such a cue entirely.
fn times(line: &str) -> Option<(f64, f64)> {
let (before, after) = line.split_once("-->")?;
let start = timestamp(before.split_whitespace().next_back().unwrap_or(""));
let end = timestamp(after.split_whitespace().next().unwrap_or(""));
(start.is_finite() && end.is_finite()).then_some((start, end))
}
/// Seconds from a `HH:MM:SS,mmm` or `MM:SS.mmm` timestamp.
///
/// Only the first comma is replaced, exactly as the original does — a
/// timestamp written with a comma for the thousands separator as well as for
/// the decimal point therefore parses as far as the second comma and no
/// further.
pub fn timestamp(text: &str) -> f64 {
let text = text.trim_matches(textkit::is_space);
let cleaned = text.replacen(',', ".", 1);
let parts: Vec<&str> = cleaned.split(':').collect();
match parts.len() {
3 => {
jsnum::number(parts[0]) * 3600.0
+ jsnum::number(parts[1]) * 60.0
+ jsnum::parse_float(parts[2])
}
2 => jsnum::number(parts[0]) * 60.0 + jsnum::parse_float(parts[1]),
_ => f64::NAN,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_timestamp_is_read_in_seconds() {
assert_eq!(timestamp("00:01:23,456"), 83.456);
assert_eq!(timestamp("01:00:00,000"), 3600.0);
assert_eq!(timestamp("02:03.500"), 123.5);
}
#[test]
fn a_timestamp_that_is_not_one_is_not_a_number() {
assert!(timestamp("").is_nan());
assert!(timestamp("nonsense").is_nan());
assert!(timestamp("00:00:00:00").is_nan());
}
#[test]
fn a_file_parses_into_cues() {
let srt = "1\n00:00:01,000 --> 00:00:03,000\nThe mill stood at the northern end.\n\n\
2\n00:00:03,000 --> 00:00:05,500\nIts wheel was undershot,\nwhich was wasteful.\n";
let cues = parse(srt);
assert_eq!(cues.len(), 2);
assert_eq!(cues[0].start, 1.0);
assert_eq!(cues[0].end, 3.0);
assert_eq!(cues[0].text, "The mill stood at the northern end.");
assert_eq!(cues[1].text, "Its wheel was undershot, which was wasteful.");
}
#[test]
fn a_block_with_no_arrow_is_skipped() {
assert!(parse("WEBVTT\n\nsome preamble\n").is_empty());
}
#[test]
fn a_block_with_unreadable_times_is_dropped_whole() {
let srt = "1\nnonsense --> also nonsense\ntext\n\n\
2\n00:00:03,000 --> 00:00:05,000\nkept\n";
let cues = parse(srt);
assert_eq!(cues.len(), 1);
assert_eq!(cues[0].text, "kept");
}
#[test]
fn a_block_with_no_text_is_dropped() {
assert!(parse("1\n00:00:01,000 --> 00:00:03,000\n[Music]\n").is_empty());
}
#[test]
fn several_blank_lines_still_separate_one_block_from_the_next() {
let srt = "1\n00:00:01,000 --> 00:00:02,000\none\n\n\n\n\
2\n00:00:02,000 --> 00:00:03,000\ntwo\n";
assert_eq!(parse(srt).len(), 2);
}
#[test]
fn webvtt_timing_and_settings_read_too() {
let vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:03.000 line:0 position:20%\nThe mill.\n";
let cues = parse(vtt);
assert_eq!(cues.len(), 1);
assert_eq!(cues[0].start, 1.0);
assert_eq!(cues[0].end, 3.0);
}
}
crates/cuekit/src/text.rs
+63
View File
@@ -0,0 +1,63 @@
//! Cleaning the markup out of a cue's text.
use std::sync::LazyLock;
use regex::Regex;
/// Styling and speaker tags: `<i>`, `<v Speaker>`, `<c.colour>`.
static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"<[^>]+>").expect("a valid pattern"));
/// Bracketed sound cues: `[Music]`, `[Applause]`, `[inaudible]`.
///
/// Not speech, and a search that matches them matches every recording with a
/// theme tune.
static BRACKETED: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\[.*?\]").expect("a valid pattern"));
/// Strip markup from a cue's text.
pub fn clean(text: &str) -> String {
let text = TAG.replace_all(text, "");
let text = BRACKETED.replace_all(&text, "");
text.trim_matches(is_js_space).to_string()
}
/// What JavaScript's `String.trim` removes: Unicode whitespace and the
/// byte-order mark.
fn is_js_space(c: char) -> bool {
c.is_whitespace() || c == '\u{feff}'
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn styling_tags_go() {
assert_eq!(clean("<i>The mill</i> stood"), "The mill stood");
assert_eq!(clean("<v Speaker>Hello"), "Hello");
}
#[test]
fn bracketed_sound_cues_go() {
assert_eq!(clean("[Music] The mill stood [Applause]"), "The mill stood");
assert_eq!(clean("[Music]"), "");
}
#[test]
fn the_bracket_match_is_not_greedy() {
assert_eq!(clean("[a] kept [b]"), "kept");
}
#[test]
fn a_stray_pair_of_angle_brackets_takes_what_is_between_them() {
// Faithful to the original, which uses the same pattern. A spoken
// line with a less-than and a greater-than in it is vanishingly rare
// next to the styling tags this exists to remove.
assert_eq!(clean("a < b and c > d"), "a d");
}
#[test]
fn a_lone_angle_bracket_is_left_alone() {
assert_eq!(clean("a < b"), "a < b");
}
}
crates/cuenorm/Cargo.toml
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "cuenorm"
description = "Any subtitle format in, one normalised cue stream out."
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
clap.workspace = true
corpus-core.workspace = true
cuekit.workspace = true
[lints]
workspace = true
crates/cuenorm/src/main.rs
+94
View File
@@ -0,0 +1,94 @@
//! Any subtitle format in, one normalised cue stream out.
use std::io::{BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use clap::Parser;
use corpus_core::{exit, ndjson};
/// Read a subtitle file — SubRip, WebVTT or SubStation Alpha — and write one
/// cue per line as JSON.
///
/// The format is worked out from the content, not from the name, because a
/// subtitle file is named for the format somebody expected it to be in.
/// Markup and bracketed sound cues are taken out, and consecutive cues with
/// identical text are merged, which is what automatic captioning needs: it
/// repeats each line across overlapping windows, so the same sentence arrives
/// three or four times over.
#[derive(Parser)]
#[command(
version,
about,
long_about = None,
after_help = "\
Example:
cuenorm episode.vtt | cuepack --seconds 60 > passages.ndjson"
)]
struct Args {
/// Subtitle file to read. Reads stdin when absent.
path: Option<PathBuf>,
/// Keep consecutive repeats instead of merging them.
#[arg(long)]
keep_repeats: bool,
}
fn main() -> ExitCode {
let args = Args::parse();
let content = match read(args.path.as_deref()) {
Ok(content) => content,
Err(e) => {
eprintln!("cuenorm: {e}");
return ExitCode::from(exit::IO);
}
};
let cues = cuekit::parse(&content);
let cues = if args.keep_repeats {
cues
} else {
cuekit::merge_repeats(&cues)
};
let mut out = BufWriter::new(std::io::stdout().lock());
for cue in &cues {
if let Err(e) = ndjson::write_line(&mut out, cue) {
// A consumer that stopped reading is not a failure.
if e.is_broken_pipe() {
return ExitCode::from(exit::OK);
}
eprintln!("cuenorm: {e}");
return ExitCode::from(exit::IO);
}
}
match out.flush() {
Ok(()) => ExitCode::from(exit::OK),
Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
Err(e) => {
eprintln!("cuenorm: {e}");
ExitCode::from(exit::IO)
}
}
}
/// Read the file as text, tolerating the encodings a subtitle file arrives in.
///
/// Not UTF-8 is common: a subtitle file downloaded from anywhere is as likely
/// to be Latin-1, and refusing it would make the tool useless for exactly the
/// archives that need it. Undecodable bytes become the replacement character
/// rather than an error, because a cue with one wrong character still has its
/// timing and most of its words.
fn read(path: Option<&Path>) -> std::io::Result<String> {
let bytes = match path {
Some(path) => std::fs::read(path)?,
None => {
let mut bytes = Vec::new();
std::io::stdin().read_to_end(&mut bytes)?;
bytes
}
};
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
parity/Cargo.toml
+1
View File
@@ -15,6 +15,7 @@ python = []
[dependencies]
chunkmd.workspace = true
cuekit.workspace = true
mdscrub.workspace = true
pdftriage.workspace = true
corpus-core.workspace = true
parity/KNOWN.md
+20
View File
@@ -18,6 +18,26 @@ affected and how often it occurs in practice.
---
## cuenorm — a cue line with trailing settings still reads
**Original:** the timestamp line is split on the arrow and each half is read
whole, so `00:00:01,000 --> 00:00:03,000 X1:040 X2:560` gives an end time
that is not a number and the cue is dropped entirely.
**Rust:** each half is reduced to the one token that could be a timestamp, so
the trailing text is ignored and the cue reads.
**Why this is acceptable:** it is what makes the same reader take WebVTT,
where positioning settings after the end time are not a malformation but the
format — `00:00:01.000 --> 00:00:03.000 line:0 position:20%` is an ordinary
cue. Without it the tool would need a second parser that differed in one
line. The divergence can only ever recover a cue that the original discarded;
there is no input where it changes a cue the original read.
**Measured:** `parity/cases/cuenorm/coordinates.srt` — one cue of two
recovered. Across the other eight cases, including deliberately malformed
timestamps, the two agree cue for cue.
## mdscrub — un-paginated text is not given a page 1 marker
**Python:** `_normalize_marker_pages` ends by wrapping any text it found no
parity/SOURCE.md
+6
View File
@@ -6,6 +6,7 @@ in this directory are the answers that Python gave, at one commit:
| Source | Commit |
|---|---|
| `book-pipeline` | `7499e3257572968b437ecfdc79feccc91b96585b` |
| `podcast-download-systemd` | `ab20139cd0defb400a0dd9ed9034e03c1d1e356d` |
Regenerating against a different commit will move the goldens. That is the
point — the diff is the review. Record the new commit here in the same change.
@@ -16,4 +17,9 @@ a checkout lives is a fact about one machine and must not be compiled in:
```sh
CATENA_PARITY_PYTHON=/path/to/venv/bin/python
CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src
CATENA_PARITY_NODE=node # only if it is not already on PATH
```
One oracle is JavaScript rather than Python, because the code it compares
against is. Its three functions are copied into `oracle/cuenorm.mjs` verbatim,
since the module they live in opens a database connection when it loads.
parity/cases/cuenorm/brackets.srt
+11
View File
@@ -0,0 +1,11 @@
1
00:00:01,000 --> 00:00:03,000
[Music]
2
00:00:03,000 --> 00:00:05,000
[Applause] The mill stood. [inaudible]
3
00:00:05,000 --> 00:00:07,000
[a] kept [b]
parity/cases/cuenorm/coordinates.srt
+7
View File
@@ -0,0 +1,7 @@
1
00:00:01,000 --> 00:00:03,000 X1:040 X2:560 Y1:050 Y2:100
The mill stood at the northern end.
2
00:00:03,000 --> 00:00:05,000
Its wheel was undershot.
parity/cases/cuenorm/crlf.srt
+8
View File
@@ -0,0 +1,8 @@
1
00:00:01,000 --> 00:00:03,000
The mill stood at the northern end,
where the fall was steepest.
2
00:00:03,000 --> 00:00:05,000
Its wheel was undershot.
parity/cases/cuenorm/malformed.srt
+20
View File
@@ -0,0 +1,20 @@
1
nonsense --> also nonsense
this block has unreadable times
2
00:00:03,000 --> 00:00:05,000
this one is fine
a block with no arrow at all
3
00:00:05,000 -->
this one has no end time
4
--> 00:00:09,000
this one has no start time
5
00:00:09,000 --> 00:00:11,000
parity/cases/cuenorm/multiline.srt
+9
View File
@@ -0,0 +1,9 @@
1
00:00:01,000 --> 00:00:03,000
The mill stood at the northern end,
where the fall of the river was steepest
and the ice came latest.
2
00:00:03,000 --> 00:00:05,000
Its wheel was undershot.
parity/cases/cuenorm/oddtimes.srt
+19
View File
@@ -0,0 +1,19 @@
1
02:03,500 --> 02:05,000
a two part timestamp
2
00:00:05,000 --> 00:00:07,000,000
an extra comma in the end time
3
0:0:8 --> 0:0:9
unpadded fields
4
00:00:09,000 --> 00:00:11,000
padded with spaces
5
1:2:3.456 --> 1:2:4.5
a point instead of a comma
parity/cases/cuenorm/plain.srt
+11
View File
@@ -0,0 +1,11 @@
1
00:00:01,000 --> 00:00:03,000
The mill stood at the northern end of the valley.
2
00:00:03,000 --> 00:00:05,500
Its wheel was undershot, which was wasteful.
3
00:00:05,500 --> 00:00:09,250
Nobody replaced it for two hundred years.
parity/cases/cuenorm/repeats.srt
+19
View File
@@ -0,0 +1,19 @@
1
00:00:00,000 --> 00:00:02,000
the mill stood at the
2
00:00:01,500 --> 00:00:03,500
the mill stood at the
3
00:00:03,000 --> 00:00:05,000
the mill stood at the
4
00:00:05,000 --> 00:00:07,000
northern end of the valley
5
00:00:07,000 --> 00:00:09,000
northern end of the valley
parity/cases/cuenorm/tags.srt
+11
View File
@@ -0,0 +1,11 @@
1
00:00:01,000 --> 00:00:03,000
<i>The mill</i> stood at the <b>northern</b> end.
2
00:00:03,000 --> 00:00:05,000
<font color="#ffffff">Its wheel was undershot.</font>
3
00:00:05,000 --> 00:00:07,000
a < b and c > d
parity/golden/cuenorm/brackets.ndjson
+2
View File
@@ -0,0 +1,2 @@
{"start":3,"end":5,"text":"The mill stood."}
{"start":5,"end":7,"text":"kept"}
parity/golden/cuenorm/coordinates.ndjson
+1
View File
@@ -0,0 +1 @@
{"start":3,"end":5,"text":"Its wheel was undershot."}
parity/golden/cuenorm/crlf.ndjson
+1
View File
@@ -0,0 +1 @@
{"start":1,"end":3,"text":"The mill stood at the northern end,\r where the fall was steepest.\r \r 2\r 00:00:03,000 --> 00:00:05,000\r Its wheel was undershot."}
parity/golden/cuenorm/malformed.ndjson
+1
View File
@@ -0,0 +1 @@
{"start":3,"end":5,"text":"this one is fine"}
parity/golden/cuenorm/multiline.ndjson
+2
View File
@@ -0,0 +1,2 @@
{"start":1,"end":3,"text":"The mill stood at the northern end, where the fall of the river was steepest and the ice came latest."}
{"start":3,"end":5,"text":"Its wheel was undershot."}
parity/golden/cuenorm/oddtimes.ndjson
+5
View File
@@ -0,0 +1,5 @@
{"start":123.5,"end":125,"text":"a two part timestamp"}
{"start":5,"end":7,"text":"an extra comma in the end time"}
{"start":8,"end":9,"text":"unpadded fields"}
{"start":9,"end":11,"text":"padded with spaces"}
{"start":3723.456,"end":3724.5,"text":"a point instead of a comma"}
parity/golden/cuenorm/plain.ndjson
+3
View File
@@ -0,0 +1,3 @@
{"start":1,"end":3,"text":"The mill stood at the northern end of the valley."}
{"start":3,"end":5.5,"text":"Its wheel was undershot, which was wasteful."}
{"start":5.5,"end":9.25,"text":"Nobody replaced it for two hundred years."}
parity/golden/cuenorm/repeats.ndjson
+2
View File
@@ -0,0 +1,2 @@
{"start":0,"end":5,"text":"the mill stood at the"}
{"start":5,"end":9,"text":"northern end of the valley"}
parity/golden/cuenorm/tags.ndjson
+3
View File
@@ -0,0 +1,3 @@
{"start":1,"end":3,"text":"The mill stood at the northern end."}
{"start":3,"end":5,"text":"Its wheel was undershot."}
{"start":5,"end":7,"text":"a d"}
parity/oracle/cuenorm.mjs
+86
View File
@@ -0,0 +1,86 @@
/**
* Parse SubRip the way the podcast archive was indexed, for comparison.
*
* The three functions below are the originals, copied out of the indexer
* verbatim and with their type annotations dropped — the module they live in
* opens a database connection when it loads, which an oracle must not do.
*
* node oracle/cuenorm.mjs < case.srt
*
* Writes one JSON cue per line.
*/
/** Parse an SRT timestamp (00:01:23,456) into seconds. */
function parseSrtTimestamp(ts) {
// SRT format: HH:MM:SS,mmm (comma separates ms)
const cleaned = ts.trim().replace(",", ".");
const parts = cleaned.split(":");
if (parts.length === 3) {
return Number(parts[0]) * 3600 + Number(parts[1]) * 60 + parseFloat(parts[2]);
}
if (parts.length === 2) {
return Number(parts[0]) * 60 + parseFloat(parts[1]);
}
return NaN;
}
/** Parse an SRT file content into cues. */
function parseSrt(content) {
const cues = [];
const blocks = content.split(/\n\n+/);
for (const block of blocks) {
const lines = block.trim().split("\n");
const tsLine = lines.find((l) => l.includes("-->"));
if (!tsLine) continue;
const [startStr, endStr] = tsLine.split("-->").map((s) => s.trim());
const startTime = parseSrtTimestamp(startStr);
const endTime = parseSrtTimestamp(endStr);
if (isNaN(startTime) || isNaN(endTime)) continue;
const tsIdx = lines.indexOf(tsLine);
const text = lines
.slice(tsIdx + 1)
.join(" ")
.replace(/<[^>]+>/g, "")
.replace(/\[.*?\]/g, "")
.trim();
if (text) {
cues.push({ startTime, endTime, text });
}
}
return cues;
}
/** Merge consecutive cues with identical text. */
function deduplicateCues(cues) {
if (cues.length === 0) return [];
const result = [{ ...cues[0] }];
for (let i = 1; i < cues.length; i++) {
const prev = result[result.length - 1];
const curr = cues[i];
if (curr.text === prev.text) {
prev.endTime = Math.max(prev.endTime, curr.endTime);
} else {
result.push({ ...curr });
}
}
return result;
}
const chunks = [];
for await (const chunk of process.stdin) chunks.push(chunk);
const content = Buffer.concat(chunks).toString("utf-8");
for (const cue of deduplicateCues(parseSrt(content))) {
process.stdout.write(
JSON.stringify({ start: cue.startTime, end: cue.endTime, text: cue.text }) + "\n",
);
}
parity/src/bin/regen.rs
+17 -1
View File
@@ -19,12 +19,13 @@ type Written = Result<Vec<String>, String>;
type Job = (&'static str, fn() -> Written);
fn main() -> ExitCode {
let jobs: [Job; 5] = [
let jobs: [Job; 6] = [
("tokens", tokens),
("unescape", unescape),
("chunkmd", chunkmd),
("mdscrub", mdscrub),
("pdftriage", pdftriage),
("cuenorm", cuenorm),
];
for (name, job) in jobs {
@@ -68,6 +69,21 @@ fn unescape() -> Written {
Ok(vec![path.display().to_string()])
}
fn cuenorm() -> Written {
let dir = parity::golden("cuenorm");
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
let mut written = Vec::new();
for case in parity::cases_with_extension("cuenorm", "srt") {
let content = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
let cues = parity::run_node_oracle("cuenorm.mjs", &content)?;
let path = dir.join(format!("{}.ndjson", parity::case_name(&case)));
std::fs::write(&path, cues).map_err(|e| e.to_string())?;
written.push(path.display().to_string());
}
Ok(written)
}
fn pdftriage() -> Written {
let dir = parity::golden("pdftriage");
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
parity/src/lib.rs
+32 -3
View File
@@ -35,6 +35,20 @@ pub fn oracle(name: &str) -> PathBuf {
dir().join("oracle").join(name)
}
/// Cases where `cuenorm` is meant to differ from the JavaScript it was
/// ported from, each one argued for in `KNOWN.md`.
pub const DIVERGENT_CUENORM: &[&str] = &["coordinates"];
/// The JavaScript runtime to run an oracle with.
pub fn node() -> String {
std::env::var("CATENA_PARITY_NODE").unwrap_or_else(|_| "node".to_string())
}
/// Run a JavaScript oracle over `input` and return its stdout.
pub fn run_node_oracle(script: &str, input: &str) -> Result<String, String> {
run_oracle_in(node(), script, &[], input)
}
/// The Python interpreter to run an oracle with.
///
/// Taken from `CATENA_PARITY_PYTHON`, because the interpreter that has the
@@ -55,18 +69,28 @@ pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
/// Run an oracle script with arguments.
pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<String, String> {
run_oracle_in(python(), script, args, input)
}
/// Run an oracle under a named runtime.
pub fn run_oracle_in(
runtime: String,
script: &str,
args: &[String],
input: &str,
) -> Result<String, String> {
use std::io::Write as _;
use std::process::{Command, Stdio};
let script = oracle(script);
let mut child = Command::new(python())
let mut child = Command::new(&runtime)
.arg(&script)
.args(args)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()
.map_err(|e| format!("running {} with {}: {e}", script.display(), python()))?;
.map_err(|e| format!("running {} with {runtime}: {e}", script.display()))?;
child
.stdin
.take()
@@ -85,12 +109,17 @@ pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<Str
/// Every case file for a tool, in a fixed order.
pub fn cases(tool: &str) -> Vec<PathBuf> {
cases_with_extension(tool, "md")
}
/// Every case file for a tool with a given extension, in a fixed order.
pub fn cases_with_extension(tool: &str, extension: &str) -> Vec<PathBuf> {
let dir = dir().join("cases").join(tool);
let mut found: Vec<PathBuf> = std::fs::read_dir(&dir)
.unwrap_or_else(|e| panic!("reading {}: {e}", dir.display()))
.filter_map(Result::ok)
.map(|entry| entry.path())
.filter(|path| path.extension().is_some_and(|ext| ext == "md"))
.filter(|path| path.extension().is_some_and(|ext| ext == extension))
.collect();
found.sort();
found
parity/tests/cuenorm.rs
+99
View File
@@ -0,0 +1,99 @@
//! Cue streams must match the JavaScript the archive was indexed with.
//!
//! A cue that shifts by a tenth of a second is a search result that lands in
//! the wrong sentence; a cue that is dropped is a sentence that cannot be
//! found at all. Both are invisible until somebody searches for exactly that
//! line.
use serde_json::Value;
/// A cue, as the two sides can be compared.
///
/// Not as JSON values: JavaScript writes a whole number as `5` where Rust
/// writes `5.0`, and the two parse to different kinds of JSON number for the
/// same quantity.
type Cue = (f64, f64, String);
fn cues(text: &str) -> Vec<Cue> {
text.lines()
.filter(|line| !line.trim().is_empty())
.map(|line| {
let cue: Value = serde_json::from_str(line).expect("valid JSON");
(
cue["start"].as_f64().expect("a start time"),
cue["end"].as_f64().expect("an end time"),
cue["text"].as_str().expect("text").to_string(),
)
})
.collect()
}
fn ours(content: &str) -> Vec<Cue> {
cuekit::merge_repeats(&cuekit::parse(content))
.iter()
.map(|cue| (cue.start, cue.end, cue.text.clone()))
.collect()
}
#[test]
fn every_case_parses_the_same_as_the_original() {
let cases = parity::cases_with_extension("cuenorm", "srt");
assert!(!cases.is_empty(), "no cases found");
for case in cases {
let name = parity::case_name(&case);
if parity::DIVERGENT_CUENORM.contains(&name.as_str()) {
continue;
}
let content = std::fs::read_to_string(&case).expect("a readable case");
let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson")));
let ours = ours(&content);
for (at, (ours, theirs)) in ours.iter().zip(&theirs).enumerate() {
assert_eq!(ours, theirs, "{name} cue {at} differs");
}
assert_eq!(
ours.len(),
theirs.len(),
"{name}: the original found {} cues, this found {}",
theirs.len(),
ours.len()
);
}
}
/// The divergence is that a cue line carrying screen coordinates after the
/// end time still reads. The original drops such a cue whole.
#[test]
fn the_documented_divergence_is_exactly_what_was_argued_for() {
for name in parity::DIVERGENT_CUENORM {
let case = parity::dir()
.join("cases/cuenorm")
.join(format!("{name}.srt"));
let content = std::fs::read_to_string(&case).expect("a readable case");
let theirs = cues(&parity::read_golden_text(&format!("cuenorm/{name}.ndjson")));
let ours = ours(&content);
assert!(
ours.len() > theirs.len(),
"{name} no longer recovers more than the original — update KNOWN.md"
);
// Everything the original did find, this finds too, unchanged.
for cue in &theirs {
assert!(ours.contains(cue), "{name} lost a cue the original kept");
}
}
}
/// Catches a golden file drifting away from an original that has moved on.
#[test]
#[cfg(feature = "python")]
fn the_golden_files_still_match_the_live_original() {
for case in parity::cases_with_extension("cuenorm", "srt") {
let name = parity::case_name(&case);
let content = std::fs::read_to_string(&case).expect("a readable case");
let live = parity::run_node_oracle("cuenorm.mjs", &content).expect("the oracle runs");
let golden = parity::read_golden_text(&format!("cuenorm/{name}.ndjson"));
assert_eq!(live, golden, "golden file for {name} is stale");
}
}