cuepack: cues into passages that keep their timestamps

Cargo.lock

@@ -187,6 +187,16 @@ dependencies = [
  "cuekit",
 ]
 
+[[package]]
+name = "cuepack"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "corpus-core",
+ "cuekit",
+ "textkit",
+]
+
 [[package]]
 name = "cueseg"
 version = "0.1.0"
@@ -283,6 +293,7 @@ dependencies = [
  "chunkmd",
  "corpus-core",
  "cuekit",
+ "cuepack",
  "cueseg",
  "mdscrub",
  "pdftriage",

Cargo.toml

@@ -19,6 +19,7 @@ pdftriage = { path = "crates/pdftriage", version = "0.1.0" }
 cuenorm = { path = "crates/cuenorm", version = "0.1.0" }
 cueseg = { path = "crates/cueseg", version = "0.1.0" }
 rrf = { path = "crates/rrf", version = "0.1.0" }
+cuepack = { path = "crates/cuepack", version = "0.1.0" }
 
 clap = { version = "4", features = ["derive"] }
 fancy-regex = "0.16"

README.md

@@ -58,6 +58,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done
 | `pdftriage` | Whether a PDF's text layer is healthy, garbled, sparse, permuted, or absent — and so whether OCR is worth its cost |
 | `cuenorm` | Any subtitle format in, one normalised cue stream out |
 | `cueseg` | A timed transcript into topic chapters |
+| `cuepack` | Cues into passages worth embedding, keeping the times they were said at |
 | `rrf` | Several ranked lists into one, without the database |
 
 ## The contract

crates/cuepack/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "cuepack"
+description = "Packs cues into passages worth embedding, keeping the times they were said at."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+cuekit.workspace = true
+textkit.workspace = true
+
+[lints]
+workspace = true

crates/cuepack/src/lib.rs

@@ -0,0 +1,197 @@
+//! Packs cues into passages worth embedding.
+//!
+//! A cue is a few seconds of speech and half a sentence. An embedding of one
+//! says almost nothing, and a search over them returns fragments. Packing
+//! them into a minute or so of continuous speech gives a passage with enough
+//! context to mean something, and — because the pack keeps the first cue's
+//! start and the last cue's end — one that can still be cited to the second.
+
+use corpus_core::Chunk;
+use cuekit::Cue;
+
+/// Where a passage is flushed, whichever comes first.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Budgets {
+    /// Seconds of audio a passage covers.
+    ///
+    /// Seventy-five is about a paragraph of speech: long enough to hold an
+    /// argument, short enough that a result points at a place in the
+    /// recording rather than at a region of it.
+    pub seconds: f64,
+    /// Tokens a passage may hold, whatever the clock says.
+    pub max_tokens: usize,
+}
+
+impl Default for Budgets {
+    fn default() -> Self {
+        Self {
+            seconds: 75.0,
+            max_tokens: 512,
+        }
+    }
+}
+
+/// Pack consecutive cues into passages.
+///
+/// A passage is flushed when the window from its first cue's start to the
+/// current cue's end reaches the seconds budget, or when adding the current
+/// cue would take it past the token budget.
+///
+/// Immediately repeated lines are dropped rather than packed, because
+/// automatic captioning repeats each line across overlapping windows and a
+/// passage padded with its own echo embeds as something nobody said.
+pub fn pack(cues: &[Cue], budgets: &Budgets) -> Vec<Chunk> {
+    let mut passages: Vec<Chunk> = Vec::new();
+    let mut buffered: Vec<&Cue> = Vec::new();
+    let mut buffered_tokens = 0usize;
+    let mut last_text: Option<&str> = None;
+
+    for cue in cues {
+        let text = cue.text.trim_matches(textkit::is_space);
+        if text.is_empty() || Some(text) == last_text {
+            continue;
+        }
+        last_text = Some(text);
+
+        let tokens = textkit::count(text);
+        let span = buffered
+            .first()
+            .map_or(0.0, |first: &&Cue| cue.end - first.start);
+        if !buffered.is_empty()
+            && (span >= budgets.seconds || buffered_tokens + tokens > budgets.max_tokens)
+        {
+            flush(&mut passages, &mut buffered, &mut buffered_tokens);
+        }
+        buffered.push(cue);
+        buffered_tokens += tokens;
+    }
+    flush(&mut passages, &mut buffered, &mut buffered_tokens);
+    passages
+}
+
+fn flush(passages: &mut Vec<Chunk>, buffered: &mut Vec<&Cue>, buffered_tokens: &mut usize) {
+    if buffered.is_empty() {
+        return;
+    }
+    let text = buffered
+        .iter()
+        .map(|cue| cue.text.trim_matches(textkit::is_space))
+        .filter(|text| !text.is_empty())
+        .collect::<Vec<_>>()
+        .join(" ");
+
+    if !text.is_empty() {
+        let tokens = textkit::count(&text);
+        let mut chunk = Chunk::new(passages.len() as u32, text, tokens as u32);
+        chunk.start_time = buffered.first().map(|cue| cue.start);
+        chunk.end_time = buffered.last().map(|cue| cue.end);
+        passages.push(chunk);
+    }
+    buffered.clear();
+    *buffered_tokens = 0;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn cue(start: f64, end: f64, text: &str) -> Cue {
+        Cue::new(start, end, text)
+    }
+
+    #[test]
+    fn cues_within_the_budgets_make_one_passage() {
+        let cues = [
+            cue(0.0, 5.0, "the mill stood"),
+            cue(5.0, 10.0, "at the northern end"),
+        ];
+        let packed = pack(&cues, &Budgets::default());
+        assert_eq!(packed.len(), 1);
+        assert_eq!(packed[0].text, "the mill stood at the northern end");
+        assert_eq!(packed[0].start_time, Some(0.0));
+        assert_eq!(packed[0].end_time, Some(10.0));
+        assert_eq!(packed[0].idx, 0);
+    }
+
+    #[test]
+    fn a_passage_is_flushed_when_it_has_run_long_enough() {
+        // The window is measured to the end of the cue being considered, so
+        // the cue that trips the budget starts the next passage rather than
+        // finishing this one.
+        let cues = [
+            cue(0.0, 40.0, "one"),
+            cue(40.0, 80.0, "two"),
+            cue(80.0, 90.0, "three"),
+        ];
+        let packed = pack(&cues, &Budgets::default());
+        assert_eq!(packed.len(), 2);
+        assert_eq!(packed[0].text, "one");
+        assert_eq!(packed[0].end_time, Some(40.0));
+        assert_eq!(packed[1].text, "two three");
+        assert_eq!(packed[1].idx, 1);
+    }
+
+    #[test]
+    fn a_passage_is_flushed_before_it_overruns_the_token_budget() {
+        let budgets = Budgets {
+            seconds: 1000.0,
+            max_tokens: 6,
+        };
+        let cues = [
+            cue(0.0, 1.0, "one two three"),
+            cue(1.0, 2.0, "four five six"),
+            cue(2.0, 3.0, "seven eight nine"),
+        ];
+        let packed = pack(&cues, &budgets);
+        assert_eq!(packed.len(), 2);
+        for passage in &packed {
+            assert!(passage.tokens <= 6, "{} tokens", passage.tokens);
+        }
+    }
+
+    #[test]
+    fn an_immediately_repeated_line_is_dropped() {
+        let cues = [
+            cue(0.0, 2.0, "the mill stood"),
+            cue(1.0, 3.0, "the mill stood"),
+            cue(3.0, 4.0, "at the northern end"),
+        ];
+        let packed = pack(&cues, &Budgets::default());
+        assert_eq!(packed[0].text, "the mill stood at the northern end");
+    }
+
+    #[test]
+    fn a_line_that_comes_back_later_is_kept() {
+        let cues = [
+            cue(0.0, 1.0, "yes"),
+            cue(1.0, 2.0, "no"),
+            cue(2.0, 3.0, "yes"),
+        ];
+        assert_eq!(pack(&cues, &Budgets::default())[0].text, "yes no yes");
+    }
+
+    #[test]
+    fn an_empty_cue_is_not_a_break_in_the_repetition() {
+        let cues = [
+            cue(0.0, 1.0, "same"),
+            cue(1.0, 2.0, "   "),
+            cue(2.0, 3.0, "same"),
+        ];
+        assert_eq!(pack(&cues, &Budgets::default())[0].text, "same");
+    }
+
+    #[test]
+    fn a_passage_carries_times_that_can_be_cited() {
+        let cues = [cue(12.5, 15.25, "the mill stood")];
+        let packed = pack(&cues, &Budgets::default());
+        assert_eq!(packed[0].start_time, Some(12.5));
+        assert_eq!(packed[0].end_time, Some(15.25));
+        assert_eq!(packed[0].page, None);
+    }
+
+    #[test]
+    fn nothing_in_nothing_out() {
+        assert!(pack(&[], &Budgets::default()).is_empty());
+        assert!(pack(&[cue(0.0, 1.0, "  ")], &Budgets::default()).is_empty());
+    }
+}

crates/cuepack/src/main.rs

@@ -0,0 +1,111 @@
+//! Packs cues into passages worth embedding.
+
+use std::io::{BufWriter, Read, Write};
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::{exit, ndjson};
+use cuekit::Cue;
+use cuepack::Budgets;
+
+/// Pack a cue stream into passages, one chunk record per line.
+///
+/// Takes either the cue stream `cuenorm` writes or a subtitle file directly.
+/// Each passage keeps the start of its first cue and the end of its last, so
+/// a result can be cited to the second — the same record a book's chunks use,
+/// with times where they have pages.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    after_help = "\
+Example:
+
+    cuenorm episode.vtt | cuepack --seconds 75 --max-tokens 512 > passages.ndjson"
+)]
+struct Args {
+    /// Cue stream or subtitle file to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+
+    /// Flush a passage once it covers this much audio.
+    #[arg(long, default_value_t = Budgets::default().seconds, value_name = "SECONDS")]
+    seconds: f64,
+
+    /// Never let a passage exceed this many tokens.
+    #[arg(long, default_value_t = Budgets::default().max_tokens, value_name = "TOKENS")]
+    max_tokens: usize,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+    let budgets = Budgets {
+        seconds: args.seconds,
+        max_tokens: args.max_tokens,
+    };
+    if !budgets.seconds.is_finite() || budgets.seconds <= 0.0 || budgets.max_tokens == 0 {
+        eprintln!("cuepack: --seconds and --max-tokens must be above zero");
+        return ExitCode::from(exit::USAGE);
+    }
+
+    let content = match read(args.path.as_deref()) {
+        Ok(content) => content,
+        Err(e) => {
+            eprintln!("cuepack: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    let cues = match cues(&content) {
+        Ok(cues) => cues,
+        Err(e) => {
+            eprintln!("cuepack: {e}");
+            return ExitCode::from(exit::DATA);
+        }
+    };
+
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    for passage in cuepack::pack(&cues, &budgets) {
+        if let Err(e) = ndjson::write_line(&mut out, &passage) {
+            // A consumer that stopped reading is not a failure.
+            if e.is_broken_pipe() {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("cuepack: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+    match out.flush() {
+        Ok(()) => ExitCode::from(exit::OK),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("cuepack: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Read cues from either encoding: a cue stream begins with a JSON object
+/// and a subtitle file does not.
+fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
+    let first = content
+        .lines()
+        .find(|line| !line.trim().is_empty())
+        .unwrap_or("");
+    if first.trim_start().starts_with('{') {
+        return ndjson::read(content.as_bytes()).collect();
+    }
+    Ok(cuekit::parse(content))
+}
+
+fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
+    let bytes = match path {
+        Some(path) => std::fs::read(path)?,
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin().read_to_end(&mut bytes)?;
+            bytes
+        }
+    };
+    Ok(String::from_utf8_lossy(&bytes).into_owned())
+}

parity/Cargo.toml

@@ -16,6 +16,7 @@ python = []
 [dependencies]
 chunkmd.workspace = true
 cuekit.workspace = true
+cuepack.workspace = true
 cueseg.workspace = true
 mdscrub.workspace = true
 pdftriage.workspace = true

parity/cases/cuepack/all-blank.ndjson

@@ -0,0 +1,2 @@
+{"start": 0.0, "end": 4.0, "text": "  "}
+{"start": 4.0, "end": 8.0, "text": ""}

parity/cases/cuepack/backwards-times.ndjson

@@ -0,0 +1,3 @@
+{"start": 10.0, "end": 5.0, "text": "an end before its start"}
+{"start": 5.0, "end": 20.0, "text": "and one that overlaps it"}
+{"start": 0.0, "end": 1.0, "text": "and one before both"}

parity/cases/cuepack/blanks.ndjson

@@ -0,0 +1,120 @@
+{"start": 0.0, "end": 4.0, "text": "   "}
+{"start": 4.0, "end": 8.0, "text": "the mill stood at the northern end of"}
+{"start": 8.0, "end": 12.0, "text": "the valley where the fall of the river"}
+{"start": 12.0, "end": 16.0, "text": "was steepest and the ice came latest its"}
+{"start": 16.0, "end": 20.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 20.0, "end": 24.0, "text": "   "}
+{"start": 24.0, "end": 28.0, "text": "replaced it for two hundred years because the"}
+{"start": 28.0, "end": 32.0, "text": "cost of replacing it fell on one household"}
+{"start": 32.0, "end": 36.0, "text": "while the benefit was spread across four hundred"}
+{"start": 36.0, "end": 40.0, "text": "families the mill stood at the northern end"}
+{"start": 40.0, "end": 44.0, "text": "   "}
+{"start": 44.0, "end": 48.0, "text": "of the valley where the fall of the"}
+{"start": 48.0, "end": 52.0, "text": "river was steepest and the ice came latest"}
+{"start": 52.0, "end": 56.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 56.0, "end": 60.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 60.0, "end": 64.0, "text": "   "}
+{"start": 64.0, "end": 68.0, "text": "the cost of replacing it fell on one"}
+{"start": 68.0, "end": 72.0, "text": "household while the benefit was spread across four"}
+{"start": 72.0, "end": 76.0, "text": "hundred families the mill stood at the northern"}
+{"start": 76.0, "end": 80.0, "text": "end of the valley where the fall of"}
+{"start": 80.0, "end": 84.0, "text": "   "}
+{"start": 84.0, "end": 88.0, "text": "the river was steepest and the ice came"}
+{"start": 88.0, "end": 92.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 92.0, "end": 96.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 96.0, "end": 100.0, "text": "because the cost of replacing it fell on"}
+{"start": 100.0, "end": 104.0, "text": "   "}
+{"start": 104.0, "end": 108.0, "text": "one household while the benefit was spread across"}
+{"start": 108.0, "end": 112.0, "text": "four hundred families the mill stood at the"}
+{"start": 112.0, "end": 116.0, "text": "northern end of the valley where the fall"}
+{"start": 116.0, "end": 120.0, "text": "of the river was steepest and the ice"}
+{"start": 120.0, "end": 124.0, "text": "   "}
+{"start": 124.0, "end": 128.0, "text": "came latest its wheel was undershot which was"}
+{"start": 128.0, "end": 132.0, "text": "wasteful and nobody replaced it for two hundred"}
+{"start": 132.0, "end": 136.0, "text": "years because the cost of replacing it fell"}
+{"start": 136.0, "end": 140.0, "text": "on one household while the benefit was spread"}
+{"start": 140.0, "end": 144.0, "text": "   "}
+{"start": 144.0, "end": 148.0, "text": "across four hundred families the mill stood at"}
+{"start": 148.0, "end": 152.0, "text": "the northern end of the valley where the"}
+{"start": 152.0, "end": 156.0, "text": "fall of the river was steepest and the"}
+{"start": 156.0, "end": 160.0, "text": "ice came latest its wheel was undershot which"}
+{"start": 160.0, "end": 164.0, "text": "   "}
+{"start": 164.0, "end": 168.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 168.0, "end": 172.0, "text": "hundred years because the cost of replacing it"}
+{"start": 172.0, "end": 176.0, "text": "fell on one household while the benefit was"}
+{"start": 176.0, "end": 180.0, "text": "spread across four hundred families the mill stood"}
+{"start": 180.0, "end": 184.0, "text": "   "}
+{"start": 184.0, "end": 188.0, "text": "at the northern end of the valley where"}
+{"start": 188.0, "end": 192.0, "text": "the fall of the river was steepest and"}
+{"start": 192.0, "end": 196.0, "text": "the ice came latest its wheel was undershot"}
+{"start": 196.0, "end": 200.0, "text": "which was wasteful and nobody replaced it for"}
+{"start": 200.0, "end": 204.0, "text": "   "}
+{"start": 204.0, "end": 208.0, "text": "two hundred years because the cost of replacing"}
+{"start": 208.0, "end": 212.0, "text": "it fell on one household while the benefit"}
+{"start": 212.0, "end": 216.0, "text": "was spread across four hundred families the mill"}
+{"start": 216.0, "end": 220.0, "text": "stood at the northern end of the valley"}
+{"start": 220.0, "end": 224.0, "text": "   "}
+{"start": 224.0, "end": 228.0, "text": "where the fall of the river was steepest"}
+{"start": 228.0, "end": 232.0, "text": "and the ice came latest its wheel was"}
+{"start": 232.0, "end": 236.0, "text": "undershot which was wasteful and nobody replaced it"}
+{"start": 236.0, "end": 240.0, "text": "for two hundred years because the cost of"}
+{"start": 240.0, "end": 244.0, "text": "   "}
+{"start": 244.0, "end": 248.0, "text": "replacing it fell on one household while the"}
+{"start": 248.0, "end": 252.0, "text": "benefit was spread across four hundred families the"}
+{"start": 252.0, "end": 256.0, "text": "mill stood at the northern end of the"}
+{"start": 256.0, "end": 260.0, "text": "valley where the fall of the river was"}
+{"start": 260.0, "end": 264.0, "text": "   "}
+{"start": 264.0, "end": 268.0, "text": "steepest and the ice came latest its wheel"}
+{"start": 268.0, "end": 272.0, "text": "was undershot which was wasteful and nobody replaced"}
+{"start": 272.0, "end": 276.0, "text": "it for two hundred years because the cost"}
+{"start": 276.0, "end": 280.0, "text": "of replacing it fell on one household while"}
+{"start": 280.0, "end": 284.0, "text": "   "}
+{"start": 284.0, "end": 288.0, "text": "the benefit was spread across four hundred families"}
+{"start": 288.0, "end": 292.0, "text": "the mill stood at the northern end of"}
+{"start": 292.0, "end": 296.0, "text": "the valley where the fall of the river"}
+{"start": 296.0, "end": 300.0, "text": "was steepest and the ice came latest its"}
+{"start": 300.0, "end": 304.0, "text": "   "}
+{"start": 304.0, "end": 308.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 308.0, "end": 312.0, "text": "replaced it for two hundred years because the"}
+{"start": 312.0, "end": 316.0, "text": "cost of replacing it fell on one household"}
+{"start": 316.0, "end": 320.0, "text": "while the benefit was spread across four hundred"}
+{"start": 320.0, "end": 324.0, "text": "   "}
+{"start": 324.0, "end": 328.0, "text": "families the mill stood at the northern end"}
+{"start": 328.0, "end": 332.0, "text": "of the valley where the fall of the"}
+{"start": 332.0, "end": 336.0, "text": "river was steepest and the ice came latest"}
+{"start": 336.0, "end": 340.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 340.0, "end": 344.0, "text": "   "}
+{"start": 344.0, "end": 348.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 348.0, "end": 352.0, "text": "the cost of replacing it fell on one"}
+{"start": 352.0, "end": 356.0, "text": "household while the benefit was spread across four"}
+{"start": 356.0, "end": 360.0, "text": "hundred families the mill stood at the northern"}
+{"start": 360.0, "end": 364.0, "text": "   "}
+{"start": 364.0, "end": 368.0, "text": "end of the valley where the fall of"}
+{"start": 368.0, "end": 372.0, "text": "the river was steepest and the ice came"}
+{"start": 372.0, "end": 376.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 376.0, "end": 380.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 380.0, "end": 384.0, "text": "   "}
+{"start": 384.0, "end": 388.0, "text": "because the cost of replacing it fell on"}
+{"start": 388.0, "end": 392.0, "text": "one household while the benefit was spread across"}
+{"start": 392.0, "end": 396.0, "text": "four hundred families the mill stood at the"}
+{"start": 396.0, "end": 400.0, "text": "northern end of the valley where the fall"}
+{"start": 400.0, "end": 404.0, "text": "   "}
+{"start": 404.0, "end": 408.0, "text": "of the river was steepest and the ice"}
+{"start": 408.0, "end": 412.0, "text": "came latest its wheel was undershot which was"}
+{"start": 412.0, "end": 416.0, "text": "wasteful and nobody replaced it for two hundred"}
+{"start": 416.0, "end": 420.0, "text": "years because the cost of replacing it fell"}
+{"start": 420.0, "end": 424.0, "text": "   "}
+{"start": 424.0, "end": 428.0, "text": "on one household while the benefit was spread"}
+{"start": 428.0, "end": 432.0, "text": "across four hundred families the mill stood at"}
+{"start": 432.0, "end": 436.0, "text": "the northern end of the valley where the"}
+{"start": 436.0, "end": 440.0, "text": "fall of the river was steepest and the"}
+{"start": 440.0, "end": 444.0, "text": "   "}
+{"start": 444.0, "end": 448.0, "text": "ice came latest its wheel was undershot which"}
+{"start": 448.0, "end": 452.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 452.0, "end": 456.0, "text": "hundred years because the cost of replacing it"}
+{"start": 456.0, "end": 460.0, "text": "fell on one household while the benefit was"}
+{"start": 460.0, "end": 464.0, "text": "   "}
+{"start": 464.0, "end": 468.0, "text": "spread across four hundred families the mill stood"}
+{"start": 468.0, "end": 472.0, "text": "at the northern end of the valley where"}
+{"start": 472.0, "end": 476.0, "text": "the fall of the river was steepest and"}
+{"start": 476.0, "end": 480.0, "text": "the ice came latest its wheel was undershot"}

parity/cases/cuepack/echoes.ndjson

@@ -0,0 +1,120 @@
+{"start": 0.0, "end": 4.0, "text": "the mill stood at the northern end of"}
+{"start": 4.0, "end": 8.0, "text": "the valley where the fall of the river"}
+{"start": 8.0, "end": 12.0, "text": "was steepest and the ice came latest its"}
+{"start": 12.0, "end": 16.0, "text": "was steepest and the ice came latest its"}
+{"start": 16.0, "end": 20.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 20.0, "end": 24.0, "text": "replaced it for two hundred years because the"}
+{"start": 24.0, "end": 28.0, "text": "replaced it for two hundred years because the"}
+{"start": 28.0, "end": 32.0, "text": "cost of replacing it fell on one household"}
+{"start": 32.0, "end": 36.0, "text": "while the benefit was spread across four hundred"}
+{"start": 36.0, "end": 40.0, "text": "while the benefit was spread across four hundred"}
+{"start": 40.0, "end": 44.0, "text": "families the mill stood at the northern end"}
+{"start": 44.0, "end": 48.0, "text": "of the valley where the fall of the"}
+{"start": 48.0, "end": 52.0, "text": "of the valley where the fall of the"}
+{"start": 52.0, "end": 56.0, "text": "river was steepest and the ice came latest"}
+{"start": 56.0, "end": 60.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 60.0, "end": 64.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 64.0, "end": 68.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 68.0, "end": 72.0, "text": "the cost of replacing it fell on one"}
+{"start": 72.0, "end": 76.0, "text": "the cost of replacing it fell on one"}
+{"start": 76.0, "end": 80.0, "text": "household while the benefit was spread across four"}
+{"start": 80.0, "end": 84.0, "text": "hundred families the mill stood at the northern"}
+{"start": 84.0, "end": 88.0, "text": "hundred families the mill stood at the northern"}
+{"start": 88.0, "end": 92.0, "text": "end of the valley where the fall of"}
+{"start": 92.0, "end": 96.0, "text": "the river was steepest and the ice came"}
+{"start": 96.0, "end": 100.0, "text": "the river was steepest and the ice came"}
+{"start": 100.0, "end": 104.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 104.0, "end": 108.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 108.0, "end": 112.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 112.0, "end": 116.0, "text": "because the cost of replacing it fell on"}
+{"start": 116.0, "end": 120.0, "text": "one household while the benefit was spread across"}
+{"start": 120.0, "end": 124.0, "text": "one household while the benefit was spread across"}
+{"start": 124.0, "end": 128.0, "text": "four hundred families the mill stood at the"}
+{"start": 128.0, "end": 132.0, "text": "northern end of the valley where the fall"}
+{"start": 132.0, "end": 136.0, "text": "northern end of the valley where the fall"}
+{"start": 136.0, "end": 140.0, "text": "of the river was steepest and the ice"}
+{"start": 140.0, "end": 144.0, "text": "came latest its wheel was undershot which was"}
+{"start": 144.0, "end": 148.0, "text": "came latest its wheel was undershot which was"}
+{"start": 148.0, "end": 152.0, "text": "wasteful and nobody replaced it for two hundred"}
+{"start": 152.0, "end": 156.0, "text": "years because the cost of replacing it fell"}
+{"start": 156.0, "end": 160.0, "text": "years because the cost of replacing it fell"}
+{"start": 160.0, "end": 164.0, "text": "on one household while the benefit was spread"}
+{"start": 164.0, "end": 168.0, "text": "across four hundred families the mill stood at"}
+{"start": 168.0, "end": 172.0, "text": "across four hundred families the mill stood at"}
+{"start": 172.0, "end": 176.0, "text": "the northern end of the valley where the"}
+{"start": 176.0, "end": 180.0, "text": "fall of the river was steepest and the"}
+{"start": 180.0, "end": 184.0, "text": "fall of the river was steepest and the"}
+{"start": 184.0, "end": 188.0, "text": "ice came latest its wheel was undershot which"}
+{"start": 188.0, "end": 192.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 192.0, "end": 196.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 196.0, "end": 200.0, "text": "hundred years because the cost of replacing it"}
+{"start": 200.0, "end": 204.0, "text": "fell on one household while the benefit was"}
+{"start": 204.0, "end": 208.0, "text": "fell on one household while the benefit was"}
+{"start": 208.0, "end": 212.0, "text": "spread across four hundred families the mill stood"}
+{"start": 212.0, "end": 216.0, "text": "at the northern end of the valley where"}
+{"start": 216.0, "end": 220.0, "text": "at the northern end of the valley where"}
+{"start": 220.0, "end": 224.0, "text": "the fall of the river was steepest and"}
+{"start": 224.0, "end": 228.0, "text": "the ice came latest its wheel was undershot"}
+{"start": 228.0, "end": 232.0, "text": "the ice came latest its wheel was undershot"}
+{"start": 232.0, "end": 236.0, "text": "which was wasteful and nobody replaced it for"}
+{"start": 236.0, "end": 240.0, "text": "two hundred years because the cost of replacing"}
+{"start": 240.0, "end": 244.0, "text": "two hundred years because the cost of replacing"}
+{"start": 244.0, "end": 248.0, "text": "it fell on one household while the benefit"}
+{"start": 248.0, "end": 252.0, "text": "was spread across four hundred families the mill"}
+{"start": 252.0, "end": 256.0, "text": "was spread across four hundred families the mill"}
+{"start": 256.0, "end": 260.0, "text": "stood at the northern end of the valley"}
+{"start": 260.0, "end": 264.0, "text": "where the fall of the river was steepest"}
+{"start": 264.0, "end": 268.0, "text": "where the fall of the river was steepest"}
+{"start": 268.0, "end": 272.0, "text": "and the ice came latest its wheel was"}
+{"start": 272.0, "end": 276.0, "text": "undershot which was wasteful and nobody replaced it"}
+{"start": 276.0, "end": 280.0, "text": "undershot which was wasteful and nobody replaced it"}
+{"start": 280.0, "end": 284.0, "text": "for two hundred years because the cost of"}
+{"start": 284.0, "end": 288.0, "text": "replacing it fell on one household while the"}
+{"start": 288.0, "end": 292.0, "text": "replacing it fell on one household while the"}
+{"start": 292.0, "end": 296.0, "text": "benefit was spread across four hundred families the"}
+{"start": 296.0, "end": 300.0, "text": "mill stood at the northern end of the"}
+{"start": 300.0, "end": 304.0, "text": "mill stood at the northern end of the"}
+{"start": 304.0, "end": 308.0, "text": "valley where the fall of the river was"}
+{"start": 308.0, "end": 312.0, "text": "steepest and the ice came latest its wheel"}
+{"start": 312.0, "end": 316.0, "text": "steepest and the ice came latest its wheel"}
+{"start": 316.0, "end": 320.0, "text": "was undershot which was wasteful and nobody replaced"}
+{"start": 320.0, "end": 324.0, "text": "it for two hundred years because the cost"}
+{"start": 324.0, "end": 328.0, "text": "it for two hundred years because the cost"}
+{"start": 328.0, "end": 332.0, "text": "of replacing it fell on one household while"}
+{"start": 332.0, "end": 336.0, "text": "the benefit was spread across four hundred families"}
+{"start": 336.0, "end": 340.0, "text": "the benefit was spread across four hundred families"}
+{"start": 340.0, "end": 344.0, "text": "the mill stood at the northern end of"}
+{"start": 344.0, "end": 348.0, "text": "the valley where the fall of the river"}
+{"start": 348.0, "end": 352.0, "text": "the valley where the fall of the river"}
+{"start": 352.0, "end": 356.0, "text": "was steepest and the ice came latest its"}
+{"start": 356.0, "end": 360.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 360.0, "end": 364.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 364.0, "end": 368.0, "text": "replaced it for two hundred years because the"}
+{"start": 368.0, "end": 372.0, "text": "cost of replacing it fell on one household"}
+{"start": 372.0, "end": 376.0, "text": "cost of replacing it fell on one household"}
+{"start": 376.0, "end": 380.0, "text": "while the benefit was spread across four hundred"}
+{"start": 380.0, "end": 384.0, "text": "families the mill stood at the northern end"}
+{"start": 384.0, "end": 388.0, "text": "families the mill stood at the northern end"}
+{"start": 388.0, "end": 392.0, "text": "of the valley where the fall of the"}
+{"start": 392.0, "end": 396.0, "text": "river was steepest and the ice came latest"}
+{"start": 396.0, "end": 400.0, "text": "river was steepest and the ice came latest"}
+{"start": 400.0, "end": 404.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 404.0, "end": 408.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 408.0, "end": 412.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 412.0, "end": 416.0, "text": "the cost of replacing it fell on one"}
+{"start": 416.0, "end": 420.0, "text": "household while the benefit was spread across four"}
+{"start": 420.0, "end": 424.0, "text": "household while the benefit was spread across four"}
+{"start": 424.0, "end": 428.0, "text": "hundred families the mill stood at the northern"}
+{"start": 428.0, "end": 432.0, "text": "end of the valley where the fall of"}
+{"start": 432.0, "end": 436.0, "text": "end of the valley where the fall of"}
+{"start": 436.0, "end": 440.0, "text": "the river was steepest and the ice came"}
+{"start": 440.0, "end": 444.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 444.0, "end": 448.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 448.0, "end": 452.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 452.0, "end": 456.0, "text": "because the cost of replacing it fell on"}
+{"start": 456.0, "end": 460.0, "text": "because the cost of replacing it fell on"}
+{"start": 460.0, "end": 464.0, "text": "one household while the benefit was spread across"}
+{"start": 464.0, "end": 468.0, "text": "four hundred families the mill stood at the"}
+{"start": 468.0, "end": 472.0, "text": "four hundred families the mill stood at the"}
+{"start": 472.0, "end": 476.0, "text": "northern end of the valley where the fall"}
+{"start": 476.0, "end": 480.0, "text": "of the river was steepest and the ice"}

parity/cases/cuepack/long-cues.ndjson

@@ -0,0 +1,40 @@
+{"start": 0.0, "end": 40.0, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the"}
+{"start": 40.0, "end": 80.0, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the"}
+{"start": 80.0, "end": 120.0, "text": "cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood"}
+{"start": 120.0, "end": 160.0, "text": "at the northern end of the valley where the fall of the river was steepest and the ice came latest"}
+{"start": 160.0, "end": 200.0, "text": "its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing"}
+{"start": 200.0, "end": 240.0, "text": "it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"start": 240.0, "end": 280.0, "text": "end of the valley where the fall of the river was steepest and the ice came latest its wheel was"}
+{"start": 280.0, "end": 320.0, "text": "undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on"}
+{"start": 320.0, "end": 360.0, "text": "one household while the benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"start": 360.0, "end": 400.0, "text": "valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"start": 400.0, "end": 440.0, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while"}
+{"start": 440.0, "end": 480.0, "text": "the benefit was spread across four hundred families the mill stood at the northern end of the valley where the"}
+{"start": 480.0, "end": 520.0, "text": "fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"start": 520.0, "end": 560.0, "text": "replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was"}
+{"start": 560.0, "end": 600.0, "text": "spread across four hundred families the mill stood at the northern end of the valley where the fall of the"}
+{"start": 600.0, "end": 640.0, "text": "river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for"}
+{"start": 640.0, "end": 680.0, "text": "two hundred years because the cost of replacing it fell on one household while the benefit was spread across four"}
+{"start": 680.0, "end": 720.0, "text": "hundred families the mill stood at the northern end of the valley where the fall of the river was steepest"}
+{"start": 720.0, "end": 760.0, "text": "and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years"}
+{"start": 760.0, "end": 800.0, "text": "because the cost of replacing it fell on one household while the benefit was spread across four hundred families the"}
+{"start": 800.0, "end": 840.0, "text": "mill stood at the northern end of the valley where the fall of the river was steepest and the ice"}
+{"start": 840.0, "end": 880.0, "text": "came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost"}
+{"start": 880.0, "end": 920.0, "text": "of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at"}
+{"start": 920.0, "end": 960.0, "text": "the northern end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"start": 960.0, "end": 1000.0, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it"}
+{"start": 1000.0, "end": 1040.0, "text": "fell on one household while the benefit was spread across four hundred families the mill stood at the northern end"}
+{"start": 1040.0, "end": 1080.0, "text": "of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot"}
+{"start": 1080.0, "end": 1120.0, "text": "which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"start": 1120.0, "end": 1160.0, "text": "household while the benefit was spread across four hundred families the mill stood at the northern end of the valley"}
+{"start": 1160.0, "end": 1200.0, "text": "where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"start": 1200.0, "end": 1240.0, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the"}
+{"start": 1240.0, "end": 1280.0, "text": "benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall"}
+{"start": 1280.0, "end": 1320.0, "text": "of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced"}
+{"start": 1320.0, "end": 1360.0, "text": "it for two hundred years because the cost of replacing it fell on one household while the benefit was spread"}
+{"start": 1360.0, "end": 1400.0, "text": "across four hundred families the mill stood at the northern end of the valley where the fall of the river"}
+{"start": 1400.0, "end": 1440.0, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"start": 1440.0, "end": 1480.0, "text": "hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred"}
+{"start": 1480.0, "end": 1520.0, "text": "families the mill stood at the northern end of the valley where the fall of the river was steepest and"}
+{"start": 1520.0, "end": 1560.0, "text": "the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"start": 1560.0, "end": 1600.0, "text": "the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill"}

parity/cases/cuepack/one-cue.ndjson

@@ -0,0 +1 @@
+{"start": 0.0, "end": 4.0, "text": "the mill stood at the northern end of"}

parity/cases/cuepack/plain.ndjson

@@ -0,0 +1,120 @@
+{"start": 0.0, "end": 4.0, "text": "the mill stood at the northern end of"}
+{"start": 4.0, "end": 8.0, "text": "the valley where the fall of the river"}
+{"start": 8.0, "end": 12.0, "text": "was steepest and the ice came latest its"}
+{"start": 12.0, "end": 16.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 16.0, "end": 20.0, "text": "replaced it for two hundred years because the"}
+{"start": 20.0, "end": 24.0, "text": "cost of replacing it fell on one household"}
+{"start": 24.0, "end": 28.0, "text": "while the benefit was spread across four hundred"}
+{"start": 28.0, "end": 32.0, "text": "families the mill stood at the northern end"}
+{"start": 32.0, "end": 36.0, "text": "of the valley where the fall of the"}
+{"start": 36.0, "end": 40.0, "text": "river was steepest and the ice came latest"}
+{"start": 40.0, "end": 44.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 44.0, "end": 48.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 48.0, "end": 52.0, "text": "the cost of replacing it fell on one"}
+{"start": 52.0, "end": 56.0, "text": "household while the benefit was spread across four"}
+{"start": 56.0, "end": 60.0, "text": "hundred families the mill stood at the northern"}
+{"start": 60.0, "end": 64.0, "text": "end of the valley where the fall of"}
+{"start": 64.0, "end": 68.0, "text": "the river was steepest and the ice came"}
+{"start": 68.0, "end": 72.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 72.0, "end": 76.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 76.0, "end": 80.0, "text": "because the cost of replacing it fell on"}
+{"start": 80.0, "end": 84.0, "text": "one household while the benefit was spread across"}
+{"start": 84.0, "end": 88.0, "text": "four hundred families the mill stood at the"}
+{"start": 88.0, "end": 92.0, "text": "northern end of the valley where the fall"}
+{"start": 92.0, "end": 96.0, "text": "of the river was steepest and the ice"}
+{"start": 96.0, "end": 100.0, "text": "came latest its wheel was undershot which was"}
+{"start": 100.0, "end": 104.0, "text": "wasteful and nobody replaced it for two hundred"}
+{"start": 104.0, "end": 108.0, "text": "years because the cost of replacing it fell"}
+{"start": 108.0, "end": 112.0, "text": "on one household while the benefit was spread"}
+{"start": 112.0, "end": 116.0, "text": "across four hundred families the mill stood at"}
+{"start": 116.0, "end": 120.0, "text": "the northern end of the valley where the"}
+{"start": 120.0, "end": 124.0, "text": "fall of the river was steepest and the"}
+{"start": 124.0, "end": 128.0, "text": "ice came latest its wheel was undershot which"}
+{"start": 128.0, "end": 132.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 132.0, "end": 136.0, "text": "hundred years because the cost of replacing it"}
+{"start": 136.0, "end": 140.0, "text": "fell on one household while the benefit was"}
+{"start": 140.0, "end": 144.0, "text": "spread across four hundred families the mill stood"}
+{"start": 144.0, "end": 148.0, "text": "at the northern end of the valley where"}
+{"start": 148.0, "end": 152.0, "text": "the fall of the river was steepest and"}
+{"start": 152.0, "end": 156.0, "text": "the ice came latest its wheel was undershot"}
+{"start": 156.0, "end": 160.0, "text": "which was wasteful and nobody replaced it for"}
+{"start": 160.0, "end": 164.0, "text": "two hundred years because the cost of replacing"}
+{"start": 164.0, "end": 168.0, "text": "it fell on one household while the benefit"}
+{"start": 168.0, "end": 172.0, "text": "was spread across four hundred families the mill"}
+{"start": 172.0, "end": 176.0, "text": "stood at the northern end of the valley"}
+{"start": 176.0, "end": 180.0, "text": "where the fall of the river was steepest"}
+{"start": 180.0, "end": 184.0, "text": "and the ice came latest its wheel was"}
+{"start": 184.0, "end": 188.0, "text": "undershot which was wasteful and nobody replaced it"}
+{"start": 188.0, "end": 192.0, "text": "for two hundred years because the cost of"}
+{"start": 192.0, "end": 196.0, "text": "replacing it fell on one household while the"}
+{"start": 196.0, "end": 200.0, "text": "benefit was spread across four hundred families the"}
+{"start": 200.0, "end": 204.0, "text": "mill stood at the northern end of the"}
+{"start": 204.0, "end": 208.0, "text": "valley where the fall of the river was"}
+{"start": 208.0, "end": 212.0, "text": "steepest and the ice came latest its wheel"}
+{"start": 212.0, "end": 216.0, "text": "was undershot which was wasteful and nobody replaced"}
+{"start": 216.0, "end": 220.0, "text": "it for two hundred years because the cost"}
+{"start": 220.0, "end": 224.0, "text": "of replacing it fell on one household while"}
+{"start": 224.0, "end": 228.0, "text": "the benefit was spread across four hundred families"}
+{"start": 228.0, "end": 232.0, "text": "the mill stood at the northern end of"}
+{"start": 232.0, "end": 236.0, "text": "the valley where the fall of the river"}
+{"start": 236.0, "end": 240.0, "text": "was steepest and the ice came latest its"}
+{"start": 240.0, "end": 244.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 244.0, "end": 248.0, "text": "replaced it for two hundred years because the"}
+{"start": 248.0, "end": 252.0, "text": "cost of replacing it fell on one household"}
+{"start": 252.0, "end": 256.0, "text": "while the benefit was spread across four hundred"}
+{"start": 256.0, "end": 260.0, "text": "families the mill stood at the northern end"}
+{"start": 260.0, "end": 264.0, "text": "of the valley where the fall of the"}
+{"start": 264.0, "end": 268.0, "text": "river was steepest and the ice came latest"}
+{"start": 268.0, "end": 272.0, "text": "its wheel was undershot which was wasteful and"}
+{"start": 272.0, "end": 276.0, "text": "nobody replaced it for two hundred years because"}
+{"start": 276.0, "end": 280.0, "text": "the cost of replacing it fell on one"}
+{"start": 280.0, "end": 284.0, "text": "household while the benefit was spread across four"}
+{"start": 284.0, "end": 288.0, "text": "hundred families the mill stood at the northern"}
+{"start": 288.0, "end": 292.0, "text": "end of the valley where the fall of"}
+{"start": 292.0, "end": 296.0, "text": "the river was steepest and the ice came"}
+{"start": 296.0, "end": 300.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 300.0, "end": 304.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 304.0, "end": 308.0, "text": "because the cost of replacing it fell on"}
+{"start": 308.0, "end": 312.0, "text": "one household while the benefit was spread across"}
+{"start": 312.0, "end": 316.0, "text": "four hundred families the mill stood at the"}
+{"start": 316.0, "end": 320.0, "text": "northern end of the valley where the fall"}
+{"start": 320.0, "end": 324.0, "text": "of the river was steepest and the ice"}
+{"start": 324.0, "end": 328.0, "text": "came latest its wheel was undershot which was"}
+{"start": 328.0, "end": 332.0, "text": "wasteful and nobody replaced it for two hundred"}
+{"start": 332.0, "end": 336.0, "text": "years because the cost of replacing it fell"}
+{"start": 336.0, "end": 340.0, "text": "on one household while the benefit was spread"}
+{"start": 340.0, "end": 344.0, "text": "across four hundred families the mill stood at"}
+{"start": 344.0, "end": 348.0, "text": "the northern end of the valley where the"}
+{"start": 348.0, "end": 352.0, "text": "fall of the river was steepest and the"}
+{"start": 352.0, "end": 356.0, "text": "ice came latest its wheel was undershot which"}
+{"start": 356.0, "end": 360.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 360.0, "end": 364.0, "text": "hundred years because the cost of replacing it"}
+{"start": 364.0, "end": 368.0, "text": "fell on one household while the benefit was"}
+{"start": 368.0, "end": 372.0, "text": "spread across four hundred families the mill stood"}
+{"start": 372.0, "end": 376.0, "text": "at the northern end of the valley where"}
+{"start": 376.0, "end": 380.0, "text": "the fall of the river was steepest and"}
+{"start": 380.0, "end": 384.0, "text": "the ice came latest its wheel was undershot"}
+{"start": 384.0, "end": 388.0, "text": "which was wasteful and nobody replaced it for"}
+{"start": 388.0, "end": 392.0, "text": "two hundred years because the cost of replacing"}
+{"start": 392.0, "end": 396.0, "text": "it fell on one household while the benefit"}
+{"start": 396.0, "end": 400.0, "text": "was spread across four hundred families the mill"}
+{"start": 400.0, "end": 404.0, "text": "stood at the northern end of the valley"}
+{"start": 404.0, "end": 408.0, "text": "where the fall of the river was steepest"}
+{"start": 408.0, "end": 412.0, "text": "and the ice came latest its wheel was"}
+{"start": 412.0, "end": 416.0, "text": "undershot which was wasteful and nobody replaced it"}
+{"start": 416.0, "end": 420.0, "text": "for two hundred years because the cost of"}
+{"start": 420.0, "end": 424.0, "text": "replacing it fell on one household while the"}
+{"start": 424.0, "end": 428.0, "text": "benefit was spread across four hundred families the"}
+{"start": 428.0, "end": 432.0, "text": "mill stood at the northern end of the"}
+{"start": 432.0, "end": 436.0, "text": "valley where the fall of the river was"}
+{"start": 436.0, "end": 440.0, "text": "steepest and the ice came latest its wheel"}
+{"start": 440.0, "end": 444.0, "text": "was undershot which was wasteful and nobody replaced"}
+{"start": 444.0, "end": 448.0, "text": "it for two hundred years because the cost"}
+{"start": 448.0, "end": 452.0, "text": "of replacing it fell on one household while"}
+{"start": 452.0, "end": 456.0, "text": "the benefit was spread across four hundred families"}
+{"start": 456.0, "end": 460.0, "text": "the mill stood at the northern end of"}
+{"start": 460.0, "end": 464.0, "text": "the valley where the fall of the river"}
+{"start": 464.0, "end": 468.0, "text": "was steepest and the ice came latest its"}
+{"start": 468.0, "end": 472.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 472.0, "end": 476.0, "text": "replaced it for two hundred years because the"}
+{"start": 476.0, "end": 480.0, "text": "cost of replacing it fell on one household"}

parity/cases/cuepack/short-cues.ndjson

@@ -0,0 +1,300 @@
+{"start": 0.0, "end": 0.5, "text": "the mill stood"}
+{"start": 0.5, "end": 1.0, "text": "at the northern"}
+{"start": 1.0, "end": 1.5, "text": "end of the"}
+{"start": 1.5, "end": 2.0, "text": "valley where the"}
+{"start": 2.0, "end": 2.5, "text": "fall of the"}
+{"start": 2.5, "end": 3.0, "text": "river was steepest"}
+{"start": 3.0, "end": 3.5, "text": "and the ice"}
+{"start": 3.5, "end": 4.0, "text": "came latest its"}
+{"start": 4.0, "end": 4.5, "text": "wheel was undershot"}
+{"start": 4.5, "end": 5.0, "text": "which was wasteful"}
+{"start": 5.0, "end": 5.5, "text": "and nobody replaced"}
+{"start": 5.5, "end": 6.0, "text": "it for two"}
+{"start": 6.0, "end": 6.5, "text": "hundred years because"}
+{"start": 6.5, "end": 7.0, "text": "the cost of"}
+{"start": 7.0, "end": 7.5, "text": "replacing it fell"}
+{"start": 7.5, "end": 8.0, "text": "on one household"}
+{"start": 8.0, "end": 8.5, "text": "while the benefit"}
+{"start": 8.5, "end": 9.0, "text": "was spread across"}
+{"start": 9.0, "end": 9.5, "text": "four hundred families"}
+{"start": 9.5, "end": 10.0, "text": "the mill stood"}
+{"start": 10.0, "end": 10.5, "text": "at the northern"}
+{"start": 10.5, "end": 11.0, "text": "end of the"}
+{"start": 11.0, "end": 11.5, "text": "valley where the"}
+{"start": 11.5, "end": 12.0, "text": "fall of the"}
+{"start": 12.0, "end": 12.5, "text": "river was steepest"}
+{"start": 12.5, "end": 13.0, "text": "and the ice"}
+{"start": 13.0, "end": 13.5, "text": "came latest its"}
+{"start": 13.5, "end": 14.0, "text": "wheel was undershot"}
+{"start": 14.0, "end": 14.5, "text": "which was wasteful"}
+{"start": 14.5, "end": 15.0, "text": "and nobody replaced"}
+{"start": 15.0, "end": 15.5, "text": "it for two"}
+{"start": 15.5, "end": 16.0, "text": "hundred years because"}
+{"start": 16.0, "end": 16.5, "text": "the cost of"}
+{"start": 16.5, "end": 17.0, "text": "replacing it fell"}
+{"start": 17.0, "end": 17.5, "text": "on one household"}
+{"start": 17.5, "end": 18.0, "text": "while the benefit"}
+{"start": 18.0, "end": 18.5, "text": "was spread across"}
+{"start": 18.5, "end": 19.0, "text": "four hundred families"}
+{"start": 19.0, "end": 19.5, "text": "the mill stood"}
+{"start": 19.5, "end": 20.0, "text": "at the northern"}
+{"start": 20.0, "end": 20.5, "text": "end of the"}
+{"start": 20.5, "end": 21.0, "text": "valley where the"}
+{"start": 21.0, "end": 21.5, "text": "fall of the"}
+{"start": 21.5, "end": 22.0, "text": "river was steepest"}
+{"start": 22.0, "end": 22.5, "text": "and the ice"}
+{"start": 22.5, "end": 23.0, "text": "came latest its"}
+{"start": 23.0, "end": 23.5, "text": "wheel was undershot"}
+{"start": 23.5, "end": 24.0, "text": "which was wasteful"}
+{"start": 24.0, "end": 24.5, "text": "and nobody replaced"}
+{"start": 24.5, "end": 25.0, "text": "it for two"}
+{"start": 25.0, "end": 25.5, "text": "hundred years because"}
+{"start": 25.5, "end": 26.0, "text": "the cost of"}
+{"start": 26.0, "end": 26.5, "text": "replacing it fell"}
+{"start": 26.5, "end": 27.0, "text": "on one household"}
+{"start": 27.0, "end": 27.5, "text": "while the benefit"}
+{"start": 27.5, "end": 28.0, "text": "was spread across"}
+{"start": 28.0, "end": 28.5, "text": "four hundred families"}
+{"start": 28.5, "end": 29.0, "text": "the mill stood"}
+{"start": 29.0, "end": 29.5, "text": "at the northern"}
+{"start": 29.5, "end": 30.0, "text": "end of the"}
+{"start": 30.0, "end": 30.5, "text": "valley where the"}
+{"start": 30.5, "end": 31.0, "text": "fall of the"}
+{"start": 31.0, "end": 31.5, "text": "river was steepest"}
+{"start": 31.5, "end": 32.0, "text": "and the ice"}
+{"start": 32.0, "end": 32.5, "text": "came latest its"}
+{"start": 32.5, "end": 33.0, "text": "wheel was undershot"}
+{"start": 33.0, "end": 33.5, "text": "which was wasteful"}
+{"start": 33.5, "end": 34.0, "text": "and nobody replaced"}
+{"start": 34.0, "end": 34.5, "text": "it for two"}
+{"start": 34.5, "end": 35.0, "text": "hundred years because"}
+{"start": 35.0, "end": 35.5, "text": "the cost of"}
+{"start": 35.5, "end": 36.0, "text": "replacing it fell"}
+{"start": 36.0, "end": 36.5, "text": "on one household"}
+{"start": 36.5, "end": 37.0, "text": "while the benefit"}
+{"start": 37.0, "end": 37.5, "text": "was spread across"}
+{"start": 37.5, "end": 38.0, "text": "four hundred families"}
+{"start": 38.0, "end": 38.5, "text": "the mill stood"}
+{"start": 38.5, "end": 39.0, "text": "at the northern"}
+{"start": 39.0, "end": 39.5, "text": "end of the"}
+{"start": 39.5, "end": 40.0, "text": "valley where the"}
+{"start": 40.0, "end": 40.5, "text": "fall of the"}
+{"start": 40.5, "end": 41.0, "text": "river was steepest"}
+{"start": 41.0, "end": 41.5, "text": "and the ice"}
+{"start": 41.5, "end": 42.0, "text": "came latest its"}
+{"start": 42.0, "end": 42.5, "text": "wheel was undershot"}
+{"start": 42.5, "end": 43.0, "text": "which was wasteful"}
+{"start": 43.0, "end": 43.5, "text": "and nobody replaced"}
+{"start": 43.5, "end": 44.0, "text": "it for two"}
+{"start": 44.0, "end": 44.5, "text": "hundred years because"}
+{"start": 44.5, "end": 45.0, "text": "the cost of"}
+{"start": 45.0, "end": 45.5, "text": "replacing it fell"}
+{"start": 45.5, "end": 46.0, "text": "on one household"}
+{"start": 46.0, "end": 46.5, "text": "while the benefit"}
+{"start": 46.5, "end": 47.0, "text": "was spread across"}
+{"start": 47.0, "end": 47.5, "text": "four hundred families"}
+{"start": 47.5, "end": 48.0, "text": "the mill stood"}
+{"start": 48.0, "end": 48.5, "text": "at the northern"}
+{"start": 48.5, "end": 49.0, "text": "end of the"}
+{"start": 49.0, "end": 49.5, "text": "valley where the"}
+{"start": 49.5, "end": 50.0, "text": "fall of the"}
+{"start": 50.0, "end": 50.5, "text": "river was steepest"}
+{"start": 50.5, "end": 51.0, "text": "and the ice"}
+{"start": 51.0, "end": 51.5, "text": "came latest its"}
+{"start": 51.5, "end": 52.0, "text": "wheel was undershot"}
+{"start": 52.0, "end": 52.5, "text": "which was wasteful"}
+{"start": 52.5, "end": 53.0, "text": "and nobody replaced"}
+{"start": 53.0, "end": 53.5, "text": "it for two"}
+{"start": 53.5, "end": 54.0, "text": "hundred years because"}
+{"start": 54.0, "end": 54.5, "text": "the cost of"}
+{"start": 54.5, "end": 55.0, "text": "replacing it fell"}
+{"start": 55.0, "end": 55.5, "text": "on one household"}
+{"start": 55.5, "end": 56.0, "text": "while the benefit"}
+{"start": 56.0, "end": 56.5, "text": "was spread across"}
+{"start": 56.5, "end": 57.0, "text": "four hundred families"}
+{"start": 57.0, "end": 57.5, "text": "the mill stood"}
+{"start": 57.5, "end": 58.0, "text": "at the northern"}
+{"start": 58.0, "end": 58.5, "text": "end of the"}
+{"start": 58.5, "end": 59.0, "text": "valley where the"}
+{"start": 59.0, "end": 59.5, "text": "fall of the"}
+{"start": 59.5, "end": 60.0, "text": "river was steepest"}
+{"start": 60.0, "end": 60.5, "text": "and the ice"}
+{"start": 60.5, "end": 61.0, "text": "came latest its"}
+{"start": 61.0, "end": 61.5, "text": "wheel was undershot"}
+{"start": 61.5, "end": 62.0, "text": "which was wasteful"}
+{"start": 62.0, "end": 62.5, "text": "and nobody replaced"}
+{"start": 62.5, "end": 63.0, "text": "it for two"}
+{"start": 63.0, "end": 63.5, "text": "hundred years because"}
+{"start": 63.5, "end": 64.0, "text": "the cost of"}
+{"start": 64.0, "end": 64.5, "text": "replacing it fell"}
+{"start": 64.5, "end": 65.0, "text": "on one household"}
+{"start": 65.0, "end": 65.5, "text": "while the benefit"}
+{"start": 65.5, "end": 66.0, "text": "was spread across"}
+{"start": 66.0, "end": 66.5, "text": "four hundred families"}
+{"start": 66.5, "end": 67.0, "text": "the mill stood"}
+{"start": 67.0, "end": 67.5, "text": "at the northern"}
+{"start": 67.5, "end": 68.0, "text": "end of the"}
+{"start": 68.0, "end": 68.5, "text": "valley where the"}
+{"start": 68.5, "end": 69.0, "text": "fall of the"}
+{"start": 69.0, "end": 69.5, "text": "river was steepest"}
+{"start": 69.5, "end": 70.0, "text": "and the ice"}
+{"start": 70.0, "end": 70.5, "text": "came latest its"}
+{"start": 70.5, "end": 71.0, "text": "wheel was undershot"}
+{"start": 71.0, "end": 71.5, "text": "which was wasteful"}
+{"start": 71.5, "end": 72.0, "text": "and nobody replaced"}
+{"start": 72.0, "end": 72.5, "text": "it for two"}
+{"start": 72.5, "end": 73.0, "text": "hundred years because"}
+{"start": 73.0, "end": 73.5, "text": "the cost of"}
+{"start": 73.5, "end": 74.0, "text": "replacing it fell"}
+{"start": 74.0, "end": 74.5, "text": "on one household"}
+{"start": 74.5, "end": 75.0, "text": "while the benefit"}
+{"start": 75.0, "end": 75.5, "text": "was spread across"}
+{"start": 75.5, "end": 76.0, "text": "four hundred families"}
+{"start": 76.0, "end": 76.5, "text": "the mill stood"}
+{"start": 76.5, "end": 77.0, "text": "at the northern"}
+{"start": 77.0, "end": 77.5, "text": "end of the"}
+{"start": 77.5, "end": 78.0, "text": "valley where the"}
+{"start": 78.0, "end": 78.5, "text": "fall of the"}
+{"start": 78.5, "end": 79.0, "text": "river was steepest"}
+{"start": 79.0, "end": 79.5, "text": "and the ice"}
+{"start": 79.5, "end": 80.0, "text": "came latest its"}
+{"start": 80.0, "end": 80.5, "text": "wheel was undershot"}
+{"start": 80.5, "end": 81.0, "text": "which was wasteful"}
+{"start": 81.0, "end": 81.5, "text": "and nobody replaced"}
+{"start": 81.5, "end": 82.0, "text": "it for two"}
+{"start": 82.0, "end": 82.5, "text": "hundred years because"}
+{"start": 82.5, "end": 83.0, "text": "the cost of"}
+{"start": 83.0, "end": 83.5, "text": "replacing it fell"}
+{"start": 83.5, "end": 84.0, "text": "on one household"}
+{"start": 84.0, "end": 84.5, "text": "while the benefit"}
+{"start": 84.5, "end": 85.0, "text": "was spread across"}
+{"start": 85.0, "end": 85.5, "text": "four hundred families"}
+{"start": 85.5, "end": 86.0, "text": "the mill stood"}
+{"start": 86.0, "end": 86.5, "text": "at the northern"}
+{"start": 86.5, "end": 87.0, "text": "end of the"}
+{"start": 87.0, "end": 87.5, "text": "valley where the"}
+{"start": 87.5, "end": 88.0, "text": "fall of the"}
+{"start": 88.0, "end": 88.5, "text": "river was steepest"}
+{"start": 88.5, "end": 89.0, "text": "and the ice"}
+{"start": 89.0, "end": 89.5, "text": "came latest its"}
+{"start": 89.5, "end": 90.0, "text": "wheel was undershot"}
+{"start": 90.0, "end": 90.5, "text": "which was wasteful"}
+{"start": 90.5, "end": 91.0, "text": "and nobody replaced"}
+{"start": 91.0, "end": 91.5, "text": "it for two"}
+{"start": 91.5, "end": 92.0, "text": "hundred years because"}
+{"start": 92.0, "end": 92.5, "text": "the cost of"}
+{"start": 92.5, "end": 93.0, "text": "replacing it fell"}
+{"start": 93.0, "end": 93.5, "text": "on one household"}
+{"start": 93.5, "end": 94.0, "text": "while the benefit"}
+{"start": 94.0, "end": 94.5, "text": "was spread across"}
+{"start": 94.5, "end": 95.0, "text": "four hundred families"}
+{"start": 95.0, "end": 95.5, "text": "the mill stood"}
+{"start": 95.5, "end": 96.0, "text": "at the northern"}
+{"start": 96.0, "end": 96.5, "text": "end of the"}
+{"start": 96.5, "end": 97.0, "text": "valley where the"}
+{"start": 97.0, "end": 97.5, "text": "fall of the"}
+{"start": 97.5, "end": 98.0, "text": "river was steepest"}
+{"start": 98.0, "end": 98.5, "text": "and the ice"}
+{"start": 98.5, "end": 99.0, "text": "came latest its"}
+{"start": 99.0, "end": 99.5, "text": "wheel was undershot"}
+{"start": 99.5, "end": 100.0, "text": "which was wasteful"}
+{"start": 100.0, "end": 100.5, "text": "and nobody replaced"}
+{"start": 100.5, "end": 101.0, "text": "it for two"}
+{"start": 101.0, "end": 101.5, "text": "hundred years because"}
+{"start": 101.5, "end": 102.0, "text": "the cost of"}
+{"start": 102.0, "end": 102.5, "text": "replacing it fell"}
+{"start": 102.5, "end": 103.0, "text": "on one household"}
+{"start": 103.0, "end": 103.5, "text": "while the benefit"}
+{"start": 103.5, "end": 104.0, "text": "was spread across"}
+{"start": 104.0, "end": 104.5, "text": "four hundred families"}
+{"start": 104.5, "end": 105.0, "text": "the mill stood"}
+{"start": 105.0, "end": 105.5, "text": "at the northern"}
+{"start": 105.5, "end": 106.0, "text": "end of the"}
+{"start": 106.0, "end": 106.5, "text": "valley where the"}
+{"start": 106.5, "end": 107.0, "text": "fall of the"}
+{"start": 107.0, "end": 107.5, "text": "river was steepest"}
+{"start": 107.5, "end": 108.0, "text": "and the ice"}
+{"start": 108.0, "end": 108.5, "text": "came latest its"}
+{"start": 108.5, "end": 109.0, "text": "wheel was undershot"}
+{"start": 109.0, "end": 109.5, "text": "which was wasteful"}
+{"start": 109.5, "end": 110.0, "text": "and nobody replaced"}
+{"start": 110.0, "end": 110.5, "text": "it for two"}
+{"start": 110.5, "end": 111.0, "text": "hundred years because"}
+{"start": 111.0, "end": 111.5, "text": "the cost of"}
+{"start": 111.5, "end": 112.0, "text": "replacing it fell"}
+{"start": 112.0, "end": 112.5, "text": "on one household"}
+{"start": 112.5, "end": 113.0, "text": "while the benefit"}
+{"start": 113.0, "end": 113.5, "text": "was spread across"}
+{"start": 113.5, "end": 114.0, "text": "four hundred families"}
+{"start": 114.0, "end": 114.5, "text": "the mill stood"}
+{"start": 114.5, "end": 115.0, "text": "at the northern"}
+{"start": 115.0, "end": 115.5, "text": "end of the"}
+{"start": 115.5, "end": 116.0, "text": "valley where the"}
+{"start": 116.0, "end": 116.5, "text": "fall of the"}
+{"start": 116.5, "end": 117.0, "text": "river was steepest"}
+{"start": 117.0, "end": 117.5, "text": "and the ice"}
+{"start": 117.5, "end": 118.0, "text": "came latest its"}
+{"start": 118.0, "end": 118.5, "text": "wheel was undershot"}
+{"start": 118.5, "end": 119.0, "text": "which was wasteful"}
+{"start": 119.0, "end": 119.5, "text": "and nobody replaced"}
+{"start": 119.5, "end": 120.0, "text": "it for two"}
+{"start": 120.0, "end": 120.5, "text": "hundred years because"}
+{"start": 120.5, "end": 121.0, "text": "the cost of"}
+{"start": 121.0, "end": 121.5, "text": "replacing it fell"}
+{"start": 121.5, "end": 122.0, "text": "on one household"}
+{"start": 122.0, "end": 122.5, "text": "while the benefit"}
+{"start": 122.5, "end": 123.0, "text": "was spread across"}
+{"start": 123.0, "end": 123.5, "text": "four hundred families"}
+{"start": 123.5, "end": 124.0, "text": "the mill stood"}
+{"start": 124.0, "end": 124.5, "text": "at the northern"}
+{"start": 124.5, "end": 125.0, "text": "end of the"}
+{"start": 125.0, "end": 125.5, "text": "valley where the"}
+{"start": 125.5, "end": 126.0, "text": "fall of the"}
+{"start": 126.0, "end": 126.5, "text": "river was steepest"}
+{"start": 126.5, "end": 127.0, "text": "and the ice"}
+{"start": 127.0, "end": 127.5, "text": "came latest its"}
+{"start": 127.5, "end": 128.0, "text": "wheel was undershot"}
+{"start": 128.0, "end": 128.5, "text": "which was wasteful"}
+{"start": 128.5, "end": 129.0, "text": "and nobody replaced"}
+{"start": 129.0, "end": 129.5, "text": "it for two"}
+{"start": 129.5, "end": 130.0, "text": "hundred years because"}
+{"start": 130.0, "end": 130.5, "text": "the cost of"}
+{"start": 130.5, "end": 131.0, "text": "replacing it fell"}
+{"start": 131.0, "end": 131.5, "text": "on one household"}
+{"start": 131.5, "end": 132.0, "text": "while the benefit"}
+{"start": 132.0, "end": 132.5, "text": "was spread across"}
+{"start": 132.5, "end": 133.0, "text": "four hundred families"}
+{"start": 133.0, "end": 133.5, "text": "the mill stood"}
+{"start": 133.5, "end": 134.0, "text": "at the northern"}
+{"start": 134.0, "end": 134.5, "text": "end of the"}
+{"start": 134.5, "end": 135.0, "text": "valley where the"}
+{"start": 135.0, "end": 135.5, "text": "fall of the"}
+{"start": 135.5, "end": 136.0, "text": "river was steepest"}
+{"start": 136.0, "end": 136.5, "text": "and the ice"}
+{"start": 136.5, "end": 137.0, "text": "came latest its"}
+{"start": 137.0, "end": 137.5, "text": "wheel was undershot"}
+{"start": 137.5, "end": 138.0, "text": "which was wasteful"}
+{"start": 138.0, "end": 138.5, "text": "and nobody replaced"}
+{"start": 138.5, "end": 139.0, "text": "it for two"}
+{"start": 139.0, "end": 139.5, "text": "hundred years because"}
+{"start": 139.5, "end": 140.0, "text": "the cost of"}
+{"start": 140.0, "end": 140.5, "text": "replacing it fell"}
+{"start": 140.5, "end": 141.0, "text": "on one household"}
+{"start": 141.0, "end": 141.5, "text": "while the benefit"}
+{"start": 141.5, "end": 142.0, "text": "was spread across"}
+{"start": 142.0, "end": 142.5, "text": "four hundred families"}
+{"start": 142.5, "end": 143.0, "text": "the mill stood"}
+{"start": 143.0, "end": 143.5, "text": "at the northern"}
+{"start": 143.5, "end": 144.0, "text": "end of the"}
+{"start": 144.0, "end": 144.5, "text": "valley where the"}
+{"start": 144.5, "end": 145.0, "text": "fall of the"}
+{"start": 145.0, "end": 145.5, "text": "river was steepest"}
+{"start": 145.5, "end": 146.0, "text": "and the ice"}
+{"start": 146.0, "end": 146.5, "text": "came latest its"}
+{"start": 146.5, "end": 147.0, "text": "wheel was undershot"}
+{"start": 147.0, "end": 147.5, "text": "which was wasteful"}
+{"start": 147.5, "end": 148.0, "text": "and nobody replaced"}
+{"start": 148.0, "end": 148.5, "text": "it for two"}
+{"start": 148.5, "end": 149.0, "text": "hundred years because"}
+{"start": 149.0, "end": 149.5, "text": "the cost of"}
+{"start": 149.5, "end": 150.0, "text": "replacing it fell"}

parity/cases/cuepack/unicode.ndjson

@@ -0,0 +1,4 @@
+{"start": 0.0, "end": 4.0, "text": "Le café était naïve first"}
+{"start": 4.0, "end": 8.0, "text": "東京から来た手紙には"}
+{"start": 8.0, "end": 12.0, "text": "Ἀθῆναι καὶ Μόσχα"}
+{"start": 12.0, "end": 16.0, "text": "مرحبا"}

parity/golden/cuepack/backwards-times.default.ndjson

@@ -0,0 +1 @@
+{"idx": 0, "start_time": 10.0, "end_time": 1.0, "tokens": 14, "text": "an end before its start and one that overlaps it and one before both"}

parity/golden/cuepack/backwards-times.small.ndjson

@@ -0,0 +1 @@
+{"idx": 0, "start_time": 10.0, "end_time": 1.0, "tokens": 14, "text": "an end before its start and one that overlaps it and one before both"}

parity/golden/cuepack/blanks.default.ndjson

@@ -0,0 +1,7 @@
+{"idx": 0, "start_time": 4.0, "end_time": 76.0, "tokens": 128, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 1, "start_time": 76.0, "end_time": 148.0, "tokens": 120, "text": "end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at"}
+{"idx": 2, "start_time": 148.0, "end_time": 220.0, "tokens": 128, "text": "the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley"}
+{"idx": 3, "start_time": 224.0, "end_time": 296.0, "tokens": 128, "text": "where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 4, "start_time": 296.0, "end_time": 368.0, "tokens": 120, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of"}
+{"idx": 5, "start_time": 368.0, "end_time": 440.0, "tokens": 130, "text": "the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the"}
+{"idx": 6, "start_time": 444.0, "end_time": 480.0, "tokens": 69, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot"}

parity/golden/cuepack/blanks.small.ndjson

@@ -0,0 +1,48 @@
+{"idx": 0, "start_time": 4.0, "end_time": 12.0, "tokens": 16, "text": "the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 1, "start_time": 12.0, "end_time": 20.0, "tokens": 20, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 2, "start_time": 24.0, "end_time": 32.0, "tokens": 17, "text": "replaced it for two hundred years because the cost of replacing it fell on one household"}
+{"idx": 3, "start_time": 32.0, "end_time": 40.0, "tokens": 16, "text": "while the benefit was spread across four hundred families the mill stood at the northern end"}
+{"idx": 4, "start_time": 44.0, "end_time": 52.0, "tokens": 18, "text": "of the valley where the fall of the river was steepest and the ice came latest"}
+{"idx": 5, "start_time": 52.0, "end_time": 60.0, "tokens": 18, "text": "its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"idx": 6, "start_time": 64.0, "end_time": 72.0, "tokens": 16, "text": "the cost of replacing it fell on one household while the benefit was spread across four"}
+{"idx": 7, "start_time": 72.0, "end_time": 80.0, "tokens": 17, "text": "hundred families the mill stood at the northern end of the valley where the fall of"}
+{"idx": 8, "start_time": 84.0, "end_time": 92.0, "tokens": 20, "text": "the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"idx": 9, "start_time": 92.0, "end_time": 100.0, "tokens": 16, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on"}
+{"idx": 10, "start_time": 104.0, "end_time": 112.0, "tokens": 16, "text": "one household while the benefit was spread across four hundred families the mill stood at the"}
+{"idx": 11, "start_time": 112.0, "end_time": 120.0, "tokens": 19, "text": "northern end of the valley where the fall of the river was steepest and the ice"}
+{"idx": 12, "start_time": 124.0, "end_time": 132.0, "tokens": 18, "text": "came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred"}
+{"idx": 13, "start_time": 132.0, "end_time": 140.0, "tokens": 16, "text": "years because the cost of replacing it fell on one household while the benefit was spread"}
+{"idx": 14, "start_time": 144.0, "end_time": 152.0, "tokens": 17, "text": "across four hundred families the mill stood at the northern end of the valley where the"}
+{"idx": 15, "start_time": 152.0, "end_time": 160.0, "tokens": 19, "text": "fall of the river was steepest and the ice came latest its wheel was undershot which"}
+{"idx": 16, "start_time": 164.0, "end_time": 172.0, "tokens": 17, "text": "was wasteful and nobody replaced it for two hundred years because the cost of replacing it"}
+{"idx": 17, "start_time": 172.0, "end_time": 180.0, "tokens": 16, "text": "fell on one household while the benefit was spread across four hundred families the mill stood"}
+{"idx": 18, "start_time": 184.0, "end_time": 192.0, "tokens": 18, "text": "at the northern end of the valley where the fall of the river was steepest and"}
+{"idx": 19, "start_time": 192.0, "end_time": 200.0, "tokens": 18, "text": "the ice came latest its wheel was undershot which was wasteful and nobody replaced it for"}
+{"idx": 20, "start_time": 204.0, "end_time": 212.0, "tokens": 16, "text": "two hundred years because the cost of replacing it fell on one household while the benefit"}
+{"idx": 21, "start_time": 212.0, "end_time": 220.0, "tokens": 16, "text": "was spread across four hundred families the mill stood at the northern end of the valley"}
+{"idx": 22, "start_time": 224.0, "end_time": 232.0, "tokens": 18, "text": "where the fall of the river was steepest and the ice came latest its wheel was"}
+{"idx": 23, "start_time": 232.0, "end_time": 240.0, "tokens": 18, "text": "undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 24, "start_time": 244.0, "end_time": 252.0, "tokens": 17, "text": "replacing it fell on one household while the benefit was spread across four hundred families the"}
+{"idx": 25, "start_time": 252.0, "end_time": 260.0, "tokens": 16, "text": "mill stood at the northern end of the valley where the fall of the river was"}
+{"idx": 26, "start_time": 264.0, "end_time": 272.0, "tokens": 20, "text": "steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced"}
+{"idx": 27, "start_time": 272.0, "end_time": 280.0, "tokens": 16, "text": "it for two hundred years because the cost of replacing it fell on one household while"}
+{"idx": 28, "start_time": 284.0, "end_time": 292.0, "tokens": 16, "text": "the benefit was spread across four hundred families the mill stood at the northern end of"}
+{"idx": 29, "start_time": 292.0, "end_time": 300.0, "tokens": 18, "text": "the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 30, "start_time": 304.0, "end_time": 312.0, "tokens": 18, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the"}
+{"idx": 31, "start_time": 312.0, "end_time": 320.0, "tokens": 16, "text": "cost of replacing it fell on one household while the benefit was spread across four hundred"}
+{"idx": 32, "start_time": 324.0, "end_time": 332.0, "tokens": 17, "text": "families the mill stood at the northern end of the valley where the fall of the"}
+{"idx": 33, "start_time": 332.0, "end_time": 340.0, "tokens": 20, "text": "river was steepest and the ice came latest its wheel was undershot which was wasteful and"}
+{"idx": 34, "start_time": 344.0, "end_time": 352.0, "tokens": 17, "text": "nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"idx": 35, "start_time": 352.0, "end_time": 360.0, "tokens": 17, "text": "household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 36, "start_time": 364.0, "end_time": 372.0, "tokens": 18, "text": "end of the valley where the fall of the river was steepest and the ice came"}
+{"idx": 37, "start_time": 372.0, "end_time": 380.0, "tokens": 18, "text": "latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years"}
+{"idx": 38, "start_time": 384.0, "end_time": 392.0, "tokens": 16, "text": "because the cost of replacing it fell on one household while the benefit was spread across"}
+{"idx": 39, "start_time": 392.0, "end_time": 400.0, "tokens": 16, "text": "four hundred families the mill stood at the northern end of the valley where the fall"}
+{"idx": 40, "start_time": 404.0, "end_time": 412.0, "tokens": 19, "text": "of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"idx": 41, "start_time": 412.0, "end_time": 420.0, "tokens": 18, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell"}
+{"idx": 42, "start_time": 424.0, "end_time": 432.0, "tokens": 16, "text": "on one household while the benefit was spread across four hundred families the mill stood at"}
+{"idx": 43, "start_time": 432.0, "end_time": 440.0, "tokens": 18, "text": "the northern end of the valley where the fall of the river was steepest and the"}
+{"idx": 44, "start_time": 444.0, "end_time": 452.0, "tokens": 18, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"idx": 45, "start_time": 452.0, "end_time": 460.0, "tokens": 17, "text": "hundred years because the cost of replacing it fell on one household while the benefit was"}
+{"idx": 46, "start_time": 464.0, "end_time": 472.0, "tokens": 16, "text": "spread across four hundred families the mill stood at the northern end of the valley where"}
+{"idx": 47, "start_time": 472.0, "end_time": 480.0, "tokens": 19, "text": "the fall of the river was steepest and the ice came latest its wheel was undershot"}

parity/golden/cuepack/echoes.default.ndjson

@@ -0,0 +1,7 @@
+{"idx": 0, "start_time": 0.0, "end_time": 72.0, "tokens": 112, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"idx": 1, "start_time": 76.0, "end_time": 144.0, "tokens": 104, "text": "household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"idx": 2, "start_time": 148.0, "end_time": 216.0, "tokens": 102, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where"}
+{"idx": 3, "start_time": 220.0, "end_time": 288.0, "tokens": 104, "text": "the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the"}
+{"idx": 4, "start_time": 292.0, "end_time": 360.0, "tokens": 105, "text": "benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 5, "start_time": 364.0, "end_time": 432.0, "tokens": 101, "text": "replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of"}
+{"idx": 6, "start_time": 436.0, "end_time": 480.0, "tokens": 70, "text": "the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice"}

parity/golden/cuepack/echoes.small.ndjson

@@ -0,0 +1,41 @@
+{"idx": 0, "start_time": 0.0, "end_time": 8.0, "tokens": 16, "text": "the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 1, "start_time": 8.0, "end_time": 12.0, "tokens": 10, "text": "was steepest and the ice came latest its"}
+{"idx": 2, "start_time": 16.0, "end_time": 24.0, "tokens": 18, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the"}
+{"idx": 3, "start_time": 28.0, "end_time": 36.0, "tokens": 16, "text": "cost of replacing it fell on one household while the benefit was spread across four hundred"}
+{"idx": 4, "start_time": 40.0, "end_time": 48.0, "tokens": 17, "text": "families the mill stood at the northern end of the valley where the fall of the"}
+{"idx": 5, "start_time": 52.0, "end_time": 60.0, "tokens": 20, "text": "river was steepest and the ice came latest its wheel was undershot which was wasteful and"}
+{"idx": 6, "start_time": 64.0, "end_time": 72.0, "tokens": 17, "text": "nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"idx": 7, "start_time": 76.0, "end_time": 84.0, "tokens": 17, "text": "household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 8, "start_time": 88.0, "end_time": 96.0, "tokens": 18, "text": "end of the valley where the fall of the river was steepest and the ice came"}
+{"idx": 9, "start_time": 100.0, "end_time": 108.0, "tokens": 18, "text": "latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years"}
+{"idx": 10, "start_time": 112.0, "end_time": 120.0, "tokens": 16, "text": "because the cost of replacing it fell on one household while the benefit was spread across"}
+{"idx": 11, "start_time": 124.0, "end_time": 132.0, "tokens": 16, "text": "four hundred families the mill stood at the northern end of the valley where the fall"}
+{"idx": 12, "start_time": 136.0, "end_time": 144.0, "tokens": 19, "text": "of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"idx": 13, "start_time": 148.0, "end_time": 156.0, "tokens": 18, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell"}
+{"idx": 14, "start_time": 160.0, "end_time": 168.0, "tokens": 16, "text": "on one household while the benefit was spread across four hundred families the mill stood at"}
+{"idx": 15, "start_time": 172.0, "end_time": 180.0, "tokens": 18, "text": "the northern end of the valley where the fall of the river was steepest and the"}
+{"idx": 16, "start_time": 184.0, "end_time": 192.0, "tokens": 18, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"idx": 17, "start_time": 196.0, "end_time": 204.0, "tokens": 17, "text": "hundred years because the cost of replacing it fell on one household while the benefit was"}
+{"idx": 18, "start_time": 208.0, "end_time": 216.0, "tokens": 16, "text": "spread across four hundred families the mill stood at the northern end of the valley where"}
+{"idx": 19, "start_time": 220.0, "end_time": 228.0, "tokens": 19, "text": "the fall of the river was steepest and the ice came latest its wheel was undershot"}
+{"idx": 20, "start_time": 232.0, "end_time": 240.0, "tokens": 17, "text": "which was wasteful and nobody replaced it for two hundred years because the cost of replacing"}
+{"idx": 21, "start_time": 244.0, "end_time": 252.0, "tokens": 16, "text": "it fell on one household while the benefit was spread across four hundred families the mill"}
+{"idx": 22, "start_time": 256.0, "end_time": 264.0, "tokens": 18, "text": "stood at the northern end of the valley where the fall of the river was steepest"}
+{"idx": 23, "start_time": 268.0, "end_time": 276.0, "tokens": 18, "text": "and the ice came latest its wheel was undershot which was wasteful and nobody replaced it"}
+{"idx": 24, "start_time": 280.0, "end_time": 288.0, "tokens": 16, "text": "for two hundred years because the cost of replacing it fell on one household while the"}
+{"idx": 25, "start_time": 292.0, "end_time": 300.0, "tokens": 17, "text": "benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"idx": 26, "start_time": 304.0, "end_time": 312.0, "tokens": 19, "text": "valley where the fall of the river was steepest and the ice came latest its wheel"}
+{"idx": 27, "start_time": 316.0, "end_time": 324.0, "tokens": 18, "text": "was undershot which was wasteful and nobody replaced it for two hundred years because the cost"}
+{"idx": 28, "start_time": 328.0, "end_time": 336.0, "tokens": 16, "text": "of replacing it fell on one household while the benefit was spread across four hundred families"}
+{"idx": 29, "start_time": 340.0, "end_time": 348.0, "tokens": 16, "text": "the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 30, "start_time": 352.0, "end_time": 360.0, "tokens": 20, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 31, "start_time": 364.0, "end_time": 372.0, "tokens": 17, "text": "replaced it for two hundred years because the cost of replacing it fell on one household"}
+{"idx": 32, "start_time": 376.0, "end_time": 384.0, "tokens": 16, "text": "while the benefit was spread across four hundred families the mill stood at the northern end"}
+{"idx": 33, "start_time": 388.0, "end_time": 396.0, "tokens": 18, "text": "of the valley where the fall of the river was steepest and the ice came latest"}
+{"idx": 34, "start_time": 400.0, "end_time": 408.0, "tokens": 18, "text": "its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"idx": 35, "start_time": 412.0, "end_time": 420.0, "tokens": 16, "text": "the cost of replacing it fell on one household while the benefit was spread across four"}
+{"idx": 36, "start_time": 424.0, "end_time": 432.0, "tokens": 17, "text": "hundred families the mill stood at the northern end of the valley where the fall of"}
+{"idx": 37, "start_time": 436.0, "end_time": 444.0, "tokens": 20, "text": "the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"idx": 38, "start_time": 448.0, "end_time": 456.0, "tokens": 16, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on"}
+{"idx": 39, "start_time": 460.0, "end_time": 468.0, "tokens": 16, "text": "one household while the benefit was spread across four hundred families the mill stood at the"}
+{"idx": 40, "start_time": 472.0, "end_time": 480.0, "tokens": 19, "text": "northern end of the valley where the fall of the river was steepest and the ice"}

parity/golden/cuepack/long-cues.default.ndjson

@@ -0,0 +1,40 @@
+{"idx": 0, "start_time": 0.0, "end_time": 40.0, "tokens": 22, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the"}
+{"idx": 1, "start_time": 40.0, "end_time": 80.0, "tokens": 22, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the"}
+{"idx": 2, "start_time": 80.0, "end_time": 120.0, "tokens": 20, "text": "cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood"}
+{"idx": 3, "start_time": 120.0, "end_time": 160.0, "tokens": 22, "text": "at the northern end of the valley where the fall of the river was steepest and the ice came latest"}
+{"idx": 4, "start_time": 160.0, "end_time": 200.0, "tokens": 22, "text": "its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing"}
+{"idx": 5, "start_time": 200.0, "end_time": 240.0, "tokens": 20, "text": "it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 6, "start_time": 240.0, "end_time": 280.0, "tokens": 22, "text": "end of the valley where the fall of the river was steepest and the ice came latest its wheel was"}
+{"idx": 7, "start_time": 280.0, "end_time": 320.0, "tokens": 22, "text": "undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on"}
+{"idx": 8, "start_time": 320.0, "end_time": 360.0, "tokens": 20, "text": "one household while the benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"idx": 9, "start_time": 360.0, "end_time": 400.0, "tokens": 24, "text": "valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"idx": 10, "start_time": 400.0, "end_time": 440.0, "tokens": 22, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while"}
+{"idx": 11, "start_time": 440.0, "end_time": 480.0, "tokens": 20, "text": "the benefit was spread across four hundred families the mill stood at the northern end of the valley where the"}
+{"idx": 12, "start_time": 480.0, "end_time": 520.0, "tokens": 24, "text": "fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 13, "start_time": 520.0, "end_time": 560.0, "tokens": 21, "text": "replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was"}
+{"idx": 14, "start_time": 560.0, "end_time": 600.0, "tokens": 20, "text": "spread across four hundred families the mill stood at the northern end of the valley where the fall of the"}
+{"idx": 15, "start_time": 600.0, "end_time": 640.0, "tokens": 24, "text": "river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for"}
+{"idx": 16, "start_time": 640.0, "end_time": 680.0, "tokens": 20, "text": "two hundred years because the cost of replacing it fell on one household while the benefit was spread across four"}
+{"idx": 17, "start_time": 680.0, "end_time": 720.0, "tokens": 23, "text": "hundred families the mill stood at the northern end of the valley where the fall of the river was steepest"}
+{"idx": 18, "start_time": 720.0, "end_time": 760.0, "tokens": 22, "text": "and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years"}
+{"idx": 19, "start_time": 760.0, "end_time": 800.0, "tokens": 20, "text": "because the cost of replacing it fell on one household while the benefit was spread across four hundred families the"}
+{"idx": 20, "start_time": 800.0, "end_time": 840.0, "tokens": 22, "text": "mill stood at the northern end of the valley where the fall of the river was steepest and the ice"}
+{"idx": 21, "start_time": 840.0, "end_time": 880.0, "tokens": 22, "text": "came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost"}
+{"idx": 22, "start_time": 880.0, "end_time": 920.0, "tokens": 20, "text": "of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at"}
+{"idx": 23, "start_time": 920.0, "end_time": 960.0, "tokens": 22, "text": "the northern end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 24, "start_time": 960.0, "end_time": 1000.0, "tokens": 22, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it"}
+{"idx": 25, "start_time": 1000.0, "end_time": 1040.0, "tokens": 20, "text": "fell on one household while the benefit was spread across four hundred families the mill stood at the northern end"}
+{"idx": 26, "start_time": 1040.0, "end_time": 1080.0, "tokens": 23, "text": "of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot"}
+{"idx": 27, "start_time": 1080.0, "end_time": 1120.0, "tokens": 21, "text": "which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"idx": 28, "start_time": 1120.0, "end_time": 1160.0, "tokens": 21, "text": "household while the benefit was spread across four hundred families the mill stood at the northern end of the valley"}
+{"idx": 29, "start_time": 1160.0, "end_time": 1200.0, "tokens": 24, "text": "where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"idx": 30, "start_time": 1200.0, "end_time": 1240.0, "tokens": 20, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the"}
+{"idx": 31, "start_time": 1240.0, "end_time": 1280.0, "tokens": 21, "text": "benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall"}
+{"idx": 32, "start_time": 1280.0, "end_time": 1320.0, "tokens": 24, "text": "of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced"}
+{"idx": 33, "start_time": 1320.0, "end_time": 1360.0, "tokens": 20, "text": "it for two hundred years because the cost of replacing it fell on one household while the benefit was spread"}
+{"idx": 34, "start_time": 1360.0, "end_time": 1400.0, "tokens": 21, "text": "across four hundred families the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 35, "start_time": 1400.0, "end_time": 1440.0, "tokens": 24, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"idx": 36, "start_time": 1440.0, "end_time": 1480.0, "tokens": 21, "text": "hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred"}
+{"idx": 37, "start_time": 1480.0, "end_time": 1520.0, "tokens": 23, "text": "families the mill stood at the northern end of the valley where the fall of the river was steepest and"}
+{"idx": 38, "start_time": 1520.0, "end_time": 1560.0, "tokens": 22, "text": "the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"idx": 39, "start_time": 1560.0, "end_time": 1600.0, "tokens": 20, "text": "the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill"}

parity/golden/cuepack/long-cues.small.ndjson

@@ -0,0 +1,40 @@
+{"idx": 0, "start_time": 0.0, "end_time": 40.0, "tokens": 22, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the"}
+{"idx": 1, "start_time": 40.0, "end_time": 80.0, "tokens": 22, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the"}
+{"idx": 2, "start_time": 80.0, "end_time": 120.0, "tokens": 20, "text": "cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood"}
+{"idx": 3, "start_time": 120.0, "end_time": 160.0, "tokens": 22, "text": "at the northern end of the valley where the fall of the river was steepest and the ice came latest"}
+{"idx": 4, "start_time": 160.0, "end_time": 200.0, "tokens": 22, "text": "its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing"}
+{"idx": 5, "start_time": 200.0, "end_time": 240.0, "tokens": 20, "text": "it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 6, "start_time": 240.0, "end_time": 280.0, "tokens": 22, "text": "end of the valley where the fall of the river was steepest and the ice came latest its wheel was"}
+{"idx": 7, "start_time": 280.0, "end_time": 320.0, "tokens": 22, "text": "undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on"}
+{"idx": 8, "start_time": 320.0, "end_time": 360.0, "tokens": 20, "text": "one household while the benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"idx": 9, "start_time": 360.0, "end_time": 400.0, "tokens": 24, "text": "valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"idx": 10, "start_time": 400.0, "end_time": 440.0, "tokens": 22, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while"}
+{"idx": 11, "start_time": 440.0, "end_time": 480.0, "tokens": 20, "text": "the benefit was spread across four hundred families the mill stood at the northern end of the valley where the"}
+{"idx": 12, "start_time": 480.0, "end_time": 520.0, "tokens": 24, "text": "fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 13, "start_time": 520.0, "end_time": 560.0, "tokens": 21, "text": "replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was"}
+{"idx": 14, "start_time": 560.0, "end_time": 600.0, "tokens": 20, "text": "spread across four hundred families the mill stood at the northern end of the valley where the fall of the"}
+{"idx": 15, "start_time": 600.0, "end_time": 640.0, "tokens": 24, "text": "river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for"}
+{"idx": 16, "start_time": 640.0, "end_time": 680.0, "tokens": 20, "text": "two hundred years because the cost of replacing it fell on one household while the benefit was spread across four"}
+{"idx": 17, "start_time": 680.0, "end_time": 720.0, "tokens": 23, "text": "hundred families the mill stood at the northern end of the valley where the fall of the river was steepest"}
+{"idx": 18, "start_time": 720.0, "end_time": 760.0, "tokens": 22, "text": "and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years"}
+{"idx": 19, "start_time": 760.0, "end_time": 800.0, "tokens": 20, "text": "because the cost of replacing it fell on one household while the benefit was spread across four hundred families the"}
+{"idx": 20, "start_time": 800.0, "end_time": 840.0, "tokens": 22, "text": "mill stood at the northern end of the valley where the fall of the river was steepest and the ice"}
+{"idx": 21, "start_time": 840.0, "end_time": 880.0, "tokens": 22, "text": "came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost"}
+{"idx": 22, "start_time": 880.0, "end_time": 920.0, "tokens": 20, "text": "of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at"}
+{"idx": 23, "start_time": 920.0, "end_time": 960.0, "tokens": 22, "text": "the northern end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 24, "start_time": 960.0, "end_time": 1000.0, "tokens": 22, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it"}
+{"idx": 25, "start_time": 1000.0, "end_time": 1040.0, "tokens": 20, "text": "fell on one household while the benefit was spread across four hundred families the mill stood at the northern end"}
+{"idx": 26, "start_time": 1040.0, "end_time": 1080.0, "tokens": 23, "text": "of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot"}
+{"idx": 27, "start_time": 1080.0, "end_time": 1120.0, "tokens": 21, "text": "which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"idx": 28, "start_time": 1120.0, "end_time": 1160.0, "tokens": 21, "text": "household while the benefit was spread across four hundred families the mill stood at the northern end of the valley"}
+{"idx": 29, "start_time": 1160.0, "end_time": 1200.0, "tokens": 24, "text": "where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"idx": 30, "start_time": 1200.0, "end_time": 1240.0, "tokens": 20, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the"}
+{"idx": 31, "start_time": 1240.0, "end_time": 1280.0, "tokens": 21, "text": "benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall"}
+{"idx": 32, "start_time": 1280.0, "end_time": 1320.0, "tokens": 24, "text": "of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced"}
+{"idx": 33, "start_time": 1320.0, "end_time": 1360.0, "tokens": 20, "text": "it for two hundred years because the cost of replacing it fell on one household while the benefit was spread"}
+{"idx": 34, "start_time": 1360.0, "end_time": 1400.0, "tokens": 21, "text": "across four hundred families the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 35, "start_time": 1400.0, "end_time": 1440.0, "tokens": 24, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"idx": 36, "start_time": 1440.0, "end_time": 1480.0, "tokens": 21, "text": "hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred"}
+{"idx": 37, "start_time": 1480.0, "end_time": 1520.0, "tokens": 23, "text": "families the mill stood at the northern end of the valley where the fall of the river was steepest and"}
+{"idx": 38, "start_time": 1520.0, "end_time": 1560.0, "tokens": 22, "text": "the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"idx": 39, "start_time": 1560.0, "end_time": 1600.0, "tokens": 20, "text": "the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill"}

parity/golden/cuepack/one-cue.default.ndjson

@@ -0,0 +1 @@
+{"idx": 0, "start_time": 0.0, "end_time": 4.0, "tokens": 8, "text": "the mill stood at the northern end of"}

parity/golden/cuepack/one-cue.small.ndjson

@@ -0,0 +1 @@
+{"idx": 0, "start_time": 0.0, "end_time": 4.0, "tokens": 8, "text": "the mill stood at the northern end of"}

parity/golden/cuepack/plain.default.ndjson

@@ -0,0 +1,7 @@
+{"idx": 0, "start_time": 0.0, "end_time": 72.0, "tokens": 156, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"idx": 1, "start_time": 72.0, "end_time": 144.0, "tokens": 152, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood"}
+{"idx": 2, "start_time": 144.0, "end_time": 216.0, "tokens": 156, "text": "at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced"}
+{"idx": 3, "start_time": 216.0, "end_time": 288.0, "tokens": 152, "text": "it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 4, "start_time": 288.0, "end_time": 360.0, "tokens": 156, "text": "end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"idx": 5, "start_time": 360.0, "end_time": 432.0, "tokens": 153, "text": "hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"idx": 6, "start_time": 432.0, "end_time": 480.0, "tokens": 105, "text": "valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household"}

parity/golden/cuepack/plain.small.ndjson

@@ -0,0 +1,60 @@
+{"idx": 0, "start_time": 0.0, "end_time": 8.0, "tokens": 16, "text": "the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 1, "start_time": 8.0, "end_time": 16.0, "tokens": 20, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 2, "start_time": 16.0, "end_time": 24.0, "tokens": 17, "text": "replaced it for two hundred years because the cost of replacing it fell on one household"}
+{"idx": 3, "start_time": 24.0, "end_time": 32.0, "tokens": 16, "text": "while the benefit was spread across four hundred families the mill stood at the northern end"}
+{"idx": 4, "start_time": 32.0, "end_time": 40.0, "tokens": 18, "text": "of the valley where the fall of the river was steepest and the ice came latest"}
+{"idx": 5, "start_time": 40.0, "end_time": 48.0, "tokens": 18, "text": "its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"idx": 6, "start_time": 48.0, "end_time": 56.0, "tokens": 16, "text": "the cost of replacing it fell on one household while the benefit was spread across four"}
+{"idx": 7, "start_time": 56.0, "end_time": 64.0, "tokens": 17, "text": "hundred families the mill stood at the northern end of the valley where the fall of"}
+{"idx": 8, "start_time": 64.0, "end_time": 72.0, "tokens": 20, "text": "the river was steepest and the ice came latest its wheel was undershot which was wasteful"}
+{"idx": 9, "start_time": 72.0, "end_time": 80.0, "tokens": 16, "text": "and nobody replaced it for two hundred years because the cost of replacing it fell on"}
+{"idx": 10, "start_time": 80.0, "end_time": 88.0, "tokens": 16, "text": "one household while the benefit was spread across four hundred families the mill stood at the"}
+{"idx": 11, "start_time": 88.0, "end_time": 96.0, "tokens": 19, "text": "northern end of the valley where the fall of the river was steepest and the ice"}
+{"idx": 12, "start_time": 96.0, "end_time": 104.0, "tokens": 18, "text": "came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred"}
+{"idx": 13, "start_time": 104.0, "end_time": 112.0, "tokens": 16, "text": "years because the cost of replacing it fell on one household while the benefit was spread"}
+{"idx": 14, "start_time": 112.0, "end_time": 120.0, "tokens": 17, "text": "across four hundred families the mill stood at the northern end of the valley where the"}
+{"idx": 15, "start_time": 120.0, "end_time": 128.0, "tokens": 19, "text": "fall of the river was steepest and the ice came latest its wheel was undershot which"}
+{"idx": 16, "start_time": 128.0, "end_time": 136.0, "tokens": 17, "text": "was wasteful and nobody replaced it for two hundred years because the cost of replacing it"}
+{"idx": 17, "start_time": 136.0, "end_time": 144.0, "tokens": 16, "text": "fell on one household while the benefit was spread across four hundred families the mill stood"}
+{"idx": 18, "start_time": 144.0, "end_time": 152.0, "tokens": 18, "text": "at the northern end of the valley where the fall of the river was steepest and"}
+{"idx": 19, "start_time": 152.0, "end_time": 160.0, "tokens": 18, "text": "the ice came latest its wheel was undershot which was wasteful and nobody replaced it for"}
+{"idx": 20, "start_time": 160.0, "end_time": 168.0, "tokens": 16, "text": "two hundred years because the cost of replacing it fell on one household while the benefit"}
+{"idx": 21, "start_time": 168.0, "end_time": 176.0, "tokens": 16, "text": "was spread across four hundred families the mill stood at the northern end of the valley"}
+{"idx": 22, "start_time": 176.0, "end_time": 184.0, "tokens": 18, "text": "where the fall of the river was steepest and the ice came latest its wheel was"}
+{"idx": 23, "start_time": 184.0, "end_time": 192.0, "tokens": 18, "text": "undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 24, "start_time": 192.0, "end_time": 200.0, "tokens": 17, "text": "replacing it fell on one household while the benefit was spread across four hundred families the"}
+{"idx": 25, "start_time": 200.0, "end_time": 208.0, "tokens": 16, "text": "mill stood at the northern end of the valley where the fall of the river was"}
+{"idx": 26, "start_time": 208.0, "end_time": 216.0, "tokens": 20, "text": "steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced"}
+{"idx": 27, "start_time": 216.0, "end_time": 224.0, "tokens": 16, "text": "it for two hundred years because the cost of replacing it fell on one household while"}
+{"idx": 28, "start_time": 224.0, "end_time": 232.0, "tokens": 16, "text": "the benefit was spread across four hundred families the mill stood at the northern end of"}
+{"idx": 29, "start_time": 232.0, "end_time": 240.0, "tokens": 18, "text": "the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 30, "start_time": 240.0, "end_time": 248.0, "tokens": 18, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the"}
+{"idx": 31, "start_time": 248.0, "end_time": 256.0, "tokens": 16, "text": "cost of replacing it fell on one household while the benefit was spread across four hundred"}
+{"idx": 32, "start_time": 256.0, "end_time": 264.0, "tokens": 17, "text": "families the mill stood at the northern end of the valley where the fall of the"}
+{"idx": 33, "start_time": 264.0, "end_time": 272.0, "tokens": 20, "text": "river was steepest and the ice came latest its wheel was undershot which was wasteful and"}
+{"idx": 34, "start_time": 272.0, "end_time": 280.0, "tokens": 17, "text": "nobody replaced it for two hundred years because the cost of replacing it fell on one"}
+{"idx": 35, "start_time": 280.0, "end_time": 288.0, "tokens": 17, "text": "household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 36, "start_time": 288.0, "end_time": 296.0, "tokens": 18, "text": "end of the valley where the fall of the river was steepest and the ice came"}
+{"idx": 37, "start_time": 296.0, "end_time": 304.0, "tokens": 18, "text": "latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years"}
+{"idx": 38, "start_time": 304.0, "end_time": 312.0, "tokens": 16, "text": "because the cost of replacing it fell on one household while the benefit was spread across"}
+{"idx": 39, "start_time": 312.0, "end_time": 320.0, "tokens": 16, "text": "four hundred families the mill stood at the northern end of the valley where the fall"}
+{"idx": 40, "start_time": 320.0, "end_time": 328.0, "tokens": 19, "text": "of the river was steepest and the ice came latest its wheel was undershot which was"}
+{"idx": 41, "start_time": 328.0, "end_time": 336.0, "tokens": 18, "text": "wasteful and nobody replaced it for two hundred years because the cost of replacing it fell"}
+{"idx": 42, "start_time": 336.0, "end_time": 344.0, "tokens": 16, "text": "on one household while the benefit was spread across four hundred families the mill stood at"}
+{"idx": 43, "start_time": 344.0, "end_time": 352.0, "tokens": 18, "text": "the northern end of the valley where the fall of the river was steepest and the"}
+{"idx": 44, "start_time": 352.0, "end_time": 360.0, "tokens": 18, "text": "ice came latest its wheel was undershot which was wasteful and nobody replaced it for two"}
+{"idx": 45, "start_time": 360.0, "end_time": 368.0, "tokens": 17, "text": "hundred years because the cost of replacing it fell on one household while the benefit was"}
+{"idx": 46, "start_time": 368.0, "end_time": 376.0, "tokens": 16, "text": "spread across four hundred families the mill stood at the northern end of the valley where"}
+{"idx": 47, "start_time": 376.0, "end_time": 384.0, "tokens": 19, "text": "the fall of the river was steepest and the ice came latest its wheel was undershot"}
+{"idx": 48, "start_time": 384.0, "end_time": 392.0, "tokens": 17, "text": "which was wasteful and nobody replaced it for two hundred years because the cost of replacing"}
+{"idx": 49, "start_time": 392.0, "end_time": 400.0, "tokens": 16, "text": "it fell on one household while the benefit was spread across four hundred families the mill"}
+{"idx": 50, "start_time": 400.0, "end_time": 408.0, "tokens": 18, "text": "stood at the northern end of the valley where the fall of the river was steepest"}
+{"idx": 51, "start_time": 408.0, "end_time": 416.0, "tokens": 18, "text": "and the ice came latest its wheel was undershot which was wasteful and nobody replaced it"}
+{"idx": 52, "start_time": 416.0, "end_time": 424.0, "tokens": 16, "text": "for two hundred years because the cost of replacing it fell on one household while the"}
+{"idx": 53, "start_time": 424.0, "end_time": 432.0, "tokens": 17, "text": "benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"idx": 54, "start_time": 432.0, "end_time": 440.0, "tokens": 19, "text": "valley where the fall of the river was steepest and the ice came latest its wheel"}
+{"idx": 55, "start_time": 440.0, "end_time": 448.0, "tokens": 18, "text": "was undershot which was wasteful and nobody replaced it for two hundred years because the cost"}
+{"idx": 56, "start_time": 448.0, "end_time": 456.0, "tokens": 16, "text": "of replacing it fell on one household while the benefit was spread across four hundred families"}
+{"idx": 57, "start_time": 456.0, "end_time": 464.0, "tokens": 16, "text": "the mill stood at the northern end of the valley where the fall of the river"}
+{"idx": 58, "start_time": 464.0, "end_time": 472.0, "tokens": 20, "text": "was steepest and the ice came latest its wheel was undershot which was wasteful and nobody"}
+{"idx": 59, "start_time": 472.0, "end_time": 480.0, "tokens": 17, "text": "replaced it for two hundred years because the cost of replacing it fell on one household"}

parity/golden/cuepack/short-cues.default.ndjson

@@ -0,0 +1,3 @@
+{"idx": 0, "start_time": 0.0, "end_time": 74.5, "tokens": 479, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household"}
+{"idx": 1, "start_time": 74.5, "end_time": 149.0, "tokens": 479, "text": "while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because"}
+{"idx": 2, "start_time": 149.0, "end_time": 150.0, "tokens": 6, "text": "the cost of replacing it fell"}

parity/golden/cuepack/short-cues.small.ndjson

@@ -0,0 +1,48 @@
+{"idx": 0, "start_time": 0.0, "end_time": 3.5, "tokens": 23, "text": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice"}
+{"idx": 1, "start_time": 3.5, "end_time": 7.0, "tokens": 23, "text": "came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 2, "start_time": 7.0, "end_time": 10.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 3, "start_time": 10.5, "end_time": 13.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 4, "start_time": 13.5, "end_time": 16.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 5, "start_time": 16.5, "end_time": 20.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 6, "start_time": 20.0, "end_time": 23.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 7, "start_time": 23.0, "end_time": 26.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 8, "start_time": 26.0, "end_time": 29.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 9, "start_time": 29.5, "end_time": 32.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 10, "start_time": 32.5, "end_time": 35.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 11, "start_time": 35.5, "end_time": 39.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 12, "start_time": 39.0, "end_time": 42.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 13, "start_time": 42.0, "end_time": 45.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 14, "start_time": 45.0, "end_time": 48.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 15, "start_time": 48.5, "end_time": 51.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 16, "start_time": 51.5, "end_time": 54.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 17, "start_time": 54.5, "end_time": 58.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 18, "start_time": 58.0, "end_time": 61.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 19, "start_time": 61.0, "end_time": 64.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 20, "start_time": 64.0, "end_time": 67.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 21, "start_time": 67.5, "end_time": 70.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 22, "start_time": 70.5, "end_time": 73.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 23, "start_time": 73.5, "end_time": 77.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 24, "start_time": 77.0, "end_time": 80.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 25, "start_time": 80.0, "end_time": 83.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 26, "start_time": 83.0, "end_time": 86.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 27, "start_time": 86.5, "end_time": 89.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 28, "start_time": 89.5, "end_time": 92.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 29, "start_time": 92.5, "end_time": 96.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 30, "start_time": 96.0, "end_time": 99.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 31, "start_time": 99.0, "end_time": 102.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 32, "start_time": 102.0, "end_time": 105.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 33, "start_time": 105.5, "end_time": 108.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 34, "start_time": 108.5, "end_time": 111.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 35, "start_time": 111.5, "end_time": 115.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 36, "start_time": 115.0, "end_time": 118.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 37, "start_time": 118.0, "end_time": 121.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 38, "start_time": 121.0, "end_time": 124.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 39, "start_time": 124.5, "end_time": 127.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 40, "start_time": 127.5, "end_time": 130.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 41, "start_time": 130.5, "end_time": 134.0, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 42, "start_time": 134.0, "end_time": 137.0, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 43, "start_time": 137.0, "end_time": 140.0, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 44, "start_time": 140.0, "end_time": 143.5, "tokens": 22, "text": "replacing it fell on one household while the benefit was spread across four hundred families the mill stood at the northern"}
+{"idx": 45, "start_time": 143.5, "end_time": 146.5, "tokens": 20, "text": "end of the valley where the fall of the river was steepest and the ice came latest its"}
+{"idx": 46, "start_time": 146.5, "end_time": 149.5, "tokens": 20, "text": "wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost of"}
+{"idx": 47, "start_time": 149.5, "end_time": 150.0, "tokens": 4, "text": "replacing it fell"}

parity/golden/cuepack/unicode.default.ndjson

@@ -0,0 +1 @@
+{"idx": 0, "start_time": 0.0, "end_time": 16.0, "tokens": 44, "text": "Le café était naïve first 東京から来た手紙には Ἀθῆναι καὶ Μόσχα مرحبا"}

parity/golden/cuepack/unicode.small.ndjson

@@ -0,0 +1,3 @@
+{"idx": 0, "start_time": 0.0, "end_time": 8.0, "tokens": 19, "text": "Le café était naïve first 東京から来た手紙には"}
+{"idx": 1, "start_time": 8.0, "end_time": 12.0, "tokens": 20, "text": "Ἀθῆναι καὶ Μόσχα"}
+{"idx": 2, "start_time": 12.0, "end_time": 16.0, "tokens": 5, "text": "مرحبا"}

parity/oracle/cuepack.py

@@ -0,0 +1,50 @@
+"""Pack cues into passages the way the Python pipeline packs them.
+
+Reads a cue stream — one `{"start", "end", "text"}` object per line — with
+the seconds and token budgets as arguments, and writes one passage per line.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/podcast-pipeline/src \
+        python3 oracle/cuepack.py 75 512 < cues.ndjson
+"""
+
+import json
+import os
+import sys
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from podcast_pipeline.transcripts import Cue, chunk_cues  # noqa: E402
+
+
+def main() -> int:
+    seconds = float(sys.argv[1])
+    max_tokens = int(sys.argv[2])
+    cues = []
+    for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+        if not line.strip():
+            continue
+        cue = json.loads(line)
+        cues.append(Cue(start=cue["start"], end=cue["end"], text=cue["text"]))
+
+    out = sys.stdout
+    for passage in chunk_cues(cues, seconds, max_tokens):
+        out.write(
+            json.dumps(
+                {
+                    "idx": passage.chunk_idx,
+                    "start_time": passage.start_time,
+                    "end_time": passage.end_time,
+                    "tokens": passage.token_count,
+                    "text": passage.text,
+                },
+                ensure_ascii=False,
+            )
+            + "\n"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 8] = [
+    let jobs: [Job; 9] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
@@ -28,6 +28,7 @@ fn main() -> ExitCode {
         ("cuenorm", cuenorm),
         ("cueseg", cueseg),
         ("rrf", rrf),
+        ("cuepack", cuepack),
     ];
 
     for (name, job) in jobs {
@@ -71,6 +72,29 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn cuepack() -> Written {
+    let dir = parity::golden("cuepack");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let budgets = [("default", 75.0, 512), ("small", 12.0, 24)];
+    let mut written = Vec::new();
+    for case in parity::cases_with_extension("cuepack", "ndjson") {
+        let name = parity::case_name(&case);
+        let cues = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
+        for (set, seconds, max_tokens) in budgets {
+            let packed = parity::run_oracle_with(
+                "cuepack.py",
+                &[seconds.to_string(), max_tokens.to_string()],
+                &cues,
+            )?;
+            let path = dir.join(format!("{name}.{set}.ndjson"));
+            std::fs::write(&path, packed).map_err(|e| e.to_string())?;
+            written.push(path.display().to_string());
+        }
+    }
+    Ok(written)
+}
+
 fn rrf() -> Written {
     let dir = parity::golden("rrf");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/cuepack.rs

@@ -0,0 +1,120 @@
+//! Passage boundaries, times and token counts must match the Python exactly.
+//!
+//! A passage that starts one cue later is a different embedding of different
+//! words at a different timestamp — which is to say a different row, and a
+//! re-ingest that replaces an index rather than extending it.
+
+use cuekit::Cue;
+use serde_json::Value;
+
+/// A passage, as the two sides can be compared.
+type Passage = (u64, f64, f64, u64, String);
+
+fn passages(text: &str) -> Vec<Passage> {
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let passage: Value = serde_json::from_str(line).expect("valid JSON");
+            (
+                passage["idx"].as_u64().expect("an index"),
+                passage["start_time"].as_f64().expect("a start"),
+                passage["end_time"].as_f64().expect("an end"),
+                passage["tokens"].as_u64().expect("a token count"),
+                passage["text"].as_str().expect("text").to_string(),
+            )
+        })
+        .collect()
+}
+
+fn cues_of(case: &std::path::Path) -> Vec<Cue> {
+    let text = std::fs::read_to_string(case).expect("a readable case");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let cue: Value = serde_json::from_str(line).expect("valid JSON");
+            Cue::new(
+                cue["start"].as_f64().expect("a start"),
+                cue["end"].as_f64().expect("an end"),
+                cue["text"].as_str().expect("text"),
+            )
+        })
+        .collect()
+}
+
+/// The budget sets the packer is pinned at: the defaults, and a small set
+/// that makes both flush conditions fire on a fixture short enough to read.
+const BUDGETS: &[(&str, cuepack::Budgets)] = &[
+    (
+        "default",
+        cuepack::Budgets {
+            seconds: 75.0,
+            max_tokens: 512,
+        },
+    ),
+    (
+        "small",
+        cuepack::Budgets {
+            seconds: 12.0,
+            max_tokens: 24,
+        },
+    ),
+];
+
+#[test]
+fn every_case_packs_the_same_as_python() {
+    let cases = parity::cases_with_extension("cuepack", "ndjson");
+    assert!(!cases.is_empty(), "no cases found");
+
+    for case in cases {
+        let name = parity::case_name(&case);
+        let cues = cues_of(&case);
+        for (set, budgets) in BUDGETS {
+            let theirs = passages(&parity::read_golden_text(&format!(
+                "cuepack/{name}.{set}.ndjson"
+            )));
+            let ours: Vec<Passage> = cuepack::pack(&cues, budgets)
+                .into_iter()
+                .map(|passage| {
+                    (
+                        passage.idx as u64,
+                        passage.start_time.expect("a start"),
+                        passage.end_time.expect("an end"),
+                        passage.tokens as u64,
+                        passage.text,
+                    )
+                })
+                .collect();
+
+            for (at, (ours, theirs)) in ours.iter().zip(&theirs).enumerate() {
+                assert_eq!(ours, theirs, "{name} [{set}] passage {at} differs");
+            }
+            assert_eq!(
+                ours.len(),
+                theirs.len(),
+                "{name} [{set}]: python packed {} passages, rust {}",
+                theirs.len(),
+                ours.len()
+            );
+        }
+    }
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_files_still_match_live_python() {
+    for case in parity::cases_with_extension("cuepack", "ndjson") {
+        let name = parity::case_name(&case);
+        let cues = std::fs::read_to_string(&case).expect("a readable case");
+        for (set, budgets) in BUDGETS {
+            let live = parity::run_oracle_with(
+                "cuepack.py",
+                &[budgets.seconds.to_string(), budgets.max_tokens.to_string()],
+                &cues,
+            )
+            .expect("the oracle runs");
+            let golden = parity::read_golden_text(&format!("cuepack/{name}.{set}.ndjson"));
+            assert_eq!(live, golden, "golden file for {name} [{set}] is stale");
+        }
+    }
+}