cueseg: topic chapters from a transcript, and a worked pipeline in the readme

Cargo.lock

@@ -187,6 +187,20 @@ dependencies = [
  "cuekit",
 ]
 
+[[package]]
+name = "cueseg"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "corpus-core",
+ "cuekit",
+ "regex",
+ "schemars",
+ "serde",
+ "serde_json",
+ "textkit",
+]
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.20"
@@ -268,6 +282,7 @@ dependencies = [
  "chunkmd",
  "corpus-core",
  "cuekit",
+ "cueseg",
  "mdscrub",
  "pdftriage",
  "serde_json",

Cargo.toml

@@ -17,6 +17,7 @@ chunkmd = { path = "crates/chunkmd", version = "0.1.0" }
 mdscrub = { path = "crates/mdscrub", version = "0.1.0" }
 pdftriage = { path = "crates/pdftriage", version = "0.1.0" }
 cuenorm = { path = "crates/cuenorm", version = "0.1.0" }
+cueseg = { path = "crates/cueseg", version = "0.1.0" }
 
 clap = { version = "4", features = ["derive"] }
 fancy-regex = "0.16"

README.md

@@ -19,6 +19,46 @@ cargo install --git https://git.pumpin.online/funkpower/catena <tool>
 
 Each tool installs by name, so you can take one without taking the rest.
 
+## A worked pipeline
+
+Converted markdown in, retrievable passages out:
+
+```
+$ pandoc -t commonmark chapter.xhtml | mdscrub | chunkmd --min-tokens 20
+{"v":1,"idx":0,"text":"The obligation ran both ways, and contemporaries said so plainly\nwhenever they were asked.4 A lord who could not\nprotect was a lord who could not command…","tokens":57,"page":41,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]}
+{"v":1,"idx":1,"text":"What the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested…","tokens":41,"page":42,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]}
+```
+
+A recording instead of a book, with chapter marks a scrubber can use:
+
+```
+$ cuenorm episode.vtt | cueseg --duration 7245
+{"v":1,"start":0.0,"end":1190.6,"title":"The Mill Stood At The Northern End","excerpt":"the mill stood at the northern end of the valley where…"}
+{"v":1,"start":1190.6,"end":2381.3,"title":"Parliament Dissolved Itself Again","excerpt":"parliament dissolved itself again amid furious objections…"}
+```
+
+And before any of that, whether a scanned PDF is worth reading at all:
+
+```
+$ pdftriage scan.pdf
+garbled, needs OCR
+  1820 characters on the opening pages
+  1820 characters over 5 sample pages, ascii text score 0.15
+  letters 0.13, latin-1 0.29
+
+$ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done
+```
+
+## The tools
+
+| | |
+|---|---|
+| `mdscrub` | Converted markdown in, clean markdown out: residual HTML, control characters, headings, page markers |
+| `chunkmd` | Markdown into passages that carry their page, chapter and role |
+| `pdftriage` | Whether a PDF's text layer is healthy, garbled, sparse, permuted, or absent — and so whether OCR is worth its cost |
+| `cuenorm` | Any subtitle format in, one normalised cue stream out |
+| `cueseg` | A timed transcript into topic chapters |
+
 ## The contract
 
 Every binary in this workspace obeys all of this, or it does not ship.

crates/cueseg/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "cueseg"
+description = "Turns a timed transcript into topic chapters."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+cuekit.workspace = true
+regex.workspace = true
+schemars.workspace = true
+serde.workspace = true
+textkit.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
+
+[lints]
+workspace = true

crates/cueseg/src/lib.rs

@@ -0,0 +1,136 @@
+//! Turns a transcript into topic chapters.
+//!
+//! No model and no network: the text is hashed into a small lexical vector a
+//! window at a time, and a topic boundary is where two consecutive windows
+//! stop resembling each other. It is crude, and it is also what makes a
+//! three-hour recording navigable — the chapters it finds are the marks a
+//! scrubber needs and the spans a summariser should be given one at a time.
+//!
+//! Every constant here is a shape decision about recorded speech, and each
+//! one carries the reason it has the value it has.
+
+pub mod segment;
+pub mod title;
+pub mod vector;
+pub mod window;
+
+use cuekit::Cue;
+
+pub use segment::Segment;
+pub use window::Window;
+
+/// How long a window of transcript is, in seconds.
+///
+/// Long enough that a window has a subject, short enough that a boundary
+/// lands within a minute of the real one.
+pub const WINDOW_SECONDS: f64 = 75.0;
+
+/// No chapter may be shorter than this.
+///
+/// A chapter mark every two minutes is not navigation, it is noise.
+pub const MIN_SEGMENT_SECONDS: f64 = 8.0 * 60.0;
+
+/// How many chapters a recording gets, as a function of how long it is.
+pub const TARGET_SECONDS_PER_CHAPTER: f64 = 15.0 * 60.0;
+/// Fewest chapters, whatever the length.
+pub const MIN_CHAPTERS: usize = 4;
+/// Most chapters, whatever the length.
+pub const MAX_CHAPTERS: usize = 20;
+
+/// How much of a chapter's text is kept as its excerpt, in characters.
+pub const EXCERPT_CHARS: usize = 900;
+
+/// Cut a transcript into chapters.
+///
+/// `duration` is how long the recording is. It is taken rather than derived
+/// because a transcript often stops before the recording does — music at the
+/// end, or a speaker who stopped talking — and a chapter that claims to run
+/// to the end of the transcript rather than the end of the file is a chapter
+/// mark that lands in the wrong place.
+pub fn segment(cues: &[Cue], duration: f64) -> Vec<Segment> {
+    let windows = window::windows(cues, WINDOW_SECONDS);
+    if windows.is_empty() {
+        return Vec::new();
+    }
+    let vectors: Vec<Vec<f64>> = windows
+        .iter()
+        .map(|window| vector::lexical(&window.text, vector::DIMENSIONS))
+        .collect();
+    let similarities = vector::consecutive_cosine(&vectors);
+    let boundaries =
+        segment::boundary_times(&windows, &similarities, duration, MIN_SEGMENT_SECONDS, None);
+    segment::from_boundaries(&windows, &boundaries, duration)
+}
+
+/// How many chapters a recording of this length should have.
+pub fn target_chapter_count(duration: f64) -> usize {
+    let target = round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER);
+    (target as usize).clamp(MIN_CHAPTERS, MAX_CHAPTERS)
+}
+
+/// Round half to even, which is what the Python this was ported from does
+/// and what Rust's own `round` does not: `round(2.5)` is 2 there and 3 here,
+/// and a recording exactly two and a half chapters long would otherwise get a
+/// different number of them.
+fn round_half_to_even(value: f64) -> f64 {
+    let rounded = value.round();
+    if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 {
+        rounded - value.signum()
+    } else {
+        rounded
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn rounding_goes_to_even_on_a_half() {
+        assert_eq!(round_half_to_even(0.5), 0.0);
+        assert_eq!(round_half_to_even(1.5), 2.0);
+        assert_eq!(round_half_to_even(2.5), 2.0);
+        assert_eq!(round_half_to_even(3.5), 4.0);
+        assert_eq!(round_half_to_even(-2.5), -2.0);
+        assert_eq!(round_half_to_even(2.4), 2.0);
+        assert_eq!(round_half_to_even(2.6), 3.0);
+    }
+
+    #[test]
+    fn the_chapter_count_follows_the_length_within_bounds() {
+        assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS);
+        assert_eq!(target_chapter_count(60.0 * 60.0), MIN_CHAPTERS);
+        assert_eq!(target_chapter_count(90.0 * 60.0), 6);
+        assert_eq!(target_chapter_count(100.0 * 60.0 * 60.0), MAX_CHAPTERS);
+    }
+
+    #[test]
+    fn a_transcript_with_no_cues_has_no_chapters() {
+        assert!(segment(&[], 3600.0).is_empty());
+    }
+
+    #[test]
+    fn a_long_transcript_is_cut_into_several_chapters() {
+        // Two subjects, an hour each, with nothing in common lexically.
+        let mut cues = Vec::new();
+        let mut at = 0.0;
+        for round in 0..2 {
+            let words = if round == 0 {
+                "the mill stood at the northern end where the fall was steepest"
+            } else {
+                "parliament dissolved itself again amid furious objections from every bench"
+            };
+            while at < (round as f64 + 1.0) * 3600.0 {
+                cues.push(Cue::new(at, at + 5.0, words));
+                at += 5.0;
+            }
+        }
+        let found = segment(&cues, at);
+        assert!(found.len() >= 2, "{} chapters", found.len());
+        assert_eq!(found[0].start, 0.0);
+        assert_eq!(found.last().expect("a chapter").end, at);
+        for pair in found.windows(2) {
+            assert_eq!(pair[0].end, pair[1].start, "chapters must not leave a gap");
+        }
+    }
+}

crates/cueseg/src/main.rs

@@ -0,0 +1,116 @@
+//! Turns a timed transcript into topic chapters.
+
+use std::io::{BufWriter, Read, Write};
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::{exit, ndjson};
+use cuekit::Cue;
+
+/// Read a transcript and write its topic chapters, one per line as JSON.
+///
+/// Takes either the cue stream `cuenorm` writes or a subtitle file directly.
+/// The chapters are found from the text alone — no model, no network — by
+/// comparing what is said in one stretch with what is said in the next.
+///
+/// The titles are extracted from the chapters' own first sentences. They are
+/// meant to be replaced by something better later, and to be usable until
+/// they are.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    long_about = None,
+    after_help = "\
+Example:
+
+    cuenorm episode.vtt | cueseg --duration 7245 > chapters.ndjson"
+)]
+struct Args {
+    /// Cue stream or subtitle file to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+
+    /// How long the recording is, in seconds. Defaults to the end of the
+    /// last cue, which is earlier than the end of the file whenever the
+    /// recording ends in music or silence.
+    #[arg(long, value_name = "SECONDS")]
+    duration: Option<f64>,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+
+    let content = match read(args.path.as_deref()) {
+        Ok(content) => content,
+        Err(e) => {
+            eprintln!("cueseg: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    let cues = match cues(&content) {
+        Ok(cues) => cues,
+        Err(e) => {
+            eprintln!("cueseg: {e}");
+            return ExitCode::from(exit::DATA);
+        }
+    };
+
+    let duration = match args.duration {
+        Some(duration) if duration.is_finite() && duration > 0.0 => duration,
+        Some(duration) => {
+            eprintln!("cueseg: --duration {duration} is not a length");
+            return ExitCode::from(exit::USAGE);
+        }
+        None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max),
+    };
+
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    for chapter in cueseg::segment(&cues, duration) {
+        if let Err(e) = ndjson::write_line(&mut out, &chapter) {
+            // A consumer that stopped reading is not a failure.
+            if e.is_broken_pipe() {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("cueseg: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+    match out.flush() {
+        Ok(()) => ExitCode::from(exit::OK),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("cueseg: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Read cues from either encoding.
+///
+/// A cue stream begins with a JSON object; a subtitle file does not. That is
+/// enough to tell them apart, and it saves a caller from having to say which
+/// of the two it has.
+fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
+    let first = content
+        .lines()
+        .find(|line| !line.trim().is_empty())
+        .unwrap_or("");
+    if first.trim_start().starts_with('{') {
+        return ndjson::read(content.as_bytes()).collect();
+    }
+    Ok(cuekit::parse(content))
+}
+
+fn read(path: Option<&Path>) -> std::io::Result<String> {
+    let bytes = match path {
+        Some(path) => std::fs::read(path)?,
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin().read_to_end(&mut bytes)?;
+            bytes
+        }
+    };
+    Ok(String::from_utf8_lossy(&bytes).into_owned())
+}

crates/cueseg/src/segment.rs

@@ -0,0 +1,353 @@
+//! Choosing where the chapters begin, and cutting them.
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::window::Window;
+use crate::{EXCERPT_CHARS, MAX_CHAPTERS, title};
+
+/// One topic chapter of a recording.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Segment {
+    /// Record version.
+    #[serde(default = "version")]
+    pub v: u32,
+    /// Seconds from the start of the recording.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub start: f64,
+    /// Seconds from the start of the recording.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub end: f64,
+    /// A title taken from the chapter's own first sentence.
+    ///
+    /// Extractive and unembarrassed about it. A model can write a better one
+    /// later; this is what makes the chapter usable before one has.
+    pub title: String,
+    /// The opening of the chapter, for a summariser or a reader.
+    pub excerpt: String,
+}
+
+fn version() -> u32 {
+    corpus_core::V
+}
+
+impl corpus_core::Record for Segment {
+    fn version(&self) -> u32 {
+        self.v
+    }
+}
+
+/// Where the chapters begin, in seconds.
+///
+/// A boundary is a valley: a pair of windows that resemble each other less
+/// than their neighbours do, and less than the recording's own average by
+/// half a standard deviation. Valleys are taken deepest first, each one at
+/// least a chapter's length from the ones already taken.
+pub fn boundary_times(
+    windows: &[Window],
+    similarities: &[f64],
+    duration: f64,
+    min_segment: f64,
+    target_count: Option<usize>,
+) -> Vec<f64> {
+    let target = target_count.unwrap_or_else(|| crate::target_chapter_count(duration));
+    if windows.len() < 2 {
+        return Vec::new();
+    }
+
+    let smoothed = moving_average(similarities, 1);
+    let average = if smoothed.is_empty() {
+        0.0
+    } else {
+        smoothed.iter().sum::<f64>() / smoothed.len() as f64
+    };
+    let deviation = if smoothed.len() < 2 {
+        0.0
+    } else {
+        let sum: f64 = smoothed.iter().map(|value| (value - average).powi(2)).sum();
+        (sum / smoothed.len() as f64).sqrt()
+    };
+    // A recording whose windows all resemble each other equally has no
+    // deviation to work from, so a fixed spread stands in for one.
+    let spread = if deviation == 0.0 { 0.15 } else { deviation };
+    let threshold = average - 0.5 * spread;
+
+    let mut valleys: Vec<(usize, f64)> = Vec::new();
+    for (at, value) in smoothed.iter().enumerate() {
+        let before = if at > 0 {
+            smoothed[at - 1]
+        } else {
+            value + 1.0
+        };
+        let after = if at < smoothed.len() - 1 {
+            smoothed[at + 1]
+        } else {
+            value + 1.0
+        };
+        if *value <= before && *value <= after && *value < threshold {
+            valleys.push((at, *value));
+        }
+    }
+    // Stable, so that two equally deep valleys are taken in the order they
+    // occur in the recording.
+    valleys.sort_by(|a, b| a.1.total_cmp(&b.1));
+
+    let needed = target.saturating_sub(1);
+    let time_of =
+        |gap: usize| -> f64 { windows.get(gap + 1).map_or(duration, |window| window.start) };
+
+    let mut chosen: Vec<usize> = Vec::new();
+    for (at, _) in valleys {
+        let time = time_of(at);
+        if time < min_segment || duration - time < min_segment {
+            continue;
+        }
+        if chosen
+            .iter()
+            .any(|taken| (time_of(*taken) - time).abs() < min_segment)
+        {
+            continue;
+        }
+        chosen.push(at);
+        if chosen.len() >= needed {
+            break;
+        }
+    }
+
+    // Nothing stood out, but the recording is long enough to want chapters
+    // anyway: fall back to even spacing, which is better than one chapter of
+    // three hours.
+    if chosen.is_empty() && duration >= min_segment * 2.0 {
+        let count = needed.max(1);
+        return (1..=count)
+            .map(|k| duration * k as f64 / (count + 1) as f64)
+            .filter(|time| *time >= min_segment && duration - *time >= min_segment)
+            .collect();
+    }
+
+    let mut times: Vec<f64> = chosen.iter().map(|at| time_of(*at)).collect();
+    times.sort_by(f64::total_cmp);
+
+    // A chapter that runs too long is no more navigable than no chapter at
+    // all, so an over-long span is divided evenly whether or not the text
+    // suggested a break there.
+    let longest = (min_segment * 2.5).max(duration / MAX_CHAPTERS as f64);
+    let mut ends = vec![0.0];
+    ends.extend(times.iter().copied());
+    ends.push(duration);
+
+    let mut extra = Vec::new();
+    for pair in ends.windows(2) {
+        let span = pair[1] - pair[0];
+        if span <= longest {
+            continue;
+        }
+        let parts = (span / longest).ceil() as usize;
+        for part in 1..parts {
+            extra.push(pair[0] + span * part as f64 / parts as f64);
+        }
+    }
+    times.extend(extra);
+    times.sort_by(f64::total_cmp);
+    times
+}
+
+/// Cut the windows into chapters at the given times.
+pub fn from_boundaries(windows: &[Window], boundaries: &[f64], duration: f64) -> Vec<Segment> {
+    if windows.is_empty() {
+        return Vec::new();
+    }
+    let mut cuts = vec![0.0];
+    cuts.extend(
+        boundaries
+            .iter()
+            .copied()
+            .filter(|time| *time > 0.0 && *time < duration),
+    );
+    cuts.push(duration);
+
+    // A cut within a second of the one before it would make a chapter nobody
+    // can click on. Compared against the unfiltered list, as the original
+    // does, so a run of near-identical cuts collapses to its first.
+    let unique: Vec<f64> = cuts
+        .iter()
+        .enumerate()
+        .filter(|(at, time)| *at == 0 || **time - cuts[at - 1] > 1.0)
+        .map(|(_, time)| *time)
+        .collect();
+
+    let mut segments = Vec::new();
+    for pair in unique.windows(2) {
+        let (start, end) = (pair[0], pair[1]);
+        let body: Vec<&str> = windows
+            .iter()
+            .filter(|window| window.start < end && window.end > start)
+            .map(|window| window.text.as_str())
+            .collect();
+        let excerpt = excerpt_of(&body.join(" "));
+        if excerpt.is_empty() {
+            continue;
+        }
+        segments.push(Segment {
+            v: corpus_core::V,
+            start,
+            end,
+            title: title::extract(&excerpt),
+            excerpt,
+        });
+    }
+    segments
+}
+
+/// Smooth a series by averaging each value with its neighbours, so that one
+/// unusual window does not read as a topic change.
+fn moving_average(values: &[f64], radius: usize) -> Vec<f64> {
+    (0..values.len())
+        .map(|at| {
+            let from = at.saturating_sub(radius);
+            let to = (at + radius + 1).min(values.len());
+            let slice = &values[from..to];
+            if slice.is_empty() {
+                0.0
+            } else {
+                slice.iter().sum::<f64>() / slice.len() as f64
+            }
+        })
+        .collect()
+}
+
+/// The opening of a chapter, cut at a word boundary where one is close
+/// enough to the limit to be worth using.
+fn excerpt_of(text: &str) -> String {
+    if text.chars().count() <= EXCERPT_CHARS {
+        return text.to_string();
+    }
+    let head: String = text.chars().take(EXCERPT_CHARS).collect();
+    match head.rfind(' ') {
+        Some(at) if head[..at].chars().count() > 200 => head[..at].trim().to_string(),
+        _ => head,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn window(start: f64, end: f64, text: &str) -> Window {
+        Window {
+            start,
+            end,
+            text: text.to_string(),
+        }
+    }
+
+    #[test]
+    fn a_single_window_has_no_boundaries() {
+        let windows = [window(0.0, 10.0, "one")];
+        assert!(boundary_times(&windows, &[], 10.0, 480.0, None).is_empty());
+    }
+
+    #[test]
+    fn a_boundary_lands_where_the_subject_changes() {
+        let windows: Vec<Window> = (0..40)
+            .map(|at| {
+                let start = at as f64 * 75.0;
+                window(start, start + 75.0, "text")
+            })
+            .collect();
+        // Similar throughout except one deep valley in the middle.
+        let mut similarities = vec![0.9; 39];
+        similarities[19] = 0.1;
+        let found = boundary_times(&windows, &similarities, 3000.0, 1200.0, Some(2));
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0], windows[20].start);
+    }
+
+    #[test]
+    fn a_chapter_that_would_run_too_long_is_divided_anyway() {
+        let windows: Vec<Window> = (0..40)
+            .map(|at| {
+                let start = at as f64 * 75.0;
+                window(start, start + 75.0, "text")
+            })
+            .collect();
+        let mut similarities = vec![0.9; 39];
+        similarities[19] = 0.1;
+        // The longest a chapter may run here is 1200 seconds, so each half of
+        // a 3000-second recording split once in the middle is still too long.
+        let found = boundary_times(&windows, &similarities, 3000.0, 480.0, Some(2));
+        assert_eq!(found.len(), 3, "{found:?}");
+        assert!(found.contains(&windows[20].start));
+        let mut ends = vec![0.0];
+        ends.extend(found.iter().copied());
+        ends.push(3000.0);
+        for pair in ends.windows(2) {
+            assert!(pair[1] - pair[0] <= 1200.0, "{:?} runs too long", pair);
+        }
+    }
+
+    #[test]
+    fn a_recording_with_no_valleys_is_cut_evenly() {
+        let windows: Vec<Window> = (0..40)
+            .map(|at| {
+                let start = at as f64 * 75.0;
+                window(start, start + 75.0, "text")
+            })
+            .collect();
+        let found = boundary_times(&windows, &vec![0.9; 39], 3000.0, 480.0, Some(3));
+        assert_eq!(found.len(), 2);
+        assert!(found[0] < found[1]);
+    }
+
+    #[test]
+    fn chapters_meet_end_to_end_and_cover_the_recording() {
+        let windows = [
+            window(
+                0.0,
+                1000.0,
+                "the mill stood at the northern end of the valley",
+            ),
+            window(
+                1000.0,
+                2000.0,
+                "parliament dissolved itself amid objections",
+            ),
+        ];
+        let found = from_boundaries(&windows, &[1000.0], 2000.0);
+        assert_eq!(found.len(), 2);
+        assert_eq!(found[0].start, 0.0);
+        assert_eq!(found[0].end, 1000.0);
+        assert_eq!(found[1].start, 1000.0);
+        assert_eq!(found[1].end, 2000.0);
+    }
+
+    #[test]
+    fn a_boundary_a_moment_after_another_is_dropped() {
+        let windows = [window(0.0, 2000.0, "text enough to excerpt")];
+        let found = from_boundaries(&windows, &[1000.0, 1000.5], 2000.0);
+        assert_eq!(found.len(), 2);
+    }
+
+    #[test]
+    fn a_boundary_outside_the_recording_is_ignored() {
+        let windows = [window(0.0, 2000.0, "text enough to excerpt")];
+        let found = from_boundaries(&windows, &[-5.0, 5000.0], 2000.0);
+        assert_eq!(found.len(), 1);
+    }
+
+    #[test]
+    fn an_excerpt_is_cut_at_a_word() {
+        let text = "word ".repeat(400);
+        let excerpt = excerpt_of(&text);
+        assert!(excerpt.chars().count() <= EXCERPT_CHARS);
+        assert!(!excerpt.ends_with(' '));
+        assert!(text.starts_with(&excerpt));
+    }
+
+    #[test]
+    fn a_short_chapter_keeps_all_its_text() {
+        assert_eq!(excerpt_of("short"), "short");
+    }
+}

crates/cueseg/src/title.rs

@@ -0,0 +1,108 @@
+//! A title taken from what was said, for a chapter nobody has named yet.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// Hesitations. A chapter called "Um Yeah So I Think" is worse than no title.
+static FILLER: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)\b(um+|uh+|ah+|er+)\b").expect("a valid pattern"));
+
+/// The most common discourse marker in recorded speech.
+static YOU_KNOW: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)\byou know\b").expect("a valid pattern"));
+
+static SPACE_RUN: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+
+/// Anything that does not belong in a title.
+static NOT_TITLE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9'-]").expect("a valid pattern"));
+
+/// How many words a title gets.
+const WORDS: usize = 8;
+
+/// Take a title from the first sentence of a chapter.
+///
+/// Extractive, and no cleverer than it looks: the first thing said in a
+/// chapter is usually what the chapter is about, and where it is not, a
+/// reader can see that at a glance and a model can replace it later.
+pub fn extract(excerpt: &str) -> String {
+    let cleaned = FILLER.replace_all(excerpt, " ");
+    let cleaned = YOU_KNOW.replace_all(&cleaned, " ");
+    let cleaned = SPACE_RUN.replace_all(&cleaned, " ");
+    let cleaned = cleaned.trim_matches(textkit::is_space);
+
+    let sentence = textkit::sentences(cleaned).first().copied().unwrap_or("");
+    let title: Vec<String> = sentence
+        .split(textkit::is_space)
+        .filter(|word| !word.is_empty())
+        .take(WORDS)
+        .map(|word| NOT_TITLE.replace_all(word, "").into_owned())
+        .filter(|word| !word.is_empty())
+        .map(|word| capitalise(&word))
+        .collect();
+
+    if title.is_empty() {
+        "Topic".to_string()
+    } else {
+        title.join(" ")
+    }
+}
+
+/// First letter up, the rest down — so that a transcript's shouting and its
+/// mid-sentence capitals come out looking like a title either way.
+fn capitalise(word: &str) -> String {
+    let mut chars = word.chars();
+    match chars.next() {
+        None => String::new(),
+        Some(first) => first.to_uppercase().collect::<String>() + &chars.as_str().to_lowercase(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_first_sentence_becomes_the_title() {
+        assert_eq!(
+            extract("The mill stood at the northern end. Its wheel was undershot."),
+            "The Mill Stood At The Northern End"
+        );
+    }
+
+    #[test]
+    fn a_title_is_at_most_eight_words() {
+        let title = extract("one two three four five six seven eight nine ten.");
+        assert_eq!(title.split(' ').count(), 8);
+    }
+
+    #[test]
+    fn fillers_and_discourse_markers_go() {
+        assert_eq!(
+            extract("Um, so, you know, uh the mill stood there."),
+            "So The Mill Stood There"
+        );
+    }
+
+    #[test]
+    fn punctuation_inside_a_word_is_kept_and_around_it_is_not() {
+        assert_eq!(
+            extract("(the) mill's well-built wheel."),
+            "The Mill's Well-built Wheel"
+        );
+    }
+
+    #[test]
+    fn shouting_comes_out_as_a_title() {
+        assert_eq!(extract("THE MILL STOOD."), "The Mill Stood");
+    }
+
+    #[test]
+    fn nothing_usable_still_gives_a_title() {
+        assert_eq!(extract(""), "Topic");
+        assert_eq!(extract("... !!! ???"), "Topic");
+        assert_eq!(extract("um uh ah"), "Topic");
+    }
+}

crates/cueseg/src/vector.rs

@@ -0,0 +1,175 @@
+//! Turning a window of speech into something two windows can be compared by.
+//!
+//! Hashed bag of words, not embeddings. The question here is only "are these
+//! two minutes about the same thing as the last two", and for that a count of
+//! which content words appear is enough — and it costs no model, no GPU and
+//! no network, which is what lets this run over an archive rather than over a
+//! few files.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// How many buckets a window's words are hashed into.
+///
+/// Small enough that two windows about the same subject collide on the same
+/// buckets, large enough that two windows about different subjects mostly do
+/// not.
+pub const DIMENSIONS: usize = 256;
+
+/// Everything that is not a letter, a digit, an apostrophe or whitespace.
+static NOT_A_WORD: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[^a-z0-9'\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+
+/// Words too common in speech to say anything about the subject.
+///
+/// Longer than an ordinary stop list because this is transcribed speech: the
+/// fillers, the discourse markers and the vague nouns that carry a
+/// conversation carry no topic at all, and leaving them in makes every window
+/// resemble every other.
+const STOP: &[&str] = &[
+    "about", "after", "ah", "all", "also", "and", "are", "back", "because", "been", "before",
+    "but", "can", "come", "could", "did", "don", "even", "for", "from", "get", "going", "gonna",
+    "got", "had", "has", "have", "here", "how", "into", "it's", "its", "just", "kind", "know",
+    "let", "lets", "like", "little", "look", "lot", "mean", "much", "need", "not", "now", "oh",
+    "ok", "okay", "one", "out", "over", "people", "really", "right", "said", "say", "says", "see",
+    "should", "so", "sort", "still", "than", "that", "the", "their", "them", "then", "there",
+    "they", "thing", "things", "think", "this", "uh", "um", "very", "want", "was", "well", "were",
+    "what", "when", "who", "why", "will", "with", "would", "yeah", "yes", "you", "your",
+];
+
+/// The content words of a text.
+pub fn tokens(text: &str) -> Vec<String> {
+    let lowered = text.to_lowercase();
+    let cleaned = NOT_A_WORD.replace_all(&lowered, " ");
+    cleaned
+        .split(textkit::is_space)
+        .filter(|word| !word.is_empty())
+        .filter(|word| word.chars().count() > 2 && !is_stop(word))
+        .map(str::to_string)
+        .collect()
+}
+
+fn is_stop(word: &str) -> bool {
+    STOP.binary_search(&word).is_ok()
+}
+
+/// Hash a text's content words into a vector of counts.
+pub fn lexical(text: &str, dimensions: usize) -> Vec<f64> {
+    let mut vector = vec![0.0; dimensions];
+    for token in tokens(text) {
+        let at = fnv1a(&token) as usize % dimensions;
+        vector[at] += 1.0;
+    }
+    vector
+}
+
+/// FNV-1a over the characters of a word.
+///
+/// Over the characters rather than the bytes, which is what the original
+/// does. It makes no difference here — the tokeniser above leaves only ASCII
+/// — but the two disagree for anything else, and a hash that disagrees puts
+/// a word in a different bucket and moves a boundary.
+fn fnv1a(word: &str) -> u32 {
+    let mut hash: u32 = 2166136261;
+    for c in word.chars() {
+        hash ^= c as u32;
+        hash = hash.wrapping_mul(16777619);
+    }
+    hash
+}
+
+/// Cosine similarity, or zero when either side has no words at all.
+pub fn cosine(a: &[f64], b: &[f64]) -> f64 {
+    let n = a.len().min(b.len());
+    let mut dot = 0.0;
+    let mut norm_a = 0.0;
+    let mut norm_b = 0.0;
+    for at in 0..n {
+        dot += a[at] * b[at];
+        norm_a += a[at] * a[at];
+        norm_b += b[at] * b[at];
+    }
+    if norm_a == 0.0 || norm_b == 0.0 {
+        return 0.0;
+    }
+    dot / (norm_a.sqrt() * norm_b.sqrt())
+}
+
+/// How much each window resembles the one after it.
+pub fn consecutive_cosine(vectors: &[Vec<f64>]) -> Vec<f64> {
+    vectors
+        .windows(2)
+        .map(|pair| cosine(&pair[0], &pair[1]))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_stop_list_is_sorted_so_it_can_be_searched() {
+        assert!(STOP.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
+    #[test]
+    fn tokens_are_content_words_only() {
+        assert_eq!(
+            tokens("So, you know, the MILL stood at the northern end!"),
+            ["mill", "stood", "northern", "end"]
+        );
+    }
+
+    #[test]
+    fn short_words_and_punctuation_go() {
+        assert_eq!(tokens("a bc def — ghi's"), ["def", "ghi's"]);
+    }
+
+    #[test]
+    fn anything_that_is_not_a_latin_letter_becomes_a_break() {
+        assert_eq!(tokens("café Москва mill"), ["caf", "mill"]);
+    }
+
+    #[test]
+    fn the_hash_is_the_one_the_original_uses() {
+        // FNV-1a over the characters, 32-bit.
+        assert_eq!(fnv1a(""), 2166136261);
+        assert_eq!(fnv1a("a"), 0xe40c292c);
+        assert_eq!(fnv1a("mill"), fnv1a("mill"));
+        assert_ne!(fnv1a("mill"), fnv1a("kiln"));
+    }
+
+    #[test]
+    fn the_same_text_is_perfectly_similar_to_itself() {
+        let vector = lexical("the mill stood at the northern end", DIMENSIONS);
+        assert!((cosine(&vector, &vector) - 1.0).abs() < 1e-12);
+    }
+
+    #[test]
+    fn different_subjects_are_not_similar() {
+        let a = lexical(
+            "the mill stood at the northern end of the valley",
+            DIMENSIONS,
+        );
+        let b = lexical(
+            "parliament dissolved itself amid furious objections",
+            DIMENSIONS,
+        );
+        assert!(cosine(&a, &b) < 0.2, "{}", cosine(&a, &b));
+    }
+
+    #[test]
+    fn an_empty_window_resembles_nothing() {
+        let empty = lexical("", DIMENSIONS);
+        let full = lexical("the mill stood", DIMENSIONS);
+        assert_eq!(cosine(&empty, &full), 0.0);
+        assert_eq!(cosine(&empty, &empty), 0.0);
+    }
+
+    #[test]
+    fn one_window_has_no_similarities_to_report() {
+        assert!(consecutive_cosine(&[vec![1.0]]).is_empty());
+        assert!(consecutive_cosine(&[]).is_empty());
+    }
+}

crates/cueseg/src/window.rs

@@ -0,0 +1,122 @@
+//! Gathering cues into windows long enough to have a subject.
+
+use std::sync::LazyLock;
+
+use cuekit::Cue;
+use regex::Regex;
+
+/// Runs of whitespace, collapsed so that a window's text is one paragraph.
+static SPACE_RUN: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+
+/// A stretch of transcript treated as one unit of subject matter.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Window {
+    /// When the window starts.
+    pub start: f64,
+    /// When it ends.
+    pub end: f64,
+    /// Everything said in it, on one line.
+    pub text: String,
+}
+
+/// Gather cues into windows of about `seconds` each.
+///
+/// A window closes when a cue starts more than `seconds` after the window
+/// began — not when it would overrun, so a window is always at least as long
+/// as the budget and never cuts a cue in half.
+pub fn windows(cues: &[Cue], seconds: f64) -> Vec<Window> {
+    let Some(first) = cues.first() else {
+        return Vec::new();
+    };
+
+    let mut windows = Vec::new();
+    let mut start = first.start;
+    let mut end = first.end;
+    let mut parts: Vec<&str> = Vec::new();
+
+    for cue in cues {
+        if !parts.is_empty() && cue.start - start >= seconds {
+            flush(&mut windows, start, end, &parts);
+            parts.clear();
+        }
+        if parts.is_empty() {
+            start = cue.start;
+        }
+        parts.push(&cue.text);
+        end = end.max(cue.end);
+    }
+    flush(&mut windows, start, end, &parts);
+    windows
+}
+
+fn flush(windows: &mut Vec<Window>, start: f64, end: f64, parts: &[&str]) {
+    let joined = parts.join(" ");
+    let text = SPACE_RUN.replace_all(&joined, " ");
+    let text = text.trim_matches(textkit::is_space);
+    if !text.is_empty() {
+        windows.push(Window {
+            start,
+            end,
+            text: text.to_string(),
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn cue(start: f64, end: f64, text: &str) -> Cue {
+        Cue::new(start, end, text)
+    }
+
+    #[test]
+    fn cues_within_the_budget_make_one_window() {
+        let cues = [cue(0.0, 5.0, "one"), cue(5.0, 10.0, "two")];
+        let found = windows(&cues, 75.0);
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0].text, "one two");
+        assert_eq!(found[0].start, 0.0);
+        assert_eq!(found[0].end, 10.0);
+    }
+
+    #[test]
+    fn a_window_closes_when_a_cue_starts_past_the_budget() {
+        let cues = [
+            cue(0.0, 5.0, "one"),
+            cue(74.0, 76.0, "two"),
+            cue(80.0, 85.0, "three"),
+        ];
+        let found = windows(&cues, 75.0);
+        assert_eq!(found.len(), 2);
+        assert_eq!(found[0].text, "one two");
+        assert_eq!(found[1].text, "three");
+        assert_eq!(found[1].start, 80.0);
+    }
+
+    #[test]
+    fn an_end_time_that_goes_backwards_does_not_shorten_a_window() {
+        let cues = [cue(0.0, 90.0, "one"), cue(10.0, 20.0, "two")];
+        assert_eq!(windows(&cues, 75.0)[0].end, 90.0);
+    }
+
+    #[test]
+    fn whitespace_inside_a_window_is_collapsed() {
+        let cues = [cue(0.0, 1.0, "one\n\n  two"), cue(1.0, 2.0, "  three ")];
+        assert_eq!(windows(&cues, 75.0)[0].text, "one two three");
+    }
+
+    #[test]
+    fn a_window_of_nothing_but_whitespace_is_dropped() {
+        let cues = [cue(0.0, 1.0, "   "), cue(100.0, 101.0, "kept")];
+        let found = windows(&cues, 75.0);
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0].text, "kept");
+    }
+
+    #[test]
+    fn nothing_in_nothing_out() {
+        assert!(windows(&[], 75.0).is_empty());
+    }
+}

parity/Cargo.toml

@@ -16,6 +16,7 @@ python = []
 [dependencies]
 chunkmd.workspace = true
 cuekit.workspace = true
+cueseg.workspace = true
 mdscrub.workspace = true
 pdftriage.workspace = true
 corpus-core.workspace = true

parity/README.md

@@ -24,7 +24,7 @@ implementation that has moved on:
 
 ```sh
 CATENA_PARITY_PYTHON=/path/to/venv/bin/python \
-CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src:/path/to/podcast-pipeline/src \
     cargo test -p parity --features python
 ```
 
@@ -39,7 +39,7 @@ read the diff, and commit it with the reason:
 
 ```sh
 CATENA_PARITY_PYTHON=/path/to/venv/bin/python \
-CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
+CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src:/path/to/podcast-pipeline/src \
     cargo run -p parity --bin regen
 ```
 

parity/SOURCE.md

@@ -7,6 +7,7 @@ in this directory are the answers that Python gave, at one commit:
 |---|---|
 | `book-pipeline` | `7499e3257572968b437ecfdc79feccc91b96585b` |
 | `podcast-download-systemd` | `ab20139cd0defb400a0dd9ed9034e03c1d1e356d` |
+| `podcast-pipeline` | `4566302444a6aa3609bd0d6685323619bd33c72b` |
 
 Regenerating against a different commit will move the goldens. That is the
 point — the diff is the review. Record the new commit here in the same change.
@@ -16,7 +17,7 @@ a checkout lives is a fact about one machine and must not be compiled in:
 
 ```sh
 CATENA_PARITY_PYTHON=/path/to/venv/bin/python
-CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src
+CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src:/path/to/podcast-pipeline/src
 CATENA_PARITY_NODE=node   # only if it is not already on PATH
 ```
 

parity/cases/cueseg/durations.json

@@ -0,0 +1,30 @@
+{
+  "gappy": {
+    "tail": 8995,
+    "tight": 7196
+  },
+  "many-topics": {
+    "tail": 22520,
+    "tight": 18016
+  },
+  "one-cue": {
+    "tail": 5,
+    "tight": 4
+  },
+  "short": {
+    "tail": 750,
+    "tight": 600
+  },
+  "sparse-text": {
+    "tail": 4505,
+    "tight": 3604
+  },
+  "two-topics": {
+    "tail": 9000,
+    "tight": 7200
+  },
+  "uniform": {
+    "tail": 6750,
+    "tight": 5400
+  }
+}

parity/cases/cueseg/one-cue.ndjson

@@ -0,0 +1 @@
+{"start": 0.0, "end": 4.0, "text": "the mill stood at the northern end"}

parity/cases/cueseg/short.ndjson

@@ -0,0 +1,150 @@
+{"start": 0.0, "end": 4.0, "text": "the mill stood at the northern end of"}
+{"start": 4.0, "end": 8.0, "text": "the valley where the fall of the river was"}
+{"start": 8.0, "end": 12.0, "text": "steepest and the ice came latest its wheel was undershot"}
+{"start": 12.0, "end": 16.0, "text": "which was wasteful and nobody replaced it for two hundred years"}
+{"start": 16.0, "end": 20.0, "text": "because the cost fell on one household and the benefit was spread"}
+{"start": 20.0, "end": 24.0, "text": "across four hundred families the mill stood at"}
+{"start": 24.0, "end": 28.0, "text": "the northern end of the valley where the fall"}
+{"start": 28.0, "end": 32.0, "text": "of the river was steepest and the ice came latest"}
+{"start": 32.0, "end": 36.0, "text": "its wheel was undershot which was wasteful and nobody replaced it"}
+{"start": 36.0, "end": 40.0, "text": "for two hundred years because the cost fell on one household and"}
+{"start": 40.0, "end": 44.0, "text": "the benefit was spread across four hundred families"}
+{"start": 44.0, "end": 48.0, "text": "the mill stood at the northern end of the"}
+{"start": 48.0, "end": 52.0, "text": "valley where the fall of the river was steepest and"}
+{"start": 52.0, "end": 56.0, "text": "the ice came latest its wheel was undershot which was wasteful"}
+{"start": 56.0, "end": 60.0, "text": "and nobody replaced it for two hundred years because the cost fell"}
+{"start": 60.0, "end": 64.0, "text": "on one household and the benefit was spread"}
+{"start": 64.0, "end": 68.0, "text": "across four hundred families the mill stood at the"}
+{"start": 68.0, "end": 72.0, "text": "northern end of the valley where the fall of the"}
+{"start": 72.0, "end": 76.0, "text": "river was steepest and the ice came latest its wheel was"}
+{"start": 76.0, "end": 80.0, "text": "undershot which was wasteful and nobody replaced it for two hundred years"}
+{"start": 80.0, "end": 84.0, "text": "because the cost fell on one household and"}
+{"start": 84.0, "end": 88.0, "text": "the benefit was spread across four hundred families the"}
+{"start": 88.0, "end": 92.0, "text": "mill stood at the northern end of the valley where"}
+{"start": 92.0, "end": 96.0, "text": "the fall of the river was steepest and the ice came"}
+{"start": 96.0, "end": 100.0, "text": "latest its wheel was undershot which was wasteful and nobody replaced it"}
+{"start": 100.0, "end": 104.0, "text": "for two hundred years because the cost fell"}
+{"start": 104.0, "end": 108.0, "text": "on one household and the benefit was spread across"}
+{"start": 108.0, "end": 112.0, "text": "four hundred families the mill stood at the northern end"}
+{"start": 112.0, "end": 116.0, "text": "of the valley where the fall of the river was steepest"}
+{"start": 116.0, "end": 120.0, "text": "and the ice came latest its wheel was undershot which was wasteful"}
+{"start": 120.0, "end": 124.0, "text": "and nobody replaced it for two hundred years"}
+{"start": 124.0, "end": 128.0, "text": "because the cost fell on one household and the"}
+{"start": 128.0, "end": 132.0, "text": "benefit was spread across four hundred families the mill stood"}
+{"start": 132.0, "end": 136.0, "text": "at the northern end of the valley where the fall of"}
+{"start": 136.0, "end": 140.0, "text": "the river was steepest and the ice came latest its wheel was"}
+{"start": 140.0, "end": 144.0, "text": "undershot which was wasteful and nobody replaced it"}
+{"start": 144.0, "end": 148.0, "text": "for two hundred years because the cost fell on"}
+{"start": 148.0, "end": 152.0, "text": "one household and the benefit was spread across four hundred"}
+{"start": 152.0, "end": 156.0, "text": "families the mill stood at the northern end of the valley"}
+{"start": 156.0, "end": 160.0, "text": "where the fall of the river was steepest and the ice came"}
+{"start": 160.0, "end": 164.0, "text": "latest its wheel was undershot which was wasteful"}
+{"start": 164.0, "end": 168.0, "text": "and nobody replaced it for two hundred years because"}
+{"start": 168.0, "end": 172.0, "text": "the cost fell on one household and the benefit was"}
+{"start": 172.0, "end": 176.0, "text": "spread across four hundred families the mill stood at the northern"}
+{"start": 176.0, "end": 180.0, "text": "end of the valley where the fall of the river was steepest"}
+{"start": 180.0, "end": 184.0, "text": "and the ice came latest its wheel was"}
+{"start": 184.0, "end": 188.0, "text": "undershot which was wasteful and nobody replaced it for"}
+{"start": 188.0, "end": 192.0, "text": "two hundred years because the cost fell on one household"}
+{"start": 192.0, "end": 196.0, "text": "and the benefit was spread across four hundred families the mill"}
+{"start": 196.0, "end": 200.0, "text": "stood at the northern end of the valley where the fall of"}
+{"start": 200.0, "end": 204.0, "text": "the river was steepest and the ice came"}
+{"start": 204.0, "end": 208.0, "text": "latest its wheel was undershot which was wasteful and"}
+{"start": 208.0, "end": 212.0, "text": "nobody replaced it for two hundred years because the cost"}
+{"start": 212.0, "end": 216.0, "text": "fell on one household and the benefit was spread across four"}
+{"start": 216.0, "end": 220.0, "text": "hundred families the mill stood at the northern end of the valley"}
+{"start": 220.0, "end": 224.0, "text": "where the fall of the river was steepest"}
+{"start": 224.0, "end": 228.0, "text": "and the ice came latest its wheel was undershot"}
+{"start": 228.0, "end": 232.0, "text": "which was wasteful and nobody replaced it for two hundred"}
+{"start": 232.0, "end": 236.0, "text": "years because the cost fell on one household and the benefit"}
+{"start": 236.0, "end": 240.0, "text": "was spread across four hundred families the mill stood at the northern"}
+{"start": 240.0, "end": 244.0, "text": "end of the valley where the fall of"}
+{"start": 244.0, "end": 248.0, "text": "the river was steepest and the ice came latest"}
+{"start": 248.0, "end": 252.0, "text": "its wheel was undershot which was wasteful and nobody replaced"}
+{"start": 252.0, "end": 256.0, "text": "it for two hundred years because the cost fell on one"}
+{"start": 256.0, "end": 260.0, "text": "household and the benefit was spread across four hundred families the mill"}
+{"start": 260.0, "end": 264.0, "text": "stood at the northern end of the valley"}
+{"start": 264.0, "end": 268.0, "text": "where the fall of the river was steepest and"}
+{"start": 268.0, "end": 272.0, "text": "the ice came latest its wheel was undershot which was"}
+{"start": 272.0, "end": 276.0, "text": "wasteful and nobody replaced it for two hundred years because the"}
+{"start": 276.0, "end": 280.0, "text": "cost fell on one household and the benefit was spread across four"}
+{"start": 280.0, "end": 284.0, "text": "hundred families the mill stood at the northern"}
+{"start": 284.0, "end": 288.0, "text": "end of the valley where the fall of the"}
+{"start": 288.0, "end": 292.0, "text": "river was steepest and the ice came latest its wheel"}
+{"start": 292.0, "end": 296.0, "text": "was undershot which was wasteful and nobody replaced it for two"}
+{"start": 296.0, "end": 300.0, "text": "hundred years because the cost fell on one household and the benefit"}
+{"start": 300.0, "end": 304.0, "text": "was spread across four hundred families the mill"}
+{"start": 304.0, "end": 308.0, "text": "stood at the northern end of the valley where"}
+{"start": 308.0, "end": 312.0, "text": "the fall of the river was steepest and the ice"}
+{"start": 312.0, "end": 316.0, "text": "came latest its wheel was undershot which was wasteful and nobody"}
+{"start": 316.0, "end": 320.0, "text": "replaced it for two hundred years because the cost fell on one"}
+{"start": 320.0, "end": 324.0, "text": "household and the benefit was spread across four"}
+{"start": 324.0, "end": 328.0, "text": "hundred families the mill stood at the northern end"}
+{"start": 328.0, "end": 332.0, "text": "of the valley where the fall of the river was"}
+{"start": 332.0, "end": 336.0, "text": "steepest and the ice came latest its wheel was undershot which"}
+{"start": 336.0, "end": 340.0, "text": "was wasteful and nobody replaced it for two hundred years because the"}
+{"start": 340.0, "end": 344.0, "text": "cost fell on one household and the benefit"}
+{"start": 344.0, "end": 348.0, "text": "was spread across four hundred families the mill stood"}
+{"start": 348.0, "end": 352.0, "text": "at the northern end of the valley where the fall"}
+{"start": 352.0, "end": 356.0, "text": "of the river was steepest and the ice came latest its"}
+{"start": 356.0, "end": 360.0, "text": "wheel was undershot which was wasteful and nobody replaced it for two"}
+{"start": 360.0, "end": 364.0, "text": "hundred years because the cost fell on one"}
+{"start": 364.0, "end": 368.0, "text": "household and the benefit was spread across four hundred"}
+{"start": 368.0, "end": 372.0, "text": "families the mill stood at the northern end of the"}
+{"start": 372.0, "end": 376.0, "text": "valley where the fall of the river was steepest and the"}
+{"start": 376.0, "end": 380.0, "text": "ice came latest its wheel was undershot which was wasteful and nobody"}
+{"start": 380.0, "end": 384.0, "text": "replaced it for two hundred years because the"}
+{"start": 384.0, "end": 388.0, "text": "cost fell on one household and the benefit was"}
+{"start": 388.0, "end": 392.0, "text": "spread across four hundred families the mill stood at the"}
+{"start": 392.0, "end": 396.0, "text": "northern end of the valley where the fall of the river"}
+{"start": 396.0, "end": 400.0, "text": "was steepest and the ice came latest its wheel was undershot which"}
+{"start": 400.0, "end": 404.0, "text": "was wasteful and nobody replaced it for two"}
+{"start": 404.0, "end": 408.0, "text": "hundred years because the cost fell on one household"}
+{"start": 408.0, "end": 412.0, "text": "and the benefit was spread across four hundred families the"}
+{"start": 412.0, "end": 416.0, "text": "mill stood at the northern end of the valley where the"}
+{"start": 416.0, "end": 420.0, "text": "fall of the river was steepest and the ice came latest its"}
+{"start": 420.0, "end": 424.0, "text": "wheel was undershot which was wasteful and nobody"}
+{"start": 424.0, "end": 428.0, "text": "replaced it for two hundred years because the cost"}
+{"start": 428.0, "end": 432.0, "text": "fell on one household and the benefit was spread across"}
+{"start": 432.0, "end": 436.0, "text": "four hundred families the mill stood at the northern end of"}
+{"start": 436.0, "end": 440.0, "text": "the valley where the fall of the river was steepest and the"}
+{"start": 440.0, "end": 444.0, "text": "ice came latest its wheel was undershot which"}
+{"start": 444.0, "end": 448.0, "text": "was wasteful and nobody replaced it for two hundred"}
+{"start": 448.0, "end": 452.0, "text": "years because the cost fell on one household and the"}
+{"start": 452.0, "end": 456.0, "text": "benefit was spread across four hundred families the mill stood at"}
+{"start": 456.0, "end": 460.0, "text": "the northern end of the valley where the fall of the river"}
+{"start": 460.0, "end": 464.0, "text": "was steepest and the ice came latest its"}
+{"start": 464.0, "end": 468.0, "text": "wheel was undershot which was wasteful and nobody replaced"}
+{"start": 468.0, "end": 472.0, "text": "it for two hundred years because the cost fell on"}
+{"start": 472.0, "end": 476.0, "text": "one household and the benefit was spread across four hundred families"}
+{"start": 476.0, "end": 480.0, "text": "the mill stood at the northern end of the valley where the"}
+{"start": 480.0, "end": 484.0, "text": "fall of the river was steepest and the"}
+{"start": 484.0, "end": 488.0, "text": "ice came latest its wheel was undershot which was"}
+{"start": 488.0, "end": 492.0, "text": "wasteful and nobody replaced it for two hundred years because"}
+{"start": 492.0, "end": 496.0, "text": "the cost fell on one household and the benefit was spread"}
+{"start": 496.0, "end": 500.0, "text": "across four hundred families the mill stood at the northern end of"}
+{"start": 500.0, "end": 504.0, "text": "the valley where the fall of the river"}
+{"start": 504.0, "end": 508.0, "text": "was steepest and the ice came latest its wheel"}
+{"start": 508.0, "end": 512.0, "text": "was undershot which was wasteful and nobody replaced it for"}
+{"start": 512.0, "end": 516.0, "text": "two hundred years because the cost fell on one household and"}
+{"start": 516.0, "end": 520.0, "text": "the benefit was spread across four hundred families the mill stood at"}
+{"start": 520.0, "end": 524.0, "text": "the northern end of the valley where the"}
+{"start": 524.0, "end": 528.0, "text": "fall of the river was steepest and the ice"}
+{"start": 528.0, "end": 532.0, "text": "came latest its wheel was undershot which was wasteful and"}
+{"start": 532.0, "end": 536.0, "text": "nobody replaced it for two hundred years because the cost fell"}
+{"start": 536.0, "end": 540.0, "text": "on one household and the benefit was spread across four hundred families"}
+{"start": 540.0, "end": 544.0, "text": "the mill stood at the northern end of"}
+{"start": 544.0, "end": 548.0, "text": "the valley where the fall of the river was"}
+{"start": 548.0, "end": 552.0, "text": "steepest and the ice came latest its wheel was undershot"}
+{"start": 552.0, "end": 556.0, "text": "which was wasteful and nobody replaced it for two hundred years"}
+{"start": 556.0, "end": 560.0, "text": "because the cost fell on one household and the benefit was spread"}
+{"start": 560.0, "end": 564.0, "text": "across four hundred families the mill stood at"}
+{"start": 564.0, "end": 568.0, "text": "the northern end of the valley where the fall"}
+{"start": 568.0, "end": 572.0, "text": "of the river was steepest and the ice came latest"}
+{"start": 572.0, "end": 576.0, "text": "its wheel was undershot which was wasteful and nobody replaced it"}
+{"start": 576.0, "end": 580.0, "text": "for two hundred years because the cost fell on one household and"}
+{"start": 580.0, "end": 584.0, "text": "the benefit was spread across four hundred families"}
+{"start": 584.0, "end": 588.0, "text": "the mill stood at the northern end of the"}
+{"start": 588.0, "end": 592.0, "text": "valley where the fall of the river was steepest and"}
+{"start": 592.0, "end": 596.0, "text": "the ice came latest its wheel was undershot which was wasteful"}
+{"start": 596.0, "end": 600.0, "text": "and nobody replaced it for two hundred years because the cost fell"}

parity/cases/cueseg/sparse-text.ndjson

@@ -0,0 +1,4 @@
+{"start": 0.0, "end": 4.0, "text": "   "}
+{"start": 4.0, "end": 8.0, "text": "the mill stood"}
+{"start": 8.0, "end": 12.0, "text": ""}
+{"start": 3600.0, "end": 3604.0, "text": "parliament dissolved itself again"}

parity/golden/cueseg/gappy.tail.ndjson

@@ -0,0 +1,8 @@
+{"start": 0.0, "end": 1194.0, "title": "The Valley Where The Fall Of The River", "excerpt": "the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and"}
+{"start": 1194.0, "end": 2388.0, "title": "Undershot Which Was Wasteful And Nobody Replaced It", "excerpt": "undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit"}
+{"start": 2388.0, "end": 3209.3333333333335, "title": "Which Was Wasteful And Nobody Replaced It For", "excerpt": "which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious"}
+{"start": 3209.3333333333335, "end": 4030.666666666667, "title": "Hear The Motion And The Clerks Recorded Nothing", "excerpt": "hear the motion and the clerks recorded nothing of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused"}
+{"start": 4030.666666666667, "end": 4852.0, "title": "The Motion And The Clerks Recorded Nothing At", "excerpt": "the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved"}
+{"start": 4852.0, "end": 5887.75, "title": "Of Wheat Trebled Before The First Ship Arrived", "excerpt": "of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports"}
+{"start": 5887.75, "end": 6923.5, "title": "Bailiff Recorded Every Bushel And The Tallies Survive", "excerpt": "bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the"}
+{"start": 6923.5, "end": 7959.25, "title": "And The Tallies Survive In The Manor Rolls", "excerpt": "and the tallies survive in the manor rolls almost complete the harvest of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying manor rolls almost complete the harvest failed in three successive years and the price of wheat"}

parity/golden/cueseg/gappy.tight.ndjson

@@ -0,0 +1,7 @@
+{"start": 0.0, "end": 1194.0, "title": "The Valley Where The Fall Of The River", "excerpt": "the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and"}
+{"start": 1194.0, "end": 2388.0, "title": "Undershot Which Was Wasteful And Nobody Replaced It", "excerpt": "undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit"}
+{"start": 2388.0, "end": 3209.3333333333335, "title": "Which Was Wasteful And Nobody Replaced It For", "excerpt": "which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious"}
+{"start": 3209.3333333333335, "end": 4030.666666666667, "title": "Hear The Motion And The Clerks Recorded Nothing", "excerpt": "hear the motion and the clerks recorded nothing of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused"}
+{"start": 4030.666666666667, "end": 4852.0, "title": "The Motion And The Clerks Recorded Nothing At", "excerpt": "the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved"}
+{"start": 4852.0, "end": 6024.0, "title": "Of Wheat Trebled Before The First Ship Arrived", "excerpt": "of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports"}
+{"start": 6024.0, "end": 7196.0, "title": "The First Ship Arrived From The Baltic Ports", "excerpt": "the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the"}

parity/golden/cueseg/many-topics.tail.ndjson

@@ -0,0 +1,17 @@
+{"start": 0.0, "end": 1102.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 1102.0, "end": 2204.0, "title": "Fall Of The River Was Steepest And The", "excerpt": "fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of"}
+{"start": 2204.0, "end": 3344.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the"}
+{"start": 3344.0, "end": 4484.0, "title": "And The Clerks Recorded Nothing At All Which", "excerpt": "and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and"}
+{"start": 4484.0, "end": 5624.0, "title": "The Motion And The Clerks Recorded Nothing At", "excerpt": "the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying"}
+{"start": 5624.0, "end": 6764.0, "title": "Tallies Survive In The Manor Rolls Almost Complete", "excerpt": "tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying"}
+{"start": 6764.0, "end": 7866.0, "title": "In A Hand Nobody Has Yet Transcribed The", "excerpt": "in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed"}
+{"start": 7866.0, "end": 8968.0, "title": "The Archdeacon Whose Court Kept Minutes In A", "excerpt": "the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of"}
+{"start": 8968.0, "end": 10146.0, "title": "In A Compromise Neither Side Recorded The Abbey", "excerpt": "in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the"}
+{"start": 10146.0, "end": 11324.0, "title": "The Justices Travelled A Circuit Of Eleven Counties", "excerpt": "the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor"}
+{"start": 11324.0, "end": 12388.0, "title": "Which Silver The Exchequer Issued A Proclamation That", "excerpt": "which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a"}
+{"start": 12388.0, "end": 13452.0, "title": "About Which Silver The Exchequer Issued A Proclamation", "excerpt": "about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a"}
+{"start": 13452.0, "end": 14592.0, "title": "About Which Silver The Exchequer Issued A Proclamation", "excerpt": "about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a"}
+{"start": 14592.0, "end": 15732.0, "title": "A Year And The Surveyor Of Highways Was", "excerpt": "a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and"}
+{"start": 15732.0, "end": 16872.0, "title": "Nobody Wanted And Everyone In Turn Was Obliged", "excerpt": "nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town"}
+{"start": 16872.0, "end": 18012.0, "title": "Town Closed Its Gates Against The Countryside Which", "excerpt": "town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside"}
+{"start": 18012.0, "end": 19139.0, "title": "Closed Its Lanes Against The Town The Death", "excerpt": "closed its lanes against the town the death roll was kept"}

parity/golden/cueseg/many-topics.tight.ndjson

@@ -0,0 +1,16 @@
+{"start": 0.0, "end": 1102.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 1102.0, "end": 2204.0, "title": "Fall Of The River Was Steepest And The", "excerpt": "fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of"}
+{"start": 2204.0, "end": 3344.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the"}
+{"start": 3344.0, "end": 4484.0, "title": "And The Clerks Recorded Nothing At All Which", "excerpt": "and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and"}
+{"start": 4484.0, "end": 5624.0, "title": "The Motion And The Clerks Recorded Nothing At", "excerpt": "the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying"}
+{"start": 5624.0, "end": 6764.0, "title": "Tallies Survive In The Manor Rolls Almost Complete", "excerpt": "tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying grain the bailiff recorded every bushel and the tallies survive in the manor rolls almost complete the harvest failed in three successive years and the price of wheat trebled before the first ship arrived from the baltic ports carrying"}
+{"start": 6764.0, "end": 7866.0, "title": "In A Hand Nobody Has Yet Transcribed The", "excerpt": "in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed"}
+{"start": 7866.0, "end": 8968.0, "title": "The Archdeacon Whose Court Kept Minutes In A", "excerpt": "the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of"}
+{"start": 8968.0, "end": 10146.0, "title": "In A Compromise Neither Side Recorded The Abbey", "excerpt": "in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed it before the archdeacon whose court kept minutes in a hand nobody has yet transcribed the dispute ran for forty years and ended in a compromise neither side recorded the abbey claimed the tithe of every sheaf and the villagers disputed the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the"}
+{"start": 10146.0, "end": 11324.0, "title": "The Justices Travelled A Circuit Of Eleven Counties", "excerpt": "the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor the justices travelled a circuit of eleven counties and their rolls are the earliest continuous record of judicial business anywhere in europe the assize met twice yearly and heard pleas of novel disseisin and mort dancestor"}
+{"start": 11324.0, "end": 12388.0, "title": "Which Silver The Exchequer Issued A Proclamation That", "excerpt": "which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a"}
+{"start": 12388.0, "end": 13452.0, "title": "About Which Silver The Exchequer Issued A Proclamation", "excerpt": "about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a"}
+{"start": 13452.0, "end": 14592.0, "title": "About Which Silver The Exchequer Issued A Proclamation", "excerpt": "about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a proclamation that nobody outside london appears to have read or obeyed the coinage was debased twice within a decade and every contract written in silver became a quarrel about which silver the exchequer issued a"}
+{"start": 14592.0, "end": 15732.0, "title": "A Year And The Surveyor Of Highways Was", "excerpt": "a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and"}
+{"start": 15732.0, "end": 16874.0, "title": "Nobody Wanted And Everyone In Turn Was Obliged", "excerpt": "nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody wanted and everyone in turn was obliged to hold the road was maintained by the parish which meant by whoever could be compelled to work on it six days a year and the surveyor of highways was an office nobody the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town"}
+{"start": 16874.0, "end": 18016.0, "title": "Town Closed Its Gates Against The Countryside Which", "excerpt": "town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside which had already closed its lanes against the town the death roll was kept by a clerk who died before he could total it the plague reached the port in autumn and the town closed its gates against the countryside"}

parity/golden/cueseg/one-cue.tail.ndjson

@@ -0,0 +1 @@
+{"start": 0.0, "end": 5.0, "title": "The Mill Stood At The Northern End", "excerpt": "the mill stood at the northern end"}

parity/golden/cueseg/one-cue.tight.ndjson

@@ -0,0 +1 @@
+{"start": 0.0, "end": 4.0, "title": "The Mill Stood At The Northern End", "excerpt": "the mill stood at the northern end"}

parity/golden/cueseg/short.tail.ndjson

@@ -0,0 +1 @@
+{"start": 0.0, "end": 750.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}

parity/golden/cueseg/short.tight.ndjson

@@ -0,0 +1 @@
+{"start": 0.0, "end": 600.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}

parity/golden/cueseg/sparse-text.tail.ndjson

@@ -0,0 +1,2 @@
+{"start": 0.0, "end": 901.0, "title": "The Mill Stood", "excerpt": "the mill stood"}
+{"start": 2703.0, "end": 3604.0, "title": "Parliament Dissolved Itself Again", "excerpt": "parliament dissolved itself again"}

parity/golden/cueseg/sparse-text.tight.ndjson

@@ -0,0 +1,2 @@
+{"start": 0.0, "end": 901.0, "title": "The Mill Stood", "excerpt": "the mill stood"}
+{"start": 2703.0, "end": 3604.0, "title": "Parliament Dissolved Itself Again", "excerpt": "parliament dissolved itself again"}

parity/golden/cueseg/two-topics.tail.ndjson

@@ -0,0 +1,7 @@
+{"start": 0.0, "end": 1190.6666666666667, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 1190.6666666666667, "end": 2381.3333333333335, "title": "On One Household And The Benefit Was Spread", "excerpt": "on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one"}
+{"start": 2381.3333333333335, "end": 3572.0, "title": "Stood At The Northern End Of The Valley", "excerpt": "stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at"}
+{"start": 3572.0, "end": 4657.6, "title": "Steepest And The Ice Came Latest Its Wheel", "excerpt": "steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to"}
+{"start": 4657.6, "end": 5743.2, "title": "Refused To Hear The Motion And The Clerks", "excerpt": "refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with"}
+{"start": 5743.2, "end": 6828.8, "title": "Objections From Every Bench The Speaker Refused To", "excerpt": "objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded"}
+{"start": 6828.8, "end": 7914.4, "title": "Journals Of Either House Parliament Dissolved Itself Again", "excerpt": "journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every"}

parity/golden/cueseg/two-topics.tight.ndjson

@@ -0,0 +1,7 @@
+{"start": 0.0, "end": 1190.6666666666667, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 1190.6666666666667, "end": 2381.3333333333335, "title": "On One Household And The Benefit Was Spread", "excerpt": "on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one"}
+{"start": 2381.3333333333335, "end": 3572.0, "title": "Stood At The Northern End Of The Valley", "excerpt": "stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at"}
+{"start": 3572.0, "end": 4479.0, "title": "Steepest And The Ice Came Latest Its Wheel", "excerpt": "steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to"}
+{"start": 4479.0, "end": 5386.0, "title": "It From The Journals Of Either House Parliament", "excerpt": "it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections"}
+{"start": 5386.0, "end": 6293.0, "title": "At All Which Left The Committee With No", "excerpt": "at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from"}
+{"start": 6293.0, "end": 7200.0, "title": "House Parliament Dissolved Itself Again Amid Furious Objections", "excerpt": "house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker refused to hear the motion and the clerks recorded nothing at all which left the committee with no record of the vote and no way to reconstruct it from the journals of either house parliament dissolved itself again amid furious objections from every bench the speaker"}

parity/golden/cueseg/uniform.tail.ndjson

@@ -0,0 +1,6 @@
+{"start": 0.0, "end": 1079.2, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 1079.2, "end": 2158.4, "title": "Fall Of The River Was Steepest And The", "excerpt": "fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of"}
+{"start": 2158.4, "end": 3237.6, "title": "Was Undershot Which Was Wasteful And Nobody Replaced", "excerpt": "was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was"}
+{"start": 3237.6, "end": 4316.8, "title": "The Cost Fell On One Household And The", "excerpt": "the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost"}
+{"start": 4316.8, "end": 5396.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 5396.0, "end": 6073.0, "title": "On One Household And The Benefit Was Spread", "excerpt": "on one household and the benefit was spread across four hundred families"}

parity/golden/cueseg/uniform.tight.ndjson

@@ -0,0 +1,6 @@
+{"start": 0.0, "end": 900.0, "title": "The Mill Stood At The Northern End Of", "excerpt": "the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill"}
+{"start": 900.0, "end": 1800.0, "title": "Hundred Years Because The Cost Fell On One", "excerpt": "hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred"}
+{"start": 1800.0, "end": 2700.0, "title": "Benefit Was Spread Across Four Hundred Families The", "excerpt": "benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit"}
+{"start": 2700.0, "end": 3600.0, "title": "The Valley Where The Fall Of The River", "excerpt": "the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the"}
+{"start": 3600.0, "end": 4500.0, "title": "Steepest And The Ice Came Latest Its Wheel", "excerpt": "steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest"}
+{"start": 4500.0, "end": 5400.0, "title": "And Nobody Replaced It For Two Hundred Years", "excerpt": "and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and nobody replaced it for two hundred years because the cost fell on one household and the benefit was spread across four hundred families the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest its wheel was undershot which was wasteful and"}

parity/oracle/cueseg.py

@@ -0,0 +1,48 @@
+"""Segment a transcript the way the Python pipeline does.
+
+Reads a cue stream — one `{"start", "end", "text"}` object per line — and a
+duration in seconds as the first argument, and writes one chapter per line.
+
+    CATENA_PARITY_PYTHONPATH=/path/to/podcast-pipeline/src \
+        python3 oracle/cueseg.py 7245 < cues.ndjson
+"""
+
+import json
+import os
+import sys
+
+for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
+    if path:
+        sys.path.insert(0, path)
+
+from podcast_pipeline.topic_segments import Cue, segment_transcript  # noqa: E402
+
+
+def main() -> int:
+    duration = float(sys.argv[1])
+    cues = []
+    for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
+        if not line.strip():
+            continue
+        cue = json.loads(line)
+        cues.append(Cue(cue["start"], cue["end"], cue["text"]))
+
+    out = sys.stdout
+    for seg in segment_transcript(cues, duration):
+        out.write(
+            json.dumps(
+                {
+                    "start": seg.start_time,
+                    "end": seg.end_time,
+                    "title": seg.fallback_title,
+                    "excerpt": seg.excerpt,
+                },
+                ensure_ascii=False,
+            )
+            + "\n"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

parity/src/bin/regen.rs

@@ -19,13 +19,14 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 6] = [
+    let jobs: [Job; 7] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
         ("mdscrub", mdscrub),
         ("pdftriage", pdftriage),
         ("cuenorm", cuenorm),
+        ("cueseg", cueseg),
     ];
 
     for (name, job) in jobs {
@@ -69,6 +70,32 @@ fn unescape() -> Written {
     Ok(vec![path.display().to_string()])
 }
 
+fn cueseg() -> Written {
+    let dir = parity::golden("cueseg");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let table = parity::dir().join("cases/cueseg/durations.json");
+    let table = std::fs::read_to_string(&table).map_err(|e| e.to_string())?;
+    let durations: std::collections::BTreeMap<String, std::collections::BTreeMap<String, f64>> =
+        serde_json::from_str(&table).map_err(|e| e.to_string())?;
+
+    let mut written = Vec::new();
+    for case in parity::cases_with_extension("cueseg", "ndjson") {
+        let name = parity::case_name(&case);
+        let Some(spans) = durations.get(&name) else {
+            continue;
+        };
+        let cues = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
+        for (span, duration) in spans {
+            let chapters = parity::run_oracle_with("cueseg.py", &[format!("{duration}")], &cues)?;
+            let path = dir.join(format!("{name}.{span}.ndjson"));
+            std::fs::write(&path, chapters).map_err(|e| e.to_string())?;
+            written.push(path.display().to_string());
+        }
+    }
+    Ok(written)
+}
+
 fn cuenorm() -> Written {
     let dir = parity::golden("cuenorm");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/cueseg.rs

@@ -0,0 +1,122 @@
+//! Chapter boundaries, titles and excerpts must match the Python exactly.
+//!
+//! The boundaries come out of float arithmetic over hashed word counts, so a
+//! difference of one word in the tokeniser or one unit in the last place in a
+//! cosine moves a chapter mark by a minute. Nothing about that is visible in
+//! the output; it just puts the mark in the wrong place.
+
+use std::collections::BTreeMap;
+
+use cuekit::Cue;
+use serde_json::Value;
+
+/// A chapter, as the two sides can be compared.
+type Chapter = (f64, f64, String, String);
+
+fn chapters(text: &str) -> Vec<Chapter> {
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let chapter: Value = serde_json::from_str(line).expect("valid JSON");
+            (
+                chapter["start"].as_f64().expect("a start"),
+                chapter["end"].as_f64().expect("an end"),
+                chapter["title"].as_str().expect("a title").to_string(),
+                chapter["excerpt"].as_str().expect("an excerpt").to_string(),
+            )
+        })
+        .collect()
+}
+
+fn ours(cues: &[Cue], duration: f64) -> Vec<Chapter> {
+    cueseg::segment(cues, duration)
+        .into_iter()
+        .map(|chapter| (chapter.start, chapter.end, chapter.title, chapter.excerpt))
+        .collect()
+}
+
+fn cues_of(case: &std::path::Path) -> Vec<Cue> {
+    let text = std::fs::read_to_string(case).expect("a readable case");
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let cue: Value = serde_json::from_str(line).expect("valid JSON");
+            Cue::new(
+                cue["start"].as_f64().expect("a start"),
+                cue["end"].as_f64().expect("an end"),
+                cue["text"].as_str().expect("text"),
+            )
+        })
+        .collect()
+}
+
+/// The two durations each case is run at: the end of the transcript, and a
+/// recording that carries on after it.
+fn durations() -> BTreeMap<String, BTreeMap<String, f64>> {
+    let text = std::fs::read_to_string(parity::dir().join("cases/cueseg/durations.json"))
+        .expect("a readable duration table");
+    serde_json::from_str(&text).expect("valid JSON")
+}
+
+#[test]
+fn every_transcript_segments_the_same_as_python() {
+    let durations = durations();
+    let cases = parity::cases_with_extension("cueseg", "ndjson");
+    assert!(!cases.is_empty(), "no cases found");
+
+    for case in cases {
+        let name = parity::case_name(&case);
+        let cues = cues_of(&case);
+        let spans = durations.get(&name).expect("a duration for every case");
+
+        for (span, duration) in spans {
+            let theirs = chapters(&parity::read_golden_text(&format!(
+                "cueseg/{name}.{span}.ndjson"
+            )));
+            let ours = ours(&cues, *duration);
+
+            for (at, (ours, theirs)) in ours.iter().zip(&theirs).enumerate() {
+                assert_eq!(
+                    ours.0, theirs.0,
+                    "{name} [{span}] chapter {at} starts elsewhere"
+                );
+                assert_eq!(
+                    ours.1, theirs.1,
+                    "{name} [{span}] chapter {at} ends elsewhere"
+                );
+                assert_eq!(
+                    ours.2, theirs.2,
+                    "{name} [{span}] chapter {at} is titled differently"
+                );
+                assert_eq!(
+                    ours.3, theirs.3,
+                    "{name} [{span}] chapter {at} excerpts differently"
+                );
+            }
+            assert_eq!(
+                ours.len(),
+                theirs.len(),
+                "{name} [{span}]: python found {} chapters, rust {}",
+                theirs.len(),
+                ours.len()
+            );
+        }
+    }
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_files_still_match_live_python() {
+    let durations = durations();
+    for case in parity::cases_with_extension("cueseg", "ndjson") {
+        let name = parity::case_name(&case);
+        let cues = std::fs::read_to_string(&case).expect("a readable case");
+        for (span, duration) in durations.get(&name).expect("a duration") {
+            let live = parity::run_oracle_with("cueseg.py", &[format!("{duration}")], &cues)
+                .expect("the oracle runs");
+            let golden = parity::read_golden_text(&format!("cueseg/{name}.{span}.ndjson"));
+            assert_eq!(live, golden, "golden file for {name} [{span}] is stale");
+        }
+    }
+}