cueseg: topic chapters from a transcript, and a worked pipeline in the readme

crates/cueseg/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "cueseg"
+description = "Turns a timed transcript into topic chapters."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+cuekit.workspace = true
+regex.workspace = true
+schemars.workspace = true
+serde.workspace = true
+textkit.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
+
+[lints]
+workspace = true

crates/cueseg/src/lib.rs

@@ -0,0 +1,136 @@
+//! Turns a transcript into topic chapters.
+//!
+//! No model and no network: the text is hashed into a small lexical vector a
+//! window at a time, and a topic boundary is where two consecutive windows
+//! stop resembling each other. It is crude, and it is also what makes a
+//! three-hour recording navigable — the chapters it finds are the marks a
+//! scrubber needs and the spans a summariser should be given one at a time.
+//!
+//! Every constant here is a shape decision about recorded speech, and each
+//! one carries the reason it has the value it has.
+
+pub mod segment;
+pub mod title;
+pub mod vector;
+pub mod window;
+
+use cuekit::Cue;
+
+pub use segment::Segment;
+pub use window::Window;
+
+/// How long a window of transcript is, in seconds.
+///
+/// Long enough that a window has a subject, short enough that a boundary
+/// lands within a minute of the real one.
+pub const WINDOW_SECONDS: f64 = 75.0;
+
+/// No chapter may be shorter than this.
+///
+/// A chapter mark every two minutes is not navigation, it is noise.
+pub const MIN_SEGMENT_SECONDS: f64 = 8.0 * 60.0;
+
+/// How many chapters a recording gets, as a function of how long it is.
+pub const TARGET_SECONDS_PER_CHAPTER: f64 = 15.0 * 60.0;
+/// Fewest chapters, whatever the length.
+pub const MIN_CHAPTERS: usize = 4;
+/// Most chapters, whatever the length.
+pub const MAX_CHAPTERS: usize = 20;
+
+/// How much of a chapter's text is kept as its excerpt, in characters.
+pub const EXCERPT_CHARS: usize = 900;
+
+/// Cut a transcript into chapters.
+///
+/// `duration` is how long the recording is. It is taken rather than derived
+/// because a transcript often stops before the recording does — music at the
+/// end, or a speaker who stopped talking — and a chapter that claims to run
+/// to the end of the transcript rather than the end of the file is a chapter
+/// mark that lands in the wrong place.
+pub fn segment(cues: &[Cue], duration: f64) -> Vec<Segment> {
+    let windows = window::windows(cues, WINDOW_SECONDS);
+    if windows.is_empty() {
+        return Vec::new();
+    }
+    let vectors: Vec<Vec<f64>> = windows
+        .iter()
+        .map(|window| vector::lexical(&window.text, vector::DIMENSIONS))
+        .collect();
+    let similarities = vector::consecutive_cosine(&vectors);
+    let boundaries =
+        segment::boundary_times(&windows, &similarities, duration, MIN_SEGMENT_SECONDS, None);
+    segment::from_boundaries(&windows, &boundaries, duration)
+}
+
+/// How many chapters a recording of this length should have.
+pub fn target_chapter_count(duration: f64) -> usize {
+    let target = round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER);
+    (target as usize).clamp(MIN_CHAPTERS, MAX_CHAPTERS)
+}
+
+/// Round half to even, which is what the Python this was ported from does
+/// and what Rust's own `round` does not: `round(2.5)` is 2 there and 3 here,
+/// and a recording exactly two and a half chapters long would otherwise get a
+/// different number of them.
+fn round_half_to_even(value: f64) -> f64 {
+    let rounded = value.round();
+    if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 {
+        rounded - value.signum()
+    } else {
+        rounded
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn rounding_goes_to_even_on_a_half() {
+        assert_eq!(round_half_to_even(0.5), 0.0);
+        assert_eq!(round_half_to_even(1.5), 2.0);
+        assert_eq!(round_half_to_even(2.5), 2.0);
+        assert_eq!(round_half_to_even(3.5), 4.0);
+        assert_eq!(round_half_to_even(-2.5), -2.0);
+        assert_eq!(round_half_to_even(2.4), 2.0);
+        assert_eq!(round_half_to_even(2.6), 3.0);
+    }
+
+    #[test]
+    fn the_chapter_count_follows_the_length_within_bounds() {
+        assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS);
+        assert_eq!(target_chapter_count(60.0 * 60.0), MIN_CHAPTERS);
+        assert_eq!(target_chapter_count(90.0 * 60.0), 6);
+        assert_eq!(target_chapter_count(100.0 * 60.0 * 60.0), MAX_CHAPTERS);
+    }
+
+    #[test]
+    fn a_transcript_with_no_cues_has_no_chapters() {
+        assert!(segment(&[], 3600.0).is_empty());
+    }
+
+    #[test]
+    fn a_long_transcript_is_cut_into_several_chapters() {
+        // Two subjects, an hour each, with nothing in common lexically.
+        let mut cues = Vec::new();
+        let mut at = 0.0;
+        for round in 0..2 {
+            let words = if round == 0 {
+                "the mill stood at the northern end where the fall was steepest"
+            } else {
+                "parliament dissolved itself again amid furious objections from every bench"
+            };
+            while at < (round as f64 + 1.0) * 3600.0 {
+                cues.push(Cue::new(at, at + 5.0, words));
+                at += 5.0;
+            }
+        }
+        let found = segment(&cues, at);
+        assert!(found.len() >= 2, "{} chapters", found.len());
+        assert_eq!(found[0].start, 0.0);
+        assert_eq!(found.last().expect("a chapter").end, at);
+        for pair in found.windows(2) {
+            assert_eq!(pair[0].end, pair[1].start, "chapters must not leave a gap");
+        }
+    }
+}

crates/cueseg/src/main.rs

@@ -0,0 +1,116 @@
+//! Turns a timed transcript into topic chapters.
+
+use std::io::{BufWriter, Read, Write};
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::{exit, ndjson};
+use cuekit::Cue;
+
+/// Read a transcript and write its topic chapters, one per line as JSON.
+///
+/// Takes either the cue stream `cuenorm` writes or a subtitle file directly.
+/// The chapters are found from the text alone — no model, no network — by
+/// comparing what is said in one stretch with what is said in the next.
+///
+/// The titles are extracted from the chapters' own first sentences. They are
+/// meant to be replaced by something better later, and to be usable until
+/// they are.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    long_about = None,
+    after_help = "\
+Example:
+
+    cuenorm episode.vtt | cueseg --duration 7245 > chapters.ndjson"
+)]
+struct Args {
+    /// Cue stream or subtitle file to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+
+    /// How long the recording is, in seconds. Defaults to the end of the
+    /// last cue, which is earlier than the end of the file whenever the
+    /// recording ends in music or silence.
+    #[arg(long, value_name = "SECONDS")]
+    duration: Option<f64>,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+
+    let content = match read(args.path.as_deref()) {
+        Ok(content) => content,
+        Err(e) => {
+            eprintln!("cueseg: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    let cues = match cues(&content) {
+        Ok(cues) => cues,
+        Err(e) => {
+            eprintln!("cueseg: {e}");
+            return ExitCode::from(exit::DATA);
+        }
+    };
+
+    let duration = match args.duration {
+        Some(duration) if duration.is_finite() && duration > 0.0 => duration,
+        Some(duration) => {
+            eprintln!("cueseg: --duration {duration} is not a length");
+            return ExitCode::from(exit::USAGE);
+        }
+        None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max),
+    };
+
+    let mut out = BufWriter::new(std::io::stdout().lock());
+    for chapter in cueseg::segment(&cues, duration) {
+        if let Err(e) = ndjson::write_line(&mut out, &chapter) {
+            // A consumer that stopped reading is not a failure.
+            if e.is_broken_pipe() {
+                return ExitCode::from(exit::OK);
+            }
+            eprintln!("cueseg: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    }
+    match out.flush() {
+        Ok(()) => ExitCode::from(exit::OK),
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("cueseg: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+/// Read cues from either encoding.
+///
+/// A cue stream begins with a JSON object; a subtitle file does not. That is
+/// enough to tell them apart, and it saves a caller from having to say which
+/// of the two it has.
+fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
+    let first = content
+        .lines()
+        .find(|line| !line.trim().is_empty())
+        .unwrap_or("");
+    if first.trim_start().starts_with('{') {
+        return ndjson::read(content.as_bytes()).collect();
+    }
+    Ok(cuekit::parse(content))
+}
+
+fn read(path: Option<&Path>) -> std::io::Result<String> {
+    let bytes = match path {
+        Some(path) => std::fs::read(path)?,
+        None => {
+            let mut bytes = Vec::new();
+            std::io::stdin().read_to_end(&mut bytes)?;
+            bytes
+        }
+    };
+    Ok(String::from_utf8_lossy(&bytes).into_owned())
+}

crates/cueseg/src/segment.rs

@@ -0,0 +1,353 @@
+//! Choosing where the chapters begin, and cutting them.
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::window::Window;
+use crate::{EXCERPT_CHARS, MAX_CHAPTERS, title};
+
+/// One topic chapter of a recording.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Segment {
+    /// Record version.
+    #[serde(default = "version")]
+    pub v: u32,
+    /// Seconds from the start of the recording.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub start: f64,
+    /// Seconds from the start of the recording.
+    #[serde(with = "corpus_core::finite")]
+    #[schemars(with = "f64")]
+    pub end: f64,
+    /// A title taken from the chapter's own first sentence.
+    ///
+    /// Extractive and unembarrassed about it. A model can write a better one
+    /// later; this is what makes the chapter usable before one has.
+    pub title: String,
+    /// The opening of the chapter, for a summariser or a reader.
+    pub excerpt: String,
+}
+
+fn version() -> u32 {
+    corpus_core::V
+}
+
+impl corpus_core::Record for Segment {
+    fn version(&self) -> u32 {
+        self.v
+    }
+}
+
+/// Where the chapters begin, in seconds.
+///
+/// A boundary is a valley: a pair of windows that resemble each other less
+/// than their neighbours do, and less than the recording's own average by
+/// half a standard deviation. Valleys are taken deepest first, each one at
+/// least a chapter's length from the ones already taken.
+pub fn boundary_times(
+    windows: &[Window],
+    similarities: &[f64],
+    duration: f64,
+    min_segment: f64,
+    target_count: Option<usize>,
+) -> Vec<f64> {
+    let target = target_count.unwrap_or_else(|| crate::target_chapter_count(duration));
+    if windows.len() < 2 {
+        return Vec::new();
+    }
+
+    let smoothed = moving_average(similarities, 1);
+    let average = if smoothed.is_empty() {
+        0.0
+    } else {
+        smoothed.iter().sum::<f64>() / smoothed.len() as f64
+    };
+    let deviation = if smoothed.len() < 2 {
+        0.0
+    } else {
+        let sum: f64 = smoothed.iter().map(|value| (value - average).powi(2)).sum();
+        (sum / smoothed.len() as f64).sqrt()
+    };
+    // A recording whose windows all resemble each other equally has no
+    // deviation to work from, so a fixed spread stands in for one.
+    let spread = if deviation == 0.0 { 0.15 } else { deviation };
+    let threshold = average - 0.5 * spread;
+
+    let mut valleys: Vec<(usize, f64)> = Vec::new();
+    for (at, value) in smoothed.iter().enumerate() {
+        let before = if at > 0 {
+            smoothed[at - 1]
+        } else {
+            value + 1.0
+        };
+        let after = if at < smoothed.len() - 1 {
+            smoothed[at + 1]
+        } else {
+            value + 1.0
+        };
+        if *value <= before && *value <= after && *value < threshold {
+            valleys.push((at, *value));
+        }
+    }
+    // Stable, so that two equally deep valleys are taken in the order they
+    // occur in the recording.
+    valleys.sort_by(|a, b| a.1.total_cmp(&b.1));
+
+    let needed = target.saturating_sub(1);
+    let time_of =
+        |gap: usize| -> f64 { windows.get(gap + 1).map_or(duration, |window| window.start) };
+
+    let mut chosen: Vec<usize> = Vec::new();
+    for (at, _) in valleys {
+        let time = time_of(at);
+        if time < min_segment || duration - time < min_segment {
+            continue;
+        }
+        if chosen
+            .iter()
+            .any(|taken| (time_of(*taken) - time).abs() < min_segment)
+        {
+            continue;
+        }
+        chosen.push(at);
+        if chosen.len() >= needed {
+            break;
+        }
+    }
+
+    // Nothing stood out, but the recording is long enough to want chapters
+    // anyway: fall back to even spacing, which is better than one chapter of
+    // three hours.
+    if chosen.is_empty() && duration >= min_segment * 2.0 {
+        let count = needed.max(1);
+        return (1..=count)
+            .map(|k| duration * k as f64 / (count + 1) as f64)
+            .filter(|time| *time >= min_segment && duration - *time >= min_segment)
+            .collect();
+    }
+
+    let mut times: Vec<f64> = chosen.iter().map(|at| time_of(*at)).collect();
+    times.sort_by(f64::total_cmp);
+
+    // A chapter that runs too long is no more navigable than no chapter at
+    // all, so an over-long span is divided evenly whether or not the text
+    // suggested a break there.
+    let longest = (min_segment * 2.5).max(duration / MAX_CHAPTERS as f64);
+    let mut ends = vec![0.0];
+    ends.extend(times.iter().copied());
+    ends.push(duration);
+
+    let mut extra = Vec::new();
+    for pair in ends.windows(2) {
+        let span = pair[1] - pair[0];
+        if span <= longest {
+            continue;
+        }
+        let parts = (span / longest).ceil() as usize;
+        for part in 1..parts {
+            extra.push(pair[0] + span * part as f64 / parts as f64);
+        }
+    }
+    times.extend(extra);
+    times.sort_by(f64::total_cmp);
+    times
+}
+
+/// Cut the windows into chapters at the given times.
+pub fn from_boundaries(windows: &[Window], boundaries: &[f64], duration: f64) -> Vec<Segment> {
+    if windows.is_empty() {
+        return Vec::new();
+    }
+    let mut cuts = vec![0.0];
+    cuts.extend(
+        boundaries
+            .iter()
+            .copied()
+            .filter(|time| *time > 0.0 && *time < duration),
+    );
+    cuts.push(duration);
+
+    // A cut within a second of the one before it would make a chapter nobody
+    // can click on. Compared against the unfiltered list, as the original
+    // does, so a run of near-identical cuts collapses to its first.
+    let unique: Vec<f64> = cuts
+        .iter()
+        .enumerate()
+        .filter(|(at, time)| *at == 0 || **time - cuts[at - 1] > 1.0)
+        .map(|(_, time)| *time)
+        .collect();
+
+    let mut segments = Vec::new();
+    for pair in unique.windows(2) {
+        let (start, end) = (pair[0], pair[1]);
+        let body: Vec<&str> = windows
+            .iter()
+            .filter(|window| window.start < end && window.end > start)
+            .map(|window| window.text.as_str())
+            .collect();
+        let excerpt = excerpt_of(&body.join(" "));
+        if excerpt.is_empty() {
+            continue;
+        }
+        segments.push(Segment {
+            v: corpus_core::V,
+            start,
+            end,
+            title: title::extract(&excerpt),
+            excerpt,
+        });
+    }
+    segments
+}
+
+/// Smooth a series by averaging each value with its neighbours, so that one
+/// unusual window does not read as a topic change.
+fn moving_average(values: &[f64], radius: usize) -> Vec<f64> {
+    (0..values.len())
+        .map(|at| {
+            let from = at.saturating_sub(radius);
+            let to = (at + radius + 1).min(values.len());
+            let slice = &values[from..to];
+            if slice.is_empty() {
+                0.0
+            } else {
+                slice.iter().sum::<f64>() / slice.len() as f64
+            }
+        })
+        .collect()
+}
+
+/// The opening of a chapter, cut at a word boundary where one is close
+/// enough to the limit to be worth using.
+fn excerpt_of(text: &str) -> String {
+    if text.chars().count() <= EXCERPT_CHARS {
+        return text.to_string();
+    }
+    let head: String = text.chars().take(EXCERPT_CHARS).collect();
+    match head.rfind(' ') {
+        Some(at) if head[..at].chars().count() > 200 => head[..at].trim().to_string(),
+        _ => head,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn window(start: f64, end: f64, text: &str) -> Window {
+        Window {
+            start,
+            end,
+            text: text.to_string(),
+        }
+    }
+
+    #[test]
+    fn a_single_window_has_no_boundaries() {
+        let windows = [window(0.0, 10.0, "one")];
+        assert!(boundary_times(&windows, &[], 10.0, 480.0, None).is_empty());
+    }
+
+    #[test]
+    fn a_boundary_lands_where_the_subject_changes() {
+        let windows: Vec<Window> = (0..40)
+            .map(|at| {
+                let start = at as f64 * 75.0;
+                window(start, start + 75.0, "text")
+            })
+            .collect();
+        // Similar throughout except one deep valley in the middle.
+        let mut similarities = vec![0.9; 39];
+        similarities[19] = 0.1;
+        let found = boundary_times(&windows, &similarities, 3000.0, 1200.0, Some(2));
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0], windows[20].start);
+    }
+
+    #[test]
+    fn a_chapter_that_would_run_too_long_is_divided_anyway() {
+        let windows: Vec<Window> = (0..40)
+            .map(|at| {
+                let start = at as f64 * 75.0;
+                window(start, start + 75.0, "text")
+            })
+            .collect();
+        let mut similarities = vec![0.9; 39];
+        similarities[19] = 0.1;
+        // The longest a chapter may run here is 1200 seconds, so each half of
+        // a 3000-second recording split once in the middle is still too long.
+        let found = boundary_times(&windows, &similarities, 3000.0, 480.0, Some(2));
+        assert_eq!(found.len(), 3, "{found:?}");
+        assert!(found.contains(&windows[20].start));
+        let mut ends = vec![0.0];
+        ends.extend(found.iter().copied());
+        ends.push(3000.0);
+        for pair in ends.windows(2) {
+            assert!(pair[1] - pair[0] <= 1200.0, "{:?} runs too long", pair);
+        }
+    }
+
+    #[test]
+    fn a_recording_with_no_valleys_is_cut_evenly() {
+        let windows: Vec<Window> = (0..40)
+            .map(|at| {
+                let start = at as f64 * 75.0;
+                window(start, start + 75.0, "text")
+            })
+            .collect();
+        let found = boundary_times(&windows, &vec![0.9; 39], 3000.0, 480.0, Some(3));
+        assert_eq!(found.len(), 2);
+        assert!(found[0] < found[1]);
+    }
+
+    #[test]
+    fn chapters_meet_end_to_end_and_cover_the_recording() {
+        let windows = [
+            window(
+                0.0,
+                1000.0,
+                "the mill stood at the northern end of the valley",
+            ),
+            window(
+                1000.0,
+                2000.0,
+                "parliament dissolved itself amid objections",
+            ),
+        ];
+        let found = from_boundaries(&windows, &[1000.0], 2000.0);
+        assert_eq!(found.len(), 2);
+        assert_eq!(found[0].start, 0.0);
+        assert_eq!(found[0].end, 1000.0);
+        assert_eq!(found[1].start, 1000.0);
+        assert_eq!(found[1].end, 2000.0);
+    }
+
+    #[test]
+    fn a_boundary_a_moment_after_another_is_dropped() {
+        let windows = [window(0.0, 2000.0, "text enough to excerpt")];
+        let found = from_boundaries(&windows, &[1000.0, 1000.5], 2000.0);
+        assert_eq!(found.len(), 2);
+    }
+
+    #[test]
+    fn a_boundary_outside_the_recording_is_ignored() {
+        let windows = [window(0.0, 2000.0, "text enough to excerpt")];
+        let found = from_boundaries(&windows, &[-5.0, 5000.0], 2000.0);
+        assert_eq!(found.len(), 1);
+    }
+
+    #[test]
+    fn an_excerpt_is_cut_at_a_word() {
+        let text = "word ".repeat(400);
+        let excerpt = excerpt_of(&text);
+        assert!(excerpt.chars().count() <= EXCERPT_CHARS);
+        assert!(!excerpt.ends_with(' '));
+        assert!(text.starts_with(&excerpt));
+    }
+
+    #[test]
+    fn a_short_chapter_keeps_all_its_text() {
+        assert_eq!(excerpt_of("short"), "short");
+    }
+}

crates/cueseg/src/title.rs

@@ -0,0 +1,108 @@
+//! A title taken from what was said, for a chapter nobody has named yet.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// Hesitations. A chapter called "Um Yeah So I Think" is worse than no title.
+static FILLER: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)\b(um+|uh+|ah+|er+)\b").expect("a valid pattern"));
+
+/// The most common discourse marker in recorded speech.
+static YOU_KNOW: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)\byou know\b").expect("a valid pattern"));
+
+static SPACE_RUN: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+
+/// Anything that does not belong in a title.
+static NOT_TITLE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9'-]").expect("a valid pattern"));
+
+/// How many words a title gets.
+const WORDS: usize = 8;
+
+/// Take a title from the first sentence of a chapter.
+///
+/// Extractive, and no cleverer than it looks: the first thing said in a
+/// chapter is usually what the chapter is about, and where it is not, a
+/// reader can see that at a glance and a model can replace it later.
+pub fn extract(excerpt: &str) -> String {
+    let cleaned = FILLER.replace_all(excerpt, " ");
+    let cleaned = YOU_KNOW.replace_all(&cleaned, " ");
+    let cleaned = SPACE_RUN.replace_all(&cleaned, " ");
+    let cleaned = cleaned.trim_matches(textkit::is_space);
+
+    let sentence = textkit::sentences(cleaned).first().copied().unwrap_or("");
+    let title: Vec<String> = sentence
+        .split(textkit::is_space)
+        .filter(|word| !word.is_empty())
+        .take(WORDS)
+        .map(|word| NOT_TITLE.replace_all(word, "").into_owned())
+        .filter(|word| !word.is_empty())
+        .map(|word| capitalise(&word))
+        .collect();
+
+    if title.is_empty() {
+        "Topic".to_string()
+    } else {
+        title.join(" ")
+    }
+}
+
+/// First letter up, the rest down — so that a transcript's shouting and its
+/// mid-sentence capitals come out looking like a title either way.
+fn capitalise(word: &str) -> String {
+    let mut chars = word.chars();
+    match chars.next() {
+        None => String::new(),
+        Some(first) => first.to_uppercase().collect::<String>() + &chars.as_str().to_lowercase(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_first_sentence_becomes_the_title() {
+        assert_eq!(
+            extract("The mill stood at the northern end. Its wheel was undershot."),
+            "The Mill Stood At The Northern End"
+        );
+    }
+
+    #[test]
+    fn a_title_is_at_most_eight_words() {
+        let title = extract("one two three four five six seven eight nine ten.");
+        assert_eq!(title.split(' ').count(), 8);
+    }
+
+    #[test]
+    fn fillers_and_discourse_markers_go() {
+        assert_eq!(
+            extract("Um, so, you know, uh the mill stood there."),
+            "So The Mill Stood There"
+        );
+    }
+
+    #[test]
+    fn punctuation_inside_a_word_is_kept_and_around_it_is_not() {
+        assert_eq!(
+            extract("(the) mill's well-built wheel."),
+            "The Mill's Well-built Wheel"
+        );
+    }
+
+    #[test]
+    fn shouting_comes_out_as_a_title() {
+        assert_eq!(extract("THE MILL STOOD."), "The Mill Stood");
+    }
+
+    #[test]
+    fn nothing_usable_still_gives_a_title() {
+        assert_eq!(extract(""), "Topic");
+        assert_eq!(extract("... !!! ???"), "Topic");
+        assert_eq!(extract("um uh ah"), "Topic");
+    }
+}

crates/cueseg/src/vector.rs

@@ -0,0 +1,175 @@
+//! Turning a window of speech into something two windows can be compared by.
+//!
+//! Hashed bag of words, not embeddings. The question here is only "are these
+//! two minutes about the same thing as the last two", and for that a count of
+//! which content words appear is enough — and it costs no model, no GPU and
+//! no network, which is what lets this run over an archive rather than over a
+//! few files.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// How many buckets a window's words are hashed into.
+///
+/// Small enough that two windows about the same subject collide on the same
+/// buckets, large enough that two windows about different subjects mostly do
+/// not.
+pub const DIMENSIONS: usize = 256;
+
+/// Everything that is not a letter, a digit, an apostrophe or whitespace.
+static NOT_A_WORD: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[^a-z0-9'\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+
+/// Words too common in speech to say anything about the subject.
+///
+/// Longer than an ordinary stop list because this is transcribed speech: the
+/// fillers, the discourse markers and the vague nouns that carry a
+/// conversation carry no topic at all, and leaving them in makes every window
+/// resemble every other.
+const STOP: &[&str] = &[
+    "about", "after", "ah", "all", "also", "and", "are", "back", "because", "been", "before",
+    "but", "can", "come", "could", "did", "don", "even", "for", "from", "get", "going", "gonna",
+    "got", "had", "has", "have", "here", "how", "into", "it's", "its", "just", "kind", "know",
+    "let", "lets", "like", "little", "look", "lot", "mean", "much", "need", "not", "now", "oh",
+    "ok", "okay", "one", "out", "over", "people", "really", "right", "said", "say", "says", "see",
+    "should", "so", "sort", "still", "than", "that", "the", "their", "them", "then", "there",
+    "they", "thing", "things", "think", "this", "uh", "um", "very", "want", "was", "well", "were",
+    "what", "when", "who", "why", "will", "with", "would", "yeah", "yes", "you", "your",
+];
+
+/// The content words of a text.
+pub fn tokens(text: &str) -> Vec<String> {
+    let lowered = text.to_lowercase();
+    let cleaned = NOT_A_WORD.replace_all(&lowered, " ");
+    cleaned
+        .split(textkit::is_space)
+        .filter(|word| !word.is_empty())
+        .filter(|word| word.chars().count() > 2 && !is_stop(word))
+        .map(str::to_string)
+        .collect()
+}
+
+fn is_stop(word: &str) -> bool {
+    STOP.binary_search(&word).is_ok()
+}
+
+/// Hash a text's content words into a vector of counts.
+pub fn lexical(text: &str, dimensions: usize) -> Vec<f64> {
+    let mut vector = vec![0.0; dimensions];
+    for token in tokens(text) {
+        let at = fnv1a(&token) as usize % dimensions;
+        vector[at] += 1.0;
+    }
+    vector
+}
+
+/// FNV-1a over the characters of a word.
+///
+/// Over the characters rather than the bytes, which is what the original
+/// does. It makes no difference here — the tokeniser above leaves only ASCII
+/// — but the two disagree for anything else, and a hash that disagrees puts
+/// a word in a different bucket and moves a boundary.
+fn fnv1a(word: &str) -> u32 {
+    let mut hash: u32 = 2166136261;
+    for c in word.chars() {
+        hash ^= c as u32;
+        hash = hash.wrapping_mul(16777619);
+    }
+    hash
+}
+
+/// Cosine similarity, or zero when either side has no words at all.
+pub fn cosine(a: &[f64], b: &[f64]) -> f64 {
+    let n = a.len().min(b.len());
+    let mut dot = 0.0;
+    let mut norm_a = 0.0;
+    let mut norm_b = 0.0;
+    for at in 0..n {
+        dot += a[at] * b[at];
+        norm_a += a[at] * a[at];
+        norm_b += b[at] * b[at];
+    }
+    if norm_a == 0.0 || norm_b == 0.0 {
+        return 0.0;
+    }
+    dot / (norm_a.sqrt() * norm_b.sqrt())
+}
+
+/// How much each window resembles the one after it.
+pub fn consecutive_cosine(vectors: &[Vec<f64>]) -> Vec<f64> {
+    vectors
+        .windows(2)
+        .map(|pair| cosine(&pair[0], &pair[1]))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_stop_list_is_sorted_so_it_can_be_searched() {
+        assert!(STOP.windows(2).all(|pair| pair[0] < pair[1]));
+    }
+
+    #[test]
+    fn tokens_are_content_words_only() {
+        assert_eq!(
+            tokens("So, you know, the MILL stood at the northern end!"),
+            ["mill", "stood", "northern", "end"]
+        );
+    }
+
+    #[test]
+    fn short_words_and_punctuation_go() {
+        assert_eq!(tokens("a bc def — ghi's"), ["def", "ghi's"]);
+    }
+
+    #[test]
+    fn anything_that_is_not_a_latin_letter_becomes_a_break() {
+        assert_eq!(tokens("café Москва mill"), ["caf", "mill"]);
+    }
+
+    #[test]
+    fn the_hash_is_the_one_the_original_uses() {
+        // FNV-1a over the characters, 32-bit.
+        assert_eq!(fnv1a(""), 2166136261);
+        assert_eq!(fnv1a("a"), 0xe40c292c);
+        assert_eq!(fnv1a("mill"), fnv1a("mill"));
+        assert_ne!(fnv1a("mill"), fnv1a("kiln"));
+    }
+
+    #[test]
+    fn the_same_text_is_perfectly_similar_to_itself() {
+        let vector = lexical("the mill stood at the northern end", DIMENSIONS);
+        assert!((cosine(&vector, &vector) - 1.0).abs() < 1e-12);
+    }
+
+    #[test]
+    fn different_subjects_are_not_similar() {
+        let a = lexical(
+            "the mill stood at the northern end of the valley",
+            DIMENSIONS,
+        );
+        let b = lexical(
+            "parliament dissolved itself amid furious objections",
+            DIMENSIONS,
+        );
+        assert!(cosine(&a, &b) < 0.2, "{}", cosine(&a, &b));
+    }
+
+    #[test]
+    fn an_empty_window_resembles_nothing() {
+        let empty = lexical("", DIMENSIONS);
+        let full = lexical("the mill stood", DIMENSIONS);
+        assert_eq!(cosine(&empty, &full), 0.0);
+        assert_eq!(cosine(&empty, &empty), 0.0);
+    }
+
+    #[test]
+    fn one_window_has_no_similarities_to_report() {
+        assert!(consecutive_cosine(&[vec![1.0]]).is_empty());
+        assert!(consecutive_cosine(&[]).is_empty());
+    }
+}

crates/cueseg/src/window.rs

@@ -0,0 +1,122 @@
+//! Gathering cues into windows long enough to have a subject.
+
+use std::sync::LazyLock;
+
+use cuekit::Cue;
+use regex::Regex;
+
+/// Runs of whitespace, collapsed so that a window's text is one paragraph.
+static SPACE_RUN: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
+
+/// A stretch of transcript treated as one unit of subject matter.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Window {
+    /// When the window starts.
+    pub start: f64,
+    /// When it ends.
+    pub end: f64,
+    /// Everything said in it, on one line.
+    pub text: String,
+}
+
+/// Gather cues into windows of about `seconds` each.
+///
+/// A window closes when a cue starts more than `seconds` after the window
+/// began — not when it would overrun, so a window is always at least as long
+/// as the budget and never cuts a cue in half.
+pub fn windows(cues: &[Cue], seconds: f64) -> Vec<Window> {
+    let Some(first) = cues.first() else {
+        return Vec::new();
+    };
+
+    let mut windows = Vec::new();
+    let mut start = first.start;
+    let mut end = first.end;
+    let mut parts: Vec<&str> = Vec::new();
+
+    for cue in cues {
+        if !parts.is_empty() && cue.start - start >= seconds {
+            flush(&mut windows, start, end, &parts);
+            parts.clear();
+        }
+        if parts.is_empty() {
+            start = cue.start;
+        }
+        parts.push(&cue.text);
+        end = end.max(cue.end);
+    }
+    flush(&mut windows, start, end, &parts);
+    windows
+}
+
+fn flush(windows: &mut Vec<Window>, start: f64, end: f64, parts: &[&str]) {
+    let joined = parts.join(" ");
+    let text = SPACE_RUN.replace_all(&joined, " ");
+    let text = text.trim_matches(textkit::is_space);
+    if !text.is_empty() {
+        windows.push(Window {
+            start,
+            end,
+            text: text.to_string(),
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn cue(start: f64, end: f64, text: &str) -> Cue {
+        Cue::new(start, end, text)
+    }
+
+    #[test]
+    fn cues_within_the_budget_make_one_window() {
+        let cues = [cue(0.0, 5.0, "one"), cue(5.0, 10.0, "two")];
+        let found = windows(&cues, 75.0);
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0].text, "one two");
+        assert_eq!(found[0].start, 0.0);
+        assert_eq!(found[0].end, 10.0);
+    }
+
+    #[test]
+    fn a_window_closes_when_a_cue_starts_past_the_budget() {
+        let cues = [
+            cue(0.0, 5.0, "one"),
+            cue(74.0, 76.0, "two"),
+            cue(80.0, 85.0, "three"),
+        ];
+        let found = windows(&cues, 75.0);
+        assert_eq!(found.len(), 2);
+        assert_eq!(found[0].text, "one two");
+        assert_eq!(found[1].text, "three");
+        assert_eq!(found[1].start, 80.0);
+    }
+
+    #[test]
+    fn an_end_time_that_goes_backwards_does_not_shorten_a_window() {
+        let cues = [cue(0.0, 90.0, "one"), cue(10.0, 20.0, "two")];
+        assert_eq!(windows(&cues, 75.0)[0].end, 90.0);
+    }
+
+    #[test]
+    fn whitespace_inside_a_window_is_collapsed() {
+        let cues = [cue(0.0, 1.0, "one\n\n  two"), cue(1.0, 2.0, "  three ")];
+        assert_eq!(windows(&cues, 75.0)[0].text, "one two three");
+    }
+
+    #[test]
+    fn a_window_of_nothing_but_whitespace_is_dropped() {
+        let cues = [cue(0.0, 1.0, "   "), cue(100.0, 101.0, "kept")];
+        let found = windows(&cues, 75.0);
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0].text, "kept");
+    }
+
+    #[test]
+    fn nothing_in_nothing_out() {
+        assert!(windows(&[], 75.0).is_empty());
+    }
+}