mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
cueseg: topic chapters from a transcript, and a worked pipeline in the readme
This commit is contained in:
funkpower
@@ -0,0 +1,136 @@
|
||||
//! Turns a transcript into topic chapters.
|
||||
//!
|
||||
//! No model and no network: the text is hashed into a small lexical vector a
|
||||
//! window at a time, and a topic boundary is where two consecutive windows
|
||||
//! stop resembling each other. It is crude, and it is also what makes a
|
||||
//! three-hour recording navigable — the chapters it finds are the marks a
|
||||
//! scrubber needs and the spans a summariser should be given one at a time.
|
||||
//!
|
||||
//! Every constant here is a shape decision about recorded speech, and each
|
||||
//! one carries the reason it has the value it has.
|
||||
|
||||
pub mod segment;
|
||||
pub mod title;
|
||||
pub mod vector;
|
||||
pub mod window;
|
||||
|
||||
use cuekit::Cue;
|
||||
|
||||
pub use segment::Segment;
|
||||
pub use window::Window;
|
||||
|
||||
/// How long a window of transcript is, in seconds.
|
||||
///
|
||||
/// Long enough that a window has a subject, short enough that a boundary
|
||||
/// lands within a minute of the real one.
|
||||
pub const WINDOW_SECONDS: f64 = 75.0;
|
||||
|
||||
/// No chapter may be shorter than this.
|
||||
///
|
||||
/// A chapter mark every two minutes is not navigation, it is noise.
|
||||
pub const MIN_SEGMENT_SECONDS: f64 = 8.0 * 60.0;
|
||||
|
||||
/// How many chapters a recording gets, as a function of how long it is.
|
||||
pub const TARGET_SECONDS_PER_CHAPTER: f64 = 15.0 * 60.0;
|
||||
/// Fewest chapters, whatever the length.
|
||||
pub const MIN_CHAPTERS: usize = 4;
|
||||
/// Most chapters, whatever the length.
|
||||
pub const MAX_CHAPTERS: usize = 20;
|
||||
|
||||
/// How much of a chapter's text is kept as its excerpt, in characters.
|
||||
pub const EXCERPT_CHARS: usize = 900;
|
||||
|
||||
/// Cut a transcript into chapters.
|
||||
///
|
||||
/// `duration` is how long the recording is. It is taken rather than derived
|
||||
/// because a transcript often stops before the recording does — music at the
|
||||
/// end, or a speaker who stopped talking — and a chapter that claims to run
|
||||
/// to the end of the transcript rather than the end of the file is a chapter
|
||||
/// mark that lands in the wrong place.
|
||||
pub fn segment(cues: &[Cue], duration: f64) -> Vec<Segment> {
|
||||
let windows = window::windows(cues, WINDOW_SECONDS);
|
||||
if windows.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let vectors: Vec<Vec<f64>> = windows
|
||||
.iter()
|
||||
.map(|window| vector::lexical(&window.text, vector::DIMENSIONS))
|
||||
.collect();
|
||||
let similarities = vector::consecutive_cosine(&vectors);
|
||||
let boundaries =
|
||||
segment::boundary_times(&windows, &similarities, duration, MIN_SEGMENT_SECONDS, None);
|
||||
segment::from_boundaries(&windows, &boundaries, duration)
|
||||
}
|
||||
|
||||
/// How many chapters a recording of this length should have.
|
||||
pub fn target_chapter_count(duration: f64) -> usize {
|
||||
let target = round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER);
|
||||
(target as usize).clamp(MIN_CHAPTERS, MAX_CHAPTERS)
|
||||
}
|
||||
|
||||
/// Round half to even, which is what the Python this was ported from does
|
||||
/// and what Rust's own `round` does not: `round(2.5)` is 2 there and 3 here,
|
||||
/// and a recording exactly two and a half chapters long would otherwise get a
|
||||
/// different number of them.
|
||||
fn round_half_to_even(value: f64) -> f64 {
|
||||
let rounded = value.round();
|
||||
if (value - value.trunc()).abs() == 0.5 && rounded % 2.0 != 0.0 {
|
||||
rounded - value.signum()
|
||||
} else {
|
||||
rounded
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rounding_goes_to_even_on_a_half() {
|
||||
assert_eq!(round_half_to_even(0.5), 0.0);
|
||||
assert_eq!(round_half_to_even(1.5), 2.0);
|
||||
assert_eq!(round_half_to_even(2.5), 2.0);
|
||||
assert_eq!(round_half_to_even(3.5), 4.0);
|
||||
assert_eq!(round_half_to_even(-2.5), -2.0);
|
||||
assert_eq!(round_half_to_even(2.4), 2.0);
|
||||
assert_eq!(round_half_to_even(2.6), 3.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_chapter_count_follows_the_length_within_bounds() {
|
||||
assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS);
|
||||
assert_eq!(target_chapter_count(60.0 * 60.0), MIN_CHAPTERS);
|
||||
assert_eq!(target_chapter_count(90.0 * 60.0), 6);
|
||||
assert_eq!(target_chapter_count(100.0 * 60.0 * 60.0), MAX_CHAPTERS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_transcript_with_no_cues_has_no_chapters() {
|
||||
assert!(segment(&[], 3600.0).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_transcript_is_cut_into_several_chapters() {
|
||||
// Two subjects, an hour each, with nothing in common lexically.
|
||||
let mut cues = Vec::new();
|
||||
let mut at = 0.0;
|
||||
for round in 0..2 {
|
||||
let words = if round == 0 {
|
||||
"the mill stood at the northern end where the fall was steepest"
|
||||
} else {
|
||||
"parliament dissolved itself again amid furious objections from every bench"
|
||||
};
|
||||
while at < (round as f64 + 1.0) * 3600.0 {
|
||||
cues.push(Cue::new(at, at + 5.0, words));
|
||||
at += 5.0;
|
||||
}
|
||||
}
|
||||
let found = segment(&cues, at);
|
||||
assert!(found.len() >= 2, "{} chapters", found.len());
|
||||
assert_eq!(found[0].start, 0.0);
|
||||
assert_eq!(found.last().expect("a chapter").end, at);
|
||||
for pair in found.windows(2) {
|
||||
assert_eq!(pair[0].end, pair[1].start, "chapters must not leave a gap");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
//! Turns a timed transcript into topic chapters.
|
||||
|
||||
use std::io::{BufWriter, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::{exit, ndjson};
|
||||
use cuekit::Cue;
|
||||
|
||||
/// Read a transcript and write its topic chapters, one per line as JSON.
|
||||
///
|
||||
/// Takes either the cue stream `cuenorm` writes or a subtitle file directly.
|
||||
/// The chapters are found from the text alone — no model, no network — by
|
||||
/// comparing what is said in one stretch with what is said in the next.
|
||||
///
|
||||
/// The titles are extracted from the chapters' own first sentences. They are
|
||||
/// meant to be replaced by something better later, and to be usable until
|
||||
/// they are.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
Example:
|
||||
|
||||
cuenorm episode.vtt | cueseg --duration 7245 > chapters.ndjson"
|
||||
)]
|
||||
struct Args {
|
||||
/// Cue stream or subtitle file to read. Reads stdin when absent.
|
||||
path: Option<PathBuf>,
|
||||
|
||||
/// How long the recording is, in seconds. Defaults to the end of the
|
||||
/// last cue, which is earlier than the end of the file whenever the
|
||||
/// recording ends in music or silence.
|
||||
#[arg(long, value_name = "SECONDS")]
|
||||
duration: Option<f64>,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args = Args::parse();
|
||||
|
||||
let content = match read(args.path.as_deref()) {
|
||||
Ok(content) => content,
|
||||
Err(e) => {
|
||||
eprintln!("cueseg: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
};
|
||||
|
||||
let cues = match cues(&content) {
|
||||
Ok(cues) => cues,
|
||||
Err(e) => {
|
||||
eprintln!("cueseg: {e}");
|
||||
return ExitCode::from(exit::DATA);
|
||||
}
|
||||
};
|
||||
|
||||
let duration = match args.duration {
|
||||
Some(duration) if duration.is_finite() && duration > 0.0 => duration,
|
||||
Some(duration) => {
|
||||
eprintln!("cueseg: --duration {duration} is not a length");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max),
|
||||
};
|
||||
|
||||
let mut out = BufWriter::new(std::io::stdout().lock());
|
||||
for chapter in cueseg::segment(&cues, duration) {
|
||||
if let Err(e) = ndjson::write_line(&mut out, &chapter) {
|
||||
// A consumer that stopped reading is not a failure.
|
||||
if e.is_broken_pipe() {
|
||||
return ExitCode::from(exit::OK);
|
||||
}
|
||||
eprintln!("cueseg: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
}
|
||||
match out.flush() {
|
||||
Ok(()) => ExitCode::from(exit::OK),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
|
||||
Err(e) => {
|
||||
eprintln!("cueseg: {e}");
|
||||
ExitCode::from(exit::IO)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read cues from either encoding.
|
||||
///
|
||||
/// A cue stream begins with a JSON object; a subtitle file does not. That is
|
||||
/// enough to tell them apart, and it saves a caller from having to say which
|
||||
/// of the two it has.
|
||||
fn cues(content: &str) -> Result<Vec<Cue>, ndjson::Error> {
|
||||
let first = content
|
||||
.lines()
|
||||
.find(|line| !line.trim().is_empty())
|
||||
.unwrap_or("");
|
||||
if first.trim_start().starts_with('{') {
|
||||
return ndjson::read(content.as_bytes()).collect();
|
||||
}
|
||||
Ok(cuekit::parse(content))
|
||||
}
|
||||
|
||||
fn read(path: Option<&Path>) -> std::io::Result<String> {
|
||||
let bytes = match path {
|
||||
Some(path) => std::fs::read(path)?,
|
||||
None => {
|
||||
let mut bytes = Vec::new();
|
||||
std::io::stdin().read_to_end(&mut bytes)?;
|
||||
bytes
|
||||
}
|
||||
};
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
//! Choosing where the chapters begin, and cutting them.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::window::Window;
|
||||
use crate::{EXCERPT_CHARS, MAX_CHAPTERS, title};
|
||||
|
||||
/// One topic chapter of a recording.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Segment {
|
||||
/// Record version.
|
||||
#[serde(default = "version")]
|
||||
pub v: u32,
|
||||
/// Seconds from the start of the recording.
|
||||
#[serde(with = "corpus_core::finite")]
|
||||
#[schemars(with = "f64")]
|
||||
pub start: f64,
|
||||
/// Seconds from the start of the recording.
|
||||
#[serde(with = "corpus_core::finite")]
|
||||
#[schemars(with = "f64")]
|
||||
pub end: f64,
|
||||
/// A title taken from the chapter's own first sentence.
|
||||
///
|
||||
/// Extractive and unembarrassed about it. A model can write a better one
|
||||
/// later; this is what makes the chapter usable before one has.
|
||||
pub title: String,
|
||||
/// The opening of the chapter, for a summariser or a reader.
|
||||
pub excerpt: String,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
corpus_core::V
|
||||
}
|
||||
|
||||
impl corpus_core::Record for Segment {
|
||||
fn version(&self) -> u32 {
|
||||
self.v
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the chapters begin, in seconds.
|
||||
///
|
||||
/// A boundary is a valley: a pair of windows that resemble each other less
|
||||
/// than their neighbours do, and less than the recording's own average by
|
||||
/// half a standard deviation. Valleys are taken deepest first, each one at
|
||||
/// least a chapter's length from the ones already taken.
|
||||
pub fn boundary_times(
|
||||
windows: &[Window],
|
||||
similarities: &[f64],
|
||||
duration: f64,
|
||||
min_segment: f64,
|
||||
target_count: Option<usize>,
|
||||
) -> Vec<f64> {
|
||||
let target = target_count.unwrap_or_else(|| crate::target_chapter_count(duration));
|
||||
if windows.len() < 2 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let smoothed = moving_average(similarities, 1);
|
||||
let average = if smoothed.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
smoothed.iter().sum::<f64>() / smoothed.len() as f64
|
||||
};
|
||||
let deviation = if smoothed.len() < 2 {
|
||||
0.0
|
||||
} else {
|
||||
let sum: f64 = smoothed.iter().map(|value| (value - average).powi(2)).sum();
|
||||
(sum / smoothed.len() as f64).sqrt()
|
||||
};
|
||||
// A recording whose windows all resemble each other equally has no
|
||||
// deviation to work from, so a fixed spread stands in for one.
|
||||
let spread = if deviation == 0.0 { 0.15 } else { deviation };
|
||||
let threshold = average - 0.5 * spread;
|
||||
|
||||
let mut valleys: Vec<(usize, f64)> = Vec::new();
|
||||
for (at, value) in smoothed.iter().enumerate() {
|
||||
let before = if at > 0 {
|
||||
smoothed[at - 1]
|
||||
} else {
|
||||
value + 1.0
|
||||
};
|
||||
let after = if at < smoothed.len() - 1 {
|
||||
smoothed[at + 1]
|
||||
} else {
|
||||
value + 1.0
|
||||
};
|
||||
if *value <= before && *value <= after && *value < threshold {
|
||||
valleys.push((at, *value));
|
||||
}
|
||||
}
|
||||
// Stable, so that two equally deep valleys are taken in the order they
|
||||
// occur in the recording.
|
||||
valleys.sort_by(|a, b| a.1.total_cmp(&b.1));
|
||||
|
||||
let needed = target.saturating_sub(1);
|
||||
let time_of =
|
||||
|gap: usize| -> f64 { windows.get(gap + 1).map_or(duration, |window| window.start) };
|
||||
|
||||
let mut chosen: Vec<usize> = Vec::new();
|
||||
for (at, _) in valleys {
|
||||
let time = time_of(at);
|
||||
if time < min_segment || duration - time < min_segment {
|
||||
continue;
|
||||
}
|
||||
if chosen
|
||||
.iter()
|
||||
.any(|taken| (time_of(*taken) - time).abs() < min_segment)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
chosen.push(at);
|
||||
if chosen.len() >= needed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing stood out, but the recording is long enough to want chapters
|
||||
// anyway: fall back to even spacing, which is better than one chapter of
|
||||
// three hours.
|
||||
if chosen.is_empty() && duration >= min_segment * 2.0 {
|
||||
let count = needed.max(1);
|
||||
return (1..=count)
|
||||
.map(|k| duration * k as f64 / (count + 1) as f64)
|
||||
.filter(|time| *time >= min_segment && duration - *time >= min_segment)
|
||||
.collect();
|
||||
}
|
||||
|
||||
let mut times: Vec<f64> = chosen.iter().map(|at| time_of(*at)).collect();
|
||||
times.sort_by(f64::total_cmp);
|
||||
|
||||
// A chapter that runs too long is no more navigable than no chapter at
|
||||
// all, so an over-long span is divided evenly whether or not the text
|
||||
// suggested a break there.
|
||||
let longest = (min_segment * 2.5).max(duration / MAX_CHAPTERS as f64);
|
||||
let mut ends = vec![0.0];
|
||||
ends.extend(times.iter().copied());
|
||||
ends.push(duration);
|
||||
|
||||
let mut extra = Vec::new();
|
||||
for pair in ends.windows(2) {
|
||||
let span = pair[1] - pair[0];
|
||||
if span <= longest {
|
||||
continue;
|
||||
}
|
||||
let parts = (span / longest).ceil() as usize;
|
||||
for part in 1..parts {
|
||||
extra.push(pair[0] + span * part as f64 / parts as f64);
|
||||
}
|
||||
}
|
||||
times.extend(extra);
|
||||
times.sort_by(f64::total_cmp);
|
||||
times
|
||||
}
|
||||
|
||||
/// Cut the windows into chapters at the given times.
|
||||
pub fn from_boundaries(windows: &[Window], boundaries: &[f64], duration: f64) -> Vec<Segment> {
|
||||
if windows.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut cuts = vec![0.0];
|
||||
cuts.extend(
|
||||
boundaries
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|time| *time > 0.0 && *time < duration),
|
||||
);
|
||||
cuts.push(duration);
|
||||
|
||||
// A cut within a second of the one before it would make a chapter nobody
|
||||
// can click on. Compared against the unfiltered list, as the original
|
||||
// does, so a run of near-identical cuts collapses to its first.
|
||||
let unique: Vec<f64> = cuts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(at, time)| *at == 0 || **time - cuts[at - 1] > 1.0)
|
||||
.map(|(_, time)| *time)
|
||||
.collect();
|
||||
|
||||
let mut segments = Vec::new();
|
||||
for pair in unique.windows(2) {
|
||||
let (start, end) = (pair[0], pair[1]);
|
||||
let body: Vec<&str> = windows
|
||||
.iter()
|
||||
.filter(|window| window.start < end && window.end > start)
|
||||
.map(|window| window.text.as_str())
|
||||
.collect();
|
||||
let excerpt = excerpt_of(&body.join(" "));
|
||||
if excerpt.is_empty() {
|
||||
continue;
|
||||
}
|
||||
segments.push(Segment {
|
||||
v: corpus_core::V,
|
||||
start,
|
||||
end,
|
||||
title: title::extract(&excerpt),
|
||||
excerpt,
|
||||
});
|
||||
}
|
||||
segments
|
||||
}
|
||||
|
||||
/// Smooth a series by averaging each value with its neighbours, so that one
|
||||
/// unusual window does not read as a topic change.
|
||||
fn moving_average(values: &[f64], radius: usize) -> Vec<f64> {
|
||||
(0..values.len())
|
||||
.map(|at| {
|
||||
let from = at.saturating_sub(radius);
|
||||
let to = (at + radius + 1).min(values.len());
|
||||
let slice = &values[from..to];
|
||||
if slice.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
slice.iter().sum::<f64>() / slice.len() as f64
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The opening of a chapter, cut at a word boundary where one is close
|
||||
/// enough to the limit to be worth using.
|
||||
fn excerpt_of(text: &str) -> String {
|
||||
if text.chars().count() <= EXCERPT_CHARS {
|
||||
return text.to_string();
|
||||
}
|
||||
let head: String = text.chars().take(EXCERPT_CHARS).collect();
|
||||
match head.rfind(' ') {
|
||||
Some(at) if head[..at].chars().count() > 200 => head[..at].trim().to_string(),
|
||||
_ => head,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn window(start: f64, end: f64, text: &str) -> Window {
|
||||
Window {
|
||||
start,
|
||||
end,
|
||||
text: text.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_single_window_has_no_boundaries() {
|
||||
let windows = [window(0.0, 10.0, "one")];
|
||||
assert!(boundary_times(&windows, &[], 10.0, 480.0, None).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_boundary_lands_where_the_subject_changes() {
|
||||
let windows: Vec<Window> = (0..40)
|
||||
.map(|at| {
|
||||
let start = at as f64 * 75.0;
|
||||
window(start, start + 75.0, "text")
|
||||
})
|
||||
.collect();
|
||||
// Similar throughout except one deep valley in the middle.
|
||||
let mut similarities = vec![0.9; 39];
|
||||
similarities[19] = 0.1;
|
||||
let found = boundary_times(&windows, &similarities, 3000.0, 1200.0, Some(2));
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0], windows[20].start);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chapter_that_would_run_too_long_is_divided_anyway() {
|
||||
let windows: Vec<Window> = (0..40)
|
||||
.map(|at| {
|
||||
let start = at as f64 * 75.0;
|
||||
window(start, start + 75.0, "text")
|
||||
})
|
||||
.collect();
|
||||
let mut similarities = vec![0.9; 39];
|
||||
similarities[19] = 0.1;
|
||||
// The longest a chapter may run here is 1200 seconds, so each half of
|
||||
// a 3000-second recording split once in the middle is still too long.
|
||||
let found = boundary_times(&windows, &similarities, 3000.0, 480.0, Some(2));
|
||||
assert_eq!(found.len(), 3, "{found:?}");
|
||||
assert!(found.contains(&windows[20].start));
|
||||
let mut ends = vec![0.0];
|
||||
ends.extend(found.iter().copied());
|
||||
ends.push(3000.0);
|
||||
for pair in ends.windows(2) {
|
||||
assert!(pair[1] - pair[0] <= 1200.0, "{:?} runs too long", pair);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_recording_with_no_valleys_is_cut_evenly() {
|
||||
let windows: Vec<Window> = (0..40)
|
||||
.map(|at| {
|
||||
let start = at as f64 * 75.0;
|
||||
window(start, start + 75.0, "text")
|
||||
})
|
||||
.collect();
|
||||
let found = boundary_times(&windows, &vec![0.9; 39], 3000.0, 480.0, Some(3));
|
||||
assert_eq!(found.len(), 2);
|
||||
assert!(found[0] < found[1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chapters_meet_end_to_end_and_cover_the_recording() {
|
||||
let windows = [
|
||||
window(
|
||||
0.0,
|
||||
1000.0,
|
||||
"the mill stood at the northern end of the valley",
|
||||
),
|
||||
window(
|
||||
1000.0,
|
||||
2000.0,
|
||||
"parliament dissolved itself amid objections",
|
||||
),
|
||||
];
|
||||
let found = from_boundaries(&windows, &[1000.0], 2000.0);
|
||||
assert_eq!(found.len(), 2);
|
||||
assert_eq!(found[0].start, 0.0);
|
||||
assert_eq!(found[0].end, 1000.0);
|
||||
assert_eq!(found[1].start, 1000.0);
|
||||
assert_eq!(found[1].end, 2000.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_boundary_a_moment_after_another_is_dropped() {
|
||||
let windows = [window(0.0, 2000.0, "text enough to excerpt")];
|
||||
let found = from_boundaries(&windows, &[1000.0, 1000.5], 2000.0);
|
||||
assert_eq!(found.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_boundary_outside_the_recording_is_ignored() {
|
||||
let windows = [window(0.0, 2000.0, "text enough to excerpt")];
|
||||
let found = from_boundaries(&windows, &[-5.0, 5000.0], 2000.0);
|
||||
assert_eq!(found.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_excerpt_is_cut_at_a_word() {
|
||||
let text = "word ".repeat(400);
|
||||
let excerpt = excerpt_of(&text);
|
||||
assert!(excerpt.chars().count() <= EXCERPT_CHARS);
|
||||
assert!(!excerpt.ends_with(' '));
|
||||
assert!(text.starts_with(&excerpt));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_short_chapter_keeps_all_its_text() {
|
||||
assert_eq!(excerpt_of("short"), "short");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
//! A title taken from what was said, for a chapter nobody has named yet.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
/// Hesitations. A chapter called "Um Yeah So I Think" is worse than no title.
|
||||
static FILLER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\b(um+|uh+|ah+|er+)\b").expect("a valid pattern"));
|
||||
|
||||
/// The most common discourse marker in recorded speech.
|
||||
static YOU_KNOW: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\byou know\b").expect("a valid pattern"));
|
||||
|
||||
static SPACE_RUN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
|
||||
|
||||
/// Anything that does not belong in a title.
|
||||
static NOT_TITLE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9'-]").expect("a valid pattern"));
|
||||
|
||||
/// How many words a title gets.
|
||||
const WORDS: usize = 8;
|
||||
|
||||
/// Take a title from the first sentence of a chapter.
|
||||
///
|
||||
/// Extractive, and no cleverer than it looks: the first thing said in a
|
||||
/// chapter is usually what the chapter is about, and where it is not, a
|
||||
/// reader can see that at a glance and a model can replace it later.
|
||||
pub fn extract(excerpt: &str) -> String {
|
||||
let cleaned = FILLER.replace_all(excerpt, " ");
|
||||
let cleaned = YOU_KNOW.replace_all(&cleaned, " ");
|
||||
let cleaned = SPACE_RUN.replace_all(&cleaned, " ");
|
||||
let cleaned = cleaned.trim_matches(textkit::is_space);
|
||||
|
||||
let sentence = textkit::sentences(cleaned).first().copied().unwrap_or("");
|
||||
let title: Vec<String> = sentence
|
||||
.split(textkit::is_space)
|
||||
.filter(|word| !word.is_empty())
|
||||
.take(WORDS)
|
||||
.map(|word| NOT_TITLE.replace_all(word, "").into_owned())
|
||||
.filter(|word| !word.is_empty())
|
||||
.map(|word| capitalise(&word))
|
||||
.collect();
|
||||
|
||||
if title.is_empty() {
|
||||
"Topic".to_string()
|
||||
} else {
|
||||
title.join(" ")
|
||||
}
|
||||
}
|
||||
|
||||
/// First letter up, the rest down — so that a transcript's shouting and its
|
||||
/// mid-sentence capitals come out looking like a title either way.
|
||||
fn capitalise(word: &str) -> String {
|
||||
let mut chars = word.chars();
|
||||
match chars.next() {
|
||||
None => String::new(),
|
||||
Some(first) => first.to_uppercase().collect::<String>() + &chars.as_str().to_lowercase(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_first_sentence_becomes_the_title() {
|
||||
assert_eq!(
|
||||
extract("The mill stood at the northern end. Its wheel was undershot."),
|
||||
"The Mill Stood At The Northern End"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_is_at_most_eight_words() {
|
||||
let title = extract("one two three four five six seven eight nine ten.");
|
||||
assert_eq!(title.split(' ').count(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fillers_and_discourse_markers_go() {
|
||||
assert_eq!(
|
||||
extract("Um, so, you know, uh the mill stood there."),
|
||||
"So The Mill Stood There"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn punctuation_inside_a_word_is_kept_and_around_it_is_not() {
|
||||
assert_eq!(
|
||||
extract("(the) mill's well-built wheel."),
|
||||
"The Mill's Well-built Wheel"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shouting_comes_out_as_a_title() {
|
||||
assert_eq!(extract("THE MILL STOOD."), "The Mill Stood");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_usable_still_gives_a_title() {
|
||||
assert_eq!(extract(""), "Topic");
|
||||
assert_eq!(extract("... !!! ???"), "Topic");
|
||||
assert_eq!(extract("um uh ah"), "Topic");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
//! Turning a window of speech into something two windows can be compared by.
|
||||
//!
|
||||
//! Hashed bag of words, not embeddings. The question here is only "are these
|
||||
//! two minutes about the same thing as the last two", and for that a count of
|
||||
//! which content words appear is enough — and it costs no model, no GPU and
|
||||
//! no network, which is what lets this run over an archive rather than over a
|
||||
//! few files.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
/// How many buckets a window's words are hashed into.
|
||||
///
|
||||
/// Small enough that two windows about the same subject collide on the same
|
||||
/// buckets, large enough that two windows about different subjects mostly do
|
||||
/// not.
|
||||
pub const DIMENSIONS: usize = 256;
|
||||
|
||||
/// Everything that is not a letter, a digit, an apostrophe or whitespace.
|
||||
static NOT_A_WORD: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[^a-z0-9'\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
|
||||
|
||||
/// Words too common in speech to say anything about the subject.
|
||||
///
|
||||
/// Longer than an ordinary stop list because this is transcribed speech: the
|
||||
/// fillers, the discourse markers and the vague nouns that carry a
|
||||
/// conversation carry no topic at all, and leaving them in makes every window
|
||||
/// resemble every other.
|
||||
const STOP: &[&str] = &[
|
||||
"about", "after", "ah", "all", "also", "and", "are", "back", "because", "been", "before",
|
||||
"but", "can", "come", "could", "did", "don", "even", "for", "from", "get", "going", "gonna",
|
||||
"got", "had", "has", "have", "here", "how", "into", "it's", "its", "just", "kind", "know",
|
||||
"let", "lets", "like", "little", "look", "lot", "mean", "much", "need", "not", "now", "oh",
|
||||
"ok", "okay", "one", "out", "over", "people", "really", "right", "said", "say", "says", "see",
|
||||
"should", "so", "sort", "still", "than", "that", "the", "their", "them", "then", "there",
|
||||
"they", "thing", "things", "think", "this", "uh", "um", "very", "want", "was", "well", "were",
|
||||
"what", "when", "who", "why", "will", "with", "would", "yeah", "yes", "you", "your",
|
||||
];
|
||||
|
||||
/// The content words of a text.
|
||||
pub fn tokens(text: &str) -> Vec<String> {
|
||||
let lowered = text.to_lowercase();
|
||||
let cleaned = NOT_A_WORD.replace_all(&lowered, " ");
|
||||
cleaned
|
||||
.split(textkit::is_space)
|
||||
.filter(|word| !word.is_empty())
|
||||
.filter(|word| word.chars().count() > 2 && !is_stop(word))
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_stop(word: &str) -> bool {
|
||||
STOP.binary_search(&word).is_ok()
|
||||
}
|
||||
|
||||
/// Hash a text's content words into a vector of counts.
|
||||
pub fn lexical(text: &str, dimensions: usize) -> Vec<f64> {
|
||||
let mut vector = vec![0.0; dimensions];
|
||||
for token in tokens(text) {
|
||||
let at = fnv1a(&token) as usize % dimensions;
|
||||
vector[at] += 1.0;
|
||||
}
|
||||
vector
|
||||
}
|
||||
|
||||
/// FNV-1a over the characters of a word.
|
||||
///
|
||||
/// Over the characters rather than the bytes, which is what the original
|
||||
/// does. It makes no difference here — the tokeniser above leaves only ASCII
|
||||
/// — but the two disagree for anything else, and a hash that disagrees puts
|
||||
/// a word in a different bucket and moves a boundary.
|
||||
fn fnv1a(word: &str) -> u32 {
|
||||
let mut hash: u32 = 2166136261;
|
||||
for c in word.chars() {
|
||||
hash ^= c as u32;
|
||||
hash = hash.wrapping_mul(16777619);
|
||||
}
|
||||
hash
|
||||
}
|
||||
|
||||
/// Cosine similarity, or zero when either side has no words at all.
|
||||
pub fn cosine(a: &[f64], b: &[f64]) -> f64 {
|
||||
let n = a.len().min(b.len());
|
||||
let mut dot = 0.0;
|
||||
let mut norm_a = 0.0;
|
||||
let mut norm_b = 0.0;
|
||||
for at in 0..n {
|
||||
dot += a[at] * b[at];
|
||||
norm_a += a[at] * a[at];
|
||||
norm_b += b[at] * b[at];
|
||||
}
|
||||
if norm_a == 0.0 || norm_b == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
dot / (norm_a.sqrt() * norm_b.sqrt())
|
||||
}
|
||||
|
||||
/// How much each window resembles the one after it.
|
||||
pub fn consecutive_cosine(vectors: &[Vec<f64>]) -> Vec<f64> {
|
||||
vectors
|
||||
.windows(2)
|
||||
.map(|pair| cosine(&pair[0], &pair[1]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_stop_list_is_sorted_so_it_can_be_searched() {
|
||||
assert!(STOP.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_are_content_words_only() {
|
||||
assert_eq!(
|
||||
tokens("So, you know, the MILL stood at the northern end!"),
|
||||
["mill", "stood", "northern", "end"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_words_and_punctuation_go() {
|
||||
assert_eq!(tokens("a bc def — ghi's"), ["def", "ghi's"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anything_that_is_not_a_latin_letter_becomes_a_break() {
|
||||
assert_eq!(tokens("café Москва mill"), ["caf", "mill"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_hash_is_the_one_the_original_uses() {
|
||||
// FNV-1a over the characters, 32-bit.
|
||||
assert_eq!(fnv1a(""), 2166136261);
|
||||
assert_eq!(fnv1a("a"), 0xe40c292c);
|
||||
assert_eq!(fnv1a("mill"), fnv1a("mill"));
|
||||
assert_ne!(fnv1a("mill"), fnv1a("kiln"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_text_is_perfectly_similar_to_itself() {
|
||||
let vector = lexical("the mill stood at the northern end", DIMENSIONS);
|
||||
assert!((cosine(&vector, &vector) - 1.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_subjects_are_not_similar() {
|
||||
let a = lexical(
|
||||
"the mill stood at the northern end of the valley",
|
||||
DIMENSIONS,
|
||||
);
|
||||
let b = lexical(
|
||||
"parliament dissolved itself amid furious objections",
|
||||
DIMENSIONS,
|
||||
);
|
||||
assert!(cosine(&a, &b) < 0.2, "{}", cosine(&a, &b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_window_resembles_nothing() {
|
||||
let empty = lexical("", DIMENSIONS);
|
||||
let full = lexical("the mill stood", DIMENSIONS);
|
||||
assert_eq!(cosine(&empty, &full), 0.0);
|
||||
assert_eq!(cosine(&empty, &empty), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_window_has_no_similarities_to_report() {
|
||||
assert!(consecutive_cosine(&[vec![1.0]]).is_empty());
|
||||
assert!(consecutive_cosine(&[]).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
//! Gathering cues into windows long enough to have a subject.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use cuekit::Cue;
|
||||
use regex::Regex;
|
||||
|
||||
/// Runs of whitespace, collapsed so that a window's text is one paragraph.
|
||||
static SPACE_RUN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[\s\x{1c}-\x{1f}]+").expect("a valid pattern"));
|
||||
|
||||
/// A stretch of transcript treated as one unit of subject matter.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Window {
|
||||
/// When the window starts.
|
||||
pub start: f64,
|
||||
/// When it ends.
|
||||
pub end: f64,
|
||||
/// Everything said in it, on one line.
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// Gather cues into windows of about `seconds` each.
|
||||
///
|
||||
/// A window closes when a cue starts more than `seconds` after the window
|
||||
/// began — not when it would overrun, so a window is always at least as long
|
||||
/// as the budget and never cuts a cue in half.
|
||||
pub fn windows(cues: &[Cue], seconds: f64) -> Vec<Window> {
|
||||
let Some(first) = cues.first() else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut windows = Vec::new();
|
||||
let mut start = first.start;
|
||||
let mut end = first.end;
|
||||
let mut parts: Vec<&str> = Vec::new();
|
||||
|
||||
for cue in cues {
|
||||
if !parts.is_empty() && cue.start - start >= seconds {
|
||||
flush(&mut windows, start, end, &parts);
|
||||
parts.clear();
|
||||
}
|
||||
if parts.is_empty() {
|
||||
start = cue.start;
|
||||
}
|
||||
parts.push(&cue.text);
|
||||
end = end.max(cue.end);
|
||||
}
|
||||
flush(&mut windows, start, end, &parts);
|
||||
windows
|
||||
}
|
||||
|
||||
fn flush(windows: &mut Vec<Window>, start: f64, end: f64, parts: &[&str]) {
|
||||
let joined = parts.join(" ");
|
||||
let text = SPACE_RUN.replace_all(&joined, " ");
|
||||
let text = text.trim_matches(textkit::is_space);
|
||||
if !text.is_empty() {
|
||||
windows.push(Window {
|
||||
start,
|
||||
end,
|
||||
text: text.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cue(start: f64, end: f64, text: &str) -> Cue {
|
||||
Cue::new(start, end, text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cues_within_the_budget_make_one_window() {
|
||||
let cues = [cue(0.0, 5.0, "one"), cue(5.0, 10.0, "two")];
|
||||
let found = windows(&cues, 75.0);
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].text, "one two");
|
||||
assert_eq!(found[0].start, 0.0);
|
||||
assert_eq!(found[0].end, 10.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_closes_when_a_cue_starts_past_the_budget() {
|
||||
let cues = [
|
||||
cue(0.0, 5.0, "one"),
|
||||
cue(74.0, 76.0, "two"),
|
||||
cue(80.0, 85.0, "three"),
|
||||
];
|
||||
let found = windows(&cues, 75.0);
|
||||
assert_eq!(found.len(), 2);
|
||||
assert_eq!(found[0].text, "one two");
|
||||
assert_eq!(found[1].text, "three");
|
||||
assert_eq!(found[1].start, 80.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_end_time_that_goes_backwards_does_not_shorten_a_window() {
|
||||
let cues = [cue(0.0, 90.0, "one"), cue(10.0, 20.0, "two")];
|
||||
assert_eq!(windows(&cues, 75.0)[0].end, 90.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_inside_a_window_is_collapsed() {
|
||||
let cues = [cue(0.0, 1.0, "one\n\n two"), cue(1.0, 2.0, " three ")];
|
||||
assert_eq!(windows(&cues, 75.0)[0].text, "one two three");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_of_nothing_but_whitespace_is_dropped() {
|
||||
let cues = [cue(0.0, 1.0, " "), cue(100.0, 101.0, "kept")];
|
||||
let found = windows(&cues, 75.0);
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].text, "kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_in_nothing_out() {
|
||||
assert!(windows(&[], 75.0).is_empty());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user