cueseg: chapter by vectors somebody else made, where there are some

The hashed bag of words asks only whether two stretches repeat the same
words, and cannot tell that ships and vessels are one subject. An embedding
can. segment_with takes one vector per window and falls back to the words
when it is given none, or the wrong number of them — a caller that embedded
some other recording's windows is asking about that recording.

windows_of gives the text to embed, in the order the vectors come back in,
so nobody has to reimplement the windowing pass to use this.

Bound in all three languages, and on the command line as --windows and
--embeddings, which pipe through emb. The C ABI version goes to 2:
catena_segment_cues gained an argument.

crates/cueseg/Cargo.toml

@@ -16,6 +16,7 @@ fancy-regex.workspace = true
 regex.workspace = true
 schemars.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 textkit.workspace = true
 
 [dev-dependencies]

crates/cueseg/README.md

@@ -64,6 +64,8 @@ cueseg --duration 7245 transcript.ndjson | jq -r '"\(.start | floor)\t\(.title)"
 
 | Option | Effect |
 |---|---|
+| `--windows` | Print the text to embed, one window per line, and stop |
+| `--embeddings <FILE>` | One vector per window, as `emb` writes them |
 | `--duration <SECONDS>` | How long the recording is |
 | `--man` | Write this page as a manual page, in roff |
 
@@ -72,6 +74,33 @@ earlier than the end of the file whenever a recording ends in music or
 silence, and a chapter that claims to run to the end of the transcript rather
 than the end of the file is a mark that lands in the wrong place.
 
+## Better boundaries, if you have embeddings
+
+By default the windows are compared by the words they repeat. That is fast,
+needs nothing, and cannot tell that a stretch about *ships* and a stretch
+about *vessels* are about one thing. An embedding can.
+
+`--windows` prints the text to embed, one window per line; `--embeddings`
+takes one vector per line back, in the same order:
+
+```sh
+cueseg --windows episode.srt | emb --model nomic-embed-text > windows.ndjson
+cueseg --embeddings windows.ndjson episode.srt
+```
+
+The count has to match, and a mismatch is refused rather than guessed at:
+
+```
+$ cueseg --embeddings two-vectors.ndjson episode.srt
+cueseg: 2 vectors for 44 windows — run --windows to see what to embed
+$ echo $?
+65
+```
+
+Everything else is the same: the same windows, the same threshold, the same
+minimum chapter length. Only the question "are these two stretches about the
+same thing" is answered better.
+
 ## Exit codes
 
 | Code | Meaning |

crates/cueseg/src/lib.rs

@@ -48,20 +48,54 @@ pub const EXCERPT_CHARS: usize = 900;
 /// to the end of the transcript rather than the end of the file is a chapter
 /// mark that lands in the wrong place.
 pub fn segment(cues: &[Cue], duration: f64) -> Vec<Segment> {
+    segment_with(cues, duration, None)
+}
+
+/// The same, given vectors somebody else computed for the windows.
+///
+/// The hashed bag of words this falls back on asks only "are these two
+/// stretches about the same thing", and answers it from repeated words. A
+/// real embedding answers the same question far better, because it knows
+/// that a stretch about *ships* and a stretch about *vessels* are about one
+/// thing, which no amount of word counting will tell you.
+///
+/// One vector per window, in order. [`windows_of`] gives the windows to embed
+/// so a caller can produce them without reimplementing the pass. A slice of
+/// the wrong length is ignored rather than trusted — a caller that embedded
+/// a different set of windows is asking a question about some other
+/// recording — and the lexical route stands in.
+pub fn segment_with(cues: &[Cue], duration: f64, embeddings: Option<&[Vec<f64>]>) -> Vec<Segment> {
     let windows = window::windows(cues, WINDOW_SECONDS);
     if windows.is_empty() {
         return Vec::new();
     }
-    let vectors: Vec<Vec<f64>> = windows
-        .iter()
-        .map(|window| vector::lexical(&window.text, vector::DIMENSIONS))
-        .collect();
-    let similarities = vector::consecutive_cosine(&vectors);
+
+    let lexical: Vec<Vec<f64>>;
+    let vectors: &[Vec<f64>] = match embeddings {
+        Some(given) if given.len() == windows.len() => given,
+        _ => {
+            lexical = windows
+                .iter()
+                .map(|window| vector::lexical(&window.text, vector::DIMENSIONS))
+                .collect();
+            &lexical
+        }
+    };
+
+    let similarities = vector::consecutive_cosine(vectors);
     let boundaries =
         segment::boundary_times(&windows, &similarities, duration, MIN_SEGMENT_SECONDS, None);
     segment::from_boundaries(&windows, &boundaries, duration)
 }
 
+/// The windows a recording is measured in, so a caller can embed them.
+///
+/// The text of each is what [`segment_with`] expects a vector for, and the
+/// order is the order the vectors must arrive in.
+pub fn windows_of(cues: &[Cue]) -> Vec<window::Window> {
+    window::windows(cues, WINDOW_SECONDS)
+}
+
 /// How many chapters a recording of this length should have.
 pub fn target_chapter_count(duration: f64) -> usize {
     let target = textkit::round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER);
@@ -72,6 +106,73 @@ pub fn target_chapter_count(duration: f64) -> usize {
 mod tests {
     use super::*;
 
+    /// A transcript that talks about two different things, in two halves.
+    fn two_halves() -> Vec<Cue> {
+        (0..1200)
+            .map(|i| Cue {
+                v: corpus_core::V,
+                start: i as f64 * 5.0,
+                end: i as f64 * 5.0 + 5.0,
+                text: match i < 600 {
+                    true => "the mill stood at the northern end of the valley".into(),
+                    false => "parliament dissolved itself again amid objections".into(),
+                },
+            })
+            .collect()
+    }
+
+    #[test]
+    fn supplied_vectors_are_used_instead_of_hashing_the_words() {
+        let cues = two_halves();
+        let windows = windows_of(&cues);
+        // Vectors that say every window is about the same thing. Nothing the
+        // words say can produce this, so if the answer changes, they were used.
+        let flat: Vec<Vec<f64>> = windows.iter().map(|_| vec![1.0, 0.0]).collect();
+        assert_ne!(
+            segment_with(&cues, 6000.0, Some(&flat)),
+            segment_with(&cues, 6000.0, None),
+            "supplied vectors made no difference"
+        );
+    }
+
+    #[test]
+    fn the_windows_to_embed_line_up_with_what_is_expected() {
+        let cues = two_halves();
+        let windows = windows_of(&cues);
+        let vectors: Vec<Vec<f64>> = windows
+            .iter()
+            .map(|w| vector::lexical(&w.text, vector::DIMENSIONS))
+            .collect();
+        // Handing back exactly what the lexical route would have computed
+        // must give exactly what the lexical route gives.
+        assert_eq!(
+            segment_with(&cues, 6000.0, Some(&vectors)),
+            segment(&cues, 6000.0)
+        );
+    }
+
+    #[test]
+    fn the_wrong_number_of_vectors_is_ignored_rather_than_trusted() {
+        // A caller that embedded some other recording's windows is asking a
+        // question about that recording, not this one.
+        let cues = two_halves();
+        let too_few = vec![vec![1.0, 0.0]; 2];
+        assert_eq!(
+            segment_with(&cues, 6000.0, Some(&too_few)),
+            segment(&cues, 6000.0)
+        );
+        assert_eq!(
+            segment_with(&cues, 6000.0, Some(&[])),
+            segment(&cues, 6000.0)
+        );
+    }
+
+    #[test]
+    fn segment_is_still_the_lexical_route() {
+        let cues = two_halves();
+        assert_eq!(segment(&cues, 6000.0), segment_with(&cues, 6000.0, None));
+    }
+
     #[test]
     fn the_chapter_count_follows_the_length_within_bounds() {
         assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS);

crates/cueseg/src/main.rs

@@ -34,6 +34,21 @@ struct Args {
     /// recording ends in music or silence.
     #[arg(long, value_name = "SECONDS")]
     duration: Option<f64>,
+
+    /// One vector per window, as a JSON array per line, in window order.
+    ///
+    /// Without this the windows are compared by the words they repeat, which
+    /// cannot tell that "ships" and "vessels" are the same subject. With it
+    /// they are compared however the vectors were made. `--windows` prints
+    /// the text to embed.
+    #[arg(long, value_name = "FILE")]
+    embeddings: Option<PathBuf>,
+
+    /// Print the window text, one per line, and stop.
+    ///
+    /// What to embed, in the order `--embeddings` expects it back.
+    #[arg(long)]
+    windows: bool,
 }
 
 fn main() -> ExitCode {
@@ -67,7 +82,58 @@ fn main() -> ExitCode {
         None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max),
     };
 
-    catena_cli::stream("cueseg", &cueseg::segment(&cues, duration))
+    if args.windows {
+        let text: String = cueseg::windows_of(&cues)
+            .iter()
+            .map(|window| format!("{}\n", window.text.replace('\n', " ")))
+            .collect();
+        return catena_cli::text("cueseg", &text);
+    }
+
+    let embeddings = match args.embeddings.as_deref() {
+        None => None,
+        Some(path) => match read_embeddings(path) {
+            Ok(vectors) => Some(vectors),
+            Err((code, why)) => {
+                eprintln!("cueseg: {why}");
+                return ExitCode::from(code);
+            }
+        },
+    };
+
+    let windows = cueseg::windows_of(&cues).len();
+    if let Some(vectors) = embeddings.as_deref() {
+        if vectors.len() != windows {
+            eprintln!(
+                "cueseg: {} vectors for {windows} windows — run --windows to see what to embed",
+                vectors.len()
+            );
+            return ExitCode::from(exit::DATA);
+        }
+    }
+
+    let chapters = cueseg::segment_with(&cues, duration, embeddings.as_deref());
+    catena_cli::stream("cueseg", &chapters)
+}
+
+/// One vector per line, as `emb` writes them.
+fn read_embeddings(path: &std::path::Path) -> Result<Vec<Vec<f64>>, (u8, String)> {
+    let text = std::fs::read_to_string(path)
+        .map_err(|e| (exit::IO, format!("{}: {e}", path.display())))?;
+    let mut out = Vec::new();
+    for (number, line) in text.lines().enumerate() {
+        if line.trim().is_empty() {
+            continue;
+        }
+        let vector: Vec<f64> = serde_json::from_str(line).map_err(|e| {
+            (
+                exit::DATA,
+                format!("{}: line {}: {e}", path.display(), number + 1),
+            )
+        })?;
+        out.push(vector);
+    }
+    Ok(out)
 }
 
 /// Read cues from either encoding.