diff --git a/crates/bindings/README.md b/crates/bindings/README.md index 7c51632..f5fbb27 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -42,6 +42,13 @@ cues = catena.parse_subtitles(open("episode.srt").read()) passages = catena.pack_cues(cues, seconds=75) catena.identify("atlas.pdf", open("atlas.pdf", "rb").read(1024))["kind"] + +# Chapters, compared by the words a window repeats … +chapters = catena.segment_cues(cues, duration=7200) + +# … or by vectors you made, which know that ships and vessels are one subject. +vectors = embed(catena.cue_windows(cues)) +chapters = catena.segment_cues(cues, duration=7200, embeddings=vectors) ``` `split_collection` raises `ValueError` when the document is one work rather @@ -65,6 +72,7 @@ except ValueError as why: | `parse_subtitles` | [`cuenorm`](../cuenorm/README.md) | | `pack_cues` | [`cuepack`](../cuepack/README.md) | | `segment_cues` | [`cueseg`](../cueseg/README.md) | +| `cue_windows` | the windows `segment_cues` measures, to embed | | `split_collection` | [`mdsplit`](../mdsplit/README.md) | | `fuse_hits` | [`rrf`](../rrf/README.md) | | `identify` | [`booksniff`](../booksniff/README.md) | diff --git a/crates/bindings/include/catena.h b/crates/bindings/include/catena.h index ef12fca..baab238 100644 --- a/crates/bindings/include/catena.h +++ b/crates/bindings/include/catena.h @@ -25,7 +25,7 @@ extern "C" { /* Bumped when the meaning or signature of anything here changes. Additions * leave it alone. Check it against CATENA_ABI_VERSION at load time. */ -#define CATENA_ABI_VERSION 1u +#define CATENA_ABI_VERSION 2u unsigned int catena_abi_version(void); @@ -58,6 +58,19 @@ char *catena_parse_subtitles(const char *content, int keep_repeats); * budget means the default. Returns a JSON array of chunks. */ char *catena_pack_cues(const char *cues, double seconds, size_t max_tokens); +/* Cut a timed transcript into topic chapters. `cues` is a JSON array of + * cues. `embeddings` is a JSON array of one vector per window, or NULL to + * compare windows by the words they repeat. A duration of zero or less is + * taken from the last cue. Returns a JSON array of segments. */ +char *catena_segment_cues(const char *cues, + double duration, + const char *embeddings); + +/* The text of each window a transcript is measured in, in the order + * catena_segment_cues expects vectors back. Returns a JSON array of + * strings. */ +char *catena_cue_windows(const char *cues); + /* Split a collection into the works inside it. Returns a JSON array of * sections, or NULL when the document is one work rather than a collection — * an answer, not a failure; the reason is in catena_last_error. */ diff --git a/crates/bindings/npm/README.md b/crates/bindings/npm/README.md index 005e15d..b067f64 100644 --- a/crates/bindings/npm/README.md +++ b/crates/bindings/npm/README.md @@ -46,6 +46,7 @@ try { | `parseSubtitles` | SubRip, WebVTT or ASS into cues | | `packCues` | Cues into passages, keeping the times they were said at | | `segmentCues` | A timed transcript into topic chapters | +| `cueWindows` | The windows `segmentCues` measures, to embed | | `splitCollection` | One markdown file into the works inside it | | `fuseHits` | Several ranked lists into one | | `identify` | What a file is, from its first bytes | diff --git a/crates/bindings/npm/typecheck.ts b/crates/bindings/npm/typecheck.ts index b8af201..85669c7 100644 --- a/crates/bindings/npm/typecheck.ts +++ b/crates/bindings/npm/typecheck.ts @@ -3,7 +3,7 @@ // check, and nothing else would notice. import { chunkMarkdown, scrub, countTokens, parseSubtitles, packCues, - splitCollection, fuseHits, identify, seriesOf, parseIdentifier, segmentCues, + splitCollection, fuseHits, identify, seriesOf, parseIdentifier, segmentCues, cueWindows, type Chunk, type Cue, type Section, type Hit, type Doc, type SeriesProposal, type Identifier, type Segment, } from "./dist/web/catena.js"; @@ -20,6 +20,8 @@ const start: number = cues[0].start; const packed: Chunk[] = packCues(cues, 75, 512); const chapters: Segment[] = segmentCues(cues, 3600); +const toEmbed: string[] = cueWindows(cues); +segmentCues(cues, 3600, toEmbed.map(() => [1, 0])); const chapterTitle: string = chapters[0].title; segmentCues(cues); diff --git a/crates/bindings/python/catena/__init__.py b/crates/bindings/python/catena/__init__.py index ca28915..73685f9 100644 --- a/crates/bindings/python/catena/__init__.py +++ b/crates/bindings/python/catena/__init__.py @@ -12,6 +12,7 @@ lines of JSON. from ._catena import ( __version__, + cue_windows, chunk_markdown, count_tokens, fuse_hits, @@ -27,6 +28,7 @@ from ._catena import ( __all__ = [ "__version__", + "cue_windows", "chunk_markdown", "count_tokens", "fuse_hits", diff --git a/crates/bindings/python/catena/__init__.pyi b/crates/bindings/python/catena/__init__.pyi index c37ae67..39a1180 100644 --- a/crates/bindings/python/catena/__init__.pyi +++ b/crates/bindings/python/catena/__init__.pyi @@ -22,8 +22,12 @@ def pack_cues( max_tokens: int | None = ..., ) -> list[dict[str, Any]]: ... def segment_cues( - cues: Iterable[Mapping[str, Any]], *, duration: float | None = ... + cues: Iterable[Mapping[str, Any]], + *, + duration: float | None = ..., + embeddings: Sequence[Sequence[float]] | None = ..., ) -> list[dict[str, Any]]: ... +def cue_windows(cues: Iterable[Mapping[str, Any]]) -> list[str]: ... def split_collection(markdown: str) -> list[dict[str, Any]]: ... def fuse_hits( hits: Iterable[Mapping[str, Any]], diff --git a/crates/bindings/src/capi.rs b/crates/bindings/src/capi.rs index 7b18224..956c327 100644 --- a/crates/bindings/src/capi.rs +++ b/crates/bindings/src/capi.rs @@ -30,7 +30,9 @@ use crate::{ChunkBudgets, CueBudgets}; /// Bumped when the meaning or signature of anything here changes. Additions /// leave it alone. -const ABI_VERSION: c_uint = 1; +/// +/// 2: `catena_segment_cues` gained an embeddings argument. +const ABI_VERSION: c_uint = 2; thread_local! { static LAST_ERROR: RefCell> = const { RefCell::new(None) }; @@ -271,6 +273,7 @@ pub unsafe extern "C" fn catena_pack_cues( pub unsafe extern "C" fn catena_segment_cues( cues: *const c_char, duration: c_double, + embeddings: *const c_char, ) -> *mut c_char { clear_error(); let Some(cues) = (unsafe { borrow(cues, "cues") }) else { @@ -279,7 +282,42 @@ pub unsafe extern "C" fn catena_segment_cues( let Some(cues) = take_json::>(cues, "cues") else { return ptr::null_mut(); }; - give_json(&crate::segment_cues(&cues, duration)) + + let vectors: Option>> = match embeddings.is_null() { + true => None, + false => { + let Some(text) = (unsafe { borrow(embeddings, "embeddings") }) else { + return ptr::null_mut(); + }; + match take_json::>>(text, "embeddings") { + Some(vectors) => Some(vectors), + None => return ptr::null_mut(), + } + } + }; + give_json(&crate::segment_cues_with( + &cues, + duration, + vectors.as_deref(), + )) +} + +/// The text of each window a transcript is measured in, to embed. Returns a +/// JSON array of strings. +/// +/// # Safety +/// +/// `cues` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_cue_windows(cues: *const c_char) -> *mut c_char { + clear_error(); + let Some(cues) = (unsafe { borrow(cues, "cues") }) else { + return ptr::null_mut(); + }; + let Some(cues) = take_json::>(cues, "cues") else { + return ptr::null_mut(); + }; + give_json(&crate::cue_windows(&cues)) } /// Split a collection into the works inside it. Returns a JSON array of @@ -550,6 +588,6 @@ mod tests { #[test] fn the_abi_version_is_the_one_the_header_declares() { - assert_eq!(catena_abi_version(), 1); + assert_eq!(catena_abi_version(), 2); } } diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 0b586ee..0d154b9 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -133,11 +133,32 @@ pub fn pack_cues(cues: &[Cue], budgets: CueBudgets) -> Vec { /// `duration` is how long the recording is, in seconds; it decides how many /// chapters to aim for. Zero or less means take it from the last cue. pub fn segment_cues(cues: &[Cue], duration: f64) -> Vec { + segment_cues_with(cues, duration, None) +} + +/// The same, given vectors somebody else computed for the windows. +/// +/// Without them the windows are compared by the words they repeat, which +/// cannot tell that "ships" and "vessels" are one subject. [`cue_windows`] +/// gives the text to embed, in the order the vectors must come back in. +pub fn segment_cues_with( + cues: &[Cue], + duration: f64, + embeddings: Option<&[Vec]>, +) -> Vec { let duration = match duration > 0.0 { true => duration, false => cues.last().map_or(0.0, |cue| cue.end), }; - cueseg::segment(cues, duration) + cueseg::segment_with(cues, duration, embeddings) +} + +/// The text of each window a transcript is measured in, to embed. +pub fn cue_windows(cues: &[Cue]) -> Vec { + cueseg::windows_of(cues) + .into_iter() + .map(|window| window.text) + .collect() } /// Split a collection into the works inside it. @@ -236,24 +257,37 @@ mod tests { #[test] fn a_transcript_comes_back_as_chapters() { - // Long enough to be worth chaptering at all: the segmenter refuses - // to cut a recording into pieces shorter than its minimum. + // Two halves about two different things, so the words themselves say + // where the boundary is. let cues: Vec = (0..1200) .map(|i| Cue { v: corpus_core::V, start: i as f64 * 5.0, end: i as f64 * 5.0 + 5.0, - text: match i % 2 { - 0 => "the mill stood at the northern end of the valley".into(), - _ => "parliament dissolved itself again amid objections".into(), + text: match i < 600 { + true => "the mill stood at the northern end of the valley".into(), + false => "parliament dissolved itself again amid objections".into(), }, }) .collect(); + let chapters = segment_cues(&cues, 6000.0); assert!(!chapters.is_empty()); assert_eq!(chapters, cueseg::segment(&cues, 6000.0)); // The duration can be left to the cues themselves. - assert_eq!(segment_cues(&cues, 0.0), cueseg::segment(&cues, 6000.0)); + assert_eq!(segment_cues(&cues, 0.0).len(), chapters.len()); + + // The windows to embed are the ones the segmenter measures, and + // vectors given for them are used rather than the words. + let windows = cue_windows(&cues); + assert_eq!(windows.len(), cueseg::windows_of(&cues).len()); + let flat: Vec> = windows.iter().map(|_| vec![1.0, 0.0]).collect(); + let with_flat = segment_cues_with(&cues, 6000.0, Some(&flat)); + assert_ne!( + with_flat.len(), + chapters.len(), + "vectors saying every window is alike gave the same chapters as the words" + ); } #[test] diff --git a/crates/bindings/src/python.rs b/crates/bindings/src/python.rs index 7e0d732..87e5c78 100644 --- a/crates/bindings/src/python.rs +++ b/crates/bindings/src/python.rs @@ -162,15 +162,28 @@ fn pack_cues<'py>( /// /// `duration` is the recording's length in seconds; omit it and the last cue /// decides. +/// `embeddings` is one vector per window, in the order `cue_windows` gives +/// them; without it the windows are compared by the words they repeat. #[pyfunction] -#[pyo3(signature = (cues, *, duration = None))] +#[pyo3(signature = (cues, *, duration = None, embeddings = None))] fn segment_cues<'py>( py: Python<'py>, cues: &Bound<'py, PyAny>, duration: Option, + embeddings: Option>>, ) -> PyResult> { let cues: Vec = from_py_list(cues)?; - list_to_py(py, &crate::segment_cues(&cues, duration.unwrap_or(0.0))) + list_to_py( + py, + &crate::segment_cues_with(&cues, duration.unwrap_or(0.0), embeddings.as_deref()), + ) +} + +/// The text of each window a transcript is measured in, to embed. +#[pyfunction] +fn cue_windows(cues: &Bound<'_, PyAny>) -> PyResult> { + let cues: Vec = from_py_list(cues)?; + Ok(crate::cue_windows(&cues)) } /// Split a collection into the works inside it. @@ -244,6 +257,7 @@ fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_subtitles, module)?)?; module.add_function(wrap_pyfunction!(pack_cues, module)?)?; module.add_function(wrap_pyfunction!(segment_cues, module)?)?; + module.add_function(wrap_pyfunction!(cue_windows, module)?)?; module.add_function(wrap_pyfunction!(split_collection, module)?)?; module.add_function(wrap_pyfunction!(fuse_hits, module)?)?; module.add_function(wrap_pyfunction!(identify, module)?)?; diff --git a/crates/bindings/src/wasm.rs b/crates/bindings/src/wasm.rs index 3fd6874..3603137 100644 --- a/crates/bindings/src/wasm.rs +++ b/crates/bindings/src/wasm.rs @@ -212,10 +212,31 @@ pub fn pack_cues( /// /// `duration` is the recording's length in seconds; omit it and the last cue /// decides. +/// `embeddings` is one vector per window, in the order `cueWindows` gives +/// them; without it the windows are compared by the words they repeat. #[wasm_bindgen(js_name = segmentCues, skip_typescript)] -pub fn segment_cues(cues: JsValue, duration: Option) -> Result { +pub fn segment_cues( + cues: JsValue, + duration: Option, + embeddings: JsValue, +) -> Result { let cues: Vec = from_js(cues, "cues")?; - to_js(&crate::segment_cues(&cues, duration.unwrap_or(0.0))) + let vectors: Option>> = match embeddings.is_undefined() || embeddings.is_null() { + true => None, + false => Some(from_js(embeddings, "embeddings")?), + }; + to_js(&crate::segment_cues_with( + &cues, + duration.unwrap_or(0.0), + vectors.as_deref(), + )) +} + +/// The text of each window a transcript is measured in, to embed. +#[wasm_bindgen(js_name = cueWindows, skip_typescript)] +pub fn cue_windows(cues: JsValue) -> Result { + let cues: Vec = from_js(cues, "cues")?; + to_js(&crate::cue_windows(&cues)) } /// Split a collection into the works inside it. @@ -288,7 +309,8 @@ const SIGNATURES: &'static str = r#" export function chunkMarkdown(markdown: string, target?: number, overlap?: number, min_tokens?: number, hard_max?: number): Chunk[]; export function parseSubtitles(content: string, keep_repeats?: boolean): Cue[]; export function packCues(cues: Cue[], seconds?: number, max_tokens?: number): Chunk[]; -export function segmentCues(cues: Cue[], duration?: number): Segment[]; +export function segmentCues(cues: Cue[], duration?: number, embeddings?: number[][]): Segment[]; +export function cueWindows(cues: Cue[]): string[]; export function splitCollection(markdown: string): Section[]; export function fuseHits(hits: Hit[], k?: number, weights?: Record, limit?: number): Hit[]; export function identify(name: string, data: Uint8Array): Doc; diff --git a/crates/cueseg/Cargo.toml b/crates/cueseg/Cargo.toml index f82f500..a8e9c60 100644 --- a/crates/cueseg/Cargo.toml +++ b/crates/cueseg/Cargo.toml @@ -16,6 +16,7 @@ fancy-regex.workspace = true regex.workspace = true schemars.workspace = true serde.workspace = true +serde_json.workspace = true textkit.workspace = true [dev-dependencies] diff --git a/crates/cueseg/README.md b/crates/cueseg/README.md index f254613..43f886b 100644 --- a/crates/cueseg/README.md +++ b/crates/cueseg/README.md @@ -64,6 +64,8 @@ cueseg --duration 7245 transcript.ndjson | jq -r '"\(.start | floor)\t\(.title)" | Option | Effect | |---|---| +| `--windows` | Print the text to embed, one window per line, and stop | +| `--embeddings ` | One vector per window, as `emb` writes them | | `--duration ` | How long the recording is | | `--man` | Write this page as a manual page, in roff | @@ -72,6 +74,33 @@ earlier than the end of the file whenever a recording ends in music or silence, and a chapter that claims to run to the end of the transcript rather than the end of the file is a mark that lands in the wrong place. +## Better boundaries, if you have embeddings + +By default the windows are compared by the words they repeat. That is fast, +needs nothing, and cannot tell that a stretch about *ships* and a stretch +about *vessels* are about one thing. An embedding can. + +`--windows` prints the text to embed, one window per line; `--embeddings` +takes one vector per line back, in the same order: + +```sh +cueseg --windows episode.srt | emb --model nomic-embed-text > windows.ndjson +cueseg --embeddings windows.ndjson episode.srt +``` + +The count has to match, and a mismatch is refused rather than guessed at: + +``` +$ cueseg --embeddings two-vectors.ndjson episode.srt +cueseg: 2 vectors for 44 windows — run --windows to see what to embed +$ echo $? +65 +``` + +Everything else is the same: the same windows, the same threshold, the same +minimum chapter length. Only the question "are these two stretches about the +same thing" is answered better. + ## Exit codes | Code | Meaning | diff --git a/crates/cueseg/src/lib.rs b/crates/cueseg/src/lib.rs index bd6e196..7748522 100644 --- a/crates/cueseg/src/lib.rs +++ b/crates/cueseg/src/lib.rs @@ -48,20 +48,54 @@ pub const EXCERPT_CHARS: usize = 900; /// to the end of the transcript rather than the end of the file is a chapter /// mark that lands in the wrong place. pub fn segment(cues: &[Cue], duration: f64) -> Vec { + segment_with(cues, duration, None) +} + +/// The same, given vectors somebody else computed for the windows. +/// +/// The hashed bag of words this falls back on asks only "are these two +/// stretches about the same thing", and answers it from repeated words. A +/// real embedding answers the same question far better, because it knows +/// that a stretch about *ships* and a stretch about *vessels* are about one +/// thing, which no amount of word counting will tell you. +/// +/// One vector per window, in order. [`windows_of`] gives the windows to embed +/// so a caller can produce them without reimplementing the pass. A slice of +/// the wrong length is ignored rather than trusted — a caller that embedded +/// a different set of windows is asking a question about some other +/// recording — and the lexical route stands in. +pub fn segment_with(cues: &[Cue], duration: f64, embeddings: Option<&[Vec]>) -> Vec { let windows = window::windows(cues, WINDOW_SECONDS); if windows.is_empty() { return Vec::new(); } - let vectors: Vec> = windows - .iter() - .map(|window| vector::lexical(&window.text, vector::DIMENSIONS)) - .collect(); - let similarities = vector::consecutive_cosine(&vectors); + + let lexical: Vec>; + let vectors: &[Vec] = match embeddings { + Some(given) if given.len() == windows.len() => given, + _ => { + lexical = windows + .iter() + .map(|window| vector::lexical(&window.text, vector::DIMENSIONS)) + .collect(); + &lexical + } + }; + + let similarities = vector::consecutive_cosine(vectors); let boundaries = segment::boundary_times(&windows, &similarities, duration, MIN_SEGMENT_SECONDS, None); segment::from_boundaries(&windows, &boundaries, duration) } +/// The windows a recording is measured in, so a caller can embed them. +/// +/// The text of each is what [`segment_with`] expects a vector for, and the +/// order is the order the vectors must arrive in. +pub fn windows_of(cues: &[Cue]) -> Vec { + window::windows(cues, WINDOW_SECONDS) +} + /// How many chapters a recording of this length should have. pub fn target_chapter_count(duration: f64) -> usize { let target = textkit::round_half_to_even(duration / TARGET_SECONDS_PER_CHAPTER); @@ -72,6 +106,73 @@ pub fn target_chapter_count(duration: f64) -> usize { mod tests { use super::*; + /// A transcript that talks about two different things, in two halves. + fn two_halves() -> Vec { + (0..1200) + .map(|i| Cue { + v: corpus_core::V, + start: i as f64 * 5.0, + end: i as f64 * 5.0 + 5.0, + text: match i < 600 { + true => "the mill stood at the northern end of the valley".into(), + false => "parliament dissolved itself again amid objections".into(), + }, + }) + .collect() + } + + #[test] + fn supplied_vectors_are_used_instead_of_hashing_the_words() { + let cues = two_halves(); + let windows = windows_of(&cues); + // Vectors that say every window is about the same thing. Nothing the + // words say can produce this, so if the answer changes, they were used. + let flat: Vec> = windows.iter().map(|_| vec![1.0, 0.0]).collect(); + assert_ne!( + segment_with(&cues, 6000.0, Some(&flat)), + segment_with(&cues, 6000.0, None), + "supplied vectors made no difference" + ); + } + + #[test] + fn the_windows_to_embed_line_up_with_what_is_expected() { + let cues = two_halves(); + let windows = windows_of(&cues); + let vectors: Vec> = windows + .iter() + .map(|w| vector::lexical(&w.text, vector::DIMENSIONS)) + .collect(); + // Handing back exactly what the lexical route would have computed + // must give exactly what the lexical route gives. + assert_eq!( + segment_with(&cues, 6000.0, Some(&vectors)), + segment(&cues, 6000.0) + ); + } + + #[test] + fn the_wrong_number_of_vectors_is_ignored_rather_than_trusted() { + // A caller that embedded some other recording's windows is asking a + // question about that recording, not this one. + let cues = two_halves(); + let too_few = vec![vec![1.0, 0.0]; 2]; + assert_eq!( + segment_with(&cues, 6000.0, Some(&too_few)), + segment(&cues, 6000.0) + ); + assert_eq!( + segment_with(&cues, 6000.0, Some(&[])), + segment(&cues, 6000.0) + ); + } + + #[test] + fn segment_is_still_the_lexical_route() { + let cues = two_halves(); + assert_eq!(segment(&cues, 6000.0), segment_with(&cues, 6000.0, None)); + } + #[test] fn the_chapter_count_follows_the_length_within_bounds() { assert_eq!(target_chapter_count(0.0), MIN_CHAPTERS); diff --git a/crates/cueseg/src/main.rs b/crates/cueseg/src/main.rs index 8befb91..f1815d4 100644 --- a/crates/cueseg/src/main.rs +++ b/crates/cueseg/src/main.rs @@ -34,6 +34,21 @@ struct Args { /// recording ends in music or silence. #[arg(long, value_name = "SECONDS")] duration: Option, + + /// One vector per window, as a JSON array per line, in window order. + /// + /// Without this the windows are compared by the words they repeat, which + /// cannot tell that "ships" and "vessels" are the same subject. With it + /// they are compared however the vectors were made. `--windows` prints + /// the text to embed. + #[arg(long, value_name = "FILE")] + embeddings: Option, + + /// Print the window text, one per line, and stop. + /// + /// What to embed, in the order `--embeddings` expects it back. + #[arg(long)] + windows: bool, } fn main() -> ExitCode { @@ -67,7 +82,58 @@ fn main() -> ExitCode { None => cues.iter().map(|cue| cue.end).fold(0.0, f64::max), }; - catena_cli::stream("cueseg", &cueseg::segment(&cues, duration)) + if args.windows { + let text: String = cueseg::windows_of(&cues) + .iter() + .map(|window| format!("{}\n", window.text.replace('\n', " "))) + .collect(); + return catena_cli::text("cueseg", &text); + } + + let embeddings = match args.embeddings.as_deref() { + None => None, + Some(path) => match read_embeddings(path) { + Ok(vectors) => Some(vectors), + Err((code, why)) => { + eprintln!("cueseg: {why}"); + return ExitCode::from(code); + } + }, + }; + + let windows = cueseg::windows_of(&cues).len(); + if let Some(vectors) = embeddings.as_deref() { + if vectors.len() != windows { + eprintln!( + "cueseg: {} vectors for {windows} windows — run --windows to see what to embed", + vectors.len() + ); + return ExitCode::from(exit::DATA); + } + } + + let chapters = cueseg::segment_with(&cues, duration, embeddings.as_deref()); + catena_cli::stream("cueseg", &chapters) +} + +/// One vector per line, as `emb` writes them. +fn read_embeddings(path: &std::path::Path) -> Result>, (u8, String)> { + let text = std::fs::read_to_string(path) + .map_err(|e| (exit::IO, format!("{}: {e}", path.display())))?; + let mut out = Vec::new(); + for (number, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let vector: Vec = serde_json::from_str(line).map_err(|e| { + ( + exit::DATA, + format!("{}: line {}: {e}", path.display(), number + 1), + ) + })?; + out.push(vector); + } + Ok(out) } /// Read cues from either encoding.