examples: programs that run, in four languages

Eight of them, each doing something a pipeline actually has to do, and each
starting from something anyone can fetch — a public domain book from Project
Gutenberg, a NASA broadcast, or a sample carried in the file.

Shell, because the tools are filters first: a book to passages and what its
notes cite; a recording to chapter marks, citable passages and the one
ffmpeg call that makes it play on a phone; passages into pgvector and a
search that asks twice and reconciles.

Rust, for the case where the caller already holds the text and a subprocess
would be serialising for nothing. Python, for the shape a pipeline stage
takes once the reading is a library call. TypeScript, for the app side —
chapter marks and deep links, and fusing two rankings in process.

Three of them end with what re-ranking means, because it is two different
things under one name: fusing ranked lists, which is what `rrf` does and
needs no model, and a cross-encoder scoring query and passage together,
which is accurate, slow, and runs over what retrieval already found.

examples/typescript/src/episode-to-chapters.ts

@@ -0,0 +1,95 @@
+// A recording's captions into chapter marks and citable passages.
+//
+// What a player needs from an episode: somewhere to jump to, and passages
+// that carry the second they were said at so a search result can be played
+// rather than just quoted.
+//
+//     npm install
+//     npm run chapters -- path/to/captions.vtt
+//
+// With no argument it reads the sample below, which is a few cues of a NASA
+// broadcast — a work of the US government, so public domain.
+
+import { readFileSync } from "node:fs";
+import { parseSubtitles, segmentCues, packCues, countTokens } from "catena";
+
+const SAMPLE = `WEBVTT
+
+00:00:00.000 --> 00:00:04.200
+Hey there, and welcome to NASA's Curious Universe.
+
+00:00:04.200 --> 00:00:08.900
+Our universe is a wild and wonderful place, and in this
+show, NASA is your tour guide.
+
+00:00:08.900 --> 00:00:15.100
+Today we are going to talk about how we find planets
+around other stars, which is harder than it sounds.
+
+00:00:15.100 --> 00:00:21.800
+A planet is small and dark and its star is enormous and
+bright, so you almost never see the planet itself.
+
+00:00:21.800 --> 00:00:28.400
+Instead you watch the star, and you wait for it to dim
+by about one part in ten thousand.
+
+00:00:28.400 --> 00:00:34.000
+That dimming is a transit, and it is how most of the
+planets we know about were found.
+`;
+
+const path = process.argv[2];
+const content = path ? readFileSync(path, "utf8") : SAMPLE;
+
+// One cue stream, whatever the format was. `keepRepeats` is off, which is
+// what throws away the re-sent lines of a rolling automatic caption track —
+// leave it on and every passage says each sentence three times.
+const cues = parseSubtitles(content);
+const runsTo = Math.max(...cues.map((cue) => cue.end));
+console.log(`== ${cues.length} cues over ${Math.round(runsTo)}s`);
+for (const cue of cues.slice(0, 3)) {
+  console.log(`   [${cue.start.toFixed(1)}s] ${cue.text.slice(0, 58)}`);
+}
+
+// Where the subject changes, from the words alone — no model, no network.
+// The titles come out of each chapter's own first sentence and are meant to
+// be replaced by something better later.
+const chapters = segmentCues(cues);
+console.log(`\n== ${chapters.length} chapter(s)`);
+for (const chapter of chapters) {
+  console.log(`   ${stamp(chapter.start)}  ${chapter.title}`);
+}
+
+// Cues packed to a size worth embedding, each keeping the seconds it spans.
+const passages = packCues(cues);
+console.log(`\n== ${passages.length} passage(s)`);
+for (const passage of passages) {
+  console.log(
+    `   [${stamp(passage.start_time ?? 0)}–${stamp(passage.end_time ?? 0)}] ` +
+      `${passage.tokens} tokens`,
+  );
+  console.log(`      ${passage.text.slice(0, 92)}…`);
+}
+
+// What a player would take: a cue sheet, and a deep link per passage.
+console.log("\n== what the player gets");
+for (const chapter of chapters) {
+  console.log(`   ${stamp(chapter.start)} ${chapter.title}`);
+}
+console.log("\n== what a search result would link to");
+for (const passage of passages.slice(0, 2)) {
+  const at = Math.floor(passage.start_time ?? 0);
+  console.log(`   ?t=${at}  ${passage.text.slice(0, 64)}…`);
+}
+
+// The same token count the chunker used, so a budget worked out here and a
+// budget enforced there agree.
+console.log(`\n   ${countTokens(cues.map((c) => c.text).join(" "))} tokens of transcript in all`);
+
+function stamp(seconds: number): string {
+  const whole = Math.floor(seconds);
+  const mm = String(Math.floor(whole / 60)).padStart(2, "0");
+  const ss = String(whole % 60).padStart(2, "0");
+  return `${String(Math.floor(whole / 3600)).padStart(2, "0")}:${mm}:${ss}`;
+}

examples/typescript/src/search-and-rerank.ts

@@ -0,0 +1,96 @@
+// Two searches, reconciled — the app-side half of hybrid retrieval.
+//
+// Fusion is the one thing in this suite worth calling in process rather than
+// as a subprocess: by the time a hit reaches your code it is already yours,
+// and serialising it to hand to a program and back is work for nothing.
+//
+//     npm install
+//     npm run search
+//
+// The hits below stand in for two queries you would run against your own
+// store — pgvector and a text index, Elasticsearch and a KNN service, or a
+// vector database and whatever else. What they came from does not matter
+// here; only where each of them placed does.
+
+import { fuseHits, type Hit } from "catena";
+
+// `payload` is whatever you want carried alongside — rrf hands it back
+// untouched, so a hit can arrive here with the row it came from attached.
+const hit = (id: string, channel: string, rank: number, score: number): Hit => ({
+  v: 1,
+  id,
+  channel,
+  rank,
+  score,
+  payload: {},
+});
+
+// Nearest in the embedding space. Finds the passage about the fall of Rome
+// for a query about decline, sharing not one word with it.
+const byMeaning: Hit[] = [
+  hit("p39", "vec", 1, 0.21),
+  hit("p127", "vec", 2, 0.24),
+  hit("p462", "vec", 3, 0.26),
+  hit("p177", "vec", 4, 0.29),
+];
+
+// Nearest by the words themselves. Finds the passage that says the word
+// where the vectors drifted to a paraphrase — and finds the front matter,
+// which says "Roman Empire" more often than any chapter of the book does.
+const byWord: Hit[] = [
+  hit("p0", "fts", 1, 0.68),
+  hit("p462", "fts", 2, 0.51),
+  hit("p8", "fts", 3, 0.44),
+  hit("p71", "fts", 4, 0.4),
+];
+
+const text: Record<string, string> = {
+  p0: "The Project Gutenberg eBook of History of the Decline and Fall…",
+  p8: "In imitation of his classical models, Gibbon places Rome as the…",
+  p39: "In the second century of the Christian Æra, the empire of Rome…",
+  p71: "Tacit. Annal. iv. 5. Those who fix a regular proportion…",
+  p127: "It was by such institutions that the nations of the empire…",
+  p177: "Among many other passages, see Pliny on the extent of the…",
+  p462: "The received calculation of Varro assigns to the foundation…",
+};
+
+for (const [label, hits] of [["by meaning", byMeaning], ["by word", byWord]] as const) {
+  console.log(`\n== ${label}`);
+  for (const hit of hits) {
+    console.log(`   ${hit.rank}. [${hit.id}] ${hit.score.toFixed(2)}  ${text[hit.id]}`);
+  }
+}
+
+// The two score columns above mean different things — a cosine distance and
+// a ts_rank_cd — and no scaling makes them comparable. So the ranks are used
+// and the scores thrown away.
+console.log("\n== fused");
+for (const hit of fuseHits([...byMeaning, ...byWord], undefined, undefined, 5)) {
+  console.log(`   ${hit.rank}. [${hit.id}] ${hit.score.toFixed(4)}  ${text[hit.id]}`);
+}
+
+// One channel can be trusted more than the other where a corpus warrants it.
+console.log("\n== fused, trusting the vectors twice as much");
+for (const hit of fuseHits([...byMeaning, ...byWord], undefined, { vec: 2 }, 5)) {
+  console.log(`   ${hit.rank}. [${hit.id}] ${hit.score.toFixed(4)}  ${text[hit.id]}`);
+}
+
+console.log(`
+   Reciprocal rank fusion scores a hit by where it placed, not by what it
+   scored: 1/(k + rank) from each list that found it, added up. p462 is
+   third in one list and second in the other and wins, because agreement
+   between two different ways of looking is evidence; p0 is first in one
+   and absent from the other, and drops.
+
+   This is fusion, and it is often called re-ranking, but a second thing
+   wears the same name. Retrieval uses a bi-encoder: the passage and the
+   query are embedded separately, which is what lets passage vectors be
+   computed once and indexed — and is also what limits it, since nothing in
+   a passage's vector knows what was asked.
+
+   A cross-encoder reads the query and the passage together and scores the
+   pair. Far more accurate, far too slow to run over a corpus, so it runs
+   over the twenty or fifty candidates retrieval already found. The two
+   stack: retrieve both ways, fuse as above, then hand the top of the fused
+   list to a cross-encoder for the final order. Nothing here loads a model,
+   so that step is yours — the fused list is what you would hand it.`);