bindings: a C ABI and an npm package alongside the wheel

crates/bindings/npm/README.md

@@ -0,0 +1,69 @@
+# catena
+
+Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles
+and rank fusion, as WebAssembly.
+
+```sh
+npm install catena
+```
+
+```js
+import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena";
+
+const chunks = chunkMarkdown(markdown, 512, 64);
+chunks[0].page;      // 41
+chunks[0].chapter;   // 'III. The Second Estate'
+
+const clean = scrub(rawMarkdown, true);
+countTokens(clean);
+
+fuseHits([...vectorHits, ...textHits], undefined, { vec: 2 }, 20);
+```
+
+Records come back as plain objects — the same documents the command-line
+tools write, with the same field names and the same stability promise.
+Nothing has to be freed.
+
+`splitCollection` throws when the document is one work rather than a
+collection, which is the usual answer:
+
+```js
+let works;
+try {
+  works = splitCollection(markdown);
+} catch (why) {
+  works = [{ idx: 0, title, text: markdown }];
+}
+```
+
+## What is here
+
+| | |
+|---|---|
+| `chunkMarkdown` | Markdown into passages that carry their page, chapter and role |
+| `scrub` | Converted markdown in, clean markdown out |
+| `countTokens` | Tokens, by the encoding the budgets count in |
+| `parseSubtitles` | SubRip, WebVTT or ASS into cues |
+| `packCues` | Cues into passages, keeping the times they were said at |
+| `splitCollection` | One markdown file into the works inside it |
+| `fuseHits` | Several ranked lists into one |
+| `identify` | What a file is, from its first bytes |
+
+The tools that read files or call another program are not here: a browser has
+neither, and a binding that pretends otherwise fails at run time rather than
+at compile time. Those stay command-line tools.
+
+## Node and the web
+
+```js
+import { chunkMarkdown } from "catena";        // resolves per environment
+import { chunkMarkdown } from "catena/node";   // or pick one
+import { chunkMarkdown } from "catena/web";
+```
+
+Two builds because the loaders differ: node reads the `.wasm` off disk at
+import, the web build takes a URL a bundler rewrites.
+
+## Licence
+
+MPL-2.0. Source: <https://git.pumpin.online/funkpower/catena>

crates/bindings/npm/build.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+# Build the npm package: one wasm module per target, and a package.json that
+# points at both.
+#
+# Two builds because the loaders differ. The node build reads the .wasm off
+# disk at import; the web build takes it as a URL a bundler rewrites. One
+# build serving both means a conditional import that every bundler gets wrong
+# in a different way.
+set -eu
+
+cd "$(dirname "$0")"
+crate=..
+out="$(pwd)/dist"
+
+command -v wasm-pack >/dev/null 2>&1 || {
+	echo "npm/build.sh: wasm-pack is not on PATH" >&2
+	exit 69
+}
+
+version=$(grep -m1 '^version' "$crate/../../Cargo.toml" | sed 's/.*"\(.*\)".*/\1/')
+
+rm -rf "$out"
+mkdir -p "$out"
+
+for target in nodejs web; do
+	case "$target" in
+	nodejs) dir=node ;;
+	*) dir=web ;;
+	esac
+	wasm-pack build "$crate" \
+		--target "$target" \
+		--out-dir "$out/$dir" \
+		--out-name catena \
+		--release \
+		-- --features wasm --no-default-features
+	# wasm-pack writes one of these per target and they disagree; the
+	# package.json below is the one that ships.
+	rm -f "$out/$dir/package.json" "$out/$dir/README.md" "$out/$dir/.gitignore"
+done
+
+# The node build is CommonJS and the package declares itself ESM, so the entry
+# has to say so in its extension or node reads `exports.x = …` as a syntax
+# error. The web build is already ESM and keeps its .js.
+mv "$out/node/catena.js" "$out/node/catena.cjs"
+mv "$out/node/catena.d.ts" "$out/node/catena.d.cts"
+
+sed "s/@VERSION@/$version/" package.json.in >"$out/package.json"
+cp README.md "$out/README.md"
+cp ../LICENSE "$out/LICENSE"
+
+echo "built $out (version $version)"
+echo "publish with: npm publish $out --access public"

crates/bindings/npm/package.json.in

@@ -0,0 +1,26 @@
+{
+  "name": "catena",
+  "version": "@VERSION@",
+  "description": "Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles and rank fusion, as WebAssembly",
+  "license": "MPL-2.0",
+  "homepage": "https://git.pumpin.online/funkpower/catena",
+  "repository": {
+    "type": "git",
+    "url": "git+https://git.pumpin.online/funkpower/catena.git"
+  },
+  "keywords": ["chunking", "markdown", "subtitles", "retrieval", "corpus", "tokens", "wasm"],
+  "type": "module",
+  "types": "./web/catena.d.ts",
+  "exports": {
+    ".": {
+      "node": { "types": "./node/catena.d.cts", "default": "./node/catena.cjs" },
+      "default": { "types": "./web/catena.d.ts", "default": "./web/catena.js" }
+    },
+    "./node": { "types": "./node/catena.d.cts", "default": "./node/catena.cjs" },
+    "./web": { "types": "./web/catena.d.ts", "default": "./web/catena.js" },
+    "./node/*": "./node/*",
+    "./web/*": "./web/*",
+    "./package.json": "./package.json"
+  },
+  "files": ["node/", "web/", "README.md", "LICENSE"]
+}

crates/bindings/npm/test.mjs

@@ -0,0 +1,36 @@
+// Exercise the built package the way a consumer would.
+import {
+  chunkMarkdown, scrub, countTokens, parseSubtitles, packCues,
+  splitCollection, fuseHits, identify,
+} from "./dist/node/catena.cjs";
+
+const md = "# Chapter One\n\n<!-- Page 41 -->\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked about it at all.\n";
+
+const chunks = chunkMarkdown(md, undefined, undefined, 10);
+console.log("chunk:     ", { idx: chunks[0].idx, page: chunks[0].page, chapter: chunks[0].chapter, tokens: chunks[0].tokens });
+
+console.log("scrub:     ", JSON.stringify(scrub("# Heading\n\n\n\nbody <b>text</b>\n")));
+console.log("tokens:    ", countTokens("a passage of prose"));
+
+const cues = parseSubtitles("1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n2\n00:00:04,000 --> 00:00:08,000\nof the valley where the road turned\n\n");
+console.log("cues:      ", cues.length, cues[0]);
+console.log("packed:    ", packCues(cues, 5)[0].text);
+
+const hits = [
+  { v: 1, id: "c81a", channel: "vec", rank: 1, score: 0.83, payload: {} },
+  { v: 1, id: "c81a", channel: "fts", rank: 3, score: 11.2, payload: {} },
+  { v: 1, id: "d42f", channel: "fts", rank: 1, score: 14.0, payload: {} },
+];
+console.log("fused:     ", fuseHits(hits, undefined, undefined, 2).map((h) => [h.id, +h.score.toFixed(5)]));
+console.log("weighted:  ", fuseHits(hits, undefined, { vec: 2 }, 2).map((h) => [h.id, +h.score.toFixed(5)]));
+
+console.log("identify:  ", identify("book.pdf", new TextEncoder().encode("%PDF-1.4\n")).kind);
+
+try { splitCollection(md); } catch (why) { console.log("split throws:", String(why).slice(0, 52)); }
+
+// The types the package promises.
+const types = await import("node:fs").then((fs) => fs.readFileSync("./dist/web/catena.d.ts", "utf8"));
+for (const name of ["Chunk", "Cue", "Section", "Hit", "Doc"]) {
+  if (!types.includes(`interface ${name}`)) throw new Error(`no ${name} in the declarations`);
+}
+console.log("types:      Chunk, Cue, Section, Hit, Doc all declared");