From 48600bd90c1865fb5769181cfeacc4c165e168d4 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 18:40:08 -0400 Subject: [PATCH] bindings: a C ABI and an npm package alongside the wheel --- .forgejo/workflows/ci.yml | 2 + .gitignore | 3 + Cargo.lock | 111 +++++++ Cargo.toml | 2 + README.md | 24 +- check.sh | 9 + crates/bindings/Cargo.toml | 35 ++- crates/bindings/README.md | 89 +++++- crates/bindings/include/catena.h | 82 +++++ crates/bindings/npm/README.md | 69 +++++ crates/bindings/npm/build.sh | 52 ++++ crates/bindings/npm/package.json.in | 26 ++ crates/bindings/npm/test.mjs | 36 +++ crates/bindings/src/capi.rs | 462 ++++++++++++++++++++++++++++ crates/bindings/src/lib.rs | 8 + crates/bindings/src/wasm.rs | 221 +++++++++++++ 16 files changed, 1221 insertions(+), 10 deletions(-) create mode 100644 crates/bindings/include/catena.h create mode 100644 crates/bindings/npm/README.md create mode 100755 crates/bindings/npm/build.sh create mode 100644 crates/bindings/npm/package.json.in create mode 100644 crates/bindings/npm/test.mjs create mode 100644 crates/bindings/src/capi.rs create mode 100644 crates/bindings/src/wasm.rs diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 4cc30dc..d076664 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -53,6 +53,8 @@ jobs: apt-get update -qq apt-get install -qq -y --no-install-recommends python3 >/dev/null rustup component add rustfmt clippy + # The WebAssembly binding is checked for its own target. + rustup target add wasm32-unknown-unknown - name: Check run: ./check.sh diff --git a/.gitignore b/.gitignore index 6006400..323264e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ target/ *.profraw .DS_Store + +# Built npm package; `crates/bindings/npm/build.sh` makes it. +crates/bindings/npm/dist/ diff --git a/Cargo.lock b/Cargo.lock index 51b3516..3f0aba5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -193,6 +193,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytecount" version = "0.6.9" @@ -219,8 +225,10 @@ dependencies = [ "pyo3", "rrf", "serde", + "serde-wasm-bindgen", "serde_json", "textkit", + "wasm-bindgen", ] [[package]] @@ -589,6 +597,30 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -793,6 +825,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1005,6 +1048,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "portable-atomic" version = "1.15.0" @@ -1316,6 +1365,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -1383,6 +1443,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.16.1" @@ -1687,6 +1753,51 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + [[package]] name = "webpki-roots" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index b7deead..a4da8e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,8 @@ emb = { path = "crates/emb", version = "0.1.0" } clap = { version = "4", features = ["derive"] } clap_mangen = "0.3" pyo3 = { version = "0.27", features = ["extension-module", "abi3-py39"] } +wasm-bindgen = "0.2" +serde-wasm-bindgen = "0.6" deunicode = "1" fancy-regex = "0.16" # 0.38 is the last release that builds on the pinned toolchain; the ones diff --git a/README.md b/README.md index e0567ad..465e896 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,29 @@ command does can be done in process instead. | [`textkit`](crates/textkit/README.md) | Token counting, sentence and paragraph splitting | | [`cuekit`](crates/cuekit/README.md) | The cue model, and the three subtitle formats it is read from | | [`catena-cli`](crates/catena-cli/README.md) | The parts of a filter every command does the same way | -| [`bindings`](crates/bindings/README.md) | The same libraries from Python, for callers already holding the text | +| [`bindings`](crates/bindings/README.md) | The same libraries from Python, C and JavaScript | + +## From another language + +The tools are filters first, and a subprocess is the right answer most of the +time. When the caller is already holding the text, the same libraries are +callable in process from three languages: + +| | Install | Import | +|---|---|---| +| **Python** | `pip install catena` | `import catena` | +| **JavaScript** | `npm install catena` | `import { chunkMarkdown } from "catena"` | +| **C** | build the `capi` feature | `#include ` | + +```python +chunks = catena.chunk_markdown(markdown, target=512) +chunks[0]["page"] # 41 +``` + +Each hands back the same record shapes the commands write — dictionaries in +Python, plain objects in JavaScript, JSON in C — so a stage that was parsing +NDJSON from a subprocess can call the function and get what it was already +parsing. Details in [`bindings`](crates/bindings/README.md). ## A worked pipeline diff --git a/check.sh b/check.sh index c5237d2..ffbd312 100755 --- a/check.sh +++ b/check.sh @@ -18,6 +18,15 @@ cargo clippy --workspace --all-targets --all-features -- -D warnings say "tests" cargo test --workspace +# The WebAssembly build is only ever compiled for wasm32, so `--all-features` +# above does not reach it: on any other target the module compiles to nothing. +say "webassembly" +if rustup target list --installed 2>/dev/null | grep -q wasm32-unknown-unknown; then + cargo clippy -p catena --features wasm --lib --target wasm32-unknown-unknown -- -D warnings +else + echo "skipped: rustup target add wasm32-unknown-unknown" +fi + say "docs" RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --quiet diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index 6689fb4..3e59ef3 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -7,6 +7,20 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +# wasm-pack ships a wasm-opt older than the WebAssembly rustc now emits, and +# it rejects the module rather than assuming the features are allowed. Naming +# them keeps the optimisation instead of turning it off; every one is years +# old and in every runtime that matters. +[package.metadata.wasm-pack.profile.release] +wasm-opt = [ + "-O", + "--enable-bulk-memory", + "--enable-nontrapping-float-to-int", + "--enable-sign-ext", + "--enable-mutable-globals", + "--enable-reference-types", +] + [lib] name = "catena" path = "src/lib.rs" @@ -28,11 +42,28 @@ textkit.workspace = true pyo3 = { workspace = true, optional = true } +# The JavaScript build. Only ever compiled for a wasm32 target; on any other +# the module is compiled out and these sit unused. +wasm-bindgen = { workspace = true, optional = true } +serde-wasm-bindgen = { workspace = true, optional = true } + [features] # Only maturin turns this on. pyo3's extension-module feature leaves the # Python symbols to be resolved by the interpreter at load time, so an # ordinary `cargo build` would fail to link if it were on by default. python = ["dep:pyo3"] +# The C ABI. Off by default so an ordinary build does not also export a +# hundred symbols nobody asked for. +capi = [] +# JavaScript and TypeScript, through WebAssembly. +wasm = ["dep:wasm-bindgen", "dep:serde-wasm-bindgen"] -[lints] -workspace = true +# Not `workspace = true`. The workspace forbids unsafe, and a C ABI is unsafe +# by definition: it takes raw pointers from a caller this cannot see. The +# allowance is the narrowest thing that compiles — every `unsafe` in this +# crate is in `src/capi.rs`, and `unsafe_op_in_unsafe_fn` keeps each one +# spelled out rather than inherited from the function signature. +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +missing_docs = "warn" diff --git a/crates/bindings/README.md b/crates/bindings/README.md index 3046da8..e136f2d 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -19,6 +19,9 @@ What comes back is the same **record shapes** the commands write to stdout, as dictionaries rather than lines of JSON. Those shapes are documented and versioned; the structs behind them are free to move. +Three of them: Python, JavaScript through WebAssembly, and a C ABI. All three +call the same Rust and hand back the same records. + ## Python ```sh @@ -69,17 +72,89 @@ The tools that are not here are the ones a binding would gain nothing over a subprocess for: they read files, shell out to another program, or talk to a server, and none of that is cheaper in process. -## Building the wheel +## JavaScript ```sh -cd crates/bindings && maturin build --release +npm install catena ``` -`cargo build --features python` does not work and is not supposed to: pyo3's -`extension-module` leaves the Python symbols to be resolved by the interpreter -at load time, so there is nothing to link against until an interpreter loads -it. `cargo clippy --features python` type-checks it without linking, which is -what the gate runs. +```js +import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena"; + +const chunks = chunkMarkdown(markdown, 512, 64); +chunks[0].page; // 41 +chunks[0].chapter; // 'III. The Second Estate' + +fuseHits([...vectorHits, ...textHits], undefined, { vec: 2 }, 20); +``` + +Records come back as plain objects, not handles: nothing to free, +`JSON.stringify` works, and an array of chunks spreads into state without +ceremony. The declarations are hand-written, so a caller gets `Chunk[]` rather +than `any`. + +`splitCollection` throws when the document is one work rather than a +collection, which is the usual answer. + +Two builds, because the loaders differ — node reads the `.wasm` off disk at +import, the web build takes a URL a bundler rewrites: + +```js +import { chunkMarkdown } from "catena"; // resolves per environment +import { chunkMarkdown } from "catena/node"; // or pick one +import { chunkMarkdown } from "catena/web"; +``` + +## C + +```c +#include + +char *chunks = catena_chunk_markdown(markdown, 512, 64, 50, 768); +if (!chunks) fprintf(stderr, "%s\n", catena_last_error()); +/* … a JSON array of chunk records … */ +catena_string_free(chunks); +``` + +Text in as NUL-terminated UTF-8, records out as NUL-terminated JSON — the same +document the command line writes. Nothing here is an opaque handle with +accessors to keep in step: the records already have a documented shape, and +every language with an FFI has a JSON parser. + +Every returned string is yours and must go back through `catena_string_free`. +A call that fails returns NULL and leaves a message in `catena_last_error`, +which is thread-local. `catena_abi_version` is bumped when anything here +changes meaning; additions leave it alone. + +A budget of zero means that budget's default, so a caller that cares about one +of them can pass zero for the rest. + +## Building + +```sh +cd crates/bindings +maturin build --profile wheel # the Python wheel +npm/build.sh # the npm package +cargo build --features capi # the C library and its header +``` + +Three things worth knowing, each of which fails confusingly otherwise: + +**`cargo build --features python` does not work and is not supposed to.** +pyo3's `extension-module` leaves the Python symbols to be resolved by the +interpreter at load time, so there is nothing to link against until an +interpreter loads it. `cargo clippy --features python` type-checks it without +linking, which is what the gate runs. + +**The wheel needs `--profile wheel`, not `--release`.** The release profile +strips, and a stripped extension module is one macOS refuses to load at all — +"mis-aligned LINKEDIT string pool". The `wheel` profile is the release profile +without it. `[tool.maturin] profile` is ignored by maturin 1.15, so it has to +be the flag. + +**The WebAssembly build is only compiled for wasm32.** On any other target the +module compiles to nothing, so `--all-features` on a host build does not reach +it; the gate checks it separately against `wasm32-unknown-unknown`. ## As a Rust library diff --git a/crates/bindings/include/catena.h b/crates/bindings/include/catena.h new file mode 100644 index 0000000..ef12fca --- /dev/null +++ b/crates/bindings/include/catena.h @@ -0,0 +1,82 @@ +/* + * catena — the libraries behind one surface, for C callers. + * + * Text goes in as NUL-terminated UTF-8; records come back as a + * NUL-terminated JSON array, the same document the command line writes and + * under the same stability promise. Release every returned string with + * catena_string_free. + * + * A function that can fail returns NULL and leaves a message in + * catena_last_error, which is thread-local. + * + * Every function is safe to call from any thread. + * + * Licence: MPL-2.0. See https://git.pumpin.online/funkpower/catena + */ +#ifndef CATENA_H +#define CATENA_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Bumped when the meaning or signature of anything here changes. Additions + * leave it alone. Check it against CATENA_ABI_VERSION at load time. */ +#define CATENA_ABI_VERSION 1u + +unsigned int catena_abi_version(void); + +/* Why the last call on this thread failed, or NULL if none did. Valid until + * the next failing call on the same thread. Do not free. */ +const char *catena_last_error(void); + +/* Release a string this library returned. NULL is allowed. */ +void catena_string_free(char *text); + +/* Cut markdown into retrievable passages. Returns a JSON array of chunks. + * Zero for a budget means that budget's default. */ +char *catena_chunk_markdown(const char *markdown, + size_t target, + size_t overlap, + size_t min_tokens, + size_t hard_max); + +/* Clean converted markdown. Non-zero `pages` normalises page markers as + * well, which is not idempotent. Returns the cleaned text. */ +char *catena_scrub(const char *markdown, int pages); + +/* How many tokens a string is. Zero on error. */ +size_t catena_count_tokens(const char *text); + +/* Read any of the subtitle formats. Returns a JSON array of cues. */ +char *catena_parse_subtitles(const char *content, int keep_repeats); + +/* Pack cues into passages. `cues` is a JSON array of cues. Zero for either + * budget means the default. Returns a JSON array of chunks. */ +char *catena_pack_cues(const char *cues, double seconds, size_t max_tokens); + +/* Split a collection into the works inside it. Returns a JSON array of + * sections, or NULL when the document is one work rather than a collection — + * an answer, not a failure; the reason is in catena_last_error. */ +char *catena_split_collection(const char *markdown); + +/* Fuse ranked lists into one. `hits` is a JSON array; `weights` a JSON object + * of channel to weight, or NULL. Zero `k` means the default, zero `limit` + * means all of them. Returns a JSON array. */ +char *catena_fuse_hits(const char *hits, + double k, + const char *weights, + size_t limit); + +/* Say what a file is from its first bytes and its name. Returns a JSON doc + * record. */ +char *catena_identify(const char *name, const uint8_t *data, size_t len); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CATENA_H */ diff --git a/crates/bindings/npm/README.md b/crates/bindings/npm/README.md new file mode 100644 index 0000000..c44e691 --- /dev/null +++ b/crates/bindings/npm/README.md @@ -0,0 +1,69 @@ +# catena + +Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles +and rank fusion, as WebAssembly. + +```sh +npm install catena +``` + +```js +import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena"; + +const chunks = chunkMarkdown(markdown, 512, 64); +chunks[0].page; // 41 +chunks[0].chapter; // 'III. The Second Estate' + +const clean = scrub(rawMarkdown, true); +countTokens(clean); + +fuseHits([...vectorHits, ...textHits], undefined, { vec: 2 }, 20); +``` + +Records come back as plain objects — the same documents the command-line +tools write, with the same field names and the same stability promise. +Nothing has to be freed. + +`splitCollection` throws when the document is one work rather than a +collection, which is the usual answer: + +```js +let works; +try { + works = splitCollection(markdown); +} catch (why) { + works = [{ idx: 0, title, text: markdown }]; +} +``` + +## What is here + +| | | +|---|---| +| `chunkMarkdown` | Markdown into passages that carry their page, chapter and role | +| `scrub` | Converted markdown in, clean markdown out | +| `countTokens` | Tokens, by the encoding the budgets count in | +| `parseSubtitles` | SubRip, WebVTT or ASS into cues | +| `packCues` | Cues into passages, keeping the times they were said at | +| `splitCollection` | One markdown file into the works inside it | +| `fuseHits` | Several ranked lists into one | +| `identify` | What a file is, from its first bytes | + +The tools that read files or call another program are not here: a browser has +neither, and a binding that pretends otherwise fails at run time rather than +at compile time. Those stay command-line tools. + +## Node and the web + +```js +import { chunkMarkdown } from "catena"; // resolves per environment +import { chunkMarkdown } from "catena/node"; // or pick one +import { chunkMarkdown } from "catena/web"; +``` + +Two builds because the loaders differ: node reads the `.wasm` off disk at +import, the web build takes a URL a bundler rewrites. + +## Licence + +MPL-2.0. Source: diff --git a/crates/bindings/npm/build.sh b/crates/bindings/npm/build.sh new file mode 100755 index 0000000..a73e851 --- /dev/null +++ b/crates/bindings/npm/build.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Build the npm package: one wasm module per target, and a package.json that +# points at both. +# +# Two builds because the loaders differ. The node build reads the .wasm off +# disk at import; the web build takes it as a URL a bundler rewrites. One +# build serving both means a conditional import that every bundler gets wrong +# in a different way. +set -eu + +cd "$(dirname "$0")" +crate=.. +out="$(pwd)/dist" + +command -v wasm-pack >/dev/null 2>&1 || { + echo "npm/build.sh: wasm-pack is not on PATH" >&2 + exit 69 +} + +version=$(grep -m1 '^version' "$crate/../../Cargo.toml" | sed 's/.*"\(.*\)".*/\1/') + +rm -rf "$out" +mkdir -p "$out" + +for target in nodejs web; do + case "$target" in + nodejs) dir=node ;; + *) dir=web ;; + esac + wasm-pack build "$crate" \ + --target "$target" \ + --out-dir "$out/$dir" \ + --out-name catena \ + --release \ + -- --features wasm --no-default-features + # wasm-pack writes one of these per target and they disagree; the + # package.json below is the one that ships. + rm -f "$out/$dir/package.json" "$out/$dir/README.md" "$out/$dir/.gitignore" +done + +# The node build is CommonJS and the package declares itself ESM, so the entry +# has to say so in its extension or node reads `exports.x = …` as a syntax +# error. The web build is already ESM and keeps its .js. +mv "$out/node/catena.js" "$out/node/catena.cjs" +mv "$out/node/catena.d.ts" "$out/node/catena.d.cts" + +sed "s/@VERSION@/$version/" package.json.in >"$out/package.json" +cp README.md "$out/README.md" +cp ../LICENSE "$out/LICENSE" + +echo "built $out (version $version)" +echo "publish with: npm publish $out --access public" diff --git a/crates/bindings/npm/package.json.in b/crates/bindings/npm/package.json.in new file mode 100644 index 0000000..aacb404 --- /dev/null +++ b/crates/bindings/npm/package.json.in @@ -0,0 +1,26 @@ +{ + "name": "catena", + "version": "@VERSION@", + "description": "Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles and rank fusion, as WebAssembly", + "license": "MPL-2.0", + "homepage": "https://git.pumpin.online/funkpower/catena", + "repository": { + "type": "git", + "url": "git+https://git.pumpin.online/funkpower/catena.git" + }, + "keywords": ["chunking", "markdown", "subtitles", "retrieval", "corpus", "tokens", "wasm"], + "type": "module", + "types": "./web/catena.d.ts", + "exports": { + ".": { + "node": { "types": "./node/catena.d.cts", "default": "./node/catena.cjs" }, + "default": { "types": "./web/catena.d.ts", "default": "./web/catena.js" } + }, + "./node": { "types": "./node/catena.d.cts", "default": "./node/catena.cjs" }, + "./web": { "types": "./web/catena.d.ts", "default": "./web/catena.js" }, + "./node/*": "./node/*", + "./web/*": "./web/*", + "./package.json": "./package.json" + }, + "files": ["node/", "web/", "README.md", "LICENSE"] +} diff --git a/crates/bindings/npm/test.mjs b/crates/bindings/npm/test.mjs new file mode 100644 index 0000000..8031906 --- /dev/null +++ b/crates/bindings/npm/test.mjs @@ -0,0 +1,36 @@ +// Exercise the built package the way a consumer would. +import { + chunkMarkdown, scrub, countTokens, parseSubtitles, packCues, + splitCollection, fuseHits, identify, +} from "./dist/node/catena.cjs"; + +const md = "# Chapter One\n\n\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked about it at all.\n"; + +const chunks = chunkMarkdown(md, undefined, undefined, 10); +console.log("chunk: ", { idx: chunks[0].idx, page: chunks[0].page, chapter: chunks[0].chapter, tokens: chunks[0].tokens }); + +console.log("scrub: ", JSON.stringify(scrub("# Heading\n\n\n\nbody text\n"))); +console.log("tokens: ", countTokens("a passage of prose")); + +const cues = parseSubtitles("1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n2\n00:00:04,000 --> 00:00:08,000\nof the valley where the road turned\n\n"); +console.log("cues: ", cues.length, cues[0]); +console.log("packed: ", packCues(cues, 5)[0].text); + +const hits = [ + { v: 1, id: "c81a", channel: "vec", rank: 1, score: 0.83, payload: {} }, + { v: 1, id: "c81a", channel: "fts", rank: 3, score: 11.2, payload: {} }, + { v: 1, id: "d42f", channel: "fts", rank: 1, score: 14.0, payload: {} }, +]; +console.log("fused: ", fuseHits(hits, undefined, undefined, 2).map((h) => [h.id, +h.score.toFixed(5)])); +console.log("weighted: ", fuseHits(hits, undefined, { vec: 2 }, 2).map((h) => [h.id, +h.score.toFixed(5)])); + +console.log("identify: ", identify("book.pdf", new TextEncoder().encode("%PDF-1.4\n")).kind); + +try { splitCollection(md); } catch (why) { console.log("split throws:", String(why).slice(0, 52)); } + +// The types the package promises. +const types = await import("node:fs").then((fs) => fs.readFileSync("./dist/web/catena.d.ts", "utf8")); +for (const name of ["Chunk", "Cue", "Section", "Hit", "Doc"]) { + if (!types.includes(`interface ${name}`)) throw new Error(`no ${name} in the declarations`); +} +console.log("types: Chunk, Cue, Section, Hit, Doc all declared"); diff --git a/crates/bindings/src/capi.rs b/crates/bindings/src/capi.rs new file mode 100644 index 0000000..c95f166 --- /dev/null +++ b/crates/bindings/src/capi.rs @@ -0,0 +1,462 @@ +//! A C ABI, so this can be embedded rather than spawned. +//! +//! # Shape +//! +//! Text goes in as NUL-terminated UTF-8 and records come back as a +//! NUL-terminated JSON array — the same document the command line writes, +//! under the same stability promise. Nothing here is an opaque handle with +//! accessors to keep in step: the records already have a documented shape, +//! and every language with an FFI has a JSON parser. +//! +//! Every string this returns is owned by the caller and must be released with +//! [`catena_string_free`]. Passing it anything else is undefined. +//! +//! # Errors +//! +//! A function that can fail returns NULL and leaves a message in +//! [`catena_last_error`], which is thread-local and valid until the next +//! failing call on the same thread. +//! +//! # Threads +//! +//! Every function is safe to call from any thread. Nothing here holds state +//! between calls but the error message, which is per-thread. + +use std::cell::RefCell; +use std::ffi::{CStr, CString, c_char, c_double, c_int, c_uint}; +use std::ptr; + +use crate::{ChunkBudgets, CueBudgets}; + +/// Bumped when the meaning or signature of anything here changes. Additions +/// leave it alone. +const ABI_VERSION: c_uint = 1; + +thread_local! { + static LAST_ERROR: RefCell> = const { RefCell::new(None) }; +} + +fn set_error(message: impl Into>) { + let message = CString::new(message).unwrap_or_else(|_| c"error".into()); + LAST_ERROR.with(|slot| *slot.borrow_mut() = Some(message)); +} + +fn clear_error() { + LAST_ERROR.with(|slot| *slot.borrow_mut() = None); +} + +/// The version of this ABI. +#[unsafe(no_mangle)] +pub extern "C" fn catena_abi_version() -> c_uint { + ABI_VERSION +} + +/// Why the last call on this thread failed, or NULL if none did. +/// +/// # Safety +/// +/// The pointer is valid until the next failing call on the same thread. Copy +/// it if you need it longer. +#[unsafe(no_mangle)] +pub extern "C" fn catena_last_error() -> *const c_char { + LAST_ERROR.with(|slot| match slot.borrow().as_ref() { + Some(message) => message.as_ptr(), + None => ptr::null(), + }) +} + +/// Release a string this library returned. +/// +/// # Safety +/// +/// `text` must be a pointer this library returned and not yet freed, or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_string_free(text: *mut c_char) { + if !text.is_null() { + drop(unsafe { CString::from_raw(text) }); + } +} + +/// Borrow a caller's string as `&str`, or fail. +/// +/// # Safety +/// +/// `text` must be NUL-terminated and valid for reads, or NULL. +unsafe fn borrow<'a>(text: *const c_char, what: &str) -> Option<&'a str> { + if text.is_null() { + set_error(format!("{what} is NULL")); + return None; + } + match unsafe { CStr::from_ptr(text) }.to_str() { + Ok(text) => Some(text), + Err(_) => { + set_error(format!("{what} is not UTF-8")); + None + } + } +} + +/// Hand a string to the caller, or NULL if it cannot be made into one. +fn give(text: String) -> *mut c_char { + match CString::new(text) { + Ok(text) => text.into_raw(), + Err(_) => { + // Only reachable if a record held a NUL, which the readers refuse. + set_error("the answer contains a NUL byte"); + ptr::null_mut() + } + } +} + +/// Serialise records to JSON and hand them over. +fn give_json(records: &T) -> *mut c_char { + match serde_json::to_string(records) { + Ok(json) => give(json), + Err(e) => { + set_error(format!("the answer does not serialise: {e}")); + ptr::null_mut() + } + } +} + +/// Read a JSON argument, or fail. +fn take_json(json: &str, what: &str) -> Option { + match serde_json::from_str(json) { + Ok(value) => Some(value), + Err(e) => { + set_error(format!("{what} is not the expected shape: {e}")); + None + } + } +} + +/// Cut markdown into retrievable passages. Returns a JSON array of chunks. +/// +/// A budget of zero means the default for that budget, so a caller that only +/// cares about one of them can pass zero for the rest. +/// +/// # Safety +/// +/// `markdown` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_chunk_markdown( + markdown: *const c_char, + target: usize, + overlap: usize, + min_tokens: usize, + hard_max: usize, +) -> *mut c_char { + clear_error(); + let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else { + return ptr::null_mut(); + }; + let default = ChunkBudgets::default(); + let budgets = ChunkBudgets { + target: or_default(target, default.target), + overlap: match overlap { + // Zero overlap is a real choice, unlike a zero budget. + _ if target == 0 && overlap == 0 => default.overlap, + overlap => overlap, + }, + min_tokens: or_default(min_tokens, default.min_tokens), + hard_max: or_default(hard_max, default.hard_max), + }; + give_json(&crate::chunk_markdown(markdown, budgets)) +} + +fn or_default(value: usize, default: usize) -> usize { + match value { + 0 => default, + value => value, + } +} + +/// Clean converted markdown. Returns the cleaned text. +/// +/// `pages` non-zero normalises page markers as well, which is not idempotent. +/// +/// # Safety +/// +/// `markdown` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_scrub(markdown: *const c_char, pages: c_int) -> *mut c_char { + clear_error(); + let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else { + return ptr::null_mut(); + }; + give(match pages { + 0 => crate::scrub_markdown(markdown), + _ => crate::scrub_markdown_pages(markdown), + }) +} + +/// How many tokens a string is. Returns zero for NULL or invalid UTF-8, and +/// leaves a message in [`catena_last_error`]. +/// +/// # Safety +/// +/// `text` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_count_tokens(text: *const c_char) -> usize { + clear_error(); + match unsafe { borrow(text, "text") } { + Some(text) => crate::count_tokens(text), + None => 0, + } +} + +/// Read any of the subtitle formats. Returns a JSON array of cues. +/// +/// # Safety +/// +/// `content` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_parse_subtitles( + content: *const c_char, + keep_repeats: c_int, +) -> *mut c_char { + clear_error(); + let Some(content) = (unsafe { borrow(content, "content") }) else { + return ptr::null_mut(); + }; + let cues = crate::parse_subtitles(content); + let cues = match keep_repeats { + 0 => crate::merge_repeats(&cues), + _ => cues, + }; + give_json(&cues) +} + +/// Pack cues into passages. `cues` is a JSON array of cues; returns a JSON +/// array of chunks. +/// +/// Zero for either budget means the default. +/// +/// # Safety +/// +/// `cues` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_pack_cues( + cues: *const c_char, + seconds: c_double, + max_tokens: usize, +) -> *mut c_char { + clear_error(); + let Some(cues) = (unsafe { borrow(cues, "cues") }) else { + return ptr::null_mut(); + }; + let Some(cues) = take_json::>(cues, "cues") else { + return ptr::null_mut(); + }; + let default = CueBudgets::default(); + let budgets = CueBudgets { + seconds: match seconds > 0.0 { + true => seconds, + false => default.seconds, + }, + max_tokens: or_default(max_tokens, default.max_tokens), + }; + give_json(&crate::pack_cues(&cues, budgets)) +} + +/// Split a collection into the works inside it. Returns a JSON array of +/// sections, or NULL when the document is one work rather than a collection — +/// which is an answer, not a failure, and the reason is in +/// [`catena_last_error`]. +/// +/// # Safety +/// +/// `markdown` must be NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_split_collection(markdown: *const c_char) -> *mut c_char { + clear_error(); + let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else { + return ptr::null_mut(); + }; + match crate::split_collection(markdown) { + Ok(sections) => give_json(§ions), + Err(why) => { + set_error(why); + ptr::null_mut() + } + } +} + +/// Fuse ranked lists into one. `hits` is a JSON array of hits and `weights` a +/// JSON object of channel to weight, or NULL for none. Returns a JSON array. +/// +/// Zero for `k` means the default; zero for `limit` means all of them. +/// +/// # Safety +/// +/// `hits` must be NUL-terminated UTF-8; `weights` likewise, or NULL. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_fuse_hits( + hits: *const c_char, + k: c_double, + weights: *const c_char, + limit: usize, +) -> *mut c_char { + clear_error(); + let Some(hits) = (unsafe { borrow(hits, "hits") }) else { + return ptr::null_mut(); + }; + let Some(hits) = take_json::>(hits, "hits") else { + return ptr::null_mut(); + }; + + let weights: Vec<(String, f64)> = if weights.is_null() { + Vec::new() + } else { + let Some(text) = (unsafe { borrow(weights, "weights") }) else { + return ptr::null_mut(); + }; + let Some(map) = take_json::>(text, "weights") else { + return ptr::null_mut(); + }; + map.into_iter().collect() + }; + + let k = match k > 0.0 { + true => k, + false => rrf::K, + }; + let limit = match limit { + 0 => None, + limit => Some(limit), + }; + give_json(&crate::fuse_hits(&hits, k, &weights, limit)) +} + +/// Say what a file is from its first bytes and its name. Returns a JSON doc +/// record. +/// +/// # Safety +/// +/// `name` must be NUL-terminated UTF-8, and `data` must be valid for `len` +/// bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn catena_identify( + name: *const c_char, + data: *const u8, + len: usize, +) -> *mut c_char { + clear_error(); + let Some(name) = (unsafe { borrow(name, "name") }) else { + return ptr::null_mut(); + }; + if data.is_null() && len != 0 { + set_error("data is NULL"); + return ptr::null_mut(); + } + let bytes: &[u8] = match len { + 0 => &[], + len => unsafe { std::slice::from_raw_parts(data, len) }, + }; + give_json(&crate::identify(name, bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Call through the ABI the way a C caller would, and take the answer + /// back as a Rust string. + fn through(call: impl FnOnce() -> *mut c_char) -> Option { + let raw = call(); + if raw.is_null() { + return None; + } + let text = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { catena_string_free(raw) }; + Some(text) + } + + fn error() -> Option { + let raw = catena_last_error(); + match raw.is_null() { + true => None, + false => Some( + unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(), + ), + } + } + + #[test] + fn chunking_comes_back_as_the_records_the_command_writes() { + let markdown = c"# One\n\n\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked at all.\n"; + let json = through(|| unsafe { catena_chunk_markdown(markdown.as_ptr(), 0, 0, 10, 0) }) + .expect("a result"); + let chunks: Vec = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!(chunks[0]["page"], 41); + assert_eq!(chunks[0]["chapter"], "One"); + } + + #[test] + fn a_zero_budget_is_the_default_budget() { + let markdown = c"# One\n\nbody\n"; + assert!( + through(|| unsafe { catena_chunk_markdown(markdown.as_ptr(), 0, 0, 0, 0) }).is_some() + ); + } + + #[test] + fn a_null_argument_is_an_error_rather_than_a_crash() { + assert!(through(|| unsafe { catena_chunk_markdown(ptr::null(), 0, 0, 0, 0) }).is_none()); + assert_eq!(error().as_deref(), Some("markdown is NULL")); + assert_eq!(unsafe { catena_count_tokens(ptr::null()) }, 0); + assert_eq!(error().as_deref(), Some("text is NULL")); + } + + #[test] + fn freeing_nothing_is_allowed() { + unsafe { catena_string_free(ptr::null_mut()) }; + } + + #[test] + fn a_document_that_is_one_work_says_why_rather_than_returning_nothing() { + let markdown = c"# Only\n\nbody\n"; + assert!(through(|| unsafe { catena_split_collection(markdown.as_ptr()) }).is_none()); + assert!(error().expect("a reason").contains("sections")); + } + + #[test] + fn a_bad_json_argument_says_which_one() { + let hits = c"not json"; + assert!( + through(|| unsafe { catena_fuse_hits(hits.as_ptr(), 0.0, ptr::null(), 0) }).is_none() + ); + assert!( + error() + .expect("a reason") + .starts_with("hits is not the expected shape") + ); + } + + #[test] + fn identifying_nothing_is_identifying_an_empty_file() { + let name = c"book.pdf"; + let json = through(|| unsafe { catena_identify(name.as_ptr(), ptr::null(), 0) }) + .expect("a result"); + assert!(json.contains("\"path\":\"book.pdf\"")); + } + + #[test] + fn subtitles_round_trip_through_the_boundary() { + let srt = c"1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n"; + let cues = through(|| unsafe { catena_parse_subtitles(srt.as_ptr(), 0) }).expect("cues"); + let packed = CString::new(cues).expect("no NUL"); + let chunks = + through(|| unsafe { catena_pack_cues(packed.as_ptr(), 0.0, 0) }).expect("chunks"); + assert!(chunks.contains("the mill stood at the northern end")); + } + + #[test] + fn the_abi_version_is_the_one_the_header_declares() { + assert_eq!(catena_abi_version(), 1); + } +} diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 40d366d..c730c5c 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -18,6 +18,14 @@ #[cfg(feature = "python")] mod python; +#[cfg(feature = "capi")] +pub mod capi; + +// Only ever compiled for a wasm32 target. The feature can be on elsewhere — +// `--all-features` turns everything on — and there it compiles to nothing. +#[cfg(all(feature = "wasm", target_arch = "wasm32"))] +pub mod wasm; + use corpus_core::{Chunk, Doc, Hit}; use cuekit::Cue; diff --git a/crates/bindings/src/wasm.rs b/crates/bindings/src/wasm.rs new file mode 100644 index 0000000..083ac14 --- /dev/null +++ b/crates/bindings/src/wasm.rs @@ -0,0 +1,221 @@ +//! JavaScript and TypeScript, through WebAssembly. +//! +//! # Shape +//! +//! Records come back as **plain objects**, not handles: the same documents +//! the command line writes, with the same field names and the same stability +//! promise. Nothing has to be freed, `JSON.stringify` works, and an array of +//! chunks spreads into state without ceremony. +//! +//! # What is not here +//! +//! The tools that read files or call another program. A browser has no +//! `pdftoppm` and no files, and a binding that pretends otherwise is one that +//! fails at run time rather than at compile time. + +use wasm_bindgen::prelude::*; + +use crate::{ChunkBudgets, CueBudgets}; + +/// Hand-written because the generated `.d.ts` would otherwise say `any`: a +/// binding that types its whole return value as `any` is a binding +/// TypeScript cannot check. +#[wasm_bindgen(typescript_custom_section)] +const TYPES: &'static str = r#" +/** A retrievable passage. The same record `chunkmd` writes. */ +export interface Chunk { + v: number; + /** Position in the stream, from zero. */ + idx: number; + text: string; + /** Tokens, by the encoding the budgets count in. */ + tokens: number; + /** The page it came from, where the source had page markers. */ + page: number | null; + /** The heading it sits under. */ + chapter: string | null; + /** "body", "front", "back", "index" or "bibliography". */ + role: string; + /** Seconds into the recording, for passages made from cues. */ + start_time: number | null; + end_time: number | null; + flags: string[]; +} + +/** One line of a transcript. */ +export interface Cue { + v: number; + start: number; + end: number; + text: string; +} + +/** One work inside a collection. */ +export interface Section { + v: number; + idx: number; + title: string; + slug: string; + text: string; +} + +/** One scored candidate, before or after fusion. */ +export interface Hit { + v: number; + id: string; + /** "vec", "fts" or "meta". */ + channel: string; + rank: number; + score: number; + payload: unknown; +} + +/** What a file is. */ +export interface Doc { + v: number; + path: string; + kind: string; + declared_ext: string | null; + bytes: number; + pages: number | null; + language: string | null; + text_layer: string | null; + needs_ocr: boolean | null; + notes: string[]; +} +"#; + +/// What the chunker does when nothing is passed. +fn chunk_budgets( + target: Option, + overlap: Option, + min_tokens: Option, + hard_max: Option, +) -> ChunkBudgets { + let default = ChunkBudgets::default(); + ChunkBudgets { + target: target.unwrap_or(default.target), + overlap: overlap.unwrap_or(default.overlap), + min_tokens: min_tokens.unwrap_or(default.min_tokens), + hard_max: hard_max.unwrap_or(default.hard_max), + } +} + +fn to_js(value: &T) -> Result { + serde_wasm_bindgen::to_value(value).map_err(|e| JsValue::from_str(&e.to_string())) +} + +fn from_js(value: JsValue, what: &str) -> Result { + serde_wasm_bindgen::from_value(value) + .map_err(|e| JsValue::from_str(&format!("{what} is not the expected shape: {e}"))) +} + +/// Cut markdown into retrievable passages. +#[wasm_bindgen(js_name = chunkMarkdown, skip_typescript)] +pub fn chunk_markdown( + markdown: &str, + target: Option, + overlap: Option, + min_tokens: Option, + hard_max: Option, +) -> Result { + let budgets = chunk_budgets(target, overlap, min_tokens, hard_max); + to_js(&crate::chunk_markdown(markdown, budgets)) +} + +/// Clean converted markdown. +/// +/// `pages` also normalises page markers, which is not idempotent. +#[wasm_bindgen(js_name = scrub)] +pub fn scrub(markdown: &str, pages: Option) -> String { + match pages.unwrap_or(false) { + true => crate::scrub_markdown_pages(markdown), + false => crate::scrub_markdown(markdown), + } +} + +/// How many tokens a string is. +#[wasm_bindgen(js_name = countTokens)] +pub fn count_tokens(text: &str) -> usize { + crate::count_tokens(text) +} + +/// Read any of the subtitle formats into cues. +#[wasm_bindgen(js_name = parseSubtitles, skip_typescript)] +pub fn parse_subtitles(content: &str, keep_repeats: Option) -> Result { + let cues = crate::parse_subtitles(content); + let cues = match keep_repeats.unwrap_or(false) { + true => cues, + false => crate::merge_repeats(&cues), + }; + to_js(&cues) +} + +/// Pack cues into passages worth embedding. +#[wasm_bindgen(js_name = packCues, skip_typescript)] +pub fn pack_cues( + cues: JsValue, + seconds: Option, + max_tokens: Option, +) -> Result { + let default = CueBudgets::default(); + let budgets = CueBudgets { + seconds: seconds.unwrap_or(default.seconds), + max_tokens: max_tokens.unwrap_or(default.max_tokens), + }; + let cues: Vec = from_js(cues, "cues")?; + to_js(&crate::pack_cues(&cues, budgets)) +} + +/// Split a collection into the works inside it. +/// +/// Throws when the document is one work rather than a collection, which is +/// the usual answer, with the reason each candidate was rejected. +#[wasm_bindgen(js_name = splitCollection, skip_typescript)] +pub fn split_collection(markdown: &str) -> Result { + match crate::split_collection(markdown) { + Ok(sections) => to_js(§ions), + Err(why) => Err(JsValue::from_str(&why)), + } +} + +/// Fuse ranked lists into one. +#[wasm_bindgen(js_name = fuseHits, skip_typescript)] +pub fn fuse_hits( + hits: JsValue, + k: Option, + weights: JsValue, + limit: Option, +) -> Result { + let hits: Vec = from_js(hits, "hits")?; + let weights: Vec<(String, f64)> = match weights.is_undefined() || weights.is_null() { + true => Vec::new(), + false => from_js::>(weights, "weights")? + .into_iter() + .collect(), + }; + to_js(&crate::fuse_hits( + &hits, + k.unwrap_or(rrf::K), + &weights, + limit, + )) +} + +/// Say what a file is from its first bytes and its name. +#[wasm_bindgen(js_name = identify, skip_typescript)] +pub fn identify(name: &str, data: &[u8]) -> Result { + to_js(&crate::identify(name, data)) +} + +/// Declarations for the functions whose return type the generator cannot +/// infer, so a caller gets `Chunk[]` rather than `any`. +#[wasm_bindgen(typescript_custom_section)] +const SIGNATURES: &'static str = r#" +export function chunkMarkdown(markdown: string, target?: number, overlap?: number, min_tokens?: number, hard_max?: number): Chunk[]; +export function parseSubtitles(content: string, keep_repeats?: boolean): Cue[]; +export function packCues(cues: Cue[], seconds?: number, max_tokens?: number): Chunk[]; +export function splitCollection(markdown: string): Section[]; +export function fuseHits(hits: Hit[], k?: number, weights?: Record, limit?: number): Hit[]; +export function identify(name: string, data: Uint8Array): Doc; +"#;