mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
bindings: a C ABI and an npm package alongside the wheel
This commit is contained in:
funkpower
@@ -53,6 +53,8 @@ jobs:
|
||||
apt-get update -qq
|
||||
apt-get install -qq -y --no-install-recommends python3 >/dev/null
|
||||
rustup component add rustfmt clippy
|
||||
# The WebAssembly binding is checked for its own target.
|
||||
rustup target add wasm32-unknown-unknown
|
||||
|
||||
- name: Check
|
||||
run: ./check.sh
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
target/
|
||||
*.profraw
|
||||
.DS_Store
|
||||
|
||||
# Built npm package; `crates/bindings/npm/build.sh` makes it.
|
||||
crates/bindings/npm/dist/
|
||||
|
||||
Generated
+111
@@ -193,6 +193,12 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "bytecount"
|
||||
version = "0.6.9"
|
||||
@@ -219,8 +225,10 @@ dependencies = [
|
||||
"pyo3",
|
||||
"rrf",
|
||||
"serde",
|
||||
"serde-wasm-bindgen",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -589,6 +597,30 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
@@ -793,6 +825,17 @@ version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
@@ -1005,6 +1048,12 @@ version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
@@ -1316,6 +1365,17 @@ dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-wasm-bindgen"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
@@ -1383,6 +1443,12 @@ version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.1"
|
||||
@@ -1687,6 +1753,51 @@ dependencies = [
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
|
||||
@@ -32,6 +32,8 @@ emb = { path = "crates/emb", version = "0.1.0" }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
pyo3 = { version = "0.27", features = ["extension-module", "abi3-py39"] }
|
||||
wasm-bindgen = "0.2"
|
||||
serde-wasm-bindgen = "0.6"
|
||||
deunicode = "1"
|
||||
fancy-regex = "0.16"
|
||||
# 0.38 is the last release that builds on the pinned toolchain; the ones
|
||||
|
||||
@@ -56,7 +56,29 @@ command does can be done in process instead.
|
||||
| [`textkit`](crates/textkit/README.md) | Token counting, sentence and paragraph splitting |
|
||||
| [`cuekit`](crates/cuekit/README.md) | The cue model, and the three subtitle formats it is read from |
|
||||
| [`catena-cli`](crates/catena-cli/README.md) | The parts of a filter every command does the same way |
|
||||
| [`bindings`](crates/bindings/README.md) | The same libraries from Python, for callers already holding the text |
|
||||
| [`bindings`](crates/bindings/README.md) | The same libraries from Python, C and JavaScript |
|
||||
|
||||
## From another language
|
||||
|
||||
The tools are filters first, and a subprocess is the right answer most of the
|
||||
time. When the caller is already holding the text, the same libraries are
|
||||
callable in process from three languages:
|
||||
|
||||
| | Install | Import |
|
||||
|---|---|---|
|
||||
| **Python** | `pip install catena` | `import catena` |
|
||||
| **JavaScript** | `npm install catena` | `import { chunkMarkdown } from "catena"` |
|
||||
| **C** | build the `capi` feature | `#include <catena.h>` |
|
||||
|
||||
```python
|
||||
chunks = catena.chunk_markdown(markdown, target=512)
|
||||
chunks[0]["page"] # 41
|
||||
```
|
||||
|
||||
Each hands back the same record shapes the commands write — dictionaries in
|
||||
Python, plain objects in JavaScript, JSON in C — so a stage that was parsing
|
||||
NDJSON from a subprocess can call the function and get what it was already
|
||||
parsing. Details in [`bindings`](crates/bindings/README.md).
|
||||
|
||||
## A worked pipeline
|
||||
|
||||
|
||||
@@ -18,6 +18,15 @@ cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
say "tests"
|
||||
cargo test --workspace
|
||||
|
||||
# The WebAssembly build is only ever compiled for wasm32, so `--all-features`
|
||||
# above does not reach it: on any other target the module compiles to nothing.
|
||||
say "webassembly"
|
||||
if rustup target list --installed 2>/dev/null | grep -q wasm32-unknown-unknown; then
|
||||
cargo clippy -p catena --features wasm --lib --target wasm32-unknown-unknown -- -D warnings
|
||||
else
|
||||
echo "skipped: rustup target add wasm32-unknown-unknown"
|
||||
fi
|
||||
|
||||
say "docs"
|
||||
RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --quiet
|
||||
|
||||
|
||||
@@ -7,6 +7,20 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
# wasm-pack ships a wasm-opt older than the WebAssembly rustc now emits, and
|
||||
# it rejects the module rather than assuming the features are allowed. Naming
|
||||
# them keeps the optimisation instead of turning it off; every one is years
|
||||
# old and in every runtime that matters.
|
||||
[package.metadata.wasm-pack.profile.release]
|
||||
wasm-opt = [
|
||||
"-O",
|
||||
"--enable-bulk-memory",
|
||||
"--enable-nontrapping-float-to-int",
|
||||
"--enable-sign-ext",
|
||||
"--enable-mutable-globals",
|
||||
"--enable-reference-types",
|
||||
]
|
||||
|
||||
[lib]
|
||||
name = "catena"
|
||||
path = "src/lib.rs"
|
||||
@@ -28,11 +42,28 @@ textkit.workspace = true
|
||||
|
||||
pyo3 = { workspace = true, optional = true }
|
||||
|
||||
# The JavaScript build. Only ever compiled for a wasm32 target; on any other
|
||||
# the module is compiled out and these sit unused.
|
||||
wasm-bindgen = { workspace = true, optional = true }
|
||||
serde-wasm-bindgen = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
# Only maturin turns this on. pyo3's extension-module feature leaves the
|
||||
# Python symbols to be resolved by the interpreter at load time, so an
|
||||
# ordinary `cargo build` would fail to link if it were on by default.
|
||||
python = ["dep:pyo3"]
|
||||
# The C ABI. Off by default so an ordinary build does not also export a
|
||||
# hundred symbols nobody asked for.
|
||||
capi = []
|
||||
# JavaScript and TypeScript, through WebAssembly.
|
||||
wasm = ["dep:wasm-bindgen", "dep:serde-wasm-bindgen"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
# Not `workspace = true`. The workspace forbids unsafe, and a C ABI is unsafe
|
||||
# by definition: it takes raw pointers from a caller this cannot see. The
|
||||
# allowance is the narrowest thing that compiles — every `unsafe` in this
|
||||
# crate is in `src/capi.rs`, and `unsafe_op_in_unsafe_fn` keeps each one
|
||||
# spelled out rather than inherited from the function signature.
|
||||
[lints.rust]
|
||||
unsafe_code = "allow"
|
||||
unsafe_op_in_unsafe_fn = "deny"
|
||||
missing_docs = "warn"
|
||||
|
||||
@@ -19,6 +19,9 @@ What comes back is the same **record shapes** the commands write to stdout,
|
||||
as dictionaries rather than lines of JSON. Those shapes are documented and
|
||||
versioned; the structs behind them are free to move.
|
||||
|
||||
Three of them: Python, JavaScript through WebAssembly, and a C ABI. All three
|
||||
call the same Rust and hand back the same records.
|
||||
|
||||
## Python
|
||||
|
||||
```sh
|
||||
@@ -69,17 +72,89 @@ The tools that are not here are the ones a binding would gain nothing over a
|
||||
subprocess for: they read files, shell out to another program, or talk to a
|
||||
server, and none of that is cheaper in process.
|
||||
|
||||
## Building the wheel
|
||||
## JavaScript
|
||||
|
||||
```sh
|
||||
cd crates/bindings && maturin build --release
|
||||
npm install catena
|
||||
```
|
||||
|
||||
`cargo build --features python` does not work and is not supposed to: pyo3's
|
||||
`extension-module` leaves the Python symbols to be resolved by the interpreter
|
||||
at load time, so there is nothing to link against until an interpreter loads
|
||||
it. `cargo clippy --features python` type-checks it without linking, which is
|
||||
what the gate runs.
|
||||
```js
|
||||
import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena";
|
||||
|
||||
const chunks = chunkMarkdown(markdown, 512, 64);
|
||||
chunks[0].page; // 41
|
||||
chunks[0].chapter; // 'III. The Second Estate'
|
||||
|
||||
fuseHits([...vectorHits, ...textHits], undefined, { vec: 2 }, 20);
|
||||
```
|
||||
|
||||
Records come back as plain objects, not handles: nothing to free,
|
||||
`JSON.stringify` works, and an array of chunks spreads into state without
|
||||
ceremony. The declarations are hand-written, so a caller gets `Chunk[]` rather
|
||||
than `any`.
|
||||
|
||||
`splitCollection` throws when the document is one work rather than a
|
||||
collection, which is the usual answer.
|
||||
|
||||
Two builds, because the loaders differ — node reads the `.wasm` off disk at
|
||||
import, the web build takes a URL a bundler rewrites:
|
||||
|
||||
```js
|
||||
import { chunkMarkdown } from "catena"; // resolves per environment
|
||||
import { chunkMarkdown } from "catena/node"; // or pick one
|
||||
import { chunkMarkdown } from "catena/web";
|
||||
```
|
||||
|
||||
## C
|
||||
|
||||
```c
|
||||
#include <catena.h>
|
||||
|
||||
char *chunks = catena_chunk_markdown(markdown, 512, 64, 50, 768);
|
||||
if (!chunks) fprintf(stderr, "%s\n", catena_last_error());
|
||||
/* … a JSON array of chunk records … */
|
||||
catena_string_free(chunks);
|
||||
```
|
||||
|
||||
Text in as NUL-terminated UTF-8, records out as NUL-terminated JSON — the same
|
||||
document the command line writes. Nothing here is an opaque handle with
|
||||
accessors to keep in step: the records already have a documented shape, and
|
||||
every language with an FFI has a JSON parser.
|
||||
|
||||
Every returned string is yours and must go back through `catena_string_free`.
|
||||
A call that fails returns NULL and leaves a message in `catena_last_error`,
|
||||
which is thread-local. `catena_abi_version` is bumped when anything here
|
||||
changes meaning; additions leave it alone.
|
||||
|
||||
A budget of zero means that budget's default, so a caller that cares about one
|
||||
of them can pass zero for the rest.
|
||||
|
||||
## Building
|
||||
|
||||
```sh
|
||||
cd crates/bindings
|
||||
maturin build --profile wheel # the Python wheel
|
||||
npm/build.sh # the npm package
|
||||
cargo build --features capi # the C library and its header
|
||||
```
|
||||
|
||||
Three things worth knowing, each of which fails confusingly otherwise:
|
||||
|
||||
**`cargo build --features python` does not work and is not supposed to.**
|
||||
pyo3's `extension-module` leaves the Python symbols to be resolved by the
|
||||
interpreter at load time, so there is nothing to link against until an
|
||||
interpreter loads it. `cargo clippy --features python` type-checks it without
|
||||
linking, which is what the gate runs.
|
||||
|
||||
**The wheel needs `--profile wheel`, not `--release`.** The release profile
|
||||
strips, and a stripped extension module is one macOS refuses to load at all —
|
||||
"mis-aligned LINKEDIT string pool". The `wheel` profile is the release profile
|
||||
without it. `[tool.maturin] profile` is ignored by maturin 1.15, so it has to
|
||||
be the flag.
|
||||
|
||||
**The WebAssembly build is only compiled for wasm32.** On any other target the
|
||||
module compiles to nothing, so `--all-features` on a host build does not reach
|
||||
it; the gate checks it separately against `wasm32-unknown-unknown`.
|
||||
|
||||
## As a Rust library
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* catena — the libraries behind one surface, for C callers.
|
||||
*
|
||||
* Text goes in as NUL-terminated UTF-8; records come back as a
|
||||
* NUL-terminated JSON array, the same document the command line writes and
|
||||
* under the same stability promise. Release every returned string with
|
||||
* catena_string_free.
|
||||
*
|
||||
* A function that can fail returns NULL and leaves a message in
|
||||
* catena_last_error, which is thread-local.
|
||||
*
|
||||
* Every function is safe to call from any thread.
|
||||
*
|
||||
* Licence: MPL-2.0. See https://git.pumpin.online/funkpower/catena
|
||||
*/
|
||||
#ifndef CATENA_H
|
||||
#define CATENA_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Bumped when the meaning or signature of anything here changes. Additions
|
||||
* leave it alone. Check it against CATENA_ABI_VERSION at load time. */
|
||||
#define CATENA_ABI_VERSION 1u
|
||||
|
||||
unsigned int catena_abi_version(void);
|
||||
|
||||
/* Why the last call on this thread failed, or NULL if none did. Valid until
|
||||
* the next failing call on the same thread. Do not free. */
|
||||
const char *catena_last_error(void);
|
||||
|
||||
/* Release a string this library returned. NULL is allowed. */
|
||||
void catena_string_free(char *text);
|
||||
|
||||
/* Cut markdown into retrievable passages. Returns a JSON array of chunks.
|
||||
* Zero for a budget means that budget's default. */
|
||||
char *catena_chunk_markdown(const char *markdown,
|
||||
size_t target,
|
||||
size_t overlap,
|
||||
size_t min_tokens,
|
||||
size_t hard_max);
|
||||
|
||||
/* Clean converted markdown. Non-zero `pages` normalises page markers as
|
||||
* well, which is not idempotent. Returns the cleaned text. */
|
||||
char *catena_scrub(const char *markdown, int pages);
|
||||
|
||||
/* How many tokens a string is. Zero on error. */
|
||||
size_t catena_count_tokens(const char *text);
|
||||
|
||||
/* Read any of the subtitle formats. Returns a JSON array of cues. */
|
||||
char *catena_parse_subtitles(const char *content, int keep_repeats);
|
||||
|
||||
/* Pack cues into passages. `cues` is a JSON array of cues. Zero for either
|
||||
* budget means the default. Returns a JSON array of chunks. */
|
||||
char *catena_pack_cues(const char *cues, double seconds, size_t max_tokens);
|
||||
|
||||
/* Split a collection into the works inside it. Returns a JSON array of
|
||||
* sections, or NULL when the document is one work rather than a collection —
|
||||
* an answer, not a failure; the reason is in catena_last_error. */
|
||||
char *catena_split_collection(const char *markdown);
|
||||
|
||||
/* Fuse ranked lists into one. `hits` is a JSON array; `weights` a JSON object
|
||||
* of channel to weight, or NULL. Zero `k` means the default, zero `limit`
|
||||
* means all of them. Returns a JSON array. */
|
||||
char *catena_fuse_hits(const char *hits,
|
||||
double k,
|
||||
const char *weights,
|
||||
size_t limit);
|
||||
|
||||
/* Say what a file is from its first bytes and its name. Returns a JSON doc
|
||||
* record. */
|
||||
char *catena_identify(const char *name, const uint8_t *data, size_t len);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* CATENA_H */
|
||||
@@ -0,0 +1,69 @@
|
||||
# catena
|
||||
|
||||
Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles
|
||||
and rank fusion, as WebAssembly.
|
||||
|
||||
```sh
|
||||
npm install catena
|
||||
```
|
||||
|
||||
```js
|
||||
import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena";
|
||||
|
||||
const chunks = chunkMarkdown(markdown, 512, 64);
|
||||
chunks[0].page; // 41
|
||||
chunks[0].chapter; // 'III. The Second Estate'
|
||||
|
||||
const clean = scrub(rawMarkdown, true);
|
||||
countTokens(clean);
|
||||
|
||||
fuseHits([...vectorHits, ...textHits], undefined, { vec: 2 }, 20);
|
||||
```
|
||||
|
||||
Records come back as plain objects — the same documents the command-line
|
||||
tools write, with the same field names and the same stability promise.
|
||||
Nothing has to be freed.
|
||||
|
||||
`splitCollection` throws when the document is one work rather than a
|
||||
collection, which is the usual answer:
|
||||
|
||||
```js
|
||||
let works;
|
||||
try {
|
||||
works = splitCollection(markdown);
|
||||
} catch (why) {
|
||||
works = [{ idx: 0, title, text: markdown }];
|
||||
}
|
||||
```
|
||||
|
||||
## What is here
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `chunkMarkdown` | Markdown into passages that carry their page, chapter and role |
|
||||
| `scrub` | Converted markdown in, clean markdown out |
|
||||
| `countTokens` | Tokens, by the encoding the budgets count in |
|
||||
| `parseSubtitles` | SubRip, WebVTT or ASS into cues |
|
||||
| `packCues` | Cues into passages, keeping the times they were said at |
|
||||
| `splitCollection` | One markdown file into the works inside it |
|
||||
| `fuseHits` | Several ranked lists into one |
|
||||
| `identify` | What a file is, from its first bytes |
|
||||
|
||||
The tools that read files or call another program are not here: a browser has
|
||||
neither, and a binding that pretends otherwise fails at run time rather than
|
||||
at compile time. Those stay command-line tools.
|
||||
|
||||
## Node and the web
|
||||
|
||||
```js
|
||||
import { chunkMarkdown } from "catena"; // resolves per environment
|
||||
import { chunkMarkdown } from "catena/node"; // or pick one
|
||||
import { chunkMarkdown } from "catena/web";
|
||||
```
|
||||
|
||||
Two builds because the loaders differ: node reads the `.wasm` off disk at
|
||||
import, the web build takes a URL a bundler rewrites.
|
||||
|
||||
## Licence
|
||||
|
||||
MPL-2.0. Source: <https://git.pumpin.online/funkpower/catena>
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
# Build the npm package: one wasm module per target, and a package.json that
|
||||
# points at both.
|
||||
#
|
||||
# Two builds because the loaders differ. The node build reads the .wasm off
|
||||
# disk at import; the web build takes it as a URL a bundler rewrites. One
|
||||
# build serving both means a conditional import that every bundler gets wrong
|
||||
# in a different way.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
crate=..
|
||||
out="$(pwd)/dist"
|
||||
|
||||
command -v wasm-pack >/dev/null 2>&1 || {
|
||||
echo "npm/build.sh: wasm-pack is not on PATH" >&2
|
||||
exit 69
|
||||
}
|
||||
|
||||
version=$(grep -m1 '^version' "$crate/../../Cargo.toml" | sed 's/.*"\(.*\)".*/\1/')
|
||||
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
|
||||
for target in nodejs web; do
|
||||
case "$target" in
|
||||
nodejs) dir=node ;;
|
||||
*) dir=web ;;
|
||||
esac
|
||||
wasm-pack build "$crate" \
|
||||
--target "$target" \
|
||||
--out-dir "$out/$dir" \
|
||||
--out-name catena \
|
||||
--release \
|
||||
-- --features wasm --no-default-features
|
||||
# wasm-pack writes one of these per target and they disagree; the
|
||||
# package.json below is the one that ships.
|
||||
rm -f "$out/$dir/package.json" "$out/$dir/README.md" "$out/$dir/.gitignore"
|
||||
done
|
||||
|
||||
# The node build is CommonJS and the package declares itself ESM, so the entry
|
||||
# has to say so in its extension or node reads `exports.x = …` as a syntax
|
||||
# error. The web build is already ESM and keeps its .js.
|
||||
mv "$out/node/catena.js" "$out/node/catena.cjs"
|
||||
mv "$out/node/catena.d.ts" "$out/node/catena.d.cts"
|
||||
|
||||
sed "s/@VERSION@/$version/" package.json.in >"$out/package.json"
|
||||
cp README.md "$out/README.md"
|
||||
cp ../LICENSE "$out/LICENSE"
|
||||
|
||||
echo "built $out (version $version)"
|
||||
echo "publish with: npm publish $out --access public"
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "catena",
|
||||
"version": "@VERSION@",
|
||||
"description": "Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles and rank fusion, as WebAssembly",
|
||||
"license": "MPL-2.0",
|
||||
"homepage": "https://git.pumpin.online/funkpower/catena",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://git.pumpin.online/funkpower/catena.git"
|
||||
},
|
||||
"keywords": ["chunking", "markdown", "subtitles", "retrieval", "corpus", "tokens", "wasm"],
|
||||
"type": "module",
|
||||
"types": "./web/catena.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"node": { "types": "./node/catena.d.cts", "default": "./node/catena.cjs" },
|
||||
"default": { "types": "./web/catena.d.ts", "default": "./web/catena.js" }
|
||||
},
|
||||
"./node": { "types": "./node/catena.d.cts", "default": "./node/catena.cjs" },
|
||||
"./web": { "types": "./web/catena.d.ts", "default": "./web/catena.js" },
|
||||
"./node/*": "./node/*",
|
||||
"./web/*": "./web/*",
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": ["node/", "web/", "README.md", "LICENSE"]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// Exercise the built package the way a consumer would.
|
||||
import {
|
||||
chunkMarkdown, scrub, countTokens, parseSubtitles, packCues,
|
||||
splitCollection, fuseHits, identify,
|
||||
} from "./dist/node/catena.cjs";
|
||||
|
||||
const md = "# Chapter One\n\n<!-- Page 41 -->\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked about it at all.\n";
|
||||
|
||||
const chunks = chunkMarkdown(md, undefined, undefined, 10);
|
||||
console.log("chunk: ", { idx: chunks[0].idx, page: chunks[0].page, chapter: chunks[0].chapter, tokens: chunks[0].tokens });
|
||||
|
||||
console.log("scrub: ", JSON.stringify(scrub("# Heading\n\n\n\nbody <b>text</b>\n")));
|
||||
console.log("tokens: ", countTokens("a passage of prose"));
|
||||
|
||||
const cues = parseSubtitles("1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n2\n00:00:04,000 --> 00:00:08,000\nof the valley where the road turned\n\n");
|
||||
console.log("cues: ", cues.length, cues[0]);
|
||||
console.log("packed: ", packCues(cues, 5)[0].text);
|
||||
|
||||
const hits = [
|
||||
{ v: 1, id: "c81a", channel: "vec", rank: 1, score: 0.83, payload: {} },
|
||||
{ v: 1, id: "c81a", channel: "fts", rank: 3, score: 11.2, payload: {} },
|
||||
{ v: 1, id: "d42f", channel: "fts", rank: 1, score: 14.0, payload: {} },
|
||||
];
|
||||
console.log("fused: ", fuseHits(hits, undefined, undefined, 2).map((h) => [h.id, +h.score.toFixed(5)]));
|
||||
console.log("weighted: ", fuseHits(hits, undefined, { vec: 2 }, 2).map((h) => [h.id, +h.score.toFixed(5)]));
|
||||
|
||||
console.log("identify: ", identify("book.pdf", new TextEncoder().encode("%PDF-1.4\n")).kind);
|
||||
|
||||
try { splitCollection(md); } catch (why) { console.log("split throws:", String(why).slice(0, 52)); }
|
||||
|
||||
// The types the package promises.
|
||||
const types = await import("node:fs").then((fs) => fs.readFileSync("./dist/web/catena.d.ts", "utf8"));
|
||||
for (const name of ["Chunk", "Cue", "Section", "Hit", "Doc"]) {
|
||||
if (!types.includes(`interface ${name}`)) throw new Error(`no ${name} in the declarations`);
|
||||
}
|
||||
console.log("types: Chunk, Cue, Section, Hit, Doc all declared");
|
||||
@@ -0,0 +1,462 @@
|
||||
//! A C ABI, so this can be embedded rather than spawned.
|
||||
//!
|
||||
//! # Shape
|
||||
//!
|
||||
//! Text goes in as NUL-terminated UTF-8 and records come back as a
|
||||
//! NUL-terminated JSON array — the same document the command line writes,
|
||||
//! under the same stability promise. Nothing here is an opaque handle with
|
||||
//! accessors to keep in step: the records already have a documented shape,
|
||||
//! and every language with an FFI has a JSON parser.
|
||||
//!
|
||||
//! Every string this returns is owned by the caller and must be released with
|
||||
//! [`catena_string_free`]. Passing it anything else is undefined.
|
||||
//!
|
||||
//! # Errors
|
||||
//!
|
||||
//! A function that can fail returns NULL and leaves a message in
|
||||
//! [`catena_last_error`], which is thread-local and valid until the next
|
||||
//! failing call on the same thread.
|
||||
//!
|
||||
//! # Threads
|
||||
//!
|
||||
//! Every function is safe to call from any thread. Nothing here holds state
|
||||
//! between calls but the error message, which is per-thread.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::ffi::{CStr, CString, c_char, c_double, c_int, c_uint};
|
||||
use std::ptr;
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
/// Bumped when the meaning or signature of anything here changes. Additions
|
||||
/// leave it alone.
|
||||
const ABI_VERSION: c_uint = 1;
|
||||
|
||||
thread_local! {
|
||||
static LAST_ERROR: RefCell<Option<CString>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn set_error(message: impl Into<Vec<u8>>) {
|
||||
let message = CString::new(message).unwrap_or_else(|_| c"error".into());
|
||||
LAST_ERROR.with(|slot| *slot.borrow_mut() = Some(message));
|
||||
}
|
||||
|
||||
fn clear_error() {
|
||||
LAST_ERROR.with(|slot| *slot.borrow_mut() = None);
|
||||
}
|
||||
|
||||
/// The version of this ABI.
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn catena_abi_version() -> c_uint {
|
||||
ABI_VERSION
|
||||
}
|
||||
|
||||
/// Why the last call on this thread failed, or NULL if none did.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The pointer is valid until the next failing call on the same thread. Copy
|
||||
/// it if you need it longer.
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn catena_last_error() -> *const c_char {
|
||||
LAST_ERROR.with(|slot| match slot.borrow().as_ref() {
|
||||
Some(message) => message.as_ptr(),
|
||||
None => ptr::null(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Release a string this library returned.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be a pointer this library returned and not yet freed, or NULL.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_string_free(text: *mut c_char) {
|
||||
if !text.is_null() {
|
||||
drop(unsafe { CString::from_raw(text) });
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow a caller's string as `&str`, or fail.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be NUL-terminated and valid for reads, or NULL.
|
||||
unsafe fn borrow<'a>(text: *const c_char, what: &str) -> Option<&'a str> {
|
||||
if text.is_null() {
|
||||
set_error(format!("{what} is NULL"));
|
||||
return None;
|
||||
}
|
||||
match unsafe { CStr::from_ptr(text) }.to_str() {
|
||||
Ok(text) => Some(text),
|
||||
Err(_) => {
|
||||
set_error(format!("{what} is not UTF-8"));
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand a string to the caller, or NULL if it cannot be made into one.
|
||||
fn give(text: String) -> *mut c_char {
|
||||
match CString::new(text) {
|
||||
Ok(text) => text.into_raw(),
|
||||
Err(_) => {
|
||||
// Only reachable if a record held a NUL, which the readers refuse.
|
||||
set_error("the answer contains a NUL byte");
|
||||
ptr::null_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialise records to JSON and hand them over.
|
||||
fn give_json<T: serde::Serialize>(records: &T) -> *mut c_char {
|
||||
match serde_json::to_string(records) {
|
||||
Ok(json) => give(json),
|
||||
Err(e) => {
|
||||
set_error(format!("the answer does not serialise: {e}"));
|
||||
ptr::null_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a JSON argument, or fail.
|
||||
fn take_json<T: serde::de::DeserializeOwned>(json: &str, what: &str) -> Option<T> {
|
||||
match serde_json::from_str(json) {
|
||||
Ok(value) => Some(value),
|
||||
Err(e) => {
|
||||
set_error(format!("{what} is not the expected shape: {e}"));
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cut markdown into retrievable passages. Returns a JSON array of chunks.
|
||||
///
|
||||
/// A budget of zero means the default for that budget, so a caller that only
|
||||
/// cares about one of them can pass zero for the rest.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `markdown` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_chunk_markdown(
|
||||
markdown: *const c_char,
|
||||
target: usize,
|
||||
overlap: usize,
|
||||
min_tokens: usize,
|
||||
hard_max: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let default = ChunkBudgets::default();
|
||||
let budgets = ChunkBudgets {
|
||||
target: or_default(target, default.target),
|
||||
overlap: match overlap {
|
||||
// Zero overlap is a real choice, unlike a zero budget.
|
||||
_ if target == 0 && overlap == 0 => default.overlap,
|
||||
overlap => overlap,
|
||||
},
|
||||
min_tokens: or_default(min_tokens, default.min_tokens),
|
||||
hard_max: or_default(hard_max, default.hard_max),
|
||||
};
|
||||
give_json(&crate::chunk_markdown(markdown, budgets))
|
||||
}
|
||||
|
||||
fn or_default(value: usize, default: usize) -> usize {
|
||||
match value {
|
||||
0 => default,
|
||||
value => value,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clean converted markdown. Returns the cleaned text.
|
||||
///
|
||||
/// `pages` non-zero normalises page markers as well, which is not idempotent.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `markdown` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_scrub(markdown: *const c_char, pages: c_int) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
give(match pages {
|
||||
0 => crate::scrub_markdown(markdown),
|
||||
_ => crate::scrub_markdown_pages(markdown),
|
||||
})
|
||||
}
|
||||
|
||||
/// How many tokens a string is. Returns zero for NULL or invalid UTF-8, and
|
||||
/// leaves a message in [`catena_last_error`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_count_tokens(text: *const c_char) -> usize {
|
||||
clear_error();
|
||||
match unsafe { borrow(text, "text") } {
|
||||
Some(text) => crate::count_tokens(text),
|
||||
None => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read any of the subtitle formats. Returns a JSON array of cues.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `content` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_parse_subtitles(
|
||||
content: *const c_char,
|
||||
keep_repeats: c_int,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(content) = (unsafe { borrow(content, "content") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let cues = crate::parse_subtitles(content);
|
||||
let cues = match keep_repeats {
|
||||
0 => crate::merge_repeats(&cues),
|
||||
_ => cues,
|
||||
};
|
||||
give_json(&cues)
|
||||
}
|
||||
|
||||
/// Pack cues into passages. `cues` is a JSON array of cues; returns a JSON
|
||||
/// array of chunks.
|
||||
///
|
||||
/// Zero for either budget means the default.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `cues` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_pack_cues(
|
||||
cues: *const c_char,
|
||||
seconds: c_double,
|
||||
max_tokens: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(cues) = (unsafe { borrow(cues, "cues") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let Some(cues) = take_json::<Vec<cuekit::Cue>>(cues, "cues") else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let default = CueBudgets::default();
|
||||
let budgets = CueBudgets {
|
||||
seconds: match seconds > 0.0 {
|
||||
true => seconds,
|
||||
false => default.seconds,
|
||||
},
|
||||
max_tokens: or_default(max_tokens, default.max_tokens),
|
||||
};
|
||||
give_json(&crate::pack_cues(&cues, budgets))
|
||||
}
|
||||
|
||||
/// Split a collection into the works inside it. Returns a JSON array of
|
||||
/// sections, or NULL when the document is one work rather than a collection —
|
||||
/// which is an answer, not a failure, and the reason is in
|
||||
/// [`catena_last_error`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `markdown` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_split_collection(markdown: *const c_char) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
match crate::split_collection(markdown) {
|
||||
Ok(sections) => give_json(§ions),
|
||||
Err(why) => {
|
||||
set_error(why);
|
||||
ptr::null_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fuse ranked lists into one. `hits` is a JSON array of hits and `weights` a
|
||||
/// JSON object of channel to weight, or NULL for none. Returns a JSON array.
|
||||
///
|
||||
/// Zero for `k` means the default; zero for `limit` means all of them.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `hits` must be NUL-terminated UTF-8; `weights` likewise, or NULL.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_fuse_hits(
|
||||
hits: *const c_char,
|
||||
k: c_double,
|
||||
weights: *const c_char,
|
||||
limit: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(hits) = (unsafe { borrow(hits, "hits") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let Some(hits) = take_json::<Vec<corpus_core::Hit>>(hits, "hits") else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
|
||||
let weights: Vec<(String, f64)> = if weights.is_null() {
|
||||
Vec::new()
|
||||
} else {
|
||||
let Some(text) = (unsafe { borrow(weights, "weights") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let Some(map) = take_json::<std::collections::HashMap<String, f64>>(text, "weights") else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
map.into_iter().collect()
|
||||
};
|
||||
|
||||
let k = match k > 0.0 {
|
||||
true => k,
|
||||
false => rrf::K,
|
||||
};
|
||||
let limit = match limit {
|
||||
0 => None,
|
||||
limit => Some(limit),
|
||||
};
|
||||
give_json(&crate::fuse_hits(&hits, k, &weights, limit))
|
||||
}
|
||||
|
||||
/// Say what a file is from its first bytes and its name. Returns a JSON doc
|
||||
/// record.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `name` must be NUL-terminated UTF-8, and `data` must be valid for `len`
|
||||
/// bytes.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_identify(
|
||||
name: *const c_char,
|
||||
data: *const u8,
|
||||
len: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(name) = (unsafe { borrow(name, "name") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
if data.is_null() && len != 0 {
|
||||
set_error("data is NULL");
|
||||
return ptr::null_mut();
|
||||
}
|
||||
let bytes: &[u8] = match len {
|
||||
0 => &[],
|
||||
len => unsafe { std::slice::from_raw_parts(data, len) },
|
||||
};
|
||||
give_json(&crate::identify(name, bytes))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Call through the ABI the way a C caller would, and take the answer
|
||||
/// back as a Rust string.
|
||||
fn through(call: impl FnOnce() -> *mut c_char) -> Option<String> {
|
||||
let raw = call();
|
||||
if raw.is_null() {
|
||||
return None;
|
||||
}
|
||||
let text = unsafe { CStr::from_ptr(raw) }
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
unsafe { catena_string_free(raw) };
|
||||
Some(text)
|
||||
}
|
||||
|
||||
fn error() -> Option<String> {
|
||||
let raw = catena_last_error();
|
||||
match raw.is_null() {
|
||||
true => None,
|
||||
false => Some(
|
||||
unsafe { CStr::from_ptr(raw) }
|
||||
.to_string_lossy()
|
||||
.into_owned(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunking_comes_back_as_the_records_the_command_writes() {
|
||||
let markdown = c"# One\n\n<!-- Page 41 -->\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked at all.\n";
|
||||
let json = through(|| unsafe { catena_chunk_markdown(markdown.as_ptr(), 0, 0, 10, 0) })
|
||||
.expect("a result");
|
||||
let chunks: Vec<serde_json::Value> = serde_json::from_str(&json).expect("valid JSON");
|
||||
assert_eq!(chunks[0]["page"], 41);
|
||||
assert_eq!(chunks[0]["chapter"], "One");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_budget_is_the_default_budget() {
|
||||
let markdown = c"# One\n\nbody\n";
|
||||
assert!(
|
||||
through(|| unsafe { catena_chunk_markdown(markdown.as_ptr(), 0, 0, 0, 0) }).is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_argument_is_an_error_rather_than_a_crash() {
|
||||
assert!(through(|| unsafe { catena_chunk_markdown(ptr::null(), 0, 0, 0, 0) }).is_none());
|
||||
assert_eq!(error().as_deref(), Some("markdown is NULL"));
|
||||
assert_eq!(unsafe { catena_count_tokens(ptr::null()) }, 0);
|
||||
assert_eq!(error().as_deref(), Some("text is NULL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn freeing_nothing_is_allowed() {
|
||||
unsafe { catena_string_free(ptr::null_mut()) };
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_that_is_one_work_says_why_rather_than_returning_nothing() {
|
||||
let markdown = c"# Only\n\nbody\n";
|
||||
assert!(through(|| unsafe { catena_split_collection(markdown.as_ptr()) }).is_none());
|
||||
assert!(error().expect("a reason").contains("sections"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_json_argument_says_which_one() {
|
||||
let hits = c"not json";
|
||||
assert!(
|
||||
through(|| unsafe { catena_fuse_hits(hits.as_ptr(), 0.0, ptr::null(), 0) }).is_none()
|
||||
);
|
||||
assert!(
|
||||
error()
|
||||
.expect("a reason")
|
||||
.starts_with("hits is not the expected shape")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifying_nothing_is_identifying_an_empty_file() {
|
||||
let name = c"book.pdf";
|
||||
let json = through(|| unsafe { catena_identify(name.as_ptr(), ptr::null(), 0) })
|
||||
.expect("a result");
|
||||
assert!(json.contains("\"path\":\"book.pdf\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subtitles_round_trip_through_the_boundary() {
|
||||
let srt = c"1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n";
|
||||
let cues = through(|| unsafe { catena_parse_subtitles(srt.as_ptr(), 0) }).expect("cues");
|
||||
let packed = CString::new(cues).expect("no NUL");
|
||||
let chunks =
|
||||
through(|| unsafe { catena_pack_cues(packed.as_ptr(), 0.0, 0) }).expect("chunks");
|
||||
assert!(chunks.contains("the mill stood at the northern end"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_abi_version_is_the_one_the_header_declares() {
|
||||
assert_eq!(catena_abi_version(), 1);
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,14 @@
|
||||
#[cfg(feature = "python")]
|
||||
mod python;
|
||||
|
||||
#[cfg(feature = "capi")]
|
||||
pub mod capi;
|
||||
|
||||
// Only ever compiled for a wasm32 target. The feature can be on elsewhere —
|
||||
// `--all-features` turns everything on — and there it compiles to nothing.
|
||||
#[cfg(all(feature = "wasm", target_arch = "wasm32"))]
|
||||
pub mod wasm;
|
||||
|
||||
use corpus_core::{Chunk, Doc, Hit};
|
||||
use cuekit::Cue;
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
//! JavaScript and TypeScript, through WebAssembly.
|
||||
//!
|
||||
//! # Shape
|
||||
//!
|
||||
//! Records come back as **plain objects**, not handles: the same documents
|
||||
//! the command line writes, with the same field names and the same stability
|
||||
//! promise. Nothing has to be freed, `JSON.stringify` works, and an array of
|
||||
//! chunks spreads into state without ceremony.
|
||||
//!
|
||||
//! # What is not here
|
||||
//!
|
||||
//! The tools that read files or call another program. A browser has no
|
||||
//! `pdftoppm` and no files, and a binding that pretends otherwise is one that
|
||||
//! fails at run time rather than at compile time.
|
||||
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
/// Hand-written because the generated `.d.ts` would otherwise say `any`: a
|
||||
/// binding that types its whole return value as `any` is a binding
|
||||
/// TypeScript cannot check.
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
const TYPES: &'static str = r#"
|
||||
/** A retrievable passage. The same record `chunkmd` writes. */
|
||||
export interface Chunk {
|
||||
v: number;
|
||||
/** Position in the stream, from zero. */
|
||||
idx: number;
|
||||
text: string;
|
||||
/** Tokens, by the encoding the budgets count in. */
|
||||
tokens: number;
|
||||
/** The page it came from, where the source had page markers. */
|
||||
page: number | null;
|
||||
/** The heading it sits under. */
|
||||
chapter: string | null;
|
||||
/** "body", "front", "back", "index" or "bibliography". */
|
||||
role: string;
|
||||
/** Seconds into the recording, for passages made from cues. */
|
||||
start_time: number | null;
|
||||
end_time: number | null;
|
||||
flags: string[];
|
||||
}
|
||||
|
||||
/** One line of a transcript. */
|
||||
export interface Cue {
|
||||
v: number;
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** One work inside a collection. */
|
||||
export interface Section {
|
||||
v: number;
|
||||
idx: number;
|
||||
title: string;
|
||||
slug: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** One scored candidate, before or after fusion. */
|
||||
export interface Hit {
|
||||
v: number;
|
||||
id: string;
|
||||
/** "vec", "fts" or "meta". */
|
||||
channel: string;
|
||||
rank: number;
|
||||
score: number;
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
/** What a file is. */
|
||||
export interface Doc {
|
||||
v: number;
|
||||
path: string;
|
||||
kind: string;
|
||||
declared_ext: string | null;
|
||||
bytes: number;
|
||||
pages: number | null;
|
||||
language: string | null;
|
||||
text_layer: string | null;
|
||||
needs_ocr: boolean | null;
|
||||
notes: string[];
|
||||
}
|
||||
"#;
|
||||
|
||||
/// What the chunker does when nothing is passed.
|
||||
fn chunk_budgets(
|
||||
target: Option<usize>,
|
||||
overlap: Option<usize>,
|
||||
min_tokens: Option<usize>,
|
||||
hard_max: Option<usize>,
|
||||
) -> ChunkBudgets {
|
||||
let default = ChunkBudgets::default();
|
||||
ChunkBudgets {
|
||||
target: target.unwrap_or(default.target),
|
||||
overlap: overlap.unwrap_or(default.overlap),
|
||||
min_tokens: min_tokens.unwrap_or(default.min_tokens),
|
||||
hard_max: hard_max.unwrap_or(default.hard_max),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_js<T: serde::Serialize>(value: &T) -> Result<JsValue, JsValue> {
|
||||
serde_wasm_bindgen::to_value(value).map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
fn from_js<T: serde::de::DeserializeOwned>(value: JsValue, what: &str) -> Result<T, JsValue> {
|
||||
serde_wasm_bindgen::from_value(value)
|
||||
.map_err(|e| JsValue::from_str(&format!("{what} is not the expected shape: {e}")))
|
||||
}
|
||||
|
||||
/// Cut markdown into retrievable passages.
|
||||
#[wasm_bindgen(js_name = chunkMarkdown, skip_typescript)]
|
||||
pub fn chunk_markdown(
|
||||
markdown: &str,
|
||||
target: Option<usize>,
|
||||
overlap: Option<usize>,
|
||||
min_tokens: Option<usize>,
|
||||
hard_max: Option<usize>,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let budgets = chunk_budgets(target, overlap, min_tokens, hard_max);
|
||||
to_js(&crate::chunk_markdown(markdown, budgets))
|
||||
}
|
||||
|
||||
/// Clean converted markdown.
|
||||
///
|
||||
/// `pages` also normalises page markers, which is not idempotent.
|
||||
#[wasm_bindgen(js_name = scrub)]
|
||||
pub fn scrub(markdown: &str, pages: Option<bool>) -> String {
|
||||
match pages.unwrap_or(false) {
|
||||
true => crate::scrub_markdown_pages(markdown),
|
||||
false => crate::scrub_markdown(markdown),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many tokens a string is.
|
||||
#[wasm_bindgen(js_name = countTokens)]
|
||||
pub fn count_tokens(text: &str) -> usize {
|
||||
crate::count_tokens(text)
|
||||
}
|
||||
|
||||
/// Read any of the subtitle formats into cues.
|
||||
#[wasm_bindgen(js_name = parseSubtitles, skip_typescript)]
|
||||
pub fn parse_subtitles(content: &str, keep_repeats: Option<bool>) -> Result<JsValue, JsValue> {
|
||||
let cues = crate::parse_subtitles(content);
|
||||
let cues = match keep_repeats.unwrap_or(false) {
|
||||
true => cues,
|
||||
false => crate::merge_repeats(&cues),
|
||||
};
|
||||
to_js(&cues)
|
||||
}
|
||||
|
||||
/// Pack cues into passages worth embedding.
|
||||
#[wasm_bindgen(js_name = packCues, skip_typescript)]
|
||||
pub fn pack_cues(
|
||||
cues: JsValue,
|
||||
seconds: Option<f64>,
|
||||
max_tokens: Option<usize>,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let default = CueBudgets::default();
|
||||
let budgets = CueBudgets {
|
||||
seconds: seconds.unwrap_or(default.seconds),
|
||||
max_tokens: max_tokens.unwrap_or(default.max_tokens),
|
||||
};
|
||||
let cues: Vec<cuekit::Cue> = from_js(cues, "cues")?;
|
||||
to_js(&crate::pack_cues(&cues, budgets))
|
||||
}
|
||||
|
||||
/// Split a collection into the works inside it.
|
||||
///
|
||||
/// Throws when the document is one work rather than a collection, which is
|
||||
/// the usual answer, with the reason each candidate was rejected.
|
||||
#[wasm_bindgen(js_name = splitCollection, skip_typescript)]
|
||||
pub fn split_collection(markdown: &str) -> Result<JsValue, JsValue> {
|
||||
match crate::split_collection(markdown) {
|
||||
Ok(sections) => to_js(§ions),
|
||||
Err(why) => Err(JsValue::from_str(&why)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fuse ranked lists into one.
|
||||
#[wasm_bindgen(js_name = fuseHits, skip_typescript)]
|
||||
pub fn fuse_hits(
|
||||
hits: JsValue,
|
||||
k: Option<f64>,
|
||||
weights: JsValue,
|
||||
limit: Option<usize>,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let hits: Vec<corpus_core::Hit> = from_js(hits, "hits")?;
|
||||
let weights: Vec<(String, f64)> = match weights.is_undefined() || weights.is_null() {
|
||||
true => Vec::new(),
|
||||
false => from_js::<std::collections::HashMap<String, f64>>(weights, "weights")?
|
||||
.into_iter()
|
||||
.collect(),
|
||||
};
|
||||
to_js(&crate::fuse_hits(
|
||||
&hits,
|
||||
k.unwrap_or(rrf::K),
|
||||
&weights,
|
||||
limit,
|
||||
))
|
||||
}
|
||||
|
||||
/// Say what a file is from its first bytes and its name.
|
||||
#[wasm_bindgen(js_name = identify, skip_typescript)]
|
||||
pub fn identify(name: &str, data: &[u8]) -> Result<JsValue, JsValue> {
|
||||
to_js(&crate::identify(name, data))
|
||||
}
|
||||
|
||||
/// Declarations for the functions whose return type the generator cannot
|
||||
/// infer, so a caller gets `Chunk[]` rather than `any`.
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
const SIGNATURES: &'static str = r#"
|
||||
export function chunkMarkdown(markdown: string, target?: number, overlap?: number, min_tokens?: number, hard_max?: number): Chunk[];
|
||||
export function parseSubtitles(content: string, keep_repeats?: boolean): Cue[];
|
||||
export function packCues(cues: Cue[], seconds?: number, max_tokens?: number): Chunk[];
|
||||
export function splitCollection(markdown: string): Section[];
|
||||
export function fuseHits(hits: Hit[], k?: number, weights?: Record<string, number>, limit?: number): Hit[];
|
||||
export function identify(name: string, data: Uint8Array): Doc;
|
||||
"#;
|
||||
Reference in New Issue
Block a user