mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
publish as catena-corpus on PyPI and npm
Both registries already have a `catena`: an HPE blockchain client on PyPI,
and a JavaScript OOP wrapper at 2.2.0 on npm. Neither is going anywhere, so
the distributions take a name that is free on both.
The Python import is unaffected — a distribution name and a module name are
independent there, so `pip install catena-corpus` still gives `import
catena`, and nothing in the examples or the bindings changes. On npm the
package name is the import specifier, so that one does change: `import
{ chunkMarkdown } from "catena-corpus"`.
The crates keep their own names. `catena` is free on crates.io.
This commit is contained in:
@@ -77,8 +77,8 @@ callable in process from three languages:
|
|||||||
|
|
||||||
| | Install | Import |
|
| | Install | Import |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| **Python** | `pip install catena` | `import catena` |
|
| **Python** | `pip install catena-corpus` | `import catena` |
|
||||||
| **JavaScript** | `npm install catena` | `import { chunkMarkdown } from "catena"` |
|
| **JavaScript** | `npm install catena-corpus` | `import { chunkMarkdown } from "catena-corpus"` |
|
||||||
| **C** | build the `capi` feature | `#include <catena.h>` |
|
| **C** | build the `capi` feature | `#include <catena.h>` |
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ call the same Rust and hand back the same records.
|
|||||||
## Python
|
## Python
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pip install catena
|
pip install catena-corpus
|
||||||
```
|
```
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -100,11 +100,11 @@ server, and none of that is cheaper in process.
|
|||||||
## JavaScript
|
## JavaScript
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
npm install catena
|
npm install catena-corpus
|
||||||
```
|
```
|
||||||
|
|
||||||
```js
|
```js
|
||||||
import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena";
|
import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena-corpus";
|
||||||
|
|
||||||
const chunks = chunkMarkdown(markdown, 512, 64);
|
const chunks = chunkMarkdown(markdown, 512, 64);
|
||||||
chunks[0].page; // 41
|
chunks[0].page; // 41
|
||||||
@@ -125,7 +125,7 @@ Two builds, because the loaders differ — node reads the `.wasm` off disk at
|
|||||||
import, the web build takes a URL a bundler rewrites:
|
import, the web build takes a URL a bundler rewrites:
|
||||||
|
|
||||||
```js
|
```js
|
||||||
import { chunkMarkdown } from "catena"; // resolves per environment
|
import { chunkMarkdown } from "catena-corpus"; // resolves per environment
|
||||||
import { chunkMarkdown } from "catena/node"; // or pick one
|
import { chunkMarkdown } from "catena/node"; // or pick one
|
||||||
import { chunkMarkdown } from "catena/web";
|
import { chunkMarkdown } from "catena/web";
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -4,11 +4,11 @@ Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles
|
|||||||
and rank fusion, as WebAssembly.
|
and rank fusion, as WebAssembly.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
npm install catena
|
npm install catena-corpus
|
||||||
```
|
```
|
||||||
|
|
||||||
```js
|
```js
|
||||||
import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena";
|
import { chunkMarkdown, scrub, countTokens, fuseHits } from "catena-corpus";
|
||||||
|
|
||||||
const chunks = chunkMarkdown(markdown, 512, 64);
|
const chunks = chunkMarkdown(markdown, 512, 64);
|
||||||
chunks[0].page; // 41
|
chunks[0].page; // 41
|
||||||
@@ -60,7 +60,7 @@ at compile time. Those stay command-line tools.
|
|||||||
## Node and the web
|
## Node and the web
|
||||||
|
|
||||||
```js
|
```js
|
||||||
import { chunkMarkdown } from "catena"; // resolves per environment
|
import { chunkMarkdown } from "catena-corpus"; // resolves per environment
|
||||||
import { chunkMarkdown } from "catena/node"; // or pick one
|
import { chunkMarkdown } from "catena/node"; // or pick one
|
||||||
import { chunkMarkdown } from "catena/web";
|
import { chunkMarkdown } from "catena/web";
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"name": "catena",
|
"name": "catena-corpus",
|
||||||
"version": "@VERSION@",
|
"version": "@VERSION@",
|
||||||
"description": "Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles and rank fusion, as WebAssembly",
|
"description": "Turn a pile of files into a searchable corpus: chunking, scrubbing, subtitles and rank fusion, as WebAssembly",
|
||||||
"license": "MPL-2.0",
|
"license": "MPL-2.0",
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ requires = ["maturin>=1.9,<2.0"]
|
|||||||
build-backend = "maturin"
|
build-backend = "maturin"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "catena"
|
name = "catena-corpus"
|
||||||
description = "The catena libraries behind one surface, for calling in process rather than as a subprocess"
|
description = "The catena libraries behind one surface, for calling in process rather than as a subprocess"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "MPL-2.0"
|
license = "MPL-2.0"
|
||||||
|
|||||||
+2
-2
@@ -75,12 +75,12 @@ convert, then loop, and what is left is the part that is actually about your
|
|||||||
schema.
|
schema.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pip install catena
|
pip install catena-corpus
|
||||||
python examples/python/book_to_citations.py
|
python examples/python/book_to_citations.py
|
||||||
```
|
```
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pip install catena psycopg
|
pip install catena-corpus psycopg
|
||||||
DATABASE_URL=postgresql://user@host/db \
|
DATABASE_URL=postgresql://user@host/db \
|
||||||
python examples/python/search_and_rerank.py "why did the empire decline"
|
python examples/python/search_and_rerank.py "why did the empire decline"
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
"search": "node --experimental-strip-types src/search-and-rerank.ts"
|
"search": "node --experimental-strip-types src/search-and-rerank.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"catena": "file:../../crates/bindings/npm/dist"
|
"catena-corpus": "file:../../crates/bindings/npm/dist"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^22"
|
"@types/node": "^22"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
// broadcast — a work of the US government, so public domain.
|
// broadcast — a work of the US government, so public domain.
|
||||||
|
|
||||||
import { readFileSync } from "node:fs";
|
import { readFileSync } from "node:fs";
|
||||||
import { parseSubtitles, segmentCues, packCues, countTokens } from "catena";
|
import { parseSubtitles, segmentCues, packCues, countTokens } from "catena-corpus";
|
||||||
|
|
||||||
const SAMPLE = `WEBVTT
|
const SAMPLE = `WEBVTT
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// vector database and whatever else. What they came from does not matter
|
// vector database and whatever else. What they came from does not matter
|
||||||
// here; only where each of them placed does.
|
// here; only where each of them placed does.
|
||||||
|
|
||||||
import { fuseHits, type Hit } from "catena";
|
import { fuseHits, type Hit } from "catena-corpus";
|
||||||
|
|
||||||
// `payload` is whatever you want carried alongside — rrf hands it back
|
// `payload` is whatever you want carried alongside — rrf hands it back
|
||||||
// untouched, so a hit can arrive here with the row it came from attached.
|
// untouched, so a hit can arrive here with the row it came from attached.
|
||||||
|
|||||||
Reference in New Issue
Block a user