mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
bindings: a C ABI and an npm package alongside the wheel
This commit is contained in:
funkpower
@@ -0,0 +1,462 @@
|
||||
//! A C ABI, so this can be embedded rather than spawned.
|
||||
//!
|
||||
//! # Shape
|
||||
//!
|
||||
//! Text goes in as NUL-terminated UTF-8 and records come back as a
|
||||
//! NUL-terminated JSON array — the same document the command line writes,
|
||||
//! under the same stability promise. Nothing here is an opaque handle with
|
||||
//! accessors to keep in step: the records already have a documented shape,
|
||||
//! and every language with an FFI has a JSON parser.
|
||||
//!
|
||||
//! Every string this returns is owned by the caller and must be released with
|
||||
//! [`catena_string_free`]. Passing it anything else is undefined.
|
||||
//!
|
||||
//! # Errors
|
||||
//!
|
||||
//! A function that can fail returns NULL and leaves a message in
|
||||
//! [`catena_last_error`], which is thread-local and valid until the next
|
||||
//! failing call on the same thread.
|
||||
//!
|
||||
//! # Threads
|
||||
//!
|
||||
//! Every function is safe to call from any thread. Nothing here holds state
|
||||
//! between calls but the error message, which is per-thread.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::ffi::{CStr, CString, c_char, c_double, c_int, c_uint};
|
||||
use std::ptr;
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
/// Bumped when the meaning or signature of anything here changes. Additions
|
||||
/// leave it alone.
|
||||
const ABI_VERSION: c_uint = 1;
|
||||
|
||||
thread_local! {
|
||||
static LAST_ERROR: RefCell<Option<CString>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn set_error(message: impl Into<Vec<u8>>) {
|
||||
let message = CString::new(message).unwrap_or_else(|_| c"error".into());
|
||||
LAST_ERROR.with(|slot| *slot.borrow_mut() = Some(message));
|
||||
}
|
||||
|
||||
fn clear_error() {
|
||||
LAST_ERROR.with(|slot| *slot.borrow_mut() = None);
|
||||
}
|
||||
|
||||
/// The version of this ABI.
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn catena_abi_version() -> c_uint {
|
||||
ABI_VERSION
|
||||
}
|
||||
|
||||
/// Why the last call on this thread failed, or NULL if none did.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The pointer is valid until the next failing call on the same thread. Copy
|
||||
/// it if you need it longer.
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn catena_last_error() -> *const c_char {
|
||||
LAST_ERROR.with(|slot| match slot.borrow().as_ref() {
|
||||
Some(message) => message.as_ptr(),
|
||||
None => ptr::null(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Release a string this library returned.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be a pointer this library returned and not yet freed, or NULL.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_string_free(text: *mut c_char) {
|
||||
if !text.is_null() {
|
||||
drop(unsafe { CString::from_raw(text) });
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow a caller's string as `&str`, or fail.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be NUL-terminated and valid for reads, or NULL.
|
||||
unsafe fn borrow<'a>(text: *const c_char, what: &str) -> Option<&'a str> {
|
||||
if text.is_null() {
|
||||
set_error(format!("{what} is NULL"));
|
||||
return None;
|
||||
}
|
||||
match unsafe { CStr::from_ptr(text) }.to_str() {
|
||||
Ok(text) => Some(text),
|
||||
Err(_) => {
|
||||
set_error(format!("{what} is not UTF-8"));
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand a string to the caller, or NULL if it cannot be made into one.
|
||||
fn give(text: String) -> *mut c_char {
|
||||
match CString::new(text) {
|
||||
Ok(text) => text.into_raw(),
|
||||
Err(_) => {
|
||||
// Only reachable if a record held a NUL, which the readers refuse.
|
||||
set_error("the answer contains a NUL byte");
|
||||
ptr::null_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialise records to JSON and hand them over.
|
||||
fn give_json<T: serde::Serialize>(records: &T) -> *mut c_char {
|
||||
match serde_json::to_string(records) {
|
||||
Ok(json) => give(json),
|
||||
Err(e) => {
|
||||
set_error(format!("the answer does not serialise: {e}"));
|
||||
ptr::null_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a JSON argument, or fail.
|
||||
fn take_json<T: serde::de::DeserializeOwned>(json: &str, what: &str) -> Option<T> {
|
||||
match serde_json::from_str(json) {
|
||||
Ok(value) => Some(value),
|
||||
Err(e) => {
|
||||
set_error(format!("{what} is not the expected shape: {e}"));
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cut markdown into retrievable passages. Returns a JSON array of chunks.
|
||||
///
|
||||
/// A budget of zero means the default for that budget, so a caller that only
|
||||
/// cares about one of them can pass zero for the rest.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `markdown` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_chunk_markdown(
|
||||
markdown: *const c_char,
|
||||
target: usize,
|
||||
overlap: usize,
|
||||
min_tokens: usize,
|
||||
hard_max: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let default = ChunkBudgets::default();
|
||||
let budgets = ChunkBudgets {
|
||||
target: or_default(target, default.target),
|
||||
overlap: match overlap {
|
||||
// Zero overlap is a real choice, unlike a zero budget.
|
||||
_ if target == 0 && overlap == 0 => default.overlap,
|
||||
overlap => overlap,
|
||||
},
|
||||
min_tokens: or_default(min_tokens, default.min_tokens),
|
||||
hard_max: or_default(hard_max, default.hard_max),
|
||||
};
|
||||
give_json(&crate::chunk_markdown(markdown, budgets))
|
||||
}
|
||||
|
||||
fn or_default(value: usize, default: usize) -> usize {
|
||||
match value {
|
||||
0 => default,
|
||||
value => value,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clean converted markdown. Returns the cleaned text.
|
||||
///
|
||||
/// `pages` non-zero normalises page markers as well, which is not idempotent.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `markdown` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_scrub(markdown: *const c_char, pages: c_int) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
give(match pages {
|
||||
0 => crate::scrub_markdown(markdown),
|
||||
_ => crate::scrub_markdown_pages(markdown),
|
||||
})
|
||||
}
|
||||
|
||||
/// How many tokens a string is. Returns zero for NULL or invalid UTF-8, and
|
||||
/// leaves a message in [`catena_last_error`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `text` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_count_tokens(text: *const c_char) -> usize {
|
||||
clear_error();
|
||||
match unsafe { borrow(text, "text") } {
|
||||
Some(text) => crate::count_tokens(text),
|
||||
None => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read any of the subtitle formats. Returns a JSON array of cues.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `content` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_parse_subtitles(
|
||||
content: *const c_char,
|
||||
keep_repeats: c_int,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(content) = (unsafe { borrow(content, "content") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let cues = crate::parse_subtitles(content);
|
||||
let cues = match keep_repeats {
|
||||
0 => crate::merge_repeats(&cues),
|
||||
_ => cues,
|
||||
};
|
||||
give_json(&cues)
|
||||
}
|
||||
|
||||
/// Pack cues into passages. `cues` is a JSON array of cues; returns a JSON
|
||||
/// array of chunks.
|
||||
///
|
||||
/// Zero for either budget means the default.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `cues` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_pack_cues(
|
||||
cues: *const c_char,
|
||||
seconds: c_double,
|
||||
max_tokens: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(cues) = (unsafe { borrow(cues, "cues") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let Some(cues) = take_json::<Vec<cuekit::Cue>>(cues, "cues") else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let default = CueBudgets::default();
|
||||
let budgets = CueBudgets {
|
||||
seconds: match seconds > 0.0 {
|
||||
true => seconds,
|
||||
false => default.seconds,
|
||||
},
|
||||
max_tokens: or_default(max_tokens, default.max_tokens),
|
||||
};
|
||||
give_json(&crate::pack_cues(&cues, budgets))
|
||||
}
|
||||
|
||||
/// Split a collection into the works inside it. Returns a JSON array of
|
||||
/// sections, or NULL when the document is one work rather than a collection —
|
||||
/// which is an answer, not a failure, and the reason is in
|
||||
/// [`catena_last_error`].
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `markdown` must be NUL-terminated UTF-8.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_split_collection(markdown: *const c_char) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(markdown) = (unsafe { borrow(markdown, "markdown") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
match crate::split_collection(markdown) {
|
||||
Ok(sections) => give_json(§ions),
|
||||
Err(why) => {
|
||||
set_error(why);
|
||||
ptr::null_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fuse ranked lists into one. `hits` is a JSON array of hits and `weights` a
|
||||
/// JSON object of channel to weight, or NULL for none. Returns a JSON array.
|
||||
///
|
||||
/// Zero for `k` means the default; zero for `limit` means all of them.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `hits` must be NUL-terminated UTF-8; `weights` likewise, or NULL.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_fuse_hits(
|
||||
hits: *const c_char,
|
||||
k: c_double,
|
||||
weights: *const c_char,
|
||||
limit: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(hits) = (unsafe { borrow(hits, "hits") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let Some(hits) = take_json::<Vec<corpus_core::Hit>>(hits, "hits") else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
|
||||
let weights: Vec<(String, f64)> = if weights.is_null() {
|
||||
Vec::new()
|
||||
} else {
|
||||
let Some(text) = (unsafe { borrow(weights, "weights") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
let Some(map) = take_json::<std::collections::HashMap<String, f64>>(text, "weights") else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
map.into_iter().collect()
|
||||
};
|
||||
|
||||
let k = match k > 0.0 {
|
||||
true => k,
|
||||
false => rrf::K,
|
||||
};
|
||||
let limit = match limit {
|
||||
0 => None,
|
||||
limit => Some(limit),
|
||||
};
|
||||
give_json(&crate::fuse_hits(&hits, k, &weights, limit))
|
||||
}
|
||||
|
||||
/// Say what a file is from its first bytes and its name. Returns a JSON doc
|
||||
/// record.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `name` must be NUL-terminated UTF-8, and `data` must be valid for `len`
|
||||
/// bytes.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn catena_identify(
|
||||
name: *const c_char,
|
||||
data: *const u8,
|
||||
len: usize,
|
||||
) -> *mut c_char {
|
||||
clear_error();
|
||||
let Some(name) = (unsafe { borrow(name, "name") }) else {
|
||||
return ptr::null_mut();
|
||||
};
|
||||
if data.is_null() && len != 0 {
|
||||
set_error("data is NULL");
|
||||
return ptr::null_mut();
|
||||
}
|
||||
let bytes: &[u8] = match len {
|
||||
0 => &[],
|
||||
len => unsafe { std::slice::from_raw_parts(data, len) },
|
||||
};
|
||||
give_json(&crate::identify(name, bytes))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Call through the ABI the way a C caller would, and take the answer
|
||||
/// back as a Rust string.
|
||||
fn through(call: impl FnOnce() -> *mut c_char) -> Option<String> {
|
||||
let raw = call();
|
||||
if raw.is_null() {
|
||||
return None;
|
||||
}
|
||||
let text = unsafe { CStr::from_ptr(raw) }
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
unsafe { catena_string_free(raw) };
|
||||
Some(text)
|
||||
}
|
||||
|
||||
fn error() -> Option<String> {
|
||||
let raw = catena_last_error();
|
||||
match raw.is_null() {
|
||||
true => None,
|
||||
false => Some(
|
||||
unsafe { CStr::from_ptr(raw) }
|
||||
.to_string_lossy()
|
||||
.into_owned(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunking_comes_back_as_the_records_the_command_writes() {
|
||||
let markdown = c"# One\n\n<!-- Page 41 -->\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked at all.\n";
|
||||
let json = through(|| unsafe { catena_chunk_markdown(markdown.as_ptr(), 0, 0, 10, 0) })
|
||||
.expect("a result");
|
||||
let chunks: Vec<serde_json::Value> = serde_json::from_str(&json).expect("valid JSON");
|
||||
assert_eq!(chunks[0]["page"], 41);
|
||||
assert_eq!(chunks[0]["chapter"], "One");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_budget_is_the_default_budget() {
|
||||
let markdown = c"# One\n\nbody\n";
|
||||
assert!(
|
||||
through(|| unsafe { catena_chunk_markdown(markdown.as_ptr(), 0, 0, 0, 0) }).is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_argument_is_an_error_rather_than_a_crash() {
|
||||
assert!(through(|| unsafe { catena_chunk_markdown(ptr::null(), 0, 0, 0, 0) }).is_none());
|
||||
assert_eq!(error().as_deref(), Some("markdown is NULL"));
|
||||
assert_eq!(unsafe { catena_count_tokens(ptr::null()) }, 0);
|
||||
assert_eq!(error().as_deref(), Some("text is NULL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn freeing_nothing_is_allowed() {
|
||||
unsafe { catena_string_free(ptr::null_mut()) };
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_that_is_one_work_says_why_rather_than_returning_nothing() {
|
||||
let markdown = c"# Only\n\nbody\n";
|
||||
assert!(through(|| unsafe { catena_split_collection(markdown.as_ptr()) }).is_none());
|
||||
assert!(error().expect("a reason").contains("sections"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_json_argument_says_which_one() {
|
||||
let hits = c"not json";
|
||||
assert!(
|
||||
through(|| unsafe { catena_fuse_hits(hits.as_ptr(), 0.0, ptr::null(), 0) }).is_none()
|
||||
);
|
||||
assert!(
|
||||
error()
|
||||
.expect("a reason")
|
||||
.starts_with("hits is not the expected shape")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifying_nothing_is_identifying_an_empty_file() {
|
||||
let name = c"book.pdf";
|
||||
let json = through(|| unsafe { catena_identify(name.as_ptr(), ptr::null(), 0) })
|
||||
.expect("a result");
|
||||
assert!(json.contains("\"path\":\"book.pdf\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subtitles_round_trip_through_the_boundary() {
|
||||
let srt = c"1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n";
|
||||
let cues = through(|| unsafe { catena_parse_subtitles(srt.as_ptr(), 0) }).expect("cues");
|
||||
let packed = CString::new(cues).expect("no NUL");
|
||||
let chunks =
|
||||
through(|| unsafe { catena_pack_cues(packed.as_ptr(), 0.0, 0) }).expect("chunks");
|
||||
assert!(chunks.contains("the mill stood at the northern end"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_abi_version_is_the_one_the_header_declares() {
|
||||
assert_eq!(catena_abi_version(), 1);
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,14 @@
|
||||
#[cfg(feature = "python")]
|
||||
mod python;
|
||||
|
||||
#[cfg(feature = "capi")]
|
||||
pub mod capi;
|
||||
|
||||
// Only ever compiled for a wasm32 target. The feature can be on elsewhere —
|
||||
// `--all-features` turns everything on — and there it compiles to nothing.
|
||||
#[cfg(all(feature = "wasm", target_arch = "wasm32"))]
|
||||
pub mod wasm;
|
||||
|
||||
use corpus_core::{Chunk, Doc, Hit};
|
||||
use cuekit::Cue;
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
//! JavaScript and TypeScript, through WebAssembly.
|
||||
//!
|
||||
//! # Shape
|
||||
//!
|
||||
//! Records come back as **plain objects**, not handles: the same documents
|
||||
//! the command line writes, with the same field names and the same stability
|
||||
//! promise. Nothing has to be freed, `JSON.stringify` works, and an array of
|
||||
//! chunks spreads into state without ceremony.
|
||||
//!
|
||||
//! # What is not here
|
||||
//!
|
||||
//! The tools that read files or call another program. A browser has no
|
||||
//! `pdftoppm` and no files, and a binding that pretends otherwise is one that
|
||||
//! fails at run time rather than at compile time.
|
||||
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
/// Hand-written because the generated `.d.ts` would otherwise say `any`: a
|
||||
/// binding that types its whole return value as `any` is a binding
|
||||
/// TypeScript cannot check.
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
const TYPES: &'static str = r#"
|
||||
/** A retrievable passage. The same record `chunkmd` writes. */
|
||||
export interface Chunk {
|
||||
v: number;
|
||||
/** Position in the stream, from zero. */
|
||||
idx: number;
|
||||
text: string;
|
||||
/** Tokens, by the encoding the budgets count in. */
|
||||
tokens: number;
|
||||
/** The page it came from, where the source had page markers. */
|
||||
page: number | null;
|
||||
/** The heading it sits under. */
|
||||
chapter: string | null;
|
||||
/** "body", "front", "back", "index" or "bibliography". */
|
||||
role: string;
|
||||
/** Seconds into the recording, for passages made from cues. */
|
||||
start_time: number | null;
|
||||
end_time: number | null;
|
||||
flags: string[];
|
||||
}
|
||||
|
||||
/** One line of a transcript. */
|
||||
export interface Cue {
|
||||
v: number;
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** One work inside a collection. */
|
||||
export interface Section {
|
||||
v: number;
|
||||
idx: number;
|
||||
title: string;
|
||||
slug: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** One scored candidate, before or after fusion. */
|
||||
export interface Hit {
|
||||
v: number;
|
||||
id: string;
|
||||
/** "vec", "fts" or "meta". */
|
||||
channel: string;
|
||||
rank: number;
|
||||
score: number;
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
/** What a file is. */
|
||||
export interface Doc {
|
||||
v: number;
|
||||
path: string;
|
||||
kind: string;
|
||||
declared_ext: string | null;
|
||||
bytes: number;
|
||||
pages: number | null;
|
||||
language: string | null;
|
||||
text_layer: string | null;
|
||||
needs_ocr: boolean | null;
|
||||
notes: string[];
|
||||
}
|
||||
"#;
|
||||
|
||||
/// What the chunker does when nothing is passed.
|
||||
fn chunk_budgets(
|
||||
target: Option<usize>,
|
||||
overlap: Option<usize>,
|
||||
min_tokens: Option<usize>,
|
||||
hard_max: Option<usize>,
|
||||
) -> ChunkBudgets {
|
||||
let default = ChunkBudgets::default();
|
||||
ChunkBudgets {
|
||||
target: target.unwrap_or(default.target),
|
||||
overlap: overlap.unwrap_or(default.overlap),
|
||||
min_tokens: min_tokens.unwrap_or(default.min_tokens),
|
||||
hard_max: hard_max.unwrap_or(default.hard_max),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_js<T: serde::Serialize>(value: &T) -> Result<JsValue, JsValue> {
|
||||
serde_wasm_bindgen::to_value(value).map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
fn from_js<T: serde::de::DeserializeOwned>(value: JsValue, what: &str) -> Result<T, JsValue> {
|
||||
serde_wasm_bindgen::from_value(value)
|
||||
.map_err(|e| JsValue::from_str(&format!("{what} is not the expected shape: {e}")))
|
||||
}
|
||||
|
||||
/// Cut markdown into retrievable passages.
|
||||
#[wasm_bindgen(js_name = chunkMarkdown, skip_typescript)]
|
||||
pub fn chunk_markdown(
|
||||
markdown: &str,
|
||||
target: Option<usize>,
|
||||
overlap: Option<usize>,
|
||||
min_tokens: Option<usize>,
|
||||
hard_max: Option<usize>,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let budgets = chunk_budgets(target, overlap, min_tokens, hard_max);
|
||||
to_js(&crate::chunk_markdown(markdown, budgets))
|
||||
}
|
||||
|
||||
/// Clean converted markdown.
|
||||
///
|
||||
/// `pages` also normalises page markers, which is not idempotent.
|
||||
#[wasm_bindgen(js_name = scrub)]
|
||||
pub fn scrub(markdown: &str, pages: Option<bool>) -> String {
|
||||
match pages.unwrap_or(false) {
|
||||
true => crate::scrub_markdown_pages(markdown),
|
||||
false => crate::scrub_markdown(markdown),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many tokens a string is.
|
||||
#[wasm_bindgen(js_name = countTokens)]
|
||||
pub fn count_tokens(text: &str) -> usize {
|
||||
crate::count_tokens(text)
|
||||
}
|
||||
|
||||
/// Read any of the subtitle formats into cues.
|
||||
#[wasm_bindgen(js_name = parseSubtitles, skip_typescript)]
|
||||
pub fn parse_subtitles(content: &str, keep_repeats: Option<bool>) -> Result<JsValue, JsValue> {
|
||||
let cues = crate::parse_subtitles(content);
|
||||
let cues = match keep_repeats.unwrap_or(false) {
|
||||
true => cues,
|
||||
false => crate::merge_repeats(&cues),
|
||||
};
|
||||
to_js(&cues)
|
||||
}
|
||||
|
||||
/// Pack cues into passages worth embedding.
|
||||
#[wasm_bindgen(js_name = packCues, skip_typescript)]
|
||||
pub fn pack_cues(
|
||||
cues: JsValue,
|
||||
seconds: Option<f64>,
|
||||
max_tokens: Option<usize>,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let default = CueBudgets::default();
|
||||
let budgets = CueBudgets {
|
||||
seconds: seconds.unwrap_or(default.seconds),
|
||||
max_tokens: max_tokens.unwrap_or(default.max_tokens),
|
||||
};
|
||||
let cues: Vec<cuekit::Cue> = from_js(cues, "cues")?;
|
||||
to_js(&crate::pack_cues(&cues, budgets))
|
||||
}
|
||||
|
||||
/// Split a collection into the works inside it.
|
||||
///
|
||||
/// Throws when the document is one work rather than a collection, which is
|
||||
/// the usual answer, with the reason each candidate was rejected.
|
||||
#[wasm_bindgen(js_name = splitCollection, skip_typescript)]
|
||||
pub fn split_collection(markdown: &str) -> Result<JsValue, JsValue> {
|
||||
match crate::split_collection(markdown) {
|
||||
Ok(sections) => to_js(§ions),
|
||||
Err(why) => Err(JsValue::from_str(&why)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fuse ranked lists into one.
|
||||
#[wasm_bindgen(js_name = fuseHits, skip_typescript)]
|
||||
pub fn fuse_hits(
|
||||
hits: JsValue,
|
||||
k: Option<f64>,
|
||||
weights: JsValue,
|
||||
limit: Option<usize>,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let hits: Vec<corpus_core::Hit> = from_js(hits, "hits")?;
|
||||
let weights: Vec<(String, f64)> = match weights.is_undefined() || weights.is_null() {
|
||||
true => Vec::new(),
|
||||
false => from_js::<std::collections::HashMap<String, f64>>(weights, "weights")?
|
||||
.into_iter()
|
||||
.collect(),
|
||||
};
|
||||
to_js(&crate::fuse_hits(
|
||||
&hits,
|
||||
k.unwrap_or(rrf::K),
|
||||
&weights,
|
||||
limit,
|
||||
))
|
||||
}
|
||||
|
||||
/// Say what a file is from its first bytes and its name.
|
||||
#[wasm_bindgen(js_name = identify, skip_typescript)]
|
||||
pub fn identify(name: &str, data: &[u8]) -> Result<JsValue, JsValue> {
|
||||
to_js(&crate::identify(name, data))
|
||||
}
|
||||
|
||||
/// Declarations for the functions whose return type the generator cannot
|
||||
/// infer, so a caller gets `Chunk[]` rather than `any`.
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
const SIGNATURES: &'static str = r#"
|
||||
export function chunkMarkdown(markdown: string, target?: number, overlap?: number, min_tokens?: number, hard_max?: number): Chunk[];
|
||||
export function parseSubtitles(content: string, keep_repeats?: boolean): Cue[];
|
||||
export function packCues(cues: Cue[], seconds?: number, max_tokens?: number): Chunk[];
|
||||
export function splitCollection(markdown: string): Section[];
|
||||
export function fuseHits(hits: Hit[], k?: number, weights?: Record<string, number>, limit?: number): Hit[];
|
||||
export function identify(name: string, data: Uint8Array): Doc;
|
||||
"#;
|
||||
Reference in New Issue
Block a user