funkpower/catena
1
0
Fork 0
mirror of https://git.pumpin.online/funkpower/catena.git synced 2026-10-22 18:57:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

textkit: token counting and the two segmenters

Browse Source
This commit is contained in:
funkpower
2026-09-19 11:04:27 -04:00
parent 00f4c38816
commit 0208b14b5d
6 changed files with 454 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+2
View File
@@ -11,6 +11,7 @@ repository = "https://git.pumpin.online/funkpower/catena"
[workspace.dependencies]
corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
textkit = { path = "crates/textkit", version = "0.1.0" }
anyhow = "1"
clap = { version = "4", features = ["derive"] }
@@ -18,6 +19,7 @@ schemars = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "2"
tiktoken-rs = "0.12"
[profile.release]
strip = true