textkit: token counting and the two segmenters

2026-10-23 03:07:13 +00:00 · 2026-09-19 11:04:27 -04:00
parent 00f4c38816
commit 0208b14b5d
6 changed files with 454 additions and 0 deletions
@@ -0,0 +1,15 @@
+[package]
+name = "textkit"
+description = "Tokenising and segmenting text, the way the catena tools agree to do it."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+tiktoken-rs.workspace = true
+
+[lints.rust]
+unsafe_code = "forbid"
+missing_docs = "warn"