mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
emb: lines into vectors, against a server that owns the model
This commit is contained in:
funkpower
Generated
+744
-3
@@ -2,6 +2,12 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
@@ -47,7 +53,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -58,7 +64,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -73,6 +79,12 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.8.0"
|
||||
@@ -110,6 +122,12 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.1.0"
|
||||
@@ -119,6 +137,22 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.1.0"
|
||||
@@ -177,6 +211,35 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.18.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie_store"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f"
|
||||
dependencies = [
|
||||
"cookie",
|
||||
"document-features",
|
||||
"idna",
|
||||
"indexmap",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"time",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.1.0"
|
||||
@@ -187,6 +250,15 @@ dependencies = [
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cuekit"
|
||||
version = "0.1.0"
|
||||
@@ -236,12 +308,61 @@ dependencies = [
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "document-features"
|
||||
version = "0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
|
||||
dependencies = [
|
||||
"litrs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "emb"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
"thiserror",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.16.2"
|
||||
@@ -264,12 +385,183 @@ dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"potential_utf",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locale_core"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_locale_core",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locale_core",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
|
||||
dependencies = [
|
||||
"idna_adapter",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
@@ -288,6 +580,30 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
|
||||
|
||||
[[package]]
|
||||
name = "litrs"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.1.0"
|
||||
@@ -305,6 +621,28 @@ version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
@@ -341,6 +679,27 @@ dependencies = [
|
||||
"unicode-general-category",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
|
||||
dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
@@ -408,6 +767,20 @@ version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rrf"
|
||||
version = "0.1.0"
|
||||
@@ -424,6 +797,41 @@ version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||
dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.2"
|
||||
@@ -503,12 +911,42 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
@@ -520,6 +958,17 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.1.0"
|
||||
@@ -555,7 +1004,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bstr",
|
||||
"fancy-regex 0.17.0",
|
||||
"lazy_static",
|
||||
@@ -563,6 +1012,47 @@ dependencies = [
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"num-conv",
|
||||
"powerfmt",
|
||||
"serde_core",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-general-category"
|
||||
version = "1.1.0"
|
||||
@@ -575,18 +1065,110 @@ version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "3.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"cookie_store",
|
||||
"flate2",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"ureq-proto",
|
||||
"utf8-zero",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ureq-proto"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8-zero"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
@@ -596,6 +1178,165 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
|
||||
|
||||
[[package]]
|
||||
name = "zerotrie"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.11.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
|
||||
@@ -22,6 +22,7 @@ cueseg = { path = "crates/cueseg", version = "0.1.0" }
|
||||
rrf = { path = "crates/rrf", version = "0.1.0" }
|
||||
cuepack = { path = "crates/cuepack", version = "0.1.0" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.1.0" }
|
||||
emb = { path = "crates/emb", version = "0.1.0" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
fancy-regex = "0.16"
|
||||
@@ -34,6 +35,7 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_json = { version = "1", features = ["float_roundtrip"] }
|
||||
thiserror = "2"
|
||||
unicode-general-category = "1"
|
||||
ureq = { version = "3", features = ["json"] }
|
||||
tiktoken-rs = "0.12"
|
||||
|
||||
[workspace.lints.rust]
|
||||
|
||||
@@ -60,6 +60,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done
|
||||
| `cuenorm` | Any subtitle format in, one normalised cue stream out |
|
||||
| `cueseg` | A timed transcript into topic chapters |
|
||||
| `cuepack` | Cues into passages worth embedding, keeping the times they were said at |
|
||||
| `emb` | Lines into vectors, against an embedding server |
|
||||
| `rrf` | Several ranked lists into one, without the database |
|
||||
|
||||
## The contract
|
||||
@@ -78,9 +79,9 @@ Every binary in this workspace obeys all of this, or it does not ship.
|
||||
"broke" are never the same number. A consumer closing the pipe is not a
|
||||
failure: `… | head -1` exits `0`, like every other filter.
|
||||
5. **No network, no database, no config file, no state.** Two exceptions, both
|
||||
declared in `--help`: a tool whose whole job is an HTTP call, and a tool
|
||||
declared in `--help`: `emb`, whose whole job is an HTTP call, and a tool
|
||||
that shells out to a named external program and fails clearly when it is
|
||||
absent.
|
||||
absent — `pdftriage` needs `pdftotext`.
|
||||
6. **No ambient knowledge.** No hostnames, no local paths, no private
|
||||
vocabularies compiled in. General scheme data — Bible book names, ISBN check
|
||||
digits — may be; anything local is a `--rules file.json`.
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "emb"
|
||||
description = "Lines in, vectors out, against an HTTP embedding endpoint."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
catena-cli.workspace = true
|
||||
clap.workspace = true
|
||||
corpus-core.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
textkit.workspace = true
|
||||
thiserror.workspace = true
|
||||
ureq.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
textkit.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,275 @@
|
||||
//! Lines in, vectors out, against an HTTP embedding endpoint.
|
||||
//!
|
||||
//! The one tool here that touches the network, and it does so for a reason:
|
||||
//! an embedding model is half a gigabyte of weights and a GPU, which belongs
|
||||
//! behind a server rather than inside a filter. What this owns is everything
|
||||
//! around the call — batching, the defensive truncation, the retry, and the
|
||||
//! assertion that the vectors coming back are the width the index expects.
|
||||
//!
|
||||
//! The request shape is the one Ollama's `/api/embed` uses, which several
|
||||
//! other servers also speak.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// How many texts go in one request.
|
||||
pub const BATCH: usize = 32;
|
||||
|
||||
/// Tokens a single input is cut to before it is sent.
|
||||
///
|
||||
/// A defensive cap, not a model limit. A server returns an error for an
|
||||
/// input past its context length and that error fails the whole batch, so one
|
||||
/// oversized passage costs thirty-one good ones. Counted with a tokeniser
|
||||
/// that is not the model's — it is within ten or fifteen per cent, which is
|
||||
/// all a safety net needs to be.
|
||||
pub const MAX_INPUT_TOKENS: usize = 1800;
|
||||
|
||||
/// How long to wait for a response.
|
||||
///
|
||||
/// Generous, because a cold server loading a model onto a GPU can take
|
||||
/// minutes before it answers at all.
|
||||
pub const TIMEOUT: Duration = Duration::from_secs(600);
|
||||
|
||||
/// How many times a failed batch is tried again.
|
||||
pub const ATTEMPTS: u32 = 3;
|
||||
|
||||
/// What the endpoint is asked.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Request<'a> {
|
||||
model: &'a str,
|
||||
input: &'a [String],
|
||||
}
|
||||
|
||||
/// What it answers.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Response {
|
||||
embeddings: Vec<Vec<f64>>,
|
||||
}
|
||||
|
||||
/// Why a batch could not be embedded.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// The server could not be reached, or did not answer well.
|
||||
#[error("{0}")]
|
||||
Http(String),
|
||||
/// The answer was not the shape this expects.
|
||||
#[error("the server's answer was not an embedding: {0}")]
|
||||
Shape(String),
|
||||
/// The vectors are not the width the caller asked for.
|
||||
#[error("vectors are {found} wide, expected {expected}")]
|
||||
Width {
|
||||
/// What came back.
|
||||
found: usize,
|
||||
/// What was asked for.
|
||||
expected: usize,
|
||||
},
|
||||
/// The server answered with fewer vectors than it was given texts.
|
||||
#[error("sent {sent} texts and got {received} vectors back")]
|
||||
Count {
|
||||
/// How many texts went out.
|
||||
sent: usize,
|
||||
/// How many vectors came back.
|
||||
received: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// Where and how to embed.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Endpoint {
|
||||
/// The server's base URL, without a trailing slash.
|
||||
pub host: String,
|
||||
/// The model to ask for.
|
||||
pub model: String,
|
||||
/// The width the vectors must be, when the caller knows it.
|
||||
pub dim: Option<usize>,
|
||||
/// Texts per request.
|
||||
pub batch: usize,
|
||||
/// Tokens per text, or `None` to send them whole.
|
||||
pub max_input_tokens: Option<usize>,
|
||||
/// How long to wait for one request.
|
||||
pub timeout: Duration,
|
||||
/// How many times to try a failing batch.
|
||||
pub attempts: u32,
|
||||
}
|
||||
|
||||
impl Endpoint {
|
||||
/// An endpoint with the usual settings.
|
||||
pub fn new(host: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
host: host.into().trim_end_matches('/').to_string(),
|
||||
model: model.into(),
|
||||
dim: None,
|
||||
batch: BATCH,
|
||||
max_input_tokens: Some(MAX_INPUT_TOKENS),
|
||||
timeout: TIMEOUT,
|
||||
attempts: ATTEMPTS,
|
||||
}
|
||||
}
|
||||
|
||||
/// The URL a request goes to.
|
||||
pub fn url(&self) -> String {
|
||||
format!("{}/api/embed", self.host)
|
||||
}
|
||||
|
||||
/// Cut a text to the input cap, if there is one.
|
||||
pub fn truncate(&self, text: &str) -> String {
|
||||
match self.max_input_tokens {
|
||||
None | Some(0) => text.to_string(),
|
||||
Some(limit) => {
|
||||
let tokens = textkit::encode(text);
|
||||
if tokens.len() <= limit {
|
||||
text.to_string()
|
||||
} else {
|
||||
textkit::decode_lossy(&tokens[..limit])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Embed one batch, retrying a failure.
|
||||
///
|
||||
/// Waits one second before the second attempt and two before the third, which
|
||||
/// is long enough for a server that is loading a model and short enough that
|
||||
/// a server that is simply down does not hold up a pipeline for a minute.
|
||||
pub fn embed_batch(
|
||||
agent: &ureq::Agent,
|
||||
endpoint: &Endpoint,
|
||||
texts: &[String],
|
||||
) -> Result<Vec<Vec<f64>>, Error> {
|
||||
let sent: Vec<String> = texts.iter().map(|text| endpoint.truncate(text)).collect();
|
||||
let mut last = Error::Http("no attempt was made".into());
|
||||
|
||||
for attempt in 0..endpoint.attempts.max(1) {
|
||||
match post(agent, endpoint, &sent) {
|
||||
Ok(vectors) => return check(endpoint, &sent, vectors),
|
||||
Err(e @ Error::Width { .. } | e @ Error::Count { .. }) => return Err(e),
|
||||
Err(e) => last = e,
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1 << attempt));
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
fn post(
|
||||
agent: &ureq::Agent,
|
||||
endpoint: &Endpoint,
|
||||
texts: &[String],
|
||||
) -> Result<Vec<Vec<f64>>, Error> {
|
||||
let request = Request {
|
||||
model: &endpoint.model,
|
||||
input: texts,
|
||||
};
|
||||
let mut response = agent
|
||||
.post(endpoint.url())
|
||||
.send_json(&request)
|
||||
.map_err(|e| Error::Http(e.to_string()))?;
|
||||
let body: Response = response
|
||||
.body_mut()
|
||||
.read_json()
|
||||
.map_err(|e| Error::Shape(e.to_string()))?;
|
||||
Ok(body.embeddings)
|
||||
}
|
||||
|
||||
fn check(
|
||||
endpoint: &Endpoint,
|
||||
sent: &[String],
|
||||
vectors: Vec<Vec<f64>>,
|
||||
) -> Result<Vec<Vec<f64>>, Error> {
|
||||
if vectors.len() != sent.len() {
|
||||
return Err(Error::Count {
|
||||
sent: sent.len(),
|
||||
received: vectors.len(),
|
||||
});
|
||||
}
|
||||
if let (Some(expected), Some(first)) = (endpoint.dim, vectors.first())
|
||||
&& first.len() != expected
|
||||
{
|
||||
return Err(Error::Width {
|
||||
found: first.len(),
|
||||
expected,
|
||||
});
|
||||
}
|
||||
Ok(vectors)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn endpoint() -> Endpoint {
|
||||
Endpoint::new("http://example.invalid/", "a-model")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_trailing_slash_does_not_double_up() {
|
||||
assert_eq!(endpoint().url(), "http://example.invalid/api/embed");
|
||||
assert_eq!(
|
||||
Endpoint::new("http://example.invalid", "m").url(),
|
||||
"http://example.invalid/api/embed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_short_text_is_sent_whole() {
|
||||
let text = "the mill stood at the northern end";
|
||||
assert_eq!(endpoint().truncate(text), text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_text_is_cut_to_the_cap() {
|
||||
let mut endpoint = endpoint();
|
||||
endpoint.max_input_tokens = Some(4);
|
||||
let cut = endpoint.truncate(&"word ".repeat(100));
|
||||
assert_eq!(textkit::count(&cut), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cap_can_be_turned_off() {
|
||||
let mut endpoint = endpoint();
|
||||
endpoint.max_input_tokens = None;
|
||||
let long = "word ".repeat(5000);
|
||||
assert_eq!(endpoint.truncate(&long), long);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_width_that_is_not_the_one_asked_for_is_refused() {
|
||||
let mut endpoint = endpoint();
|
||||
endpoint.dim = Some(1024);
|
||||
let sent = vec!["a".to_string()];
|
||||
let e = check(&endpoint, &sent, vec![vec![0.0; 768]]).expect_err("should refuse");
|
||||
assert!(matches!(
|
||||
e,
|
||||
Error::Width {
|
||||
found: 768,
|
||||
expected: 1024
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_vector_is_refused() {
|
||||
let sent = vec!["a".to_string(), "b".to_string()];
|
||||
let e = check(&endpoint(), &sent, vec![vec![0.0; 8]]).expect_err("should refuse");
|
||||
assert!(matches!(
|
||||
e,
|
||||
Error::Count {
|
||||
sent: 2,
|
||||
received: 1
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vectors_of_the_right_shape_come_straight_back() {
|
||||
let mut endpoint = endpoint();
|
||||
endpoint.dim = Some(3);
|
||||
let sent = vec!["a".to_string()];
|
||||
let vectors = vec![vec![1.0, 2.0, 3.0]];
|
||||
assert_eq!(
|
||||
check(&endpoint, &sent, vectors.clone()).expect("should pass"),
|
||||
vectors
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
//! Lines in, vectors out, against an HTTP embedding endpoint.
|
||||
|
||||
use std::io::Write;
|
||||
use std::process::ExitCode;
|
||||
use std::time::Duration;
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
use emb::Endpoint;
|
||||
|
||||
/// Embed each line of the input and write its vector.
|
||||
///
|
||||
/// One JSON array per line, in the order the lines arrived, so the output
|
||||
/// lines up with the input line for line. Empty lines are sent as they are:
|
||||
/// dropping them would make the two disagree.
|
||||
///
|
||||
/// The only tool here that uses the network. It never loads a model — the
|
||||
/// model belongs behind a server, where it can be shared and where its GPU
|
||||
/// is somebody else's problem.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 every line was embedded, 65 the server answered something that
|
||||
was not an embedding, 69 the server could not be reached, 74 the input could
|
||||
not be read.
|
||||
|
||||
Example:
|
||||
|
||||
jq -r .text chunks.ndjson | emb --model bge-m3 --dim 1024 > vectors.ndjson"
|
||||
)]
|
||||
struct Args {
|
||||
/// File of texts, one per line. Reads stdin when absent.
|
||||
path: Option<std::path::PathBuf>,
|
||||
|
||||
/// The server to ask.
|
||||
#[arg(long, default_value = "http://localhost:11434", value_name = "URL")]
|
||||
host: String,
|
||||
|
||||
/// The model to ask for.
|
||||
#[arg(long, value_name = "NAME")]
|
||||
model: String,
|
||||
|
||||
/// Refuse vectors that are not this wide.
|
||||
#[arg(long, value_name = "WIDTH")]
|
||||
dim: Option<usize>,
|
||||
|
||||
/// Texts per request.
|
||||
#[arg(long, default_value_t = emb::BATCH, value_name = "COUNT")]
|
||||
batch: usize,
|
||||
|
||||
/// Cut each text to this many tokens before sending it. Zero sends them
|
||||
/// whole.
|
||||
#[arg(long, default_value_t = emb::MAX_INPUT_TOKENS, value_name = "TOKENS")]
|
||||
max_input_tokens: usize,
|
||||
|
||||
/// How long to wait for one request.
|
||||
#[arg(long, default_value_t = emb::TIMEOUT.as_secs(), value_name = "SECONDS")]
|
||||
timeout: u64,
|
||||
|
||||
/// How many times to try a failing request.
|
||||
#[arg(long, default_value_t = emb::ATTEMPTS, value_name = "COUNT")]
|
||||
attempts: u32,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
if args.batch == 0 {
|
||||
eprintln!("emb: --batch must be above zero");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
|
||||
let endpoint = Endpoint {
|
||||
dim: args.dim,
|
||||
batch: args.batch,
|
||||
max_input_tokens: Some(args.max_input_tokens),
|
||||
timeout: Duration::from_secs(args.timeout),
|
||||
attempts: args.attempts,
|
||||
..Endpoint::new(&args.host, &args.model)
|
||||
};
|
||||
|
||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||
Ok(text) => text,
|
||||
Err((code, why)) => {
|
||||
eprintln!("emb: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
// A trailing newline ends the last line rather than starting an empty one.
|
||||
let lines: Vec<String> = text
|
||||
.strip_suffix('\n')
|
||||
.unwrap_or(&text)
|
||||
.split('\n')
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
if text.is_empty() {
|
||||
return ExitCode::from(exit::OK);
|
||||
}
|
||||
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.timeout_global(Some(endpoint.timeout))
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let mut out = std::io::BufWriter::new(std::io::stdout().lock());
|
||||
for batch in lines.chunks(endpoint.batch) {
|
||||
let vectors = match emb::embed_batch(&agent, &endpoint, batch) {
|
||||
Ok(vectors) => vectors,
|
||||
Err(e) => {
|
||||
eprintln!("emb: {e}");
|
||||
return ExitCode::from(match e {
|
||||
emb::Error::Http(_) => exit::UNAVAILABLE,
|
||||
_ => exit::DATA,
|
||||
});
|
||||
}
|
||||
};
|
||||
for vector in vectors {
|
||||
let line = serde_json::to_string(&vector).expect("a vector serialises");
|
||||
match writeln!(out, "{line}").and_then(|()| out.flush()) {
|
||||
Ok(()) => {}
|
||||
// A consumer that stopped reading is not a failure.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => {
|
||||
return ExitCode::from(exit::OK);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("emb: {e}");
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catena_cli::finish("emb", out)
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//! The tool against a server, without needing one.
|
||||
//!
|
||||
//! The server here is twenty lines of `TcpListener` that answers whatever the
|
||||
//! test wants it to. That is enough to check the things that actually go
|
||||
//! wrong: the request shape, the batching, the retry, and what happens when
|
||||
//! the answer is not what was asked for.
|
||||
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::process::{Command, Output};
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
|
||||
/// What a request asked for, as the test wants to see it.
|
||||
struct Asked {
|
||||
model: String,
|
||||
inputs: Vec<String>,
|
||||
}
|
||||
|
||||
/// Answer `replies` in turn, one per request, and report what was asked.
|
||||
fn server(replies: Vec<(u16, String)>) -> (String, mpsc::Receiver<Asked>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("a free port");
|
||||
let host = format!("http://{}", listener.local_addr().expect("an address"));
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
|
||||
thread::spawn(move || {
|
||||
for (at, connection) in listener.incoming().enumerate() {
|
||||
let Ok(mut connection) = connection else {
|
||||
break;
|
||||
};
|
||||
if let Some(asked) = serve(&mut connection, replies.get(at)) {
|
||||
let _ = sender.send(asked);
|
||||
}
|
||||
if at + 1 >= replies.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
(host, receiver)
|
||||
}
|
||||
|
||||
fn serve(connection: &mut TcpStream, reply: Option<&(u16, String)>) -> Option<Asked> {
|
||||
let mut reader = BufReader::new(connection.try_clone().expect("a clone"));
|
||||
let mut length = 0usize;
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
if reader.read_line(&mut line).ok()? == 0 {
|
||||
return None;
|
||||
}
|
||||
if let Some(value) = line.to_lowercase().strip_prefix("content-length:") {
|
||||
length = value.trim().parse().unwrap_or(0);
|
||||
}
|
||||
if line == "\r\n" || line == "\n" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut body = vec![0; length];
|
||||
reader.read_exact(&mut body).ok()?;
|
||||
let request: serde_json::Value = serde_json::from_slice(&body).ok()?;
|
||||
|
||||
let (status, reply) = reply.cloned().unwrap_or((500, String::new()));
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}",
|
||||
reply.len()
|
||||
);
|
||||
let _ = connection.write_all(response.as_bytes());
|
||||
let _ = connection.flush();
|
||||
|
||||
Some(Asked {
|
||||
model: request["model"].as_str().unwrap_or_default().to_string(),
|
||||
inputs: request["input"]
|
||||
.as_array()
|
||||
.map(|inputs| {
|
||||
inputs
|
||||
.iter()
|
||||
.map(|input| input.as_str().unwrap_or_default().to_string())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
fn vectors(count: usize, width: usize) -> String {
|
||||
let one = format!("[{}]", vec!["0.5"; width].join(","));
|
||||
format!("{{\"embeddings\":[{}]}}", vec![one; count].join(","))
|
||||
}
|
||||
|
||||
fn run(args: &[&str], stdin: &str) -> Output {
|
||||
use std::process::Stdio;
|
||||
let mut child = Command::new(env!("CARGO_BIN_EXE_emb"))
|
||||
.args(args)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("the binary should run");
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.expect("stdin was piped")
|
||||
.write_all(stdin.as_bytes())
|
||||
.expect("the input should write");
|
||||
child.wait_with_output().expect("the binary should finish")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_line_comes_back_as_a_vector() {
|
||||
let (host, asked) = server(vec![(200, vectors(2, 4))]);
|
||||
let out = run(
|
||||
&["--host", &host, "--model", "a-model"],
|
||||
"the mill stood\nat the northern end\n",
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
out.status.code(),
|
||||
Some(0),
|
||||
"{:?}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let lines: Vec<&str> = std::str::from_utf8(&out.stdout)
|
||||
.expect("utf-8")
|
||||
.lines()
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 2);
|
||||
assert_eq!(lines[0], "[0.5,0.5,0.5,0.5]");
|
||||
|
||||
let asked = asked.recv().expect("a request");
|
||||
assert_eq!(asked.model, "a-model");
|
||||
assert_eq!(asked.inputs, ["the mill stood", "at the northern end"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_lines_are_sent_in_batches() {
|
||||
let (host, asked) = server(vec![(200, vectors(2, 2)), (200, vectors(1, 2))]);
|
||||
let out = run(
|
||||
&["--host", &host, "--model", "m", "--batch", "2"],
|
||||
"one\ntwo\nthree\n",
|
||||
);
|
||||
|
||||
assert_eq!(out.status.code(), Some(0));
|
||||
assert_eq!(out.stdout.lines().count(), 3);
|
||||
assert_eq!(asked.recv().expect("a request").inputs.len(), 2);
|
||||
assert_eq!(asked.recv().expect("a request").inputs.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_vector_of_the_wrong_width_is_refused() {
|
||||
let (host, _asked) = server(vec![(200, vectors(1, 768))]);
|
||||
let out = run(&["--host", &host, "--model", "m", "--dim", "1024"], "one\n");
|
||||
|
||||
assert_eq!(out.status.code(), Some(65), "bad data must exit 65");
|
||||
assert!(out.stdout.is_empty(), "a bad answer must write nothing");
|
||||
let said = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(said.contains("768"), "{said}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_that_is_not_there_is_reported_as_unavailable() {
|
||||
// Port 1 on the loopback: nothing is listening and nothing will be.
|
||||
let out = run(
|
||||
&[
|
||||
"--host",
|
||||
"http://127.0.0.1:1",
|
||||
"--model",
|
||||
"m",
|
||||
"--attempts",
|
||||
"1",
|
||||
],
|
||||
"one\n",
|
||||
);
|
||||
assert_eq!(out.status.code(), Some(69), "unreachable must exit 69");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_request_is_tried_again() {
|
||||
let (host, asked) = server(vec![(503, "{}".into()), (200, vectors(1, 2))]);
|
||||
let out = run(
|
||||
&["--host", &host, "--model", "m", "--attempts", "2"],
|
||||
"one\n",
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
out.status.code(),
|
||||
Some(0),
|
||||
"{:?}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
assert_eq!(out.stdout.lines().count(), 1);
|
||||
assert_eq!(asked.recv().expect("a first request").inputs, ["one"]);
|
||||
assert_eq!(asked.recv().expect("a second request").inputs, ["one"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_line_is_cut_before_it_is_sent() {
|
||||
let (host, asked) = server(vec![(200, vectors(1, 2))]);
|
||||
let long = "word ".repeat(100);
|
||||
run(
|
||||
&["--host", &host, "--model", "m", "--max-input-tokens", "5"],
|
||||
&format!("{long}\n"),
|
||||
);
|
||||
|
||||
let asked = asked.recv().expect("a request");
|
||||
assert_eq!(textkit::count(&asked.inputs[0]), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_input_asks_the_server_nothing() {
|
||||
let out = run(&["--host", "http://127.0.0.1:1", "--model", "m"], "");
|
||||
assert_eq!(out.status.code(), Some(0));
|
||||
assert!(out.stdout.is_empty());
|
||||
}
|
||||
Reference in New Issue
Block a user