From c6575b2d42c08286a022a1b633ab39bb00bef3f6 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 12:43:20 -0400 Subject: [PATCH] emb: lines into vectors, against a server that owns the model --- Cargo.lock | 747 +++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + README.md | 5 +- crates/emb/Cargo.toml | 25 ++ crates/emb/src/lib.rs | 275 +++++++++++++++ crates/emb/src/main.rs | 137 ++++++++ crates/emb/tests/cli.rs | 212 ++++++++++++ 7 files changed, 1398 insertions(+), 5 deletions(-) create mode 100644 crates/emb/Cargo.toml create mode 100644 crates/emb/src/lib.rs create mode 100644 crates/emb/src/main.rs create mode 100644 crates/emb/tests/cli.rs diff --git a/Cargo.lock b/Cargo.lock index 89ec1a1..51a591e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -47,7 +53,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -58,7 +64,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -73,6 +79,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.8.0" @@ -110,6 +122,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "catena-cli" version = "0.1.0" @@ -119,6 +137,22 @@ dependencies = [ "serde", ] +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + [[package]] name = "chunkmd" version = "0.1.0" @@ -177,6 +211,35 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "corpus-core" version = "0.1.0" @@ -187,6 +250,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "cuekit" version = "0.1.0" @@ -236,12 +308,61 @@ dependencies = [ "textkit", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "emb" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "serde", + "serde_json", + "textkit", + "thiserror", + "ureq", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "fancy-regex" version = "0.16.2" @@ -264,12 +385,183 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -288,6 +580,30 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + [[package]] name = "mdscrub" version = "0.1.0" @@ -305,6 +621,28 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" @@ -341,6 +679,27 @@ dependencies = [ "unicode-general-category", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -408,6 +767,20 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rrf" version = "0.1.0" @@ -424,6 +797,41 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "schemars" version = "1.2.2" @@ -503,12 +911,42 @@ dependencies = [ "zmij", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "3.0.6" @@ -520,6 +958,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "textkit" version = "0.1.0" @@ -555,7 +1004,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bstr", "fancy-regex 0.17.0", "lazy_static", @@ -563,6 +1012,47 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -575,18 +1065,110 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64 0.23.1", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -596,6 +1178,165 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index f8a7710..c38edcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ cueseg = { path = "crates/cueseg", version = "0.1.0" } rrf = { path = "crates/rrf", version = "0.1.0" } cuepack = { path = "crates/cuepack", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } +emb = { path = "crates/emb", version = "0.1.0" } clap = { version = "4", features = ["derive"] } fancy-regex = "0.16" @@ -34,6 +35,7 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["float_roundtrip"] } thiserror = "2" unicode-general-category = "1" +ureq = { version = "3", features = ["json"] } tiktoken-rs = "0.12" [workspace.lints.rust] diff --git a/README.md b/README.md index 5b770d4..2ec695e 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done | `cuenorm` | Any subtitle format in, one normalised cue stream out | | `cueseg` | A timed transcript into topic chapters | | `cuepack` | Cues into passages worth embedding, keeping the times they were said at | +| `emb` | Lines into vectors, against an embedding server | | `rrf` | Several ranked lists into one, without the database | ## The contract @@ -78,9 +79,9 @@ Every binary in this workspace obeys all of this, or it does not ship. "broke" are never the same number. A consumer closing the pipe is not a failure: `… | head -1` exits `0`, like every other filter. 5. **No network, no database, no config file, no state.** Two exceptions, both - declared in `--help`: a tool whose whole job is an HTTP call, and a tool + declared in `--help`: `emb`, whose whole job is an HTTP call, and a tool that shells out to a named external program and fails clearly when it is - absent. + absent — `pdftriage` needs `pdftotext`. 6. **No ambient knowledge.** No hostnames, no local paths, no private vocabularies compiled in. General scheme data — Bible book names, ISBN check digits — may be; anything local is a `--rules file.json`. diff --git a/crates/emb/Cargo.toml b/crates/emb/Cargo.toml new file mode 100644 index 0000000..c91c58c --- /dev/null +++ b/crates/emb/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "emb" +description = "Lines in, vectors out, against an HTTP embedding endpoint." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +serde.workspace = true +serde_json.workspace = true +textkit.workspace = true +thiserror.workspace = true +ureq.workspace = true + +[dev-dependencies] +serde_json.workspace = true +textkit.workspace = true + +[lints] +workspace = true diff --git a/crates/emb/src/lib.rs b/crates/emb/src/lib.rs new file mode 100644 index 0000000..e6a34e5 --- /dev/null +++ b/crates/emb/src/lib.rs @@ -0,0 +1,275 @@ +//! Lines in, vectors out, against an HTTP embedding endpoint. +//! +//! The one tool here that touches the network, and it does so for a reason: +//! an embedding model is half a gigabyte of weights and a GPU, which belongs +//! behind a server rather than inside a filter. What this owns is everything +//! around the call — batching, the defensive truncation, the retry, and the +//! assertion that the vectors coming back are the width the index expects. +//! +//! The request shape is the one Ollama's `/api/embed` uses, which several +//! other servers also speak. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +/// How many texts go in one request. +pub const BATCH: usize = 32; + +/// Tokens a single input is cut to before it is sent. +/// +/// A defensive cap, not a model limit. A server returns an error for an +/// input past its context length and that error fails the whole batch, so one +/// oversized passage costs thirty-one good ones. Counted with a tokeniser +/// that is not the model's — it is within ten or fifteen per cent, which is +/// all a safety net needs to be. +pub const MAX_INPUT_TOKENS: usize = 1800; + +/// How long to wait for a response. +/// +/// Generous, because a cold server loading a model onto a GPU can take +/// minutes before it answers at all. +pub const TIMEOUT: Duration = Duration::from_secs(600); + +/// How many times a failed batch is tried again. +pub const ATTEMPTS: u32 = 3; + +/// What the endpoint is asked. +#[derive(Debug, Serialize)] +struct Request<'a> { + model: &'a str, + input: &'a [String], +} + +/// What it answers. +#[derive(Debug, Deserialize)] +struct Response { + embeddings: Vec>, +} + +/// Why a batch could not be embedded. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The server could not be reached, or did not answer well. + #[error("{0}")] + Http(String), + /// The answer was not the shape this expects. + #[error("the server's answer was not an embedding: {0}")] + Shape(String), + /// The vectors are not the width the caller asked for. + #[error("vectors are {found} wide, expected {expected}")] + Width { + /// What came back. + found: usize, + /// What was asked for. + expected: usize, + }, + /// The server answered with fewer vectors than it was given texts. + #[error("sent {sent} texts and got {received} vectors back")] + Count { + /// How many texts went out. + sent: usize, + /// How many vectors came back. + received: usize, + }, +} + +/// Where and how to embed. +#[derive(Debug, Clone)] +pub struct Endpoint { + /// The server's base URL, without a trailing slash. + pub host: String, + /// The model to ask for. + pub model: String, + /// The width the vectors must be, when the caller knows it. + pub dim: Option, + /// Texts per request. + pub batch: usize, + /// Tokens per text, or `None` to send them whole. + pub max_input_tokens: Option, + /// How long to wait for one request. + pub timeout: Duration, + /// How many times to try a failing batch. + pub attempts: u32, +} + +impl Endpoint { + /// An endpoint with the usual settings. + pub fn new(host: impl Into, model: impl Into) -> Self { + Self { + host: host.into().trim_end_matches('/').to_string(), + model: model.into(), + dim: None, + batch: BATCH, + max_input_tokens: Some(MAX_INPUT_TOKENS), + timeout: TIMEOUT, + attempts: ATTEMPTS, + } + } + + /// The URL a request goes to. + pub fn url(&self) -> String { + format!("{}/api/embed", self.host) + } + + /// Cut a text to the input cap, if there is one. + pub fn truncate(&self, text: &str) -> String { + match self.max_input_tokens { + None | Some(0) => text.to_string(), + Some(limit) => { + let tokens = textkit::encode(text); + if tokens.len() <= limit { + text.to_string() + } else { + textkit::decode_lossy(&tokens[..limit]) + } + } + } + } +} + +/// Embed one batch, retrying a failure. +/// +/// Waits one second before the second attempt and two before the third, which +/// is long enough for a server that is loading a model and short enough that +/// a server that is simply down does not hold up a pipeline for a minute. +pub fn embed_batch( + agent: &ureq::Agent, + endpoint: &Endpoint, + texts: &[String], +) -> Result>, Error> { + let sent: Vec = texts.iter().map(|text| endpoint.truncate(text)).collect(); + let mut last = Error::Http("no attempt was made".into()); + + for attempt in 0..endpoint.attempts.max(1) { + match post(agent, endpoint, &sent) { + Ok(vectors) => return check(endpoint, &sent, vectors), + Err(e @ Error::Width { .. } | e @ Error::Count { .. }) => return Err(e), + Err(e) => last = e, + } + std::thread::sleep(Duration::from_secs(1 << attempt)); + } + Err(last) +} + +fn post( + agent: &ureq::Agent, + endpoint: &Endpoint, + texts: &[String], +) -> Result>, Error> { + let request = Request { + model: &endpoint.model, + input: texts, + }; + let mut response = agent + .post(endpoint.url()) + .send_json(&request) + .map_err(|e| Error::Http(e.to_string()))?; + let body: Response = response + .body_mut() + .read_json() + .map_err(|e| Error::Shape(e.to_string()))?; + Ok(body.embeddings) +} + +fn check( + endpoint: &Endpoint, + sent: &[String], + vectors: Vec>, +) -> Result>, Error> { + if vectors.len() != sent.len() { + return Err(Error::Count { + sent: sent.len(), + received: vectors.len(), + }); + } + if let (Some(expected), Some(first)) = (endpoint.dim, vectors.first()) + && first.len() != expected + { + return Err(Error::Width { + found: first.len(), + expected, + }); + } + Ok(vectors) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn endpoint() -> Endpoint { + Endpoint::new("http://example.invalid/", "a-model") + } + + #[test] + fn a_trailing_slash_does_not_double_up() { + assert_eq!(endpoint().url(), "http://example.invalid/api/embed"); + assert_eq!( + Endpoint::new("http://example.invalid", "m").url(), + "http://example.invalid/api/embed" + ); + } + + #[test] + fn a_short_text_is_sent_whole() { + let text = "the mill stood at the northern end"; + assert_eq!(endpoint().truncate(text), text); + } + + #[test] + fn a_long_text_is_cut_to_the_cap() { + let mut endpoint = endpoint(); + endpoint.max_input_tokens = Some(4); + let cut = endpoint.truncate(&"word ".repeat(100)); + assert_eq!(textkit::count(&cut), 4); + } + + #[test] + fn the_cap_can_be_turned_off() { + let mut endpoint = endpoint(); + endpoint.max_input_tokens = None; + let long = "word ".repeat(5000); + assert_eq!(endpoint.truncate(&long), long); + } + + #[test] + fn a_width_that_is_not_the_one_asked_for_is_refused() { + let mut endpoint = endpoint(); + endpoint.dim = Some(1024); + let sent = vec!["a".to_string()]; + let e = check(&endpoint, &sent, vec![vec![0.0; 768]]).expect_err("should refuse"); + assert!(matches!( + e, + Error::Width { + found: 768, + expected: 1024 + } + )); + } + + #[test] + fn a_missing_vector_is_refused() { + let sent = vec!["a".to_string(), "b".to_string()]; + let e = check(&endpoint(), &sent, vec![vec![0.0; 8]]).expect_err("should refuse"); + assert!(matches!( + e, + Error::Count { + sent: 2, + received: 1 + } + )); + } + + #[test] + fn vectors_of_the_right_shape_come_straight_back() { + let mut endpoint = endpoint(); + endpoint.dim = Some(3); + let sent = vec!["a".to_string()]; + let vectors = vec![vec![1.0, 2.0, 3.0]]; + assert_eq!( + check(&endpoint, &sent, vectors.clone()).expect("should pass"), + vectors + ); + } +} diff --git a/crates/emb/src/main.rs b/crates/emb/src/main.rs new file mode 100644 index 0000000..de4f456 --- /dev/null +++ b/crates/emb/src/main.rs @@ -0,0 +1,137 @@ +//! Lines in, vectors out, against an HTTP embedding endpoint. + +use std::io::Write; +use std::process::ExitCode; +use std::time::Duration; + +use clap::Parser; +use corpus_core::exit; +use emb::Endpoint; + +/// Embed each line of the input and write its vector. +/// +/// One JSON array per line, in the order the lines arrived, so the output +/// lines up with the input line for line. Empty lines are sent as they are: +/// dropping them would make the two disagree. +/// +/// The only tool here that uses the network. It never loads a model — the +/// model belongs behind a server, where it can be shared and where its GPU +/// is somebody else's problem. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 every line was embedded, 65 the server answered something that +was not an embedding, 69 the server could not be reached, 74 the input could +not be read. + +Example: + + jq -r .text chunks.ndjson | emb --model bge-m3 --dim 1024 > vectors.ndjson" +)] +struct Args { + /// File of texts, one per line. Reads stdin when absent. + path: Option, + + /// The server to ask. + #[arg(long, default_value = "http://localhost:11434", value_name = "URL")] + host: String, + + /// The model to ask for. + #[arg(long, value_name = "NAME")] + model: String, + + /// Refuse vectors that are not this wide. + #[arg(long, value_name = "WIDTH")] + dim: Option, + + /// Texts per request. + #[arg(long, default_value_t = emb::BATCH, value_name = "COUNT")] + batch: usize, + + /// Cut each text to this many tokens before sending it. Zero sends them + /// whole. + #[arg(long, default_value_t = emb::MAX_INPUT_TOKENS, value_name = "TOKENS")] + max_input_tokens: usize, + + /// How long to wait for one request. + #[arg(long, default_value_t = emb::TIMEOUT.as_secs(), value_name = "SECONDS")] + timeout: u64, + + /// How many times to try a failing request. + #[arg(long, default_value_t = emb::ATTEMPTS, value_name = "COUNT")] + attempts: u32, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + if args.batch == 0 { + eprintln!("emb: --batch must be above zero"); + return ExitCode::from(exit::USAGE); + } + + let endpoint = Endpoint { + dim: args.dim, + batch: args.batch, + max_input_tokens: Some(args.max_input_tokens), + timeout: Duration::from_secs(args.timeout), + attempts: args.attempts, + ..Endpoint::new(&args.host, &args.model) + }; + + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("emb: {why}"); + return ExitCode::from(code); + } + }; + // A trailing newline ends the last line rather than starting an empty one. + let lines: Vec = text + .strip_suffix('\n') + .unwrap_or(&text) + .split('\n') + .map(str::to_string) + .collect(); + if text.is_empty() { + return ExitCode::from(exit::OK); + } + + let agent: ureq::Agent = ureq::Agent::config_builder() + .timeout_global(Some(endpoint.timeout)) + .build() + .into(); + + let mut out = std::io::BufWriter::new(std::io::stdout().lock()); + for batch in lines.chunks(endpoint.batch) { + let vectors = match emb::embed_batch(&agent, &endpoint, batch) { + Ok(vectors) => vectors, + Err(e) => { + eprintln!("emb: {e}"); + return ExitCode::from(match e { + emb::Error::Http(_) => exit::UNAVAILABLE, + _ => exit::DATA, + }); + } + }; + for vector in vectors { + let line = serde_json::to_string(&vector).expect("a vector serialises"); + match writeln!(out, "{line}").and_then(|()| out.flush()) { + Ok(()) => {} + // A consumer that stopped reading is not a failure. + Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => { + return ExitCode::from(exit::OK); + } + Err(e) => { + eprintln!("emb: {e}"); + return ExitCode::from(exit::IO); + } + } + } + } + catena_cli::finish("emb", out) +} diff --git a/crates/emb/tests/cli.rs b/crates/emb/tests/cli.rs new file mode 100644 index 0000000..c87c7d3 --- /dev/null +++ b/crates/emb/tests/cli.rs @@ -0,0 +1,212 @@ +//! The tool against a server, without needing one. +//! +//! The server here is twenty lines of `TcpListener` that answers whatever the +//! test wants it to. That is enough to check the things that actually go +//! wrong: the request shape, the batching, the retry, and what happens when +//! the answer is not what was asked for. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::process::{Command, Output}; +use std::sync::mpsc; +use std::thread; + +/// What a request asked for, as the test wants to see it. +struct Asked { + model: String, + inputs: Vec, +} + +/// Answer `replies` in turn, one per request, and report what was asked. +fn server(replies: Vec<(u16, String)>) -> (String, mpsc::Receiver) { + let listener = TcpListener::bind("127.0.0.1:0").expect("a free port"); + let host = format!("http://{}", listener.local_addr().expect("an address")); + let (sender, receiver) = mpsc::channel(); + + thread::spawn(move || { + for (at, connection) in listener.incoming().enumerate() { + let Ok(mut connection) = connection else { + break; + }; + if let Some(asked) = serve(&mut connection, replies.get(at)) { + let _ = sender.send(asked); + } + if at + 1 >= replies.len() { + break; + } + } + }); + + (host, receiver) +} + +fn serve(connection: &mut TcpStream, reply: Option<&(u16, String)>) -> Option { + let mut reader = BufReader::new(connection.try_clone().expect("a clone")); + let mut length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).ok()? == 0 { + return None; + } + if let Some(value) = line.to_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap_or(0); + } + if line == "\r\n" || line == "\n" { + break; + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).ok()?; + let request: serde_json::Value = serde_json::from_slice(&body).ok()?; + + let (status, reply) = reply.cloned().unwrap_or((500, String::new())); + let response = format!( + "HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", + reply.len() + ); + let _ = connection.write_all(response.as_bytes()); + let _ = connection.flush(); + + Some(Asked { + model: request["model"].as_str().unwrap_or_default().to_string(), + inputs: request["input"] + .as_array() + .map(|inputs| { + inputs + .iter() + .map(|input| input.as_str().unwrap_or_default().to_string()) + .collect() + }) + .unwrap_or_default(), + }) +} + +fn vectors(count: usize, width: usize) -> String { + let one = format!("[{}]", vec!["0.5"; width].join(",")); + format!("{{\"embeddings\":[{}]}}", vec![one; count].join(",")) +} + +fn run(args: &[&str], stdin: &str) -> Output { + use std::process::Stdio; + let mut child = Command::new(env!("CARGO_BIN_EXE_emb")) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("the binary should run"); + child + .stdin + .take() + .expect("stdin was piped") + .write_all(stdin.as_bytes()) + .expect("the input should write"); + child.wait_with_output().expect("the binary should finish") +} + +#[test] +fn every_line_comes_back_as_a_vector() { + let (host, asked) = server(vec![(200, vectors(2, 4))]); + let out = run( + &["--host", &host, "--model", "a-model"], + "the mill stood\nat the northern end\n", + ); + + assert_eq!( + out.status.code(), + Some(0), + "{:?}", + String::from_utf8_lossy(&out.stderr) + ); + let lines: Vec<&str> = std::str::from_utf8(&out.stdout) + .expect("utf-8") + .lines() + .collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0], "[0.5,0.5,0.5,0.5]"); + + let asked = asked.recv().expect("a request"); + assert_eq!(asked.model, "a-model"); + assert_eq!(asked.inputs, ["the mill stood", "at the northern end"]); +} + +#[test] +fn the_lines_are_sent_in_batches() { + let (host, asked) = server(vec![(200, vectors(2, 2)), (200, vectors(1, 2))]); + let out = run( + &["--host", &host, "--model", "m", "--batch", "2"], + "one\ntwo\nthree\n", + ); + + assert_eq!(out.status.code(), Some(0)); + assert_eq!(out.stdout.lines().count(), 3); + assert_eq!(asked.recv().expect("a request").inputs.len(), 2); + assert_eq!(asked.recv().expect("a request").inputs.len(), 1); +} + +#[test] +fn a_vector_of_the_wrong_width_is_refused() { + let (host, _asked) = server(vec![(200, vectors(1, 768))]); + let out = run(&["--host", &host, "--model", "m", "--dim", "1024"], "one\n"); + + assert_eq!(out.status.code(), Some(65), "bad data must exit 65"); + assert!(out.stdout.is_empty(), "a bad answer must write nothing"); + let said = String::from_utf8_lossy(&out.stderr); + assert!(said.contains("768"), "{said}"); +} + +#[test] +fn a_server_that_is_not_there_is_reported_as_unavailable() { + // Port 1 on the loopback: nothing is listening and nothing will be. + let out = run( + &[ + "--host", + "http://127.0.0.1:1", + "--model", + "m", + "--attempts", + "1", + ], + "one\n", + ); + assert_eq!(out.status.code(), Some(69), "unreachable must exit 69"); +} + +#[test] +fn a_failed_request_is_tried_again() { + let (host, asked) = server(vec![(503, "{}".into()), (200, vectors(1, 2))]); + let out = run( + &["--host", &host, "--model", "m", "--attempts", "2"], + "one\n", + ); + + assert_eq!( + out.status.code(), + Some(0), + "{:?}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(out.stdout.lines().count(), 1); + assert_eq!(asked.recv().expect("a first request").inputs, ["one"]); + assert_eq!(asked.recv().expect("a second request").inputs, ["one"]); +} + +#[test] +fn a_long_line_is_cut_before_it_is_sent() { + let (host, asked) = server(vec![(200, vectors(1, 2))]); + let long = "word ".repeat(100); + run( + &["--host", &host, "--model", "m", "--max-input-tokens", "5"], + &format!("{long}\n"), + ); + + let asked = asked.recv().expect("a request"); + assert_eq!(textkit::count(&asked.inputs[0]), 5); +} + +#[test] +fn no_input_asks_the_server_nothing() { + let out = run(&["--host", "http://127.0.0.1:1", "--model", "m"], ""); + assert_eq!(out.status.code(), Some(0)); + assert!(out.stdout.is_empty()); +}