From cf126dddc379ac323036640673f71118a50f77db Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 17:54:18 -0400 Subject: [PATCH] bindings: the libraries in process, for Python --- Cargo.lock | 127 +++++++ Cargo.toml | 10 + README.md | 1 + crates/bindings/Cargo.toml | 38 +++ crates/bindings/LICENSE | 373 +++++++++++++++++++++ crates/bindings/README.md | 99 ++++++ crates/bindings/pyproject.toml | 47 +++ crates/bindings/python/catena/__init__.py | 35 ++ crates/bindings/python/catena/__init__.pyi | 32 ++ crates/bindings/python/catena/py.typed | 0 crates/bindings/src/lib.rs | 235 +++++++++++++ crates/bindings/src/python.rs | 208 ++++++++++++ 12 files changed, 1205 insertions(+) create mode 100644 crates/bindings/Cargo.toml create mode 100644 crates/bindings/LICENSE create mode 100644 crates/bindings/README.md create mode 100644 crates/bindings/pyproject.toml create mode 100644 crates/bindings/python/catena/__init__.py create mode 100644 crates/bindings/python/catena/__init__.pyi create mode 100644 crates/bindings/python/catena/py.typed create mode 100644 crates/bindings/src/lib.rs create mode 100644 crates/bindings/src/python.rs diff --git a/Cargo.lock b/Cargo.lock index d85ee49..51b3516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,6 +84,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "avplan" version = "0.1.0" @@ -199,6 +205,24 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "catena" +version = "0.1.0" +dependencies = [ + "booksniff", + "chunkmd", + "corpus-core", + "cuekit", + "cuepack", + "mdscrub", + "mdsplit", + "pyo3", + "rrf", + "serde", + "serde_json", + "textkit", +] + [[package]] name = "catena-cli" version = "0.1.0" @@ -738,6 +762,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "inout" version = "0.1.4" @@ -862,6 +895,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -963,6 +1005,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "potential_utf" version = "0.1.6" @@ -996,6 +1044,67 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pyo3" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +dependencies = [ + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.119", +] + [[package]] name = "quote" version = "1.0.47" @@ -1166,6 +1275,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "schemars" version = "1.2.2" @@ -1336,6 +1451,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + [[package]] name = "textkit" version = "0.1.0" @@ -1471,6 +1592,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index e94470d..b7deead 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,13 @@ cuepack = { path = "crates/cuepack", version = "0.1.0" } bibclean = { path = "crates/bibclean", version = "0.1.0" } avplan = { path = "crates/avplan", version = "0.1.0" } pdfgutter = { path = "crates/pdfgutter", version = "0.1.0" } +catena = { path = "crates/bindings", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } clap = { version = "4", features = ["derive"] } clap_mangen = "0.3" +pyo3 = { version = "0.27", features = ["extension-module", "abi3-py39"] } deunicode = "1" fancy-regex = "0.16" # 0.38 is the last release that builds on the pinned toolchain; the ones @@ -56,3 +58,11 @@ missing_docs = "warn" strip = true lto = true codegen-units = 1 + +# The Python extension module, which cannot be stripped the way a binary can. +# Stripping a dylib's symbols leaves one macOS refuses to load at all — +# "mis-aligned LINKEDIT string pool" — and there is nothing to strip for +# anyway, since the interpreter resolves its symbols at load time. +[profile.wheel] +inherits = "release" +strip = false diff --git a/README.md b/README.md index c8c1260..e0567ad 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ command does can be done in process instead. | [`textkit`](crates/textkit/README.md) | Token counting, sentence and paragraph splitting | | [`cuekit`](crates/cuekit/README.md) | The cue model, and the three subtitle formats it is read from | | [`catena-cli`](crates/catena-cli/README.md) | The parts of a filter every command does the same way | +| [`bindings`](crates/bindings/README.md) | The same libraries from Python, for callers already holding the text | ## A worked pipeline diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml new file mode 100644 index 0000000..6689fb4 --- /dev/null +++ b/crates/bindings/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "catena" +description = "The catena libraries behind one surface, for calling in process rather than as a subprocess." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "catena" +path = "src/lib.rs" +# `cdylib` for the bindings, `rlib` so the workspace can still depend on it. +crate-type = ["cdylib", "rlib"] + +[dependencies] +booksniff.workspace = true +chunkmd.workspace = true +corpus-core.workspace = true +cuekit.workspace = true +cuepack.workspace = true +mdscrub.workspace = true +mdsplit.workspace = true +rrf.workspace = true +serde.workspace = true +serde_json.workspace = true +textkit.workspace = true + +pyo3 = { workspace = true, optional = true } + +[features] +# Only maturin turns this on. pyo3's extension-module feature leaves the +# Python symbols to be resolved by the interpreter at load time, so an +# ordinary `cargo build` would fail to link if it were on by default. +python = ["dep:pyo3"] + +[lints] +workspace = true diff --git a/crates/bindings/LICENSE b/crates/bindings/LICENSE new file mode 100644 index 0000000..a612ad9 --- /dev/null +++ b/crates/bindings/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/crates/bindings/README.md b/crates/bindings/README.md new file mode 100644 index 0000000..3046da8 --- /dev/null +++ b/crates/bindings/README.md @@ -0,0 +1,99 @@ +# catena (bindings) + +The catena libraries behind one surface, for calling in process rather than as +a subprocess. + +Part of [catena](../../README.md). + +## What it does + +Every tool in this workspace is a library with a thin command on top. A +pipeline written in another language can have either. + +A subprocess is the right answer most of the time: it is a real process +boundary, it composes with a shell, and it costs a few milliseconds. In +process is the right answer when the caller is already holding the text — a +stage that would otherwise write a file so that it can read it straight back. + +What comes back is the same **record shapes** the commands write to stdout, +as dictionaries rather than lines of JSON. Those shapes are documented and +versioned; the structs behind them are free to move. + +## Python + +```sh +pip install catena +``` + +```python +import catena + +chunks = catena.chunk_markdown(markdown, target=512, overlap=64) +chunks[0]["page"] # 41 +chunks[0]["chapter"] # 'III. The Second Estate' + +clean = catena.scrub(raw_markdown, pages=True) +catena.count_tokens(clean) + +cues = catena.parse_subtitles(open("episode.srt").read()) +passages = catena.pack_cues(cues, seconds=75) + +catena.identify("atlas.pdf", open("atlas.pdf", "rb").read(1024))["kind"] +``` + +`split_collection` raises `ValueError` when the document is one work rather +than a collection — the usual answer, with the reason each candidate was +rejected: + +```python +try: + works = catena.split_collection(markdown) +except ValueError as why: + works = [{"idx": 0, "title": title, "text": markdown}] +``` + +### What is bound + +| Function | The tool it is | +|---|---| +| `chunk_markdown` | [`chunkmd`](../chunkmd/README.md) | +| `scrub` | [`mdscrub`](../mdscrub/README.md) | +| `count_tokens` | [`textkit`](../textkit/README.md) | +| `parse_subtitles` | [`cuenorm`](../cuenorm/README.md) | +| `pack_cues` | [`cuepack`](../cuepack/README.md) | +| `split_collection` | [`mdsplit`](../mdsplit/README.md) | +| `fuse_hits` | [`rrf`](../rrf/README.md) | +| `identify` | [`booksniff`](../booksniff/README.md) | + +The tools that are not here are the ones a binding would gain nothing over a +subprocess for: they read files, shell out to another program, or talk to a +server, and none of that is cheaper in process. + +## Building the wheel + +```sh +cd crates/bindings && maturin build --release +``` + +`cargo build --features python` does not work and is not supposed to: pyo3's +`extension-module` leaves the Python symbols to be resolved by the interpreter +at load time, so there is nothing to link against until an interpreter loads +it. `cargo clippy --features python` type-checks it without linking, which is +what the gate runs. + +## As a Rust library + +The same surface, without a binding in the way: + +```rust +let chunks = catena::chunk_markdown(&markdown, catena::ChunkBudgets::default()); +``` + +Nothing here grows logic of its own. If a binding needs something the +libraries cannot do, that is a missing library function rather than a binding +feature. + +## See also + +- [The records](../../docs/records.md) — the shapes these hand back. +- [Recipes](../../docs/recipes.md) — the same pipelines from a shell. diff --git a/crates/bindings/pyproject.toml b/crates/bindings/pyproject.toml new file mode 100644 index 0000000..1109047 --- /dev/null +++ b/crates/bindings/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["maturin>=1.9,<2.0"] +build-backend = "maturin" + +[project] +name = "catena" +description = "The catena libraries behind one surface, for calling in process rather than as a subprocess" +readme = "README.md" +license = "MPL-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.9" +keywords = ["chunking", "markdown", "subtitles", "retrieval", "corpus", "tokens"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Rust", + "Topic :: Text Processing :: Markup", + "Topic :: Software Development :: Libraries", + "Typing :: Typed", +] +# maturin takes it from Cargo.toml, so there is one place to bump. +dynamic = ["version"] + +[project.urls] +Homepage = "https://git.pumpin.online/funkpower/catena" +Repository = "https://git.pumpin.online/funkpower/catena" + +[tool.maturin] +bindings = "pyo3" +module-name = "catena._catena" +# Mixed layout: the compiled module is private and python/catena wraps it, +# which is what lets the wheel carry py.typed and a stub file. A bare +# extension module has nowhere to put either. +python-source = "python" +# The feature is optional so that an ordinary `cargo build` does not try to +# link against a Python that is not there. This is the only place it is +# turned on. +features = ["python"] +profile = "wheel" +manifest-path = "Cargo.toml" diff --git a/crates/bindings/python/catena/__init__.py b/crates/bindings/python/catena/__init__.py new file mode 100644 index 0000000..bdecad5 --- /dev/null +++ b/crates/bindings/python/catena/__init__.py @@ -0,0 +1,35 @@ +"""The catena libraries, in process. + +Every function here is the same code the command-line tool runs, and hands +back the same record shapes it writes to stdout — dictionaries rather than +lines of JSON. + + >>> import catena + >>> chunks = catena.chunk_markdown(markdown, target=512) + >>> chunks[0]["page"] + 41 +""" + +from ._catena import ( + __version__, + chunk_markdown, + count_tokens, + fuse_hits, + identify, + pack_cues, + parse_subtitles, + scrub, + split_collection, +) + +__all__ = [ + "__version__", + "chunk_markdown", + "count_tokens", + "fuse_hits", + "identify", + "pack_cues", + "parse_subtitles", + "scrub", + "split_collection", +] diff --git a/crates/bindings/python/catena/__init__.pyi b/crates/bindings/python/catena/__init__.pyi new file mode 100644 index 0000000..c3bd903 --- /dev/null +++ b/crates/bindings/python/catena/__init__.pyi @@ -0,0 +1,32 @@ +from typing import Any, Iterable, Mapping, Sequence + +__version__: str + +def chunk_markdown( + markdown: str, + *, + target: int | None = ..., + overlap: int | None = ..., + min_tokens: int | None = ..., + hard_max: int | None = ..., +) -> list[dict[str, Any]]: ... +def scrub(markdown: str, *, pages: bool = ...) -> str: ... +def count_tokens(text: str) -> int: ... +def parse_subtitles( + content: str, *, keep_repeats: bool = ... +) -> list[dict[str, Any]]: ... +def pack_cues( + cues: Iterable[Mapping[str, Any]], + *, + seconds: float | None = ..., + max_tokens: int | None = ..., +) -> list[dict[str, Any]]: ... +def split_collection(markdown: str) -> list[dict[str, Any]]: ... +def fuse_hits( + hits: Iterable[Mapping[str, Any]], + *, + k: float | None = ..., + weights: Mapping[str, float] | None = ..., + limit: int | None = ..., +) -> list[dict[str, Any]]: ... +def identify(name: str, data: bytes) -> dict[str, Any]: ... diff --git a/crates/bindings/python/catena/py.typed b/crates/bindings/python/catena/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs new file mode 100644 index 0000000..40d366d --- /dev/null +++ b/crates/bindings/src/lib.rs @@ -0,0 +1,235 @@ +//! The catena libraries behind one surface, for calling in process. +//! +//! Every tool in this workspace is a library with a thin command on top, and +//! a pipeline written in another language can have either. A subprocess is +//! the right answer most of the time: it is a process boundary, it composes +//! with a shell, and it costs a few milliseconds. In process is the right +//! answer when the caller is already holding the text — a stage that would +//! otherwise write a file so that it can read it straight back. +//! +//! What is bound here is the **record shapes**, not the internals. Those +//! shapes are documented, versioned, and what every tool already writes to +//! stdout; binding the structs behind them would publish field names that are +//! free to move and leave the two forms disagreeing about the same chunk. +//! +//! Nothing here grows logic of its own. If a binding needs something the +//! libraries cannot do, that is a missing library function. + +#[cfg(feature = "python")] +mod python; + +use corpus_core::{Chunk, Doc, Hit}; +use cuekit::Cue; + +/// Where chunks are flushed, and how large they may get. +#[derive(Debug, Clone, Copy)] +pub struct ChunkBudgets { + /// Flush a chunk at this many tokens. + pub target: usize, + /// Carry this much of each chunk into the next one. + pub overlap: usize, + /// Drop chunks smaller than this. + pub min_tokens: usize, + /// Never emit a chunk larger than this. + pub hard_max: usize, +} + +impl Default for ChunkBudgets { + fn default() -> Self { + let budgets = chunkmd::Budgets::default(); + Self { + target: budgets.target, + overlap: budgets.overlap, + min_tokens: budgets.min_tokens, + hard_max: budgets.hard_max, + } + } +} + +impl From for chunkmd::Budgets { + fn from(budgets: ChunkBudgets) -> Self { + chunkmd::Budgets { + target: budgets.target, + overlap: budgets.overlap, + min_tokens: budgets.min_tokens, + hard_max: budgets.hard_max, + } + } +} + +/// How much of a recording a passage covers. +#[derive(Debug, Clone, Copy)] +pub struct CueBudgets { + /// Seconds of audio a passage covers. + pub seconds: f64, + /// Tokens a passage may hold, whatever the clock says. + pub max_tokens: usize, +} + +impl Default for CueBudgets { + fn default() -> Self { + let budgets = cuepack::Budgets::default(); + Self { + seconds: budgets.seconds, + max_tokens: budgets.max_tokens, + } + } +} + +/// Cut markdown into retrievable passages. +pub fn chunk_markdown(markdown: &str, budgets: ChunkBudgets) -> Vec { + chunkmd::chunk(markdown, &budgets.into()) +} + +/// Clean converted markdown. +pub fn scrub_markdown(markdown: &str) -> String { + mdscrub::scrub(markdown) +} + +/// Clean converted markdown, normalising page markers as well. +/// +/// Separate because it is not idempotent: running it twice renumbers the +/// pages it moved the first time. +pub fn scrub_markdown_pages(markdown: &str) -> String { + mdscrub::pages::normalize(&mdscrub::scrub(markdown)) +} + +/// How many tokens a string is, by the encoding every budget here counts in. +pub fn count_tokens(text: &str) -> usize { + textkit::count(text) +} + +/// Read any of the subtitle formats into cues. +pub fn parse_subtitles(content: &str) -> Vec { + cuekit::parse(content) +} + +/// Collapse the runs of repeated cues a live caption track leaves behind. +pub fn merge_repeats(cues: &[Cue]) -> Vec { + cuekit::merge_repeats(cues) +} + +/// Pack cues into passages worth embedding. +pub fn pack_cues(cues: &[Cue], budgets: CueBudgets) -> Vec { + cuepack::pack( + cues, + &cuepack::Budgets { + seconds: budgets.seconds, + max_tokens: budgets.max_tokens, + }, + ) +} + +/// Split a collection into the works inside it. +/// +/// `Err` when the document is one work rather than a collection, which is the +/// usual answer and not a failure. The message says what each candidate +/// produced. +pub fn split_collection(markdown: &str) -> Result, String> { + mdsplit::split(markdown).map_err(|why| why.to_string()) +} + +/// Fuse ranked lists into one. +/// +/// `weights` is read as channel name to weight; a channel not named is worth +/// one. An unknown name is ignored rather than refused, because a caller +/// fusing two channels should not have to know about a third. +pub fn fuse_hits( + hits: &[Hit], + k: f64, + weights: &[(String, f64)], + limit: Option, +) -> Vec { + let mut channels = std::collections::HashMap::new(); + for (name, weight) in weights { + if let Some(channel) = channel_named(name) { + channels.insert(channel, *weight); + } + } + let fused = rrf::fuse(hits, &rrf::Weights { k, channels }); + match limit { + Some(limit) => fused.into_iter().take(limit).collect(), + None => fused, + } +} + +/// The channel a name refers to, spelled as the records spell it. +fn channel_named(name: &str) -> Option { + match name { + "vec" => Some(corpus_core::Channel::Vector), + "fts" => Some(corpus_core::Channel::Fts), + "meta" => Some(corpus_core::Channel::Meta), + _ => None, + } +} + +/// Say what a file is from its bytes and its name. +pub fn identify(name: &str, bytes: &[u8]) -> Doc { + booksniff::sniff(&bytes, name) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MARKDOWN: &str = "# Chapter One\n\n\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked. A lord who could not protect was a lord who could not command.\n"; + + #[test] + fn the_surface_agrees_with_the_library_underneath() { + assert_eq!( + chunk_markdown(MARKDOWN, ChunkBudgets::default()), + chunkmd::chunk(MARKDOWN, &chunkmd::Budgets::default()) + ); + assert_eq!(scrub_markdown(MARKDOWN), mdscrub::scrub(MARKDOWN)); + assert_eq!(count_tokens("a passage"), textkit::count("a passage")); + } + + #[test] + fn the_default_budgets_are_the_tools_defaults() { + let ours = ChunkBudgets::default(); + let theirs = chunkmd::Budgets::default(); + assert_eq!( + (ours.target, ours.overlap, ours.min_tokens, ours.hard_max), + ( + theirs.target, + theirs.overlap, + theirs.min_tokens, + theirs.hard_max + ) + ); + assert_eq!( + CueBudgets::default().seconds, + cuepack::Budgets::default().seconds + ); + } + + #[test] + fn a_document_that_is_one_work_says_so_rather_than_splitting() { + assert!(split_collection(MARKDOWN).is_err()); + } + + #[test] + fn subtitles_come_back_as_cues_and_pack_into_passages() { + let srt = "1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n"; + let cues = parse_subtitles(srt); + assert_eq!(cues.len(), 1); + let packed = pack_cues(&cues, CueBudgets::default()); + assert_eq!(packed.len(), 1); + assert_eq!(packed[0].start_time, Some(1.0)); + } + + #[test] + fn a_channel_nobody_here_knows_is_ignored_rather_than_refused() { + assert_eq!(channel_named("vec"), Some(corpus_core::Channel::Vector)); + assert_eq!(channel_named("lexical"), None); + } + + #[test] + fn bytes_are_identified_the_same_way_a_file_is() { + let pdf = b"%PDF-1.4\n"; + assert_eq!( + identify("book.pdf", pdf).kind, + booksniff::sniff(&&pdf[..], "book.pdf").kind + ); + } +} diff --git a/crates/bindings/src/python.rs b/crates/bindings/src/python.rs new file mode 100644 index 0000000..9ec8307 --- /dev/null +++ b/crates/bindings/src/python.rs @@ -0,0 +1,208 @@ +//! Python bindings. +//! +//! These hand back the record shapes as dictionaries — the same JSON every +//! tool writes to stdout, spelled the way Python spells things. A stage that +//! was reading NDJSON from a subprocess can call the function instead and get +//! the dictionaries it was already parsing. +//! +//! Built only under the `python` feature, which maturin turns on. See +//! Cargo.toml for why it cannot be on by default. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList}; + +use crate::{ChunkBudgets, CueBudgets}; + +/// A record to a Python dictionary, by way of its JSON. +/// +/// Through JSON rather than field by field so that the two forms cannot drift +/// apart: if a record gains a field, it appears here without anybody +/// remembering to add it. +fn to_py<'py, T: serde::Serialize>(py: Python<'py>, value: &T) -> PyResult> { + let json = serde_json::to_value(value) + .map_err(|e| PyValueError::new_err(format!("record does not serialise: {e}")))?; + json_to_py(py, &json) +} + +/// Every record in a list, likewise. +fn list_to_py<'py, T: serde::Serialize>( + py: Python<'py>, + values: &[T], +) -> PyResult> { + let mut out = Vec::with_capacity(values.len()); + for value in values { + out.push(to_py(py, value)?); + } + PyList::new(py, out) +} + +fn json_to_py<'py>(py: Python<'py>, value: &serde_json::Value) -> PyResult> { + Ok(match value { + serde_json::Value::Null => py.None().into_bound(py), + serde_json::Value::Bool(b) => b.into_pyobject(py)?.to_owned().into_any(), + serde_json::Value::Number(n) => match (n.as_u64(), n.as_i64(), n.as_f64()) { + (Some(u), _, _) => u.into_pyobject(py)?.into_any(), + (_, Some(i), _) => i.into_pyobject(py)?.into_any(), + (_, _, Some(f)) => f.into_pyobject(py)?.into_any(), + // JSON cannot hold a number this is not one of, so this is + // unreachable rather than a case worth inventing a value for. + _ => return Err(PyValueError::new_err(format!("{n} is not a number"))), + }, + serde_json::Value::String(s) => s.into_pyobject(py)?.into_any(), + serde_json::Value::Array(items) => { + let mut out = Vec::with_capacity(items.len()); + for item in items { + out.push(json_to_py(py, item)?); + } + PyList::new(py, out)?.into_any() + } + serde_json::Value::Object(fields) => { + let dict = PyDict::new(py); + for (key, value) in fields { + dict.set_item(key, json_to_py(py, value)?)?; + } + dict.into_any() + } + }) +} + +/// A Python dictionary back to a record, the same way round. +fn from_py(value: &Bound<'_, PyAny>) -> PyResult { + let json: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + serde_json::from_str(&json) + .map_err(|e| PyValueError::new_err(format!("not a record of the expected shape: {e}"))) +} + +/// Every dictionary in a sequence. +fn from_py_list(values: &Bound<'_, PyAny>) -> PyResult> { + let mut out = Vec::new(); + for value in values.try_iter()? { + out.push(from_py(&value?)?); + } + Ok(out) +} + +/// Cut markdown into retrievable passages, one dictionary each. +#[pyfunction] +#[pyo3(signature = (markdown, *, target = None, overlap = None, min_tokens = None, hard_max = None))] +fn chunk_markdown<'py>( + py: Python<'py>, + markdown: &str, + target: Option, + overlap: Option, + min_tokens: Option, + hard_max: Option, +) -> PyResult> { + let default = ChunkBudgets::default(); + let budgets = ChunkBudgets { + target: target.unwrap_or(default.target), + overlap: overlap.unwrap_or(default.overlap), + min_tokens: min_tokens.unwrap_or(default.min_tokens), + hard_max: hard_max.unwrap_or(default.hard_max), + }; + list_to_py(py, &crate::chunk_markdown(markdown, budgets)) +} + +/// Clean converted markdown. +#[pyfunction] +#[pyo3(signature = (markdown, *, pages = false))] +fn scrub(markdown: &str, pages: bool) -> String { + match pages { + true => crate::scrub_markdown_pages(markdown), + false => crate::scrub_markdown(markdown), + } +} + +/// How many tokens a string is. +#[pyfunction] +fn count_tokens(text: &str) -> usize { + crate::count_tokens(text) +} + +/// Read any of the subtitle formats into cues. +#[pyfunction] +#[pyo3(signature = (content, *, keep_repeats = false))] +fn parse_subtitles<'py>( + py: Python<'py>, + content: &str, + keep_repeats: bool, +) -> PyResult> { + let cues = crate::parse_subtitles(content); + let cues = match keep_repeats { + true => cues, + false => crate::merge_repeats(&cues), + }; + list_to_py(py, &cues) +} + +/// Pack cues into passages worth embedding. +#[pyfunction] +#[pyo3(signature = (cues, *, seconds = None, max_tokens = None))] +fn pack_cues<'py>( + py: Python<'py>, + cues: &Bound<'py, PyAny>, + seconds: Option, + max_tokens: Option, +) -> PyResult> { + let default = CueBudgets::default(); + let budgets = CueBudgets { + seconds: seconds.unwrap_or(default.seconds), + max_tokens: max_tokens.unwrap_or(default.max_tokens), + }; + let cues: Vec = from_py_list(cues)?; + list_to_py(py, &crate::pack_cues(&cues, budgets)) +} + +/// Split a collection into the works inside it. +/// +/// Raises `ValueError` when the document is one work rather than a +/// collection, with the reason each candidate was rejected. +#[pyfunction] +fn split_collection<'py>(py: Python<'py>, markdown: &str) -> PyResult> { + match crate::split_collection(markdown) { + Ok(sections) => list_to_py(py, §ions), + Err(why) => Err(PyValueError::new_err(why)), + } +} + +/// Fuse ranked lists into one. +#[pyfunction] +#[pyo3(signature = (hits, *, k = None, weights = None, limit = None))] +fn fuse_hits<'py>( + py: Python<'py>, + hits: &Bound<'py, PyAny>, + k: Option, + weights: Option>, + limit: Option, +) -> PyResult> { + let hits: Vec = from_py_list(hits)?; + let weights: Vec<(String, f64)> = weights.unwrap_or_default().into_iter().collect(); + let fused = crate::fuse_hits(&hits, k.unwrap_or(rrf::K), &weights, limit); + list_to_py(py, &fused) +} + +/// Say what a file is from its bytes and its name. +#[pyfunction] +fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult> { + to_py(py, &crate::identify(name, data)) +} + +/// The module Python imports. +#[pymodule] +fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add("__version__", env!("CARGO_PKG_VERSION"))?; + module.add_function(wrap_pyfunction!(chunk_markdown, module)?)?; + module.add_function(wrap_pyfunction!(scrub, module)?)?; + module.add_function(wrap_pyfunction!(count_tokens, module)?)?; + module.add_function(wrap_pyfunction!(parse_subtitles, module)?)?; + module.add_function(wrap_pyfunction!(pack_cues, module)?)?; + module.add_function(wrap_pyfunction!(split_collection, module)?)?; + module.add_function(wrap_pyfunction!(fuse_hits, module)?)?; + module.add_function(wrap_pyfunction!(identify, module)?)?; + Ok(()) +}