mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
bindings: the libraries in process, for Python
This commit is contained in:
funkpower
Generated
+127
@@ -84,6 +84,12 @@ version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "avplan"
|
||||
version = "0.1.0"
|
||||
@@ -199,6 +205,24 @@ version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "catena"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"booksniff",
|
||||
"chunkmd",
|
||||
"corpus-core",
|
||||
"cuekit",
|
||||
"cuepack",
|
||||
"mdscrub",
|
||||
"mdsplit",
|
||||
"pyo3",
|
||||
"rrf",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.1.0"
|
||||
@@ -738,6 +762,15 @@ dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indoc"
|
||||
version = "2.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.4"
|
||||
@@ -862,6 +895,15 @@ version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
@@ -963,6 +1005,12 @@ version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.6"
|
||||
@@ -996,6 +1044,67 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d"
|
||||
dependencies = [
|
||||
"indoc",
|
||||
"libc",
|
||||
"memoffset",
|
||||
"once_cell",
|
||||
"portable-atomic",
|
||||
"pyo3-build-config",
|
||||
"pyo3-ffi",
|
||||
"pyo3-macros",
|
||||
"unindent",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-build-config"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6"
|
||||
dependencies = [
|
||||
"target-lexicon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-ffi"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pyo3-build-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-macros"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"pyo3-macros-backend",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-macros-backend"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"pyo3-build-config",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
@@ -1166,6 +1275,12 @@ dependencies = [
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.2"
|
||||
@@ -1336,6 +1451,12 @@ dependencies = [
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.1.0"
|
||||
@@ -1471,6 +1592,12 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
||||
|
||||
[[package]]
|
||||
name = "unindent"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
|
||||
+10
@@ -25,11 +25,13 @@ cuepack = { path = "crates/cuepack", version = "0.1.0" }
|
||||
bibclean = { path = "crates/bibclean", version = "0.1.0" }
|
||||
avplan = { path = "crates/avplan", version = "0.1.0" }
|
||||
pdfgutter = { path = "crates/pdfgutter", version = "0.1.0" }
|
||||
catena = { path = "crates/bindings", version = "0.1.0" }
|
||||
booksniff = { path = "crates/booksniff", version = "0.1.0" }
|
||||
emb = { path = "crates/emb", version = "0.1.0" }
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
clap_mangen = "0.3"
|
||||
pyo3 = { version = "0.27", features = ["extension-module", "abi3-py39"] }
|
||||
deunicode = "1"
|
||||
fancy-regex = "0.16"
|
||||
# 0.38 is the last release that builds on the pinned toolchain; the ones
|
||||
@@ -56,3 +58,11 @@ missing_docs = "warn"
|
||||
strip = true
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
|
||||
# The Python extension module, which cannot be stripped the way a binary can.
|
||||
# Stripping a dylib's symbols leaves one macOS refuses to load at all —
|
||||
# "mis-aligned LINKEDIT string pool" — and there is nothing to strip for
|
||||
# anyway, since the interpreter resolves its symbols at load time.
|
||||
[profile.wheel]
|
||||
inherits = "release"
|
||||
strip = false
|
||||
|
||||
@@ -56,6 +56,7 @@ command does can be done in process instead.
|
||||
| [`textkit`](crates/textkit/README.md) | Token counting, sentence and paragraph splitting |
|
||||
| [`cuekit`](crates/cuekit/README.md) | The cue model, and the three subtitle formats it is read from |
|
||||
| [`catena-cli`](crates/catena-cli/README.md) | The parts of a filter every command does the same way |
|
||||
| [`bindings`](crates/bindings/README.md) | The same libraries from Python, for callers already holding the text |
|
||||
|
||||
## A worked pipeline
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
[package]
|
||||
name = "catena"
|
||||
description = "The catena libraries behind one surface, for calling in process rather than as a subprocess."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "catena"
|
||||
path = "src/lib.rs"
|
||||
# `cdylib` for the bindings, `rlib` so the workspace can still depend on it.
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[dependencies]
|
||||
booksniff.workspace = true
|
||||
chunkmd.workspace = true
|
||||
corpus-core.workspace = true
|
||||
cuekit.workspace = true
|
||||
cuepack.workspace = true
|
||||
mdscrub.workspace = true
|
||||
mdsplit.workspace = true
|
||||
rrf.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
textkit.workspace = true
|
||||
|
||||
pyo3 = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
# Only maturin turns this on. pyo3's extension-module feature leaves the
|
||||
# Python symbols to be resolved by the interpreter at load time, so an
|
||||
# ordinary `cargo build` would fail to link if it were on by default.
|
||||
python = ["dep:pyo3"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,373 @@
|
||||
Mozilla Public License Version 2.0
|
||||
==================================
|
||||
|
||||
1. Definitions
|
||||
--------------
|
||||
|
||||
1.1. "Contributor"
|
||||
means each individual or legal entity that creates, contributes to
|
||||
the creation of, or owns Covered Software.
|
||||
|
||||
1.2. "Contributor Version"
|
||||
means the combination of the Contributions of others (if any) used
|
||||
by a Contributor and that particular Contributor's Contribution.
|
||||
|
||||
1.3. "Contribution"
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. "Covered Software"
|
||||
means Source Code Form to which the initial Contributor has attached
|
||||
the notice in Exhibit A, the Executable Form of such Source Code
|
||||
Form, and Modifications of such Source Code Form, in each case
|
||||
including portions thereof.
|
||||
|
||||
1.5. "Incompatible With Secondary Licenses"
|
||||
means
|
||||
|
||||
(a) that the initial Contributor has attached the notice described
|
||||
in Exhibit B to the Covered Software; or
|
||||
|
||||
(b) that the Covered Software was made available under the terms of
|
||||
version 1.1 or earlier of the License, but not also under the
|
||||
terms of a Secondary License.
|
||||
|
||||
1.6. "Executable Form"
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. "Larger Work"
|
||||
means a work that combines Covered Software with other material, in
|
||||
a separate file or files, that is not Covered Software.
|
||||
|
||||
1.8. "License"
|
||||
means this document.
|
||||
|
||||
1.9. "Licensable"
|
||||
means having the right to grant, to the maximum extent possible,
|
||||
whether at the time of the initial grant or subsequently, any and
|
||||
all of the rights conveyed by this License.
|
||||
|
||||
1.10. "Modifications"
|
||||
means any of the following:
|
||||
|
||||
(a) any file in Source Code Form that results from an addition to,
|
||||
deletion from, or modification of the contents of Covered
|
||||
Software; or
|
||||
|
||||
(b) any new file in Source Code Form that contains any Covered
|
||||
Software.
|
||||
|
||||
1.11. "Patent Claims" of a Contributor
|
||||
means any patent claim(s), including without limitation, method,
|
||||
process, and apparatus claims, in any patent Licensable by such
|
||||
Contributor that would be infringed, but for the grant of the
|
||||
License, by the making, using, selling, offering for sale, having
|
||||
made, import, or transfer of either its Contributions or its
|
||||
Contributor Version.
|
||||
|
||||
1.12. "Secondary License"
|
||||
means either the GNU General Public License, Version 2.0, the GNU
|
||||
Lesser General Public License, Version 2.1, the GNU Affero General
|
||||
Public License, Version 3.0, or any later versions of those
|
||||
licenses.
|
||||
|
||||
1.13. "Source Code Form"
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. "You" (or "Your")
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, "You" includes any entity that
|
||||
controls, is controlled by, or is under common control with You. For
|
||||
purposes of this definition, "control" means (a) the power, direct
|
||||
or indirect, to cause the direction or management of such entity,
|
||||
whether by contract or otherwise, or (b) ownership of more than
|
||||
fifty percent (50%) of the outstanding shares or beneficial
|
||||
ownership of such entity.
|
||||
|
||||
2. License Grants and Conditions
|
||||
--------------------------------
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
(a) under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or
|
||||
as part of a Larger Work; and
|
||||
|
||||
(b) under Patent Claims of such Contributor to make, use, sell, offer
|
||||
for sale, have made, import, and otherwise transfer either its
|
||||
Contributions or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution
|
||||
become effective for each Contribution on the date the Contributor first
|
||||
distributes such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under
|
||||
this License. No additional rights or licenses will be implied from the
|
||||
distribution or licensing of Covered Software under this License.
|
||||
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
||||
Contributor:
|
||||
|
||||
(a) for any code that a Contributor has removed from Covered Software;
|
||||
or
|
||||
|
||||
(b) for infringements caused by: (i) Your and any other third party's
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
(c) under Patent Claims infringed by Covered Software in the absence of
|
||||
its Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks,
|
||||
or logos of any Contributor (except as may be necessary to comply with
|
||||
the notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this
|
||||
License (see Section 10.2) or under the terms of a Secondary License (if
|
||||
permitted under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its
|
||||
Contributions are its original creation(s) or it has sufficient rights
|
||||
to grant the rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under
|
||||
applicable copyright doctrines of fair use, fair dealing, or other
|
||||
equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
|
||||
in Section 2.1.
|
||||
|
||||
3. Responsibilities
|
||||
-------------------
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under
|
||||
the terms of this License. You must inform recipients that the Source
|
||||
Code Form of the Covered Software is governed by the terms of this
|
||||
License, and how they can obtain a copy of this License. You may not
|
||||
attempt to alter or restrict the recipients' rights in the Source Code
|
||||
Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
(a) such Covered Software must also be made available in Source Code
|
||||
Form, as described in Section 3.1, and You must inform recipients of
|
||||
the Executable Form how they can obtain a copy of such Source Code
|
||||
Form by reasonable means in a timely manner, at a charge no more
|
||||
than the cost of distribution to the recipient; and
|
||||
|
||||
(b) You may distribute such Executable Form under the terms of this
|
||||
License, or sublicense it under different terms, provided that the
|
||||
license for the Executable Form does not attempt to limit or alter
|
||||
the recipients' rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for
|
||||
the Covered Software. If the Larger Work is a combination of Covered
|
||||
Software with a work governed by one or more Secondary Licenses, and the
|
||||
Covered Software is not Incompatible With Secondary Licenses, this
|
||||
License permits You to additionally distribute such Covered Software
|
||||
under the terms of such Secondary License(s), so that the recipient of
|
||||
the Larger Work may, at their option, further distribute the Covered
|
||||
Software under the terms of either this License or such Secondary
|
||||
License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices
|
||||
(including copyright notices, patent notices, disclaimers of warranty,
|
||||
or limitations of liability) contained within the Source Code Form of
|
||||
the Covered Software, except that You may alter any license notices to
|
||||
the extent required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on
|
||||
behalf of any Contributor. You must make it absolutely clear that any
|
||||
such warranty, support, indemnity, or liability obligation is offered by
|
||||
You alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
---------------------------------------------------
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this
|
||||
License with respect to some or all of the Covered Software due to
|
||||
statute, judicial order, or regulation then You must: (a) comply with
|
||||
the terms of this License to the maximum extent possible; and (b)
|
||||
describe the limitations and the code they affect. Such description must
|
||||
be placed in a text file included with all distributions of the Covered
|
||||
Software under this License. Except to the extent prohibited by statute
|
||||
or regulation, such description must be sufficiently detailed for a
|
||||
recipient of ordinary skill to be able to understand it.
|
||||
|
||||
5. Termination
|
||||
--------------
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically
|
||||
if You fail to comply with any of its terms. However, if You become
|
||||
compliant, then the rights granted under this License from a particular
|
||||
Contributor are reinstated (a) provisionally, unless and until such
|
||||
Contributor explicitly and finally terminates Your grants, and (b) on an
|
||||
ongoing basis, if such Contributor fails to notify You of the
|
||||
non-compliance by some reasonable means prior to 60 days after You have
|
||||
come back into compliance. Moreover, Your grants from a particular
|
||||
Contributor are reinstated on an ongoing basis if such Contributor
|
||||
notifies You of the non-compliance by some reasonable means, this is the
|
||||
first time You have received notice of non-compliance with this License
|
||||
from such Contributor, and You become compliant prior to 30 days after
|
||||
Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions,
|
||||
counter-claims, and cross-claims) alleging that a Contributor Version
|
||||
directly or indirectly infringes any patent, then the rights granted to
|
||||
You by any and all Contributors for the Covered Software under Section
|
||||
2.1 of this License shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
|
||||
end user license agreements (excluding distributors and resellers) which
|
||||
have been validly granted by You or Your distributors under this License
|
||||
prior to termination shall survive termination.
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 6. Disclaimer of Warranty *
|
||||
* ------------------------- *
|
||||
* *
|
||||
* Covered Software is provided under this License on an "as is" *
|
||||
* basis, without warranty of any kind, either expressed, implied, or *
|
||||
* statutory, including, without limitation, warranties that the *
|
||||
* Covered Software is free of defects, merchantable, fit for a *
|
||||
* particular purpose or non-infringing. The entire risk as to the *
|
||||
* quality and performance of the Covered Software is with You. *
|
||||
* Should any Covered Software prove defective in any respect, You *
|
||||
* (not any Contributor) assume the cost of any necessary servicing, *
|
||||
* repair, or correction. This disclaimer of warranty constitutes an *
|
||||
* essential part of this License. No use of any Covered Software is *
|
||||
* authorized under this License except under this disclaimer. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 7. Limitation of Liability *
|
||||
* -------------------------- *
|
||||
* *
|
||||
* Under no circumstances and under no legal theory, whether tort *
|
||||
* (including negligence), contract, or otherwise, shall any *
|
||||
* Contributor, or anyone who distributes Covered Software as *
|
||||
* permitted above, be liable to You for any direct, indirect, *
|
||||
* special, incidental, or consequential damages of any character *
|
||||
* including, without limitation, damages for lost profits, loss of *
|
||||
* goodwill, work stoppage, computer failure or malfunction, or any *
|
||||
* and all other commercial damages or losses, even if such party *
|
||||
* shall have been informed of the possibility of such damages. This *
|
||||
* limitation of liability shall not apply to liability for death or *
|
||||
* personal injury resulting from such party's negligence to the *
|
||||
* extent applicable law prohibits such limitation. Some *
|
||||
* jurisdictions do not allow the exclusion or limitation of *
|
||||
* incidental or consequential damages, so this exclusion and *
|
||||
* limitation may not apply to You. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
8. Litigation
|
||||
-------------
|
||||
|
||||
Any litigation relating to this License may be brought only in the
|
||||
courts of a jurisdiction where the defendant maintains its principal
|
||||
place of business and such litigation shall be governed by laws of that
|
||||
jurisdiction, without reference to its conflict-of-law provisions.
|
||||
Nothing in this Section shall prevent a party's ability to bring
|
||||
cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
----------------
|
||||
|
||||
This License represents the complete agreement concerning the subject
|
||||
matter hereof. If any provision of this License is held to be
|
||||
unenforceable, such provision shall be reformed only to the extent
|
||||
necessary to make it enforceable. Any law or regulation which provides
|
||||
that the language of a contract shall be construed against the drafter
|
||||
shall not be used to construe this License against a Contributor.
|
||||
|
||||
10. Versions of the License
|
||||
---------------------------
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version
|
||||
of the License under which You originally received the Covered Software,
|
||||
or under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a
|
||||
modified version of this License if you rename the license and remove
|
||||
any references to the name of the license steward (except to note that
|
||||
such modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
||||
Licenses
|
||||
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
-------------------------------------------
|
||||
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular
|
||||
file, then You may include the notice in a location (such as a LICENSE
|
||||
file in a relevant directory) where a recipient would be likely to look
|
||||
for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
||||
---------------------------------------------------------
|
||||
|
||||
This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||
defined by the Mozilla Public License, v. 2.0.
|
||||
@@ -0,0 +1,99 @@
|
||||
# catena (bindings)
|
||||
|
||||
The catena libraries behind one surface, for calling in process rather than as
|
||||
a subprocess.
|
||||
|
||||
Part of [catena](../../README.md).
|
||||
|
||||
## What it does
|
||||
|
||||
Every tool in this workspace is a library with a thin command on top. A
|
||||
pipeline written in another language can have either.
|
||||
|
||||
A subprocess is the right answer most of the time: it is a real process
|
||||
boundary, it composes with a shell, and it costs a few milliseconds. In
|
||||
process is the right answer when the caller is already holding the text — a
|
||||
stage that would otherwise write a file so that it can read it straight back.
|
||||
|
||||
What comes back is the same **record shapes** the commands write to stdout,
|
||||
as dictionaries rather than lines of JSON. Those shapes are documented and
|
||||
versioned; the structs behind them are free to move.
|
||||
|
||||
## Python
|
||||
|
||||
```sh
|
||||
pip install catena
|
||||
```
|
||||
|
||||
```python
|
||||
import catena
|
||||
|
||||
chunks = catena.chunk_markdown(markdown, target=512, overlap=64)
|
||||
chunks[0]["page"] # 41
|
||||
chunks[0]["chapter"] # 'III. The Second Estate'
|
||||
|
||||
clean = catena.scrub(raw_markdown, pages=True)
|
||||
catena.count_tokens(clean)
|
||||
|
||||
cues = catena.parse_subtitles(open("episode.srt").read())
|
||||
passages = catena.pack_cues(cues, seconds=75)
|
||||
|
||||
catena.identify("atlas.pdf", open("atlas.pdf", "rb").read(1024))["kind"]
|
||||
```
|
||||
|
||||
`split_collection` raises `ValueError` when the document is one work rather
|
||||
than a collection — the usual answer, with the reason each candidate was
|
||||
rejected:
|
||||
|
||||
```python
|
||||
try:
|
||||
works = catena.split_collection(markdown)
|
||||
except ValueError as why:
|
||||
works = [{"idx": 0, "title": title, "text": markdown}]
|
||||
```
|
||||
|
||||
### What is bound
|
||||
|
||||
| Function | The tool it is |
|
||||
|---|---|
|
||||
| `chunk_markdown` | [`chunkmd`](../chunkmd/README.md) |
|
||||
| `scrub` | [`mdscrub`](../mdscrub/README.md) |
|
||||
| `count_tokens` | [`textkit`](../textkit/README.md) |
|
||||
| `parse_subtitles` | [`cuenorm`](../cuenorm/README.md) |
|
||||
| `pack_cues` | [`cuepack`](../cuepack/README.md) |
|
||||
| `split_collection` | [`mdsplit`](../mdsplit/README.md) |
|
||||
| `fuse_hits` | [`rrf`](../rrf/README.md) |
|
||||
| `identify` | [`booksniff`](../booksniff/README.md) |
|
||||
|
||||
The tools that are not here are the ones a binding would gain nothing over a
|
||||
subprocess for: they read files, shell out to another program, or talk to a
|
||||
server, and none of that is cheaper in process.
|
||||
|
||||
## Building the wheel
|
||||
|
||||
```sh
|
||||
cd crates/bindings && maturin build --release
|
||||
```
|
||||
|
||||
`cargo build --features python` does not work and is not supposed to: pyo3's
|
||||
`extension-module` leaves the Python symbols to be resolved by the interpreter
|
||||
at load time, so there is nothing to link against until an interpreter loads
|
||||
it. `cargo clippy --features python` type-checks it without linking, which is
|
||||
what the gate runs.
|
||||
|
||||
## As a Rust library
|
||||
|
||||
The same surface, without a binding in the way:
|
||||
|
||||
```rust
|
||||
let chunks = catena::chunk_markdown(&markdown, catena::ChunkBudgets::default());
|
||||
```
|
||||
|
||||
Nothing here grows logic of its own. If a binding needs something the
|
||||
libraries cannot do, that is a missing library function rather than a binding
|
||||
feature.
|
||||
|
||||
## See also
|
||||
|
||||
- [The records](../../docs/records.md) — the shapes these hand back.
|
||||
- [Recipes](../../docs/recipes.md) — the same pipelines from a shell.
|
||||
@@ -0,0 +1,47 @@
|
||||
[build-system]
|
||||
requires = ["maturin>=1.9,<2.0"]
|
||||
build-backend = "maturin"
|
||||
|
||||
[project]
|
||||
name = "catena"
|
||||
description = "The catena libraries behind one surface, for calling in process rather than as a subprocess"
|
||||
readme = "README.md"
|
||||
license = "MPL-2.0"
|
||||
license-files = ["LICENSE"]
|
||||
requires-python = ">=3.9"
|
||||
keywords = ["chunking", "markdown", "subtitles", "retrieval", "corpus", "tokens"]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
"Programming Language :: Rust",
|
||||
"Topic :: Text Processing :: Markup",
|
||||
"Topic :: Software Development :: Libraries",
|
||||
"Typing :: Typed",
|
||||
]
|
||||
# maturin takes it from Cargo.toml, so there is one place to bump.
|
||||
dynamic = ["version"]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://git.pumpin.online/funkpower/catena"
|
||||
Repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[tool.maturin]
|
||||
bindings = "pyo3"
|
||||
module-name = "catena._catena"
|
||||
# Mixed layout: the compiled module is private and python/catena wraps it,
|
||||
# which is what lets the wheel carry py.typed and a stub file. A bare
|
||||
# extension module has nowhere to put either.
|
||||
python-source = "python"
|
||||
# The feature is optional so that an ordinary `cargo build` does not try to
|
||||
# link against a Python that is not there. This is the only place it is
|
||||
# turned on.
|
||||
features = ["python"]
|
||||
profile = "wheel"
|
||||
manifest-path = "Cargo.toml"
|
||||
@@ -0,0 +1,35 @@
|
||||
"""The catena libraries, in process.
|
||||
|
||||
Every function here is the same code the command-line tool runs, and hands
|
||||
back the same record shapes it writes to stdout — dictionaries rather than
|
||||
lines of JSON.
|
||||
|
||||
>>> import catena
|
||||
>>> chunks = catena.chunk_markdown(markdown, target=512)
|
||||
>>> chunks[0]["page"]
|
||||
41
|
||||
"""
|
||||
|
||||
from ._catena import (
|
||||
__version__,
|
||||
chunk_markdown,
|
||||
count_tokens,
|
||||
fuse_hits,
|
||||
identify,
|
||||
pack_cues,
|
||||
parse_subtitles,
|
||||
scrub,
|
||||
split_collection,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"chunk_markdown",
|
||||
"count_tokens",
|
||||
"fuse_hits",
|
||||
"identify",
|
||||
"pack_cues",
|
||||
"parse_subtitles",
|
||||
"scrub",
|
||||
"split_collection",
|
||||
]
|
||||
@@ -0,0 +1,32 @@
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
|
||||
__version__: str
|
||||
|
||||
def chunk_markdown(
|
||||
markdown: str,
|
||||
*,
|
||||
target: int | None = ...,
|
||||
overlap: int | None = ...,
|
||||
min_tokens: int | None = ...,
|
||||
hard_max: int | None = ...,
|
||||
) -> list[dict[str, Any]]: ...
|
||||
def scrub(markdown: str, *, pages: bool = ...) -> str: ...
|
||||
def count_tokens(text: str) -> int: ...
|
||||
def parse_subtitles(
|
||||
content: str, *, keep_repeats: bool = ...
|
||||
) -> list[dict[str, Any]]: ...
|
||||
def pack_cues(
|
||||
cues: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
seconds: float | None = ...,
|
||||
max_tokens: int | None = ...,
|
||||
) -> list[dict[str, Any]]: ...
|
||||
def split_collection(markdown: str) -> list[dict[str, Any]]: ...
|
||||
def fuse_hits(
|
||||
hits: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
k: float | None = ...,
|
||||
weights: Mapping[str, float] | None = ...,
|
||||
limit: int | None = ...,
|
||||
) -> list[dict[str, Any]]: ...
|
||||
def identify(name: str, data: bytes) -> dict[str, Any]: ...
|
||||
@@ -0,0 +1,235 @@
|
||||
//! The catena libraries behind one surface, for calling in process.
|
||||
//!
|
||||
//! Every tool in this workspace is a library with a thin command on top, and
|
||||
//! a pipeline written in another language can have either. A subprocess is
|
||||
//! the right answer most of the time: it is a process boundary, it composes
|
||||
//! with a shell, and it costs a few milliseconds. In process is the right
|
||||
//! answer when the caller is already holding the text — a stage that would
|
||||
//! otherwise write a file so that it can read it straight back.
|
||||
//!
|
||||
//! What is bound here is the **record shapes**, not the internals. Those
|
||||
//! shapes are documented, versioned, and what every tool already writes to
|
||||
//! stdout; binding the structs behind them would publish field names that are
|
||||
//! free to move and leave the two forms disagreeing about the same chunk.
|
||||
//!
|
||||
//! Nothing here grows logic of its own. If a binding needs something the
|
||||
//! libraries cannot do, that is a missing library function.
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
mod python;
|
||||
|
||||
use corpus_core::{Chunk, Doc, Hit};
|
||||
use cuekit::Cue;
|
||||
|
||||
/// Where chunks are flushed, and how large they may get.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ChunkBudgets {
|
||||
/// Flush a chunk at this many tokens.
|
||||
pub target: usize,
|
||||
/// Carry this much of each chunk into the next one.
|
||||
pub overlap: usize,
|
||||
/// Drop chunks smaller than this.
|
||||
pub min_tokens: usize,
|
||||
/// Never emit a chunk larger than this.
|
||||
pub hard_max: usize,
|
||||
}
|
||||
|
||||
impl Default for ChunkBudgets {
|
||||
fn default() -> Self {
|
||||
let budgets = chunkmd::Budgets::default();
|
||||
Self {
|
||||
target: budgets.target,
|
||||
overlap: budgets.overlap,
|
||||
min_tokens: budgets.min_tokens,
|
||||
hard_max: budgets.hard_max,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ChunkBudgets> for chunkmd::Budgets {
|
||||
fn from(budgets: ChunkBudgets) -> Self {
|
||||
chunkmd::Budgets {
|
||||
target: budgets.target,
|
||||
overlap: budgets.overlap,
|
||||
min_tokens: budgets.min_tokens,
|
||||
hard_max: budgets.hard_max,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How much of a recording a passage covers.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct CueBudgets {
|
||||
/// Seconds of audio a passage covers.
|
||||
pub seconds: f64,
|
||||
/// Tokens a passage may hold, whatever the clock says.
|
||||
pub max_tokens: usize,
|
||||
}
|
||||
|
||||
impl Default for CueBudgets {
|
||||
fn default() -> Self {
|
||||
let budgets = cuepack::Budgets::default();
|
||||
Self {
|
||||
seconds: budgets.seconds,
|
||||
max_tokens: budgets.max_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cut markdown into retrievable passages.
|
||||
pub fn chunk_markdown(markdown: &str, budgets: ChunkBudgets) -> Vec<Chunk> {
|
||||
chunkmd::chunk(markdown, &budgets.into())
|
||||
}
|
||||
|
||||
/// Clean converted markdown.
|
||||
pub fn scrub_markdown(markdown: &str) -> String {
|
||||
mdscrub::scrub(markdown)
|
||||
}
|
||||
|
||||
/// Clean converted markdown, normalising page markers as well.
|
||||
///
|
||||
/// Separate because it is not idempotent: running it twice renumbers the
|
||||
/// pages it moved the first time.
|
||||
pub fn scrub_markdown_pages(markdown: &str) -> String {
|
||||
mdscrub::pages::normalize(&mdscrub::scrub(markdown))
|
||||
}
|
||||
|
||||
/// How many tokens a string is, by the encoding every budget here counts in.
|
||||
pub fn count_tokens(text: &str) -> usize {
|
||||
textkit::count(text)
|
||||
}
|
||||
|
||||
/// Read any of the subtitle formats into cues.
|
||||
pub fn parse_subtitles(content: &str) -> Vec<Cue> {
|
||||
cuekit::parse(content)
|
||||
}
|
||||
|
||||
/// Collapse the runs of repeated cues a live caption track leaves behind.
|
||||
pub fn merge_repeats(cues: &[Cue]) -> Vec<Cue> {
|
||||
cuekit::merge_repeats(cues)
|
||||
}
|
||||
|
||||
/// Pack cues into passages worth embedding.
|
||||
pub fn pack_cues(cues: &[Cue], budgets: CueBudgets) -> Vec<Chunk> {
|
||||
cuepack::pack(
|
||||
cues,
|
||||
&cuepack::Budgets {
|
||||
seconds: budgets.seconds,
|
||||
max_tokens: budgets.max_tokens,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Split a collection into the works inside it.
|
||||
///
|
||||
/// `Err` when the document is one work rather than a collection, which is the
|
||||
/// usual answer and not a failure. The message says what each candidate
|
||||
/// produced.
|
||||
pub fn split_collection(markdown: &str) -> Result<Vec<mdsplit::Section>, String> {
|
||||
mdsplit::split(markdown).map_err(|why| why.to_string())
|
||||
}
|
||||
|
||||
/// Fuse ranked lists into one.
|
||||
///
|
||||
/// `weights` is read as channel name to weight; a channel not named is worth
|
||||
/// one. An unknown name is ignored rather than refused, because a caller
|
||||
/// fusing two channels should not have to know about a third.
|
||||
pub fn fuse_hits(
|
||||
hits: &[Hit],
|
||||
k: f64,
|
||||
weights: &[(String, f64)],
|
||||
limit: Option<usize>,
|
||||
) -> Vec<Hit> {
|
||||
let mut channels = std::collections::HashMap::new();
|
||||
for (name, weight) in weights {
|
||||
if let Some(channel) = channel_named(name) {
|
||||
channels.insert(channel, *weight);
|
||||
}
|
||||
}
|
||||
let fused = rrf::fuse(hits, &rrf::Weights { k, channels });
|
||||
match limit {
|
||||
Some(limit) => fused.into_iter().take(limit).collect(),
|
||||
None => fused,
|
||||
}
|
||||
}
|
||||
|
||||
/// The channel a name refers to, spelled as the records spell it.
|
||||
fn channel_named(name: &str) -> Option<corpus_core::Channel> {
|
||||
match name {
|
||||
"vec" => Some(corpus_core::Channel::Vector),
|
||||
"fts" => Some(corpus_core::Channel::Fts),
|
||||
"meta" => Some(corpus_core::Channel::Meta),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Say what a file is from its bytes and its name.
|
||||
pub fn identify(name: &str, bytes: &[u8]) -> Doc {
|
||||
booksniff::sniff(&bytes, name)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const MARKDOWN: &str = "# Chapter One\n\n<!-- Page 41 -->\n\nThe obligation ran both ways, and contemporaries said so plainly whenever they were asked. A lord who could not protect was a lord who could not command.\n";
|
||||
|
||||
#[test]
|
||||
fn the_surface_agrees_with_the_library_underneath() {
|
||||
assert_eq!(
|
||||
chunk_markdown(MARKDOWN, ChunkBudgets::default()),
|
||||
chunkmd::chunk(MARKDOWN, &chunkmd::Budgets::default())
|
||||
);
|
||||
assert_eq!(scrub_markdown(MARKDOWN), mdscrub::scrub(MARKDOWN));
|
||||
assert_eq!(count_tokens("a passage"), textkit::count("a passage"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_default_budgets_are_the_tools_defaults() {
|
||||
let ours = ChunkBudgets::default();
|
||||
let theirs = chunkmd::Budgets::default();
|
||||
assert_eq!(
|
||||
(ours.target, ours.overlap, ours.min_tokens, ours.hard_max),
|
||||
(
|
||||
theirs.target,
|
||||
theirs.overlap,
|
||||
theirs.min_tokens,
|
||||
theirs.hard_max
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
CueBudgets::default().seconds,
|
||||
cuepack::Budgets::default().seconds
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_that_is_one_work_says_so_rather_than_splitting() {
|
||||
assert!(split_collection(MARKDOWN).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subtitles_come_back_as_cues_and_pack_into_passages() {
|
||||
let srt = "1\n00:00:01,000 --> 00:00:04,000\nthe mill stood at the northern end\n\n";
|
||||
let cues = parse_subtitles(srt);
|
||||
assert_eq!(cues.len(), 1);
|
||||
let packed = pack_cues(&cues, CueBudgets::default());
|
||||
assert_eq!(packed.len(), 1);
|
||||
assert_eq!(packed[0].start_time, Some(1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_channel_nobody_here_knows_is_ignored_rather_than_refused() {
|
||||
assert_eq!(channel_named("vec"), Some(corpus_core::Channel::Vector));
|
||||
assert_eq!(channel_named("lexical"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bytes_are_identified_the_same_way_a_file_is() {
|
||||
let pdf = b"%PDF-1.4\n";
|
||||
assert_eq!(
|
||||
identify("book.pdf", pdf).kind,
|
||||
booksniff::sniff(&&pdf[..], "book.pdf").kind
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
//! Python bindings.
|
||||
//!
|
||||
//! These hand back the record shapes as dictionaries — the same JSON every
|
||||
//! tool writes to stdout, spelled the way Python spells things. A stage that
|
||||
//! was reading NDJSON from a subprocess can call the function instead and get
|
||||
//! the dictionaries it was already parsing.
|
||||
//!
|
||||
//! Built only under the `python` feature, which maturin turns on. See
|
||||
//! Cargo.toml for why it cannot be on by default.
|
||||
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyList};
|
||||
|
||||
use crate::{ChunkBudgets, CueBudgets};
|
||||
|
||||
/// A record to a Python dictionary, by way of its JSON.
|
||||
///
|
||||
/// Through JSON rather than field by field so that the two forms cannot drift
|
||||
/// apart: if a record gains a field, it appears here without anybody
|
||||
/// remembering to add it.
|
||||
fn to_py<'py, T: serde::Serialize>(py: Python<'py>, value: &T) -> PyResult<Bound<'py, PyAny>> {
|
||||
let json = serde_json::to_value(value)
|
||||
.map_err(|e| PyValueError::new_err(format!("record does not serialise: {e}")))?;
|
||||
json_to_py(py, &json)
|
||||
}
|
||||
|
||||
/// Every record in a list, likewise.
|
||||
fn list_to_py<'py, T: serde::Serialize>(
|
||||
py: Python<'py>,
|
||||
values: &[T],
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let mut out = Vec::with_capacity(values.len());
|
||||
for value in values {
|
||||
out.push(to_py(py, value)?);
|
||||
}
|
||||
PyList::new(py, out)
|
||||
}
|
||||
|
||||
fn json_to_py<'py>(py: Python<'py>, value: &serde_json::Value) -> PyResult<Bound<'py, PyAny>> {
|
||||
Ok(match value {
|
||||
serde_json::Value::Null => py.None().into_bound(py),
|
||||
serde_json::Value::Bool(b) => b.into_pyobject(py)?.to_owned().into_any(),
|
||||
serde_json::Value::Number(n) => match (n.as_u64(), n.as_i64(), n.as_f64()) {
|
||||
(Some(u), _, _) => u.into_pyobject(py)?.into_any(),
|
||||
(_, Some(i), _) => i.into_pyobject(py)?.into_any(),
|
||||
(_, _, Some(f)) => f.into_pyobject(py)?.into_any(),
|
||||
// JSON cannot hold a number this is not one of, so this is
|
||||
// unreachable rather than a case worth inventing a value for.
|
||||
_ => return Err(PyValueError::new_err(format!("{n} is not a number"))),
|
||||
},
|
||||
serde_json::Value::String(s) => s.into_pyobject(py)?.into_any(),
|
||||
serde_json::Value::Array(items) => {
|
||||
let mut out = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
out.push(json_to_py(py, item)?);
|
||||
}
|
||||
PyList::new(py, out)?.into_any()
|
||||
}
|
||||
serde_json::Value::Object(fields) => {
|
||||
let dict = PyDict::new(py);
|
||||
for (key, value) in fields {
|
||||
dict.set_item(key, json_to_py(py, value)?)?;
|
||||
}
|
||||
dict.into_any()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// A Python dictionary back to a record, the same way round.
|
||||
fn from_py<T: serde::de::DeserializeOwned>(value: &Bound<'_, PyAny>) -> PyResult<T> {
|
||||
let json: String = value
|
||||
.py()
|
||||
.import("json")?
|
||||
.call_method1("dumps", (value,))?
|
||||
.extract()?;
|
||||
serde_json::from_str(&json)
|
||||
.map_err(|e| PyValueError::new_err(format!("not a record of the expected shape: {e}")))
|
||||
}
|
||||
|
||||
/// Every dictionary in a sequence.
|
||||
fn from_py_list<T: serde::de::DeserializeOwned>(values: &Bound<'_, PyAny>) -> PyResult<Vec<T>> {
|
||||
let mut out = Vec::new();
|
||||
for value in values.try_iter()? {
|
||||
out.push(from_py(&value?)?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Cut markdown into retrievable passages, one dictionary each.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (markdown, *, target = None, overlap = None, min_tokens = None, hard_max = None))]
|
||||
fn chunk_markdown<'py>(
|
||||
py: Python<'py>,
|
||||
markdown: &str,
|
||||
target: Option<usize>,
|
||||
overlap: Option<usize>,
|
||||
min_tokens: Option<usize>,
|
||||
hard_max: Option<usize>,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let default = ChunkBudgets::default();
|
||||
let budgets = ChunkBudgets {
|
||||
target: target.unwrap_or(default.target),
|
||||
overlap: overlap.unwrap_or(default.overlap),
|
||||
min_tokens: min_tokens.unwrap_or(default.min_tokens),
|
||||
hard_max: hard_max.unwrap_or(default.hard_max),
|
||||
};
|
||||
list_to_py(py, &crate::chunk_markdown(markdown, budgets))
|
||||
}
|
||||
|
||||
/// Clean converted markdown.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (markdown, *, pages = false))]
|
||||
fn scrub(markdown: &str, pages: bool) -> String {
|
||||
match pages {
|
||||
true => crate::scrub_markdown_pages(markdown),
|
||||
false => crate::scrub_markdown(markdown),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many tokens a string is.
|
||||
#[pyfunction]
|
||||
fn count_tokens(text: &str) -> usize {
|
||||
crate::count_tokens(text)
|
||||
}
|
||||
|
||||
/// Read any of the subtitle formats into cues.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (content, *, keep_repeats = false))]
|
||||
fn parse_subtitles<'py>(
|
||||
py: Python<'py>,
|
||||
content: &str,
|
||||
keep_repeats: bool,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let cues = crate::parse_subtitles(content);
|
||||
let cues = match keep_repeats {
|
||||
true => cues,
|
||||
false => crate::merge_repeats(&cues),
|
||||
};
|
||||
list_to_py(py, &cues)
|
||||
}
|
||||
|
||||
/// Pack cues into passages worth embedding.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (cues, *, seconds = None, max_tokens = None))]
|
||||
fn pack_cues<'py>(
|
||||
py: Python<'py>,
|
||||
cues: &Bound<'py, PyAny>,
|
||||
seconds: Option<f64>,
|
||||
max_tokens: Option<usize>,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let default = CueBudgets::default();
|
||||
let budgets = CueBudgets {
|
||||
seconds: seconds.unwrap_or(default.seconds),
|
||||
max_tokens: max_tokens.unwrap_or(default.max_tokens),
|
||||
};
|
||||
let cues: Vec<cuekit::Cue> = from_py_list(cues)?;
|
||||
list_to_py(py, &crate::pack_cues(&cues, budgets))
|
||||
}
|
||||
|
||||
/// Split a collection into the works inside it.
|
||||
///
|
||||
/// Raises `ValueError` when the document is one work rather than a
|
||||
/// collection, with the reason each candidate was rejected.
|
||||
#[pyfunction]
|
||||
fn split_collection<'py>(py: Python<'py>, markdown: &str) -> PyResult<Bound<'py, PyList>> {
|
||||
match crate::split_collection(markdown) {
|
||||
Ok(sections) => list_to_py(py, §ions),
|
||||
Err(why) => Err(PyValueError::new_err(why)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fuse ranked lists into one.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (hits, *, k = None, weights = None, limit = None))]
|
||||
fn fuse_hits<'py>(
|
||||
py: Python<'py>,
|
||||
hits: &Bound<'py, PyAny>,
|
||||
k: Option<f64>,
|
||||
weights: Option<std::collections::HashMap<String, f64>>,
|
||||
limit: Option<usize>,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let hits: Vec<corpus_core::Hit> = from_py_list(hits)?;
|
||||
let weights: Vec<(String, f64)> = weights.unwrap_or_default().into_iter().collect();
|
||||
let fused = crate::fuse_hits(&hits, k.unwrap_or(rrf::K), &weights, limit);
|
||||
list_to_py(py, &fused)
|
||||
}
|
||||
|
||||
/// Say what a file is from its bytes and its name.
|
||||
#[pyfunction]
|
||||
fn identify<'py>(py: Python<'py>, name: &str, data: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
to_py(py, &crate::identify(name, data))
|
||||
}
|
||||
|
||||
/// The module Python imports.
|
||||
#[pymodule]
|
||||
fn _catena(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
module.add("__version__", env!("CARGO_PKG_VERSION"))?;
|
||||
module.add_function(wrap_pyfunction!(chunk_markdown, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(scrub, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(count_tokens, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(parse_subtitles, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(pack_cues, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(split_collection, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(fuse_hits, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(identify, module)?)?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user