mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
workspace, licence, and the three record types
This commit is contained in:
funkpower
@@ -0,0 +1,3 @@
|
|||||||
|
target/
|
||||||
|
*.profraw
|
||||||
|
.DS_Store
|
||||||
Generated
+191
@@ -0,0 +1,191 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "corpus-core"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"schemars",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dyn-clone"
|
||||||
|
version = "1.0.20"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.107"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ref-cast"
|
||||||
|
version = "1.0.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||||
|
dependencies = [
|
||||||
|
"ref-cast-impl",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ref-cast-impl"
|
||||||
|
version = "1.0.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "schemars"
|
||||||
|
version = "1.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
||||||
|
dependencies = [
|
||||||
|
"dyn-clone",
|
||||||
|
"ref-cast",
|
||||||
|
"schemars_derive",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "schemars_derive"
|
||||||
|
version = "1.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"serde_derive_internals",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive_internals"
|
||||||
|
version = "0.30.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.151"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "3.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror"
|
||||||
|
version = "2.0.20"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
|
||||||
|
dependencies = [
|
||||||
|
"thiserror-impl",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror-impl"
|
||||||
|
version = "2.0.20"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.26"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
+25
@@ -0,0 +1,25 @@
|
|||||||
|
[workspace]
|
||||||
|
resolver = "3"
|
||||||
|
members = ["crates/*"]
|
||||||
|
|
||||||
|
[workspace.package]
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
rust-version = "1.85"
|
||||||
|
license = "MPL-2.0"
|
||||||
|
repository = "https://git.pumpin.online/funkpower/catena"
|
||||||
|
|
||||||
|
[workspace.dependencies]
|
||||||
|
corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
|
||||||
|
|
||||||
|
anyhow = "1"
|
||||||
|
clap = { version = "4", features = ["derive"] }
|
||||||
|
schemars = "1"
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_json = "1"
|
||||||
|
thiserror = "2"
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
strip = true
|
||||||
|
lto = true
|
||||||
|
codegen-units = 1
|
||||||
@@ -0,0 +1,373 @@
|
|||||||
|
Mozilla Public License Version 2.0
|
||||||
|
==================================
|
||||||
|
|
||||||
|
1. Definitions
|
||||||
|
--------------
|
||||||
|
|
||||||
|
1.1. "Contributor"
|
||||||
|
means each individual or legal entity that creates, contributes to
|
||||||
|
the creation of, or owns Covered Software.
|
||||||
|
|
||||||
|
1.2. "Contributor Version"
|
||||||
|
means the combination of the Contributions of others (if any) used
|
||||||
|
by a Contributor and that particular Contributor's Contribution.
|
||||||
|
|
||||||
|
1.3. "Contribution"
|
||||||
|
means Covered Software of a particular Contributor.
|
||||||
|
|
||||||
|
1.4. "Covered Software"
|
||||||
|
means Source Code Form to which the initial Contributor has attached
|
||||||
|
the notice in Exhibit A, the Executable Form of such Source Code
|
||||||
|
Form, and Modifications of such Source Code Form, in each case
|
||||||
|
including portions thereof.
|
||||||
|
|
||||||
|
1.5. "Incompatible With Secondary Licenses"
|
||||||
|
means
|
||||||
|
|
||||||
|
(a) that the initial Contributor has attached the notice described
|
||||||
|
in Exhibit B to the Covered Software; or
|
||||||
|
|
||||||
|
(b) that the Covered Software was made available under the terms of
|
||||||
|
version 1.1 or earlier of the License, but not also under the
|
||||||
|
terms of a Secondary License.
|
||||||
|
|
||||||
|
1.6. "Executable Form"
|
||||||
|
means any form of the work other than Source Code Form.
|
||||||
|
|
||||||
|
1.7. "Larger Work"
|
||||||
|
means a work that combines Covered Software with other material, in
|
||||||
|
a separate file or files, that is not Covered Software.
|
||||||
|
|
||||||
|
1.8. "License"
|
||||||
|
means this document.
|
||||||
|
|
||||||
|
1.9. "Licensable"
|
||||||
|
means having the right to grant, to the maximum extent possible,
|
||||||
|
whether at the time of the initial grant or subsequently, any and
|
||||||
|
all of the rights conveyed by this License.
|
||||||
|
|
||||||
|
1.10. "Modifications"
|
||||||
|
means any of the following:
|
||||||
|
|
||||||
|
(a) any file in Source Code Form that results from an addition to,
|
||||||
|
deletion from, or modification of the contents of Covered
|
||||||
|
Software; or
|
||||||
|
|
||||||
|
(b) any new file in Source Code Form that contains any Covered
|
||||||
|
Software.
|
||||||
|
|
||||||
|
1.11. "Patent Claims" of a Contributor
|
||||||
|
means any patent claim(s), including without limitation, method,
|
||||||
|
process, and apparatus claims, in any patent Licensable by such
|
||||||
|
Contributor that would be infringed, but for the grant of the
|
||||||
|
License, by the making, using, selling, offering for sale, having
|
||||||
|
made, import, or transfer of either its Contributions or its
|
||||||
|
Contributor Version.
|
||||||
|
|
||||||
|
1.12. "Secondary License"
|
||||||
|
means either the GNU General Public License, Version 2.0, the GNU
|
||||||
|
Lesser General Public License, Version 2.1, the GNU Affero General
|
||||||
|
Public License, Version 3.0, or any later versions of those
|
||||||
|
licenses.
|
||||||
|
|
||||||
|
1.13. "Source Code Form"
|
||||||
|
means the form of the work preferred for making modifications.
|
||||||
|
|
||||||
|
1.14. "You" (or "Your")
|
||||||
|
means an individual or a legal entity exercising rights under this
|
||||||
|
License. For legal entities, "You" includes any entity that
|
||||||
|
controls, is controlled by, or is under common control with You. For
|
||||||
|
purposes of this definition, "control" means (a) the power, direct
|
||||||
|
or indirect, to cause the direction or management of such entity,
|
||||||
|
whether by contract or otherwise, or (b) ownership of more than
|
||||||
|
fifty percent (50%) of the outstanding shares or beneficial
|
||||||
|
ownership of such entity.
|
||||||
|
|
||||||
|
2. License Grants and Conditions
|
||||||
|
--------------------------------
|
||||||
|
|
||||||
|
2.1. Grants
|
||||||
|
|
||||||
|
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||||
|
non-exclusive license:
|
||||||
|
|
||||||
|
(a) under intellectual property rights (other than patent or trademark)
|
||||||
|
Licensable by such Contributor to use, reproduce, make available,
|
||||||
|
modify, display, perform, distribute, and otherwise exploit its
|
||||||
|
Contributions, either on an unmodified basis, with Modifications, or
|
||||||
|
as part of a Larger Work; and
|
||||||
|
|
||||||
|
(b) under Patent Claims of such Contributor to make, use, sell, offer
|
||||||
|
for sale, have made, import, and otherwise transfer either its
|
||||||
|
Contributions or its Contributor Version.
|
||||||
|
|
||||||
|
2.2. Effective Date
|
||||||
|
|
||||||
|
The licenses granted in Section 2.1 with respect to any Contribution
|
||||||
|
become effective for each Contribution on the date the Contributor first
|
||||||
|
distributes such Contribution.
|
||||||
|
|
||||||
|
2.3. Limitations on Grant Scope
|
||||||
|
|
||||||
|
The licenses granted in this Section 2 are the only rights granted under
|
||||||
|
this License. No additional rights or licenses will be implied from the
|
||||||
|
distribution or licensing of Covered Software under this License.
|
||||||
|
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
||||||
|
Contributor:
|
||||||
|
|
||||||
|
(a) for any code that a Contributor has removed from Covered Software;
|
||||||
|
or
|
||||||
|
|
||||||
|
(b) for infringements caused by: (i) Your and any other third party's
|
||||||
|
modifications of Covered Software, or (ii) the combination of its
|
||||||
|
Contributions with other software (except as part of its Contributor
|
||||||
|
Version); or
|
||||||
|
|
||||||
|
(c) under Patent Claims infringed by Covered Software in the absence of
|
||||||
|
its Contributions.
|
||||||
|
|
||||||
|
This License does not grant any rights in the trademarks, service marks,
|
||||||
|
or logos of any Contributor (except as may be necessary to comply with
|
||||||
|
the notice requirements in Section 3.4).
|
||||||
|
|
||||||
|
2.4. Subsequent Licenses
|
||||||
|
|
||||||
|
No Contributor makes additional grants as a result of Your choice to
|
||||||
|
distribute the Covered Software under a subsequent version of this
|
||||||
|
License (see Section 10.2) or under the terms of a Secondary License (if
|
||||||
|
permitted under the terms of Section 3.3).
|
||||||
|
|
||||||
|
2.5. Representation
|
||||||
|
|
||||||
|
Each Contributor represents that the Contributor believes its
|
||||||
|
Contributions are its original creation(s) or it has sufficient rights
|
||||||
|
to grant the rights to its Contributions conveyed by this License.
|
||||||
|
|
||||||
|
2.6. Fair Use
|
||||||
|
|
||||||
|
This License is not intended to limit any rights You have under
|
||||||
|
applicable copyright doctrines of fair use, fair dealing, or other
|
||||||
|
equivalents.
|
||||||
|
|
||||||
|
2.7. Conditions
|
||||||
|
|
||||||
|
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
|
||||||
|
in Section 2.1.
|
||||||
|
|
||||||
|
3. Responsibilities
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
3.1. Distribution of Source Form
|
||||||
|
|
||||||
|
All distribution of Covered Software in Source Code Form, including any
|
||||||
|
Modifications that You create or to which You contribute, must be under
|
||||||
|
the terms of this License. You must inform recipients that the Source
|
||||||
|
Code Form of the Covered Software is governed by the terms of this
|
||||||
|
License, and how they can obtain a copy of this License. You may not
|
||||||
|
attempt to alter or restrict the recipients' rights in the Source Code
|
||||||
|
Form.
|
||||||
|
|
||||||
|
3.2. Distribution of Executable Form
|
||||||
|
|
||||||
|
If You distribute Covered Software in Executable Form then:
|
||||||
|
|
||||||
|
(a) such Covered Software must also be made available in Source Code
|
||||||
|
Form, as described in Section 3.1, and You must inform recipients of
|
||||||
|
the Executable Form how they can obtain a copy of such Source Code
|
||||||
|
Form by reasonable means in a timely manner, at a charge no more
|
||||||
|
than the cost of distribution to the recipient; and
|
||||||
|
|
||||||
|
(b) You may distribute such Executable Form under the terms of this
|
||||||
|
License, or sublicense it under different terms, provided that the
|
||||||
|
license for the Executable Form does not attempt to limit or alter
|
||||||
|
the recipients' rights in the Source Code Form under this License.
|
||||||
|
|
||||||
|
3.3. Distribution of a Larger Work
|
||||||
|
|
||||||
|
You may create and distribute a Larger Work under terms of Your choice,
|
||||||
|
provided that You also comply with the requirements of this License for
|
||||||
|
the Covered Software. If the Larger Work is a combination of Covered
|
||||||
|
Software with a work governed by one or more Secondary Licenses, and the
|
||||||
|
Covered Software is not Incompatible With Secondary Licenses, this
|
||||||
|
License permits You to additionally distribute such Covered Software
|
||||||
|
under the terms of such Secondary License(s), so that the recipient of
|
||||||
|
the Larger Work may, at their option, further distribute the Covered
|
||||||
|
Software under the terms of either this License or such Secondary
|
||||||
|
License(s).
|
||||||
|
|
||||||
|
3.4. Notices
|
||||||
|
|
||||||
|
You may not remove or alter the substance of any license notices
|
||||||
|
(including copyright notices, patent notices, disclaimers of warranty,
|
||||||
|
or limitations of liability) contained within the Source Code Form of
|
||||||
|
the Covered Software, except that You may alter any license notices to
|
||||||
|
the extent required to remedy known factual inaccuracies.
|
||||||
|
|
||||||
|
3.5. Application of Additional Terms
|
||||||
|
|
||||||
|
You may choose to offer, and to charge a fee for, warranty, support,
|
||||||
|
indemnity or liability obligations to one or more recipients of Covered
|
||||||
|
Software. However, You may do so only on Your own behalf, and not on
|
||||||
|
behalf of any Contributor. You must make it absolutely clear that any
|
||||||
|
such warranty, support, indemnity, or liability obligation is offered by
|
||||||
|
You alone, and You hereby agree to indemnify every Contributor for any
|
||||||
|
liability incurred by such Contributor as a result of warranty, support,
|
||||||
|
indemnity or liability terms You offer. You may include additional
|
||||||
|
disclaimers of warranty and limitations of liability specific to any
|
||||||
|
jurisdiction.
|
||||||
|
|
||||||
|
4. Inability to Comply Due to Statute or Regulation
|
||||||
|
---------------------------------------------------
|
||||||
|
|
||||||
|
If it is impossible for You to comply with any of the terms of this
|
||||||
|
License with respect to some or all of the Covered Software due to
|
||||||
|
statute, judicial order, or regulation then You must: (a) comply with
|
||||||
|
the terms of this License to the maximum extent possible; and (b)
|
||||||
|
describe the limitations and the code they affect. Such description must
|
||||||
|
be placed in a text file included with all distributions of the Covered
|
||||||
|
Software under this License. Except to the extent prohibited by statute
|
||||||
|
or regulation, such description must be sufficiently detailed for a
|
||||||
|
recipient of ordinary skill to be able to understand it.
|
||||||
|
|
||||||
|
5. Termination
|
||||||
|
--------------
|
||||||
|
|
||||||
|
5.1. The rights granted under this License will terminate automatically
|
||||||
|
if You fail to comply with any of its terms. However, if You become
|
||||||
|
compliant, then the rights granted under this License from a particular
|
||||||
|
Contributor are reinstated (a) provisionally, unless and until such
|
||||||
|
Contributor explicitly and finally terminates Your grants, and (b) on an
|
||||||
|
ongoing basis, if such Contributor fails to notify You of the
|
||||||
|
non-compliance by some reasonable means prior to 60 days after You have
|
||||||
|
come back into compliance. Moreover, Your grants from a particular
|
||||||
|
Contributor are reinstated on an ongoing basis if such Contributor
|
||||||
|
notifies You of the non-compliance by some reasonable means, this is the
|
||||||
|
first time You have received notice of non-compliance with this License
|
||||||
|
from such Contributor, and You become compliant prior to 30 days after
|
||||||
|
Your receipt of the notice.
|
||||||
|
|
||||||
|
5.2. If You initiate litigation against any entity by asserting a patent
|
||||||
|
infringement claim (excluding declaratory judgment actions,
|
||||||
|
counter-claims, and cross-claims) alleging that a Contributor Version
|
||||||
|
directly or indirectly infringes any patent, then the rights granted to
|
||||||
|
You by any and all Contributors for the Covered Software under Section
|
||||||
|
2.1 of this License shall terminate.
|
||||||
|
|
||||||
|
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
|
||||||
|
end user license agreements (excluding distributors and resellers) which
|
||||||
|
have been validly granted by You or Your distributors under this License
|
||||||
|
prior to termination shall survive termination.
|
||||||
|
|
||||||
|
************************************************************************
|
||||||
|
* *
|
||||||
|
* 6. Disclaimer of Warranty *
|
||||||
|
* ------------------------- *
|
||||||
|
* *
|
||||||
|
* Covered Software is provided under this License on an "as is" *
|
||||||
|
* basis, without warranty of any kind, either expressed, implied, or *
|
||||||
|
* statutory, including, without limitation, warranties that the *
|
||||||
|
* Covered Software is free of defects, merchantable, fit for a *
|
||||||
|
* particular purpose or non-infringing. The entire risk as to the *
|
||||||
|
* quality and performance of the Covered Software is with You. *
|
||||||
|
* Should any Covered Software prove defective in any respect, You *
|
||||||
|
* (not any Contributor) assume the cost of any necessary servicing, *
|
||||||
|
* repair, or correction. This disclaimer of warranty constitutes an *
|
||||||
|
* essential part of this License. No use of any Covered Software is *
|
||||||
|
* authorized under this License except under this disclaimer. *
|
||||||
|
* *
|
||||||
|
************************************************************************
|
||||||
|
|
||||||
|
************************************************************************
|
||||||
|
* *
|
||||||
|
* 7. Limitation of Liability *
|
||||||
|
* -------------------------- *
|
||||||
|
* *
|
||||||
|
* Under no circumstances and under no legal theory, whether tort *
|
||||||
|
* (including negligence), contract, or otherwise, shall any *
|
||||||
|
* Contributor, or anyone who distributes Covered Software as *
|
||||||
|
* permitted above, be liable to You for any direct, indirect, *
|
||||||
|
* special, incidental, or consequential damages of any character *
|
||||||
|
* including, without limitation, damages for lost profits, loss of *
|
||||||
|
* goodwill, work stoppage, computer failure or malfunction, or any *
|
||||||
|
* and all other commercial damages or losses, even if such party *
|
||||||
|
* shall have been informed of the possibility of such damages. This *
|
||||||
|
* limitation of liability shall not apply to liability for death or *
|
||||||
|
* personal injury resulting from such party's negligence to the *
|
||||||
|
* extent applicable law prohibits such limitation. Some *
|
||||||
|
* jurisdictions do not allow the exclusion or limitation of *
|
||||||
|
* incidental or consequential damages, so this exclusion and *
|
||||||
|
* limitation may not apply to You. *
|
||||||
|
* *
|
||||||
|
************************************************************************
|
||||||
|
|
||||||
|
8. Litigation
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Any litigation relating to this License may be brought only in the
|
||||||
|
courts of a jurisdiction where the defendant maintains its principal
|
||||||
|
place of business and such litigation shall be governed by laws of that
|
||||||
|
jurisdiction, without reference to its conflict-of-law provisions.
|
||||||
|
Nothing in this Section shall prevent a party's ability to bring
|
||||||
|
cross-claims or counter-claims.
|
||||||
|
|
||||||
|
9. Miscellaneous
|
||||||
|
----------------
|
||||||
|
|
||||||
|
This License represents the complete agreement concerning the subject
|
||||||
|
matter hereof. If any provision of this License is held to be
|
||||||
|
unenforceable, such provision shall be reformed only to the extent
|
||||||
|
necessary to make it enforceable. Any law or regulation which provides
|
||||||
|
that the language of a contract shall be construed against the drafter
|
||||||
|
shall not be used to construe this License against a Contributor.
|
||||||
|
|
||||||
|
10. Versions of the License
|
||||||
|
---------------------------
|
||||||
|
|
||||||
|
10.1. New Versions
|
||||||
|
|
||||||
|
Mozilla Foundation is the license steward. Except as provided in Section
|
||||||
|
10.3, no one other than the license steward has the right to modify or
|
||||||
|
publish new versions of this License. Each version will be given a
|
||||||
|
distinguishing version number.
|
||||||
|
|
||||||
|
10.2. Effect of New Versions
|
||||||
|
|
||||||
|
You may distribute the Covered Software under the terms of the version
|
||||||
|
of the License under which You originally received the Covered Software,
|
||||||
|
or under the terms of any subsequent version published by the license
|
||||||
|
steward.
|
||||||
|
|
||||||
|
10.3. Modified Versions
|
||||||
|
|
||||||
|
If you create software not governed by this License, and you want to
|
||||||
|
create a new license for such software, you may create and use a
|
||||||
|
modified version of this License if you rename the license and remove
|
||||||
|
any references to the name of the license steward (except to note that
|
||||||
|
such modified license differs from this License).
|
||||||
|
|
||||||
|
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
||||||
|
Licenses
|
||||||
|
|
||||||
|
If You choose to distribute Source Code Form that is Incompatible With
|
||||||
|
Secondary Licenses under the terms of this version of the License, the
|
||||||
|
notice described in Exhibit B of this License must be attached.
|
||||||
|
|
||||||
|
Exhibit A - Source Code Form License Notice
|
||||||
|
-------------------------------------------
|
||||||
|
|
||||||
|
This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
If it is not possible or desirable to put the notice in a particular
|
||||||
|
file, then You may include the notice in a location (such as a LICENSE
|
||||||
|
file in a relevant directory) where a recipient would be likely to look
|
||||||
|
for such a notice.
|
||||||
|
|
||||||
|
You may add additional accurate notices of copyright ownership.
|
||||||
|
|
||||||
|
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
||||||
|
---------------------------------------------------------
|
||||||
|
|
||||||
|
This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||||
|
defined by the Mozilla Public License, v. 2.0.
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
# catena
|
||||||
|
|
||||||
|
Small programs that each do one part of turning a pile of files into a
|
||||||
|
searchable corpus, and hand the next one text on stdout.
|
||||||
|
|
||||||
|
A catena is a commentary assembled out of excerpts drawn from many sources and
|
||||||
|
chained in order, each one carrying the name of who said it and where. That is
|
||||||
|
both what these programs produce and how they run.
|
||||||
|
|
||||||
|
Nothing here needs a database, a GPU, a queue or a config file. Each tool is a
|
||||||
|
filter: bytes in, records out, diagnostics on stderr, and an exit code that
|
||||||
|
means something.
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo install --git https://git.pumpin.online/funkpower/catena corpus-core
|
||||||
|
```
|
||||||
|
|
||||||
|
Each tool installs by name, so you can take one without taking the rest.
|
||||||
|
|
||||||
|
## The contract
|
||||||
|
|
||||||
|
Every binary in this workspace obeys all of this, or it does not ship.
|
||||||
|
|
||||||
|
1. **One job.** A tool that needs a `--mode` flag to change what it *is* is two
|
||||||
|
tools.
|
||||||
|
2. **stdin to stdout.** Input on stdin or a path argument, result on stdout,
|
||||||
|
diagnostics on stderr only — never content and chatter on the same stream.
|
||||||
|
3. **Streams, not documents.** Anything list-shaped is NDJSON: one record per
|
||||||
|
line, flushed as it is produced, so a slow stage never blocks the pipe.
|
||||||
|
4. **Predicates answer with exit codes.** `0` yes, `1` no, `2` could not tell.
|
||||||
|
Failures use the `sysexits` range — `64` bad usage, `65` bad input, `69` a
|
||||||
|
missing external program, `74` I/O — so "no" and "broke" are never the same
|
||||||
|
number.
|
||||||
|
5. **No network, no database, no config file, no state.** Two exceptions, both
|
||||||
|
declared in `--help`: a tool whose whole job is an HTTP call, and a tool
|
||||||
|
that shells out to a named external program and fails clearly when it is
|
||||||
|
absent.
|
||||||
|
6. **No ambient knowledge.** No hostnames, no local paths, no private
|
||||||
|
vocabularies compiled in. General scheme data — Bible book names, ISBN check
|
||||||
|
digits — may be; anything local is a `--rules file.json`.
|
||||||
|
7. **Deterministic.** Same bytes in, same bytes out, across runs and machines.
|
||||||
|
8. **`--json` everywhere.** The human-readable default is for a person at a
|
||||||
|
terminal; `--json` is the contract, and it does not change shape without a
|
||||||
|
major version.
|
||||||
|
|
||||||
|
## The records
|
||||||
|
|
||||||
|
Three shapes travel between the tools, defined in `corpus-core` and versioned
|
||||||
|
by a `v` field.
|
||||||
|
|
||||||
|
**`doc`** — what a file is.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"v":1,"path":"scan.pdf","kind":"pdf","declared_ext":"pdf","bytes":9261184,
|
||||||
|
"pages":412,"language":"eng","text_layer":"garbled","needs_ocr":true,"notes":[]}
|
||||||
|
```
|
||||||
|
|
||||||
|
**`chunk`** — a retrievable passage with enough provenance to cite it.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"v":1,"idx":7,"text":"…","tokens":512,"page":37,"chapter":"III. The Second Estate",
|
||||||
|
"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
```
|
||||||
|
|
||||||
|
`page` and `chapter` for documents, `start_time` and `end_time` for
|
||||||
|
recordings. One shape either way, so a single index serves a book and a
|
||||||
|
lecture and returns both with a location you can cite.
|
||||||
|
|
||||||
|
**`hit`** — one scored candidate, on its way into a fusion.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"v":1,"id":"c81a","channel":"vec","rank":1,"score":0.83,"payload":{}}
|
||||||
|
```
|
||||||
|
|
||||||
|
The JSON Schema for all three:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo run -p corpus-core -- schema all
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo test
|
||||||
|
cargo clippy --all-targets -- -D warnings
|
||||||
|
cargo fmt --check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Licence
|
||||||
|
|
||||||
|
MPL-2.0. See `LICENSE`.
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "corpus-core"
|
||||||
|
description = "Record types and stream plumbing shared by the catena tools."
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
schemars.workspace = true
|
||||||
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
|
thiserror.workspace = true
|
||||||
|
|
||||||
|
[lints.rust]
|
||||||
|
unsafe_code = "forbid"
|
||||||
|
missing_docs = "warn"
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
//! A retrievable passage, with enough provenance to cite it.
|
||||||
|
|
||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// One passage of a document or a recording.
|
||||||
|
///
|
||||||
|
/// Documents fill `page` and `chapter`; media fill `start_time` and
|
||||||
|
/// `end_time`. The shape is the same either way, so one index serves a book
|
||||||
|
/// and a lecture recording and can cite both.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct Chunk {
|
||||||
|
/// Record version.
|
||||||
|
#[serde(default = "crate::v")]
|
||||||
|
pub v: u32,
|
||||||
|
/// Position in the stream this chunk was cut from, from zero.
|
||||||
|
pub idx: u32,
|
||||||
|
/// The passage.
|
||||||
|
pub text: String,
|
||||||
|
/// Token count of `text`, by the tokeniser the chunker used.
|
||||||
|
pub tokens: u32,
|
||||||
|
/// Page the passage starts on, for paginated sources.
|
||||||
|
pub page: Option<u32>,
|
||||||
|
/// Nearest enclosing heading, as written.
|
||||||
|
pub chapter: Option<String>,
|
||||||
|
/// What part of the work the passage belongs to.
|
||||||
|
pub role: Role,
|
||||||
|
/// Seconds from the start of the recording, for timed sources.
|
||||||
|
pub start_time: Option<f64>,
|
||||||
|
/// Seconds from the start of the recording, for timed sources.
|
||||||
|
pub end_time: Option<f64>,
|
||||||
|
/// Set tags, sorted and deduplicated; see [`flags`].
|
||||||
|
#[serde(default)]
|
||||||
|
pub flags: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Chunk {
|
||||||
|
/// A body chunk with no provenance yet.
|
||||||
|
pub fn new(idx: u32, text: impl Into<String>, tokens: u32) -> Self {
|
||||||
|
Self {
|
||||||
|
v: crate::V,
|
||||||
|
idx,
|
||||||
|
text: text.into(),
|
||||||
|
tokens,
|
||||||
|
page: None,
|
||||||
|
chapter: None,
|
||||||
|
role: Role::Body,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
flags: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set a flag. Flags stay sorted and unique so the same input always
|
||||||
|
/// serialises to the same bytes.
|
||||||
|
pub fn flag(&mut self, flag: impl Into<String>) {
|
||||||
|
let flag = flag.into();
|
||||||
|
if let Err(at) = self.flags.binary_search(&flag) {
|
||||||
|
self.flags.insert(at, flag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a flag is set.
|
||||||
|
pub fn has_flag(&self, flag: &str) -> bool {
|
||||||
|
self.flags
|
||||||
|
.binary_search_by(|f| f.as_str().cmp(flag))
|
||||||
|
.is_ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The flag names the tools in this workspace set.
|
||||||
|
pub mod flags {
|
||||||
|
/// The passage's footnote markers were resolved against a citation list.
|
||||||
|
pub const FOOTNOTE_CITATIONS: &str = "footnote_citations";
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What part of a work a passage belongs to.
|
||||||
|
///
|
||||||
|
/// Retrieval wants body text; an index or a bibliography matches lexically
|
||||||
|
/// far too well for what it is worth, which is why the distinction is carried
|
||||||
|
/// on the record rather than rediscovered at query time.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum Role {
|
||||||
|
/// Running text.
|
||||||
|
#[default]
|
||||||
|
Body,
|
||||||
|
/// A works-cited or references list.
|
||||||
|
Bibliography,
|
||||||
|
/// A back-of-book index.
|
||||||
|
Index,
|
||||||
|
/// A table of contents.
|
||||||
|
Toc,
|
||||||
|
/// Title page, copyright page, dedication, and the rest of the front.
|
||||||
|
Frontmatter,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn role_spellings_are_the_wire_spellings() {
|
||||||
|
for (role, wire) in [
|
||||||
|
(Role::Body, "\"body\""),
|
||||||
|
(Role::Bibliography, "\"bibliography\""),
|
||||||
|
(Role::Index, "\"index\""),
|
||||||
|
(Role::Toc, "\"toc\""),
|
||||||
|
(Role::Frontmatter, "\"frontmatter\""),
|
||||||
|
] {
|
||||||
|
assert_eq!(serde_json::to_string(&role).unwrap(), wire);
|
||||||
|
assert_eq!(serde_json::from_str::<Role>(wire).unwrap(), role);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_field_is_emitted_even_when_empty() {
|
||||||
|
let chunk = Chunk::new(0, "hello", 1);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&chunk).unwrap(),
|
||||||
|
r#"{"v":1,"idx":0,"text":"hello","tokens":1,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}"#
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn flags_stay_sorted_and_unique() {
|
||||||
|
let mut chunk = Chunk::new(0, "x", 1);
|
||||||
|
chunk.flag("zebra");
|
||||||
|
chunk.flag(flags::FOOTNOTE_CITATIONS);
|
||||||
|
chunk.flag("zebra");
|
||||||
|
assert_eq!(chunk.flags, ["footnote_citations", "zebra"]);
|
||||||
|
assert!(chunk.has_flag("zebra"));
|
||||||
|
assert!(!chunk.has_flag("aardvark"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
//! What a file is.
|
||||||
|
|
||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// A source file, as identified by inspection rather than by its name.
|
||||||
|
///
|
||||||
|
/// Fields a given tool cannot know are `None`: a format sniffer fills `kind`
|
||||||
|
/// and leaves `text_layer` empty, a triage tool does the reverse.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct Doc {
|
||||||
|
/// Record version.
|
||||||
|
#[serde(default = "crate::v")]
|
||||||
|
pub v: u32,
|
||||||
|
/// Path the file was read from, as given.
|
||||||
|
pub path: String,
|
||||||
|
/// What the bytes say the file is.
|
||||||
|
pub kind: Kind,
|
||||||
|
/// Extension the name claimed, lowercased, without the dot.
|
||||||
|
pub declared_ext: Option<String>,
|
||||||
|
/// Size on disk.
|
||||||
|
pub bytes: u64,
|
||||||
|
/// Page count, for paginated formats.
|
||||||
|
pub pages: Option<u32>,
|
||||||
|
/// ISO 639-2/B language code, when one was detected.
|
||||||
|
pub language: Option<String>,
|
||||||
|
/// Verdict on the embedded text layer.
|
||||||
|
pub text_layer: Option<TextLayer>,
|
||||||
|
/// Whether the file has to be OCRed to be searchable.
|
||||||
|
pub needs_ocr: Option<bool>,
|
||||||
|
/// Human-readable observations, in the order they were made.
|
||||||
|
pub notes: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Doc {
|
||||||
|
/// A doc with only what inspection of the bytes always yields.
|
||||||
|
pub fn new(path: impl Into<String>, kind: Kind, bytes: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
v: crate::V,
|
||||||
|
path: path.into(),
|
||||||
|
kind,
|
||||||
|
declared_ext: None,
|
||||||
|
bytes,
|
||||||
|
pages: None,
|
||||||
|
language: None,
|
||||||
|
text_layer: None,
|
||||||
|
needs_ocr: None,
|
||||||
|
notes: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append an observation.
|
||||||
|
pub fn note(&mut self, note: impl Into<String>) {
|
||||||
|
self.notes.push(note.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The format a file actually is.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum Kind {
|
||||||
|
/// Portable Document Format.
|
||||||
|
Pdf,
|
||||||
|
/// EPUB, any version.
|
||||||
|
Epub,
|
||||||
|
/// DjVu.
|
||||||
|
Djvu,
|
||||||
|
/// Amazon KF8.
|
||||||
|
Azw3,
|
||||||
|
/// Amazon MOBI.
|
||||||
|
Mobi,
|
||||||
|
/// Plain text.
|
||||||
|
Txt,
|
||||||
|
/// HTML or XHTML.
|
||||||
|
Html,
|
||||||
|
/// An OLE2 compound document — a `.doc`, whatever the name says.
|
||||||
|
Ole2,
|
||||||
|
/// A zip archive whose only real content is a single PDF.
|
||||||
|
ZipWrappedPdf,
|
||||||
|
/// Recognised as none of the above.
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The state of a PDF's embedded text layer, and therefore whether OCR is
|
||||||
|
/// worth its cost.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum TextLayer {
|
||||||
|
/// Extractable and readable as it stands.
|
||||||
|
Healthy,
|
||||||
|
/// Present but mojibake — extraction yields characters, not words.
|
||||||
|
Garbled,
|
||||||
|
/// Present but too thin to be the whole page; a scan with a caption layer.
|
||||||
|
Sparse,
|
||||||
|
/// Present and permuted by a non-standard font encoding, recoverable
|
||||||
|
/// without OCR once the permutation is undone.
|
||||||
|
ReflectedFont,
|
||||||
|
/// No text at all; a scan.
|
||||||
|
Absent,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn kind_spellings_are_the_wire_spellings() {
|
||||||
|
let cases = [
|
||||||
|
(Kind::Pdf, "\"pdf\""),
|
||||||
|
(Kind::Ole2, "\"ole2\""),
|
||||||
|
(Kind::Azw3, "\"azw3\""),
|
||||||
|
(Kind::ZipWrappedPdf, "\"zip-wrapped-pdf\""),
|
||||||
|
(Kind::Unknown, "\"unknown\""),
|
||||||
|
];
|
||||||
|
for (kind, wire) in cases {
|
||||||
|
assert_eq!(serde_json::to_string(&kind).unwrap(), wire);
|
||||||
|
assert_eq!(serde_json::from_str::<Kind>(wire).unwrap(), kind);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn text_layer_spellings_are_the_wire_spellings() {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&TextLayer::ReflectedFont).unwrap(),
|
||||||
|
"\"reflected-font\""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_field_is_emitted_even_when_empty() {
|
||||||
|
let doc = Doc::new("/tmp/a.pdf", Kind::Pdf, 12);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&doc).unwrap(),
|
||||||
|
r#"{"v":1,"path":"/tmp/a.pdf","kind":"pdf","declared_ext":null,"bytes":12,"pages":null,"language":null,"text_layer":null,"needs_ocr":null,"notes":[]}"#
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_record_written_without_a_version_reads_as_version_one() {
|
||||||
|
let doc: Doc =
|
||||||
|
serde_json::from_str(r#"{"path":"a","kind":"txt","bytes":1,"notes":[]}"#).unwrap();
|
||||||
|
assert_eq!(doc.v, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
//! What a tool's exit code means.
|
||||||
|
//!
|
||||||
|
//! A predicate answers through its status, so it is usable in a shell `if`
|
||||||
|
//! without parsing anything. That only works if "no" and "broke" are
|
||||||
|
//! different numbers, which is why the failure codes below start where
|
||||||
|
//! `sysexits.h` puts them and leave 0, 1 and 2 to the answer.
|
||||||
|
|
||||||
|
/// Success, or a predicate answering yes.
|
||||||
|
pub const OK: i32 = 0;
|
||||||
|
/// A predicate answering no.
|
||||||
|
pub const NO: i32 = 1;
|
||||||
|
/// A predicate that could not tell — a damaged file, a format it does not
|
||||||
|
/// know. Distinct from no, because a caller may want to retry or escalate.
|
||||||
|
pub const INDETERMINATE: i32 = 2;
|
||||||
|
/// The command line was wrong.
|
||||||
|
pub const USAGE: i32 = 64;
|
||||||
|
/// The input was not what the tool can work on.
|
||||||
|
pub const DATA: i32 = 65;
|
||||||
|
/// An external program the tool needs is not on `PATH`.
|
||||||
|
pub const UNAVAILABLE: i32 = 69;
|
||||||
|
/// Reading or writing failed.
|
||||||
|
pub const IO: i32 = 74;
|
||||||
|
|
||||||
|
/// The answer a predicate tool gives.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum Verdict {
|
||||||
|
/// Yes.
|
||||||
|
Yes,
|
||||||
|
/// No.
|
||||||
|
No,
|
||||||
|
/// Could not tell.
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Verdict {
|
||||||
|
/// The process exit code for this answer.
|
||||||
|
pub const fn code(self) -> i32 {
|
||||||
|
match self {
|
||||||
|
Verdict::Yes => OK,
|
||||||
|
Verdict::No => NO,
|
||||||
|
Verdict::Unknown => INDETERMINATE,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<bool> for Verdict {
|
||||||
|
fn from(yes: bool) -> Self {
|
||||||
|
if yes { Verdict::Yes } else { Verdict::No }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_three_answers_are_zero_one_two() {
|
||||||
|
assert_eq!(Verdict::Yes.code(), 0);
|
||||||
|
assert_eq!(Verdict::No.code(), 1);
|
||||||
|
assert_eq!(Verdict::Unknown.code(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failure_codes_never_collide_with_an_answer() {
|
||||||
|
for code in [USAGE, DATA, UNAVAILABLE, IO] {
|
||||||
|
assert!(code > INDETERMINATE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
//! One scored candidate, on its way into a fusion.
|
||||||
|
|
||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
|
/// A single result from a single retrieval channel.
|
||||||
|
///
|
||||||
|
/// `rank` is the candidate's position within its own channel, from one, and
|
||||||
|
/// is what rank fusion consumes; `score` is the channel's raw number, carried
|
||||||
|
/// along because it is useful to a human reading a trace and useless for
|
||||||
|
/// comparing across channels.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct Hit {
|
||||||
|
/// Record version.
|
||||||
|
#[serde(default = "crate::v")]
|
||||||
|
pub v: u32,
|
||||||
|
/// Identifier of the thing retrieved, opaque to fusion.
|
||||||
|
pub id: String,
|
||||||
|
/// Which channel produced this hit.
|
||||||
|
pub channel: Channel,
|
||||||
|
/// Position within the channel's own ranking, from one.
|
||||||
|
pub rank: u32,
|
||||||
|
/// The channel's raw score, on whatever scale that channel uses.
|
||||||
|
pub score: f64,
|
||||||
|
/// Anything the producer wants to carry through fusion untouched.
|
||||||
|
#[serde(default)]
|
||||||
|
pub payload: Map<String, Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Hit {
|
||||||
|
/// A hit with an empty payload.
|
||||||
|
pub fn new(id: impl Into<String>, channel: Channel, rank: u32, score: f64) -> Self {
|
||||||
|
Self {
|
||||||
|
v: crate::V,
|
||||||
|
id: id.into(),
|
||||||
|
channel,
|
||||||
|
rank,
|
||||||
|
score,
|
||||||
|
payload: Map::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A retrieval channel.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum Channel {
|
||||||
|
/// Nearest neighbours in an embedding space.
|
||||||
|
#[serde(rename = "vec")]
|
||||||
|
Vector,
|
||||||
|
/// Full-text search.
|
||||||
|
Fts,
|
||||||
|
/// A match on metadata — title, author, tag.
|
||||||
|
Meta,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn channel_spellings_are_the_wire_spellings() {
|
||||||
|
for (channel, wire) in [
|
||||||
|
(Channel::Vector, "\"vec\""),
|
||||||
|
(Channel::Fts, "\"fts\""),
|
||||||
|
(Channel::Meta, "\"meta\""),
|
||||||
|
] {
|
||||||
|
assert_eq!(serde_json::to_string(&channel).unwrap(), wire);
|
||||||
|
assert_eq!(serde_json::from_str::<Channel>(wire).unwrap(), channel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_field_is_emitted_even_when_empty() {
|
||||||
|
let hit = Hit::new("abc", Channel::Fts, 1, 0.5);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&hit).unwrap(),
|
||||||
|
r#"{"v":1,"id":"abc","channel":"fts","rank":1,"score":0.5,"payload":{}}"#
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn payload_keys_serialise_in_a_fixed_order() {
|
||||||
|
let mut hit = Hit::new("abc", Channel::Vector, 2, 0.25);
|
||||||
|
hit.payload.insert("zulu".into(), Value::from(1));
|
||||||
|
hit.payload.insert("alpha".into(), Value::from(2));
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&hit).unwrap(),
|
||||||
|
r#"{"v":1,"id":"abc","channel":"vec","rank":2,"score":0.25,"payload":{"alpha":2,"zulu":1}}"#
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
//! Record types and stream plumbing shared by every catena tool.
|
||||||
|
//!
|
||||||
|
//! Three records travel between the tools: a [`Doc`] is what a file is, a
|
||||||
|
//! [`Chunk`] is a retrievable passage with enough provenance to cite it, and a
|
||||||
|
//! [`Hit`] is one scored candidate on its way into a fusion. Each carries a
|
||||||
|
//! version field so a reader can refuse a stream it does not understand.
|
||||||
|
//!
|
||||||
|
//! Every list-shaped stream is NDJSON — see [`ndjson`] — and every predicate
|
||||||
|
//! answers through an exit code — see [`exit`].
|
||||||
|
|
||||||
|
pub mod chunk;
|
||||||
|
pub mod doc;
|
||||||
|
pub mod exit;
|
||||||
|
pub mod hit;
|
||||||
|
pub mod ndjson;
|
||||||
|
|
||||||
|
pub use chunk::{Chunk, Role};
|
||||||
|
pub use doc::{Doc, Kind, TextLayer};
|
||||||
|
pub use hit::{Channel, Hit};
|
||||||
|
pub use ndjson::{Error, read, write_line};
|
||||||
|
|
||||||
|
/// The record version these types serialise as.
|
||||||
|
pub const V: u32 = 1;
|
||||||
|
|
||||||
|
/// Serde default for the `v` field, so a record written by an older tool that
|
||||||
|
/// omitted it still reads back as version 1 rather than 0.
|
||||||
|
pub(crate) const fn v() -> u32 {
|
||||||
|
V
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
//! Prints the JSON Schema for the record types, so a consumer in another
|
||||||
|
//! language can generate its own bindings instead of reading Rust.
|
||||||
|
|
||||||
|
use std::process::ExitCode;
|
||||||
|
|
||||||
|
use corpus_core::{Chunk, Doc, Hit, exit};
|
||||||
|
|
||||||
|
const USAGE: &str = "\
|
||||||
|
usage: corpus-core schema [doc|chunk|hit|all]
|
||||||
|
|
||||||
|
Prints the JSON Schema for one record type, or an object of all three.
|
||||||
|
|
||||||
|
cargo run -p corpus-core -- schema chunk > chunk.schema.json
|
||||||
|
";
|
||||||
|
|
||||||
|
fn main() -> ExitCode {
|
||||||
|
let mut args = std::env::args().skip(1);
|
||||||
|
match args.next().as_deref() {
|
||||||
|
Some("schema") => {
|
||||||
|
let which = args.next().unwrap_or_else(|| "all".into());
|
||||||
|
match schema(&which) {
|
||||||
|
Some(json) => {
|
||||||
|
println!("{json}");
|
||||||
|
ExitCode::from(exit::OK as u8)
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
eprintln!("corpus-core: unknown record type {which:?}\n\n{USAGE}");
|
||||||
|
ExitCode::from(exit::USAGE as u8)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some("--help" | "-h") => {
|
||||||
|
println!("{USAGE}");
|
||||||
|
ExitCode::from(exit::OK as u8)
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
eprintln!("{USAGE}");
|
||||||
|
ExitCode::from(exit::USAGE as u8)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn schema(which: &str) -> Option<String> {
|
||||||
|
let json = match which {
|
||||||
|
"doc" => serde_json::to_string_pretty(&schemars::schema_for!(Doc)),
|
||||||
|
"chunk" => serde_json::to_string_pretty(&schemars::schema_for!(Chunk)),
|
||||||
|
"hit" => serde_json::to_string_pretty(&schemars::schema_for!(Hit)),
|
||||||
|
"all" => serde_json::to_string_pretty(&serde_json::json!({
|
||||||
|
"doc": schemars::schema_for!(Doc),
|
||||||
|
"chunk": schemars::schema_for!(Chunk),
|
||||||
|
"hit": schemars::schema_for!(Hit),
|
||||||
|
})),
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
Some(json.expect("a schema is always serialisable"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
//! Newline-delimited JSON, read lazily and written flushed.
|
||||||
|
//!
|
||||||
|
//! Every list-shaped stream between these tools is NDJSON: one record per
|
||||||
|
//! line, produced and consumed one at a time, so a slow stage never has to
|
||||||
|
//! hold the whole list and never blocks the pipe behind it.
|
||||||
|
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
|
||||||
|
use serde::{Serialize, de::DeserializeOwned};
|
||||||
|
|
||||||
|
/// Something went wrong reading or writing a stream.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum Error {
|
||||||
|
/// A line was not the record the reader expected.
|
||||||
|
#[error("line {line}: {source}")]
|
||||||
|
Parse {
|
||||||
|
/// One-based line number within the stream, counting blank lines.
|
||||||
|
line: u64,
|
||||||
|
/// What serde made of it.
|
||||||
|
source: serde_json::Error,
|
||||||
|
},
|
||||||
|
/// A record could not be turned into JSON at all.
|
||||||
|
#[error("encoding a record: {0}")]
|
||||||
|
Encode(#[source] serde_json::Error),
|
||||||
|
/// The stream itself failed.
|
||||||
|
#[error("{0}")]
|
||||||
|
Io(#[from] std::io::Error),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a stream of records, one per line.
|
||||||
|
///
|
||||||
|
/// Blank lines are skipped, so a stream that was concatenated or hand-edited
|
||||||
|
/// still reads. The iterator stops at the first error rather than skipping
|
||||||
|
/// it: a malformed line means the producer and the consumer disagree about
|
||||||
|
/// the contract, and continuing would bury that.
|
||||||
|
pub fn read<T, R>(reader: R) -> impl Iterator<Item = Result<T, Error>>
|
||||||
|
where
|
||||||
|
T: DeserializeOwned,
|
||||||
|
R: BufRead,
|
||||||
|
{
|
||||||
|
let mut line_no = 0u64;
|
||||||
|
reader.lines().filter_map(move |line| {
|
||||||
|
line_no += 1;
|
||||||
|
match line {
|
||||||
|
Err(e) => Some(Err(Error::Io(e))),
|
||||||
|
Ok(line) if line.trim().is_empty() => None,
|
||||||
|
Ok(line) => Some(serde_json::from_str(&line).map_err(|source| Error::Parse {
|
||||||
|
line: line_no,
|
||||||
|
source,
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write one record and flush it.
|
||||||
|
///
|
||||||
|
/// The flush is the point: a consumer downstream should see a record as soon
|
||||||
|
/// as it exists, not when the producer's buffer happens to fill.
|
||||||
|
pub fn write_line<T, W>(writer: &mut W, record: &T) -> Result<(), Error>
|
||||||
|
where
|
||||||
|
T: Serialize,
|
||||||
|
W: Write,
|
||||||
|
{
|
||||||
|
serde_json::to_writer(&mut *writer, record).map_err(Error::Encode)?;
|
||||||
|
writer.write_all(b"\n")?;
|
||||||
|
writer.flush()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::{Channel, Hit};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reads_a_record_per_line_and_skips_blanks() {
|
||||||
|
let input = "{\"id\":\"a\",\"channel\":\"fts\",\"rank\":1,\"score\":1.0}\n\n\
|
||||||
|
{\"id\":\"b\",\"channel\":\"vec\",\"rank\":2,\"score\":0.5}\n";
|
||||||
|
let hits: Vec<Hit> = read(input.as_bytes()).collect::<Result<_, _>>().unwrap();
|
||||||
|
assert_eq!(hits.len(), 2);
|
||||||
|
assert_eq!(hits[1].id, "b");
|
||||||
|
assert_eq!(hits[1].channel, Channel::Vector);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_bad_line_reports_its_line_number_and_stops_the_stream() {
|
||||||
|
let input = "{\"id\":\"a\",\"channel\":\"fts\",\"rank\":1,\"score\":1.0}\nnot json\n";
|
||||||
|
let mut stream = read::<Hit, _>(input.as_bytes());
|
||||||
|
assert!(stream.next().unwrap().is_ok());
|
||||||
|
match stream.next().unwrap() {
|
||||||
|
Err(Error::Parse { line, .. }) => assert_eq!(line, 2),
|
||||||
|
other => panic!("expected a parse error, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn writes_one_compact_line_per_record() {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
write_line(&mut out, &Hit::new("a", Channel::Meta, 1, 0.25)).unwrap();
|
||||||
|
write_line(&mut out, &Hit::new("b", Channel::Meta, 2, 0.5)).unwrap();
|
||||||
|
let text = String::from_utf8(out).unwrap();
|
||||||
|
assert_eq!(text.lines().count(), 2);
|
||||||
|
assert!(text.ends_with('\n'));
|
||||||
|
assert!(!text.contains("\n "));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn what_is_written_reads_back_identical() {
|
||||||
|
let hit = Hit::new("a", Channel::Vector, 3, 0.125);
|
||||||
|
let mut out = Vec::new();
|
||||||
|
write_line(&mut out, &hit).unwrap();
|
||||||
|
let back: Vec<Hit> = read(out.as_slice()).collect::<Result<_, _>>().unwrap();
|
||||||
|
assert_eq!(back, vec![hit]);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user