mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
workspace, licence, and the three record types
This commit is contained in:
funkpower
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
*.profraw
|
||||
.DS_Store
|
||||
Generated
+191
@@ -0,0 +1,191 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MPL-2.0"
|
||||
repository = "https://git.pumpin.online/funkpower/catena"
|
||||
|
||||
[workspace.dependencies]
|
||||
corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
|
||||
|
||||
anyhow = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
schemars = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
thiserror = "2"
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
@@ -0,0 +1,373 @@
|
||||
Mozilla Public License Version 2.0
|
||||
==================================
|
||||
|
||||
1. Definitions
|
||||
--------------
|
||||
|
||||
1.1. "Contributor"
|
||||
means each individual or legal entity that creates, contributes to
|
||||
the creation of, or owns Covered Software.
|
||||
|
||||
1.2. "Contributor Version"
|
||||
means the combination of the Contributions of others (if any) used
|
||||
by a Contributor and that particular Contributor's Contribution.
|
||||
|
||||
1.3. "Contribution"
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. "Covered Software"
|
||||
means Source Code Form to which the initial Contributor has attached
|
||||
the notice in Exhibit A, the Executable Form of such Source Code
|
||||
Form, and Modifications of such Source Code Form, in each case
|
||||
including portions thereof.
|
||||
|
||||
1.5. "Incompatible With Secondary Licenses"
|
||||
means
|
||||
|
||||
(a) that the initial Contributor has attached the notice described
|
||||
in Exhibit B to the Covered Software; or
|
||||
|
||||
(b) that the Covered Software was made available under the terms of
|
||||
version 1.1 or earlier of the License, but not also under the
|
||||
terms of a Secondary License.
|
||||
|
||||
1.6. "Executable Form"
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. "Larger Work"
|
||||
means a work that combines Covered Software with other material, in
|
||||
a separate file or files, that is not Covered Software.
|
||||
|
||||
1.8. "License"
|
||||
means this document.
|
||||
|
||||
1.9. "Licensable"
|
||||
means having the right to grant, to the maximum extent possible,
|
||||
whether at the time of the initial grant or subsequently, any and
|
||||
all of the rights conveyed by this License.
|
||||
|
||||
1.10. "Modifications"
|
||||
means any of the following:
|
||||
|
||||
(a) any file in Source Code Form that results from an addition to,
|
||||
deletion from, or modification of the contents of Covered
|
||||
Software; or
|
||||
|
||||
(b) any new file in Source Code Form that contains any Covered
|
||||
Software.
|
||||
|
||||
1.11. "Patent Claims" of a Contributor
|
||||
means any patent claim(s), including without limitation, method,
|
||||
process, and apparatus claims, in any patent Licensable by such
|
||||
Contributor that would be infringed, but for the grant of the
|
||||
License, by the making, using, selling, offering for sale, having
|
||||
made, import, or transfer of either its Contributions or its
|
||||
Contributor Version.
|
||||
|
||||
1.12. "Secondary License"
|
||||
means either the GNU General Public License, Version 2.0, the GNU
|
||||
Lesser General Public License, Version 2.1, the GNU Affero General
|
||||
Public License, Version 3.0, or any later versions of those
|
||||
licenses.
|
||||
|
||||
1.13. "Source Code Form"
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. "You" (or "Your")
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, "You" includes any entity that
|
||||
controls, is controlled by, or is under common control with You. For
|
||||
purposes of this definition, "control" means (a) the power, direct
|
||||
or indirect, to cause the direction or management of such entity,
|
||||
whether by contract or otherwise, or (b) ownership of more than
|
||||
fifty percent (50%) of the outstanding shares or beneficial
|
||||
ownership of such entity.
|
||||
|
||||
2. License Grants and Conditions
|
||||
--------------------------------
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
(a) under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or
|
||||
as part of a Larger Work; and
|
||||
|
||||
(b) under Patent Claims of such Contributor to make, use, sell, offer
|
||||
for sale, have made, import, and otherwise transfer either its
|
||||
Contributions or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution
|
||||
become effective for each Contribution on the date the Contributor first
|
||||
distributes such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under
|
||||
this License. No additional rights or licenses will be implied from the
|
||||
distribution or licensing of Covered Software under this License.
|
||||
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
||||
Contributor:
|
||||
|
||||
(a) for any code that a Contributor has removed from Covered Software;
|
||||
or
|
||||
|
||||
(b) for infringements caused by: (i) Your and any other third party's
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
(c) under Patent Claims infringed by Covered Software in the absence of
|
||||
its Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks,
|
||||
or logos of any Contributor (except as may be necessary to comply with
|
||||
the notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this
|
||||
License (see Section 10.2) or under the terms of a Secondary License (if
|
||||
permitted under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its
|
||||
Contributions are its original creation(s) or it has sufficient rights
|
||||
to grant the rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under
|
||||
applicable copyright doctrines of fair use, fair dealing, or other
|
||||
equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
|
||||
in Section 2.1.
|
||||
|
||||
3. Responsibilities
|
||||
-------------------
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under
|
||||
the terms of this License. You must inform recipients that the Source
|
||||
Code Form of the Covered Software is governed by the terms of this
|
||||
License, and how they can obtain a copy of this License. You may not
|
||||
attempt to alter or restrict the recipients' rights in the Source Code
|
||||
Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
(a) such Covered Software must also be made available in Source Code
|
||||
Form, as described in Section 3.1, and You must inform recipients of
|
||||
the Executable Form how they can obtain a copy of such Source Code
|
||||
Form by reasonable means in a timely manner, at a charge no more
|
||||
than the cost of distribution to the recipient; and
|
||||
|
||||
(b) You may distribute such Executable Form under the terms of this
|
||||
License, or sublicense it under different terms, provided that the
|
||||
license for the Executable Form does not attempt to limit or alter
|
||||
the recipients' rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for
|
||||
the Covered Software. If the Larger Work is a combination of Covered
|
||||
Software with a work governed by one or more Secondary Licenses, and the
|
||||
Covered Software is not Incompatible With Secondary Licenses, this
|
||||
License permits You to additionally distribute such Covered Software
|
||||
under the terms of such Secondary License(s), so that the recipient of
|
||||
the Larger Work may, at their option, further distribute the Covered
|
||||
Software under the terms of either this License or such Secondary
|
||||
License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices
|
||||
(including copyright notices, patent notices, disclaimers of warranty,
|
||||
or limitations of liability) contained within the Source Code Form of
|
||||
the Covered Software, except that You may alter any license notices to
|
||||
the extent required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on
|
||||
behalf of any Contributor. You must make it absolutely clear that any
|
||||
such warranty, support, indemnity, or liability obligation is offered by
|
||||
You alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
---------------------------------------------------
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this
|
||||
License with respect to some or all of the Covered Software due to
|
||||
statute, judicial order, or regulation then You must: (a) comply with
|
||||
the terms of this License to the maximum extent possible; and (b)
|
||||
describe the limitations and the code they affect. Such description must
|
||||
be placed in a text file included with all distributions of the Covered
|
||||
Software under this License. Except to the extent prohibited by statute
|
||||
or regulation, such description must be sufficiently detailed for a
|
||||
recipient of ordinary skill to be able to understand it.
|
||||
|
||||
5. Termination
|
||||
--------------
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically
|
||||
if You fail to comply with any of its terms. However, if You become
|
||||
compliant, then the rights granted under this License from a particular
|
||||
Contributor are reinstated (a) provisionally, unless and until such
|
||||
Contributor explicitly and finally terminates Your grants, and (b) on an
|
||||
ongoing basis, if such Contributor fails to notify You of the
|
||||
non-compliance by some reasonable means prior to 60 days after You have
|
||||
come back into compliance. Moreover, Your grants from a particular
|
||||
Contributor are reinstated on an ongoing basis if such Contributor
|
||||
notifies You of the non-compliance by some reasonable means, this is the
|
||||
first time You have received notice of non-compliance with this License
|
||||
from such Contributor, and You become compliant prior to 30 days after
|
||||
Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions,
|
||||
counter-claims, and cross-claims) alleging that a Contributor Version
|
||||
directly or indirectly infringes any patent, then the rights granted to
|
||||
You by any and all Contributors for the Covered Software under Section
|
||||
2.1 of this License shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
|
||||
end user license agreements (excluding distributors and resellers) which
|
||||
have been validly granted by You or Your distributors under this License
|
||||
prior to termination shall survive termination.
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 6. Disclaimer of Warranty *
|
||||
* ------------------------- *
|
||||
* *
|
||||
* Covered Software is provided under this License on an "as is" *
|
||||
* basis, without warranty of any kind, either expressed, implied, or *
|
||||
* statutory, including, without limitation, warranties that the *
|
||||
* Covered Software is free of defects, merchantable, fit for a *
|
||||
* particular purpose or non-infringing. The entire risk as to the *
|
||||
* quality and performance of the Covered Software is with You. *
|
||||
* Should any Covered Software prove defective in any respect, You *
|
||||
* (not any Contributor) assume the cost of any necessary servicing, *
|
||||
* repair, or correction. This disclaimer of warranty constitutes an *
|
||||
* essential part of this License. No use of any Covered Software is *
|
||||
* authorized under this License except under this disclaimer. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 7. Limitation of Liability *
|
||||
* -------------------------- *
|
||||
* *
|
||||
* Under no circumstances and under no legal theory, whether tort *
|
||||
* (including negligence), contract, or otherwise, shall any *
|
||||
* Contributor, or anyone who distributes Covered Software as *
|
||||
* permitted above, be liable to You for any direct, indirect, *
|
||||
* special, incidental, or consequential damages of any character *
|
||||
* including, without limitation, damages for lost profits, loss of *
|
||||
* goodwill, work stoppage, computer failure or malfunction, or any *
|
||||
* and all other commercial damages or losses, even if such party *
|
||||
* shall have been informed of the possibility of such damages. This *
|
||||
* limitation of liability shall not apply to liability for death or *
|
||||
* personal injury resulting from such party's negligence to the *
|
||||
* extent applicable law prohibits such limitation. Some *
|
||||
* jurisdictions do not allow the exclusion or limitation of *
|
||||
* incidental or consequential damages, so this exclusion and *
|
||||
* limitation may not apply to You. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
8. Litigation
|
||||
-------------
|
||||
|
||||
Any litigation relating to this License may be brought only in the
|
||||
courts of a jurisdiction where the defendant maintains its principal
|
||||
place of business and such litigation shall be governed by laws of that
|
||||
jurisdiction, without reference to its conflict-of-law provisions.
|
||||
Nothing in this Section shall prevent a party's ability to bring
|
||||
cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
----------------
|
||||
|
||||
This License represents the complete agreement concerning the subject
|
||||
matter hereof. If any provision of this License is held to be
|
||||
unenforceable, such provision shall be reformed only to the extent
|
||||
necessary to make it enforceable. Any law or regulation which provides
|
||||
that the language of a contract shall be construed against the drafter
|
||||
shall not be used to construe this License against a Contributor.
|
||||
|
||||
10. Versions of the License
|
||||
---------------------------
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version
|
||||
of the License under which You originally received the Covered Software,
|
||||
or under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a
|
||||
modified version of this License if you rename the license and remove
|
||||
any references to the name of the license steward (except to note that
|
||||
such modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
||||
Licenses
|
||||
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
-------------------------------------------
|
||||
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular
|
||||
file, then You may include the notice in a location (such as a LICENSE
|
||||
file in a relevant directory) where a recipient would be likely to look
|
||||
for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
||||
---------------------------------------------------------
|
||||
|
||||
This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||
defined by the Mozilla Public License, v. 2.0.
|
||||
@@ -0,0 +1,93 @@
|
||||
# catena
|
||||
|
||||
Small programs that each do one part of turning a pile of files into a
|
||||
searchable corpus, and hand the next one text on stdout.
|
||||
|
||||
A catena is a commentary assembled out of excerpts drawn from many sources and
|
||||
chained in order, each one carrying the name of who said it and where. That is
|
||||
both what these programs produce and how they run.
|
||||
|
||||
Nothing here needs a database, a GPU, a queue or a config file. Each tool is a
|
||||
filter: bytes in, records out, diagnostics on stderr, and an exit code that
|
||||
means something.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
cargo install --git https://git.pumpin.online/funkpower/catena corpus-core
|
||||
```
|
||||
|
||||
Each tool installs by name, so you can take one without taking the rest.
|
||||
|
||||
## The contract
|
||||
|
||||
Every binary in this workspace obeys all of this, or it does not ship.
|
||||
|
||||
1. **One job.** A tool that needs a `--mode` flag to change what it *is* is two
|
||||
tools.
|
||||
2. **stdin to stdout.** Input on stdin or a path argument, result on stdout,
|
||||
diagnostics on stderr only — never content and chatter on the same stream.
|
||||
3. **Streams, not documents.** Anything list-shaped is NDJSON: one record per
|
||||
line, flushed as it is produced, so a slow stage never blocks the pipe.
|
||||
4. **Predicates answer with exit codes.** `0` yes, `1` no, `2` could not tell.
|
||||
Failures use the `sysexits` range — `64` bad usage, `65` bad input, `69` a
|
||||
missing external program, `74` I/O — so "no" and "broke" are never the same
|
||||
number.
|
||||
5. **No network, no database, no config file, no state.** Two exceptions, both
|
||||
declared in `--help`: a tool whose whole job is an HTTP call, and a tool
|
||||
that shells out to a named external program and fails clearly when it is
|
||||
absent.
|
||||
6. **No ambient knowledge.** No hostnames, no local paths, no private
|
||||
vocabularies compiled in. General scheme data — Bible book names, ISBN check
|
||||
digits — may be; anything local is a `--rules file.json`.
|
||||
7. **Deterministic.** Same bytes in, same bytes out, across runs and machines.
|
||||
8. **`--json` everywhere.** The human-readable default is for a person at a
|
||||
terminal; `--json` is the contract, and it does not change shape without a
|
||||
major version.
|
||||
|
||||
## The records
|
||||
|
||||
Three shapes travel between the tools, defined in `corpus-core` and versioned
|
||||
by a `v` field.
|
||||
|
||||
**`doc`** — what a file is.
|
||||
|
||||
```json
|
||||
{"v":1,"path":"scan.pdf","kind":"pdf","declared_ext":"pdf","bytes":9261184,
|
||||
"pages":412,"language":"eng","text_layer":"garbled","needs_ocr":true,"notes":[]}
|
||||
```
|
||||
|
||||
**`chunk`** — a retrievable passage with enough provenance to cite it.
|
||||
|
||||
```json
|
||||
{"v":1,"idx":7,"text":"…","tokens":512,"page":37,"chapter":"III. The Second Estate",
|
||||
"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||
```
|
||||
|
||||
`page` and `chapter` for documents, `start_time` and `end_time` for
|
||||
recordings. One shape either way, so a single index serves a book and a
|
||||
lecture and returns both with a location you can cite.
|
||||
|
||||
**`hit`** — one scored candidate, on its way into a fusion.
|
||||
|
||||
```json
|
||||
{"v":1,"id":"c81a","channel":"vec","rank":1,"score":0.83,"payload":{}}
|
||||
```
|
||||
|
||||
The JSON Schema for all three:
|
||||
|
||||
```sh
|
||||
cargo run -p corpus-core -- schema all
|
||||
```
|
||||
|
||||
## Build
|
||||
|
||||
```sh
|
||||
cargo test
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
cargo fmt --check
|
||||
```
|
||||
|
||||
## Licence
|
||||
|
||||
MPL-2.0. See `LICENSE`.
|
||||
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "corpus-core"
|
||||
description = "Record types and stream plumbing shared by the catena tools."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
schemars.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
missing_docs = "warn"
|
||||
@@ -0,0 +1,135 @@
|
||||
//! A retrievable passage, with enough provenance to cite it.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// One passage of a document or a recording.
|
||||
///
|
||||
/// Documents fill `page` and `chapter`; media fill `start_time` and
|
||||
/// `end_time`. The shape is the same either way, so one index serves a book
|
||||
/// and a lecture recording and can cite both.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Chunk {
|
||||
/// Record version.
|
||||
#[serde(default = "crate::v")]
|
||||
pub v: u32,
|
||||
/// Position in the stream this chunk was cut from, from zero.
|
||||
pub idx: u32,
|
||||
/// The passage.
|
||||
pub text: String,
|
||||
/// Token count of `text`, by the tokeniser the chunker used.
|
||||
pub tokens: u32,
|
||||
/// Page the passage starts on, for paginated sources.
|
||||
pub page: Option<u32>,
|
||||
/// Nearest enclosing heading, as written.
|
||||
pub chapter: Option<String>,
|
||||
/// What part of the work the passage belongs to.
|
||||
pub role: Role,
|
||||
/// Seconds from the start of the recording, for timed sources.
|
||||
pub start_time: Option<f64>,
|
||||
/// Seconds from the start of the recording, for timed sources.
|
||||
pub end_time: Option<f64>,
|
||||
/// Set tags, sorted and deduplicated; see [`flags`].
|
||||
#[serde(default)]
|
||||
pub flags: Vec<String>,
|
||||
}
|
||||
|
||||
impl Chunk {
|
||||
/// A body chunk with no provenance yet.
|
||||
pub fn new(idx: u32, text: impl Into<String>, tokens: u32) -> Self {
|
||||
Self {
|
||||
v: crate::V,
|
||||
idx,
|
||||
text: text.into(),
|
||||
tokens,
|
||||
page: None,
|
||||
chapter: None,
|
||||
role: Role::Body,
|
||||
start_time: None,
|
||||
end_time: None,
|
||||
flags: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a flag. Flags stay sorted and unique so the same input always
|
||||
/// serialises to the same bytes.
|
||||
pub fn flag(&mut self, flag: impl Into<String>) {
|
||||
let flag = flag.into();
|
||||
if let Err(at) = self.flags.binary_search(&flag) {
|
||||
self.flags.insert(at, flag);
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a flag is set.
|
||||
pub fn has_flag(&self, flag: &str) -> bool {
|
||||
self.flags
|
||||
.binary_search_by(|f| f.as_str().cmp(flag))
|
||||
.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// The flag names the tools in this workspace set.
|
||||
pub mod flags {
|
||||
/// The passage's footnote markers were resolved against a citation list.
|
||||
pub const FOOTNOTE_CITATIONS: &str = "footnote_citations";
|
||||
}
|
||||
|
||||
/// What part of a work a passage belongs to.
|
||||
///
|
||||
/// Retrieval wants body text; an index or a bibliography matches lexically
|
||||
/// far too well for what it is worth, which is why the distinction is carried
|
||||
/// on the record rather than rediscovered at query time.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum Role {
|
||||
/// Running text.
|
||||
#[default]
|
||||
Body,
|
||||
/// A works-cited or references list.
|
||||
Bibliography,
|
||||
/// A back-of-book index.
|
||||
Index,
|
||||
/// A table of contents.
|
||||
Toc,
|
||||
/// Title page, copyright page, dedication, and the rest of the front.
|
||||
Frontmatter,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn role_spellings_are_the_wire_spellings() {
|
||||
for (role, wire) in [
|
||||
(Role::Body, "\"body\""),
|
||||
(Role::Bibliography, "\"bibliography\""),
|
||||
(Role::Index, "\"index\""),
|
||||
(Role::Toc, "\"toc\""),
|
||||
(Role::Frontmatter, "\"frontmatter\""),
|
||||
] {
|
||||
assert_eq!(serde_json::to_string(&role).unwrap(), wire);
|
||||
assert_eq!(serde_json::from_str::<Role>(wire).unwrap(), role);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_field_is_emitted_even_when_empty() {
|
||||
let chunk = Chunk::new(0, "hello", 1);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&chunk).unwrap(),
|
||||
r#"{"v":1,"idx":0,"text":"hello","tokens":1,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}"#
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_stay_sorted_and_unique() {
|
||||
let mut chunk = Chunk::new(0, "x", 1);
|
||||
chunk.flag("zebra");
|
||||
chunk.flag(flags::FOOTNOTE_CITATIONS);
|
||||
chunk.flag("zebra");
|
||||
assert_eq!(chunk.flags, ["footnote_citations", "zebra"]);
|
||||
assert!(chunk.has_flag("zebra"));
|
||||
assert!(!chunk.has_flag("aardvark"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! What a file is.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A source file, as identified by inspection rather than by its name.
|
||||
///
|
||||
/// Fields a given tool cannot know are `None`: a format sniffer fills `kind`
|
||||
/// and leaves `text_layer` empty, a triage tool does the reverse.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Doc {
|
||||
/// Record version.
|
||||
#[serde(default = "crate::v")]
|
||||
pub v: u32,
|
||||
/// Path the file was read from, as given.
|
||||
pub path: String,
|
||||
/// What the bytes say the file is.
|
||||
pub kind: Kind,
|
||||
/// Extension the name claimed, lowercased, without the dot.
|
||||
pub declared_ext: Option<String>,
|
||||
/// Size on disk.
|
||||
pub bytes: u64,
|
||||
/// Page count, for paginated formats.
|
||||
pub pages: Option<u32>,
|
||||
/// ISO 639-2/B language code, when one was detected.
|
||||
pub language: Option<String>,
|
||||
/// Verdict on the embedded text layer.
|
||||
pub text_layer: Option<TextLayer>,
|
||||
/// Whether the file has to be OCRed to be searchable.
|
||||
pub needs_ocr: Option<bool>,
|
||||
/// Human-readable observations, in the order they were made.
|
||||
pub notes: Vec<String>,
|
||||
}
|
||||
|
||||
impl Doc {
|
||||
/// A doc with only what inspection of the bytes always yields.
|
||||
pub fn new(path: impl Into<String>, kind: Kind, bytes: u64) -> Self {
|
||||
Self {
|
||||
v: crate::V,
|
||||
path: path.into(),
|
||||
kind,
|
||||
declared_ext: None,
|
||||
bytes,
|
||||
pages: None,
|
||||
language: None,
|
||||
text_layer: None,
|
||||
needs_ocr: None,
|
||||
notes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Append an observation.
|
||||
pub fn note(&mut self, note: impl Into<String>) {
|
||||
self.notes.push(note.into());
|
||||
}
|
||||
}
|
||||
|
||||
/// The format a file actually is.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum Kind {
|
||||
/// Portable Document Format.
|
||||
Pdf,
|
||||
/// EPUB, any version.
|
||||
Epub,
|
||||
/// DjVu.
|
||||
Djvu,
|
||||
/// Amazon KF8.
|
||||
Azw3,
|
||||
/// Amazon MOBI.
|
||||
Mobi,
|
||||
/// Plain text.
|
||||
Txt,
|
||||
/// HTML or XHTML.
|
||||
Html,
|
||||
/// An OLE2 compound document — a `.doc`, whatever the name says.
|
||||
Ole2,
|
||||
/// A zip archive whose only real content is a single PDF.
|
||||
ZipWrappedPdf,
|
||||
/// Recognised as none of the above.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// The state of a PDF's embedded text layer, and therefore whether OCR is
|
||||
/// worth its cost.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum TextLayer {
|
||||
/// Extractable and readable as it stands.
|
||||
Healthy,
|
||||
/// Present but mojibake — extraction yields characters, not words.
|
||||
Garbled,
|
||||
/// Present but too thin to be the whole page; a scan with a caption layer.
|
||||
Sparse,
|
||||
/// Present and permuted by a non-standard font encoding, recoverable
|
||||
/// without OCR once the permutation is undone.
|
||||
ReflectedFont,
|
||||
/// No text at all; a scan.
|
||||
Absent,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn kind_spellings_are_the_wire_spellings() {
|
||||
let cases = [
|
||||
(Kind::Pdf, "\"pdf\""),
|
||||
(Kind::Ole2, "\"ole2\""),
|
||||
(Kind::Azw3, "\"azw3\""),
|
||||
(Kind::ZipWrappedPdf, "\"zip-wrapped-pdf\""),
|
||||
(Kind::Unknown, "\"unknown\""),
|
||||
];
|
||||
for (kind, wire) in cases {
|
||||
assert_eq!(serde_json::to_string(&kind).unwrap(), wire);
|
||||
assert_eq!(serde_json::from_str::<Kind>(wire).unwrap(), kind);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_layer_spellings_are_the_wire_spellings() {
|
||||
assert_eq!(
|
||||
serde_json::to_string(&TextLayer::ReflectedFont).unwrap(),
|
||||
"\"reflected-font\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_field_is_emitted_even_when_empty() {
|
||||
let doc = Doc::new("/tmp/a.pdf", Kind::Pdf, 12);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&doc).unwrap(),
|
||||
r#"{"v":1,"path":"/tmp/a.pdf","kind":"pdf","declared_ext":null,"bytes":12,"pages":null,"language":null,"text_layer":null,"needs_ocr":null,"notes":[]}"#
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_written_without_a_version_reads_as_version_one() {
|
||||
let doc: Doc =
|
||||
serde_json::from_str(r#"{"path":"a","kind":"txt","bytes":1,"notes":[]}"#).unwrap();
|
||||
assert_eq!(doc.v, 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//! What a tool's exit code means.
|
||||
//!
|
||||
//! A predicate answers through its status, so it is usable in a shell `if`
|
||||
//! without parsing anything. That only works if "no" and "broke" are
|
||||
//! different numbers, which is why the failure codes below start where
|
||||
//! `sysexits.h` puts them and leave 0, 1 and 2 to the answer.
|
||||
|
||||
/// Success, or a predicate answering yes.
|
||||
pub const OK: i32 = 0;
|
||||
/// A predicate answering no.
|
||||
pub const NO: i32 = 1;
|
||||
/// A predicate that could not tell — a damaged file, a format it does not
|
||||
/// know. Distinct from no, because a caller may want to retry or escalate.
|
||||
pub const INDETERMINATE: i32 = 2;
|
||||
/// The command line was wrong.
|
||||
pub const USAGE: i32 = 64;
|
||||
/// The input was not what the tool can work on.
|
||||
pub const DATA: i32 = 65;
|
||||
/// An external program the tool needs is not on `PATH`.
|
||||
pub const UNAVAILABLE: i32 = 69;
|
||||
/// Reading or writing failed.
|
||||
pub const IO: i32 = 74;
|
||||
|
||||
/// The answer a predicate tool gives.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
/// Yes.
|
||||
Yes,
|
||||
/// No.
|
||||
No,
|
||||
/// Could not tell.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
/// The process exit code for this answer.
|
||||
pub const fn code(self) -> i32 {
|
||||
match self {
|
||||
Verdict::Yes => OK,
|
||||
Verdict::No => NO,
|
||||
Verdict::Unknown => INDETERMINATE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<bool> for Verdict {
|
||||
fn from(yes: bool) -> Self {
|
||||
if yes { Verdict::Yes } else { Verdict::No }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_three_answers_are_zero_one_two() {
|
||||
assert_eq!(Verdict::Yes.code(), 0);
|
||||
assert_eq!(Verdict::No.code(), 1);
|
||||
assert_eq!(Verdict::Unknown.code(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_codes_never_collide_with_an_answer() {
|
||||
for code in [USAGE, DATA, UNAVAILABLE, IO] {
|
||||
assert!(code > INDETERMINATE);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
//! One scored candidate, on its way into a fusion.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// A single result from a single retrieval channel.
|
||||
///
|
||||
/// `rank` is the candidate's position within its own channel, from one, and
|
||||
/// is what rank fusion consumes; `score` is the channel's raw number, carried
|
||||
/// along because it is useful to a human reading a trace and useless for
|
||||
/// comparing across channels.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Hit {
|
||||
/// Record version.
|
||||
#[serde(default = "crate::v")]
|
||||
pub v: u32,
|
||||
/// Identifier of the thing retrieved, opaque to fusion.
|
||||
pub id: String,
|
||||
/// Which channel produced this hit.
|
||||
pub channel: Channel,
|
||||
/// Position within the channel's own ranking, from one.
|
||||
pub rank: u32,
|
||||
/// The channel's raw score, on whatever scale that channel uses.
|
||||
pub score: f64,
|
||||
/// Anything the producer wants to carry through fusion untouched.
|
||||
#[serde(default)]
|
||||
pub payload: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl Hit {
|
||||
/// A hit with an empty payload.
|
||||
pub fn new(id: impl Into<String>, channel: Channel, rank: u32, score: f64) -> Self {
|
||||
Self {
|
||||
v: crate::V,
|
||||
id: id.into(),
|
||||
channel,
|
||||
rank,
|
||||
score,
|
||||
payload: Map::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A retrieval channel.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum Channel {
|
||||
/// Nearest neighbours in an embedding space.
|
||||
#[serde(rename = "vec")]
|
||||
Vector,
|
||||
/// Full-text search.
|
||||
Fts,
|
||||
/// A match on metadata — title, author, tag.
|
||||
Meta,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn channel_spellings_are_the_wire_spellings() {
|
||||
for (channel, wire) in [
|
||||
(Channel::Vector, "\"vec\""),
|
||||
(Channel::Fts, "\"fts\""),
|
||||
(Channel::Meta, "\"meta\""),
|
||||
] {
|
||||
assert_eq!(serde_json::to_string(&channel).unwrap(), wire);
|
||||
assert_eq!(serde_json::from_str::<Channel>(wire).unwrap(), channel);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_field_is_emitted_even_when_empty() {
|
||||
let hit = Hit::new("abc", Channel::Fts, 1, 0.5);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&hit).unwrap(),
|
||||
r#"{"v":1,"id":"abc","channel":"fts","rank":1,"score":0.5,"payload":{}}"#
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payload_keys_serialise_in_a_fixed_order() {
|
||||
let mut hit = Hit::new("abc", Channel::Vector, 2, 0.25);
|
||||
hit.payload.insert("zulu".into(), Value::from(1));
|
||||
hit.payload.insert("alpha".into(), Value::from(2));
|
||||
assert_eq!(
|
||||
serde_json::to_string(&hit).unwrap(),
|
||||
r#"{"v":1,"id":"abc","channel":"vec","rank":2,"score":0.25,"payload":{"alpha":2,"zulu":1}}"#
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//! Record types and stream plumbing shared by every catena tool.
|
||||
//!
|
||||
//! Three records travel between the tools: a [`Doc`] is what a file is, a
|
||||
//! [`Chunk`] is a retrievable passage with enough provenance to cite it, and a
|
||||
//! [`Hit`] is one scored candidate on its way into a fusion. Each carries a
|
||||
//! version field so a reader can refuse a stream it does not understand.
|
||||
//!
|
||||
//! Every list-shaped stream is NDJSON — see [`ndjson`] — and every predicate
|
||||
//! answers through an exit code — see [`exit`].
|
||||
|
||||
pub mod chunk;
|
||||
pub mod doc;
|
||||
pub mod exit;
|
||||
pub mod hit;
|
||||
pub mod ndjson;
|
||||
|
||||
pub use chunk::{Chunk, Role};
|
||||
pub use doc::{Doc, Kind, TextLayer};
|
||||
pub use hit::{Channel, Hit};
|
||||
pub use ndjson::{Error, read, write_line};
|
||||
|
||||
/// The record version these types serialise as.
|
||||
pub const V: u32 = 1;
|
||||
|
||||
/// Serde default for the `v` field, so a record written by an older tool that
|
||||
/// omitted it still reads back as version 1 rather than 0.
|
||||
pub(crate) const fn v() -> u32 {
|
||||
V
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
//! Prints the JSON Schema for the record types, so a consumer in another
|
||||
//! language can generate its own bindings instead of reading Rust.
|
||||
|
||||
use std::process::ExitCode;
|
||||
|
||||
use corpus_core::{Chunk, Doc, Hit, exit};
|
||||
|
||||
const USAGE: &str = "\
|
||||
usage: corpus-core schema [doc|chunk|hit|all]
|
||||
|
||||
Prints the JSON Schema for one record type, or an object of all three.
|
||||
|
||||
cargo run -p corpus-core -- schema chunk > chunk.schema.json
|
||||
";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let mut args = std::env::args().skip(1);
|
||||
match args.next().as_deref() {
|
||||
Some("schema") => {
|
||||
let which = args.next().unwrap_or_else(|| "all".into());
|
||||
match schema(&which) {
|
||||
Some(json) => {
|
||||
println!("{json}");
|
||||
ExitCode::from(exit::OK as u8)
|
||||
}
|
||||
None => {
|
||||
eprintln!("corpus-core: unknown record type {which:?}\n\n{USAGE}");
|
||||
ExitCode::from(exit::USAGE as u8)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("--help" | "-h") => {
|
||||
println!("{USAGE}");
|
||||
ExitCode::from(exit::OK as u8)
|
||||
}
|
||||
_ => {
|
||||
eprintln!("{USAGE}");
|
||||
ExitCode::from(exit::USAGE as u8)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn schema(which: &str) -> Option<String> {
|
||||
let json = match which {
|
||||
"doc" => serde_json::to_string_pretty(&schemars::schema_for!(Doc)),
|
||||
"chunk" => serde_json::to_string_pretty(&schemars::schema_for!(Chunk)),
|
||||
"hit" => serde_json::to_string_pretty(&schemars::schema_for!(Hit)),
|
||||
"all" => serde_json::to_string_pretty(&serde_json::json!({
|
||||
"doc": schemars::schema_for!(Doc),
|
||||
"chunk": schemars::schema_for!(Chunk),
|
||||
"hit": schemars::schema_for!(Hit),
|
||||
})),
|
||||
_ => return None,
|
||||
};
|
||||
Some(json.expect("a schema is always serialisable"))
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
//! Newline-delimited JSON, read lazily and written flushed.
|
||||
//!
|
||||
//! Every list-shaped stream between these tools is NDJSON: one record per
|
||||
//! line, produced and consumed one at a time, so a slow stage never has to
|
||||
//! hold the whole list and never blocks the pipe behind it.
|
||||
|
||||
use std::io::{BufRead, Write};
|
||||
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
|
||||
/// Something went wrong reading or writing a stream.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// A line was not the record the reader expected.
|
||||
#[error("line {line}: {source}")]
|
||||
Parse {
|
||||
/// One-based line number within the stream, counting blank lines.
|
||||
line: u64,
|
||||
/// What serde made of it.
|
||||
source: serde_json::Error,
|
||||
},
|
||||
/// A record could not be turned into JSON at all.
|
||||
#[error("encoding a record: {0}")]
|
||||
Encode(#[source] serde_json::Error),
|
||||
/// The stream itself failed.
|
||||
#[error("{0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
/// Read a stream of records, one per line.
|
||||
///
|
||||
/// Blank lines are skipped, so a stream that was concatenated or hand-edited
|
||||
/// still reads. The iterator stops at the first error rather than skipping
|
||||
/// it: a malformed line means the producer and the consumer disagree about
|
||||
/// the contract, and continuing would bury that.
|
||||
pub fn read<T, R>(reader: R) -> impl Iterator<Item = Result<T, Error>>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
R: BufRead,
|
||||
{
|
||||
let mut line_no = 0u64;
|
||||
reader.lines().filter_map(move |line| {
|
||||
line_no += 1;
|
||||
match line {
|
||||
Err(e) => Some(Err(Error::Io(e))),
|
||||
Ok(line) if line.trim().is_empty() => None,
|
||||
Ok(line) => Some(serde_json::from_str(&line).map_err(|source| Error::Parse {
|
||||
line: line_no,
|
||||
source,
|
||||
})),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Write one record and flush it.
|
||||
///
|
||||
/// The flush is the point: a consumer downstream should see a record as soon
|
||||
/// as it exists, not when the producer's buffer happens to fill.
|
||||
pub fn write_line<T, W>(writer: &mut W, record: &T) -> Result<(), Error>
|
||||
where
|
||||
T: Serialize,
|
||||
W: Write,
|
||||
{
|
||||
serde_json::to_writer(&mut *writer, record).map_err(Error::Encode)?;
|
||||
writer.write_all(b"\n")?;
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{Channel, Hit};
|
||||
|
||||
#[test]
|
||||
fn reads_a_record_per_line_and_skips_blanks() {
|
||||
let input = "{\"id\":\"a\",\"channel\":\"fts\",\"rank\":1,\"score\":1.0}\n\n\
|
||||
{\"id\":\"b\",\"channel\":\"vec\",\"rank\":2,\"score\":0.5}\n";
|
||||
let hits: Vec<Hit> = read(input.as_bytes()).collect::<Result<_, _>>().unwrap();
|
||||
assert_eq!(hits.len(), 2);
|
||||
assert_eq!(hits[1].id, "b");
|
||||
assert_eq!(hits[1].channel, Channel::Vector);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_line_reports_its_line_number_and_stops_the_stream() {
|
||||
let input = "{\"id\":\"a\",\"channel\":\"fts\",\"rank\":1,\"score\":1.0}\nnot json\n";
|
||||
let mut stream = read::<Hit, _>(input.as_bytes());
|
||||
assert!(stream.next().unwrap().is_ok());
|
||||
match stream.next().unwrap() {
|
||||
Err(Error::Parse { line, .. }) => assert_eq!(line, 2),
|
||||
other => panic!("expected a parse error, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_one_compact_line_per_record() {
|
||||
let mut out = Vec::new();
|
||||
write_line(&mut out, &Hit::new("a", Channel::Meta, 1, 0.25)).unwrap();
|
||||
write_line(&mut out, &Hit::new("b", Channel::Meta, 2, 0.5)).unwrap();
|
||||
let text = String::from_utf8(out).unwrap();
|
||||
assert_eq!(text.lines().count(), 2);
|
||||
assert!(text.ends_with('\n'));
|
||||
assert!(!text.contains("\n "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_written_reads_back_identical() {
|
||||
let hit = Hit::new("a", Channel::Vector, 3, 0.125);
|
||||
let mut out = Vec::new();
|
||||
write_line(&mut out, &hit).unwrap();
|
||||
let back: Vec<Hit> = read(out.as_slice()).collect::<Result<_, _>>().unwrap();
|
||||
assert_eq!(back, vec![hit]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user