mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
funkpower
58be244478
Eight of them, each doing something a pipeline actually has to do, and each starting from something anyone can fetch — a public domain book from Project Gutenberg, a NASA broadcast, or a sample carried in the file. Shell, because the tools are filters first: a book to passages and what its notes cite; a recording to chapter marks, citable passages and the one ffmpeg call that makes it play on a phone; passages into pgvector and a search that asks twice and reconciles. Rust, for the case where the caller already holds the text and a subprocess would be serialising for nothing. Python, for the shape a pipeline stage takes once the reading is a library call. TypeScript, for the app side — chapter marks and deep links, and fusing two rankings in process. Three of them end with what re-ranking means, because it is two different things under one name: fusing ranked lists, which is what `rrf` does and needs no model, and a cross-encoder scoring query and passage together, which is accurate, slow, and runs over what retrieval already found.
94 lines
3.6 KiB
Bash
Executable File
94 lines
3.6 KiB
Bash
Executable File
#!/bin/sh
|
|
# A book file to passages worth embedding, and what its notes cite.
|
|
#
|
|
# The starting point is Gibbon's first volume from Project Gutenberg: public
|
|
# domain, a real table of contents, and a few hundred pages of footnotes.
|
|
#
|
|
# examples/shell/01-book-to-passages.sh [work-dir]
|
|
#
|
|
# Needs pandoc and jq, and the catena commands on PATH.
|
|
set -eu
|
|
|
|
work="${1:-./work}"
|
|
mkdir -p "$work"
|
|
book="$work/gibbon.epub"
|
|
|
|
if [ ! -s "$book" ]; then
|
|
echo "fetching Gibbon vol. 1 from Project Gutenberg"
|
|
curl -sSL --max-time 120 -o "$book" \
|
|
"https://www.gutenberg.org/cache/epub/731/pg731.epub"
|
|
fi
|
|
|
|
# What the file actually is. Extensions lie, and a `.epub` that is really a
|
|
# zip of page scans fails four stages later, in a way that looks like a bug
|
|
# in the converter.
|
|
echo
|
|
echo "== what it is"
|
|
booksniff "$book" | jq -r '" \(.kind), \(.bytes) bytes"'
|
|
|
|
# A PDF gets one more question first: exit 0 from pdftriage means the text
|
|
# layer is not worth reading and the file has to be OCRed before any of this.
|
|
if [ "$(booksniff "$book" | jq -r .kind)" = "pdf" ] && pdftriage --quiet "$book"; then
|
|
echo " the text layer is garbled; OCR it before going on"
|
|
exit 0
|
|
fi
|
|
|
|
# What the file already says about itself — the publisher's own metadata,
|
|
# which beats anything guessed from a filename, and costs no lookup.
|
|
echo
|
|
echo "== what it says about itself"
|
|
bookmeta "$book" | jq -r '" \(.title)\n \(.authors | join("; "))\n \(.language // "?") · \(.date // "no date")"'
|
|
|
|
# The chapters the book declares, rather than any inferred from the text.
|
|
echo
|
|
echo "== the contents it declares"
|
|
tocfind "$book" --outline | sed -n '6,10p' | sed 's/^/ /'
|
|
echo " … $(tocfind "$book" | wc -l | tr -d ' ') entries"
|
|
|
|
# Convert, scrub, chunk. `commonmark` rather than `commonmark_x`, because the
|
|
# extended flavour writes anchors as bracketed spans and they end up in the
|
|
# passages. The scrub is what takes out the rest of the converter's leavings.
|
|
echo
|
|
echo "== passages"
|
|
pandoc -f epub -t commonmark --wrap=none "$book" -o "$work/book.md"
|
|
mdscrub < "$work/book.md" > "$work/clean.md"
|
|
chunkmd < "$work/clean.md" > "$work/chunks.ndjson"
|
|
|
|
jq -s -r '
|
|
" \(length) passages over \([.[].chapter] | unique | length) chapters",
|
|
" roles: \([.[].role] | group_by(.) | map("\(length) \(.[0])") | join(", "))",
|
|
" \([.[] | select(.flags | index("footnote_citations"))] | length) carry citation material"
|
|
' "$work/chunks.ndjson"
|
|
|
|
echo
|
|
echo " a passage, with where it came from:"
|
|
jq -s -r '[.[] | select(.role == "body" and .chapter != null)][40]
|
|
| " [\(.idx)] \(.tokens) tokens · \(.chapter)\n \(.text[0:150])…"' \
|
|
"$work/chunks.ndjson"
|
|
|
|
# The notes. A passage flagged `footnote_citations` is a page of them, and
|
|
# each is read on its own — the flag says which passages are worth the ask,
|
|
# which is the whole reason chunkmd sets it.
|
|
echo
|
|
echo "== what the notes cite"
|
|
: > "$work/references.ndjson"
|
|
jq -c 'select(.flags | index("footnote_citations")) | .text' "$work/chunks.ndjson" \
|
|
| while IFS= read -r line; do
|
|
printf '%s' "$line" | jq -r . | citefind >> "$work/references.ndjson" || true
|
|
done
|
|
|
|
jq -s -r '
|
|
" \(length) entries read",
|
|
" \([.[] | select(.title)] | length) name a work, \([.[] | select(.authors | length > 0)] | length) name an author",
|
|
" \([.[] | select(.unsettled | length > 0)] | length) came back with something unread"
|
|
' "$work/references.ndjson"
|
|
|
|
echo
|
|
echo " Gibbon cites in the classical short form — \"Tacit. Annal. xiii. 5.\" —"
|
|
echo " so most entries name the passage rather than a book, and say so:"
|
|
jq -r 'select(.unsettled | length > 0) | " \(.text[0:58]) → unread: \(.unsettled | join(", "))"' \
|
|
"$work/references.ndjson" | head -4
|
|
|
|
echo
|
|
echo "written to $work/"
|