mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
funkpower
18cc217c0d
book-pipeline reaches for colophon and bibfetch on every book it repairs. Shelling out would mean a process per book per tool across five thousand books, and parsing back the JSON it is about to parse anyway. bibfetch's fetch loop lived in main.rs, so binding it meant either lifting it or writing a second one. A second implementation of 'which register answers about this kind of identifier, and in what order' would drift from the first, and the copy nobody runs from a terminal would drift silently. It is now bibfetch::ask, and the command line calls it too — verified by running the CLI before and after and diffing. fetch_identifier releases the GIL around the request. It is the only function here that opens a socket, and holding the interpreter for the length of an HTTP call would stop every other thread in it. bibfetch is optional and only the python feature turns it on: its HTTP client does not build for wasm32, and an unconditional dependency would have taken the JavaScript package down with it.
90 lines
1.8 KiB
Python
90 lines
1.8 KiB
Python
"""The catena libraries, in process.
|
|
|
|
Every function here is the same code the command-line tool runs, and hands
|
|
back the same record shapes it writes to stdout — dictionaries rather than
|
|
lines of JSON.
|
|
|
|
>>> import catena
|
|
>>> chunks = catena.chunk_markdown(markdown, target=512)
|
|
>>> chunks[0]["page"]
|
|
41
|
|
"""
|
|
|
|
from ._catena import (
|
|
__version__,
|
|
canonical_subject,
|
|
chunk_markdown,
|
|
citation_prompt,
|
|
clean_record,
|
|
clean_underscores,
|
|
compose_subjects,
|
|
count_tokens,
|
|
cue_windows,
|
|
fetch_identifier,
|
|
find_citations,
|
|
fuse_hits,
|
|
identify,
|
|
is_shouting,
|
|
isbn_from_filename,
|
|
isbn_from_text,
|
|
looks_like_footnote_citations,
|
|
looks_like_person,
|
|
merge_citation_reply,
|
|
needs_subject,
|
|
pack_cues,
|
|
parse_identifier,
|
|
parse_subtitles,
|
|
read_colophon,
|
|
record_problems,
|
|
record_severity,
|
|
same_work,
|
|
scrub,
|
|
segment_cues,
|
|
series_of,
|
|
shelve,
|
|
split_collection,
|
|
suggest_author,
|
|
title_case,
|
|
title_in_author,
|
|
valid_isbn,
|
|
)
|
|
|
|
__all__ = [
|
|
"__version__",
|
|
"canonical_subject",
|
|
"chunk_markdown",
|
|
"citation_prompt",
|
|
"clean_record",
|
|
"clean_underscores",
|
|
"compose_subjects",
|
|
"count_tokens",
|
|
"cue_windows",
|
|
"fetch_identifier",
|
|
"find_citations",
|
|
"fuse_hits",
|
|
"identify",
|
|
"is_shouting",
|
|
"isbn_from_filename",
|
|
"isbn_from_text",
|
|
"looks_like_footnote_citations",
|
|
"looks_like_person",
|
|
"merge_citation_reply",
|
|
"needs_subject",
|
|
"pack_cues",
|
|
"parse_identifier",
|
|
"parse_subtitles",
|
|
"read_colophon",
|
|
"record_problems",
|
|
"record_severity",
|
|
"same_work",
|
|
"scrub",
|
|
"segment_cues",
|
|
"series_of",
|
|
"shelve",
|
|
"split_collection",
|
|
"suggest_author",
|
|
"title_case",
|
|
"title_in_author",
|
|
"valid_isbn",
|
|
]
|