mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
pdftriage: text layer verdicts, including the reflected-font recovery
This commit is contained in:
funkpower
@@ -16,6 +16,7 @@ python = []
|
||||
[dependencies]
|
||||
chunkmd.workspace = true
|
||||
mdscrub.workspace = true
|
||||
pdftriage.workspace = true
|
||||
corpus-core.workspace = true
|
||||
serde_json.workspace = true
|
||||
textkit.workspace = true
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{"name": "empty", "text": ""}
|
||||
{"name": "whitespace", "text": " \n\t \n"}
|
||||
{"name": "prose-short", "text": "The mill stood at the northern end."}
|
||||
{"name": "prose", "text": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. "}
|
||||
{"name": "prose-reflected", "text": "̸» »½±²¼ »¬¿¬» ¸»´¼ ·¬ ´¿²¼ ·² »¨½¸¿²¹» º±® »®ª·½»ô ¿²¼ ¬¸» »®ª·½» ©¿ ³·´·¬¿®§ò ̸¿¬ ³«½¸ · ¿¹®»»¼å ©¸¿¬ º±´´±© · ²±¬ò ̸» ½±«®¬ ®±´´ ¼± ²±¬ ¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿ ¬»¬»¼ô ¾»½¿«» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»¬ò ̸· · ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ±«®½»ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±© ¼»°»²¼ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò ̸» »½±²¼ »¬¿¬» ¸»´¼ ·¬ ´¿²¼ ·² »¨½¸¿²¹» º±® »®ª·½»ô ¿²¼ ¬¸» »®ª·½» ©¿ ³·´·¬¿®§ò ̸¿¬ ³«½¸ · ¿¹®»»¼å ©¸¿¬ º±´´±© · ²±¬ò ̸» ½±«®¬ ®±´´ ¼± ²±¬ ¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿ ¬»¬»¼ô ¾»½¿«» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»¬ò ̸· · ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ±«®½»ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±© ¼»°»²¼ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò ̸» »½±²¼ »¬¿¬» ¸»´¼ ·¬ ´¿²¼ ·² »¨½¸¿²¹» º±® »®ª·½»ô ¿²¼ ¬¸» »®ª·½» ©¿ ³·´·¬¿®§ò ̸¿¬ ³«½¸ · ¿¹®»»¼å ©¸¿¬ º±´´±© · ²±¬ò ̸» ½±«®¬ ®±´´ ¼± ²±¬ ¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿ ¬»¬»¼ô ¾»½¿«» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»¬ò ̸· · ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ±«®½»ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±© ¼»°»²¼ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò "}
|
||||
{"name": "prose-reflected-short", "text": "̸» »½±²¼ »¬¿¬» ¸»´¼ ·¬ ´¿²¼ ·² »¨½¸¿²¹» º±® »®ª·½»ô ¿²¼ ¬¸» »®ª·½» ©¿ ³·´·¬¿®§ò ̸¿¬ ³«½¸ · ¿¹®»»¼å ©¸¿¬ º±´´±© · ²±¬ò ̸» ½±«®¬ ®±´´ ¼± ²±¬ ¿§ ¸±© ±º¬»² ¬¸» ±¾´·¹¿¬·±² ©¿ ¬»¬»¼ô ¾»½¿«» ¬¸» ½´»®µ ©¸± ®»½±®¼»¼ ·¬ ¸¿¼ ²± ª±½¿¾«´¿®§ º±® ¬¸» ¼·ºº»®»²½» ¾»¬©»»² ¿ ®»¾»´´·±² ¿²¼ ¿ ¾¿¼ ¸¿®ª»¬ò ̸· · ¬¸» ½»²¬®¿´ ¼·ºº·½«´¬§ ±º ¬¸» ±«®½»ô ¿²¼ »ª»®§¬¸·²¹ ¬¸¿¬ º±´´±© ¼»°»²¼ ±² ®»¿¼·²¹ ¿®±«²¼ ¬¸¿¬ ´·³·¬¿¬·±² ®¿¬¸»® ¬¸¿² ¬¸®±«¹¸ ·¬ò "}
|
||||
{"name": "german", "text": "Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikte schweigen die Quellen. "}
|
||||
{"name": "german-reflected", "text": "Ü·» Ù®«²¼¸»®®½¸¿º¬ ©¿® µ»·² Û·¹»²¬«³ ·³ ®ö³·½¸»² Í·²²»ô ±²¼»®² »·² Þü²¼»´ ª±² ß²°®ü½¸»²ò Ü·» Þ¿«»®² ½¸«´¼»¬»² ß¾¹¿¾»² «²¼ Ú®±²¼·»²¬»å ¼»® Ø»®® ½¸«´¼»¬» ͽ¸«¬¦ò ܾ»® ¼·» Ø䫺·¹µ»·¬ ¼»® Õ±²º´·µ¬» ½¸©»·¹»² ¼·» Ï«»´´»²ò Ü·» Ù®«²¼¸»®®½¸¿º¬ ©¿® µ»·² Û·¹»²¬«³ ·³ ®ö³·½¸»² Í·²²»ô ±²¼»®² »·² Þü²¼»´ ª±² ß²°®ü½¸»²ò Ü·» Þ¿«»®² ½¸«´¼»¬»² ß¾¹¿¾»² «²¼ Ú®±²¼·»²¬»å ¼»® Ø»®® ½¸«´¼»¬» ͽ¸«¬¦ò ܾ»® ¼·» Ø䫺·¹µ»·¬ ¼»® Õ±²º´·µ¬» ½¸©»·¹»² ¼·» Ï«»´´»²ò Ü·» Ù®«²¼¸»®®½¸¿º¬ ©¿® µ»·² Û·¹»²¬«³ ·³ ®ö³·½¸»² Í·²²»ô ±²¼»®² »·² Þü²¼»´ ª±² ß²°®ü½¸»²ò Ü·» Þ¿«»®² ½¸«´¼»¬»² ß¾¹¿¾»² «²¼ Ú®±²¼·»²¬»å ¼»® Ø»®® ½¸«´¼»¬» ͽ¸«¬¦ò ܾ»® ¼·» Ø䫺·¹µ»·¬ ¼»® Õ±²º´·µ¬» ½¸©»·¹»² ¼·» Ï«»´´»²ò "}
|
||||
{"name": "greek", "text": "Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Ἀθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. "}
|
||||
{"name": "cyrillic", "text": "Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. Въ лѣто 6390 поиде Олегъ поемъ воя многи варяги. "}
|
||||
{"name": "chess", "text": "1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O Be7 6. Re1 b5 7. Bb3 d6 8. c3 O-O "}
|
||||
{"name": "running-heads", "text": "The Second Estate 41\nThe Second Estate 41\nThe Second Estate 41\nThe Second Estate 41\nThe Second Estate 41\nThe Second Estate 41\nThe Second Estate 41\nThe Second Estate 41\n"}
|
||||
{"name": "combining-letters", "text": "wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. wiſt ouer ſee aͣnd aͤ laͥnd wiͦth moͧre. "}
|
||||
{"name": "nonsense-latin1", "text": "¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíô"}
|
||||
{"name": "nonsense-reflected", "text": "¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíôû£ª±¸¿ÆÍÔÛâéð÷þ¦´»ÂÉÐ×Þåìóú¢©°·¾ÅÌÓÚáèïöý¥¬³ºÁÈÏÖÝäëòù¡¨¯¶½ÄËÒÙàçîõü¤«²¹ÀÇÎÕÜãêñøÿ§®µ¼ÃÊÑØßæíô"}
|
||||
{"name": "mixed-scripts", "text": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, becauseἈθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikteτὰ 東京 ١٢٣ The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, becauseἈθῆναι καὶ Λακεδαίμων καὶ τὰ λοιπὰ τῶν Ἑλλήνων. Die Grundherrschaft war kein Eigentum im römischen Sinne, sondern ein Bündel von Ansprüchen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Über die Häufigkeit der Konflikteτὰ 東京 ١٢٣ "}
|
||||
{"name": "all-caps", "text": "THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY THE SECOND ESTATE HELD ITS LAND IN EXCHANGE FOR SERVICE AND THE SERVICE WAS MILITARY "}
|
||||
{"name": "mixed-case-junk", "text": "zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x zgW$c_ iXYc$f\\KO qPzR$t mNbV$x "}
|
||||
{"name": "digits", "text": "1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 1234567890 "}
|
||||
{"name": "punctuation", "text": ".,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- .,;:!?()[]{}- "}
|
||||
{"name": "emdash-prose", "text": "The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. The obligation — and it was an obligation — ran both ways. "}
|
||||
{"name": "nbsp-prose", "text": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. "}
|
||||
{"name": "control-chars", "text": "text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c text\u0001with\u000bcontrol\u001fchars\u001c "}
|
||||
{"name": "ligatures", "text": "The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. The first flight of the figure was confirmed. "}
|
||||
@@ -0,0 +1,24 @@
|
||||
{"ascii": 0.0, "body": 0, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "empty", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.0, "body": 0, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "whitespace", "tidy": null, "vowelled": null}
|
||||
{"ascii": 1.0, "body": 29, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.9655172413793104, "name": "prose-short", "tidy": null, "vowelled": null}
|
||||
{"ascii": 1.0, "body": 1104, "decoded": null, "english": 0.42105263157894735, "latin1": 0.0, "letters": 0.9782608695652174, "name": "prose", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 0.0, "body": 1104, "decoded": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. ", "english": 0.0, "latin1": 1.0, "letters": 0.07608695652173914, "name": "prose-reflected", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.0, "body": 368, "decoded": "The second estate held its land in exchange for service, and the service was military. That much is agreed; what follows is not. The court rolls do not say how often the obligation was tested, because the clerk who recorded it had no vocabulary for the difference between a rebellion and a bad harvest. This is the central difficulty of the sources, and everything that follows depends on reading around that limitation rather than through it. ", "english": null, "latin1": 1.0, "letters": 0.07608695652173914, "name": "prose-reflected-short", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.9740932642487047, "body": 579, "decoded": null, "english": 0.0, "latin1": 0.025906735751295335, "letters": 0.9740932642487047, "name": "german", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 0.0, "body": 579, "decoded": "Die Grundherrschaft war kein Eigentum im r*mischen Sinne, sondern ein B$ndel von Anspr$chen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Dber die H<ufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im r*mischen Sinne, sondern ein B$ndel von Anspr$chen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Dber die H<ufigkeit der Konflikte schweigen die Quellen. Die Grundherrschaft war kein Eigentum im r*mischen Sinne, sondern ein B$ndel von Anspr$chen. Die Bauern schuldeten Abgaben und Frondienste; der Herr schuldete Schutz. Dber die H<ufigkeit der Konflikte schweigen die Quellen. ", "english": 0.0, "latin1": 1.0, "letters": 0.16580310880829016, "name": "german-reflected", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.025, "body": 240, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.975, "name": "greek", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.125, "body": 240, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.875, "name": "cyrillic", "tidy": null, "vowelled": null}
|
||||
{"ascii": 1.0, "body": 348, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.4482758620689655, "name": "chess", "tidy": null, "vowelled": null}
|
||||
{"ascii": 1.0, "body": 136, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.8823529411764706, "name": "running-heads", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.7878787878787878, "body": 660, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.8181818181818182, "name": "combining-letters", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.0, "body": 800, "decoded": null, "english": null, "latin1": 1.0, "letters": 0.685, "name": "nonsense-latin1", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.0, "body": 800, "decoded": null, "english": null, "latin1": 1.0, "letters": 0.685, "name": "nonsense-reflected", "tidy": null, "vowelled": null}
|
||||
{"ascii": 0.8675324675324675, "body": 770, "decoded": null, "english": 0.1917808219178082, "latin1": 0.012987012987012988, "letters": 0.9662337662337662, "name": "mixed-scripts", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 1.0, "body": 280, "decoded": null, "english": null, "latin1": 0.0, "letters": 1.0, "name": "all-caps", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 1.0, "body": 1080, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.7777777777777778, "name": "mixed-case-junk", "tidy": 0.0, "vowelled": 0.25}
|
||||
{"ascii": 1.0, "body": 400, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "digits", "tidy": null, "vowelled": null}
|
||||
{"ascii": 1.0, "body": 520, "decoded": null, "english": null, "latin1": 0.0, "letters": 0.0, "name": "punctuation", "tidy": null, "vowelled": null}
|
||||
{"ascii": 1.0, "body": 564, "decoded": null, "english": 0.4166666666666667, "latin1": 0.0, "letters": 0.9361702127659575, "name": "emdash-prose", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 1.0, "body": 736, "decoded": null, "english": 0.42105263157894735, "latin1": 0.0, "letters": 0.9782608695652174, "name": "nbsp-prose", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 0.9523809523809523, "body": 630, "decoded": null, "english": 0.0, "latin1": 0.0, "letters": 0.9523809523809523, "name": "control-chars", "tidy": 1.0, "vowelled": 1.0}
|
||||
{"ascii": 0.8823529411764706, "body": 680, "decoded": null, "english": 0.5, "latin1": 0.0, "letters": 0.9705882352941176, "name": "ligatures", "tidy": 1.0, "vowelled": 0.875}
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Measure text layers the way the Python pipeline measures them.
|
||||
|
||||
Reads one `{"name", "text"}` JSON object per line and writes back the numbers
|
||||
each gate is decided on, so the judgement can be compared without shipping any
|
||||
book. Two of the measurements — the letter share and the Latin-1 share — are
|
||||
written inline in the Python's garbled gate rather than in a function of their
|
||||
own, so they are spelled out again here, copied line for line.
|
||||
|
||||
CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
|
||||
python3 oracle/pdftriage.py < cases/pdftriage/samples.ndjson
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
|
||||
if path:
|
||||
sys.path.insert(0, path)
|
||||
|
||||
from book_pipeline import convert # noqa: E402
|
||||
|
||||
|
||||
def letter_share(body: list[str]) -> float:
|
||||
if not body:
|
||||
return 0.0
|
||||
return sum(1 for c in body if c.isalpha()) / len(body)
|
||||
|
||||
|
||||
def latin1_share(body: list[str]) -> float:
|
||||
if not body:
|
||||
return 0.0
|
||||
return sum(1 for c in body if 0xA1 <= ord(c) <= 0xFF) / len(body)
|
||||
|
||||
|
||||
def english_share(text: str, min_words: int = 80) -> float | None:
|
||||
words = [w.strip(".,;:!?()[]\"'").lower() for w in text.split()]
|
||||
words = [w for w in words if w]
|
||||
if len(words) < min_words:
|
||||
return None
|
||||
return sum(1 for w in words if w in convert._COMMON_WORDS) / len(words)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
out = sys.stdout
|
||||
for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
case = json.loads(line)
|
||||
text = case["text"]
|
||||
body = [c for c in text if not c.isspace()]
|
||||
shape = convert._word_shape(text)
|
||||
decoded = convert.undo_reflected_font(text)
|
||||
record = {
|
||||
"name": case["name"],
|
||||
"body": len(body),
|
||||
"ascii": convert._ascii_text_score(text),
|
||||
"letters": letter_share(body),
|
||||
"latin1": latin1_share(body),
|
||||
"vowelled": None if shape is None else shape[0],
|
||||
"tidy": None if shape is None else shape[1],
|
||||
"english": english_share(text),
|
||||
"decoded": decoded,
|
||||
}
|
||||
out.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+14
-1
@@ -19,11 +19,12 @@ type Written = Result<Vec<String>, String>;
|
||||
type Job = (&'static str, fn() -> Written);
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let jobs: [Job; 4] = [
|
||||
let jobs: [Job; 5] = [
|
||||
("tokens", tokens),
|
||||
("unescape", unescape),
|
||||
("chunkmd", chunkmd),
|
||||
("mdscrub", mdscrub),
|
||||
("pdftriage", pdftriage),
|
||||
];
|
||||
|
||||
for (name, job) in jobs {
|
||||
@@ -67,6 +68,18 @@ fn unescape() -> Written {
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn pdftriage() -> Written {
|
||||
let dir = parity::golden("pdftriage");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
let cases = parity::dir().join("cases/pdftriage/samples.ndjson");
|
||||
let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
|
||||
let measured = parity::run_oracle("pdftriage.py", &text)?;
|
||||
let path = dir.join("samples.ndjson");
|
||||
std::fs::write(&path, measured).map_err(|e| e.to_string())?;
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn mdscrub() -> Written {
|
||||
let dir = parity::golden("mdscrub");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
//! Every number a text-layer verdict is decided on must match the Python
|
||||
//! exactly.
|
||||
//!
|
||||
//! The thresholds these feed were measured against three thousand real PDFs,
|
||||
//! so a score that is a little different is a verdict that is occasionally
|
||||
//! different — and a wrong "garbled" costs an hour of GPU time while a wrong
|
||||
//! "healthy" puts unreadable text in the index for good. The samples are
|
||||
//! synthetic: a parity test over a text-layer classifier needs the score
|
||||
//! inputs, not anybody's book.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// Both sides compute a count divided by a count, so agreement is exact or
|
||||
/// the counts differ.
|
||||
fn same(ours: f64, theirs: &Value) -> bool {
|
||||
match theirs {
|
||||
Value::Null => false,
|
||||
value => value.as_f64() == Some(ours),
|
||||
}
|
||||
}
|
||||
|
||||
fn cases() -> Vec<Value> {
|
||||
let text = std::fs::read_to_string(parity::dir().join("cases/pdftriage/samples.ndjson"))
|
||||
.expect("a readable case file");
|
||||
text.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| serde_json::from_str(line).expect("valid JSON"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_measurement_matches_python() {
|
||||
let golden: Vec<Value> = parity::read_golden_text("pdftriage/samples.ndjson")
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| serde_json::from_str(line).expect("valid JSON"))
|
||||
.collect();
|
||||
let cases = cases();
|
||||
assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
|
||||
|
||||
let mut wrong: Vec<String> = Vec::new();
|
||||
for (case, theirs) in cases.iter().zip(&golden) {
|
||||
let name = case["name"].as_str().expect("a name");
|
||||
assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
|
||||
let text = case["text"].as_str().expect("text");
|
||||
let body = pdftriage::score::body(text);
|
||||
|
||||
let mut note = |field: &str, ours: String, theirs: &Value| {
|
||||
wrong.push(format!("{name}.{field}: python {theirs}, rust {ours}"));
|
||||
};
|
||||
|
||||
if !same(body.len() as f64, &theirs["body"]) {
|
||||
note("body", body.len().to_string(), &theirs["body"]);
|
||||
}
|
||||
if !same(pdftriage::score::ascii_text_score(text), &theirs["ascii"]) {
|
||||
let ours = pdftriage::score::ascii_text_score(text);
|
||||
note("ascii", ours.to_string(), &theirs["ascii"]);
|
||||
}
|
||||
if !same(pdftriage::score::letter_share(&body), &theirs["letters"]) {
|
||||
let ours = pdftriage::score::letter_share(&body);
|
||||
note("letters", ours.to_string(), &theirs["letters"]);
|
||||
}
|
||||
if !same(pdftriage::score::latin1_share(&body), &theirs["latin1"]) {
|
||||
let ours = pdftriage::score::latin1_share(&body);
|
||||
note("latin1", ours.to_string(), &theirs["latin1"]);
|
||||
}
|
||||
|
||||
let shape = pdftriage::score::word_shape(text, 40);
|
||||
match (shape, theirs["vowelled"].as_f64(), theirs["tidy"].as_f64()) {
|
||||
(None, None, None) => {}
|
||||
(Some((vowelled, tidy)), Some(their_vowelled), Some(their_tidy)) => {
|
||||
if vowelled != their_vowelled {
|
||||
note("vowelled", vowelled.to_string(), &theirs["vowelled"]);
|
||||
}
|
||||
if tidy != their_tidy {
|
||||
note("tidy", tidy.to_string(), &theirs["tidy"]);
|
||||
}
|
||||
}
|
||||
(ours, _, _) => note("word_shape", format!("{ours:?}"), &theirs["vowelled"]),
|
||||
}
|
||||
|
||||
let english = pdftriage::english_prose_share(text, 80);
|
||||
match (english, theirs["english"].as_f64()) {
|
||||
(None, None) => {}
|
||||
(Some(ours), Some(theirs_value)) if ours == theirs_value => {}
|
||||
(ours, _) => note("english", format!("{ours:?}"), &theirs["english"]),
|
||||
}
|
||||
|
||||
let decoded = pdftriage::reflect::undo(text);
|
||||
let their_decoded = theirs["decoded"].as_str();
|
||||
if decoded.as_deref() != their_decoded {
|
||||
note(
|
||||
"decoded",
|
||||
match &decoded {
|
||||
None => "none".to_string(),
|
||||
Some(text) => format!("{} characters", text.chars().count()),
|
||||
},
|
||||
&theirs["decoded"],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
wrong.is_empty(),
|
||||
"{} measurements differ:\n{}",
|
||||
wrong.len(),
|
||||
wrong
|
||||
.iter()
|
||||
.take(20)
|
||||
.map(|line| line.chars().take(200).collect::<String>())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Catches a golden file drifting away from a Python that has moved on.
|
||||
#[test]
|
||||
#[cfg(feature = "python")]
|
||||
fn the_golden_file_still_matches_live_python() {
|
||||
let cases = std::fs::read_to_string(parity::dir().join("cases/pdftriage/samples.ndjson"))
|
||||
.expect("a readable case file");
|
||||
let live = parity::run_oracle("pdftriage.py", &cases).expect("the oracle runs");
|
||||
assert_eq!(live, parity::read_golden_text("pdftriage/samples.ndjson"));
|
||||
}
|
||||
Reference in New Issue
Block a user