ported patterns mean in rust what they mean in python

crates/textkit/Cargo.toml

@@ -10,5 +10,8 @@ repository.workspace = true
 [dependencies]
 tiktoken-rs.workspace = true
 
+[dev-dependencies]
+regex.workspace = true
+
 [lints]
 workspace = true

crates/textkit/src/lib.rs

@@ -11,6 +11,7 @@
 //! matches what is already in an index.
 
 pub mod number;
+pub mod pattern;
 pub mod segment;
 pub mod tokens;
 

crates/textkit/src/pattern.rs

@@ -0,0 +1,59 @@
+//! Character classes that mean in Rust what they mean in Python.
+//!
+//! The two regex engines disagree about `\s`, `\w` and therefore `\b`, and
+//! every disagreement is a heuristic that fires on one side and not the
+//! other. The differences are small and entirely real:
+//!
+//! - Python's `\s` also matches `U+001C`–`U+001F`, the four ASCII
+//!   information separators.
+//! - Python's `\w` is the letters, the numbers and the underscore. Rust's
+//!   adds every combining mark, every connector punctuation mark and the
+//!   join controls, and leaves out the letter-like and other numbers — so
+//!   `Gödel` written in decomposed form is one word to Rust and two to
+//!   Python, and `²` is a word character to Python and not to Rust.
+//! - `\b` is defined in terms of `\w`, so it inherits all of that.
+//!
+//! Spell a ported pattern with these and it means the same thing on both
+//! sides. The boundary forms are lookarounds, so a pattern using them needs
+//! an engine that has them.
+
+/// Python's `\s`.
+pub const SPACE: &str = r"[\s\x{1c}-\x{1f}]";
+
+/// Python's `\w`.
+pub const WORD: &str = r"[\p{L}\p{N}_]";
+
+/// Python's `\b`, before a word.
+pub const WORD_START: &str = r"(?<![\p{L}\p{N}_])";
+
+/// Python's `\b`, after a word.
+pub const WORD_END: &str = r"(?![\p{L}\p{N}_])";
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_space_class_covers_what_python_trims() {
+        let space = regex::Regex::new(SPACE).expect("a valid pattern");
+        for c in [' ', '\t', '\n', '\u{a0}', '\u{1c}', '\u{1f}', '\u{2028}'] {
+            assert!(space.is_match(&c.to_string()), "{c:?}");
+            assert!(crate::is_space(c), "{c:?}");
+        }
+        for c in ['a', '-', '\u{200b}'] {
+            assert!(!space.is_match(&c.to_string()), "{c:?}");
+        }
+    }
+
+    #[test]
+    fn the_word_class_is_letters_numbers_and_underscore() {
+        let word = regex::Regex::new(&format!("^{WORD}$")).expect("a valid pattern");
+        for c in ['a', 'Z', '0', '_', 'é', 'Ω', '²', 'Ⅷ'] {
+            assert!(word.is_match(&c.to_string()), "{c:?}");
+        }
+        // The combining marks Rust's own `\w` would accept.
+        for c in ['\u{301}', '\u{363}', '-', ' '] {
+            assert!(!word.is_match(&c.to_string()), "{c:?}");
+        }
+    }
+}