tantivy-tokenizer-api

use tantivy_tokenizer_api::Tokenizer; fn main() { // Example: using a tokenizer (implementation not included) let tokenizer = MyTokenizer::default(); let mut stream = tokenizer.token_stream("hello world"); while let Some(token) = stream.next() { println!("{:?}", token); } }