chonkie-core

from chonkie_core import Chunker, chunk, chunk_offsets text = "This is the first sentence. This is the second sentence! And this is the third sentence, with a comma. Finally, the last one. Here is some Japanese: これは日本語のテキストです。句読点も含まれます。" # Using Chunker class with delimiters and patterns print("--- Using Chunker ---") chunks_obj = list(Chunker(text, delimiters="\n.?!", patterns=["。", "，", "！"])) for c in chunks_obj: print(f"'{c.text}' (len: {len(c.text)})\nOffset range: {c.offset_range}") # Using convenience function `chunk` print("\n--- Using chunk function ---") for c in chunk(text, delimiters=".", patterns=["。"]): # The 'chunk' function returns Chunk objects print(f"'{c.text}' (len: {len(c.text)})\nOffset range: {c.offset_range}") # Getting offsets directly print("\n--- Using chunk_offsets function ---") offsets = chunk_offsets(text, delimiters=".", patterns=["。"]) print(f"Offsets: {offsets}")