pytorch-tokenizers

import os import tempfile import sentencepiece as spm # Required for generating dummy model from pytorch_tokenizers import SentencePieceTokenizer # 1. Create a dummy SentencePiece model file for demonstration # In real-world scenarios, you would use an existing pre-trained model. model_prefix = os.path.join(tempfile.gettempdir(), 'm_test') model_file = f'{model_prefix}.model' vocab_file = f'{model_prefix}.vocab' # Ensure clean slate for temporary files if os.path.exists(model_file): os.remove(model_file) if os.path.exists(vocab_file): os.remove(vocab_file) text_data = "Hello world. This is a test sentence. SentencePiece is great!" with open(f'{model_prefix}.txt', 'w') as f: f.write(text_data) spm.SentencePieceTrainer.train( input=f'{model_prefix}.txt', model_prefix=model_prefix, vocab_size=10, model_type='bpe' ) # 2. Instantiate the SentencePieceTokenizer from the created model file tokenizer = SentencePieceTokenizer.from_file(model_file) # 3. Encode text input_text = "This is a sample text for tokenization." encoded_tokens = tokenizer.encode(input_text) print(f"Original text: {input_text}") print(f"Encoded token IDs: {encoded_tokens}") # 4. Decode tokens decoded_text = tokenizer.decode(encoded_tokens) print(f"Decoded text: {decoded_text}") # Clean up temporary files os.remove(f'{model_prefix}.txt') os.remove(model_file) os.remove(vocab_file)