llmcompressor

from transformers import AutoTokenizer from llmcompressor.models import AutoModelForCausalLM from llmcompressor.recipes import SparseMLRecipe from llmcompressor.compression import Compressor import torch # 1. Load a pre-trained model and tokenizer model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForCausalLM.from_pretrained(model_name) # 2. Define a compression recipe (e.g., for 8-bit quantization) # This YAML describes a simple post-training quantization (PTQ) modifier. # For full functionality, specific targets and calibration data would be needed. recipe_yaml = """ quantization_modifiers: - !QuantizationModifier start: 0.0 scheme_args: num_bits: 8 symmetric: False per_channel: True """ # 3. Parse the recipe recipe = SparseMLRecipe.parse_yaml(recipe_yaml) # 4. Create a Compressor instance # The model will be modified in-place when compression is applied. # For PTQ, a calibration dataloader is typically required for `compressor.compress()`. compressor = Compressor(recipe=recipe, model=model, tokenizer=tokenizer) # 5. Apply compression (requires calibration data for true PTQ) print("Compressor initialized. To apply compression with Post-Training Quantization (PTQ),") print("you would typically call: compressor.compress(dataloader=your_calibration_dataloader)") print("For this quickstart, we've demonstrated the setup without running full PTQ.") # Example of saving (after actual compression) # compressor.save_compressed_model("path/to/save/compressed_model")