xformers

import torch from xformers.ops import memory_efficient_attention, LowerTriangularMask # Ensure tensors are on CUDA if available device = "cuda" if torch.cuda.is_available() else "cpu" # Assume batch_size=2, seq_len=128, num_heads=8, head_dim=64 batch_size = 2 seq_len = 128 num_heads = 8 head_dim = 64 # Create dummy query, key, value tensors # xFormers memory_efficient_attention typically expects (batch_size, seq_len, num_heads, head_dim) query = torch.randn(batch_size, seq_len, num_heads, head_dim, device=device) key = torch.randn(batch_size, seq_len, num_heads, head_dim, device=device) value = torch.randn(batch_size, seq_len, num_heads, head_dim, device=device) # It's common to use float16 (half precision) for performance with xFormers query = query.half() key = key.half() value = value.half() # Example 1: Standard memory-efficient attention # xFormers automatically dispatches to the best available operator output_attn = memory_efficient_attention(query, key, value) print(f"Output attention shape (standard): {output_attn.shape}") # Example 2: Causal attention with a lower triangular mask # Note: The attn_bias argument structure has changed in newer versions (e.g., v0.0.21+) # For LowerTriangularMask, it often handles internal expansion if num_heads is implicitly available. attn_bias = LowerTriangularMask() output_causal_attn = memory_efficient_attention(query, key, value, attn_bias=attn_bias) print(f"Output attention shape (causal): {output_causal_attn.shape}") # To verify installation and available kernels: # import subprocess # subprocess.run(["python", "-m", "xformers.info"])