untokenize

import tokenize import io import untokenize source_code_with_whitespace = "def hello( name ):\n print(f'Hello, {name}!') # A comment" # Tokenize the source code using the standard library's generator # The readline callable should return bytes for tokenize.tokenize tokens_generator = tokenize.tokenize(io.BytesIO(source_code_with_whitespace.encode('utf-8')).readline) # Convert generator to a list of tokens tokens = list(tokens_generator) # Use untokenize library to reconstruct source code, preserving whitespace reconstructed_code = untokenize.untokenize(tokens) print("Original:") print(source_code_with_whitespace) print("\nReconstructed (with untokenize library):") print(reconstructed_code) # For comparison, standard library's untokenize (might format differently) # from tokenize import untokenize as std_untokenize # std_reconstructed_code = std_untokenize(tokens) # print("\nReconstructed (with stdlib tokenize.untokenize):") # print(std_reconstructed_code)