azure-datalake-store

import os from azure.datalake.store import core # Set these environment variables for authentication # Ensure AZURE_TENANT_ID, AZURE_USERNAME, AZURE_PASSWORD, AZURE_STORE_NAME are set # For testing, use placeholder values if not connecting to a real ADLS Gen1 tenant_id = os.environ.get('AZURE_TENANT_ID', 'YOUR_TENANT_ID') username = os.environ.get('AZURE_USERNAME', 'YOUR_USERNAME') password = os.environ.get('AZURE_PASSWORD', 'YOUR_PASSWORD') store_name = os.environ.get('AZURE_STORE_NAME', 'youradlstorename') try: # Authenticate (lib.auth now uses generic Azure token credentials internally) token = core.lib.auth(tenant_id, username, password) # Initialize the Data Lake Store filesystem client adl = core.AzureDLFileSystem(store_name, token=token) # Example: List contents of the root directory print(f"Listing contents of / in {store_name}:") items = adl.ls('/', detail=True) if items: for item in items: print(item) else: print("Directory is empty or path does not exist.") # Example: Create a directory and a file test_dir = 'mytestdir' test_file = f'{test_dir}/testfile.txt' if not adl.exists(test_dir): adl.mkdir(test_dir) print(f"Created directory: {test_dir}") with adl.open(test_file, 'wb') as f: f.write(b"Hello from Azure Data Lake Store Gen1!") print(f"Created and wrote to file: {test_file}") # Example: Read the file with adl.open(test_file, 'rb') as f: content = f.read() print(f"Content of {test_file}: {content.decode('utf-8')}") # Example: Delete the file and directory adl.rm(test_file) print(f"Deleted file: {test_file}") adl.rmdir(test_dir) print(f"Deleted directory: {test_dir}") except Exception as e: print(f"An error occurred: {e}") print("Please ensure environment variables for ADLS Gen1 authentication (AZURE_TENANT_ID, AZURE_USERNAME, AZURE_PASSWORD, AZURE_STORE_NAME) are correctly set, or replace placeholders.")