dask-kubernetes

from dask_kubernetes import KubeCluster from dask.distributed import Client # Ensure you have a kubectl context configured for a running Kubernetes cluster. # KubeCluster automatically detects the current context. # Create a Dask cluster on Kubernetes cluster = KubeCluster(name="my-dask-cluster", n_workers=3) print(f"Dashboard link: {cluster.dashboard_link}") # Connect a Dask client to the cluster client = Client(cluster) # Perform a simple computation def inc(x): return x + 1 def add(x, y): return x + y futures = client.map(inc, range(10)) total = client.submit(add, *futures) print(f"Result of computation: {total.result()}") # Scale the cluster (optional) cluster.scale(5) print(f"Cluster scaled to {len(cluster.workers)} workers.") # Close the client and cluster when done client.close() cluster.close() print("Dask cluster and client closed.")