dagster-databricks

import os from dagster import Definitions, job from dagster_databricks import databricks_resource, databricks_op # Configure Databricks resource using environment variables # DATABRICKS_HOST should be 'https://<workspace-url>.cloud.databricks.com' # DATABRICKS_TOKEN is your personal access token databricks_config = { "host": os.environ.get("DATABRICKS_HOST", ""), "token": os.environ.get("DATABRICKS_TOKEN", "") } # Create a configured Databricks resource configured_databricks_resource = databricks_resource.configured(databricks_config) @databricks_op( name="run_example_databricks_notebook", # These parameters map directly to the Databricks Jobs API 'notebook_task' # Replace with your actual notebook path and cluster configuration notebook_task={ "notebook_path": "/Users/your_user@example.com/my_dagster_notebook" }, new_cluster={ "spark_version": "12.2.x-scala2.12", "node_type_id": "i3.xlarge", "num_workers": 1 } ) def run_databricks_notebook(context): """An op to run a Databricks notebook job.""" context.log.info("Databricks notebook job submitted via Dagster.") @job def databricks_example_job(): run_databricks_notebook() # Define a repository using Definitions defs = Definitions( jobs=[databricks_example_job], resources={ "databricks_resource": configured_databricks_resource } ) # To run this example: # 1. Set DATABRICKS_HOST and DATABRICKS_TOKEN environment variables. # 2. Ensure '/Users/your_user@example.com/my_dagster_notebook' exists in your Databricks workspace. # 3. Save this code as a Python file (e.g., `databricks_repo.py`). # 4. Run `dagster dev -f databricks_repo.py` and launch the job from Dagit.