openlineage-dbt

# 1. Install openlineage-dbt and your target dbt adapter (e.g., dbt-snowflake) # pip install openlineage-dbt dbt-snowflake # 2. Configure your dbt profiles.yml # (e.g., ~/.dbt/profiles.yml) # Add an OpenLineage profile that wraps your actual adapter: # my_openlineage_profile: # target: dev # outputs: # dev: # type: openlineage # adapter: snowflake # The actual dbt adapter you are using # account: <your_snowflake_account> # user: <your_snowflake_user> # password: <your_snowflake_password> # role: <your_snowflake_role> # warehouse: <your_snowflake_warehouse> # database: <your_snowflake_database> # schema: <your_snowflake_schema> # threads: 1 # 3. Set OpenLineage URL (and optionally API Key) as environment variables # or pass as CLI arguments. import os os.environ['OPENLINEAGE_URL'] = os.environ.get('OPENLINEAGE_URL', 'http://localhost:5000') # Or your Marque URL os.environ['OPENLINEAGE_API_KEY'] = os.environ.get('OPENLINEAGE_API_KEY', 'your_api_key_if_needed') # 4. Run dbt with the OpenLineage-enabled profile # Create a dummy dbt project structure for demonstration # (assuming you have a dbt project initialized, e.g., 'dbt init my_project') # For example, create models/example.sql in your dbt project: # -- models/example.sql # -- {{ config(materialized='table') }} # -- select 1 as id print(f"Running dbt with OpenLineage at {os.environ['OPENLINEAGE_URL']}...") # Example dbt command (execute from your dbt project root): # dbt build --profile my_openlineage_profile --target dev # Or, to ensure lineage is sent: # dbt build --profile my_openlineage_profile --target dev \ # --openlineage-url $OPENLINEAGE_URL \ # --openlineage-api-key $OPENLINEAGE_API_KEY print("Please execute the dbt command from your terminal to see lineage generation.") print("Example: dbt build --profile my_openlineage_profile --target dev")