dagster-pandas

import os import pandas as pd from dagster import asset, Definitions, load_assets_from_modules # --- defs/data/sample_data.csv --- # name,age,city # Alice,25,New York # Bob,35,San Francisco # Charlie,45,Chicago # Diana,28,Boston # --- defs/assets.py --- @asset def raw_data_csv() -> pd.DataFrame: # In a real scenario, this would read from a persistent store, e.g., S3 or a database # For quickstart, we simulate by creating a DataFrame data = { 'name': ['Alice', 'Bob', 'Charlie', 'Diana'], 'age': [25, 35, 45, 28], 'city': ['New York', 'San Francisco', 'Chicago', 'Boston'] } return pd.DataFrame(data) @asset def processed_data(raw_data_csv: pd.DataFrame) -> pd.DataFrame: return raw_data_csv[raw_data_csv['age'] > 30].copy() # --- definitions.py --- # Assuming assets.py is in a 'defs' directory or in the same file for quickstart all_assets = load_assets_from_modules([__name__]) # Load assets from this file defs = Definitions(assets=all_assets) # To run this, save as a Python file (e.g., my_project.py) and run: # dagster dev -f my_project.py # Then open http://localhost:3000 and materialize 'processed_data'