soda-core-spark

import os from pyspark.sql import SparkSession from sodaspark import scan # Initialize Spark Session spark_session = SparkSession.builder.appName("SodaSparkExample").getOrCreate() # Create a sample DataFrame df = spark_session.createDataFrame([ {"id": "1", "name": "Alice", "age": 30}, {"id": "2", "name": "Bob", "age": None}, {"id": "3", "name": "Charlie", "age": 35}, {"id": "4", "name": "David", "age": 22} ]) # Define data quality checks in YAML format # For deprecated soda-spark, checks are passed as a string. # For modern Soda Core, these would typically be in a separate .yml file. scan_definition = """ table_name: my_dataframe metrics: - row_count - missing_count(age) - avg(age) checks: - row_count > 0 - missing_count(age) < 1 - avg(age) between 20 and 40 """ # Execute the scan # Note: data_source_name should be set if connecting to Soda Cloud, # but for local programmatic scans, it's often 'spark_df' by default. scan_results = scan.execute( data_frame=df, scan_definition=scan_definition, data_source_name="spark_df" # Can be customized ) print("Scan Results:") print(scan_results.get_json_representation()) # Stop Spark Session spark_session.stop() # IMPORTANT: This quickstart uses the deprecated `sodaspark` library. # For current Spark integration, please refer to Soda Core documentation and use # `from soda.scan import Scan` and `scan.add_spark_session(...)`.