AQF 1.1__py3-none-any.whl

AQF/__init__.py

@@ -0,0 +1 @@
+import AQF.aqf_dataset_rules as aqf_dataset_rules, AQF.aqf_engine as aqf_engine, AQF.aqf_logging as aqf_logging, AQF.aqf_row_rules as aqf_row_rules, AQF.aqf_rule_retriever as aqf_rule_retriever, AQF.aqf_utils as aqf_utils

AQF/aqf_dataset_rules.py

@@ -0,0 +1,138 @@
+from pyspark.sql import DataFrame
+from .aqf_utils import create_spark, Status
+from pyspark.sql.functions import col
+from functools import reduce
+import operator
+
+def join_basic_inner_count_check(test, df: DataFrame, **kwargs) -> dict:
+    """
+    Validates referential integrity by performing inner join with lookup table.
+    
+    Checks if records in the source DataFrame have corresponding matches in a reference table.
+    This test ensures data consistency across related tables and identifies orphaned records.
+    
+    Args:
+        rule: Rule object containing test configuration with attributes:
+              - join_table: Target table name for join operation
+              - column_name: Source column for join condition
+              - join_column: Target column for join condition
+              - rule_id, test_type_id, description, table_name, bad_data_action
+        df (DataFrame): Source DataFrame to validate
+    
+    Returns:
+        (dict): Single-element list containing status code:
+                  [1] = PASS (all records have matches)
+                  [0] = FAIL (some records lack matches) 
+                  [2] = ERROR (exception occurred)
+    
+    Test Logic:
+        - Performs inner join between source and reference table
+        - Compares join result count with source record count
+        - PASS if join_count > 0, FAIL otherwise
+    """
+
+    spark = kwargs["spark"]
+
+    # Load reference table for join operation
+    df_join_target = spark.sql(f"SELECT * FROM {test.join_table}").alias("df_join_target")
+    df_comp = df.alias("df_comp")
+
+
+    # prepare string to join tables on all columns
+    columns = eval(test.columns)
+    join_columns = eval(test.join_column)
+
+    if len(columns) != len(join_columns):
+        raise Exception("column count doesn't match join column count")
+
+    # Build join conditions
+    join_condition = reduce(
+        operator.and_,
+        [col(f"df_comp.{c1}") == col(f"df_join_target.{c2}") for c1, c2 in zip(columns, join_columns)]
+    )
+
+    # Perform inner join
+    joined_df = df_comp.join(
+        df_join_target,
+        join_condition,
+        "inner"
+    )
+
+    # Calculate metrics for test evaluation
+    join_count = joined_df.count()       # Records with valid references
+    new_data_count = df_comp.count()          # Total source records
+    
+
+    # Prepare metrics for result logging
+    result_values = {
+        "status": Status.PASS if join_count > 0 else Status.WARNING,  # PASS if any matches found
+        "new_data_count": new_data_count,
+        "join_count": join_count,
+    }
+    return result_values
+
+#----------------------------------------------------------------------------------
+
+
+def unique_check(test, df: DataFrame) -> dict:
+    """
+    Validates data uniqueness by checking for duplicate values in specified column.
+    
+    Ensures primary key constraints and data integrity by identifying duplicate records.
+    Supports automatic deduplication based on configured bad data action.
+
+    !!! Removing/quarantining any data doesnt make sense here. We cant decide what the bad data is, based on only one column. !!!
+    
+    Args:
+        rule: Rule object containing test configuration with attributes:
+              - column_name: Target column for uniqueness validation0293
+              - bad_data_action: Action for duplicate records ('exclude', 'process')
+              - rule_id, test_type_id, description, table_name
+        df (DataFrame): Source DataFrame to validate
+    
+    Returns:
+        List[Union[int, DataFrame]]: Status code and optionally deduplicated DataFrame:
+                                   [status, clean_df] if deduplication applied
+                                   [status] if no filtering
+                                   
+                                   Status codes:
+                                   1 = PASS (all values unique)
+                                   0 = FAIL (duplicates found and handled)
+                                   2 = ERROR (exception occurred)
+    
+    Deduplication Logic:
+        - 'exclude': Remove duplicate records keeping first occurrence
+        - 'process': Allow duplicates to continue through pipeline
+        
+    Note:
+        Current implementation uses hardcoded 'id' column for deduplication.
+        Consider making this configurable via rule.column_name.
+    """
+    columns = eval(test.columns)
+    if not columns:
+        unique_count = df.distinct().count()
+    else:
+        unique_count = df.select(columns).distinct().count()  # Unique values in target column
+
+    # Calculate uniqueness metrics
+    total_count = df.count()  # Total records in DataFrame
+    
+    ""
+    if unique_count == total_count:
+        # Perfect uniqueness - all values are distinct
+        status = Status.PASS  # PASS
+        return_df = df
+    # What happens with quarantine? All duplicates should be moved to quarantine table
+    else:
+        status = Status.WARNING
+
+    # Prepare detailed metrics for analysis
+    result_values = {
+        "status": status,
+        "total_count": total_count,
+        "unique_count": unique_count,
+        "difference": total_count - unique_count  # Number of duplicate records
+    }
+
+    return result_values
+

AQF/aqf_engine.py

@@ -0,0 +1,377 @@
+from .aqf_utils import generate_run_id, create_spark, Status, normalize_timestamps
+from .aqf_rule_retriever import *
+from .aqf_dataset_rules import *
+from .aqf_row_rules import *
+from .aqf_logging import result_writer, bad_data_writer
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import max
+from functools import reduce
+from datetime import datetime, timezone
+
+import notebookutils
+
+
+class AQF_Engine():
+
+    """
+    Class that manages the DQF testing procedure
+
+    Args:
+        run_id: id string to make every test run unique
+        job_id: id for every hsi session?
+        config: name of the variable in the workspace
+        spark: Optional SparkSession to use. If not provided, the active session is used.
+    
+    """
+
+    def __init__(
+        self,
+        job_id: str,
+        run_id: str,
+        config: str,
+        spark: SparkSession | None = None
+        ):
+        self.spark = create_spark() if spark is None else spark
+        self.run_id = run_id
+
+        # list of the outcomes of each test
+        self.status_list = []
+        
+        # env variables
+        self.config = notebookutils.variableLibrary.getLibrary(config)
+        
+        # represents the tested and updated dataframe
+        self.consolidated_df = None
+
+        self.cancel = False
+    
+
+    def run_tests(
+        self,
+        table_path: str,
+        table_name: str,
+        df: DataFrame,
+        check_type: str | None = None,
+        spark: SparkSession | None = None,
+        ) -> DataFrame:
+        """
+        Main orchestration function for the Data Quality Framework (DQF).
+        
+        Executes all configured data quality tests for a given table and consolidates results.
+        This function retrieves rules, executes corresponding tests, and tracks success/failure rates.
+        
+        Args:
+            config_id: 
+            table_name: string for the table name, only used for quaratine table name
+            table_path:
+            df (DataFrame): Input Spark DataFrame to validate
+            check_type: ?
+
+            
+        Returns:
+            DataFrame: Consolidated DataFrame containing only records that passed all tests
+                    (when bad_data_action is 'quarantine' or 'delete')
+        
+        Notes:
+            - Test results are automatically logged to dbo.dqf.dqf_test_log table
+            - Supports multiple bad data actions: 'process', 'delete', 'quarantine', ignore
+            - Generates unique run_id for tracking test execution sessions
+        
+        Example:
+           >>> clean_df = engine.run_tests(test_table="dqf_tests", df=df, table_path="lh_bronze", table_name="bronze_hsi", check_type="")
+        """
+
+        if spark:
+            self.spark = spark
+        elif not self.spark:
+            self.spark = create_spark()
+
+        # table (str): Fully qualified table name (e.g., 'catalog.schema.table')
+        #        Used to retrieve applicable DQ rules from metadata table
+        self.table_id = table_path + "." + table_name
+        self.table_name = table_name
+
+        # clean df for return after tests are done
+        self.consolidated_df = df
+
+        print(f"Starting Data Quality checks for {self.table_id}")
+        
+        # Retrieve all configured DQ rules for the specified table
+        df_tests = retrieve_aqf_tests_by_table_id(table_id=self.table_id, test_table=self.config.test_table, spark=self.spark, jdbc_url=self.config.jdbc_url)
+        tests = df_tests.collect()
+        number_of_tests = len(tests)
+        if not number_of_tests:
+            print(f"No tests found for {self.table_id}")
+            return df
+
+        test_list = df_tests.toPandas()["rule_id"].tolist()
+        rules = retrieve_aqf_rules_by_id(test_list, spark=self.spark, jdbc_url=self.config.jdbc_url, rule_table=self.config.rule_table)
+
+        # Execute each rule sequentially
+        for test in tests:
+            print(test)
+            rule = rules.filter(rules.rule_id.isin([test.rule_id])).collect()
+            self.testing(test=test, df=df, rule=rule[0])
+            
+            if self.status_list[-1] == Status.FAIL:
+                self.cancel = True
+
+        self.output_results(number_of_tests)
+        
+        return self.consolidated_df
+    
+
+
+    def consolidate(self, new_df: DataFrame):
+        """
+        Takes the dataframe that just got reduced by some row level test and removes the missing rows also from the consolidated df
+
+        Args:
+            new_df: the dataframe that got changed by a test
+        """
+        # Verwende INTERSECT statt UNION
+        self.consolidated_df = self.consolidated_df.intersect(new_df)
+        #self.consolidated_df.show()
+
+
+    def output_results(self, number_of_tests: int):
+        """
+        Might get scrapped or changed to something else since output to notebook doesnt make sense in a regular environment
+        
+        Evaluates how the test session went
+
+        Args:
+            number_of_tests: number of tests is inferred from the test retrievel at the start
+        """
+        # Calculate final test statistics
+        incomplete_tests = self.status_list.count(Status.ERROR)  # ERROR status
+        passed_tests = self.status_list.count(Status.PASS)       # PASS status
+        failed_tests = self.status_list.count(Status.FAIL)      # FAIL status
+        warnings = self.status_list.count(Status.WARNING)
+        #canceled = number_of_tests - len(self.status_list)
+
+        # Summary output for monitoring (minimal logging)
+        print(f"AQF Results - Tests: {number_of_tests} | Passed: {passed_tests} | Failed: {failed_tests+warnings} | Errors: {incomplete_tests}")
+
+
+    def get_kwargs(self, connection: str) -> dict:
+        """
+        Some rules require an additional table so this puts the connections into the extra parameters
+        """
+        con = eval(connection)
+        kwargs = {}
+        for c in con:
+            match c:
+                case "spark":
+                    kwargs["spark"] = self.spark
+                case "sql":
+                    kwargs["jdbc_url"] = self.config.jdbc_url
+        return kwargs
+    
+    def row_level(self, test: dict, df: DataFrame, rule: dict, **kwargs) -> dict:
+        """
+        this is only for tests that are row based
+        checks each cell in a column with a value
+        reference value str optional
+
+        Args:
+            test: dict with the test info
+            df: Dataframe to evaluate
+            rule: dict with the rule info
+
+        Returns:
+            logging data
+        """
+        # results = (result_values, good_df, bad_df)
+        results = eval(rule.name)(test=test, df=df, **kwargs)
+
+        # Write bad data to quarantine table for later analysis
+        # Table naming: utility.bad_data.{source_table_name}
+        if results[2]:
+            bad_data_writer(table_name=self.table_name, df=results[2], run_id=self.run_id, spark=self.spark, jdbc_url=self.config.jdbc_url, quarantine_table=self.config.quarantine_table)
+            if results[1]:
+                self.consolidate(results[1])
+
+        return results[0]
+
+
+    def testing(self, test: dict, df: DataFrame, rule: dict):
+        """
+        This manages a single test
+        It runs it and logs it
+
+        Args:
+            test: dict with the test info
+            df: Dataframe to evaluate
+            rule: dict with the rule info
+        """
+
+        start_time = datetime.now(timezone.utc)
+        start_time.strftime("%Y-%m-%d %H:%M:%S.%f")
+        try:
+            result_writer(
+                    log_path = self.config.log_table,
+                    run_id = self.run_id,
+                    test_id = test.test_id,
+                    rule_id = test.rule_id,
+                    description = test.description,
+                    table_id = test.table_id,
+                    start_time = start_time,
+                    result_values = {},
+                    bad_data_action = test.bad_data_action,
+                    criticality = test.criticality,
+                    spark = self.spark,
+                    jdbc_url = self.config.jdbc_url
+            )
+            print("logging (start) successful")
+        except:
+            pass
+        
+
+        kwargs = self.get_kwargs(rule.connection)
+
+        try:
+            # result: result_values
+            if rule.rule_type == "dataset":
+                # dataset level
+                result_values = eval(rule.name)(test, df, **kwargs)
+            elif rule.rule_type == "row":
+                # row level
+                result_values = self.row_level(test=test, df=df, rule=rule, **kwargs)
+            else:
+                raise Exception(f"Invalid rule type {rule.rule_type}")
+            #print(results)
+            if result_values["status"] == Status.WARNING and test.criticality:
+                result_values["status"] = Status.FAIL
+            
+            self.status_list.append(result_values["status"])
+            end_time = datetime.now(timezone.utc)
+            end_time.strftime("%Y-%m-%d %H:%M:%S.%f")
+            error_message = None
+
+            print(f"Test successful: {test.test_id}")
+        except Exception as e:
+            # Handle unexpected errors during test execution
+            print(f"Testing (id: {test.test_id}) failed: {e}")
+            error_message = str(e)
+            result_values = {"status": Status.ERROR}
+            end_time = datetime.now(timezone.utc)
+            end_time.strftime("%Y-%m-%d %H:%M:%S.%f")
+            self.status_list.append(Status.ERROR)
+
+        try:
+            result_writer(
+                    log_path = self.config.log_table,
+                    run_id = self.run_id,
+                    test_id = test.test_id,
+                    rule_id = test.rule_id,
+                    description = test.description,
+                    table_id = test.table_id,
+                    start_time = start_time,
+                    end_time = end_time,
+                    result_values = result_values,
+                    error_message = error_message,
+                    bad_data_action = test.bad_data_action,
+                    criticality = test.criticality,
+                    spark = self.spark,
+                    jdbc_url = self.config.jdbc_url
+            )
+            print("Logging successful")
+        except Exception as e:
+            print(f"Logging failed: {e}")
+        
+        
+    def is_critical(self):
+        """
+        Returns wether a critical test failed
+        """
+        return self.cancel
+
+    def get_fail_count(self):
+        return self.status_list.count(Status.FAIL)
+
+    def get_log_table(self):
+        spark = self.spark
+        return spark.read.option("url", self.config.jdbc_url).mssql(self.config.log_table)
+
+def create_test(
+    jdbc_url: str,
+    test_table: str,
+    rule_table: str,
+    rule_id: int,
+    stage: str,
+    table_id: str,
+    desc: str = None,
+    columns: list[str] = None,
+    expression: str = None,
+    join_table: str = None,
+    join_column: str = None,
+    bad_data_action: str = "process", # in case no bad data action is given for row based test
+    citicality: bool = False, # in case no criticality is given
+    spark: SparkSession | None = None
+    ) -> int:
+    """
+    Create an entry in the test table with all necessary values
+    """
+    spark = create_spark() if not spark else spark
+
+    # create new test_id
+    tests = spark.read.option("url", jdbc_url).mssql(test_table)
+    test_id = tests.select(max(tests.test_id).alias("test_id")).collect()[0][0]+1
+    
+    # check rule_id validity
+    rules = spark.read.option("url", jdbc_url).mssql(rule_table)
+    if not rules.filter(rules.rule_id == rule_id):
+        raise Exception(f"rule {rule_id} doesn't exist")
+        return -1
+    
+    #check for table
+    try:
+        df = spark.read.format("delta").load(table_id)
+    except:
+        try:
+            query = f"""SELECT * FROM {table_id}"""
+            df = spark.sql(query)
+        except:
+            try:
+                ws = notebookutils.runtime.context.get("currentWorkspaceName")
+                query = f"""SELECT * FROM {ws}.{table_id}"""
+                df = spark.sql(query)
+            except:
+                raise Exception(f"table {table} doesn't exist")
+                return -1
+
+    # create table
+    schema = StructType([
+        StructField("test_id",         LongType(), True),
+        StructField("rule_id",         LongType(), True),
+        StructField("description",     StringType(),  True),
+        StructField("stage",           StringType(),  True),
+        StructField("table_id",        StringType(),  True),
+        StructField("columns",         StringType(),  True),
+        StructField("expression",      StringType(),  True),
+        StructField("join_table",      StringType(),  True),
+        StructField("join_column",     StringType(),  True),
+        StructField("bad_data_action", StringType(),  True),
+        StructField("criticality",     BooleanType(),  True),
+    ])
+
+    data = (
+        test_id,
+        rule_id,
+        desc,
+        stage,
+        table_id,
+        columns,
+        expression,
+        join_table,
+        join_columns,
+        bad_data_action,
+        criticality
+    )
+
+    new_test = spark.createDataframe(data=[data], schema=schema)
+    tests.write.option("url", jdbc_url).mode("append").mssql(test_table)
+
+    return test_id

AQF/aqf_logging.py

@@ -0,0 +1,192 @@
+from datetime import datetime, timezone
+import pandas as pd
+import json
+import ast
+
+from pyspark.sql.functions import lit
+from pyspark.sql.types import StructType, StructField, StringType, LongType, BooleanType, TimestampType
+from typing import Optional
+from pyspark.sql import DataFrame, SparkSession, Row
+import notebookutils
+from .aqf_utils import create_spark, Status, normalize_timestamps
+
+
+
+def result_writer(
+    log_path: str,
+    run_id: str,
+    test_id: int,
+    rule_id: int,
+    description: str, 
+    table_id: str, 
+    start_time: datetime, 
+    criticality: bool,
+    bad_data_action: str, 
+    result_values,
+    jdbc_url: str,
+    spark: SparkSession,
+    end_time: Optional[datetime] = None,
+    error_message: Optional[str] = None, 
+    job_run_id: int = 0
+    
+) -> None:
+    
+    """
+    Persists data quality test results to the DQF audit trail.
+    
+    Records comprehensive test execution metadata including performance metrics,
+    test outcomes, and error details for monitoring and compliance reporting.
+    
+    Args:
+        run_id: unique identifier for each test session
+        test_id (int): Unique identifier of the executed test
+        rule_id (int): Unique identifier of the executed rule
+        description (str): Human-readable description of the test
+        table (str): name of the tested table
+        start_time (datetime): Test execution start timestamp (UTC)
+        end_time (datetime): Test execution completion timestamp (UTC)
+        criticality: what if a test fails
+        bad_data_action (Optional[str]): Action taken for failed records
+                                        ('process', 'delete', 'ignore', 'quarantine')
+        result_values (Dict): Test-specific metrics and measurements
+                             Examples: {"status": Status.PASS, "null_count": 5, "total_count": 1000}
+        error_message (Optional[str]): Exception details if status=ERROR
+        job_id (int): Job run identifier for grouping related tests
+                     Defaults to 0 if not specified
+    
+    Result Table Schema:
+        - run_id: Auto-incrementing primary key per test execution
+        - job_id: Groups related tests in the same batch run
+        - Execution metadata: rule_id, test_type_id, description, config_id
+        - Timing data: start_time, end_time (for performance analysis)
+        - Outcome data: status, result_values, error_message
+        - Action taken: bad_data_action
+    
+    Usage:
+        Called automatically by test functions in test_engine.py to ensure
+        comprehensive audit trail of all DQ validations.
+    
+    Note:
+        Uses string interpolation in SQL. Consider JSON serialization
+        for result_values to handle complex data types safely.
+    """
+
+
+    #insert optional values that are dependend on the test
+    if "new_data_count" in result_values:
+        new_data_count = result_values["new_data_count"]
+    else:
+        new_data_count = None
+    
+    if "rows_test_failed" in result_values:
+        rows_test_failed = result_values["rows_test_failed"]
+    else:
+        rows_test_failed = None
+    
+    if "rows_test_passed" in result_values:
+        rows_test_passed = result_values["rows_test_passed"]
+    else:
+        rows_test_passed = None
+    
+
+    #start_time = start_time.astimezone(timezone.utc).replace(tzinfo=None)
+    #end_time = end_time.astimezone(timezone.utc).replace(tzinfo=None)
+
+    if not end_time:
+        status = "running"
+    else:
+        status = result_values["status"].as_name()
+    
+    # Dictionary für DataFrame
+    df_dqf_test_log = (
+        run_id,                         # Unique identifier for each test engine run
+        job_run_id,                 # Batch execution identifier
+        test_id,                       # Reference to rule configuration
+        rule_id,             # Test function type for categorization
+        criticality,               
+        description,               # Human-readable test description
+        table_id,                 # Target table for auditing 
+        start_time,                 # Execution start for performance tracking
+        end_time,                     # Execution end for duration calculation
+        status,                         # Test outcome (PASS/FAIL/ERROR)
+        json.dumps(result_values),      # Test-specific metrics as VARIANT (STRUCT, ARRAY, "JSON" etc.)    
+        new_data_count,
+        rows_test_failed,
+        rows_test_passed, 
+        bad_data_action,       # Action taken for quality violations
+        error_message            # Exception details for troubleshooting
+    )
+    #print(df_dqf_test_log)
+    #print(type(run_id))
+    schema = StructType([
+        StructField('run_id', StringType(), True), 
+        StructField('job_run_id', LongType(), True), 
+        StructField('test_id', LongType(), True), 
+        StructField('rule_id', LongType(), True), 
+        StructField('criticality', BooleanType(), True), 
+        StructField('description', StringType(), True), 
+        StructField('table_id', StringType(), True), 
+        StructField('start_time', TimestampType(), True), 
+        StructField('end_time', TimestampType(), True), 
+        StructField('status', StringType(), True), 
+        StructField('result_values', StringType(), True), 
+        StructField('new_data_count', LongType(), True), 
+        StructField('rows_test_failed', LongType(), True), 
+        StructField('rows_test_passed', LongType(), True), 
+        StructField('bad_data_action', StringType(), True), 
+        StructField('error_message', StringType(), True)
+    ])
+
+
+    #old = spark.read.option("url", jdbc_url).mssql("dbo.dqf_test_log")
+
+    log_df = spark.createDataFrame(data=[df_dqf_test_log], schema=schema)
+    
+    log_df = normalize_timestamps(log_df)
+
+    log_df.write.option("url", jdbc_url).mode("append").mssql(f"dbo.{log_path}")
+
+
+def bad_data_writer(table_name: str, df: DataFrame, run_id: str, spark: SparkSession, jdbc_url: str, quarantine_table: str) -> None:
+    """
+    Persists data quality test results to the DQF audit trail.
+    
+    Records comprehensive test execution metadata including performance metrics,
+    test outcomes, and error details for monitoring and compliance reporting.
+    
+    Args:
+        table_name (str): name for the quarantine table
+        df (DataFrame): DataFrame containing the test results
+        run_id (str): Unique identifier for the test run
+        spark: Sparksession
+        jdbc_url: connections string to sql db
+        quarantine_table: path to quarantine lakehouse
+    
+    Result Table Schema:
+        - run_id: Unique identifier per test execution
+        - Additional columns from the input DataFrame
+    
+    Usage:
+        Called automatically by row test functions if they fail to ensure
+        comprehensive audit trail of all DQ validations.
+    """
+    
+    # Insert test execution record into audit table
+    # All test executions are logged regardless of outcome for compliance
+    df = df.withColumn("run_id", lit(run_id))
+
+    
+    try:
+        # date table
+        ddl_error_table = (
+        f"""CREATE TABLE IF NOT EXISTS {quarantine_table}.{table_name}""")
+        spark.sql(ddl_error_table)
+    
+        df.write.mode("append").option("mergeSchema", "true").saveAsTable(f"{quarantine_table}.{table_name}")
+    except:
+        ws = notebookutils.runtime.context.get("currentWorkspaceName")
+        ddl_error_table = (
+        f"""CREATE TABLE IF NOT EXISTS {ws}.{quarantine_table}.dbo.{table_name}""")
+        spark.sql(ddl_error_table)
+    
+        df.write.mode("append").option("mergeSchema", "true").saveAsTable(f"{ws}.{quarantine_table}.dbo.{table_name}")

AQF/aqf_row_rules.py

@@ -0,0 +1,168 @@
+from pyspark.sql import DataFrame
+from .aqf_utils import Status
+from datetime import datetime, timezone
+
+def null_check(test: dict, df: DataFrame) -> (dict, DataFrame, DataFrame):
+    """
+    Validates data completeness by checking for NULL values in specified column.
+    
+    Identifies and optionally quarantines records with NULL values in critical fields.
+    Supports data quality enforcement through configurable bad data actions.
+    
+    Args:
+        test: Test object containing test configuration with attributes:
+              - column_name: Target column for NULL validation
+              - bad_data_action: Action for NULL records ('quarantine', 'exclude', 'process')
+              - rule_id, test_type_id, description, table_name
+        df (DataFrame): Source DataFrame to validate
+    
+    Returns:
+       Tuple(
+          result_values: status and additional infos for logging
+          return_df: df that has the failed rows removed
+          df_bad_data: df that has the failed rows
+       )
+    
+    Bad Data Actions:
+        - 'quarantine': Move NULL records to bad_data table, continue with clean data
+        - 'delete': Remove NULL records from processing pipeline
+        - 'ignore': Allow NULL records to continue (no filtering)
+        - 'process': Copy NULL records to bad_data_table, dont remove bda data from df
+
+
+    """
+    # create df with only null values
+    df_bad_data = df.filter(df[eval(test.columns)[0]].isNull())
+    null_count = df_bad_data.count()
+
+    if not df_bad_data.count():
+        # Perfect data quality - no NULLs detected
+        status = Status.PASS  # PASS
+        return_df = None
+    else:
+        match test.bad_data_action:
+            case "quarantine" | "delete":
+                return_df = df.filter(df[eval(test.columns)[0]].isNotNull())
+            case _: # ignore, process
+                return_df = df
+
+        status = Status.WARNING  # FAIL (but handled appropriately)
+    
+    if return_df:
+        new_data_count = return_df.count()
+    else:
+        new_data_count = 0
+    
+    # Prepare comprehensive metrics for analysis
+    result_values = {
+        "status": status,
+        "new_data_count": new_data_count,
+        "null_count": null_count,
+        "none_null_count": df.count() - null_count
+    }
+    return (result_values, return_df, df_bad_data)
+
+# ---------------------------------------------------------------------------------------
+
+def compare(test: dict, df: DataFrame):
+    """
+    Validates data by comparing a value to a given expression
+
+    Args:
+        test: information about the test
+        df: dataframe to validate
+    
+    Returns:
+       Tuple(
+          result_values: status and additional infos for logging
+          return_df: df that has the failed rows removed
+          df_bad_data: df that has the failed rows
+       )
+    Bad Data Actions:
+        - 'quarantine': Move NULL records to bad_data table, continue with clean data
+        - 'delete': Remove NULL records from processing pipeline
+        - 'ignore': Allow NULL records to continue (no filtering)
+        - 'process': Copy NULL records to bad_data_table, dont remove bda data from df
+    """
+    comp = ""
+    comp += "df[eval(test.columns)[0]]" + test.expression
+
+    df_bad_data = df.filter(~eval(comp))
+    fail_count = df_bad_data.count()
+
+    if not df_bad_data.count():
+        # Perfect data quality - no NULLs detected
+        status = Status.PASS  # PASS
+        return_df = None
+    else:
+        match test.bad_data_action:
+            case "quarantine" | "delete":
+                return_df = df.filter(eval(comp))
+            case _: # ignore, process
+                return_df = df
+
+        status = Status.WARNING  # FAIL (but handled appropriately)
+    
+    if return_df:
+        new_data_count = return_df.count()
+    else:
+        new_data_count = 0
+
+    # Prepare comprehensive metrics for analysis
+    result_values = {
+        "status": status,
+        "new_data_count": new_data_count,
+        "fail_count": fail_count,
+        "pass_count": df.count() - fail_count
+    }
+    return (result_values, return_df, df_bad_data)
+
+# -------------------------------------------------------------------------------------
+
+def is_not_in_future(test: dict, df: DataFrame):
+    """
+    Validates data by checking if a date or timestamp is a valid past date.
+
+    Args:
+        test: information about the test
+        df: dataframe to validate
+
+    Return:
+        Tuple(
+          result_values: status and additional infos for logging
+          return_df: df that has the failed rows removed
+          df_bad_data: df that has the failed rows
+       )
+    Bad Data Actions:
+        - 'quarantine': Move NULL records to bad_data table, continue with clean data
+        - 'delete': Remove NULL records from processing pipeline
+        - 'ignore': Allow NULL records to continue (no filtering)
+        - 'process': Copy NULL records to bad_data_table, dont remove bda data from df
+    """
+    now = datetime.now(timezone.utc)
+
+    df_bad_data = df.filter(df[eval(test.columns)[0]] > now)
+    fail_count = df_bad_data.count()
+
+    if not df_bad_data.count():
+        # Perfect data quality - no NULLs detected
+        status = Status.PASS  # PASS
+    else:
+        match test.bad_data_action:
+            case "quarantine" | "delete":
+                return_df = df.filter(df[eval(test.columns)[0]] <= now)
+            case _: # ignore, process
+                return_df = df
+
+        status = Status.WARNING  # FAIL (but handled appropriately)
+    
+    new_data_count = return_df.count()
+
+    # Prepare comprehensive metrics for analysis
+    result_values = {
+        "status": status,
+        "new_data_count": new_data_count,
+        "fail_count": fail_count,
+        "pass_count": df.count() - fail_count
+    }
+    return (result_values, return_df, df_bad_data)

AQF/aqf_rule_retriever.py

@@ -0,0 +1,97 @@
+from pyspark.sql import DataFrame, SparkSession
+from .aqf_utils import create_spark
+import com.microsoft.sqlserver.jdbc.spark
+from pyspark.sql.types import StructType
+
+def retrieve_aqf_tests_by_table_id(table_id: str, test_table: str, spark: SparkSession, jdbc_url: str) -> DataFrame:
+    """
+    Retrieves all configured Data Quality Framework rules for a specific table.
+    
+    Queries the DQF metadata repository to fetch test configurations associated
+    with the specified table. This enables dynamic test execution based on
+    stored rule definitions rather than hardcoded test logic.
+    
+    Args:
+        config_id (int): id
+        test_table: Fully qualified table name (e.g., 'catalog.schema.table')
+        spark: Sparksession
+        jdbc_url: connections string to the sql db of the test table
+        
+    
+    Returns:
+        DataFrame: Spark DataFrame containing rule configurations with columns:
+                  - test_id: Unique identifier for the test
+                  - rule_id: ID for the rule
+                  - description: Human-readable test description
+                  - table: Target table for validation
+                  - columns: Target columns
+                  - expression: Join strategy (inner/left/right) for referential tests or comparison based expression ("<10")
+                  - join_table: Reference table for join operations
+                  - join_column: Reference column for join operations
+                  - bad_data_action: Action for failed validations ('process'/'exclude'/'quarantine')
+                  - criticality: action in case the test fails
+    
+    Usage:
+        Rules are typically configured via rule_writer.py and stored in utility.dqf.dqf_rules.
+        The engine uses this function to discover applicable tests at runtime.
+    
+    Example:
+        >>> rules_df = retrieve_dqf_rules_by_table_name('bronze.sales.orders')
+        >>> rules_list = rules_df.collect()  # Convert to list for iteration
+    """
+    # Query DQF metadata table for rules matching the specified table
+    # Selects all rule configuration fields needed for test execution
+    all_tests = spark.read.option("url", jdbc_url).mssql(f"dbo.{test_table}")
+    all_tests.createOrReplaceTempView("pdf")
+    tests_sql = f"""
+        SELECT   
+            test_id,           -- Unique test identifier
+            rule_id,           -- Maps to rule
+            description,       -- Human-readable test description,
+            stage,
+            table_id,          -- full Target table name
+            columns,           -- Target columns (nullable for table-level tests)
+            expression,        -- Join strategy (inner/left/right) for referential tests or comparison based expression ("<10")
+            join_table,        -- Reference table for join operations
+            join_column,       -- Reference column for join operations
+            bad_data_action,   -- Bad data handling strategy
+            criticality
+        FROM pdf 
+        WHERE table_id = '{table_id}'
+        ORDER BY rule_id      -- Ensure consistent execution order
+    """
+    tests = spark.sql(tests_sql)
+
+    if tests.rdd.isEmpty():
+        print(f"Warning: No DQF tests for table '{table_id}' found.")
+        
+    print("Retrieving of tests successful")
+    return tests
+
+
+def retrieve_aqf_rules_by_id(id_list, spark: SparkSession, jdbc_url: str, rule_table: str) -> DataFrame:
+    """
+    retrieves the rules based on their id
+
+    Args:
+        test_table: Fully qualified table name (e.g., 'catalog.schema.table')
+        spark: Sparksession
+        jdbc_url: connections string to the sql db of the test table
+
+    Returns:
+        Dataframe with the info for the rules
+    """
+
+    try:
+        all_rules = spark.read.option("url", jdbc_url).mssql(f"dbo.{rule_table}")
+        rules = all_rules.filter(all_rules.rule_id.isin(id_list))
+
+        if rules.rdd.isEmpty():
+            return spark.createDataFrame([], schema=["rule_id", "rule_type_id", "reference_type_id", "name", "descripton", "connection"])
+        
+        print("Retrieving of rules successful")
+        return rules
+
+    except Exception as e:
+        print(f"Error retrieving rules: {e}")
+        return spark.createDataFrame(data=[], schema=["rule_id", "rule_type_id", "reference_type_id", "name", "descripton", "connection"])

AQF/aqf_utils.py

@@ -0,0 +1,74 @@
+import uuid
+from pyspark.sql import SparkSession
+from enum import IntEnum
+from pyspark.sql import functions as F
+
+
+def generate_run_id() -> str:
+    """
+    Generate a globally unique identifier for DQF test execution sessions.
+    
+    Creates a UUID1 string that uniquely identifies each invocation of the
+    DQF engine. This enables tracking and correlation of test results across
+    related validations within the same processing batch.
+    
+    Returns:
+        str: UUID1 string in format 'xxxxxxxx-xxxx-1xxx-yxxx-xxxxxxxxxxxx'
+             Example: '550e8400-e29b-11d4-a716-446655440000'
+    
+    Usage:
+        Used by engine.py to tag all test executions within a single run,
+        enabling batch-level reporting and correlation of results.
+    
+    Benefits:
+        - Enables grouping of related test executions
+        - Supports distributed processing scenarios
+        - Provides audit trail for compliance reporting
+        - Facilitates troubleshooting of failed validation runs
+    
+    Example:
+        >>> run_id = generate_run_id()
+        >>> print(f"Starting DQF execution with run_id: {run_id}")
+    """
+    return str(uuid.uuid1())
+
+def create_spark():
+    """
+    creates spark session if not directly run in notebook
+    """
+    return SparkSession.builder.getOrCreate()
+
+def normalize_timestamps(df):
+    """
+    helper to fix timestamp issues
+    """
+    for col, dtype in df.dtypes:
+        if dtype == "timestamp":
+            df = df.withColumn(
+                col,
+                F.to_timestamp(F.date_format(F.col(col), "yyyy-MM-dd HH:mm:ss"))
+            )
+    return df
+
+
+class Status(IntEnum):
+    """
+    User friendly enum for status type
+    """
+    PASS = 0
+    WARNING = 1
+    FAIL = 2
+    ERROR = 3
+
+    def as_name(self):
+        match self:
+            case Status.PASS:
+                return "pass"
+            case Status.WARNING:
+                return "warning"
+            case Status.FAIL:
+                return "fail"
+            case Status.ERROR:
+                return "error"
+            case _:
+                "no status"

aqf-1.1.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: AQF
+Version: 1.1
+Summary: Data Quality Framework for Spark-based pipelines
+Author: qjener
+Author-email: dasxxq@gmail.com
+Dynamic: author
+Dynamic: author-email
+Dynamic: summary

aqf-1.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+AQF/__init__.py,sha256=_wKo_YyN6sPfUK8RUyLKZ-x5jZp-Nki8mxXh9DMAKKU,221
+AQF/aqf_dataset_rules.py,sha256=ATZTgap8FOXqcYbKoUndRK9mqD8v2c3xXYZwOlNVK8I,5224
+AQF/aqf_engine.py,sha256=3Se7x1HyZhiGpC9tIV1iDcGtPnTnGlbQPsOzlhz7fmA,13369
+AQF/aqf_logging.py,sha256=HUCDQ5ddOM3VmeQvl94VD8zTW3HxZkN2dz-Bh0UB_Sg,7672
+AQF/aqf_row_rules.py,sha256=h4jtIqSOZ8UXgoIJcHFKx11tRSoroDLSPFnZMfXsp8k,5848
+AQF/aqf_rule_retriever.py,sha256=6hqQukdWXdcBdCXkw4hqQBDXCE0os7AdsSe0MGMyWkE,4338
+AQF/aqf_utils.py,sha256=QrIC6hM2XrUI4b0QcsdlF9wqaSSNEx5KzOagNrctFyI,2086
+aqf-1.1.dist-info/METADATA,sha256=KlhD8zP1gE3Mqn2SMqt5TLbZ3wQYeIGRadwOuLbCOnw,204
+aqf-1.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+aqf-1.1.dist-info/top_level.txt,sha256=kd_ZM7nabvEJmgyyqllhL26UZV7OC_PxvShFSjS_XvQ,4
+aqf-1.1.dist-info/RECORD,,

aqf-1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

aqf-1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+AQF