databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/reconcile/recon_capture.py

@@ -0,0 +1,635 @@
+import logging
+from datetime import datetime
+from functools import reduce
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import col, collect_list, create_map, lit
+from pyspark.sql.types import StringType, StructField, StructType
+from pyspark.errors import PySparkException
+from sqlglot import Dialect
+
+from databricks.labs.lakebridge.config import DatabaseConfig, Table, ReconcileMetadataConfig
+from databricks.labs.lakebridge.reconcile.recon_config import TableThresholds
+from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_key_from_dialect
+from databricks.labs.lakebridge.reconcile.exception import (
+    WriteToTableException,
+    ReadAndWriteWithVolumeException,
+    CleanFromVolumeException,
+)
+from databricks.labs.lakebridge.reconcile.recon_output_config import (
+    DataReconcileOutput,
+    ReconcileOutput,
+    ReconcileProcessDuration,
+    ReconcileTableOutput,
+    SchemaReconcileOutput,
+    StatusOutput,
+    ReconcileRecordCount,
+    AggregateQueryOutput,
+)
+from databricks.sdk import WorkspaceClient
+
+logger = logging.getLogger(__name__)
+
+_RECON_TABLE_NAME = "main"
+_RECON_METRICS_TABLE_NAME = "metrics"
+_RECON_DETAILS_TABLE_NAME = "details"
+_RECON_AGGREGATE_RULES_TABLE_NAME = "aggregate_rules"
+_RECON_AGGREGATE_METRICS_TABLE_NAME = "aggregate_metrics"
+_RECON_AGGREGATE_DETAILS_TABLE_NAME = "aggregate_details"
+
+
+class ReconIntermediatePersist:
+
+    def __init__(self, spark: SparkSession, path: str):
+        self.spark = spark
+        self.path = path
+
+    def _write_unmatched_df_to_volumes(
+        self,
+        unmatched_df: DataFrame,
+    ) -> None:
+        unmatched_df.write.format("parquet").mode("overwrite").save(self.path)
+
+    def _read_unmatched_df_from_volumes(self) -> DataFrame:
+        return self.spark.read.format("parquet").load(self.path)
+
+    def clean_unmatched_df_from_volume(self):
+        try:
+            # TODO: for now we are overwriting the intermediate cache path. We should delete the volume in future
+            # workspace_client.dbfs.get_status(path)
+            # workspace_client.dbfs.delete(path, recursive=True)
+            empty_df = self.spark.createDataFrame([], schema=StructType([StructField("empty", StringType(), True)]))
+            empty_df.write.format("parquet").mode("overwrite").save(self.path)
+            logger.warning(f"Unmatched DF cleaned up from {self.path} successfully.")
+        except PySparkException as e:
+            message = f"Error cleaning up unmatched DF from {self.path} volumes --> {e}"
+            logger.error(message)
+            raise CleanFromVolumeException(message) from e
+
+    def write_and_read_unmatched_df_with_volumes(
+        self,
+        unmatched_df: DataFrame,
+    ) -> DataFrame:
+        try:
+            self._write_unmatched_df_to_volumes(unmatched_df)
+            return self._read_unmatched_df_from_volumes()
+        except PySparkException as e:
+            message = f"Exception in reading or writing unmatched DF with volumes {self.path} --> {e}"
+            logger.error(message)
+            raise ReadAndWriteWithVolumeException(message) from e
+
+
+def _write_df_to_delta(df: DataFrame, table_name: str, mode="append"):
+    try:
+        df.write.mode(mode).saveAsTable(table_name)
+        logger.info(f"Data written to {table_name} successfully.")
+    except Exception as e:
+        message = f"Error writing data to {table_name}: {e}"
+        logger.error(message)
+        raise WriteToTableException(message) from e
+
+
+def generate_final_reconcile_output(
+    recon_id: str,
+    spark: SparkSession,
+    metadata_config: ReconcileMetadataConfig = ReconcileMetadataConfig(),
+    local_test_run: bool = False,
+) -> ReconcileOutput:
+    _db_prefix = "default" if local_test_run else f"{metadata_config.catalog}.{metadata_config.schema}"
+    recon_df = spark.sql(
+        f"""
+    SELECT
+    CASE
+        WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME)
+        ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME)
+    END AS SOURCE_TABLE,
+    CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS TARGET_TABLE,
+    CASE WHEN lower(MAIN.report_type) in ('all', 'row', 'data') THEN
+    CASE
+        WHEN METRICS.recon_metrics.row_comparison.missing_in_source = 0 AND METRICS.recon_metrics.row_comparison.missing_in_target = 0 THEN TRUE
+        ELSE FALSE
+    END
+    ELSE NULL END AS ROW,
+    CASE WHEN lower(MAIN.report_type) in ('all', 'data') THEN
+    CASE
+        WHEN (METRICS.run_metrics.status = true) or
+         (METRICS.recon_metrics.column_comparison.absolute_mismatch = 0 AND METRICS.recon_metrics.column_comparison.threshold_mismatch = 0 AND METRICS.recon_metrics.column_comparison.mismatch_columns = '') THEN TRUE
+        ELSE FALSE
+    END
+    ELSE NULL END AS COLUMN,
+    CASE WHEN lower(MAIN.report_type) in ('all', 'schema') THEN
+    CASE
+        WHEN METRICS.recon_metrics.schema_comparison = true THEN TRUE
+        ELSE FALSE
+    END
+    ELSE NULL END AS SCHEMA,
+    METRICS.run_metrics.exception_message AS EXCEPTION_MESSAGE
+    FROM
+        {_db_prefix}.{_RECON_TABLE_NAME} MAIN
+    INNER JOIN
+        {_db_prefix}.{_RECON_METRICS_TABLE_NAME} METRICS
+    ON
+        (MAIN.recon_table_id = METRICS.recon_table_id)
+    WHERE
+        MAIN.recon_id = '{recon_id}'
+    """
+    )
+    table_output = []
+    for row in recon_df.collect():
+        if row.EXCEPTION_MESSAGE is not None and row.EXCEPTION_MESSAGE != "":
+            table_output.append(
+                ReconcileTableOutput(
+                    target_table_name=row.TARGET_TABLE,
+                    source_table_name=row.SOURCE_TABLE,
+                    status=StatusOutput(),
+                    exception_message=row.EXCEPTION_MESSAGE,
+                )
+            )
+        else:
+            table_output.append(
+                ReconcileTableOutput(
+                    target_table_name=row.TARGET_TABLE,
+                    source_table_name=row.SOURCE_TABLE,
+                    status=StatusOutput(row=row.ROW, column=row.COLUMN, schema=row.SCHEMA),
+                    exception_message=row.EXCEPTION_MESSAGE,
+                )
+            )
+    final_reconcile_output = ReconcileOutput(recon_id=recon_id, results=table_output)
+    logger.info(f"Final reconcile output: {final_reconcile_output}")
+    return final_reconcile_output
+
+
+def generate_final_reconcile_aggregate_output(
+    recon_id: str,
+    spark: SparkSession,
+    metadata_config: ReconcileMetadataConfig = ReconcileMetadataConfig(),
+    local_test_run: bool = False,
+) -> ReconcileOutput:
+    _db_prefix = "default" if local_test_run else f"{metadata_config.catalog}.{metadata_config.schema}"
+    recon_df = spark.sql(
+        f"""
+        SELECT source_table,
+         target_table,
+          EVERY(status) AS status,
+           ARRAY_JOIN(COLLECT_SET(exception_message), '\n') AS exception_message
+        FROM
+        (SELECT
+            IF(ISNULL(main.source_table.catalog)
+                , CONCAT_WS('.', main.source_table.schema, main.source_table.table_name)
+                , CONCAT_WS('.', main.source_table.catalog, main.source_table.schema, main.source_table.table_name)) AS source_table,
+            CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS target_table,
+            IF(metrics.run_metrics.status='true', TRUE , FALSE) AS status,
+            metrics.run_metrics.exception_message AS exception_message
+            FROM
+                {_db_prefix}.{_RECON_TABLE_NAME} main
+            INNER JOIN
+                {_db_prefix}.{_RECON_AGGREGATE_METRICS_TABLE_NAME} metrics
+            ON
+                (MAIN.recon_table_id = METRICS.recon_table_id
+                AND MAIN.operation_name = 'aggregates-reconcile')
+            WHERE
+                MAIN.recon_id = '{recon_id}'
+        )
+        GROUP BY source_table, target_table;
+    """
+    )
+    table_output = []
+    for row in recon_df.collect():
+        if row.exception_message is not None and row.exception_message != "":
+            table_output.append(
+                ReconcileTableOutput(
+                    target_table_name=row.target_table,
+                    source_table_name=row.source_table,
+                    status=StatusOutput(),
+                    exception_message=row.exception_message,
+                )
+            )
+        else:
+            table_output.append(
+                ReconcileTableOutput(
+                    target_table_name=row.target_table,
+                    source_table_name=row.source_table,
+                    status=StatusOutput(aggregate=row.status),
+                    exception_message=row.exception_message,
+                )
+            )
+    final_reconcile_output = ReconcileOutput(recon_id=recon_id, results=table_output)
+    logger.info(f"Final reconcile output: {final_reconcile_output}")
+    return final_reconcile_output
+
+
+class ReconCapture:
+
+    def __init__(
+        self,
+        database_config: DatabaseConfig,
+        recon_id: str,
+        report_type: str,
+        source_dialect: Dialect,
+        ws: WorkspaceClient,
+        spark: SparkSession,
+        metadata_config: ReconcileMetadataConfig = ReconcileMetadataConfig(),
+        local_test_run: bool = False,
+    ):
+        self.database_config = database_config
+        self.recon_id = recon_id
+        self.report_type = report_type
+        self.source_dialect = source_dialect
+        self.ws = ws
+        self.spark = spark
+        self._db_prefix = "default" if local_test_run else f"{metadata_config.catalog}.{metadata_config.schema}"
+
+    def _generate_recon_main_id(
+        self,
+        table_conf: Table,
+    ) -> int:
+        full_source_table = (
+            f"{self.database_config.source_schema}.{table_conf.source_name}"
+            if self.database_config.source_catalog is None
+            else f"{self.database_config.source_catalog}.{self.database_config.source_schema}.{table_conf.source_name}"
+        )
+        full_target_table = (
+            f"{self.database_config.target_catalog}.{self.database_config.target_schema}.{table_conf.target_name}"
+        )
+        return hash(f"{self.recon_id}{full_source_table}{full_target_table}")
+
+    def _insert_into_main_table(
+        self,
+        recon_table_id: int,
+        table_conf: Table,
+        recon_process_duration: ReconcileProcessDuration,
+        operation_name: str = "reconcile",
+    ) -> None:
+        source_dialect_key = get_key_from_dialect(self.source_dialect)
+        df = self.spark.sql(
+            f"""
+                select {recon_table_id} as recon_table_id,
+                '{self.recon_id}' as recon_id,
+                case
+                    when '{source_dialect_key}' = 'databricks' then 'Databricks'
+                    when '{source_dialect_key}' = 'snowflake' then 'Snowflake'
+                    when '{source_dialect_key}' = 'oracle' then 'Oracle'
+                    else '{source_dialect_key}'
+                end as source_type,
+                named_struct(
+                    'catalog', case when '{self.database_config.source_catalog}' = 'None' then null else '{self.database_config.source_catalog}' end,
+                    'schema', '{self.database_config.source_schema}',
+                    'table_name', '{table_conf.source_name}'
+                ) as source_table,
+                named_struct(
+                    'catalog', '{self.database_config.target_catalog}',
+                    'schema', '{self.database_config.target_schema}',
+                    'table_name', '{table_conf.target_name}'
+                ) as target_table,
+                '{self.report_type}' as report_type,
+                '{operation_name}' as operation_name,
+                cast('{recon_process_duration.start_ts}' as timestamp) as start_ts,
+                cast('{recon_process_duration.end_ts}' as timestamp) as end_ts
+            """
+        )
+        _write_df_to_delta(df, f"{self._db_prefix}.{_RECON_TABLE_NAME}")
+
+    @classmethod
+    def _is_mismatch_within_threshold_limits(
+        cls, data_reconcile_output: DataReconcileOutput, table_conf: Table, record_count: ReconcileRecordCount
+    ):
+        total_mismatch_count = (
+            data_reconcile_output.mismatch_count + data_reconcile_output.threshold_output.threshold_mismatch_count
+        )
+        logger.info(f"total_mismatch_count : {total_mismatch_count}")
+        logger.warning(f"reconciled_record_count : {record_count}")
+        # if the mismatch count is 0 then no need of checking bounds.
+        if total_mismatch_count == 0:
+            return True
+        # pull out table thresholds
+        thresholds: list[TableThresholds] = (
+            [threshold for threshold in table_conf.table_thresholds if threshold.model == "mismatch"]
+            if table_conf.table_thresholds
+            else []
+        )
+        # if not table thresholds are provided return false
+        if not thresholds:
+            return False
+
+        res = None
+        for threshold in thresholds:
+            mode = threshold.get_mode()
+            lower_bound = int(threshold.lower_bound.replace("%", ""))
+            upper_bound = int(threshold.upper_bound.replace("%", ""))
+            if mode == "absolute":
+                res = lower_bound <= total_mismatch_count <= upper_bound
+            if mode == "percentage":
+                lower_bound = int(round((lower_bound / 100) * record_count.source))
+                upper_bound = int(round((upper_bound / 100) * record_count.source))
+                res = lower_bound <= total_mismatch_count <= upper_bound
+
+        return res
+
+    def _insert_into_metrics_table(
+        self,
+        recon_table_id: int,
+        data_reconcile_output: DataReconcileOutput,
+        schema_reconcile_output: SchemaReconcileOutput,
+        table_conf: Table,
+        record_count: ReconcileRecordCount,
+    ) -> None:
+        status = False
+        if data_reconcile_output.exception in {None, ''} and schema_reconcile_output.exception in {None, ''}:
+            status = (
+                # validate for both exact mismatch and threshold mismatch
+                self._is_mismatch_within_threshold_limits(
+                    data_reconcile_output=data_reconcile_output, table_conf=table_conf, record_count=record_count
+                )
+                and data_reconcile_output.missing_in_src_count == 0
+                and data_reconcile_output.missing_in_tgt_count == 0
+                and schema_reconcile_output.is_valid
+            )
+
+        exception_msg = ""
+        if schema_reconcile_output.exception is not None:
+            exception_msg = schema_reconcile_output.exception.replace("'", '').replace('"', '')
+        if data_reconcile_output.exception is not None:
+            exception_msg = data_reconcile_output.exception.replace("'", '').replace('"', '')
+
+        insertion_time = str(datetime.now())
+        mismatch_columns = []
+        if data_reconcile_output.mismatch and data_reconcile_output.mismatch.mismatch_columns:
+            mismatch_columns = data_reconcile_output.mismatch.mismatch_columns
+
+        df = self.spark.sql(
+            f"""
+                select {recon_table_id} as recon_table_id,
+                named_struct(
+                    'row_comparison', case when '{self.report_type.lower()}' in ('all', 'row', 'data')
+                        and '{exception_msg}' = '' then
+                     named_struct(
+                        'missing_in_source', cast({data_reconcile_output.missing_in_src_count} as bigint),
+                        'missing_in_target', cast({data_reconcile_output.missing_in_tgt_count} as bigint)
+                    ) else null end,
+                    'column_comparison', case when '{self.report_type.lower()}' in ('all', 'data')
+                        and '{exception_msg}' = '' then
+                    named_struct(
+                        'absolute_mismatch', cast({data_reconcile_output.mismatch_count} as bigint),
+                        'threshold_mismatch', cast({data_reconcile_output.threshold_output.threshold_mismatch_count} as bigint),
+                        'mismatch_columns', '{",".join(mismatch_columns)}'
+                    ) else null end,
+                    'schema_comparison', case when '{self.report_type.lower()}' in ('all', 'schema')
+                        and '{exception_msg}' = '' then
+                        {schema_reconcile_output.is_valid} else null end
+                ) as recon_metrics,
+                named_struct(
+                    'status', {status},
+                    'run_by_user', '{self.ws.current_user.me().user_name}',
+                    'exception_message', "{exception_msg}"
+                ) as run_metrics,
+                cast('{insertion_time}' as timestamp) as inserted_ts
+            """
+        )
+        _write_df_to_delta(df, f"{self._db_prefix}.{_RECON_METRICS_TABLE_NAME}")
+
+    @classmethod
+    def _create_map_column(
+        cls,
+        recon_table_id: int,
+        df: DataFrame,
+        recon_type: str,
+        status: bool,
+    ) -> DataFrame:
+        columns = df.columns
+        # Create a list of column names and their corresponding column values
+        map_args = []
+        for column in columns:
+            map_args.extend([lit(column).alias(column + "_key"), col(column).cast("string").alias(column + "_value")])
+        # Create a new DataFrame with a map column
+        df = df.select(create_map(*map_args).alias("data"))
+        df = (
+            df.withColumn("recon_table_id", lit(recon_table_id))
+            .withColumn("recon_type", lit(recon_type))
+            .withColumn("status", lit(status))
+            .withColumn("inserted_ts", lit(datetime.now()))
+        )
+        return (
+            df.groupBy("recon_table_id", "recon_type", "status", "inserted_ts")
+            .agg(collect_list("data").alias("data"))
+            .selectExpr("recon_table_id", "recon_type", "status", "data", "inserted_ts")
+        )
+
+    def _create_map_column_and_insert(
+        self,
+        recon_table_id: int,
+        df: DataFrame,
+        recon_type: str,
+        status: bool,
+    ) -> None:
+        df = self._create_map_column(recon_table_id, df, recon_type, status)
+        _write_df_to_delta(df, f"{self._db_prefix}.{_RECON_DETAILS_TABLE_NAME}")
+
+    def _insert_into_details_table(
+        self,
+        recon_table_id: int,
+        reconcile_output: DataReconcileOutput,
+        schema_output: SchemaReconcileOutput,
+    ):
+        if reconcile_output.mismatch_count > 0 and reconcile_output.mismatch.mismatch_df:
+            self._create_map_column_and_insert(
+                recon_table_id,
+                reconcile_output.mismatch.mismatch_df,
+                "mismatch",
+                False,
+            )
+
+        if reconcile_output.missing_in_src_count > 0 and reconcile_output.missing_in_src:
+            self._create_map_column_and_insert(
+                recon_table_id,
+                reconcile_output.missing_in_src,
+                "missing_in_source",
+                False,
+            )
+
+        if reconcile_output.missing_in_tgt_count > 0 and reconcile_output.missing_in_tgt:
+            self._create_map_column_and_insert(
+                recon_table_id,
+                reconcile_output.missing_in_tgt,
+                "missing_in_target",
+                False,
+            )
+
+        if (
+            reconcile_output.threshold_output.threshold_mismatch_count > 0
+            and reconcile_output.threshold_output.threshold_df
+        ):
+            self._create_map_column_and_insert(
+                recon_table_id,
+                reconcile_output.threshold_output.threshold_df,
+                "threshold_mismatch",
+                False,
+            )
+
+        if schema_output.compare_df is not None:
+            self._create_map_column_and_insert(
+                recon_table_id, schema_output.compare_df, "schema", schema_output.is_valid
+            )
+
+    def _get_df(
+        self,
+        recon_table_id: int,
+        agg_data: DataReconcileOutput,
+        recon_type: str,
+    ):
+
+        column_count = agg_data.mismatch_count
+        agg_df = agg_data.mismatch.mismatch_df
+        match recon_type:
+            case "missing_in_source":
+                column_count = agg_data.missing_in_src_count
+                agg_df = agg_data.missing_in_src
+            case "missing_in_target":
+                column_count = agg_data.missing_in_tgt_count
+                agg_df = agg_data.missing_in_tgt
+
+        if column_count > 0 and agg_df:
+            return self._create_map_column(
+                recon_table_id,
+                agg_df,
+                recon_type,
+                False,
+            )
+        return None
+
+    @classmethod
+    def _union_dataframes(cls, df_list: list[DataFrame]) -> DataFrame:
+        return reduce(lambda agg_df, df: agg_df.unionByName(df), df_list)
+
+    def _insert_aggregates_into_metrics_table(
+        self,
+        recon_table_id: int,
+        reconcile_agg_output_list: list[AggregateQueryOutput],
+    ) -> None:
+
+        agg_metrics_df_list = []
+        for agg_output in reconcile_agg_output_list:
+            agg_data = agg_output.reconcile_output
+
+            status = False
+            if agg_data.exception in {None, ''}:
+                status = not (
+                    agg_data.mismatch_count > 0
+                    or agg_data.missing_in_src_count > 0
+                    or agg_data.missing_in_tgt_count > 0
+                )
+
+            exception_msg = ""
+            if agg_data.exception is not None:
+                exception_msg = agg_data.exception.replace("'", '').replace('"', '')
+
+            insertion_time = str(datetime.now())
+
+            # If there is any exception while running the Query,
+            # each rule is stored, with the Exception message in the metrics table
+            assert agg_output.rule, "Aggregate Rule must be present for storing the metrics"
+            rule_id = hash(f"{recon_table_id}_{agg_output.rule.column_from_rule}")
+
+            agg_metrics_df = self.spark.sql(
+                f"""
+                    select {recon_table_id} as recon_table_id,
+                    {rule_id}  as rule_id,
+                    if('{exception_msg}' = '', named_struct(
+                            'missing_in_source', {agg_data.missing_in_src_count},
+                            'missing_in_target', {agg_data.missing_in_tgt_count},
+                            'mismatch', {agg_data.mismatch_count}
+                    ), null) as recon_metrics,
+                    named_struct(
+                        'status', {status},
+                        'run_by_user', '{self.ws.current_user.me().user_name}',
+                        'exception_message', "{exception_msg}"
+                    ) as run_metrics,
+                    cast('{insertion_time}' as timestamp) as inserted_ts
+                """
+            )
+            agg_metrics_df_list.append(agg_metrics_df)
+
+        agg_metrics_table_df = self._union_dataframes(agg_metrics_df_list)
+        _write_df_to_delta(agg_metrics_table_df, f"{self._db_prefix}.{_RECON_AGGREGATE_METRICS_TABLE_NAME}")
+
+    def _insert_aggregates_into_details_table(
+        self, recon_table_id: int, reconcile_agg_output_list: list[AggregateQueryOutput]
+    ):
+        agg_details_df_list = []
+        for agg_output in reconcile_agg_output_list:
+            agg_details_rule_df_list = []
+
+            mismatch_df = self._get_df(recon_table_id, agg_output.reconcile_output, "mismatch")
+            if mismatch_df and not mismatch_df.isEmpty():
+                agg_details_rule_df_list.append(mismatch_df)
+
+            missing_src_df = self._get_df(recon_table_id, agg_output.reconcile_output, "missing_in_source")
+            if missing_src_df and not missing_src_df.isEmpty():
+                agg_details_rule_df_list.append(missing_src_df)
+
+            missing_tgt_df = self._get_df(recon_table_id, agg_output.reconcile_output, "missing_in_target")
+            if missing_tgt_df and not missing_tgt_df.isEmpty():
+                agg_details_rule_df_list.append(missing_tgt_df)
+
+            if agg_details_rule_df_list:
+                agg_details_rule_df = self._union_dataframes(agg_details_rule_df_list)
+                if agg_output.rule:
+                    rule_id = hash(f"{recon_table_id}_{agg_output.rule.column_from_rule}")
+                    agg_details_rule_df = agg_details_rule_df.withColumn("rule_id", lit(rule_id)).select(
+                        "recon_table_id", "rule_id", "recon_type", "data", "inserted_ts"
+                    )
+                    agg_details_df_list.append(agg_details_rule_df)
+            else:
+                logger.warning("Aggregate Details Rules are empty")
+
+        if agg_details_df_list:
+            agg_details_table_df = self._union_dataframes(agg_details_df_list)
+            _write_df_to_delta(agg_details_table_df, f"{self._db_prefix}.{_RECON_AGGREGATE_DETAILS_TABLE_NAME}")
+
+    def start(
+        self,
+        data_reconcile_output: DataReconcileOutput,
+        schema_reconcile_output: SchemaReconcileOutput,
+        table_conf: Table,
+        recon_process_duration: ReconcileProcessDuration,
+        record_count: ReconcileRecordCount,
+    ) -> None:
+        recon_table_id = self._generate_recon_main_id(table_conf)
+        self._insert_into_main_table(recon_table_id, table_conf, recon_process_duration)
+        self._insert_into_metrics_table(
+            recon_table_id, data_reconcile_output, schema_reconcile_output, table_conf, record_count
+        )
+        self._insert_into_details_table(recon_table_id, data_reconcile_output, schema_reconcile_output)
+
+    def store_aggregates_metrics(
+        self,
+        table_conf: Table,
+        recon_process_duration: ReconcileProcessDuration,
+        reconcile_agg_output_list: list[AggregateQueryOutput],
+    ) -> None:
+        recon_table_id = self._generate_recon_main_id(table_conf)
+        self._insert_into_main_table(recon_table_id, table_conf, recon_process_duration, 'aggregates-reconcile')
+        self._insert_into_rules_table(recon_table_id, reconcile_agg_output_list)
+        self._insert_aggregates_into_metrics_table(recon_table_id, reconcile_agg_output_list)
+        self._insert_aggregates_into_details_table(
+            recon_table_id,
+            reconcile_agg_output_list,
+        )
+
+    def _insert_into_rules_table(self, recon_table_id: int, reconcile_agg_output_list: list[AggregateQueryOutput]):
+
+        rule_df_list = []
+        for agg_output in reconcile_agg_output_list:
+            if not agg_output.rule:
+                logger.error("Aggregate Rule must be present for storing the rules")
+                continue
+            rule_id = hash(f"{recon_table_id}_{agg_output.rule.column_from_rule}")
+            rule_query = agg_output.rule.get_rule_query(rule_id)
+            rule_df_list.append(
+                self.spark.sql(rule_query)
+                .withColumn("inserted_ts", lit(datetime.now()))
+                .select("rule_id", "rule_type", "rule_info", "inserted_ts")
+            )
+
+        if rule_df_list:
+            rules_table_df = self._union_dataframes(rule_df_list)
+            _write_df_to_delta(rules_table_df, f"{self._db_prefix}.{_RECON_AGGREGATE_RULES_TABLE_NAME}")