databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/reconcile/sampler.py

@@ -0,0 +1,239 @@
+import logging
+from abc import ABC, abstractmethod
+
+from pyspark.sql import DataFrame
+from pyspark.sql import functions as F
+
+from databricks.labs.lakebridge.reconcile.constants import SamplingOptionMethod, SamplingSpecificationsType
+from databricks.labs.lakebridge.reconcile.recon_config import SamplingOptions, SamplingSpecifications
+
+logger = logging.getLogger(__name__)
+
+_MIN_SAMPLE_COUNT = 50
+_MAX_SAMPLE_COUNT = 400
+
+_MIN_BUCKET_LIMIT = 2
+_MAX_BUCKET_LIMIT = 50
+
+
+class Sampler(ABC):
+    def __init__(self, sampling_options: SamplingOptions):
+        self._sampling_options = sampling_options
+
+    @abstractmethod
+    def _validate_sampling_options(self):
+        return NotImplemented
+
+    @abstractmethod
+    def sample(
+        self,
+        keys_df: DataFrame,
+        keys_df_count: int,
+        key_columns: list[str],
+        target_table: DataFrame,
+    ) -> DataFrame:
+        return NotImplemented
+
+
+class RandomSampler(Sampler):
+    """
+    RandomSampler is a subclass of Sampler that performs random sampling on a given DataFrame.
+    """
+
+    def __init__(self, sampling_options: SamplingOptions, seed: int = 100):
+        """
+        Attributes:
+        sampling_options (SamplingOptions): The options for sampling, including method, specifications.
+        seed (int): The seed for random sampling to ensure reproducibility.
+        """
+        super().__init__(sampling_options)
+        self.seed = seed
+
+    def _validate_sampling_options(self):
+        if self._sampling_options.method != SamplingOptionMethod.RANDOM:
+            raise ValueError("RandomSampler: Only 'random' sampling method is supported")
+
+        specs = self._sampling_options.specifications
+        if specs.type == SamplingSpecificationsType.COUNT and (
+            specs.value is None or (specs.value < _MIN_SAMPLE_COUNT)
+        ):
+            logger.info(
+                f"RandomSampler: Sample count must be >= {_MIN_SAMPLE_COUNT}, " f"flooring to {_MIN_SAMPLE_COUNT}"
+            )
+            self._sampling_options.specifications.value = _MIN_SAMPLE_COUNT
+
+        elif specs.type == SamplingSpecificationsType.COUNT and specs.value > _MAX_SAMPLE_COUNT:
+            logger.info(
+                f"RandomSampler: Sample count must be <= {_MAX_SAMPLE_COUNT}, " f"capping to {_MAX_SAMPLE_COUNT}"
+            )
+            self._sampling_options.specifications.value = _MAX_SAMPLE_COUNT
+
+    def sample(
+        self, keys_df: DataFrame, keys_df_count: int, key_columns: list[str], target_table: DataFrame
+    ) -> DataFrame:
+        """
+        Performs random sampling on the given DataFrame based on the specified options.
+        - Validates the sampling options.
+        - Uses pre-calculated `keys_df_count` from `reconcile_output.mismatch_count` to avoid from recomputing `keys_df`.
+        - If the specifications type is FRACTION, samples the DataFrame based on the fraction value.
+        - If the specifications type is COUNT, calculates the fraction and samples the DataFrame accordingly,
+          then limits the sample size to the specified count.
+        - Returns the sampled DataFrame.
+        """
+
+        self._validate_sampling_options()
+        specs = self._sampling_options.specifications
+
+        default_sampled_df = keys_df.limit(_MIN_SAMPLE_COUNT)
+
+        if specs.type == SamplingSpecificationsType.FRACTION:
+            sampled_df = keys_df.sample(fraction=specs.value, seed=self.seed)
+        elif specs.type == SamplingSpecificationsType.COUNT:
+            total_count = keys_df_count
+            sample_size = int(specs.value)
+            fraction = min(1.0, sample_size / total_count)
+            sampled_df = keys_df.sample(fraction=fraction, seed=self.seed).limit(sample_size)
+        else:
+            return default_sampled_df
+
+        return sampled_df
+
+
+class StratifiedSampler(Sampler):
+    """
+    StratifiedSampler is a subclass of Sampler that performs stratified sampling on a given DataFrame.
+    """
+
+    def __init__(self, sampling_options: SamplingOptions, seed: int = 100):
+        """
+        Attributes:
+        sampling_options (SamplingOptions): The options for sampling, including method, specifications,
+                                            stratified columns, and stratified buckets.
+        seed (int): The seed for random sampling to ensure reproducibility.
+        """
+        super().__init__(sampling_options)
+        self.seed = seed
+
+    def _validate_sampling_options(self):
+        if self._sampling_options.method != SamplingOptionMethod.STRATIFIED:
+            raise ValueError("StratifiedSampler: Only 'stratified' sampling method is supported")
+
+        specs = self._sampling_options.specifications
+        stratified_buckets = self._sampling_options.stratified_buckets
+
+        if specs.type == SamplingSpecificationsType.COUNT and (
+            specs.value is None or (specs.value < _MIN_SAMPLE_COUNT)
+        ):
+            logger.info(
+                f"StratifiedSampler: Sample count must be >= {_MIN_SAMPLE_COUNT}, " f"flooring to {_MIN_SAMPLE_COUNT}"
+            )
+            self._sampling_options.specifications.value = _MIN_SAMPLE_COUNT
+
+        elif specs.type == SamplingSpecificationsType.COUNT and specs.value > _MAX_SAMPLE_COUNT:
+            logger.info(
+                f"StratifiedSampler: Sample count must be <= {_MAX_SAMPLE_COUNT}, " f"capping to {_MAX_SAMPLE_COUNT}"
+            )
+            self._sampling_options.specifications.value = _MAX_SAMPLE_COUNT
+
+        if stratified_buckets < _MIN_BUCKET_LIMIT:
+            logger.info(
+                f"StratifiedSampler: Stratified buckets must be >= {_MIN_BUCKET_LIMIT}, "
+                f"flooring to {_MIN_BUCKET_LIMIT}"
+            )
+            self._sampling_options.stratified_buckets = _MIN_BUCKET_LIMIT
+        elif stratified_buckets > _MAX_BUCKET_LIMIT:
+            logger.info(
+                f"StratifiedSampler: Stratified buckets must be <= {_MAX_BUCKET_LIMIT}, "
+                f"capping to {_MAX_BUCKET_LIMIT}"
+            )
+            self._sampling_options.stratified_buckets = _MAX_BUCKET_LIMIT
+
+    def sample(
+        self, keys_df: DataFrame, keys_df_count: int, key_columns: list[str], target_table: DataFrame
+    ) -> DataFrame:
+        """
+        Performs stratified sampling on the given DataFrame based on the specified options.
+        - Joins the keys_df with the target_table on the key_columns.
+        - Creates a hash bucket column based on the stratified columns.
+        - Adds the bucket column to the joined DataFrame.
+        - If specifications type is FRACTION, calculates fractions for each bucket and samples accordingly.
+        - If specifications type is COUNT, calculates fractions for each bucket and samples accordingly,
+          then limits the sample size to the specified count.
+        - Returns the sampled DataFrame.
+        """
+        self._validate_sampling_options()
+
+        specs = self._sampling_options.specifications
+        stratified_columns = self._sampling_options.stratified_columns
+        non_key_stratified_columns = [
+            col for col in (self._sampling_options.stratified_columns or []) if col not in key_columns
+        ]
+        stratified_buckets = self._sampling_options.stratified_buckets or _MIN_BUCKET_LIMIT
+
+        keys_df.select(*key_columns)
+        default_sampled_df = keys_df.limit(_MIN_SAMPLE_COUNT)
+
+        # Join the mismatched_df with target_table_df
+        joined_df = keys_df.join(
+            target_table, [keys_df[col] == target_table[col] for col in key_columns], "inner"
+        ).select(*[keys_df[col] for col in key_columns], *[target_table[col] for col in non_key_stratified_columns])
+
+        # Create a hash bucket column based on the stratified columns
+        bucket_col = F.pmod(F.abs(F.hash(*[F.col(c) for c in (stratified_columns or [])])), stratified_buckets)
+
+        # Add the bucket column to the joined_df
+        bucketed_df = joined_df.withColumn("bucket", bucket_col)
+
+        if specs.type == SamplingSpecificationsType.FRACTION:
+            # Calculate fractions for each bucket
+            unique_values = bucketed_df.select("bucket").distinct().collect()
+            fractions = {row["bucket"]: specs.value for row in unique_values}
+            sampled_df = bucketed_df.sampleBy("bucket", fractions=fractions, seed=self.seed)
+
+        elif specs.type == SamplingSpecificationsType.COUNT:
+            # Calculate fractions for each bucket
+            sample_size = int(specs.value)
+            bucket_counts = bucketed_df.groupBy("bucket").count().collect()
+            total_count = sum(row['count'] for row in bucket_counts)
+            fractions = {
+                row["bucket"]: min(1.0, (sample_size * row['count'] / total_count) / row['count'])
+                for row in bucket_counts
+            }
+            sampled_df = bucketed_df.sampleBy("bucket", fractions=fractions, seed=self.seed).limit(sample_size)
+
+        else:
+            return default_sampled_df.select(*key_columns)
+
+        return sampled_df.select(*key_columns)
+
+
+# TODO: Move away from SamplerFactory to a context-driven approach
+class SamplerFactory:
+    @staticmethod
+    def get_sampler(sampling_options: SamplingOptions, seed: int = 100) -> Sampler:
+        # If no sampling options provided, use default
+        if sampling_options is None:
+            default_sampling_options = SamplingOptions(
+                method=SamplingOptionMethod.RANDOM,
+                specifications=SamplingSpecifications(type=SamplingSpecificationsType.COUNT, value=_MIN_SAMPLE_COUNT),
+                stratified_columns=None,
+                stratified_buckets=None,
+            )
+            logger.info(
+                f"SamplerFactory: No sampling options provided, using default options: " f"{default_sampling_options}"
+            )
+            sampling_options = default_sampling_options
+
+        else:
+            logger.info(f"SamplerFactory: Creating sampler using provided options: " f"{sampling_options}")
+
+        # Use a dictionary-based dispatch for better extensibility
+        sampler_map = {SamplingOptionMethod.RANDOM: RandomSampler, SamplingOptionMethod.STRATIFIED: StratifiedSampler}
+
+        # Get the sampler class
+        sampler_class = sampler_map.get(sampling_options.method)
+
+        if sampler_class is None:
+            raise ValueError(f"SamplerFactory : Unsupported sampling method: {sampling_options.method}")
+
+        return sampler_class(sampling_options, seed)

databricks/labs/lakebridge/reconcile/schema_compare.py

@@ -0,0 +1,126 @@
+import logging
+from dataclasses import asdict
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.types import BooleanType, StringType, StructField, StructType
+from sqlglot import Dialect, parse_one
+
+from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect
+from databricks.labs.lakebridge.reconcile.recon_config import Schema, Table
+from databricks.labs.lakebridge.reconcile.recon_output_config import SchemaMatchResult, SchemaReconcileOutput
+from databricks.labs.lakebridge.transpiler.sqlglot.generator.databricks import Databricks
+
+logger = logging.getLogger(__name__)
+
+
+class SchemaCompare:
+    def __init__(
+        self,
+        spark: SparkSession,
+    ):
+        self.spark = spark
+
+    # Define the schema for the schema compare DataFrame
+    _schema_compare_schema: StructType = StructType(
+        [
+            StructField("source_column", StringType(), False),
+            StructField("source_datatype", StringType(), False),
+            StructField("databricks_column", StringType(), True),
+            StructField("databricks_datatype", StringType(), True),
+            StructField("is_valid", BooleanType(), False),
+        ]
+    )
+
+    @classmethod
+    def _build_master_schema(
+        cls,
+        source_schema: list[Schema],
+        databricks_schema: list[Schema],
+        table_conf: Table,
+    ) -> list[SchemaMatchResult]:
+        master_schema = source_schema
+        if table_conf.select_columns:
+            master_schema = [schema for schema in master_schema if schema.column_name in table_conf.select_columns]
+        if table_conf.drop_columns:
+            master_schema = [sschema for sschema in master_schema if sschema.column_name not in table_conf.drop_columns]
+
+        target_column_map = table_conf.to_src_col_map or {}
+        master_schema_match_res = [
+            SchemaMatchResult(
+                source_column=s.column_name,
+                databricks_column=target_column_map.get(s.column_name, s.column_name),
+                source_datatype=s.data_type,
+                databricks_datatype=next(
+                    (
+                        tgt.data_type
+                        for tgt in databricks_schema
+                        if tgt.column_name == target_column_map.get(s.column_name, s.column_name)
+                    ),
+                    "",
+                ),
+            )
+            for s in master_schema
+        ]
+        return master_schema_match_res
+
+    def _create_dataframe(self, data: list, schema: StructType) -> DataFrame:
+        """
+        :param data: Expectation is list of dataclass
+        :param schema: Target schema
+        :return: DataFrame
+        """
+        data = [tuple(asdict(item).values()) for item in data]
+        df = self.spark.createDataFrame(data, schema)
+
+        return df
+
+    @classmethod
+    def _parse(cls, source: Dialect, column: str, data_type: str) -> str:
+        return (
+            parse_one(f"create table dummy ({column} {data_type})", read=source)
+            .sql(dialect=get_dialect("databricks"))
+            .replace(", ", ",")
+        )
+
+    @classmethod
+    def _table_schema_status(cls, schema_compare_maps: list[SchemaMatchResult]) -> bool:
+        return bool(all(x.is_valid for x in schema_compare_maps))
+
+    @classmethod
+    def _validate_parsed_query(cls, master: SchemaMatchResult, parsed_query) -> None:
+        databricks_query = f"create table dummy ({master.source_column} {master.databricks_datatype})"
+        logger.info(
+            f"""
+        Source datatype: create table dummy ({master.source_column} {master.source_datatype})
+        Parse datatype: {parsed_query}
+        Databricks datatype: {databricks_query}
+        """
+        )
+        if parsed_query.lower() != databricks_query.lower():
+            master.is_valid = False
+
+    def compare(
+        self,
+        source_schema: list[Schema],
+        databricks_schema: list[Schema],
+        source: Dialect,
+        table_conf: Table,
+    ) -> SchemaReconcileOutput:
+        """
+        This method compares the source schema and the Databricks schema. It checks if the data types of the columns in the source schema
+        match with the corresponding columns in the Databricks schema by parsing using remorph transpile.
+
+        Returns:
+            SchemaReconcileOutput: A dataclass object containing a boolean indicating the overall result of the comparison and a DataFrame with the comparison details.
+        """
+        master_schema = self._build_master_schema(source_schema, databricks_schema, table_conf)
+        for master in master_schema:
+            if not isinstance(source, Databricks):
+                parsed_query = self._parse(source, master.source_column, master.source_datatype)
+                self._validate_parsed_query(master, parsed_query)
+            elif master.source_datatype.lower() != master.databricks_datatype.lower():
+                master.is_valid = False
+
+        df = self._create_dataframe(master_schema, self._schema_compare_schema)
+        final_result = self._table_schema_status(master_schema)
+        return SchemaReconcileOutput(final_result, df)

databricks/labs/lakebridge/resources/config/credentials.yml

@@ -0,0 +1,33 @@
+secret_vault_type: local | databricks | env
+secret_vault_name: null
+snowflake:
+  account: example_account
+  connect_retries: 1
+  connect_timeout: null
+  host: null
+  insecure_mode: false
+  oauth_client_id: null
+  oauth_client_secret: null
+  password: null
+  port: null
+  private_key: null
+  private_key_passphrase: null
+  private_key_path: null
+  role: null
+  token: null
+  user: null
+  warehouse: null
+
+mssql:
+  #TODO Expand to support sqlpools, and legacy dwh
+  database: DB_NAME
+  driver: ODBC Driver 18 for SQL Server
+  server: example_host
+  port: null
+  user: null
+  password: null
+
+
+
+
+

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/00_0_aggregate_recon_header.md

@@ -0,0 +1,6 @@
+# Aggregates Reconcile Table Metrics
+### It provides the following information:
+
+* Mismatch
+* Missing in Source
+* Missing in Target

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml

@@ -0,0 +1,6 @@
+columns:
+- recon_id
+- dd_recon_id
+type: MULTI_SELECT
+title: Recon Id
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_executed_by.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- executed_by
+type: MULTI_SELECT
+title: Executed by
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_2_started_at.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- start_ts
+title: Started At
+type: DATE_RANGE_PICKER
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_source_type.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- source_type
+type: MULTI_SELECT
+title: Source Type
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_table.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- source_table
+type: MULTI_SELECT
+title: Source Table Name
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_2_target_table.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- target_table
+type: MULTI_SELECT
+title: Target Table Name
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/04_0_aggregate_summary_table.sql

@@ -0,0 +1,46 @@
+/* --title 'Aggregates Summary Table' --width 6 --height 6 */
+SELECT
+    main.recon_id,
+    main.source_type,
+    main.source_table.`catalog` AS source_catalog,
+    main.source_table.`schema` AS source_schema,
+    main.source_table.table_name AS source_table_name,
+    IF(
+            ISNULL(source_catalog),
+            CONCAT_WS('.', source_schema, source_table_name),
+            CONCAT_WS(
+                    '.',
+                    source_catalog,
+                    source_schema,
+                    source_table_name
+            )
+    ) AS source_table,
+    main.target_table.`catalog` AS target_catalog,
+    main.target_table.`schema` AS target_schema,
+    main.target_table.table_name AS target_table_name,
+    CONCAT_WS(
+            '.',
+            target_catalog,
+            target_schema,
+            target_table_name
+    ) AS target_table,
+    UPPER(rules.rule_info.agg_type) || CONCAT('(', rules.rule_info.agg_column, ')') AS aggregate_column,
+    rules.rule_info.group_by_columns,
+    metrics.run_metrics.status AS status,
+    metrics.run_metrics.exception_message AS exception,
+    metrics.recon_metrics.missing_in_source AS missing_in_source,
+    metrics.recon_metrics.missing_in_target AS missing_in_target,
+    metrics.recon_metrics.mismatch AS mismatch,
+    metrics.run_metrics.run_by_user AS executed_by,
+    main.start_ts AS start_ts,
+    main.end_ts AS end_ts
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.aggregate_metrics metrics
+        INNER JOIN remorph.reconcile.aggregate_rules rules
+                   ON main.recon_table_id = metrics.recon_table_id
+                       AND rules.rule_id = metrics.rule_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/05_0_aggregate_recon_drilldown_header.md

@@ -0,0 +1,2 @@
+# Drill Down
+### The Aggregates Reconcile details table contains all the sample records information of mismatches and missing entries.

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_0_recon_id.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- dd_recon_id
+type: MULTI_SELECT
+title: Recon Id
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_1_category.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- dd_recon_type
+type: MULTI_SELECT
+title: Category
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_2_aggregate_type.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- dd_aggregate_type
+type: MULTI_SELECT
+title: Aggregate Type
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_0_target_table.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- dd_target_table
+type: MULTI_SELECT
+title: Target Table Name

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_1_source_table.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- dd_source_table
+type: MULTI_SELECT
+title: Source Table Name

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/08_0_aggregate_details_table.sql

@@ -0,0 +1,92 @@
+/* --title 'Aggregates Reconciliation Details' --width 6 --height 6 */
+WITH details_view AS (
+    SELECT
+        recon_table_id,
+        rule_id,
+        recon_type,
+        explode(data) AS agg_details
+    FROM
+        remorph.reconcile.aggregate_details
+),
+     metrics_view AS (
+         SELECT
+             recon_table_id,
+             rule_id,
+             recon_metrics,
+             run_metrics
+         FROM
+             remorph.reconcile.aggregate_metrics
+     )
+SELECT
+    recon_id AS dd_recon_id,
+    source_table AS dd_source_table,
+    target_table AS dd_target_table,
+    recon_type AS dd_recon_type,
+    aggregate_type AS dd_aggregate_type,
+    rule AS aggregate_column,
+    source_value,
+    target_value,
+    zip_with(rule_group_by_columns, group_by_column_values, (groupby, value) -> CONCAT_WS(':', TRIM(groupby), value)) AS group_by_columns,
+    COALESCE(status, 'false') AS status
+FROM (
+         SELECT
+             main.recon_id,
+             main.source_table.`catalog` AS source_catalog,
+             main.source_table.`schema` AS source_schema,
+             main.source_table.table_name AS source_table_name,
+             IF(
+                     ISNULL(source_catalog),
+                     CONCAT_WS('.', source_schema, source_table_name),
+                     CONCAT_WS(
+                             '.',
+                             source_catalog,
+                             source_schema,
+                             source_table_name
+                     )
+             ) AS source_table,
+             main.target_table.`catalog` AS target_catalog,
+             main.target_table.`schema` AS target_schema,
+             main.target_table.table_name AS target_table_name,
+             CONCAT_WS(
+                     '.',
+                     target_catalog,
+                     target_schema,
+                     target_table_name
+             ) AS target_table,
+             dtl.recon_type,
+             rul.rule_info.agg_type AS aggregate_type,
+             UPPER(rul.rule_info.agg_type) || CONCAT('(', rul.rule_info.agg_column, ')') AS rule,
+             CONCAT_WS(
+                     '_',
+                     'source',
+                     rul.rule_info.agg_type,
+                     rul.rule_info.agg_column
+             ) AS source_agg_column,
+             dtl.agg_details[source_agg_column] AS source_value,
+             CONCAT_WS(
+                     '_',
+                     'target',
+                     rul.rule_info.agg_type,
+                     rul.rule_info.agg_column
+             ) AS target_agg_column,
+             dtl.agg_details[target_agg_column] AS target_value,
+             SPLIT(rul.rule_info.group_by_columns, ',') AS rule_group_by_columns,
+             TRANSFORM(rule_group_by_columns, colm ->
+                 COALESCE(dtl.agg_details[CONCAT('source_group_by_', TRIM(colm))],
+                          dtl.agg_details[CONCAT('target_group_by_', TRIM(colm))])) AS group_by_column_values,
+             CONCAT_WS(
+                     '_',
+                     'match',
+                     rul.rule_info.agg_type,
+                     rul.rule_info.agg_column
+             ) AS status_column,
+             dtl.agg_details[status_column] AS status
+         FROM
+             metrics_view mtc
+                 INNER JOIN remorph.reconcile.main main ON main.recon_table_id = mtc.recon_table_id
+                 INNER JOIN details_view dtl ON mtc.recon_table_id = dtl.recon_table_id
+                 INNER JOIN remorph.reconcile.aggregate_rules rul ON mtc.rule_id = dtl.rule_id
+                 AND dtl.rule_id = rul.rule_id
+     )
+ORDER BY
+    recon_id

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/09_0_aggregate_missing_mismatch_header.md

@@ -0,0 +1 @@
+# Visualization of Missing and Mismatched Records

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/10_0_aggr_mismatched_records.sql

@@ -0,0 +1,19 @@
+/* --title 'Mismatched Records' --width 6 */
+SELECT
+    main.recon_id,
+    CONCAT_WS(
+            '.',
+            main.target_table.`catalog`,
+            main.target_table.`schema`,
+            main.target_table.table_name
+    ) AS target_table,
+    main.start_ts,
+    metrics.recon_metrics.mismatch AS mismatch
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.aggregate_metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name