databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/11_0_aggr_missing_in_databricks.sql

@@ -0,0 +1,19 @@
+/* --title 'Missing in Databricks' --width 3 */
+SELECT
+    main.recon_id,
+    CONCAT_WS(
+            '.',
+            main.target_table.`catalog`,
+            main.target_table.`schema`,
+            main.target_table.table_name
+    ) AS target_table,
+    main.start_ts,
+    metrics.recon_metrics.missing_in_target AS missing_in_target
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.aggregate_metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/11_1_aggr_missing_in_source.sql

@@ -0,0 +1,19 @@
+/* --title 'Missing in Source' --width 3 */
+SELECT
+    main.recon_id,
+    CONCAT_WS(
+            '.',
+            main.target_table.`catalog`,
+            main.target_table.`schema`,
+            main.target_table.table_name
+    ) AS target_table,
+    main.start_ts,
+    metrics.recon_metrics.missing_in_source AS missing_in_source
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.aggregate_metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml

@@ -0,0 +1,365 @@
+display_name: "Aggregate Reconciliation Metrics"
+tiles:
+  04_0_aggregate_summary_table:
+    overrides:
+      spec:
+        withRowNumber: true
+        encodings:
+          columns:
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: recon_id
+              title: recon_id
+              type: string
+              cellFormat:
+                default:
+                  foregroundColor:
+                rules:
+                  - if:
+                      column: status
+                      fn: '='
+                      literal: 'true'
+                    value:
+                      foregroundColor: '#3BD973'
+                  - if:
+                      column: status
+                      fn: '='
+                      literal: 'false'
+                    value:
+                      foregroundColor: '#E92828'
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_type
+              title: source_type
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_catalog
+              title: source_catalog
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_schema
+              title: source_schema
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_table_name
+              title: source_table_name
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_table
+              title: source_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_catalog
+              title: target_catalog
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_schema
+              title: target_schema
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_table_name
+              title: target_table_name
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_table
+              title: target_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: aggregate_column
+              title: aggregate_column
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: group_by_columns
+              title: group_by_columns
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: status
+              title: status
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: exception
+              title: exception
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: missing_in_source
+              title: missing_in_source
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: missing_in_target
+              title: missing_in_target
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: mismatch
+              title: mismatch
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: executed_by
+              title: executed_by
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: datetime
+              fieldName: start_ts
+              title: start_ts
+              type: datetime
+              dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS'
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: datetime
+              fieldName: end_ts
+              title: end_ts
+              type: datetime
+              dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS'
+  08_0_aggregate_details_table:
+    overrides:
+      spec:
+        withRowNumber: true
+        encodings:
+          columns:
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: dd_recon_id
+              title: recon_id
+              type: string
+              cellFormat:
+                default:
+                  foregroundColor:
+                rules:
+                  - if:
+                      column: status
+                      fn: '='
+                      literal: 'true'
+                    value:
+                      foregroundColor: '#3BD973'
+                  - if:
+                      column: status
+                      fn: '='
+                      literal: 'false'
+                    value:
+                      foregroundColor: '#E92828'
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: dd_source_table
+              title: source_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: dd_target_table
+              title: target_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: dd_recon_type
+              title: recon_type
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: dd_aggregate_type
+              title: aggregate_type
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: aggregate_column
+              title: aggregate_column
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_value
+              title: source_value
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_value
+              title: target_value
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: group_by_columns
+              title: group_by_columns
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: status
+              title: status
+              type: string
+  10_0_aggr_mismatched_records:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 10_0_aggr_mismatched_records
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: mismatch
+                expression: '`mismatch`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: mismatch
+            scale:
+              type: quantitative
+            displayName: mismatch
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false
+  11_0_aggr_missing_in_databricks:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 11_0_aggr_missing_in_databricks
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: missing_in_target
+                expression: '`missing_in_target`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: missing_in_target
+            scale:
+              type: quantitative
+            displayName: missing_in_target
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false
+  11_1_aggr_missing_in_source:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 11_1_aggr_missing_in_source
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: missing_in_source
+                expression: '`missing_in_source`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: missing_in_source
+            scale:
+              type: quantitative
+            displayName: missing_in_source
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false
+
+

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md

@@ -0,0 +1,3 @@
+# Main Reconciliation Table
+
+### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records.

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml

@@ -0,0 +1,6 @@
+columns:
+- recon_id
+- dd_recon_id
+type: MULTI_SELECT
+title: Recon Id
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/01_1_report_type.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- report_type
+type: MULTI_SELECT
+title: Report Type
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/01_2_executed_by.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- executed_by
+type: MULTI_SELECT
+title: Executed by
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- source_type
+type: MULTI_SELECT
+title: Source Type
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_table.filter.yml

@@ -0,0 +1,6 @@
+columns:
+- source_table
+- dd_source_table
+type: MULTI_SELECT
+title: Source Table Name
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/02_2_target_table.filter.yml

@@ -0,0 +1,6 @@
+columns:
+- target_table
+- dd_target_table
+type: MULTI_SELECT
+title: Target Table Name
+width: 2

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml

@@ -0,0 +1,5 @@
+columns:
+- start_ts
+title: Started At
+type: DATE_RANGE_PICKER
+width: 6

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql

@@ -0,0 +1,38 @@
+/* --title 'Summary Table' --width 6 --height 6 */
+SELECT main.recon_id,
+       main.source_type,
+       main.report_type,
+       main.source_table.`catalog` AS source_catalog,
+       main.source_table.`schema` AS source_schema,
+       main.source_table.table_name AS source_table_name,
+       IF(
+           ISNULL(source_catalog),
+           CONCAT_WS('.', source_schema, source_table_name),
+           CONCAT_WS(
+               '.',
+               source_catalog,
+               source_schema,
+               source_table_name
+           )
+       ) AS source_table,
+       main.target_table.`catalog` AS target_catalog,
+       main.target_table.`schema` AS target_schema,
+       main.target_table.table_name AS target_table_name,
+       CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table,
+       metrics.run_metrics.status AS status,
+       metrics.run_metrics.exception_message AS exception,
+       metrics.recon_metrics.row_comparison.missing_in_source AS missing_in_source,
+       metrics.recon_metrics.row_comparison.missing_in_target AS missing_in_target,
+       metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch,
+       metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch,
+       metrics.recon_metrics.column_comparison.mismatch_columns AS mismatch_columns,
+       metrics.recon_metrics.schema_comparison AS schema_comparison,
+       metrics.run_metrics.run_by_user AS executed_by,
+       main.start_ts AS start_ts,
+       main.end_ts AS end_ts
+FROM remorph.reconcile.main main
+         INNER JOIN remorph.reconcile.metrics metrics
+                    ON main.recon_table_id = metrics.recon_table_id
+ORDER BY metrics.inserted_ts DESC,
+         main.recon_id,
+         main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_comparison_header.md

@@ -0,0 +1,3 @@
+# Schema Comparison Details
+
+### This table provides a detailed view of schema mismatches.

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql

@@ -0,0 +1,42 @@
+/* --title 'Schema Details' --width 6 */
+WITH tmp AS (
+    SELECT
+        recon_table_id,
+        inserted_ts,
+        explode(data) AS schema_data
+    FROM
+        remorph.reconcile.details
+    WHERE
+        recon_type = 'schema'
+)
+SELECT
+    main.recon_id,
+    main.source_table.`catalog` AS source_catalog,
+    main.source_table.`schema` AS source_schema,
+    main.source_table.table_name AS source_table_name,
+    IF(
+        ISNULL(source_catalog),
+        CONCAT_WS('.', source_schema, source_table_name),
+        CONCAT_WS(
+            '.',
+            source_catalog,
+            source_schema,
+            source_table_name
+        )
+    ) AS source_table,
+    main.target_table.`catalog` AS target_catalog,
+    main.target_table.`schema` AS target_schema,
+    main.target_table.table_name AS target_table_name,
+    CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table,
+    schema_data['source_column'] AS source_column,
+    schema_data['source_datatype'] AS source_datatype,
+    schema_data['databricks_column'] AS databricks_column,
+    schema_data['databricks_datatype'] AS databricks_datatype,
+    schema_data['is_valid'] AS is_valid
+FROM
+    remorph.reconcile.main main
+        INNER JOIN tmp ON main.recon_table_id = tmp.recon_table_id
+ORDER BY
+    tmp.inserted_ts DESC,
+    main.recon_id,
+    main.target_table

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/08_0_drill_down_header.md

@@ -0,0 +1,3 @@
+# Drill Down
+
+### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues.

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/09_0_recon_id.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- dd_recon_id
+type: MULTI_SELECT
+title: Recon Id

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/09_1_category.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- dd_recon_type
+type: MULTI_SELECT
+title: Category

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/10_0_target_table.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- dd_target_table
+type: MULTI_SELECT
+title: Target Table Name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/10_1_source_table.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- dd_source_table
+type: MULTI_SELECT
+title: Source Table Name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql

@@ -0,0 +1,40 @@
+/* --title 'Recon Details Drill Down' --height 6 --width 6 */
+WITH tmp AS (
+    SELECT
+        recon_table_id,
+        inserted_ts,
+        recon_type,
+        explode(data) AS data,
+        row_number() OVER (PARTITION BY recon_table_id, recon_type ORDER BY recon_table_id) AS rn
+    FROM
+        remorph.reconcile.details
+    WHERE
+        recon_type != 'schema'
+)
+SELECT
+    main.recon_id AS dd_recon_id,
+    main.source_table.`catalog` AS source_catalog,
+    main.source_table.`schema` AS source_schema,
+    main.source_table.table_name AS source_table_name,
+    IF(
+        ISNULL(source_catalog),
+        CONCAT_WS('.', source_schema, source_table_name),
+        CONCAT_WS(
+            '.',
+            source_catalog,
+            source_schema,
+            source_table_name
+        )
+    ) AS dd_source_table,
+    main.target_table.`catalog` AS target_catalog,
+    main.target_table.`schema` AS target_schema,
+    main.target_table.table_name AS target_table_name,
+    CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS dd_target_table,
+    recon_type AS dd_recon_type,
+    key,
+    value,
+    rn
+FROM tmp
+         INNER JOIN remorph.reconcile.main main
+                    ON main.recon_table_id = tmp.recon_table_id
+    LATERAL VIEW explode(data) exploded_data AS key, value

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/12_0_daily_data_validation_issue_header.md

@@ -0,0 +1,3 @@
+# Daily Data Validation Issues Report
+
+### This summary report provides an overview of all data validation runs conducted on a specific day. It highlights whether each table has encountered any validation issues, without delving into the low-level details. This report aims to give a quick and clear status of data integrity across all tables for the day.

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/13_0_success_fail_.filter.yml

@@ -0,0 +1,4 @@
+columns:
+- start_date
+type: DATE_RANGE_PICKER
+width: 6

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/14_0_failed_recon_ids.sql

@@ -0,0 +1,15 @@
+/* --title 'Number of Distinct Recon IDs per Target Table Failed' --width 6 */
+SELECT
+    main.recon_id AS rec_id,
+    CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS t_table,
+    DATE(main.start_ts) AS start_date
+FROM
+    remorph.reconcile.main main
+    INNER JOIN remorph.reconcile.metrics metrics
+ON main.recon_table_id = metrics.recon_table_id
+WHERE
+    metrics.run_metrics.status = FALSE
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/15_0_total_failed_runs.sql

@@ -0,0 +1,10 @@
+/* --title 'Total number of runs failed' --width 2 */
+SELECT
+    main.recon_id AS rec_id,
+    DATE(main.start_ts) AS start_date
+FROM
+    remorph.reconcile.main main
+    INNER JOIN remorph.reconcile.metrics metrics
+ON main.recon_table_id = metrics.recon_table_id
+WHERE
+    metrics.run_metrics.status = FALSE

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql

@@ -0,0 +1,10 @@
+/* --title 'Unique target tables failed' --width 2 */
+SELECT
+    CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS t_table,
+    DATE(main.start_ts) AS start_date
+FROM
+    remorph.reconcile.main main
+    INNER JOIN remorph.reconcile.metrics metrics
+ON main.recon_table_id = metrics.recon_table_id
+WHERE
+    metrics.run_metrics.status = FALSE

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql

@@ -0,0 +1,10 @@
+/* --title 'Unique target tables successful' --width 2 */
+SELECT
+    CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS t_table,
+    DATE(main.start_ts) AS start_date
+FROM
+    remorph.reconcile.main main
+    INNER JOIN remorph.reconcile.metrics metrics
+ON main.recon_table_id = metrics.recon_table_id
+WHERE
+    metrics.run_metrics.status = TRUE