databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/16_0_missing_mismatch_header.md

@@ -0,0 +1 @@
+# Visualization of Missing and Mismatched Records

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql

@@ -0,0 +1,14 @@
+/* --title 'Mismatched Records' --width 3 */
+SELECT
+    main.recon_id,
+    CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS target_table,
+    metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch,
+    main.start_ts AS start_ts
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql

@@ -0,0 +1,14 @@
+/* --title 'Threshold Mismatches' --width 3 */
+SELECT
+    main.recon_id,
+    CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS target_table,
+    metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch,
+    main.start_ts AS start_ts
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/18_0_missing_in_databricks.sql

@@ -0,0 +1,14 @@
+/* --title 'Missing in Databricks' --width 3 */
+SELECT
+    main.recon_id,
+    CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table,
+    metrics.recon_metrics.row_comparison.missing_in_target AS missing_in_target,
+    main.start_ts AS start_ts
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/18_1_missing_in_source.sql

@@ -0,0 +1,14 @@
+/* --title 'Missing in Source' --width 3 */
+SELECT
+    main.recon_id,
+    CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table,
+    metrics.recon_metrics.row_comparison.missing_in_source AS missing_in_source,
+    main.start_ts AS start_ts
+FROM
+    remorph.reconcile.main main
+        INNER JOIN remorph.reconcile.metrics metrics
+                   ON main.recon_table_id = metrics.recon_table_id
+ORDER BY
+    metrics.inserted_ts DESC,
+    main.recon_id,
+    main.target_table.table_name

databricks/labs/lakebridge/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml

@@ -0,0 +1,545 @@
+display_name: "Reconciliation Metrics"
+tiles:
+  05_0_summary_table:
+    overrides:
+      spec:
+        withRowNumber: true
+        encodings:
+          columns:
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: recon_id
+              title: recon_id
+              type: string
+              cellFormat:
+                default:
+                  foregroundColor:
+                rules:
+                  - if:
+                      column: status
+                      fn: '='
+                      literal: 'true'
+                    value:
+                      foregroundColor: '#3BD973'
+                  - if:
+                      column: status
+                      fn: '='
+                      literal: 'false'
+                    value:
+                      foregroundColor: '#E92828'
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_type
+              title: source_type
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: report_type
+              title: report_type
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_catalog
+              title: source_catalog
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_schema
+              title: source_schema
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_table_name
+              title: source_table_name
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_table
+              title: source_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_catalog
+              title: target_catalog
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_schema
+              title: target_schema
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_table_name
+              title: target_table_name
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_table
+              title: target_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: status
+              title: status
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: exception
+              title: exception
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: missing_in_source
+              title: missing_in_source
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: missing_in_target
+              title: missing_in_target
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: absolute_mismatch
+              title: absolute_mismatch
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: threshold_mismatch
+              title: threshold_mismatch
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: mismatch_columns
+              title: mismatch_columns
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: schema_comparison
+              title: schema_comparison
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: executed_by
+              title: executed_by
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: datetime
+              fieldName: start_ts
+              title: start_ts
+              type: datetime
+              dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS'
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: datetime
+              fieldName: end_ts
+              title: end_ts
+              type: datetime
+              dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS'
+  07_0_schema_details_table:
+    overrides:
+      spec:
+        withRowNumber: true
+        encodings:
+          columns:
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: recon_id
+              title: recon_id
+              type: string
+              cellFormat:
+                default:
+                  foregroundColor:
+                rules:
+                  - if:
+                      column: is_valid
+                      fn: '='
+                      literal: 'false'
+                    value:
+                      foregroundColor: '#E92828'
+                  - if:
+                      column: is_valid
+                      fn: '='
+                      literal: 'true'
+                    value:
+                      foregroundColor: '#3BD973'
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_catalog
+              title: source_catalog
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_schema
+              title: source_schema
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_table_name
+              title: source_table_name
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_table
+              title: source_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_catalog
+              title: target_catalog
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_schema
+              title: target_schema
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_table_name
+              title: target_table_name
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: target_table
+              title: target_table
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_column
+              title: source_column
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: source_datatype
+              title: source_datatype
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: databricks_column
+              title: databricks_column
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: databricks_datatype
+              title: databricks_datatype
+              type: string
+            - booleanValues:
+                - 'false'
+                - 'true'
+              displayAs: string
+              fieldName: is_valid
+              title: is_valid
+              type: string
+  11_0_recon_details_pivot:
+    overrides:
+      spec:
+        version: 3
+        widgetType: pivot
+        encodings:
+          rows:
+            - fieldName: dd_recon_id
+              displayName: recon_id
+            - fieldName: dd_source_table
+              displayName: source_table
+            - fieldName: dd_target_table
+              displayName: target_table
+            - fieldName: dd_recon_type
+              displayName: recon_type
+            - fieldName: rn
+              displayName: rn
+          columns:
+            - fieldName: key
+              displayName: key
+          cell:
+            fieldName: value
+            cellType: text
+            displayName: value
+  14_0_failed_recon_ids:
+    overrides:
+      spec:
+        version: 3
+        widgetType: bar
+        encodings:
+          x:
+            fieldName: t_table
+            scale:
+              type: categorical
+              sort:
+                by: y-reversed
+            displayName: Target table
+          'y':
+            fieldName: countdistinct(rec_id)
+            scale:
+              type: quantitative
+            displayName: Count of Unique Recon Ids
+          label:
+            show: true
+      queries:
+        - name: main_query
+          query:
+            datasetName: 14_0_failed_recon_ids
+            fields:
+              - name: t_table
+                expression: '`t_table`'
+              - name: countdistinct(rec_id)
+                expression: COUNT(DISTINCT `rec_id`)
+            disaggregated: false
+  15_0_total_failed_runs:
+    overrides:
+      spec:
+        version: 2
+        widgetType: counter
+        encodings:
+          value:
+            fieldName: countdistinct(rec_id)
+            displayName: countdistinct(rec_id)
+      queries:
+        - name: main_query
+          query:
+            datasetName: 15_0_total_failed_runs
+            fields:
+              - name: countdistinct(rec_id)
+                expression: 'COUNT(DISTINCT `rec_id`)'
+            disaggregated: false
+  15_1_failed_targets:
+    overrides:
+      spec:
+        version: 2
+        widgetType: counter
+        encodings:
+          value:
+            fieldName: countdistinct(t_table)
+            displayName: countdistinct(t_table)
+      queries:
+        - name: main_query
+          query:
+            datasetName: 15_1_failed_targets
+            fields:
+              - name: countdistinct(t_table)
+                expression: 'COUNT(DISTINCT `t_table`)'
+            disaggregated: false
+  15_2_successful_targets:
+    overrides:
+      spec:
+        version: 2
+        widgetType: counter
+        encodings:
+          value:
+            fieldName: countdistinct(t_table)
+            displayName: countdistinct(t_table)
+      queries:
+        - name: main_query
+          query:
+            datasetName: 15_2_successful_targets
+            fields:
+              - name: countdistinct(t_table)
+                expression: 'COUNT(DISTINCT `t_table`)'
+            disaggregated: false
+  17_0_mismatched_records:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 17_0_mismatched_records
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: absolute_mismatch
+                expression: '`absolute_mismatch`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: absolute_mismatch
+            scale:
+              type: quantitative
+            displayName: absolute_mismatch
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false
+  17_1_threshold_mismatches:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 17_1_threshold_mismatches
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: threshold_mismatch
+                expression: '`threshold_mismatch`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: threshold_mismatch
+            scale:
+              type: quantitative
+            displayName: threshold_mismatch
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false
+  18_0_missing_in_databricks:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 18_0_missing_in_databricks
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: missing_in_target
+                expression: '`missing_in_target`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: missing_in_target
+            scale:
+              type: quantitative
+            displayName: missing_in_target
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false
+  18_1_missing_in_source:
+    overrides:
+      queries:
+        - name: main_query
+          query:
+            datasetName: 18_1_missing_in_source
+            fields:
+              - name: target_table
+                expression: '`target_table`'
+              - name: hourly(start_ts)
+                expression: 'DATE_TRUNC("HOUR", `start_ts`)'
+              - name: missing_in_source
+                expression: '`missing_in_source`'
+            disaggregated: true
+      spec:
+        version: 3
+        widgetType: area
+        encodings:
+          x:
+            fieldName: hourly(start_ts)
+            scale:
+              type: temporal
+            displayName: start_ts
+          'y':
+            fieldName: missing_in_source
+            scale:
+              type: quantitative
+            displayName: missing_in_source
+          color:
+            fieldName: target_table
+            scale:
+              type: categorical
+            displayName: target_table
+          label:
+            show: false

databricks/labs/lakebridge/resources/reconcile/queries/installation/aggregate_details.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS aggregate_details (
+    recon_table_id BIGINT NOT NULL,
+    rule_id BIGINT NOT NULL,
+    recon_type STRING NOT NULL,
+    data ARRAY<MAP<STRING, STRING>> NOT NULL,
+    inserted_ts TIMESTAMP NOT NULL
+);

databricks/labs/lakebridge/resources/reconcile/queries/installation/aggregate_metrics.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS aggregate_metrics (
+    recon_table_id BIGINT NOT NULL,
+    rule_id BIGINT NOT NULL,
+    recon_metrics STRUCT<
+                                                missing_in_source: INTEGER,
+                                                missing_in_target: INTEGER,
+                                                mismatch: INTEGER
+                                           >,
+    run_metrics STRUCT<
+                        status: BOOLEAN NOT NULL,
+                        run_by_user: STRING NOT NULL,
+                        exception_message: STRING
+                       > NOT NULL,
+    inserted_ts TIMESTAMP NOT NULL
+);

databricks/labs/lakebridge/resources/reconcile/queries/installation/aggregate_rules.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS aggregate_rules (
+    rule_id BIGINT NOT NULL,
+    rule_type STRING NOT NULL,
+    rule_info MAP<STRING, STRING> NOT NULL,
+    inserted_ts TIMESTAMP NOT NULL
+);

databricks/labs/lakebridge/resources/reconcile/queries/installation/details.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS details (
+    recon_table_id BIGINT NOT NULL,
+    recon_type STRING NOT NULL,
+    status BOOLEAN NOT NULL,
+    data ARRAY<MAP<STRING, STRING>> NOT NULL,
+    inserted_ts TIMESTAMP NOT NULL
+);

databricks/labs/lakebridge/resources/reconcile/queries/installation/main.sql

@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS main (
+    recon_table_id BIGINT NOT NULL,
+    recon_id STRING NOT NULL,
+    source_type STRING NOT NULL,
+    source_table STRUCT<
+                         catalog: STRING,
+                         schema: STRING NOT NULL,
+                         table_name: STRING NOT NULL
+                        > ,
+    target_table STRUCT<
+                         catalog: STRING NOT NULL,
+                         schema: STRING NOT NULL,
+                         table_name: STRING NOT NULL
+                        > NOT NULL,
+    report_type STRING NOT NULL,
+    operation_name  STRING NOT NULL,
+    start_ts TIMESTAMP,
+    end_ts TIMESTAMP
+)
+TBLPROPERTIES (
+    'delta.columnMapping.mode' = 'name',
+    'delta.minReaderVersion' = '2',
+    'delta.minWriterVersion' = '5'
+);