databricks-labs-lakebridge 0.10.7__py3-none-any.whl → 0.10.8__py3-none-any.whl

databricks/labs/lakebridge/__about__.py

@@ -1,2 +1,2 @@
 # DO NOT MODIFY THIS FILE
-__version__ = "0.10.7"
+__version__ = "0.10.8"

databricks/labs/lakebridge/assessments/profiler_validator.py

@@ -0,0 +1,103 @@
+import os
+from dataclasses import dataclass
+from duckdb import DuckDBPyConnection
+
+from databricks.labs.lakebridge.assessments.pipeline import PipelineClass
+
+PROFILER_DB_NAME = "profiler_extract.db"
+
+
+@dataclass(frozen=True)
+class ValidationOutcome:
+    """A data class that holds the outcome of a table validation check."""
+
+    table: str
+    column: str | None
+    strategy: str
+    outcome: str
+    severity: str
+
+
+class ValidationStrategy:
+    """Abstract class for validating a Profiler table"""
+
+    def validate(self, connection: DuckDBPyConnection) -> ValidationOutcome:
+        raise NotImplementedError
+
+
+class NullValidationCheck(ValidationStrategy):
+    """Concrete class for validating null values in a profiler table"""
+
+    def __init__(self, table, column, severity="WARN"):
+        self.name = self.__class__.__name__
+        self.table = table
+        self.column = column
+        self.severity = severity
+
+    def validate(self, connection: DuckDBPyConnection) -> ValidationOutcome:
+        """
+        Validates that a column does not contain null values.
+        input:
+          connection: a DuckDB connection object
+        """
+        result = connection.execute(f"SELECT COUNT(*) FROM {self.table} WHERE {self.column} IS NULL").fetchone()
+        if result:
+            row_count = result[0]
+            outcome = "FAIL" if row_count > 0 else "PASS"
+        else:
+            outcome = "FAIL"
+        return ValidationOutcome(self.table, self.column, self.name, outcome, self.severity)
+
+
+class EmptyTableValidationCheck(ValidationStrategy):
+    """Concrete class for validating empty tables from a profiler run."""
+
+    def __init__(self, table, severity="WARN"):
+        self.name = self.__class__.__name__
+        self.table = table
+        self.severity = severity
+
+    def validate(self, connection) -> ValidationOutcome:
+        """Validates that a table is not empty.
+        input:
+          connection: a DuckDB connection object
+        returns:
+          a ValidationOutcome object
+        """
+        result = connection.execute(f"SELECT COUNT(*) FROM {self.table}").fetchone()
+        if result:
+            row_count = result[0]
+            outcome = "PASS" if row_count > 0 else "FAIL"
+        else:
+            outcome = "FAIL"
+        return ValidationOutcome(self.table, None, self.name, outcome, self.severity)
+
+
+def get_profiler_extract_path(pipeline_config_path: str) -> str:
+    """
+    Returns the filesystem path of the profiler extract database.
+    input:
+       pipeline_config_path: the location of the pipeline definition .yml file
+    returns:
+       the filesystem path to the profiler extract database
+    """
+    pipeline_config = PipelineClass.load_config_from_yaml(pipeline_config_path)
+    normalized_db_path = os.path.normpath(pipeline_config.extract_folder)
+    database_path = f"{normalized_db_path}/{PROFILER_DB_NAME}"
+    return database_path
+
+
+def build_validation_report(
+    validations: list[ValidationStrategy], connection: DuckDBPyConnection
+) -> list[ValidationOutcome]:
+    """
+    Builds a list of ValidationOutcomes from list of validation checks.
+    input:
+      validations: a list of ValidationStrategy objects
+      connection: a DuckDB connection object
+    returns: a list of ValidationOutcomes
+    """
+    validation_report = []
+    for validation in validations:
+        validation_report.append(validation.validate(connection))
+    return validation_report

databricks/labs/lakebridge/base_install.py

@@ -19,11 +19,7 @@ def main() -> None:
         WorkspaceClient(product="lakebridge", product_version=__version__),
         transpiler_repository=TranspilerRepository.user_home(),
     )
-    if installer.has_installed_transpilers():
-        logger.warning(
-            "Detected existing Lakebridge transpilers; run 'databricks labs lakebridge install-transpile' to upgrade them."
-        )
-    else:
+    if not installer.upgrade_installed_transpilers():
         logger.debug("No existing Lakebridge transpilers detected; assuming fresh installation.")
 
     logger.info("Successfully Setup Lakebridge Components Locally")

databricks/labs/lakebridge/cli.py

@@ -74,6 +74,7 @@ def _remove_warehouse(ws: WorkspaceClient, warehouse_id: str):
 
 @lakebridge.command
 def transpile(
+    *,
     w: WorkspaceClient,
     transpiler_config_path: str | None = None,
     source_dialect: str | None = None,
@@ -340,6 +341,8 @@ class _TranspileConfigChecker:
                 supported_dialects = ", ".join(self._transpiler_repository.all_dialects())
                 msg = f"{msg_prefix}: {source_dialect!r} (supported dialects: {supported_dialects})"
                 raise_validation_exception(msg)
+            else:
+                self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
         else:
             # Check the source dialect against the engine.
             if source_dialect not in engine.supported_dialects:
@@ -366,6 +369,7 @@ class _TranspileConfigChecker:
                 source_dialect = self._prompts.choice("Select the source dialect:", list(supported_dialects))
         engine = self._configure_transpiler_config_path(source_dialect)
         assert engine is not None, "No transpiler engine available for a supported dialect; configuration is invalid."
+        self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
         return engine
 
     def _check_lsp_engine(self) -> TranspileEngine:
@@ -518,7 +522,7 @@ def _override_workspace_client_config(ctx: ApplicationContext, overrides: dict[s
 
 
 @lakebridge.command
-def reconcile(w: WorkspaceClient) -> None:
+def reconcile(*, w: WorkspaceClient) -> None:
     """[EXPERIMENTAL] Reconciles source to Databricks datasets"""
     with_user_agent_extra("cmd", "execute-reconcile")
     ctx = ApplicationContext(w)
@@ -534,7 +538,7 @@ def reconcile(w: WorkspaceClient) -> None:
 
 
 @lakebridge.command
-def aggregates_reconcile(w: WorkspaceClient) -> None:
+def aggregates_reconcile(*, w: WorkspaceClient) -> None:
     """[EXPERIMENTAL] Reconciles Aggregated source to Databricks datasets"""
     with_user_agent_extra("cmd", "execute-aggregates-reconcile")
     ctx = ApplicationContext(w)
@@ -552,8 +556,8 @@ def aggregates_reconcile(w: WorkspaceClient) -> None:
 
 @lakebridge.command
 def generate_lineage(
-    w: WorkspaceClient,
     *,
+    w: WorkspaceClient,
     source_dialect: str | None = None,
     input_source: str,
     output_folder: str,
@@ -578,7 +582,7 @@ def generate_lineage(
 
 
 @lakebridge.command
-def configure_secrets(w: WorkspaceClient) -> None:
+def configure_secrets(*, w: WorkspaceClient) -> None:
     """Setup reconciliation connection profile details as Secrets on Databricks Workspace"""
     recon_conf = ReconConfigPrompts(w)
 
@@ -604,8 +608,9 @@ def configure_database_profiler() -> None:
     assessment.run()
 
 
-@lakebridge.command()
+@lakebridge.command
 def install_transpile(
+    *,
     w: WorkspaceClient,
     artifact: str | None = None,
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
@@ -622,6 +627,7 @@ def install_transpile(
 
 @lakebridge.command(is_unauthenticated=False)
 def configure_reconcile(
+    *,
     w: WorkspaceClient,
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ) -> None:
@@ -637,8 +643,9 @@ def configure_reconcile(
     reconcile_installer.run(module="reconcile")
 
 
-@lakebridge.command()
+@lakebridge.command
 def analyze(
+    *,
     w: WorkspaceClient,
     source_directory: str | None = None,
     report_file: str | None = None,

databricks/labs/lakebridge/helpers/validation.py

@@ -37,19 +37,21 @@ class Validator:
             config.catalog_name,
             config.schema_name,
         )
+        # Some errors doesn't return the query test alon with the error message so need to handle those separately
+        static_errors_lkp = ["[UNRESOLVED_ROUTINE]", "[UNRESOLVED_COLUMN.WITHOUT_SUGGESTION]"]
         if is_valid:
             result = sql_text
             if exception_type is not None:
                 exception_msg = f"[{exception_type.upper()}]: {exception_msg}"
         else:
             query = ""
-            if "[UNRESOLVED_ROUTINE]" in str(exception_msg):
+            if any(err in str(exception_msg) for err in static_errors_lkp):
                 query = sql_text
             buffer = StringIO()
             buffer.write("-------------- Exception Start-------------------\n")
-            buffer.write("/* \n")
+            buffer.write("/*\n")
             buffer.write(str(exception_msg))
-            buffer.write("\n */ \n")
+            buffer.write("\n*/\n")
             buffer.write(query)
             buffer.write("\n ---------------Exception End --------------------\n")