databricks-labs-lakebridge 0.10.4__tar.gz → 0.10.6__tar.gz

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-labs-lakebridge
-Version: 0.10.4
+Version: 0.10.6
 Summary: Fast and predictable migrations to Databricks Lakehouse Platform. This tool is designed to help you migrate your data and workloads to the Databricks Lakehouse Platform in a fast, predictable, and reliable way. It provides a set of tools and utilities to help you reconcile your data and workloads, assess your current state, and plan your migration.
 Project-URL: Documentation, https://databrickslabs.github.io/lakebridge
 Project-URL: Issues, https://github.com/databrickslabs/lakebridge/issues
@@ -26,7 +26,7 @@ Classifier: Topic :: Utilities
 Requires-Python: >=3.10
 Requires-Dist: cryptography<45.1.0,>=44.0.2
 Requires-Dist: databricks-bb-analyzer~=0.1.9
-Requires-Dist: databricks-labs-blueprint[yaml]<0.12.0,>=0.11.2
+Requires-Dist: databricks-labs-blueprint[yaml]<0.12.0,>=0.11.3
 Requires-Dist: databricks-labs-lsql==0.16.0
 Requires-Dist: databricks-sdk~=0.51.0
 Requires-Dist: duckdb~=1.2.2

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/databricks/labs/lakebridge/__about__.py

@@ -1,2 +1,2 @@
 # DO NOT MODIFY THIS FILE
-__version__ = "0.10.4"
+__version__ = "0.10.6"

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/databricks/labs/lakebridge/cli.py

@@ -33,14 +33,12 @@ from databricks.labs.lakebridge.contexts.application import ApplicationContext
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
 from databricks.labs.lakebridge.install import WorkspaceInstaller
-from databricks.labs.lakebridge.install import TranspilerInstaller
 from databricks.labs.lakebridge.reconcile.runner import ReconcileRunner
 from databricks.labs.lakebridge.lineage import lineage_generator
 from databricks.labs.lakebridge.reconcile.recon_config import RECONCILE_OPERATION_NAME, AGG_RECONCILE_OPERATION_NAME
 from databricks.labs.lakebridge.transpiler.execute import transpile as do_transpile
-
-
 from databricks.labs.lakebridge.transpiler.lsp.lsp_engine import LSPEngine
+from databricks.labs.lakebridge.transpiler.repository import TranspilerRepository
 from databricks.labs.lakebridge.transpiler.sqlglot.sqlglot_engine import SqlglotEngine
 from databricks.labs.lakebridge.transpiler.transpile_engine import TranspileEngine
 
@@ -54,7 +52,7 @@ def raise_validation_exception(msg: str) -> NoReturn:
     raise ValueError(msg)
 
 
-def _installer(ws: WorkspaceClient) -> WorkspaceInstaller:
+def _installer(ws: WorkspaceClient, transpiler_repository: TranspilerRepository) -> WorkspaceInstaller:
     app_context = ApplicationContext(_verify_workspace_client(ws))
     return WorkspaceInstaller(
         app_context.workspace_client,
@@ -64,6 +62,7 @@ def _installer(ws: WorkspaceClient) -> WorkspaceInstaller:
         app_context.product_info,
         app_context.resource_configurator,
         app_context.workspace_installation,
+        transpiler_repository=transpiler_repository,
     )
 
 
@@ -114,12 +113,13 @@ def transpile(
     skip_validation: str | None = None,
     catalog_name: str | None = None,
     schema_name: str | None = None,
+    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ):
     """Transpiles source dialect to databricks dialect"""
     ctx = ApplicationContext(w)
     logger.debug(f"Preconfigured transpiler config: {ctx.transpile_config!r}")
     with_user_agent_extra("cmd", "execute-transpile")
-    checker = _TranspileConfigChecker(ctx.transpile_config, ctx.prompts)
+    checker = _TranspileConfigChecker(ctx.transpile_config, ctx.prompts, transpiler_repository)
     checker.use_transpiler_config_path(transpiler_config_path)
     checker.use_source_dialect(source_dialect)
     checker.use_input_source(input_source)
@@ -188,14 +188,19 @@ class _TranspileConfigChecker:
 
     _config: TranspileConfig
     """The workspace configuration for transpiling, updated from command-line arguments."""
-    # _engine: TranspileEngine | None
-    # """The transpiler engine to use for transpiling, lazily loaded based on the configuration."""
     _prompts: Prompts
     """Prompting system, for requesting configuration that hasn't been provided."""
     _source_dialect_override: str | None = None
     """The source dialect provided on the command-line, if any."""
-
-    def __init__(self, config: TranspileConfig | None, prompts: Prompts) -> None:
+    _transpiler_repository: TranspilerRepository
+    """The repository where available transpilers are installed."""
+
+    def __init__(
+        self,
+        config: TranspileConfig | None,
+        prompts: Prompts,
+        transpiler_repository: TranspilerRepository,
+    ) -> None:
         if config is None:
             logger.warning(
                 "No workspace transpile configuration, use 'install-transpile' to (re)install and configure; using defaults for now."
@@ -203,6 +208,7 @@ class _TranspileConfigChecker:
             config = TranspileConfig()
         self._config = config
         self._prompts = prompts
+        self._transpiler_repository = transpiler_repository
         self._source_dialect_override = None
 
     @staticmethod
@@ -334,7 +340,7 @@ class _TranspileConfigChecker:
     def _configure_transpiler_config_path(self, source_dialect: str) -> TranspileEngine | None:
         """Configure the transpiler config path based on the requested source dialect."""
         # Names of compatible transpiler engines for the given dialect.
-        compatible_transpilers = TranspilerInstaller.transpilers_with_dialect(source_dialect)
+        compatible_transpilers = self._transpiler_repository.transpilers_with_dialect(source_dialect)
         match len(compatible_transpilers):
             case 0:
                 # Nothing found for the specified dialect, fail.
@@ -349,7 +355,7 @@ class _TranspileConfigChecker:
                     f"Multiple transpilers available for dialect {source_dialect!r}: {compatible_transpilers!r}"
                 )
                 transpiler_name = self._prompts.choice("Select the transpiler:", list(compatible_transpilers))
-        transpiler_config_path = TranspilerInstaller.transpiler_config_path(transpiler_name)
+        transpiler_config_path = self._transpiler_repository.transpiler_config_path(transpiler_name)
         logger.info(f"Lakebridge will use the {transpiler_name} transpiler.")
         self._config = dataclasses.replace(self._config, transpiler_config_path=str(transpiler_config_path))
         return TranspileEngine.load_engine(transpiler_config_path)
@@ -361,7 +367,7 @@ class _TranspileConfigChecker:
         if engine is None:
             engine = self._configure_transpiler_config_path(source_dialect)
             if engine is None:
-                supported_dialects = ", ".join(TranspilerInstaller.all_dialects())
+                supported_dialects = ", ".join(self._transpiler_repository.all_dialects())
                 msg = f"{msg_prefix}: {source_dialect!r} (supported dialects: {supported_dialects})"
                 raise_validation_exception(msg)
         else:
@@ -375,7 +381,7 @@ class _TranspileConfigChecker:
 
     def _prompt_source_dialect(self) -> TranspileEngine:
         # This is similar to the post-install prompting for the source dialect.
-        supported_dialects = TranspilerInstaller.all_dialects()
+        supported_dialects = self._transpiler_repository.all_dialects()
         match len(supported_dialects):
             case 0:
                 msg = "No transpilers are available, install using 'install-transpile' or use --transpiler-conf-path'."
@@ -522,7 +528,7 @@ async def _transpile(ctx: ApplicationContext, config: TranspileConfig, engine: T
     return [status]
 
 
-def _override_workspace_client_config(ctx: ApplicationContext, overrides: dict[str, str] | None):
+def _override_workspace_client_config(ctx: ApplicationContext, overrides: dict[str, str] | None) -> None:
     """
     Override the Workspace client's SDK config with the user provided SDK config.
     Users can provide the cluster_id and warehouse_id during the installation.
@@ -541,7 +547,7 @@ def _override_workspace_client_config(ctx: ApplicationContext, overrides: dict[s
 
 
 @lakebridge.command
-def reconcile(w: WorkspaceClient):
+def reconcile(w: WorkspaceClient) -> None:
     """[EXPERIMENTAL] Reconciles source to Databricks datasets"""
     with_user_agent_extra("cmd", "execute-reconcile")
     ctx = ApplicationContext(w)
@@ -557,7 +563,7 @@ def reconcile(w: WorkspaceClient):
 
 
 @lakebridge.command
-def aggregates_reconcile(w: WorkspaceClient):
+def aggregates_reconcile(w: WorkspaceClient) -> None:
     """[EXPERIMENTAL] Reconciles Aggregated source to Databricks datasets"""
     with_user_agent_extra("cmd", "execute-aggregates-reconcile")
     ctx = ApplicationContext(w)
@@ -574,7 +580,13 @@ def aggregates_reconcile(w: WorkspaceClient):
 
 
 @lakebridge.command
-def generate_lineage(w: WorkspaceClient, *, source_dialect: str | None = None, input_source: str, output_folder: str):
+def generate_lineage(
+    w: WorkspaceClient,
+    *,
+    source_dialect: str | None = None,
+    input_source: str,
+    output_folder: str,
+) -> None:
     """[Experimental] Generates a lineage of source SQL files or folder"""
     ctx = ApplicationContext(w)
     logger.debug(f"User: {ctx.current_user}")
@@ -595,7 +607,7 @@ def generate_lineage(w: WorkspaceClient, *, source_dialect: str | None = None, i
 
 
 @lakebridge.command
-def configure_secrets(w: WorkspaceClient):
+def configure_secrets(w: WorkspaceClient) -> None:
     """Setup reconciliation connection profile details as Secrets on Databricks Workspace"""
     recon_conf = ReconConfigPrompts(w)
 
@@ -607,7 +619,7 @@ def configure_secrets(w: WorkspaceClient):
 
 
 @lakebridge.command(is_unauthenticated=True)
-def configure_database_profiler():
+def configure_database_profiler() -> None:
     """[Experimental] Install the lakebridge Assessment package"""
     prompts = Prompts()
 
@@ -622,32 +634,40 @@ def configure_database_profiler():
 
 
 @lakebridge.command()
-def install_transpile(w: WorkspaceClient, artifact: str | None = None):
+def install_transpile(
+    w: WorkspaceClient,
+    artifact: str | None = None,
+    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
+) -> None:
     """Install the Lakebridge transpilers"""
     with_user_agent_extra("cmd", "install-transpile")
     if artifact:
         with_user_agent_extra("artifact-overload", Path(artifact).name)
     user = w.current_user
     logger.debug(f"User: {user}")
-    installer = _installer(w)
+    installer = _installer(w, transpiler_repository)
     installer.run(module="transpile", artifact=artifact)
 
 
 @lakebridge.command(is_unauthenticated=False)
-def configure_reconcile(w: WorkspaceClient):
+def configure_reconcile(
+    w: WorkspaceClient,
+    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
+) -> None:
     """Configure the Lakebridge reconciliation module"""
     with_user_agent_extra("cmd", "configure-reconcile")
     user = w.current_user
     logger.debug(f"User: {user}")
-    dbsql_id = _create_warehouse(w)
-    w.config.warehouse_id = dbsql_id
-    installer = _installer(w)
+    if not w.config.warehouse_id:
+        dbsql_id = _create_warehouse(w)
+        w.config.warehouse_id = dbsql_id
+    logger.debug(f"Warehouse ID used for configuring reconcile: {w.config.warehouse_id}.")
+    installer = _installer(w, transpiler_repository)
     installer.run(module="reconcile")
-    _remove_warehouse(w, dbsql_id)
 
 
 @lakebridge.command()
-def analyze(w: WorkspaceClient, source_directory: str, report_file: str, source_tech: str | None = None):
+def analyze(w: WorkspaceClient, source_directory: str, report_file: str, source_tech: str | None = None) -> None:
     """Run the Analyzer"""
     with_user_agent_extra("cmd", "analyze")
     ctx = ApplicationContext(w)

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/databricks/labs/lakebridge/config.py

@@ -185,6 +185,6 @@ class ReconcileConfig:
 
 
 @dataclass
-class RemorphConfigs:
+class LakebridgeConfiguration:
     transpile: TranspileConfig | None = None
     reconcile: ReconcileConfig | None = None

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/databricks/labs/lakebridge/contexts/application.py

@@ -12,7 +12,7 @@ from databricks.sdk.config import Config
 from databricks.sdk.errors import NotFound
 from databricks.sdk.service.iam import User
 
-from databricks.labs.lakebridge.config import TranspileConfig, ReconcileConfig, RemorphConfigs
+from databricks.labs.lakebridge.config import TranspileConfig, ReconcileConfig, LakebridgeConfiguration
 from databricks.labs.lakebridge.deployment.configurator import ResourceConfigurator
 from databricks.labs.lakebridge.deployment.dashboard import DashboardDeployment
 from databricks.labs.lakebridge.deployment.installation import WorkspaceInstallation
@@ -42,7 +42,7 @@ class ApplicationContext:
 
     @cached_property
     def product_info(self) -> ProductInfo:
-        return ProductInfo.from_class(RemorphConfigs)
+        return ProductInfo.from_class(LakebridgeConfiguration)
 
     @cached_property
     def installation(self) -> Installation:
@@ -65,8 +65,8 @@ class ApplicationContext:
             return None
 
     @cached_property
-    def remorph_config(self) -> RemorphConfigs:
-        return RemorphConfigs(transpile=self.transpile_config, reconcile=self.recon_config)
+    def remorph_config(self) -> LakebridgeConfiguration:
+        return LakebridgeConfiguration(transpile=self.transpile_config, reconcile=self.recon_config)
 
     @cached_property
     def connect_config(self) -> Config:

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/databricks/labs/lakebridge/deployment/dashboard.py

@@ -54,7 +54,8 @@ class DashboardDeployment:
 
         valid_dashboard_refs = set()
         for dashboard_folder in folder.iterdir():
-            if not dashboard_folder.is_dir():
+            # Make sure the directory contains a dashboard
+            if not (dashboard_folder.is_dir() and dashboard_folder.joinpath("dashboard.yml").exists()):
                 continue
             valid_dashboard_refs.add(self._dashboard_reference(dashboard_folder))
             dashboard = self._update_or_create_dashboard(dashboard_folder, parent_path, config.metadata_config)

{databricks_labs_lakebridge-0.10.4 → databricks_labs_lakebridge-0.10.6}/databricks/labs/lakebridge/deployment/installation.py

@@ -11,7 +11,7 @@ from databricks.sdk.errors import NotFound
 from databricks.sdk.mixins.compute import SemVer
 from databricks.sdk.errors.platform import InvalidParameterValue, ResourceDoesNotExist
 
-from databricks.labs.lakebridge.config import RemorphConfigs
+from databricks.labs.lakebridge.config import LakebridgeConfiguration
 from databricks.labs.lakebridge.deployment.recon import ReconDeployment
 
 logger = logging.getLogger("databricks.labs.lakebridge.install")
@@ -54,14 +54,14 @@ class WorkspaceInstallation:
         return Version(
             version=local_installed_version,
             date=local_installed_date,
-            wheel=f"databricks_labs_remorph-{local_installed_version}-py3-none-any.whl",
+            wheel=f"databricks_labs_lakebridge-{local_installed_version}-py3-none-any.whl",
         )
 
     def _get_ws_version(self):
         try:
             return self._installation.load(Version)
-        except ResourceDoesNotExist as err:
-            logger.warning(f"Unable to get Workspace Version due to: {err}")
+        except ResourceDoesNotExist:
+            logger.debug("No existing version found in workspace; assuming fresh installation.")
             return None
 
     def _apply_upgrades(self):
@@ -91,21 +91,21 @@ class WorkspaceInstallation:
             wheel_paths = [f"/Workspace{wheel}" for wheel in wheel_paths]
             return wheel_paths
 
-    def install(self, config: RemorphConfigs):
+    def install(self, config: LakebridgeConfiguration):
         self._apply_upgrades()
         wheel_paths: list[str] = self._upload_wheel()
         if config.reconcile:
-            logger.info("Installing Remorph reconcile Metadata components.")
+            logger.info("Installing Lakebridge reconcile Metadata components.")
             self._recon_deployment.install(config.reconcile, wheel_paths)
 
-    def uninstall(self, config: RemorphConfigs):
-        # This will remove all the Remorph modules
+    def uninstall(self, config: LakebridgeConfiguration):
+        # This will remove all the Lakebridge modules
         if not self._prompts.confirm(
-            "Do you want to uninstall Remorph from the workspace too, this would "
-            "remove Remorph project folder, jobs, metadata and dashboards"
+            "Do you want to uninstall Lakebridge from the workspace too, this would "
+            "remove Lakebridge project folder, jobs, metadata and dashboards"
         ):
             return
-        logger.info(f"Uninstalling Remorph from {self._ws.config.host}.")
+        logger.info(f"Uninstalling Lakebridge from {self._ws.config.host}.")
         try:
             self._installation.files()
         except NotFound: