databricks-labs-lakebridge 0.10.6__py3-none-any.whl → 0.10.7__py3-none-any.whl

databricks/labs/lakebridge/__about__.py

@@ -1,2 +1,2 @@
 # DO NOT MODIFY THIS FILE
-__version__ = "0.10.6"
+__version__ = "0.10.7"

databricks/labs/lakebridge/analyzer/lakebridge_analyzer.py

@@ -0,0 +1,95 @@
+import tempfile
+from pathlib import Path
+
+from databricks.sdk.service.iam import User
+from databricks.sdk.core import with_user_agent_extra
+
+from databricks.labs.blueprint.entrypoint import get_logger
+from databricks.labs.blueprint.tui import Prompts
+
+from databricks.labs.bladespector.analyzer import Analyzer, _PLATFORM_TO_SOURCE_TECHNOLOGY
+
+from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
+from databricks.labs.lakebridge.helpers.file_utils import check_path, move_tmp_file
+
+logger = get_logger(__file__)
+
+
+class LakebridgeAnalyzer(Analyzer):
+    def __init__(self, current_user: User, prompts: Prompts, is_debug: bool = False):
+        self._current_user = current_user
+        self._prompts = prompts
+        self._is_debug = is_debug
+        super().__init__()
+
+    def _get_source_directory(self) -> Path:
+        """Get and validate the source directory from user input."""
+        directory_str = self._prompts.question(
+            "Enter full path to the source directory",
+            default=Path.cwd().as_posix(),
+            validate=check_path,
+        )
+        return Path(directory_str).resolve()
+
+    def _get_result_file_path(self, directory: Path) -> Path:
+        """Get the result file path - accepts either filename or full path."""
+        filename = self._prompts.question(
+            "Enter report file name or custom export path including file name without extension",
+            default=f"{directory.as_posix()}/lakebridge-analyzer-results.xlsx",
+            validate=check_path,
+        )
+        return directory / Path(filename) if len(filename.split("/")) == 1 else Path(filename)
+
+    def _get_source_tech(self, platform: str | None = None) -> str:
+        """Validate source technology or prompt for a valid source"""
+        if platform is None or platform not in self.supported_source_technologies():
+            if platform is not None:
+                logger.warning(f"Invalid source technology {platform}")
+            platform = self._prompts.choice("Select the source technology", self.supported_source_technologies())
+            with_user_agent_extra("analyzer_source_tech", make_alphanum_or_semver(platform))
+            logger.debug(f"User: {self._current_user}")
+        return _PLATFORM_TO_SOURCE_TECHNOLOGY[platform]
+
+    @staticmethod
+    def _temp_xlsx_path(results_dir: Path | str) -> Path:
+        return (Path(tempfile.mkdtemp()) / Path(results_dir).name).with_suffix(".xlsx")
+
+    def _run_prompt_analyzer(self):
+        """Run the analyzer: prompt guided"""
+        source_dir = self._get_source_directory()
+        results_dir = self._get_result_file_path(source_dir)
+        tmp_dir = self._temp_xlsx_path(results_dir)
+        technology = self._get_source_tech()
+
+        self._run_binary(source_dir, tmp_dir, technology, self._is_debug)
+
+        move_tmp_file(tmp_dir, results_dir)
+
+        logger.info(f"Successfully Analyzed files in ${source_dir} for ${technology} and saved report to {results_dir}")
+
+    def _run_arg_analyzer(self, source_dir: str | None, results_dir: str | None, technology: str | None):
+        """Run the analyzer: arg guided"""
+        if source_dir is None or results_dir is None or technology is None:
+            logger.error("All arguments (--source-directory, --report-file, --source-tech) must be provided")
+            return
+
+        if check_path(source_dir) and check_path(results_dir):
+            tmp_dir = self._temp_xlsx_path(results_dir)
+            technology = self._get_source_tech(technology)
+            self._run_binary(Path(source_dir), tmp_dir, technology, self._is_debug)
+
+            move_tmp_file(tmp_dir, Path(results_dir))
+
+            logger.info(
+                f"Successfully Analyzed files in ${source_dir} for ${technology} and saved report to {results_dir}"
+            )
+
+    def run_analyzer(
+        self, source_dir: str | None = None, results_dir: str | None = None, technology: str | None = None
+    ):
+        """Run the analyzer."""
+        if not any([source_dir, results_dir, technology]):
+            self._run_prompt_analyzer()
+            return
+
+        self._run_arg_analyzer(source_dir, results_dir, technology)

databricks/labs/lakebridge/base_install.py

@@ -1,13 +1,34 @@
 from databricks.labs.blueprint.logger import install_logger
 from databricks.labs.blueprint.entrypoint import get_logger
+from databricks.sdk import WorkspaceClient
 from databricks.sdk.core import with_user_agent_extra
 
-install_logger()
-with_user_agent_extra("cmd", "install")
+from databricks.labs.lakebridge import __version__
+from databricks.labs.lakebridge.install import installer as _installer
+from databricks.labs.lakebridge.transpiler.repository import TranspilerRepository
+
+
+def main() -> None:
+    install_logger()
+    with_user_agent_extra("cmd", "install")
 
-if __name__ == "__main__":
     logger = get_logger(__file__)
     logger.setLevel("INFO")
 
+    installer = _installer(
+        WorkspaceClient(product="lakebridge", product_version=__version__),
+        transpiler_repository=TranspilerRepository.user_home(),
+    )
+    if installer.has_installed_transpilers():
+        logger.warning(
+            "Detected existing Lakebridge transpilers; run 'databricks labs lakebridge install-transpile' to upgrade them."
+        )
+    else:
+        logger.debug("No existing Lakebridge transpilers detected; assuming fresh installation.")
+
     logger.info("Successfully Setup Lakebridge Components Locally")
     logger.info("For more information, please visit https://databrickslabs.github.io/lakebridge/")
+
+
+if __name__ == "__main__":
+    main()

databricks/labs/lakebridge/cli.py

@@ -19,20 +19,17 @@ from databricks.labs.blueprint.entrypoint import get_logger, is_in_debug
 from databricks.labs.blueprint.installation import RootJsonValue
 from databricks.labs.blueprint.tui import Prompts
 
-from databricks.labs.bladespector.analyzer import Analyzer
-
 
 from databricks.labs.lakebridge.assessments.configure_assessment import (
     create_assessment_configurator,
     PROFILER_SOURCE_SYSTEM,
 )
 
-from databricks.labs.lakebridge.__about__ import __version__
 from databricks.labs.lakebridge.config import TranspileConfig
 from databricks.labs.lakebridge.contexts.application import ApplicationContext
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
-from databricks.labs.lakebridge.install import WorkspaceInstaller
+from databricks.labs.lakebridge.install import installer
 from databricks.labs.lakebridge.reconcile.runner import ReconcileRunner
 from databricks.labs.lakebridge.lineage import lineage_generator
 from databricks.labs.lakebridge.reconcile.recon_config import RECONCILE_OPERATION_NAME, AGG_RECONCILE_OPERATION_NAME
@@ -52,20 +49,6 @@ def raise_validation_exception(msg: str) -> NoReturn:
     raise ValueError(msg)
 
 
-def _installer(ws: WorkspaceClient, transpiler_repository: TranspilerRepository) -> WorkspaceInstaller:
-    app_context = ApplicationContext(_verify_workspace_client(ws))
-    return WorkspaceInstaller(
-        app_context.workspace_client,
-        app_context.prompts,
-        app_context.installation,
-        app_context.install_state,
-        app_context.product_info,
-        app_context.resource_configurator,
-        app_context.workspace_installation,
-        transpiler_repository=transpiler_repository,
-    )
-
-
 def _create_warehouse(ws: WorkspaceClient) -> str:
 
     dbsql = ws.warehouses.create_and_wait(
@@ -89,19 +72,6 @@ def _remove_warehouse(ws: WorkspaceClient, warehouse_id: str):
     logger.info(f"Removed warehouse post installation with id: {warehouse_id}")
 
 
-def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient:
-    """
-    [Private] Verifies and updates the workspace client configuration.
-    """
-
-    # Using reflection to set right value for _product_info for telemetry
-    product_info = getattr(ws.config, '_product_info')
-    if product_info[0] != "lakebridge":
-        setattr(ws.config, '_product_info', ('lakebridge', __version__))
-
-    return ws
-
-
 @lakebridge.command
 def transpile(
     w: WorkspaceClient,
@@ -358,7 +328,7 @@ class _TranspileConfigChecker:
         transpiler_config_path = self._transpiler_repository.transpiler_config_path(transpiler_name)
         logger.info(f"Lakebridge will use the {transpiler_name} transpiler.")
         self._config = dataclasses.replace(self._config, transpiler_config_path=str(transpiler_config_path))
-        return TranspileEngine.load_engine(transpiler_config_path)
+        return LSPEngine.from_config_path(transpiler_config_path)
 
     def _configure_source_dialect(
         self, source_dialect: str, engine: TranspileEngine | None, msg_prefix: str
@@ -426,14 +396,15 @@ class _TranspileConfigChecker:
         #
 
         # Step 1: Check the transpiler config path.
+        engine: TranspileEngine | None
         transpiler_config_path = self._config.transpiler_config_path
         if transpiler_config_path is not None:
             self._validate_transpiler_config_path(
                 transpiler_config_path,
-                f"Invalid transpiler path configured, path does not exist: {transpiler_config_path}",
+                f"Error: Invalid value for '--transpiler-config-path': '{str(transpiler_config_path)}', file does not exist.",
             )
             path = Path(transpiler_config_path)
-            engine = TranspileEngine.load_engine(path)
+            engine = LSPEngine.from_config_path(path)
         else:
             engine = None
         del transpiler_config_path
@@ -639,14 +610,14 @@ def install_transpile(
     artifact: str | None = None,
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ) -> None:
-    """Install the Lakebridge transpilers"""
+    """Install or upgrade the Lakebridge transpilers."""
     with_user_agent_extra("cmd", "install-transpile")
     if artifact:
         with_user_agent_extra("artifact-overload", Path(artifact).name)
     user = w.current_user
     logger.debug(f"User: {user}")
-    installer = _installer(w, transpiler_repository)
-    installer.run(module="transpile", artifact=artifact)
+    transpile_installer = installer(w, transpiler_repository)
+    transpile_installer.run(module="transpile", artifact=artifact)
 
 
 @lakebridge.command(is_unauthenticated=False)
@@ -662,28 +633,23 @@ def configure_reconcile(
         dbsql_id = _create_warehouse(w)
         w.config.warehouse_id = dbsql_id
     logger.debug(f"Warehouse ID used for configuring reconcile: {w.config.warehouse_id}.")
-    installer = _installer(w, transpiler_repository)
-    installer.run(module="reconcile")
+    reconcile_installer = installer(w, transpiler_repository)
+    reconcile_installer.run(module="reconcile")
 
 
 @lakebridge.command()
-def analyze(w: WorkspaceClient, source_directory: str, report_file: str, source_tech: str | None = None) -> None:
+def analyze(
+    w: WorkspaceClient,
+    source_directory: str | None = None,
+    report_file: str | None = None,
+    source_tech: str | None = None,
+):
     """Run the Analyzer"""
     with_user_agent_extra("cmd", "analyze")
     ctx = ApplicationContext(w)
-    prompts = ctx.prompts
-    output_file = report_file
-    input_folder = source_directory
-    if source_tech is None:
-        source_tech = prompts.choice("Select the source technology", Analyzer.supported_source_technologies())
-    with_user_agent_extra("analyzer_source_tech", make_alphanum_or_semver(source_tech))
-    user = ctx.current_user
-    logger.debug(f"User: {user}")
-    is_debug = logger.getEffectiveLevel() == logging.DEBUG
-    Analyzer.analyze(Path(input_folder), Path(output_file), source_tech, is_debug=is_debug)
-    logger.info(
-        f"Successfully Analyzed files in ${source_directory} for ${source_tech} and saved report to {report_file}"
-    )
+
+    logger.debug(f"User: {ctx.current_user}")
+    ctx.analyzer.run_analyzer(source_directory, report_file, source_tech)
 
 
 if __name__ == "__main__":

databricks/labs/lakebridge/contexts/application.py

@@ -12,6 +12,7 @@ from databricks.sdk.config import Config
 from databricks.sdk.errors import NotFound
 from databricks.sdk.service.iam import User
 
+from databricks.labs.lakebridge.analyzer.lakebridge_analyzer import LakebridgeAnalyzer
 from databricks.labs.lakebridge.config import TranspileConfig, ReconcileConfig, LakebridgeConfiguration
 from databricks.labs.lakebridge.deployment.configurator import ResourceConfigurator
 from databricks.labs.lakebridge.deployment.dashboard import DashboardDeployment
@@ -22,6 +23,7 @@ from databricks.labs.lakebridge.helpers.metastore import CatalogOperations
 logger = logging.getLogger(__name__)
 
 
+# pylint: disable=too-many-public-methods
 class ApplicationContext:
     def __init__(self, ws: WorkspaceClient):
         self._ws = ws
@@ -131,3 +133,8 @@ class ApplicationContext:
     @cached_property
     def upgrades(self):
         return Upgrades(self.product_info, self.installation)
+
+    @cached_property
+    def analyzer(self):
+        is_debug = logger.getEffectiveLevel() == logging.DEBUG
+        return LakebridgeAnalyzer(self.current_user, self.prompts, is_debug)

databricks/labs/lakebridge/deployment/job.py

@@ -112,8 +112,8 @@ class JobDeployment:
         libraries = [
             compute.Library(whl=remorph_wheel_path),
         ]
-        source = recon_config.data_source
-        if source == ReconSourceType.ORACLE.value:
+
+        if recon_config.data_source == ReconSourceType.ORACLE.value:
             # TODO: Automatically fetch a version list for `ojdbc8`
             oracle_driver_version = "23.4.0.24.05"
             libraries.append(

databricks/labs/lakebridge/helpers/file_utils.py

@@ -1,8 +1,13 @@
 import contextlib
+import logging
 import os
 from pathlib import Path
+from shutil import move, Error
+from datetime import datetime
 from collections.abc import Generator
 
+logger = logging.getLogger(__name__)
+
 
 def is_sql_file(file: str | Path) -> bool:
     """
@@ -63,3 +68,34 @@ def chdir(new_path: Path) -> Generator[None, None, None]:
         yield
     finally:
         os.chdir(saved_path)
+
+
+def check_path(path: str) -> bool:
+    """Validates a path for both existing files and writable files."""
+    try:
+        path_obj = Path(path) if not isinstance(path, Path) else path
+
+        if path_obj.exists():
+            return os.access(path_obj, os.W_OK)
+
+        parent = path_obj.parent
+        return parent.exists() and os.access(parent, os.W_OK)
+
+    except OSError as e:
+        logger.warning(f"Could not validate path: {path}, error: {e}")
+        return False
+
+
+def move_tmp_file(tmp_path: Path, output_path: Path) -> None:
+    """Process file from a temp directory"""
+    try:
+        move(tmp_path, output_path.parent)
+    except (FileExistsError, Error):
+        timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        new_output_path = output_path.parent / timestamp
+        new_output_path.mkdir(exist_ok=True)
+
+        move(tmp_path, new_output_path)
+    finally:
+        tmp_path.parent.rmdir()
+        logger.info(f"Results store at {output_path}")