databricks-labs-lakebridge 0.10.2__py3-none-any.whl → 0.10.3__py3-none-any.whl

databricks/labs/lakebridge/__about__.py

@@ -1,2 +1,2 @@
 # DO NOT MODIFY THIS FILE
-__version__ = "0.10.2"
+__version__ = "0.10.3"

databricks/labs/lakebridge/cli.py

@@ -4,9 +4,11 @@ import itertools
 import json
 import logging
 import os
+import re
 import time
+from collections.abc import Mapping
 from pathlib import Path
-from typing import NoReturn, cast
+from typing import NoReturn
 
 from databricks.sdk.core import with_user_agent_extra
 from databricks.sdk.service.sql import CreateWarehouseRequestWarehouseType
@@ -14,7 +16,7 @@ from databricks.sdk import WorkspaceClient
 
 from databricks.labs.blueprint.cli import App
 from databricks.labs.blueprint.entrypoint import get_logger, is_in_debug
-from databricks.labs.blueprint.installation import JsonValue
+from databricks.labs.blueprint.installation import RootJsonValue
 from databricks.labs.blueprint.tui import Prompts
 
 from databricks.labs.bladespector.analyzer import Analyzer
@@ -26,7 +28,7 @@ from databricks.labs.lakebridge.assessments.configure_assessment import (
 )
 
 from databricks.labs.lakebridge.__about__ import __version__
-from databricks.labs.lakebridge.config import TranspileConfig, LSPConfigOptionV1
+from databricks.labs.lakebridge.config import TranspileConfig
 from databricks.labs.lakebridge.contexts.application import ApplicationContext
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
@@ -38,7 +40,7 @@ from databricks.labs.lakebridge.reconcile.recon_config import RECONCILE_OPERATIO
 from databricks.labs.lakebridge.transpiler.execute import transpile as do_transpile
 
 
-from databricks.labs.lakebridge.transpiler.lsp.lsp_engine import LSPConfig
+from databricks.labs.lakebridge.transpiler.lsp.lsp_engine import LSPEngine
 from databricks.labs.lakebridge.transpiler.sqlglot.sqlglot_engine import SqlglotEngine
 from databricks.labs.lakebridge.transpiler.transpile_engine import TranspileEngine
 
@@ -115,194 +117,379 @@ def transpile(
 ):
     """Transpiles source dialect to databricks dialect"""
     ctx = ApplicationContext(w)
-    logger.debug(f"Application transpiler config: {ctx.transpile_config}")
+    logger.debug(f"Preconfigured transpiler config: {ctx.transpile_config!r}")
+    with_user_agent_extra("cmd", "execute-transpile")
     checker = _TranspileConfigChecker(ctx.transpile_config, ctx.prompts)
-    checker.check_input_source(input_source)
-    checker.check_source_dialect(source_dialect)
-    checker.check_transpiler_config_path(transpiler_config_path)
-    checker.check_transpiler_config_options()
-    checker.check_output_folder(output_folder)
-    checker.check_error_file_path(error_file_path)
-    checker.check_skip_validation(skip_validation)
-    checker.check_catalog_name(catalog_name)
-    checker.check_schema_name(schema_name)
+    checker.use_transpiler_config_path(transpiler_config_path)
+    checker.use_source_dialect(source_dialect)
+    checker.use_input_source(input_source)
+    checker.use_output_folder(output_folder)
+    checker.use_error_file_path(error_file_path)
+    checker.use_skip_validation(skip_validation)
+    checker.use_catalog_name(catalog_name)
+    checker.use_schema_name(schema_name)
     config, engine = checker.check()
+    logger.debug(f"Final configuration for transpilation: {config!r}")
+
+    assert config.source_dialect is not None, "Source dialect has been validated by this point."
+    with_user_agent_extra("transpiler_source_tech", config.source_dialect)
+    plugin_name = engine.transpiler_name
+    plugin_name = re.sub(r"\s+", "_", plugin_name)
+    with_user_agent_extra("transpiler_plugin_name", plugin_name)
+    user = ctx.current_user
+    logger.debug(f"User: {user}")
+
     result = asyncio.run(_transpile(ctx, config, engine))
     # DO NOT Modify this print statement, it is used by the CLI to display results in GO Table Template
     print(json.dumps(result))
 
 
 class _TranspileConfigChecker:
-
-    def __init__(self, config: TranspileConfig | None, prompts: Prompts):
-        if not config:
-            raise SystemExit("Installed transpile config not found. Please install lakebridge transpile first.")
-        self._config: TranspileConfig = config
+    """Helper class for the 'transpile' command to check and consolidate the configuration."""
+
+    #
+    # Configuration parameters can come from 3 sources:
+    #  - Command-line arguments (e.g., --input-source, --output-folder, etc.)
+    #  - The configuration file, stored in the user's workspace home directory.
+    #  - User prompts.
+    #
+    # The conventions are:
+    #  - Command-line arguments take precedence over the configuration file.
+    #  - Prompting is a last resort, only used when a required configuration value has not been provided and does not
+    #    have a default value.
+    #  - An invalid value results in a halt, with the error message indicating the source of the invalid value. We do
+    #    NOT attempt to recover from invalid values by looking for another source:
+    #     - Prompting unexpectedly will break scripting and automation.
+    #     - Using an alternate value will need to confusion because the behaviour will not be what the user expects.
+    #
+    # This ensures that we distinguish between:
+    #  - Invalid command-line arguments:
+    #    Resolution: fix the command-line argument value.
+    #  - Invalid prompt responses:
+    #    Resolution: provide a valid response to the prompt.
+    #  - Invalid configuration file values:
+    #    Resolution: fix the configuration file value, or provide the command-line argument to override it.
+    #
+    # Implementation details:
+    #  - For command-line arguments and prompted values, we:
+    #     - Log the raw values (prior to validation) at DEBUG level, using the repr() rendering.
+    #     - Validate the values immediately, with the error message on failure mentioning the source of the value.
+    #     - Only update the configuration if the validation passes.
+    #  - Prompting only occurs when a value is required, but not provided via the command-line argument or the
+    #    configuration file.
+    #  - In addition to the above, a final validation of everything is required: this ensures that values from the
+    #    configuration file are validated, and if we have a failure we know that's the source because other sources
+    #    were already checked.
+    #  - The interplay between the source dialect and the transpiler config path is handled with care:
+    #      - The source dialect, needs to be consistent with the engine that transpiler config path, refers to.
+    #      - The source dialect can be used to infer the transpiler config path.
+    #
+    # TODO: Refactor this class to eliminate a lof of the boilerplate and handle this more elegantly.
+
+    _config: TranspileConfig
+    """The workspace configuration for transpiling, updated from command-line arguments."""
+    # _engine: TranspileEngine | None
+    # """The transpiler engine to use for transpiling, lazily loaded based on the configuration."""
+    _prompts: Prompts
+    """Prompting system, for requesting configuration that hasn't been provided."""
+    _source_dialect_override: str | None = None
+    """The source dialect provided on the command-line, if any."""
+
+    def __init__(self, config: TranspileConfig | None, prompts: Prompts) -> None:
+        if config is None:
+            logger.warning(
+                "No workspace transpile configuration, use 'install-transpile' to (re)install and configure; using defaults for now."
+            )
+            config = TranspileConfig()
+        self._config = config
         self._prompts = prompts
+        self._source_dialect_override = None
+
+    @staticmethod
+    def _validate_transpiler_config_path(transpiler_config_path: str, msg: str) -> None:
+        """Validate the transpiler config path: it must be a valid path that exists."""
+        # Note: the content is not validated here, but during loading of the engine.
+        if not Path(transpiler_config_path).exists():
+            raise_validation_exception(msg)
+
+    def use_transpiler_config_path(self, transpiler_config_path: str | None) -> None:
+        if transpiler_config_path is not None:
+            logger.debug(f"Setting transpiler_config_path to: {transpiler_config_path!r}")
+            self._validate_transpiler_config_path(
+                transpiler_config_path,
+                f"Invalid path for '--transpiler-config-path', does not exist: {transpiler_config_path}",
+            )
+            self._config = dataclasses.replace(self._config, transpiler_config_path=transpiler_config_path)
+
+    def use_source_dialect(self, source_dialect: str | None) -> None:
+        if source_dialect is not None:
+            # Defer validation: depends on the transpiler config path, we'll deal with this later.
+            logger.debug(f"Pending source_dialect override: {source_dialect!r}")
+            self._source_dialect_override = source_dialect
+
+    @staticmethod
+    def _validate_input_source(input_source: str, msg: str) -> None:
+        """Validate the input source: it must be a path that exists."""
+        if not Path(input_source).exists():
+            raise_validation_exception(msg)
+
+    def use_input_source(self, input_source: str | None) -> None:
+        if input_source is not None:
+            logger.debug(f"Setting input_source to: {input_source!r}")
+            self._validate_input_source(
+                input_source, f"Invalid path for '--input-source', does not exist: {input_source}"
+            )
+            self._config = dataclasses.replace(self._config, input_source=input_source)
+
+    def _prompt_input_source(self) -> None:
+        prompted_input_source = self._prompts.question("Enter input SQL path (directory/file)").strip()
+        logger.debug(f"Setting input_source to: {prompted_input_source!r}")
+        self._validate_input_source(
+            prompted_input_source, f"Invalid input source, path does not exist: {prompted_input_source}"
+        )
+        self._config = dataclasses.replace(self._config, input_source=prompted_input_source)
+
+    def _check_input_source(self) -> None:
+        config_input_source = self._config.input_source
+        if config_input_source is None:
+            self._prompt_input_source()
+        else:
+            self._validate_input_source(
+                config_input_source, f"Invalid input source path configured, does not exist: {config_input_source}"
+            )
 
-    def check_input_source(self, input_source: str | None):
-        if input_source == "None":
-            input_source = None
-        if not input_source:
-            input_source = self._config.input_source
-        if not input_source:
-            input_source = self._prompts.question("Enter input SQL path (directory/file)")
-            input_source = input_source.strip()
-        if not input_source:
-            raise_validation_exception("Missing '--input-source'")
-        if not os.path.exists(input_source):
-            raise_validation_exception(f"Invalid value for '--input-source': Path '{input_source}' does not exist.")
-        logger.debug(f"Setting input_source to '{input_source}'")
-        self._config = dataclasses.replace(self._config, input_source=input_source)
-
-    def check_source_dialect(self, source_dialect: str | None):
-        if source_dialect == "None":
-            source_dialect = None
-        if not source_dialect:
-            source_dialect = self._config.source_dialect
-        all_dialects = sorted(TranspilerInstaller.all_dialects())
-        if source_dialect and source_dialect not in all_dialects:
-            logger.error(f"'{source_dialect}' is not a supported dialect. Selecting a supported one...")
-            source_dialect = None
-        if not source_dialect:
-            source_dialect = self._prompts.choice("Select the source dialect:", all_dialects)
-        if not source_dialect:
-            raise_validation_exception("Missing '--source-dialect'")
-        logger.debug(f"Setting source_dialect to '{source_dialect}'")
-        self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
-
-    def check_transpiler_config_path(self, transpiler_config_path: str | None):
-        if transpiler_config_path == "None":
-            transpiler_config_path = None
-        if not transpiler_config_path:
-            transpiler_config_path = self._config.transpiler_config_path
-        # we allow pointing to a loose transpiler config (i.e. not installed under .databricks)
-        if transpiler_config_path:
-            if not os.path.exists(transpiler_config_path):
-                logger.error(f"The transpiler configuration does not exist '{transpiler_config_path}'.")
-                transpiler_config_path = None
-        if transpiler_config_path:
-            config = LSPConfig.load(Path(transpiler_config_path))
-            if self._config.source_dialect not in config.remorph.dialects:
-                logger.error(f"The configured transpiler does not support dialect '{self._config.source_dialect}'.")
-                transpiler_config_path = None
-        if not transpiler_config_path:
-            transpiler_names = TranspilerInstaller.transpilers_with_dialect(cast(str, self._config.source_dialect))
-            if len(transpiler_names) > 1:
-                transpiler_name = self._prompts.choice("Select the transpiler:", list(transpiler_names))
-            else:
-                transpiler_name = next(name for name in transpiler_names)
-                logger.info(f"Lakebridge will use the {transpiler_name} transpiler")
-            transpiler_config_path = str(TranspilerInstaller.transpiler_config_path(transpiler_name))
-        logger.debug(f"Setting transpiler_config_path to '{transpiler_config_path}'")
-        self._config = dataclasses.replace(self._config, transpiler_config_path=cast(str, transpiler_config_path))
-
-    def check_transpiler_config_options(self):
-        lsp_config = LSPConfig.load(Path(self._config.transpiler_config_path))
-        options_to_configure = lsp_config.options_for_dialect(self._config.source_dialect) or []
-        transpiler_options = self._config.transpiler_options or {}
-        if len(options_to_configure) == 0:
-            transpiler_options = None
+    @staticmethod
+    def _validate_output_folder(output_folder: str, msg: str) -> None:
+        """Validate the output folder: it doesn't have to exist, but its parent must."""
+        if not Path(output_folder).parent.exists():
+            raise_validation_exception(msg)
+
+    def use_output_folder(self, output_folder: str | None) -> None:
+        if output_folder is not None:
+            logger.debug(f"Setting output_folder to: {output_folder!r}")
+            self._validate_output_folder(
+                output_folder, f"Invalid path for '--output-folder', parent does not exist for: {output_folder}"
+            )
+            self._config = dataclasses.replace(self._config, output_folder=output_folder)
+
+    def _prompt_output_folder(self) -> None:
+        prompted_output_folder = self._prompts.question("Enter output folder path (directory)").strip()
+        logger.debug(f"Setting output_folder to: {prompted_output_folder!r}")
+        self._validate_output_folder(
+            prompted_output_folder, f"Invalid output folder path, parent does not exist for: {prompted_output_folder}"
+        )
+        self._config = dataclasses.replace(self._config, output_folder=prompted_output_folder)
+
+    def _check_output_folder(self) -> None:
+        config_output_folder = self._config.output_folder
+        if config_output_folder is None:
+            self._prompt_output_folder()
         else:
-            # TODO delete stale options ?
-            for option in options_to_configure:
-                self._check_transpiler_config_option(option, transpiler_options)
-        logger.debug(f"Setting transpiler_options to {transpiler_options}")
-        self._config = dataclasses.replace(self._config, transpiler_options=transpiler_options)
-
-    def _check_transpiler_config_option(self, option: LSPConfigOptionV1, values: dict[str, JsonValue]):
-        if option.flag in values.keys():
-            return
-        values[option.flag] = option.prompt_for_value(self._prompts)
-
-    def check_output_folder(self, output_folder: str | None):
-        output_folder = output_folder if output_folder else self._config.output_folder
-        if not output_folder:
-            raise_validation_exception("Missing '--output-folder'")
-        if not os.path.exists(output_folder):
-            os.makedirs(output_folder, exist_ok=True)
-        logger.debug(f"Setting output_folder to '{output_folder}'")
-        self._config = dataclasses.replace(self._config, output_folder=output_folder)
-
-    def check_error_file_path(self, error_file_path: str | None):
-        error_file_path = error_file_path if error_file_path else self._config.error_file_path
-        if not error_file_path or error_file_path == "None":
-            raise_validation_exception("Missing '--error-file-path'")
-        if error_file_path == "errors.log":
-            error_file_path = str(Path.cwd() / "errors.log")
-        if not os.path.exists(Path(error_file_path).parent):
-            os.makedirs(Path(error_file_path).parent, exist_ok=True)
-
-        logger.debug(f"Setting error_file_path to '{error_file_path}'")
-        self._config = dataclasses.replace(self._config, error_file_path=error_file_path)
-
-    def check_skip_validation(self, skip_validation_str: str | None):
-        skip_validation: bool | None = None
-        if skip_validation_str == "None":
-            skip_validation_str = None
-        if skip_validation_str is not None:
-            if skip_validation_str.lower() not in {"true", "false"}:
-                raise_validation_exception(
-                    f"Invalid value for '--skip-validation': '{skip_validation_str}' is not one of 'true', 'false'."
-                )
-            skip_validation = skip_validation_str.lower() == "true"
-        if skip_validation is None:
-            skip_validation = self._config.skip_validation
-        if skip_validation is None:
-            skip_validation = self._prompts.confirm(
-                "Would you like to validate the syntax and semantics of the transpiled queries?"
+            self._validate_output_folder(
+                config_output_folder,
+                f"Invalid output folder configured, parent does not exist for: {config_output_folder}",
             )
-        logger.debug(f"Setting skip_validation to '{skip_validation}'")
-        self._config = dataclasses.replace(self._config, skip_validation=skip_validation)
 
-    def check_catalog_name(self, catalog_name: str | None):
-        if self._config.skip_validation:
-            return
-        if catalog_name == "None":
-            catalog_name = None
-        if not catalog_name:
-            catalog_name = self._config.catalog_name
-        if not catalog_name:
-            raise_validation_exception(
-                "Missing '--catalog-name', please run 'databricks labs lakebridge install-transpile' to configure one"
+    @staticmethod
+    def _validate_error_file_path(error_file_path: str | None, msg: str) -> None:
+        """Value the error file path: it doesn't have to exist, but its parent must."""
+        if error_file_path is not None and not Path(error_file_path).parent.exists():
+            raise_validation_exception(msg)
+
+    def use_error_file_path(self, error_file_path: str | None) -> None:
+        if error_file_path is not None:
+            logger.debug(f"Setting error_file_path to: {error_file_path!r}")
+            self._validate_error_file_path(
+                error_file_path, f"Invalid path for '--error-file-path', parent does not exist: {error_file_path}"
+            )
+            self._config = dataclasses.replace(self._config, error_file_path=error_file_path)
+
+    def _check_error_file_path(self) -> None:
+        config_error_file_path = self._config.error_file_path
+        self._validate_error_file_path(
+            config_error_file_path,
+            f"Invalid error file path configured, parent does not exist for: {config_error_file_path}",
+        )
+
+    def use_skip_validation(self, skip_validation: str | None) -> None:
+        if skip_validation is not None:
+            skip_validation_lower = skip_validation.lower()
+            if skip_validation_lower not in {"true", "false"}:
+                msg = f"Invalid value for '--skip-validation': {skip_validation!r} must be 'true' or 'false'."
+                raise_validation_exception(msg)
+            new_skip_validation = skip_validation_lower == "true"
+            logger.debug(f"Setting skip_validation to: {new_skip_validation!r}")
+            self._config = dataclasses.replace(self._config, skip_validation=new_skip_validation)
+
+    def use_catalog_name(self, catalog_name: str | None) -> None:
+        if catalog_name:
+            logger.debug(f"Setting catalog_name to: {catalog_name!r}")
+            self._config = dataclasses.replace(self._config, catalog_name=catalog_name)
+
+    def use_schema_name(self, schema_name: str | None) -> None:
+        if schema_name:
+            logger.debug(f"Setting schema_name to: {schema_name!r}")
+            self._config = dataclasses.replace(self._config, schema_name=schema_name)
+
+    def _configure_transpiler_config_path(self, source_dialect: str) -> TranspileEngine | None:
+        """Configure the transpiler config path based on the requested source dialect."""
+        # Names of compatible transpiler engines for the given dialect.
+        compatible_transpilers = TranspilerInstaller.transpilers_with_dialect(source_dialect)
+        match len(compatible_transpilers):
+            case 0:
+                # Nothing found for the specified dialect, fail.
+                return None
+            case 1:
+                # Only one transpiler available for the specified dialect, use it.
+                transpiler_name = compatible_transpilers.pop()
+                logger.debug(f"Using only transpiler available for dialect {source_dialect!r}: {transpiler_name!r}")
+            case _:
+                # Multiple transpilers available for the specified dialect, prompt for which to use.
+                logger.debug(
+                    f"Multiple transpilers available for dialect {source_dialect!r}: {compatible_transpilers!r}"
+                )
+                transpiler_name = self._prompts.choice("Select the transpiler:", list(compatible_transpilers))
+        transpiler_config_path = TranspilerInstaller.transpiler_config_path(transpiler_name)
+        logger.info(f"Lakebridge will use the {transpiler_name} transpiler.")
+        self._config = dataclasses.replace(self._config, transpiler_config_path=str(transpiler_config_path))
+        return TranspileEngine.load_engine(transpiler_config_path)
+
+    def _configure_source_dialect(
+        self, source_dialect: str, engine: TranspileEngine | None, msg_prefix: str
+    ) -> TranspileEngine:
+        """Configure the source dialect, if possible, and return the transpiler engine."""
+        if engine is None:
+            engine = self._configure_transpiler_config_path(source_dialect)
+            if engine is None:
+                supported_dialects = ", ".join(TranspilerInstaller.all_dialects())
+                msg = f"{msg_prefix}: {source_dialect!r} (supported dialects: {supported_dialects})"
+                raise_validation_exception(msg)
+        else:
+            # Check the source dialect against the engine.
+            if source_dialect not in engine.supported_dialects:
+                supported_dialects_description = ", ".join(engine.supported_dialects)
+                msg = f"Invalid value for '--source-dialect': {source_dialect!r} must be one of: {supported_dialects_description}"
+                raise_validation_exception(msg)
+            self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
+        return engine
+
+    def _prompt_source_dialect(self) -> TranspileEngine:
+        # This is similar to the post-install prompting for the source dialect.
+        supported_dialects = TranspilerInstaller.all_dialects()
+        match len(supported_dialects):
+            case 0:
+                msg = "No transpilers are available, install using 'install-transpile' or use --transpiler-conf-path'."
+                raise_validation_exception(msg)
+            case 1:
+                # Only one dialect available, use it.
+                source_dialect = supported_dialects.pop()
+                logger.debug(f"Using only source dialect available: {source_dialect!r}")
+            case _:
+                # Multiple dialects available, prompt for which to use.
+                logger.debug(f"Multiple source dialects available, choice required: {supported_dialects!r}")
+                source_dialect = self._prompts.choice("Select the source dialect:", list(supported_dialects))
+        engine = self._configure_transpiler_config_path(source_dialect)
+        assert engine is not None, "No transpiler engine available for a supported dialect; configuration is invalid."
+        return engine
+
+    def _check_lsp_engine(self) -> TranspileEngine:
+        #
+        # This is somewhat complicated:
+        #  - If there is no transpiler config path, we need to try to infer it from the source dialect.
+        #  - If there is no source dialect, we need to prompt for it: but that depends on the transpiler config path.
+        #
+        # With this in mind, the steps here are:
+        # 1. If the transpiler config path is set, check it exists and load the engine.
+        # 2. If the source dialect is set,
+        #      - If the transpiler config path is set: validate the source dialect against the engine.
+        #      - If the transpiler config path is not set: search for a transpiler that satisfies the dialect:
+        #          * If one is found, we're good to go.
+        #          * If more than one is found, prompt for the transpiler config path.
+        #          * If none are found, fail: no transpilers available for the specified dialect.
+        #    At this point we have either halted, or we have a valid transpiler path and source dialect.
+        # 3. If the source dialect is not set, we need to:
+        #      a) Load the set of available dialects: just for the engine if transpiler config path is set, or for all
+        #         available transpilers if not.
+        #      b) Depending on the available dialects:
+        #          - If there is only one dialect available, set it as the source dialect.
+        #          - If there are multiple dialects available, prompt for which to use.
+        #          - If there are no dialects available, fail: no transpilers available.
+        #    At this point we have either halted, or we have a valid transpiler path and source dialect.
+        #
+        # TODO: Deal with the transpiler options, and filtering them for the engine.
+        #
+
+        # Step 1: Check the transpiler config path.
+        transpiler_config_path = self._config.transpiler_config_path
+        if transpiler_config_path is not None:
+            self._validate_transpiler_config_path(
+                transpiler_config_path,
+                f"Invalid transpiler path configured, path does not exist: {transpiler_config_path}",
             )
-        logger.debug(f"Setting catalog_name to '{catalog_name}'")
-        self._config = dataclasses.replace(self._config, catalog_name=catalog_name)
+            path = Path(transpiler_config_path)
+            engine = TranspileEngine.load_engine(path)
+        else:
+            engine = None
+        del transpiler_config_path
+
+        # Step 2: Check the source dialect, assuming it has been specified, and infer the transpiler config path if necessary.
+        source_dialect = self._source_dialect_override
+        if source_dialect is not None:
+            logger.debug(f"Setting source_dialect override: {source_dialect!r}")
+            engine = self._configure_source_dialect(source_dialect, engine, "Invalid value for '--source-dialect'")
+        else:
+            source_dialect = self._config.source_dialect
+            if source_dialect is not None:
+                logger.debug(f"Using configured source_dialect: {source_dialect!r}")
+                engine = self._configure_source_dialect(source_dialect, engine, "Invalid configured source dialect")
+            else:
+                # Step 3: Source dialect is not set, we need to prompt for it.
+                logger.debug("No source_dialect available, prompting.")
+                engine = self._prompt_source_dialect()
+        return engine
 
-    def check_schema_name(self, schema_name: str | None):
-        if self._config.skip_validation:
+    def _check_transpiler_options(self, engine: TranspileEngine) -> None:
+        if not isinstance(engine, LSPEngine):
             return
-        if schema_name == "None":
-            schema_name = None
-        if not schema_name:
-            schema_name = self._config.schema_name
-        if not schema_name:
-            raise_validation_exception(
-                "Missing '--schema-name', please run 'databricks labs lakebridge install-transpile' to configure one"
+        assert self._config.source_dialect is not None, "Source dialect must be set before checking transpiler options."
+        options_for_dialect = engine.options_for_dialect(self._config.source_dialect)
+        transpiler_options = self._config.transpiler_options
+        if not isinstance(transpiler_options, Mapping):
+            return
+        checked_options = {
+            option.flag: (
+                transpiler_options[option.flag]
+                if option.flag in transpiler_options
+                else option.prompt_for_value(self._prompts)
             )
-        logger.debug(f"Setting schema_name to '{schema_name}'")
-        self._config = dataclasses.replace(self._config, schema_name=schema_name)
+            for option in options_for_dialect
+        }
+        self._config = dataclasses.replace(self._config, transpiler_options=checked_options)
 
     def check(self) -> tuple[TranspileConfig, TranspileEngine]:
-        logger.debug(f"Checking config: {self!s}")
-        # not using os.path.exists because it sometimes fails mysteriously...
-        transpiler_path = self._config.transpiler_path
-        if not transpiler_path or not transpiler_path.exists():
-            raise_validation_exception(
-                f"Invalid value for '--transpiler-config-path': Path '{self._config.transpiler_config_path}' does not exist."
-            )
-        engine = TranspileEngine.load_engine(transpiler_path)
-        engine.check_source_dialect(self._config.source_dialect)
-        if not self._config.input_source or not os.path.exists(self._config.input_source):
-            raise_validation_exception(
-                f"Invalid value for '--input-source': Path '{self._config.input_source}' does not exist."
-            )
-        # 'transpiled' will be used as output_folder if not specified
-        # 'errors.log' will be used as errors file if not specified
-        return self._config, engine
+        """Checks that all configuration parameters are present and valid."""
+        logger.debug(f"Checking config: {self._config!r}")
+
+        self._check_input_source()
+        self._check_output_folder()
+        self._check_error_file_path()
+        # No validation here required for:
+        #   - skip_validation: it is a boolean flag, mandatory, and has a default: so no further validation is needed.
+        #   - catalog_name and schema_name: they are mandatory, but have a default.
+        # TODO: if validation is enabled, we should check that the catalog and schema names are valid.
+
+        # This covers: transpiler_config_path, source_dialect
+        engine = self._check_lsp_engine()
+
+        # Last thing: the configuration may have transpiler-specific options, check them.
+        self._check_transpiler_options(engine)
+
+        config = self._config
+        logger.debug(f"Validated config: {config!r}")
+        return config, engine
 
 
-async def _transpile(ctx: ApplicationContext, config: TranspileConfig, engine: TranspileEngine):
+async def _transpile(ctx: ApplicationContext, config: TranspileConfig, engine: TranspileEngine) -> RootJsonValue:
     """Transpiles source dialect to databricks dialect"""
     with_user_agent_extra("cmd", "execute-transpile")
     user = ctx.current_user
@@ -387,16 +574,22 @@ def aggregates_reconcile(w: WorkspaceClient):
 
 
 @lakebridge.command
-def generate_lineage(w: WorkspaceClient, source_dialect: str, input_source: str, output_folder: str):
+def generate_lineage(w: WorkspaceClient, *, source_dialect: str | None = None, input_source: str, output_folder: str):
     """[Experimental] Generates a lineage of source SQL files or folder"""
     ctx = ApplicationContext(w)
     logger.debug(f"User: {ctx.current_user}")
+    if not os.path.exists(input_source):
+        raise_validation_exception(f"Invalid path for '--input-source': Path '{input_source}' does not exist.")
+    if not os.path.exists(output_folder):
+        raise_validation_exception(f"Invalid path for '--output-folder': Path '{output_folder}' does not exist.")
+    if source_dialect is None:
+        raise_validation_exception("Value for '--source-dialect' must be provided.")
     engine = SqlglotEngine()
-    engine.check_source_dialect(source_dialect)
-    if not input_source or not os.path.exists(input_source):
-        raise_validation_exception(f"Invalid value for '--input-source': Path '{input_source}' does not exist.")
-    if not os.path.exists(output_folder) or output_folder in {None, ""}:
-        raise_validation_exception(f"Invalid value for '--output-folder': Path '{output_folder}' does not exist.")
+    supported_dialects = engine.supported_dialects
+    if source_dialect not in supported_dialects:
+        supported_dialects_description = ", ".join(supported_dialects)
+        msg = f"Unsupported source dialect provided for '--source-dialect': '{source_dialect}' (supported: {supported_dialects_description})"
+        raise_validation_exception(msg)
 
     lineage_generator(engine, source_dialect, input_source, output_folder)
 
@@ -430,8 +623,10 @@ def configure_database_profiler():
 
 @lakebridge.command()
 def install_transpile(w: WorkspaceClient, artifact: str | None = None):
-    """Install the lakebridge Transpilers"""
+    """Install the Lakebridge transpilers"""
     with_user_agent_extra("cmd", "install-transpile")
+    if artifact:
+        with_user_agent_extra("artifact-overload", Path(artifact).name)
     user = w.current_user
     logger.debug(f"User: {user}")
     installer = _installer(w)
@@ -440,7 +635,7 @@ def install_transpile(w: WorkspaceClient, artifact: str | None = None):
 
 @lakebridge.command(is_unauthenticated=False)
 def configure_reconcile(w: WorkspaceClient):
-    """Configure the lakebridge Reconcile Package"""
+    """Configure the Lakebridge reconciliation module"""
     with_user_agent_extra("cmd", "configure-reconcile")
     user = w.current_user
     logger.debug(f"User: {user}")

databricks/labs/lakebridge/config.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import logging
 from dataclasses import dataclass
 from enum import Enum, auto
@@ -31,11 +29,11 @@ class LSPConfigOptionV1:
     default: Any = None
 
     @classmethod
-    def parse_all(cls, data: dict[str, Any]) -> dict[str, list[LSPConfigOptionV1]]:
+    def parse_all(cls, data: dict[str, Any]) -> dict[str, list["LSPConfigOptionV1"]]:
         return {key: list(LSPConfigOptionV1.parse(item) for item in value) for (key, value) in data.items()}
 
     @classmethod
-    def parse(cls, data: Any) -> LSPConfigOptionV1:
+    def parse(cls, data: Any) -> "LSPConfigOptionV1":
         if not isinstance(data, dict):
             raise ValueError(f"Invalid transpiler config option, expecting a dict entry, got {data}")
         flag: str = data.get("flag", "")
@@ -79,7 +77,7 @@ class TranspileConfig:
     output_folder: str | None = None
     error_file_path: str | None = None
     sdk_config: dict[str, str] | None = None
-    skip_validation: bool | None = False
+    skip_validation: bool = False
     catalog_name: str = "remorph"
     schema_name: str = "transpiler"
     transpiler_options: JsonValue = None

databricks/labs/lakebridge/helpers/file_utils.py

@@ -1,3 +1,5 @@
+import contextlib
+import os
 from pathlib import Path
 from collections.abc import Generator
 
@@ -51,3 +53,13 @@ def get_sql_file(input_path: str | Path) -> Generator[Path, None, None]:
         for filename in files:
             if is_sql_file(filename):
                 yield filename
+
+
+@contextlib.contextmanager
+def chdir(new_path: Path) -> Generator[None, None, None]:
+    saved_path = Path.cwd()
+    try:
+        os.chdir(new_path)
+        yield
+    finally:
+        os.chdir(saved_path)

databricks/labs/lakebridge/install.py

@@ -37,6 +37,7 @@ from databricks.labs.lakebridge.config import (
 
 from databricks.labs.lakebridge.deployment.configurator import ResourceConfigurator
 from databricks.labs.lakebridge.deployment.installation import WorkspaceInstallation
+from databricks.labs.lakebridge.helpers.file_utils import chdir
 from databricks.labs.lakebridge.reconcile.constants import ReconReportType, ReconSourceType
 from databricks.labs.lakebridge.transpiler.lsp.lsp_engine import LSPConfig
 
@@ -251,12 +252,8 @@ class WheelInstaller(TranspilerInstaller):
         return self._post_install(version)
 
     def _create_venv(self) -> None:
-        cwd = os.getcwd()
-        try:
-            os.chdir(self._install_path)
+        with chdir(self._install_path):
             self._unsafe_create_venv()
-        finally:
-            os.chdir(cwd)
 
     def _unsafe_create_venv(self) -> None:
         # using the venv module doesn't work (maybe it's not possible to create a venv from a venv ?)
@@ -298,16 +295,12 @@ class WheelInstaller(TranspilerInstaller):
         raise ValueError(f"Could not locate 'site-packages' for {self._venv!s}")
 
     def _install_with_pip(self) -> None:
-        cwd = os.getcwd()
-        try:
-            os.chdir(self._install_path)
+        with chdir(self._install_path):
             # the way to call pip from python is highly sensitive to os and source type
             if self._artifact:
                 self._install_local_artifact()
             else:
                 self._install_remote_artifact()
-        finally:
-            os.chdir(cwd)
 
     def _install_local_artifact(self) -> None:
         pip = self._locate_pip()
@@ -557,10 +550,9 @@ class WorkspaceInstaller:
 
     @classmethod
     def install_morpheus(cls, artifact: Path | None = None):
-        java_version = cls.get_java_version()
-        if java_version is None or java_version < (11, 0, 0, 0):
-            logger.warning(
-                "This software requires Java 11 or above. Please install Java and re-run 'install-transpile'."
+        if not cls.is_java_version_okay():
+            logger.error(
+                "The morpheus transpiler requires Java 11 or above. Please install Java and re-run 'install-transpile'."
             )
             return
         product_name = "databricks-morph-plugin"
@@ -568,6 +560,26 @@ class WorkspaceInstaller:
         artifact_id = product_name
         TranspilerInstaller.install_from_maven(product_name, group_id, artifact_id, artifact)
 
+    @classmethod
+    def is_java_version_okay(cls) -> bool:
+        detected_java = cls.find_java()
+        match detected_java:
+            case None:
+                logger.warning("No Java executable found in the system PATH.")
+                return False
+            case (java_executable, None):
+                logger.warning(f"Java found, but could not determine the version: {java_executable}.")
+                return False
+            case (java_executable, bytes(raw_version)):
+                logger.warning(f"Java found ({java_executable}), but could not parse the version:\n{raw_version}")
+                return False
+            case (java_executable, tuple(old_version)) if old_version < (11, 0, 0, 0):
+                version_str = ".".join(str(v) for v in old_version)
+                logger.warning(f"Java found ({java_executable}), but version {version_str} is too old.")
+                return False
+            case _:
+                return True
+
     @classmethod
     def install_artifact(cls, artifact: str):
         path = Path(artifact)
@@ -582,25 +594,41 @@ class WorkspaceInstaller:
             logger.fatal(f"Cannot install unsupported artifact: {artifact}")
 
     @classmethod
-    def get_java_version(cls) -> tuple[int, int, int, int] | None:
+    def find_java(cls) -> tuple[Path, tuple[int, int, int, int] | bytes | None] | None:
+        """Locate Java and return its version, as reported by `java -version`.
+
+        The java executable is currently located by searching the system PATH. Its version is parsed from the output of
+        the `java -version` command, which has been standardized since Java 10.
+
+        Returns:
+            a tuple of its path and the version as a tuple of integers (feature, interim, update, patch), if the java
+            executable could be located. If the version cannot be parsed, instead the raw version information is
+            returned, or `None` as a last resort. When no java executable is found, `None` is returned instead of a
+            tuple.
+        """
         # Platform-independent way to reliably locate the java executable.
         # Reference: https://docs.python.org/3.10/library/subprocess.html#popen-constructor
         java_executable = shutil.which("java")
         if java_executable is None:
             return None
+        java_executable_path = Path(java_executable)
+        logger.debug(f"Using java executable: {java_executable_path!r}")
         try:
-            completed = run([java_executable, "-version"], shell=False, capture_output=True, check=True)
+            completed = run([str(java_executable_path), "-version"], shell=False, capture_output=True, check=True)
         except CalledProcessError as e:
             logger.debug(
                 f"Failed to run {e.args!r} (exit-code={e.returncode}, stdout={e.stdout!r}, stderr={e.stderr!r})",
                 exc_info=e,
             )
-            return None
+            return java_executable_path, None
         # It might not be ascii, but the bits we care about are so this will never fail.
-        java_version_output = completed.stderr.decode("ascii", errors="ignore")
+        raw_output = completed.stderr
+        java_version_output = raw_output.decode("ascii", errors="ignore")
         java_version = cls._parse_java_version(java_version_output)
+        if java_version is None:
+            return java_executable_path, raw_output.strip()
         logger.debug(f"Detected java version: {java_version}")
-        return java_version
+        return java_executable_path, java_version
 
     # Pattern to match a Java version string, compiled at import time to ensure it's valid.
     # Ref: https://docs.oracle.com/en/java/javase/11/install/version-string-format.html

databricks/labs/lakebridge/reconcile/connectors/source_adapter.py

@@ -6,7 +6,7 @@ from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSour
 from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource
 from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource
 from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource
-from databricks.labs.lakebridge.reconcile.connectors.sql_server import SQLServerDataSource
+from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource
 from databricks.labs.lakebridge.transpiler.sqlglot.generator.databricks import Databricks
 from databricks.labs.lakebridge.transpiler.sqlglot.parsers.oracle import Oracle
 from databricks.labs.lakebridge.transpiler.sqlglot.parsers.snowflake import Snowflake
@@ -26,5 +26,5 @@ def create_adapter(
     if isinstance(engine, Databricks):
         return DatabricksDataSource(engine, spark, ws, secret_scope)
     if isinstance(engine, TSQL):
-        return SQLServerDataSource(engine, spark, ws, secret_scope)
+        return TSQLServerDataSource(engine, spark, ws, secret_scope)
     raise ValueError(f"Unsupported source type --> {engine}")

databricks/labs/lakebridge/reconcile/connectors/{sql_server.py → tsql.py}

@@ -47,7 +47,7 @@ _SCHEMA_QUERY = """SELECT
               """
 
 
-class SQLServerDataSource(DataSource, SecretsMixin, JDBCReaderMixin):
+class TSQLServerDataSource(DataSource, SecretsMixin, JDBCReaderMixin):
     _DRIVER = "sqlserver"
 
     def __init__(

databricks/labs/lakebridge/reconcile/constants.py

@@ -18,6 +18,7 @@ class ReconSourceType(AutoName):
     SNOWFLAKE = auto()
     ORACLE = auto()
     DATABRICKS = auto()
+    TSQL = auto()
 
 
 class ReconReportType(AutoName):

databricks/labs/lakebridge/transpiler/execute.py

@@ -80,8 +80,9 @@ async def _process_one_file(context: TranspilingContext) -> tuple[int, list[Tran
     error_list = list(transpile_result.error_list)
     context = dataclasses.replace(context, transpiled_code=transpile_result.transpiled_code)
 
-    output_path = cast(Path, context.output_path)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path = context.output_path
+    assert output_path is not None, "Output path must be set in the context"
+    output_path.parent.mkdir(exist_ok=True)
 
     if _is_combined_result(transpile_result):
         _process_combined_result(context, error_list)

databricks/labs/lakebridge/transpiler/lsp/lsp_engine.py

@@ -35,7 +35,7 @@ from pygls.lsp.client import BaseLanguageClient
 from databricks.labs.blueprint.wheels import ProductInfo
 from databricks.labs.lakebridge.config import LSPConfigOptionV1, TranspileConfig, TranspileResult
 from databricks.labs.lakebridge.errors.exceptions import IllegalStateException
-from databricks.labs.lakebridge.helpers.file_utils import is_sql_file, is_dbt_project_file
+from databricks.labs.lakebridge.helpers.file_utils import chdir, is_dbt_project_file, is_sql_file
 from databricks.labs.lakebridge.transpiler.transpile_engine import TranspileEngine
 from databricks.labs.lakebridge.transpiler.transpile_status import (
     CodePosition,
@@ -389,6 +389,14 @@ class LSPEngine(TranspileEngine):
         self._client = _LanguageClient(name, version)
         self._init_response: InitializeResult | None = None
 
+    @property
+    def transpiler_name(self) -> str:
+        return self._config.name
+
+    def options_for_dialect(self, source_dialect: str) -> list[LSPConfigOptionV1]:
+        """Get the options supported when transpiling a given source dialect."""
+        return self._config.options_for_dialect(source_dialect)
+
     @property
     def supported_dialects(self) -> list[str]:
         return self._config.remorph.dialects
@@ -400,15 +408,14 @@ class LSPEngine(TranspileEngine):
     async def initialize(self, config: TranspileConfig) -> None:
         if self.is_alive:
             raise IllegalStateException("LSP engine is already initialized")
-        cwd = os.getcwd()
         try:
-            os.chdir(self._workdir)
-            await self._do_initialize(config)
+            # TODO: Avoid this by setting the working directory when launching the child process.
+            with chdir(self._workdir):
+                await self._do_initialize(config)
             await self._await_for_transpile_capability()
         # it is good practice to catch broad exceptions raised by launching a child process
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error("LSP initialization failed", exc_info=e)
-            os.chdir(cwd)
 
     async def _do_initialize(self, config: TranspileConfig) -> None:
         await self._start_server()

databricks/labs/lakebridge/transpiler/sqlglot/sqlglot_engine.py

@@ -39,6 +39,10 @@ class SqlglotEngine(TranspileEngine):
     def supported_dialects(self) -> list[str]:
         return sorted(SQLGLOT_DIALECTS.keys())
 
+    @property
+    def transpiler_name(self) -> str:
+        return "sqlglot"
+
     def _partial_transpile(
         self,
         read_dialect: Dialect,

databricks/labs/lakebridge/transpiler/transpile_engine.py

@@ -37,13 +37,11 @@ class TranspileEngine(abc.ABC):
 
     @property
     @abc.abstractmethod
-    def supported_dialects(self) -> list[str]: ...
+    def transpiler_name(self) -> str: ...
 
-    def check_source_dialect(self, source_dialect: str | None) -> None:
-        if source_dialect not in self.supported_dialects:
-            raise ValueError(
-                f"Invalid value for '--source-dialect': '{source_dialect}' is not one of {self.supported_dialects}."
-            )
+    @property
+    @abc.abstractmethod
+    def supported_dialects(self) -> list[str]: ...
 
     @abc.abstractmethod
     def is_supported_file(self, file: Path) -> bool: ...

{databricks_labs_lakebridge-0.10.2.dist-info → databricks_labs_lakebridge-0.10.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-labs-lakebridge
-Version: 0.10.2
+Version: 0.10.3
 Summary: Fast and predictable migrations to Databricks Lakehouse Platform. This tool is designed to help you migrate your data and workloads to the Databricks Lakehouse Platform in a fast, predictable, and reliable way. It provides a set of tools and utilities to help you reconcile your data and workloads, assess your current state, and plan your migration.
 Project-URL: Documentation, https://databrickslabs.github.io/lakebridge
 Project-URL: Issues, https://github.com/databrickslabs/lakebridge/issues
@@ -25,8 +25,8 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Utilities
 Requires-Python: >=3.10
 Requires-Dist: cryptography<45.1.0,>=44.0.2
-Requires-Dist: databricks-bb-analyzer~=0.1.8
-Requires-Dist: databricks-labs-blueprint[yaml]<0.12.0,>=0.11.1
+Requires-Dist: databricks-bb-analyzer~=0.1.9
+Requires-Dist: databricks-labs-blueprint[yaml]<0.12.0,>=0.11.2
 Requires-Dist: databricks-labs-lsql==0.16.0
 Requires-Dist: databricks-sdk~=0.51.0
 Requires-Dist: duckdb~=1.2.2

{databricks_labs_lakebridge-0.10.2.dist-info → databricks_labs_lakebridge-0.10.3.dist-info}/RECORD

@@ -1,4 +1,5 @@
 docs/lakebridge/src/components/Button.tsx,sha256=5l_irZl4AGwK7k1e2rdOb_W2-305Q1mjwXA3iP8CqaM,3159
+docs/lakebridge/src/components/ReconcileTabs.tsx,sha256=xJD0nq_raoYv70YLEnG2iuAUTSXXvDpmtmjX7X9Tw9E,2665
 docs/lakebridge/src/css/custom.css,sha256=-XnDdVlHqJZXJmKarH7zCUMnnlAfpxIpZyr8FNJ4q0A,4024
 docs/lakebridge/src/css/table.css,sha256=_MAyY7hyhfFrSNVAvCA2QlqdbeBi4Kr9Ue93bSyhKSE,315
 docs/lakebridge/src/pages/index.tsx,sha256=fQRA9ZbKsPxZbXuSa1LMDk1xfYg2YXCFgsgzqus0NLc,1789
@@ -7,12 +8,12 @@ docs/lakebridge/src/theme/Footer/index.tsx,sha256=Jj8zY5WDiTLXwF_mAgld8Dh1A3MY1H
 docs/lakebridge/src/theme/Layout/index.tsx,sha256=IkdLr13jKmLxT0jWQqrwqrjVXc8Rwd_kWNpTd1t2sc0,592
 databricks/__init__.py,sha256=YqH8Hy8lHJxd0hLMZF6kWirUDdPiX90LRDX6S6yTMn0,261
 databricks/labs/__init__.py,sha256=YqH8Hy8lHJxd0hLMZF6kWirUDdPiX90LRDX6S6yTMn0,261
-databricks/labs/lakebridge/__about__.py,sha256=1jZlIfQdn0z-MHgWnSw-KUImADj-pX_Qj-zRbJOsOb8,49
+databricks/labs/lakebridge/__about__.py,sha256=LBCN0OI_6vUqxgIo75HVdQc1TP5LOmy5HIs3OKrDIpk,49
 databricks/labs/lakebridge/__init__.py,sha256=nUNECqNvyfpT0aeWwlqG0ADT8U8ScCLb8WWpLydppcA,464
 databricks/labs/lakebridge/base_install.py,sha256=8NxXsNpgqXnuADKXVFh5oQL3osdvygRMY1amJwKfU08,490
-databricks/labs/lakebridge/cli.py,sha256=BgN1pz4dtLB_Y0C16_JhcRVxIfU7srZk24tUBSLJPAs,20597
-databricks/labs/lakebridge/config.py,sha256=Kxl_Yzo5ooiFrt95Gp7AwyLlPZopa4MmQKpBfpHso2Y,5872
-databricks/labs/lakebridge/install.py,sha256=e7MdIWFHHr2-sp-NNe6inJ44s_jgL-b_g_Y4MJXtqNQ,38724
+databricks/labs/lakebridge/cli.py,sha256=6exPUJs7c2qVo-X9VXFg5VM3XqOCdlk0_5OXfPw6nbY,31578
+databricks/labs/lakebridge/config.py,sha256=IjxvphM9fRQHQ2FAxwZ23deJGgSemJ3rMV0sp1Ob6e8,5833
+databricks/labs/lakebridge/install.py,sha256=EmtzbC-pOeiK7lqn4wxSRoeODlkqB_lQBJ9Mj4E0kjE,40536
 databricks/labs/lakebridge/jvmproxy.py,sha256=F9pXpemzdaJXwpshHxVM9PYU_eNn4zTCUFQ5vc9WIhA,1573
 databricks/labs/lakebridge/lineage.py,sha256=Q2oky4RkODRHWMwIQIwbYXSdZTmRkMWwEh6RssBiQxY,1843
 databricks/labs/lakebridge/uninstall.py,sha256=hf36YgeW9XO2cRvvn6AXUZdihQ1ZMHnR38OVEF5sfRw,759
@@ -46,7 +47,7 @@ databricks/labs/lakebridge/errors/exceptions.py,sha256=PIj8wRJpxrBXOLMMt9HQhBfhZ
 databricks/labs/lakebridge/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 databricks/labs/lakebridge/helpers/db_sql.py,sha256=chFHpn6XIuC0GrJ3a30_Y7tcXd4KZ5qO9zCAI4d7TR0,806
 databricks/labs/lakebridge/helpers/execution_time.py,sha256=8oLEYh0AKz1fuiQMyDTWDymhxh6xUKlcFpINWzKnOy4,533
-databricks/labs/lakebridge/helpers/file_utils.py,sha256=0EISLVIleoKe0bzdvhd6RRWjbauX7eNB1eHSV9-2SIo,1676
+databricks/labs/lakebridge/helpers/file_utils.py,sha256=1X3ri7_kyZibOFq36mX8fiERhE3tru_7VZIat1jjzOc,1911
 databricks/labs/lakebridge/helpers/metastore.py,sha256=1SKsIfNtiu3jUFjaXZ5B1fBZigVYqS1Q2OWhdn9qa8U,6425
 databricks/labs/lakebridge/helpers/recon_config_utils.py,sha256=1Nq_pIonE2tz08kdVpSDS-NVKGZ1p_kGRZBUQFFWZAs,7404
 databricks/labs/lakebridge/helpers/string_utils.py,sha256=TKW0BHmOZ2G8EebCohQRJLYglqeJajHgQ2BLehf9qsE,1169
@@ -58,7 +59,7 @@ databricks/labs/lakebridge/intermediate/engine_adapter.py,sha256=47DEQpj8HBSa-_T
 databricks/labs/lakebridge/intermediate/root_tables.py,sha256=G9PFU22qJ0BgV1FGZPK5bWNdEa8Xpo_gyEvMmATHkTw,1524
 databricks/labs/lakebridge/reconcile/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 databricks/labs/lakebridge/reconcile/compare.py,sha256=P9ABIT95TeS7BVRYVbzjpaEsynF2h2m5M8f9-he8A3A,16136
-databricks/labs/lakebridge/reconcile/constants.py,sha256=TrWuQFgSoBhRZkpiuscewG5eaZKV-FsErUQt4JO0cxo,798
+databricks/labs/lakebridge/reconcile/constants.py,sha256=ZXhGp0hxNdCWTN0iOfaIiDvRkxMZm4E7vtL-tVDsImM,816
 databricks/labs/lakebridge/reconcile/exception.py,sha256=kA-1KVAgZfWzxhcUwYha_8OapmFajJG0iY5TxPUPJyQ,1463
 databricks/labs/lakebridge/reconcile/execute.py,sha256=13yDonKuOcGytIDEySgAF--8VC_zLR4-hLudD2EkE0g,35111
 databricks/labs/lakebridge/reconcile/recon_capture.py,sha256=mlrKSzeTQnq3_ncbTunE1OyIFA2bLKlwiuDMicQRf5c,27317
@@ -74,8 +75,8 @@ databricks/labs/lakebridge/reconcile/connectors/jdbc_reader.py,sha256=SsY1rkeLo4
 databricks/labs/lakebridge/reconcile/connectors/oracle.py,sha256=LBqlK5WbgB4XaQNJ_DomTHXazdHJNu4vkIic_z6UENw,4795
 databricks/labs/lakebridge/reconcile/connectors/secrets.py,sha256=vue72BaYVaaeUfTOaqIEwP-I3TApgbPiuq69Z6I2u3k,1125
 databricks/labs/lakebridge/reconcile/connectors/snowflake.py,sha256=ARooTfPo6Vvrrj1n3KQ6aW-raAkoY_Z_qHB6epa5WVI,8086
-databricks/labs/lakebridge/reconcile/connectors/source_adapter.py,sha256=N_qbIuJsU7q3M3WWLv36DTKH-zYqV7js7Vcdx9Z7fLo,1449
-databricks/labs/lakebridge/reconcile/connectors/sql_server.py,sha256=vvjNyw8ZhO-IqMMCuSVcD19vPOrWTh2WH5ndoxKybHE,5675
+databricks/labs/lakebridge/reconcile/connectors/source_adapter.py,sha256=I6LBE0C8e80lMm_lVBVIrW9g9ogIgZ53J_EFRNkcSWY,1445
+databricks/labs/lakebridge/reconcile/connectors/tsql.py,sha256=71ChvUvDWSp6qftl4cJ7B_ztnchpU7uXo1_zLl5cDbc,5676
 databricks/labs/lakebridge/reconcile/query_builder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 databricks/labs/lakebridge/reconcile/query_builder/aggregate_query.py,sha256=zCPmLBLWeKwn0E2QMs0ua2CIJ6cnxmn77mNt4lvauTw,13783
 databricks/labs/lakebridge/reconcile/query_builder/base.py,sha256=J1LSemcN6bn-0K5U1PhXaQj22axOmqHUv-s9WwLQZOk,5293
@@ -146,16 +147,16 @@ databricks/labs/lakebridge/resources/reconcile/queries/installation/details.sql,
 databricks/labs/lakebridge/resources/reconcile/queries/installation/main.sql,sha256=s_A0YyGSX_pCWnQsQnY65VYFcbNvq2qKJvYxU6zam6E,794
 databricks/labs/lakebridge/resources/reconcile/queries/installation/metrics.sql,sha256=FdvjQp7gCwsbcu4UrOuJN-bBLJFpvUIyxH6PQvg04Wo,1006
 databricks/labs/lakebridge/transpiler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-databricks/labs/lakebridge/transpiler/execute.py,sha256=IUlz682ptMndolCI7IIGRTXJpJnGXHaEZXZnCLdCEDA,16956
-databricks/labs/lakebridge/transpiler/transpile_engine.py,sha256=9o-MXAnCChbFxv9Kg8kGLXdc8BZmtlwV5JdMPiuTQNk,1827
+databricks/labs/lakebridge/transpiler/execute.py,sha256=7DpeIixATOPryyt4TD93-sdwE1C_fIwuo6bKwClaF_s,17007
+databricks/labs/lakebridge/transpiler/transpile_engine.py,sha256=5zC8fkpBBlt9RjE_BeA_Sd6vaRxA3mBdhTqoRGFTc_Y,1616
 databricks/labs/lakebridge/transpiler/transpile_status.py,sha256=MO-Ju-ki3FCY15WxgwfPV9EC7Ma9q8aIfSTgHAmnkGU,1715
 databricks/labs/lakebridge/transpiler/lsp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-databricks/labs/lakebridge/transpiler/lsp/lsp_engine.py,sha256=Whdb3Usi0_lq7i94c_D2XNXBTm-bfy3bso4nKIEq_qk,22205
+databricks/labs/lakebridge/transpiler/lsp/lsp_engine.py,sha256=osT4RXpYqBNcAQ8mcoFt8m2dygs5TcmYnQq57KN_kw4,22580
 databricks/labs/lakebridge/transpiler/sqlglot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 databricks/labs/lakebridge/transpiler/sqlglot/dialect_utils.py,sha256=GhXXWGA_2PlmHKjxrjryZpA5xaVZ81Vrw3b7DzjpFFI,1033
 databricks/labs/lakebridge/transpiler/sqlglot/lca_utils.py,sha256=vpDLGhE-wFMah1VTXkMg6gI_QnzdzpYZf0h9DUd8zcI,5154
 databricks/labs/lakebridge/transpiler/sqlglot/local_expression.py,sha256=V69eEJHyZKxmyaham6OulYnwQRqkbGUrdiWm1EWP8YE,3825
-databricks/labs/lakebridge/transpiler/sqlglot/sqlglot_engine.py,sha256=3-YTEDPCk3Q7nT6A09D0p_IH8ftC3wr4sPF23Yy_OJM,10053
+databricks/labs/lakebridge/transpiler/sqlglot/sqlglot_engine.py,sha256=1uqpYIB-6vhuFqco80lXyBqqdkVkZkk9xuqFAvf2kXI,10131
 databricks/labs/lakebridge/transpiler/sqlglot/generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 databricks/labs/lakebridge/transpiler/sqlglot/generator/databricks.py,sha256=tF38z3J-P0mDnGeDmCzAiowAUoShiosimM6nfR_-3Ro,30653
 databricks/labs/lakebridge/transpiler/sqlglot/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -164,9 +165,9 @@ databricks/labs/lakebridge/transpiler/sqlglot/parsers/presto.py,sha256=bY6Ku8ZPW
 databricks/labs/lakebridge/transpiler/sqlglot/parsers/snowflake.py,sha256=dZ7BdOlBZlkbiN9G9bu4l2c456265Gx9WoWUPRa7Ffg,23203
 databricks/labs/lakebridge/upgrades/v0.4.0_add_main_table_operation_name_column.py,sha256=wMTbj1q5td4fa5DCk0tWFJ-OmhhzsExRLYUe4PKmk0s,3527
 databricks/labs/lakebridge/upgrades/v0.6.0_alter_metrics_datatype.py,sha256=hnTHRtqzwPSF5Judzh6ss-uB5h3IFtm2ylWduwRNq5Y,2424
-databricks_labs_lakebridge-0.10.2.dist-info/METADATA,sha256=vZS_ckQImF3YDBo5xH0S3KXPOPn78Stn689GBu1dR-Q,3078
-databricks_labs_lakebridge-0.10.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-databricks_labs_lakebridge-0.10.2.dist-info/entry_points.txt,sha256=Idr1CT73b8wShdr287yu1hheGbDbhBvucVUlZcbpiPo,75
-databricks_labs_lakebridge-0.10.2.dist-info/licenses/LICENSE,sha256=1hG0Cvw6mp9nL9qRoHFcCUk9fYqhcnj2vgJ75rt3BxA,3862
-databricks_labs_lakebridge-0.10.2.dist-info/licenses/NOTICE,sha256=wtxMsNvTkw1hAEkkWHz8A8JrYySAUSt1tOTcqddkWEg,1797
-databricks_labs_lakebridge-0.10.2.dist-info/RECORD,,
+databricks_labs_lakebridge-0.10.3.dist-info/METADATA,sha256=e7yr--8po1oLKE8_BRRTFbv_y2fJ0Fw5F95wT2bnn8U,3078
+databricks_labs_lakebridge-0.10.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+databricks_labs_lakebridge-0.10.3.dist-info/entry_points.txt,sha256=Idr1CT73b8wShdr287yu1hheGbDbhBvucVUlZcbpiPo,75
+databricks_labs_lakebridge-0.10.3.dist-info/licenses/LICENSE,sha256=1hG0Cvw6mp9nL9qRoHFcCUk9fYqhcnj2vgJ75rt3BxA,3862
+databricks_labs_lakebridge-0.10.3.dist-info/licenses/NOTICE,sha256=wtxMsNvTkw1hAEkkWHz8A8JrYySAUSt1tOTcqddkWEg,1797
+databricks_labs_lakebridge-0.10.3.dist-info/RECORD,,

docs/lakebridge/src/components/ReconcileTabs.tsx

@@ -0,0 +1,86 @@
+import React, {useRef,useEffect} from 'react';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import useBaseUrl from '@docusaurus/useBaseUrl';
+
+type FrameTabProps = {
+    src: string;
+    label: string;
+};
+
+const frameStyle: React.CSSProperties = {
+    width: '100%',
+    height: '500px',
+    border: '1px solid #ccc',
+    borderRadius: '6px',
+    marginBottom: '1em',
+    overflow: 'auto',
+};
+
+const FrameTab: React.FC<FrameTabProps> = ({ src, label }) => {
+    const iframeRef = useRef<HTMLIFrameElement>(null);
+    const url = useBaseUrl(src);
+
+    useEffect(() => {
+        const iframe = iframeRef.current;
+        if (!iframe) return;
+
+        const onLoad = () => {
+            try {
+                const doc = iframe.contentDocument || iframe.contentWindow?.document;
+                if (!doc) return;
+
+                const elementsToRemove = doc.querySelectorAll(
+                    'div[data-testid="content-spacer"], div[data-testid="extra-whitespace"]'
+                );
+
+                elementsToRemove.forEach(el => {
+                    if (!el.children.length) {
+                        el.remove();
+                    }
+                });
+            } catch (err) {
+                console.warn(err);
+            }
+        };
+
+        iframe.addEventListener('load', onLoad);
+
+        return () => {
+            iframe.removeEventListener('load', onLoad);
+        };
+    }, [url]);
+
+    return (
+        <div>
+            <div style={{ marginBottom: '0.5em' }}>
+                <a href={url} target="_blank" rel="noopener noreferrer">
+                    Open notebook in new tab
+                </a>
+            </div>
+            <iframe ref={iframeRef} src={url} style={frameStyle} title={label} />
+        </div>
+    );
+};
+
+const LakebridgeTabs: React.FC = () => (
+    <Tabs>
+        {/*        <TabItem value="Readme" label="Readme" default>
+            <FrameTab src="/lakebridge_reconcile/Readme.html" label="Readme" />
+        </TabItem>*/}
+        <TabItem value="Recon Main" label="Recon Main">
+            <FrameTab src="/lakebridge_reconcile/lakebridge_recon_main.html" label="Recon Main" />
+        </TabItem>
+        <TabItem value="Recon Wrapper" label="Recon Wrapper">
+            <FrameTab src="/lakebridge_reconcile/recon_wrapper_nb.html" label="Recon Wrapper" />
+        </TabItem>
+        <TabItem value="Snowflake Example" label="Transformation Query Generator">
+            <FrameTab
+                src="/lakebridge_reconcile/snowflake_transformation_query_generator.html"
+                label="Query Generator"
+            />
+        </TabItem>
+    </Tabs>
+);
+
+export default LakebridgeTabs;