databricks-labs-lakebridge 0.10.6__py3-none-any.whl → 0.10.7__py3-none-any.whl

databricks/labs/lakebridge/install.py

@@ -1,40 +1,41 @@
 import re
 import abc
 import dataclasses
-import shutil
-from json import loads, dump
 import logging
 import os
-from shutil import rmtree, move
-from subprocess import run, CalledProcessError
+import shutil
 import sys
-from typing import Any, cast
-from urllib import request
-from urllib.error import URLError, HTTPError
+import venv
 import webbrowser
+import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
+from json import loads, dump
 from pathlib import Path
-import xml.etree.ElementTree as ET
+from shutil import rmtree, move
+from subprocess import run, CalledProcessError
+from typing import Any, Literal, cast
+from urllib import request
+from urllib.error import URLError, HTTPError
 from zipfile import ZipFile
 
-from databricks.labs.blueprint.installation import Installation, JsonValue
-from databricks.labs.blueprint.installation import SerdeError
+from databricks.labs.blueprint.installation import Installation, JsonValue, SerdeError
 from databricks.labs.blueprint.installer import InstallState
 from databricks.labs.blueprint.tui import Prompts
 from databricks.labs.blueprint.wheels import ProductInfo
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.errors import NotFound, PermissionDenied
 
+from databricks.labs.lakebridge.__about__ import __version__
 from databricks.labs.lakebridge.config import (
-    TranspileConfig,
-    ReconcileConfig,
     DatabaseConfig,
+    ReconcileConfig,
     LakebridgeConfiguration,
     ReconcileMetadataConfig,
+    TranspileConfig,
 )
+from databricks.labs.lakebridge.contexts.application import ApplicationContext
 from databricks.labs.lakebridge.deployment.configurator import ResourceConfigurator
 from databricks.labs.lakebridge.deployment.installation import WorkspaceInstallation
-from databricks.labs.lakebridge.helpers.file_utils import chdir
 from databricks.labs.lakebridge.reconcile.constants import ReconReportType, ReconSourceType
 from databricks.labs.lakebridge.transpiler.repository import TranspilerRepository
 
@@ -43,9 +44,70 @@ logger = logging.getLogger(__name__)
 TRANSPILER_WAREHOUSE_PREFIX = "Lakebridge Transpiler Validation"
 
 
+class _PathBackup:
+    """A context manager to preserve a path before performing an operation, and optionally restore it afterwards."""
+
+    def __init__(self, path: Path) -> None:
+        self._path = path
+        self._backup_path: Path | None = None
+        self._finished = False
+
+    def __enter__(self) -> "_PathBackup":
+        self.start()
+        return self
+
+    def start(self) -> None:
+        """Start the backup process by creating a backup of the path, if it already exists."""
+        backup_path = self._path.with_name(f"{self._path.name}-saved")
+        if backup_path.exists():
+            logger.debug(f"Existing backup found, removing: {backup_path}")
+            rmtree(backup_path)
+        if self._path.exists():
+            logger.debug(f"Backing up existing path: {self._path} -> {backup_path}")
+            os.rename(self._path, backup_path)
+            self._backup_path = backup_path
+        else:
+            self._backup_path = None
+
+    def rollback(self) -> None:
+        """Rollback the operation by restoring the backup path, if it exists."""
+        assert not self._finished, "Can only rollback/commit once."
+        logger.debug(f"Removing path: {self._path}")
+        rmtree(self._path)
+        if self._backup_path is not None:
+            logger.debug(f"Restoring previous path: {self._backup_path} -> {self._path}")
+            os.rename(self._backup_path, self._path)
+            self._backup_path = None
+        self._finished = True
+
+    def commit(self) -> None:
+        """Commit the operation by removing the backup path, if it exists."""
+        assert not self._finished, "Can only rollback/commit once."
+        if self._backup_path is not None:
+            logger.debug(f"Removing backup path: {self._backup_path}")
+            rmtree(self._backup_path)
+            self._backup_path = None
+        self._finished = True
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
+        if not self._finished:
+            # Automatically commit or rollback based on whether an exception is underway.
+            if exc_val is None:
+                self.commit()
+            else:
+                self.rollback()
+        return False  # Do not suppress any exception underway
+
+
 class TranspilerInstaller(abc.ABC):
-    def __init__(self, repository: TranspilerRepository) -> None:
+
+    # TODO: Remove these properties when post-install is removed.
+    _install_path: Path
+    """The path where the transpiler is being installed, once this starts."""
+
+    def __init__(self, repository: TranspilerRepository, product_name: str) -> None:
         self._repository = repository
+        self._product_name = product_name
 
     _version_pattern = re.compile(r"[_-](\d+(?:[.\-_]\w*\d+)+)")
 
@@ -75,9 +137,44 @@ class TranspilerInstaller(abc.ABC):
             dump(version_data, f)
             f.write("\n")
 
+    def _install_version_with_backup(self, version: str) -> Path | None:
+        """Install a specific version of the transpiler, with backup handling."""
+        logger.info(f"Installing Databricks {self._product_name} transpiler (v{version})")
+        product_path = self._repository.transpilers_path() / self._product_name
+        with _PathBackup(product_path) as backup:
+            self._install_path = product_path / "lib"
+            self._install_path.mkdir(parents=True, exist_ok=True)
+            try:
+                result = self._install_version(version)
+            except (CalledProcessError, KeyError, ValueError) as e:
+                # Warning: if you end up here under the IntelliJ/PyCharm debugger, it can be because the debugger is
+                # trying to inject itself into the subprocess. Try disabling:
+                #   Settings | Build, Execution, Deployment | Python Debugger | Attach to subprocess automatically while debugging
+                # Note: Subprocess output is not captured, and should already be visible in the console.
+                logger.error(f"Failed to install {self._product_name} transpiler (v{version})", exc_info=e)
+                result = False
+
+            if result:
+                logger.info(f"Successfully installed {self._product_name} transpiler (v{version})")
+                self._store_product_state(product_path=product_path, version=version)
+                backup.commit()
+                return product_path
+            backup.rollback()
+        return None
+
+    @abc.abstractmethod
+    def _install_version(self, version: str) -> bool:
+        """Install a specific version of the transpiler, returning True if successful."""
+
 
 class WheelInstaller(TranspilerInstaller):
 
+    _venv_exec_cmd: Path
+    """Once created, the command to run the virtual environment's Python executable."""
+
+    _site_packages: Path
+    """Once created, the path to the site-packages directory in the virtual environment."""
+
     @classmethod
     def get_latest_artifact_version_from_pypi(cls, product_name: str) -> str | None:
         try:
@@ -96,8 +193,7 @@ class WheelInstaller(TranspilerInstaller):
         pypi_name: str,
         artifact: Path | None = None,
     ) -> None:
-        super().__init__(repository)
-        self._product_name = product_name
+        super().__init__(repository, product_name)
         self._pypi_name = pypi_name
         self._artifact = artifact
 
@@ -118,116 +214,45 @@ class WheelInstaller(TranspilerInstaller):
         if installed_version == latest_version:
             logger.info(f"{self._pypi_name} v{latest_version} already installed")
             return None
-        return self._install_latest_version(latest_version)
-
-    def _install_latest_version(self, version: str) -> Path | None:
-        logger.info(f"Installing Databricks {self._product_name} transpiler v{version}")
-        self._product_path = self._repository.transpilers_path() / self._product_name
-        backup_path = Path(f"{self._product_path!s}-saved")
-        if self._product_path.exists():
-            os.rename(self._product_path, backup_path)
-        self._install_path = self._product_path / "lib"
-        self._install_path.mkdir(parents=True, exist_ok=True)
-        try:
-            result = self._unsafe_install_latest_version(version)
-            logger.info(f"Successfully installed {self._pypi_name} v{version}")
-            if backup_path.exists():
-                rmtree(backup_path)
-            return result
-        except (CalledProcessError, ValueError) as e:
-            logger.error(f"Failed to install {self._pypi_name} v{version}", exc_info=e)
-            rmtree(self._product_path)
-            if backup_path.exists():
-                os.rename(backup_path, self._product_path)
-            return None
+        return self._install_version_with_backup(latest_version)
 
-    def _unsafe_install_latest_version(self, version: str) -> Path | None:
+    def _install_version(self, version: str) -> bool:
         self._create_venv()
         self._install_with_pip()
         self._copy_lsp_resources()
-        return self._post_install(version)
+        return self._post_install() is not None
 
     def _create_venv(self) -> None:
-        with chdir(self._install_path):
-            self._unsafe_create_venv()
-
-    def _unsafe_create_venv(self) -> None:
-        # using the venv module doesn't work (maybe it's not possible to create a venv from a venv ?)
-        # so falling back to something that works
-        # for some reason this requires shell=True, so pass full cmd line
-        cmd_line = f"{sys.executable} -m venv .venv"
-        completed = run(cmd_line, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, shell=True, check=False)
-        if completed.returncode:
-            logger.error(f"Failed to create venv, error code: {completed.returncode}")
-            if completed.stdout:
-                for line in completed.stdout:
-                    logger.error(line)
-            if completed.stderr:
-                for line in completed.stderr:
-                    logger.error(line)
-        completed.check_returncode()
-        self._venv = self._install_path / ".venv"
-        self._site_packages = self._locate_site_packages()
-
-    def _locate_site_packages(self) -> Path:
-        # can't use sysconfig because it only works for currently running python
-        if sys.platform == "win32":
-            return self._locate_site_packages_windows()
-        return self._locate_site_packages_linux_or_macos()
-
-    def _locate_site_packages_windows(self) -> Path:
-        packages = self._venv / "Lib" / "site-packages"
-        if packages.exists():
-            return packages
-        raise ValueError(f"Could not locate 'site-packages' for {self._venv!s}")
-
-    def _locate_site_packages_linux_or_macos(self) -> Path:
-        lib = self._venv / "lib"
-        for dir_ in os.listdir(lib):
-            if dir_.startswith("python"):
-                packages = lib / dir_ / "site-packages"
-                if packages.exists():
-                    return packages
-        raise ValueError(f"Could not locate 'site-packages' for {self._venv!s}")
-
-    def _install_with_pip(self) -> None:
-        with chdir(self._install_path):
-            # the way to call pip from python is highly sensitive to os and source type
-            if self._artifact:
-                self._install_local_artifact()
-            else:
-                self._install_remote_artifact()
-
-    def _install_local_artifact(self) -> None:
-        pip = self._locate_pip()
-        pip = pip.relative_to(self._install_path)
-        target = self._site_packages
-        target = target.relative_to(self._install_path)
-        if sys.platform == "win32":
-            command = f"{pip!s} install {self._artifact!s} -t {target!s}"
-            completed = run(command, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, shell=False, check=False)
+        venv_path = self._install_path / ".venv"
+        # Sadly, some platform-specific variations need to be dealt with:
+        #   - Windows venvs do not use symlinks, but rather copies, when populating the venv.
+        #   - The library path is different.
+        if use_symlinks := sys.platform != "win32":
+            major, minor = sys.version_info[:2]
+            lib_path = venv_path / "lib" / f"python{major}.{minor}" / "site-packages"
         else:
-            command = f"'{pip!s}' install '{self._artifact!s}' -t '{target!s}'"
-            completed = run(command, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, shell=True, check=False)
-        # checking return code later makes debugging easier
-        completed.check_returncode()
-
-    def _install_remote_artifact(self) -> None:
-        pip = self._locate_pip()
-        pip = pip.relative_to(self._install_path)
-        target = self._site_packages
-        target = target.relative_to(self._install_path)
-        if sys.platform == "win32":
-            args = [str(pip), "install", self._pypi_name, "-t", str(target)]
-            completed = run(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, shell=False, check=False)
-        else:
-            command = f"'{pip!s}' install {self._pypi_name} -t '{target!s}'"
-            completed = run(command, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, shell=True, check=False)
-        # checking return code later makes debugging easier
-        completed.check_returncode()
+            lib_path = venv_path / "Lib" / "site-packages"
+        builder = venv.EnvBuilder(with_pip=True, prompt=f"{self._product_name}", symlinks=use_symlinks)
+        builder.create(venv_path)
+        context = builder.ensure_directories(venv_path)
+        logger.debug(f"Created virtual environment with context: {context}")
+        self._venv_exec_cmd = context.env_exec_cmd
+        self._site_packages = lib_path
 
-    def _locate_pip(self) -> Path:
-        return self._venv / "Scripts" / "pip3.exe" if sys.platform == "win32" else self._venv / "bin" / "pip3"
+    def _install_with_pip(self) -> None:
+        # Based on: https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
+        # (But with venv_exec_cmd instead of sys.executable, so that we use the venv's pip.)
+        to_install: Path | str = self._artifact if self._artifact is not None else self._pypi_name
+        command: list[Path | str] = [
+            self._venv_exec_cmd,
+            "-m",
+            "pip",
+            "--disable-pip-version-check",
+            "install",
+            to_install,
+        ]
+        result = run(command, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, check=False)
+        result.check_returncode()
 
     def _copy_lsp_resources(self):
         lsp = self._site_packages / "lsp"
@@ -235,21 +260,20 @@ class WheelInstaller(TranspilerInstaller):
             raise ValueError("Installed transpiler is missing a 'lsp' folder")
         shutil.copytree(lsp, self._install_path, dirs_exist_ok=True)
 
-    def _post_install(self, version: str) -> Path | None:
+    def _post_install(self) -> Path | None:
         config = self._install_path / "config.yml"
         if not config.exists():
             raise ValueError("Installed transpiler is missing a 'config.yml' file in its 'lsp' folder")
         install_ext = "ps1" if sys.platform == "win32" else "sh"
         install_script = f"installer.{install_ext}"
-        installer = self._install_path / install_script
-        if installer.exists():
-            self._run_custom_installer(installer)
-        self._store_product_state(product_path=self._product_path, version=version)
+        installer_path = self._install_path / install_script
+        if installer_path.exists():
+            self._run_custom_installer(installer_path)
         return self._install_path
 
-    def _run_custom_installer(self, installer):
-        args = [str(installer)]
-        run(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, cwd=str(self._install_path), check=True)
+    def _run_custom_installer(self, installer_path: Path) -> None:
+        args = [installer_path]
+        run(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, cwd=self._install_path, check=True)
 
 
 class MavenInstaller(TranspilerInstaller):
@@ -336,8 +360,7 @@ class MavenInstaller(TranspilerInstaller):
         artifact_id: str,
         artifact: Path | None = None,
     ) -> None:
-        super().__init__(repository)
-        self._product_name = product_name
+        super().__init__(repository, product_name)
         self._group_id = group_id
         self._artifact_id = artifact_id
         self._artifact = artifact
@@ -358,40 +381,15 @@ class MavenInstaller(TranspilerInstaller):
         if installed_version == latest_version:
             logger.info(f"Databricks {self._product_name} transpiler v{latest_version} already installed")
             return None
-        return self._install_version(latest_version)
+        return self._install_version_with_backup(latest_version)
 
-    def _install_version(self, version: str) -> Path | None:
-        logger.info(f"Installing Databricks {self._product_name} transpiler v{version}")
-        self._product_path = self._repository.transpilers_path() / self._product_name
-        backup_path = Path(f"{self._product_path!s}-saved")
-        if backup_path.exists():
-            rmtree(backup_path)
-        if self._product_path.exists():
-            os.rename(self._product_path, backup_path)
-        self._product_path.mkdir(parents=True)
-        self._install_path = self._product_path / "lib"
-        self._install_path.mkdir()
-        try:
-            if self._unsafe_install_version(version):
-                logger.info(f"Successfully installed {self._product_name} v{version}")
-                self._store_product_state(self._product_path, version)
-                if backup_path.exists():
-                    rmtree(backup_path)
-                return self._product_path
-        except (KeyError, ValueError) as e:
-            logger.error(f"Failed to install Databricks {self._product_name} transpiler v{version}", exc_info=e)
-        rmtree(self._product_path)
-        if backup_path.exists():
-            os.rename(backup_path, self._product_path)
-        return None
-
-    def _unsafe_install_version(self, version: str) -> bool:
+    def _install_version(self, version: str) -> bool:
         jar_file_path = self._install_path / f"{self._artifact_id}.jar"
         if self._artifact:
-            logger.debug(f"Copying '{self._artifact!s}' to '{jar_file_path!s}'")
+            logger.debug(f"Copying: {self._artifact} -> {jar_file_path}")
             shutil.copyfile(self._artifact, jar_file_path)
         elif not self.download_artifact_from_maven(self._group_id, self._artifact_id, version, jar_file_path):
-            logger.error(f"Failed to install Databricks {self._product_name} transpiler v{version}")
+            logger.error(f"Failed to install Databricks {self._product_name} transpiler (v{version})")
             return False
         self._copy_lsp_config(jar_file_path)
         return True
@@ -449,6 +447,13 @@ class WorkspaceInstaller:
         logger.info("Installation completed successfully! Please refer to the documentation for the next steps.")
         return config
 
+    def has_installed_transpilers(self) -> bool:
+        """Detect whether there are transpilers currently installed."""
+        installed_transpilers = self._transpiler_repository.all_transpiler_names()
+        if installed_transpilers:
+            logger.info(f"Detected installed transpilers: {sorted(installed_transpilers)}")
+        return bool(installed_transpilers)
+
     def install_bladebridge(self, artifact: Path | None = None) -> None:
         local_name = "bladebridge"
         pypi_name = "databricks-bb-plugin"
@@ -802,3 +807,28 @@ class WorkspaceInstaller:
 
     def _has_necessary_access(self, catalog_name: str, schema_name: str, volume_name: str | None = None):
         self._resource_configurator.has_necessary_access(catalog_name, schema_name, volume_name)
+
+
+def installer(ws: WorkspaceClient, transpiler_repository: TranspilerRepository) -> WorkspaceInstaller:
+    app_context = ApplicationContext(_verify_workspace_client(ws))
+    return WorkspaceInstaller(
+        app_context.workspace_client,
+        app_context.prompts,
+        app_context.installation,
+        app_context.install_state,
+        app_context.product_info,
+        app_context.resource_configurator,
+        app_context.workspace_installation,
+        transpiler_repository=transpiler_repository,
+    )
+
+
+def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient:
+    """Verifies the workspace client configuration, ensuring it has the correct product info."""
+
+    # Using reflection to set right value for _product_info for telemetry
+    product_info = getattr(ws.config, '_product_info')
+    if product_info[0] != "lakebridge":
+        setattr(ws.config, '_product_info', ('lakebridge', __version__))
+
+    return ws

databricks/labs/lakebridge/reconcile/compare.py

@@ -3,6 +3,7 @@ from functools import reduce
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import col, expr, lit
 
+from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils
 from databricks.labs.lakebridge.reconcile.exception import ColumnMismatchException
 from databricks.labs.lakebridge.reconcile.recon_capture import (
     ReconIntermediatePersist,
@@ -22,7 +23,7 @@ _HASH_COLUMN_NAME = "hash_value_recon"
 _SAMPLE_ROWS = 50
 
 
-def raise_column_mismatch_exception(msg: str, source_missing: list[str], target_missing: list[str]) -> Exception:
+def _raise_column_mismatch_exception(msg: str, source_missing: list[str], target_missing: list[str]) -> Exception:
     error_msg = (
         f"{msg}\n"
         f"columns missing in source: {','.join(source_missing) if source_missing else None}\n"
@@ -33,12 +34,25 @@ def raise_column_mismatch_exception(msg: str, source_missing: list[str], target_
 
 def _generate_join_condition(source_alias, target_alias, key_columns):
     conditions = [
-        col(f"{source_alias}.{key_column}").eqNullSafe(col(f"{target_alias}.{key_column}"))
+        col(f"{source_alias}.{DialectUtils.ansi_normalize_identifier(key_column)}").eqNullSafe(
+            col(f"{target_alias}.{DialectUtils.ansi_normalize_identifier(key_column)}")
+        )
         for key_column in key_columns
     ]
     return reduce(lambda a, b: a & b, conditions)
 
 
+def _build_column_selector(table_name, column_name):
+    alias = DialectUtils.ansi_normalize_identifier(f"{table_name}_{DialectUtils.unnormalize_identifier(column_name)}")
+    return f'{table_name}.{DialectUtils.ansi_normalize_identifier(column_name)} as {alias}'
+
+
+def _build_mismatch_column(table, column):
+    return col(DialectUtils.ansi_normalize_identifier(column)).alias(
+        DialectUtils.unnormalize_identifier(column.replace(f'{table}_', '').lower())
+    )
+
+
 def reconcile_data(
     source: DataFrame,
     target: DataFrame,
@@ -59,14 +73,14 @@ def reconcile_data(
             how="full",
         )
         .selectExpr(
-            *[f'{source_alias}.{col_name} as {source_alias}_{col_name}' for col_name in source.columns],
-            *[f'{target_alias}.{col_name} as {target_alias}_{col_name}' for col_name in target.columns],
+            *[f'{_build_column_selector(source_alias, col_name)}' for col_name in source.columns],
+            *[f'{_build_column_selector(target_alias, col_name)}' for col_name in target.columns],
         )
     )
 
     # Write unmatched df to volume
     df = ReconIntermediatePersist(spark, path).write_and_read_unmatched_df_with_volumes(df)
-    logger.warning(f"Unmatched data is written to {path} successfully")
+    logger.warning(f"Unmatched data was written to {path} successfully")
 
     mismatch = _get_mismatch_data(df, source_alias, target_alias) if report_type in {"all", "data"} else None
 
@@ -74,24 +88,24 @@ def reconcile_data(
         df.filter(col(f"{source_alias}_{_HASH_COLUMN_NAME}").isNull())
         .select(
             *[
-                col(col_name).alias(col_name.replace(f'{target_alias}_', '').lower())
+                _build_mismatch_column(target_alias, col_name)
                 for col_name in df.columns
                 if col_name.startswith(f'{target_alias}_')
             ]
         )
-        .drop(_HASH_COLUMN_NAME)
+        .drop(f"{_HASH_COLUMN_NAME}")
     )
 
     missing_in_tgt = (
         df.filter(col(f"{target_alias}_{_HASH_COLUMN_NAME}").isNull())
         .select(
             *[
-                col(col_name).alias(col_name.replace(f'{source_alias}_', '').lower())
+                _build_mismatch_column(source_alias, col_name)
                 for col_name in df.columns
                 if col_name.startswith(f'{source_alias}_')
             ]
         )
-        .drop(_HASH_COLUMN_NAME)
+        .drop(f"{_HASH_COLUMN_NAME}")
     )
     mismatch_count = 0
     if mismatch:
@@ -123,23 +137,27 @@ def _get_mismatch_data(df: DataFrame, src_alias: str, tgt_alias: str) -> DataFra
         .filter(col("hash_match") == lit(False))
         .select(
             *[
-                col(col_name).alias(col_name.replace(f'{src_alias}_', '').lower())
+                _build_mismatch_column(src_alias, col_name)
                 for col_name in df.columns
                 if col_name.startswith(f'{src_alias}_')
             ]
         )
-        .drop(_HASH_COLUMN_NAME)
+        .drop(f"{_HASH_COLUMN_NAME}")
     )
 
 
-def _convert_columns_to_lowercase(df: DataFrame) -> DataFrame:
-    lowercased_columns = [col(column).alias(column.lower()) for column in df.columns]
-    return df.select(*lowercased_columns)
+def _build_capture_df(df: DataFrame) -> DataFrame:
+    columns = [
+        col(DialectUtils.ansi_normalize_identifier(column)).alias(DialectUtils.unnormalize_identifier(column))
+        for column in df.columns
+    ]
+    return df.select(*columns)
 
 
 def capture_mismatch_data_and_columns(source: DataFrame, target: DataFrame, key_columns: list[str]) -> MismatchOutput:
-    source_df = _convert_columns_to_lowercase(source)
-    target_df = _convert_columns_to_lowercase(target)
+    source_df = _build_capture_df(source)
+    target_df = _build_capture_df(target)
+    unnormalized_key_columns = [DialectUtils.unnormalize_identifier(column) for column in key_columns]
 
     source_columns = source_df.columns
     target_columns = target_df.columns
@@ -148,10 +166,10 @@ def capture_mismatch_data_and_columns(source: DataFrame, target: DataFrame, key_
         message = "source and target should have same columns for capturing the mismatch data"
         source_missing = [column for column in target_columns if column not in source_columns]
         target_missing = [column for column in source_columns if column not in target_columns]
-        raise raise_column_mismatch_exception(message, source_missing, target_missing)
+        raise _raise_column_mismatch_exception(message, source_missing, target_missing)
 
-    check_columns = [column for column in source_columns if column not in key_columns]
-    mismatch_df = _get_mismatch_df(source_df, target_df, key_columns, check_columns)
+    check_columns = [column for column in source_columns if column not in unnormalized_key_columns]
+    mismatch_df = _get_mismatch_df(source_df, target_df, unnormalized_key_columns, check_columns)
     mismatch_columns = _get_mismatch_columns(mismatch_df, check_columns)
     return MismatchOutput(mismatch_df, mismatch_columns)
 
@@ -167,31 +185,50 @@ def _get_mismatch_columns(df: DataFrame, columns: list[str]):
     return mismatch_columns
 
 
+def _normalize_mismatch_df_col(column, suffix):
+    unnormalized = DialectUtils.unnormalize_identifier(column) + suffix
+    return DialectUtils.ansi_normalize_identifier(unnormalized)
+
+
+def _unnormalize_mismatch_df_col(column, suffix):
+    unnormalized = DialectUtils.unnormalize_identifier(column) + suffix
+    return unnormalized
+
+
 def _get_mismatch_df(source: DataFrame, target: DataFrame, key_columns: list[str], column_list: list[str]):
-    source_aliased = [col('base.' + column).alias(column + '_base') for column in column_list]
-    target_aliased = [col('compare.' + column).alias(column + '_compare') for column in column_list]
+    source_aliased = [
+        col('base.' + DialectUtils.ansi_normalize_identifier(column)).alias(
+            _unnormalize_mismatch_df_col(column, '_base')
+        )
+        for column in column_list
+    ]
+    target_aliased = [
+        col('compare.' + DialectUtils.ansi_normalize_identifier(column)).alias(
+            _unnormalize_mismatch_df_col(column, '_compare')
+        )
+        for column in column_list
+    ]
 
-    match_expr = [expr(f"{column}_base=={column}_compare").alias(column + "_match") for column in column_list]
-    key_cols = [col(column) for column in key_columns]
+    match_expr = [
+        expr(f"{_normalize_mismatch_df_col(column,'_base')}=={_normalize_mismatch_df_col(column,'_compare')}").alias(
+            _unnormalize_mismatch_df_col(column, '_match')
+        )
+        for column in column_list
+    ]
+    key_cols = [col(DialectUtils.ansi_normalize_identifier(column)) for column in key_columns]
     select_expr = key_cols + source_aliased + target_aliased + match_expr
 
-    filter_columns = " and ".join([column + "_match" for column in column_list])
-    filter_expr = ~expr(filter_columns)
-
     logger.info(f"KEY COLUMNS: {key_columns}")
-    logger.info(f"FILTER COLUMNS: {filter_expr}")
     logger.info(f"SELECT COLUMNS: {select_expr}")
 
     mismatch_df = (
         source.alias('base').join(other=target.alias('compare'), on=key_columns, how="inner").select(*select_expr)
     )
 
-    compare_columns = [column for column in mismatch_df.columns if column not in key_columns]
-    return mismatch_df.select(*key_columns + sorted(compare_columns))
-
-
-def alias_column_str(alias: str, columns: list[str]) -> list[str]:
-    return [f"{alias}.{column}" for column in columns]
+    compare_columns = [
+        DialectUtils.ansi_normalize_identifier(column) for column in mismatch_df.columns if column not in key_columns
+    ]
+    return mismatch_df.select(*key_cols + sorted(compare_columns))
 
 
 def _generate_agg_join_condition(source_alias: str, target_alias: str, key_columns: list[str]):