databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/reconcile/connectors/data_source.py

@@ -0,0 +1,72 @@
+import logging
+from abc import ABC, abstractmethod
+
+from pyspark.sql import DataFrame
+
+from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException
+from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema
+
+logger = logging.getLogger(__name__)
+
+
+class DataSource(ABC):
+
+    @abstractmethod
+    def read_data(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+        query: str,
+        options: JdbcReaderOptions | None,
+    ) -> DataFrame:
+        return NotImplemented
+
+    @abstractmethod
+    def get_schema(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+    ) -> list[Schema]:
+        return NotImplemented
+
+    @classmethod
+    def log_and_throw_exception(cls, exception: Exception, fetch_type: str, query: str):
+        error_msg = f"Runtime exception occurred while fetching {fetch_type} using {query} : {exception}"
+        logger.warning(error_msg)
+        raise DataSourceRuntimeException(error_msg) from exception
+
+
+class MockDataSource(DataSource):
+
+    def __init__(
+        self,
+        dataframe_repository: dict[tuple[str, str, str], DataFrame],
+        schema_repository: dict[tuple[str, str, str], list[Schema]],
+        exception: Exception = RuntimeError("Mock Exception"),
+    ):
+        self._dataframe_repository: dict[tuple[str, str, str], DataFrame] = dataframe_repository
+        self._schema_repository: dict[tuple[str, str, str], list[Schema]] = schema_repository
+        self._exception = exception
+
+    def read_data(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+        query: str,
+        options: JdbcReaderOptions | None,
+    ) -> DataFrame:
+        catalog_str = catalog if catalog else ""
+        mock_df = self._dataframe_repository.get((catalog_str, schema, query))
+        if not mock_df:
+            return self.log_and_throw_exception(self._exception, "data", f"({catalog}, {schema}, {query})")
+        return mock_df
+
+    def get_schema(self, catalog: str | None, schema: str, table: str) -> list[Schema]:
+        catalog_str = catalog if catalog else ""
+        mock_schema = self._schema_repository.get((catalog_str, schema, table))
+        if not mock_schema:
+            return self.log_and_throw_exception(self._exception, "schema", f"({catalog}, {schema}, {table})")
+        return mock_schema

databricks/labs/lakebridge/reconcile/connectors/databricks.py

@@ -0,0 +1,87 @@
+import logging
+import re
+from datetime import datetime
+
+from pyspark.errors import PySparkException
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import col
+from sqlglot import Dialect
+
+from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource
+from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin
+from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema
+from databricks.sdk import WorkspaceClient
+
+logger = logging.getLogger(__name__)
+
+
+def _get_schema_query(catalog: str, schema: str, table: str):
+    # TODO: Ensure that the target_catalog in the configuration is not set to "hive_metastore". The source_catalog
+    #  can only be set to "hive_metastore" if the source type is "databricks".
+    if schema == "global_temp":
+        return f"describe table global_temp.{table}"
+    if catalog == "hive_metastore":
+        return f"describe table {catalog}.{schema}.{table}"
+
+    query = f"""select
+                            lower(column_name) as col_name,
+                             full_data_type as data_type
+                       from {catalog}.information_schema.columns
+                       where lower(table_catalog)='{catalog}'
+                                    and lower(table_schema)='{schema}'
+                                     and lower(table_name) ='{table}'
+                       order by col_name"""
+    return re.sub(r'\s+', ' ', query)
+
+
+class DatabricksDataSource(DataSource, SecretsMixin):
+
+    def __init__(
+        self,
+        engine: Dialect,
+        spark: SparkSession,
+        ws: WorkspaceClient,
+        secret_scope: str,
+    ):
+        self._engine = engine
+        self._spark = spark
+        self._ws = ws
+        self._secret_scope = secret_scope
+
+    def read_data(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+        query: str,
+        options: JdbcReaderOptions | None,
+    ) -> DataFrame:
+        namespace_catalog = "hive_metastore" if not catalog else catalog
+        if schema == "global_temp":
+            namespace_catalog = "global_temp"
+        else:
+            namespace_catalog = f"{namespace_catalog}.{schema}"
+        table_with_namespace = f"{namespace_catalog}.{table}"
+        table_query = query.replace(":tbl", table_with_namespace)
+        try:
+            df = self._spark.sql(table_query)
+            return df.select([col(column).alias(column.lower()) for column in df.columns])
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "data", table_query)
+
+    def get_schema(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+    ) -> list[Schema]:
+        catalog_str = catalog if catalog else "hive_metastore"
+        schema_query = _get_schema_query(catalog_str, schema, table)
+        try:
+            logger.debug(f"Fetching schema using query: \n`{schema_query}`")
+            logger.info(f"Fetching Schema: Started at: {datetime.now()}")
+            schema_metadata = self._spark.sql(schema_query).where("col_name not like '#%'").distinct().collect()
+            logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}")
+            return [Schema(field.col_name.lower(), field.data_type.lower()) for field in schema_metadata]
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "schema", schema_query)

databricks/labs/lakebridge/reconcile/connectors/jdbc_reader.py

@@ -0,0 +1,41 @@
+from typing import Any
+
+from pyspark.sql import SparkSession
+
+from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions
+
+
+class JDBCReaderMixin:
+    _spark: SparkSession
+
+    # TODO update the url
+    def _get_jdbc_reader(self, query, jdbc_url, driver, prepare_query=None):
+        driver_class = {
+            "oracle": "oracle.jdbc.driver.OracleDriver",
+            "snowflake": "net.snowflake.client.jdbc.SnowflakeDriver",
+            "sqlserver": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+        }
+        reader = (
+            self._spark.read.format("jdbc")
+            .option("url", jdbc_url)
+            .option("driver", driver_class.get(driver, driver))
+            .option("dbtable", f"({query}) tmp")
+        )
+        if prepare_query is not None:
+            reader = reader.option('prepareQuery', prepare_query)
+        return reader
+
+    @staticmethod
+    def _get_jdbc_reader_options(options: JdbcReaderOptions):
+        option_dict: dict[str, Any] = {}
+        if options.number_partitions:
+            option_dict["numPartitions"] = options.number_partitions
+        if options.partition_column:
+            option_dict["partitionColumn"] = options.partition_column
+        if options.lower_bound:
+            option_dict["lowerBound"] = options.lower_bound
+        if options.upper_bound:
+            option_dict["upperBound"] = options.upper_bound
+        if options.fetch_size:
+            option_dict["fetchsize"] = options.fetch_size
+        return option_dict

databricks/labs/lakebridge/reconcile/connectors/oracle.py

@@ -0,0 +1,108 @@
+import re
+import logging
+from datetime import datetime
+
+from pyspark.errors import PySparkException
+from pyspark.sql import DataFrame, DataFrameReader, SparkSession
+from pyspark.sql.functions import col
+from sqlglot import Dialect
+
+from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource
+from databricks.labs.lakebridge.reconcile.connectors.jdbc_reader import JDBCReaderMixin
+from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin
+from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema
+from databricks.sdk import WorkspaceClient
+
+logger = logging.getLogger(__name__)
+
+
+class OracleDataSource(DataSource, SecretsMixin, JDBCReaderMixin):
+    _DRIVER = "oracle"
+    _SCHEMA_QUERY = """select column_name, case when (data_precision is not null
+                                              and data_scale <> 0)
+                                              then data_type || '(' || data_precision || ',' || data_scale || ')'
+                                              when (data_precision is not null and data_scale = 0)
+                                              then data_type || '(' || data_precision || ')'
+                                              when data_precision is null and (lower(data_type) in ('date') or
+                                              lower(data_type) like 'timestamp%') then  data_type
+                                              when CHAR_LENGTH = 0 then data_type
+                                              else data_type || '(' || CHAR_LENGTH || ')'
+                                              end data_type
+                                              FROM ALL_TAB_COLUMNS
+                            WHERE lower(TABLE_NAME) = '{table}' and lower(owner) = '{owner}'"""
+
+    def __init__(
+        self,
+        engine: Dialect,
+        spark: SparkSession,
+        ws: WorkspaceClient,
+        secret_scope: str,
+    ):
+        self._engine = engine
+        self._spark = spark
+        self._ws = ws
+        self._secret_scope = secret_scope
+
+    @property
+    def get_jdbc_url(self) -> str:
+        return (
+            f"jdbc:{OracleDataSource._DRIVER}:thin:{self._get_secret('user')}"
+            f"/{self._get_secret('password')}@//{self._get_secret('host')}"
+            f":{self._get_secret('port')}/{self._get_secret('database')}"
+        )
+
+    def read_data(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+        query: str,
+        options: JdbcReaderOptions | None,
+    ) -> DataFrame:
+        table_query = query.replace(":tbl", f"{schema}.{table}")
+        try:
+            if options is None:
+                return self.reader(table_query).options(**self._get_timestamp_options()).load()
+            reader_options = self._get_jdbc_reader_options(options) | self._get_timestamp_options()
+            df = self.reader(table_query).options(**reader_options).load()
+            logger.warning(f"Fetching data using query: \n`{table_query}`")
+
+            # Convert all column names to lower case
+            df = df.select([col(c).alias(c.lower()) for c in df.columns])
+            return df
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "data", table_query)
+
+    def get_schema(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+    ) -> list[Schema]:
+        schema_query = re.sub(
+            r'\s+',
+            ' ',
+            OracleDataSource._SCHEMA_QUERY.format(table=table, owner=schema),
+        )
+        try:
+            logger.debug(f"Fetching schema using query: \n`{schema_query}`")
+            logger.info(f"Fetching Schema: Started at: {datetime.now()}")
+            df = self.reader(schema_query).load()
+            schema_metadata = df.select([col(c).alias(c.lower()) for c in df.columns]).collect()
+            logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}")
+            logger.debug(f"schema_metadata: ${schema_metadata}")
+            return [Schema(field.column_name.lower(), field.data_type.lower()) for field in schema_metadata]
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "schema", schema_query)
+
+    @staticmethod
+    def _get_timestamp_options() -> dict[str, str]:
+        return {
+            "oracle.jdbc.mapDateToTimestamp": "False",
+            "sessionInitStatement": "BEGIN dbms_session.set_nls('nls_date_format', "
+            "'''YYYY-MM-DD''');dbms_session.set_nls('nls_timestamp_format', '''YYYY-MM-DD "
+            "HH24:MI:SS''');END;",
+        }
+
+    def reader(self, query: str) -> DataFrameReader:
+        return self._get_jdbc_reader(query, self.get_jdbc_url, OracleDataSource._DRIVER)

databricks/labs/lakebridge/reconcile/connectors/secrets.py

@@ -0,0 +1,30 @@
+import base64
+import logging
+
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.errors import NotFound
+
+logger = logging.getLogger(__name__)
+
+
+class SecretsMixin:
+    _ws: WorkspaceClient
+    _secret_scope: str
+
+    def _get_secret(self, secret_key: str) -> str:
+        """Get the secret value given a secret scope & secret key. Log a warning if secret does not exist"""
+        try:
+            # Return the decoded secret value in string format
+            secret = self._ws.secrets.get_secret(self._secret_scope, secret_key)
+            assert secret.value is not None
+            return base64.b64decode(secret.value).decode("utf-8")
+        except NotFound as e:
+            raise NotFound(f'Secret does not exist with scope: {self._secret_scope} and key: {secret_key} : {e}') from e
+        except UnicodeDecodeError as e:
+            raise UnicodeDecodeError(
+                "utf-8",
+                secret_key.encode(),
+                0,
+                1,
+                f"Secret {self._secret_scope}/{secret_key} has Base64 bytes that cannot be decoded to utf-8 string: {e}.",
+            ) from e

databricks/labs/lakebridge/reconcile/connectors/snowflake.py

@@ -0,0 +1,173 @@
+import logging
+import re
+from datetime import datetime
+
+from pyspark.errors import PySparkException
+from pyspark.sql import DataFrame, DataFrameReader, SparkSession
+from pyspark.sql.functions import col
+from sqlglot import Dialect
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+
+from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource
+from databricks.labs.lakebridge.reconcile.connectors.jdbc_reader import JDBCReaderMixin
+from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin
+from databricks.labs.lakebridge.reconcile.exception import InvalidSnowflakePemPrivateKey
+from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.errors import NotFound
+
+logger = logging.getLogger(__name__)
+
+
+class SnowflakeDataSource(DataSource, SecretsMixin, JDBCReaderMixin):
+    _DRIVER = "snowflake"
+    """
+       * INFORMATION_SCHEMA:
+          - see https://docs.snowflake.com/en/sql-reference/info-schema#considerations-for-replacing-show-commands-with-information-schema-views
+       * DATA:
+          - only unquoted identifiers are treated as case-insensitive and are stored in uppercase.
+          - for quoted identifiers refer:
+             https://docs.snowflake.com/en/sql-reference/identifiers-syntax#double-quoted-identifiers
+       * ORDINAL_POSITION:
+          - indicates the sequential order of a column within a table or view,
+             starting from 1 based on the order of column definition.
+    """
+    _SCHEMA_QUERY = """select column_name,
+                                                      case
+                                                            when numeric_precision is not null and numeric_scale is not null
+                                                            then
+                                                                concat(data_type, '(', numeric_precision, ',' , numeric_scale, ')')
+                                                            when lower(data_type) = 'text'
+                                                            then
+                                                                concat('varchar', '(', CHARACTER_MAXIMUM_LENGTH, ')')
+                                                            else data_type
+                                                      end as data_type
+                                                      from {catalog}.INFORMATION_SCHEMA.COLUMNS
+                                                      where lower(table_name)='{table}' and table_schema = '{schema}'
+                                                      order by ordinal_position"""
+
+    def __init__(
+        self,
+        engine: Dialect,
+        spark: SparkSession,
+        ws: WorkspaceClient,
+        secret_scope: str,
+    ):
+        self._engine = engine
+        self._spark = spark
+        self._ws = ws
+        self._secret_scope = secret_scope
+
+    @property
+    def get_jdbc_url(self) -> str:
+        try:
+            sf_password = self._get_secret('sfPassword')
+        except (NotFound, KeyError) as e:
+            message = "sfPassword is mandatory for jdbc connectivity with Snowflake."
+            logger.error(message)
+            raise NotFound(message) from e
+
+        return (
+            f"jdbc:{SnowflakeDataSource._DRIVER}://{self._get_secret('sfAccount')}.snowflakecomputing.com"
+            f"/?user={self._get_secret('sfUser')}&password={sf_password}"
+            f"&db={self._get_secret('sfDatabase')}&schema={self._get_secret('sfSchema')}"
+            f"&warehouse={self._get_secret('sfWarehouse')}&role={self._get_secret('sfRole')}"
+        )
+
+    @staticmethod
+    def get_private_key(pem_private_key: str) -> str:
+        try:
+            private_key_bytes = pem_private_key.encode("UTF-8")
+            p_key = serialization.load_pem_private_key(
+                private_key_bytes,
+                password=None,
+                backend=default_backend(),
+            )
+            pkb = p_key.private_bytes(
+                encoding=serialization.Encoding.PEM,
+                format=serialization.PrivateFormat.PKCS8,
+                encryption_algorithm=serialization.NoEncryption(),
+            )
+            pkb_str = pkb.decode("UTF-8")
+            # Remove the first and last lines (BEGIN/END markers)
+            private_key_pem_lines = pkb_str.strip().split('\n')[1:-1]
+            # Join the lines to form the base64 encoded string
+            private_key_pem_str = ''.join(private_key_pem_lines)
+            return private_key_pem_str
+        except Exception as e:
+            message = f"Failed to load or process the provided PEM private key. --> {e}"
+            logger.error(message)
+            raise InvalidSnowflakePemPrivateKey(message) from e
+
+    def read_data(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+        query: str,
+        options: JdbcReaderOptions | None,
+    ) -> DataFrame:
+        table_query = query.replace(":tbl", f"{catalog}.{schema}.{table}")
+        try:
+            if options is None:
+                df = self.reader(table_query).load()
+            else:
+                options = self._get_jdbc_reader_options(options)
+                df = (
+                    self._get_jdbc_reader(table_query, self.get_jdbc_url, SnowflakeDataSource._DRIVER)
+                    .options(**options)
+                    .load()
+                )
+            return df.select([col(column).alias(column.lower()) for column in df.columns])
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "data", table_query)
+
+    def get_schema(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+    ) -> list[Schema]:
+        """
+        Fetch the Schema from the INFORMATION_SCHEMA.COLUMNS table in Snowflake.
+
+        If the user's current role does not have the necessary privileges to access the specified
+        Information Schema object, RunTimeError will be raised:
+        "SQL access control error: Insufficient privileges to operate on schema 'INFORMATION_SCHEMA' "
+        """
+        schema_query = re.sub(
+            r'\s+',
+            ' ',
+            SnowflakeDataSource._SCHEMA_QUERY.format(catalog=catalog, schema=schema.upper(), table=table),
+        )
+        try:
+            logger.debug(f"Fetching schema using query: \n`{schema_query}`")
+            logger.info(f"Fetching Schema: Started at: {datetime.now()}")
+            schema_metadata = self.reader(schema_query).load().collect()
+            logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}")
+            return [Schema(field.COLUMN_NAME.lower(), field.DATA_TYPE.lower()) for field in schema_metadata]
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "schema", schema_query)
+
+    def reader(self, query: str) -> DataFrameReader:
+        options = {
+            "sfUrl": self._get_secret('sfUrl'),
+            "sfUser": self._get_secret('sfUser'),
+            "sfDatabase": self._get_secret('sfDatabase'),
+            "sfSchema": self._get_secret('sfSchema'),
+            "sfWarehouse": self._get_secret('sfWarehouse'),
+            "sfRole": self._get_secret('sfRole'),
+        }
+        try:
+            options["pem_private_key"] = SnowflakeDataSource.get_private_key(self._get_secret('pem_private_key'))
+        except (NotFound, KeyError):
+            logger.warning("pem_private_key not found. Checking for sfPassword")
+            try:
+                options["sfPassword"] = self._get_secret('sfPassword')
+            except (NotFound, KeyError) as e:
+                message = "sfPassword and pem_private_key not found. Either one is required for snowflake auth."
+                logger.error(message)
+                raise NotFound(message) from e
+
+        return self._spark.read.format("snowflake").option("dbtable", f"({query}) as tmp").options(**options)

databricks/labs/lakebridge/reconcile/connectors/source_adapter.py

@@ -0,0 +1,30 @@
+from pyspark.sql import SparkSession
+from sqlglot import Dialect
+from sqlglot.dialects import TSQL
+
+from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource
+from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource
+from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource
+from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource
+from databricks.labs.lakebridge.reconcile.connectors.sql_server import SQLServerDataSource
+from databricks.labs.lakebridge.transpiler.sqlglot.generator.databricks import Databricks
+from databricks.labs.lakebridge.transpiler.sqlglot.parsers.oracle import Oracle
+from databricks.labs.lakebridge.transpiler.sqlglot.parsers.snowflake import Snowflake
+from databricks.sdk import WorkspaceClient
+
+
+def create_adapter(
+    engine: Dialect,
+    spark: SparkSession,
+    ws: WorkspaceClient,
+    secret_scope: str,
+) -> DataSource:
+    if isinstance(engine, Snowflake):
+        return SnowflakeDataSource(engine, spark, ws, secret_scope)
+    if isinstance(engine, Oracle):
+        return OracleDataSource(engine, spark, ws, secret_scope)
+    if isinstance(engine, Databricks):
+        return DatabricksDataSource(engine, spark, ws, secret_scope)
+    if isinstance(engine, TSQL):
+        return SQLServerDataSource(engine, spark, ws, secret_scope)
+    raise ValueError(f"Unsupported source type --> {engine}")

databricks/labs/lakebridge/reconcile/connectors/sql_server.py

@@ -0,0 +1,132 @@
+import re
+import logging
+from datetime import datetime
+
+from pyspark.errors import PySparkException
+from pyspark.sql import DataFrame, DataFrameReader, SparkSession
+from pyspark.sql.functions import col
+from sqlglot import Dialect
+
+from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource
+from databricks.labs.lakebridge.reconcile.connectors.jdbc_reader import JDBCReaderMixin
+from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin
+from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema
+from databricks.sdk import WorkspaceClient
+
+logger = logging.getLogger(__name__)
+
+_SCHEMA_QUERY = """SELECT
+                     COLUMN_NAME,
+                     CASE
+                        WHEN DATA_TYPE IN ('int', 'bigint')
+                            THEN DATA_TYPE
+                        WHEN DATA_TYPE IN ('smallint', 'tinyint')
+                            THEN 'smallint'
+                        WHEN DATA_TYPE IN ('decimal' ,'numeric')
+                            THEN 'decimal(' +
+                                CAST(NUMERIC_PRECISION AS VARCHAR) + ',' +
+                                CAST(NUMERIC_SCALE AS VARCHAR) + ')'
+                        WHEN DATA_TYPE IN ('float', 'real')
+                                THEN 'double'
+                        WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE IN ('varchar','char','text','nchar','nvarchar','ntext')
+                                THEN DATA_TYPE
+                        WHEN DATA_TYPE IN ('date','time','datetime', 'datetime2','smalldatetime','datetimeoffset')
+                                THEN DATA_TYPE
+                        WHEN DATA_TYPE IN ('bit')
+                                THEN 'boolean'
+                        WHEN DATA_TYPE IN ('binary','varbinary')
+                                THEN 'binary'
+                        ELSE DATA_TYPE
+                    END AS 'DATA_TYPE'
+                    FROM
+                        INFORMATION_SCHEMA.COLUMNS
+                    WHERE
+                    LOWER(TABLE_NAME) = LOWER('{table}')
+                    AND LOWER(TABLE_SCHEMA) = LOWER('{schema}')
+                    AND LOWER(TABLE_CATALOG) = LOWER('{catalog}')
+              """
+
+
+class SQLServerDataSource(DataSource, SecretsMixin, JDBCReaderMixin):
+    _DRIVER = "sqlserver"
+
+    def __init__(
+        self,
+        engine: Dialect,
+        spark: SparkSession,
+        ws: WorkspaceClient,
+        secret_scope: str,
+    ):
+        self._engine = engine
+        self._spark = spark
+        self._ws = ws
+        self._secret_scope = secret_scope
+
+    @property
+    def get_jdbc_url(self) -> str:
+        # Construct the JDBC URL
+        return (
+            f"jdbc:{self._DRIVER}://{self._get_secret('host')}:{self._get_secret('port')};"
+            f"databaseName={self._get_secret('database')};"
+            f"user={self._get_secret('user')};"
+            f"password={self._get_secret('password')};"
+            f"encrypt={self._get_secret('encrypt')};"
+            f"trustServerCertificate={self._get_secret('trustServerCertificate')};"
+        )
+
+    def read_data(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+        query: str,
+        options: JdbcReaderOptions | None,
+    ) -> DataFrame:
+        table_query = query.replace(":tbl", f"{catalog}.{schema}.{table}")
+        with_clause_pattern = re.compile(r'WITH\s+.*?\)\s*(?=SELECT)', re.IGNORECASE | re.DOTALL)
+        match = with_clause_pattern.search(table_query)
+        if match:
+            prepare_query_string = match.group(0)
+            query = table_query.replace(match.group(0), '')
+        else:
+            query = table_query
+            prepare_query_string = ""
+        try:
+            if options is None:
+                df = self.reader(query, prepare_query_string).load()
+            else:
+                options = self._get_jdbc_reader_options(options)
+                df = self._get_jdbc_reader(table_query, self.get_jdbc_url, self._DRIVER).options(**options).load()
+            return df.select([col(column).alias(column.lower()) for column in df.columns])
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "data", table_query)
+
+    def get_schema(
+        self,
+        catalog: str | None,
+        schema: str,
+        table: str,
+    ) -> list[Schema]:
+        """
+        Fetch the Schema from the INFORMATION_SCHEMA.COLUMNS table in SQL Server.
+
+        If the user's current role does not have the necessary privileges to access the specified
+        Information Schema object, RunTimeError will be raised:
+        "SQL access control error: Insufficient privileges to operate on schema 'INFORMATION_SCHEMA' "
+        """
+        schema_query = re.sub(
+            r'\s+',
+            ' ',
+            _SCHEMA_QUERY.format(catalog=catalog, schema=schema, table=table),
+        )
+        try:
+            logger.debug(f"Fetching schema using query: \n`{schema_query}`")
+            logger.info(f"Fetching Schema: Started at: {datetime.now()}")
+            schema_metadata = self.reader(schema_query).load().collect()
+            logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}")
+            return [Schema(field.COLUMN_NAME.lower(), field.DATA_TYPE.lower()) for field in schema_metadata]
+        except (RuntimeError, PySparkException) as e:
+            return self.log_and_throw_exception(e, "schema", schema_query)
+
+    def reader(self, query: str, prepare_query_str="") -> DataFrameReader:
+        return self._get_jdbc_reader(query, self.get_jdbc_url, self._DRIVER, prepare_query_str)