soda-databricks 4.0.5__tar.gz

soda_databricks-4.0.5/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: soda-databricks
+Version: 4.0.5
+Requires-Dist: soda-core==4.0.5
+Requires-Dist: databricks-sql-connector
+Dynamic: requires-dist

soda_databricks-4.0.5/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

soda_databricks-4.0.5/setup.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+package_name = "soda-databricks"
+package_version = "4.0.5"
+description = "Soda Databricks V4"
+
+requires = [
+    f"soda-core=={package_version}",
+    "databricks-sql-connector",
+]
+
+setup(
+    name=package_name,
+    version=package_version,
+    install_requires=requires,
+    package_dir={"": "src"},
+    entry_points={
+        "soda.plugins.data_source.databricks": [
+            "DatabricksDataSourceImpl = soda_databricks.common.data_sources.databricks_data_source:DatabricksDataSourceImpl",
+        ],
+    },
+)

soda_databricks-4.0.5/src/soda_databricks/common/data_sources/databricks_data_source.py

@@ -0,0 +1,298 @@
+from logging import Logger
+from typing import Optional
+
+from soda_core.common.data_source_connection import DataSourceConnection
+from soda_core.common.data_source_impl import DataSourceImpl
+from soda_core.common.data_source_results import QueryResult
+from soda_core.common.logging_constants import soda_logger
+from soda_core.common.metadata_types import (
+    ColumnMetadata,
+    DataSourceNamespace,
+    SodaDataTypeName,
+)
+from soda_core.common.sql_ast import (
+    ALTER_TABLE_ADD_COLUMN,
+    ALTER_TABLE_DROP_COLUMN,
+    CREATE_TABLE_COLUMN,
+)
+from soda_core.common.sql_dialect import SqlDialect
+from soda_core.common.statements.metadata_tables_query import MetadataTablesQuery
+from soda_core.common.statements.table_types import TableType
+from soda_databricks.common.data_sources.databricks_data_source_connection import (
+    DatabricksDataSourceConnection,
+)
+from soda_databricks.common.statements.hive_metadata_tables_query import (
+    HiveMetadataTablesQuery,
+)
+from soda_databricks.model.data_source.databricks_data_source import (
+    DatabricksDataSource as DatabricksDataSourceModel,
+)
+
+logger: Logger = soda_logger
+
+
+class DatabricksDataSourceImpl(DataSourceImpl, model_class=DatabricksDataSourceModel):
+    def __init__(self, data_source_model: DatabricksDataSourceModel, connection: Optional[DataSourceConnection] = None):
+        super().__init__(data_source_model=data_source_model, connection=connection)
+
+    def _create_sql_dialect(self) -> SqlDialect:
+        if self.__is_hive_catalog():
+            return DatabricksHiveSqlDialect(data_source_impl=self)
+        return DatabricksSqlDialect(data_source_impl=self)
+
+    def _create_data_source_connection(self) -> DataSourceConnection:
+        return DatabricksDataSourceConnection(
+            name=self.data_source_model.name, connection_properties=self.data_source_model.connection_properties
+        )
+
+    def create_metadata_tables_query(self) -> MetadataTablesQuery:
+        if self.__is_hive_catalog():
+            return HiveMetadataTablesQuery(
+                sql_dialect=self.sql_dialect, data_source_connection=self.data_source_connection
+            )
+        else:
+            return super().create_metadata_tables_query()
+
+    def __is_hive_catalog(self):
+        # Check the connection "catalog"
+        catalog: Optional[str] = self.data_source_model.connection_properties.catalog
+        if catalog and catalog.lower() == "hive_metastore":
+            return True
+        # All other catalogs should be treated as "unity catalogs"
+        return False
+
+    def get_columns_metadata(self, dataset_prefixes: list[str], dataset_name: str) -> list[ColumnMetadata]:
+        try:
+            return super().get_columns_metadata(dataset_prefixes, dataset_name)
+        except Exception as e:
+            logger.warning(f"Error getting columns metadata for {dataset_name}: {e}\n\nReturning empty list.")
+            return []
+
+    def test_schema_exists(self, prefixes: list[str]) -> bool:
+        if not self.__is_hive_catalog():
+            return super().test_schema_exists(prefixes)
+
+        schema_name: str = prefixes[1]
+
+        result = self.execute_query(
+            f"SHOW SCHEMAS LIKE '{schema_name}'"
+        ).rows  # We only need to check the schema name, as the catalog name is always the same
+        for row in result:
+            if row[0] and row[0].lower() == schema_name.lower():
+                return True
+        return False
+
+
+class DatabricksSqlDialect(SqlDialect):
+    DEFAULT_QUOTE_CHAR = "`"
+
+    SODA_DATA_TYPE_SYNONYMS = (
+        (SodaDataTypeName.TEXT, SodaDataTypeName.VARCHAR, SodaDataTypeName.CHAR),
+        (SodaDataTypeName.NUMERIC, SodaDataTypeName.DECIMAL),
+        (SodaDataTypeName.TIMESTAMP_TZ, SodaDataTypeName.TIMESTAMP),
+    )
+
+    def _get_data_type_name_synonyms(self) -> list[list[str]]:
+        return [
+            ["int", "integer"],
+        ]
+
+    def column_data_type(self) -> str:
+        return self.default_casify("data_type")
+
+    def supports_data_type_character_maximum_length(self) -> bool:
+        return False
+
+    def supports_data_type_numeric_precision(self) -> bool:
+        return False
+
+    def supports_data_type_numeric_scale(self) -> bool:
+        return False
+
+    def supports_data_type_datetime_precision(self) -> bool:
+        return False
+
+    def get_data_source_data_type_name_by_soda_data_type_names(self) -> dict:
+        return {
+            SodaDataTypeName.CHAR: "string",
+            SodaDataTypeName.VARCHAR: "string",
+            SodaDataTypeName.TEXT: "string",
+            SodaDataTypeName.SMALLINT: "smallint",
+            SodaDataTypeName.INTEGER: "int",
+            SodaDataTypeName.BIGINT: "bigint",
+            SodaDataTypeName.NUMERIC: "decimal",
+            SodaDataTypeName.DECIMAL: "decimal",
+            SodaDataTypeName.FLOAT: "float",
+            SodaDataTypeName.DOUBLE: "double",
+            SodaDataTypeName.TIMESTAMP: "timestamp",
+            SodaDataTypeName.TIMESTAMP_TZ: "timestamp",
+            SodaDataTypeName.DATE: "date",
+            SodaDataTypeName.TIME: "string",  # no native TIME type in Databricks
+            SodaDataTypeName.BOOLEAN: "boolean",
+        }
+
+    def get_soda_data_type_name_by_data_source_data_type_names(self) -> dict[str, SodaDataTypeName]:
+        return {
+            "string": SodaDataTypeName.TEXT,
+            "varchar": SodaDataTypeName.VARCHAR,
+            "char": SodaDataTypeName.CHAR,
+            "tinyint": SodaDataTypeName.SMALLINT,
+            "short": SodaDataTypeName.SMALLINT,
+            "smallint": SodaDataTypeName.SMALLINT,
+            "int": SodaDataTypeName.INTEGER,
+            "integer": SodaDataTypeName.INTEGER,
+            "bigint": SodaDataTypeName.BIGINT,
+            "long": SodaDataTypeName.BIGINT,
+            "decimal": SodaDataTypeName.DECIMAL,
+            "numeric": SodaDataTypeName.NUMERIC,
+            "float": SodaDataTypeName.FLOAT,
+            "real": SodaDataTypeName.FLOAT,
+            "float4": SodaDataTypeName.FLOAT,
+            "double": SodaDataTypeName.DOUBLE,
+            "double precision": SodaDataTypeName.DOUBLE,
+            "float8": SodaDataTypeName.DOUBLE,
+            "timestamp": SodaDataTypeName.TIMESTAMP,
+            "timestamp without time zone": SodaDataTypeName.TIMESTAMP,
+            "timestamp_ntz": SodaDataTypeName.TIMESTAMP,  # If there is explicitly stated that the timestamp is without time zone, we consider it to be the same as TIMESTAMP
+            "timestamptz": SodaDataTypeName.TIMESTAMP_TZ,
+            "timestamp with time zone": SodaDataTypeName.TIMESTAMP_TZ,
+            "date": SodaDataTypeName.DATE,
+            "boolean": SodaDataTypeName.BOOLEAN,
+            # Not supported -> will be converted to varchar
+            # "binary"
+            # "interval",
+            # "array",
+            # "map",
+            # "struct"
+        }
+
+    def escape_string(self, value: str):
+        raw_string = rf"{value}"
+        string_literal: str = raw_string.replace(r"'", r"\'")
+        return string_literal
+
+    def encode_string_for_sql(self, string: str) -> str:
+        """This escapes values that contain newlines correctly."""
+        # For databricks, we don't need to encode the string, it's able to handle the newlines correctly.
+        # In fact, when we encode the string, we run into issues with the escape characters for the quotes
+        return string
+
+    def column_data_type_numeric_scale(self) -> str:
+        return self.default_casify("numeric_scale")
+
+    def column_data_type_datetime_precision(self) -> str:
+        return self.default_casify("datetime_precision")
+
+    def _build_create_table_column_type(self, create_table_column: CREATE_TABLE_COLUMN) -> str:
+        # Databricks will complain if string lengths or datetime precisions are passed in, so strip if they are provided
+        if create_table_column.type.name == "string":
+            create_table_column.type.character_maximum_length = None
+        if create_table_column.type.name in ["timestamp_ntz", "timestamp"]:
+            create_table_column.type.datetime_precision = None
+        return super()._build_create_table_column_type(create_table_column=create_table_column)
+
+    def _get_data_type_name_synonyms(self) -> list[list[str]]:
+        return [
+            ["varchar", "char", "string"],
+            ["smallint", "int2"],
+            ["integer", "int", "int4"],
+            ["bigint", "int8"],
+            ["real", "float4", "float"],
+            ["double precision", "float8", "double"],
+            ["timestamp", "timestamp without time zone"],
+            ["timestamptz", "timestamp with time zone"],
+            ["time", "time without time zone"],
+        ]
+
+    # Explicitly leaving this here for reference. See comment below why we moved to DESCRIBE TABLE.
+    # def build_columns_metadata_query_str(self, table_namespace: DataSourceNamespace, table_name: str) -> str:
+    # Unity catalog only stores things in lower case,
+    # even though create table may have been quoted and with mixed case
+    # table_name_lower: str = table_name.lower()
+    # return super().build_columns_metadata_query_str(table_namespace, table_name_lower)
+
+    # We move to DESCRIBE TABLE, that is a more up-to-date way to get the columns metadata. (information_schema is lagging behind sometimes, and does not always return the correct columns)
+    def build_columns_metadata_query_str(self, table_namespace: DataSourceNamespace, table_name: str) -> str:
+        database_name: str | None = table_namespace.get_database_for_metadata_query()
+        schema_name: str = table_namespace.get_schema_for_metadata_query()
+        return f"DESCRIBE {database_name}.{schema_name}.{table_name}"
+
+    def build_column_metadatas_from_query_result(self, query_result: QueryResult) -> list[ColumnMetadata]:
+        # Filter out dataset description rows (first such line starts with #, ignore the rest) or empty
+        filtered_rows = []
+        for row in query_result.rows:
+            if row[0].startswith("#"):  # ignore all description rows
+                break
+            if not row[0] and not row[1]:  # empty row
+                continue
+
+            # Trim data type details, e.g. decimal(10,0) -> decimal. Only decimal supports it anyway.
+            data_type = row[1]
+            if "(" in data_type:
+                data_type = data_type[: data_type.index("(")].strip()
+            row = (row[0], data_type) + row[2:]
+            filtered_rows.append(row)
+
+        return super().build_column_metadatas_from_query_result(
+            QueryResult(rows=filtered_rows, columns=query_result.columns)
+        )
+
+    def post_schema_create_sql(self, prefixes: list[str]) -> Optional[list[str]]:
+        assert len(prefixes) == 2, f"Expected 2 prefixes, got {len(prefixes)}"
+        catalog_name: str = self.quote_default(prefixes[0])
+        schema_name: str = self.quote_default(prefixes[1])
+        return [f"GRANT SELECT, USAGE, CREATE, MANAGE ON SCHEMA {catalog_name}.{schema_name} TO `account users`;"]
+        #      f"GRANT SELECT ON FUTURE TABLES IN SCHEMA {catalog_name}.{schema_name} TO `account users`;"]
+
+    @classmethod
+    def is_same_soda_data_type_with_synonyms(cls, expected: SodaDataTypeName, actual: SodaDataTypeName) -> bool:
+        # Special case of a 1-way synonym: TEXT is allowed where TIME is expected
+        if expected == SodaDataTypeName.TIME and actual == SodaDataTypeName.TEXT:
+            logger.debug(
+                f"In is_same_soda_data_type_with_synonyms, Expected {expected} and actual {actual} are the same"
+            )
+            return True
+        return super().is_same_soda_data_type_with_synonyms(expected, actual)
+
+    def _build_alter_table_add_column_sql(
+        self, alter_table: ALTER_TABLE_ADD_COLUMN, add_semicolon: bool = True, add_parenthesis: bool = False
+    ) -> str:
+        return super()._build_alter_table_add_column_sql(alter_table, add_semicolon=add_semicolon, add_parenthesis=True)
+
+    def _get_add_column_sql_expr(self) -> str:
+        return "ADD COLUMNS"
+
+    def _build_alter_table_drop_column_sql(
+        self, alter_table: ALTER_TABLE_DROP_COLUMN, add_semicolon: bool = True
+    ) -> str:
+        column_name_quoted: str = self._quote_column_for_create_table(alter_table.column_name)
+        return f"ALTER TABLE {alter_table.fully_qualified_table_name} DROP COLUMNS ({column_name_quoted})" + (
+            ";" if add_semicolon else ""
+        )
+
+    def drop_column_supported(self) -> bool:
+        return False  # Note, this is technically supported. But we need to change the delta table mapping mode name for this (out of scope at the time of writing)
+
+    def convert_table_type_to_enum(self, table_type: str) -> TableType:
+        if table_type == "MANAGED":
+            return TableType.TABLE
+        elif table_type == "VIEW":
+            return TableType.VIEW
+        elif table_type == "MATERIALIZED_VIEW":
+            return TableType.VIEW  # For now, a materialized view is treated as a view.
+        else:
+            # Default to TABLE if the table type is not recognized (so we're backwards compatible with existing code)
+            logger.warning(f"Invalid table type: {table_type}, defaulting to TABLE")
+            return TableType.TABLE
+
+    def metadata_casify(self, identifier: str) -> str:
+        return identifier.lower()
+
+
+class DatabricksHiveSqlDialect(DatabricksSqlDialect):
+    def post_schema_create_sql(self, prefixes: list[str]) -> Optional[list[str]]:
+        assert len(prefixes) == 2, f"Expected 2 prefixes, got {len(prefixes)}"
+        catalog_name: str = self.quote_default(prefixes[0])
+        schema_name: str = self.quote_default(prefixes[1])
+
+        return [f"GRANT SELECT, USAGE, CREATE ON SCHEMA {catalog_name}.{schema_name} TO `users`;"]

soda_databricks-4.0.5/src/soda_databricks/common/data_sources/databricks_data_source_connection.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import logging
+
+from databricks import sql
+from soda_core.common.data_source_connection import DataSourceConnection
+from soda_core.common.logging_constants import soda_logger
+from soda_core.model.data_source.data_source_connection_properties import (
+    DataSourceConnectionProperties,
+)
+from soda_databricks.model.data_source.databricks_connection_properties import (
+    DatabricksConnectionProperties,
+)
+
+logger: logging.Logger = soda_logger
+
+
+class DatabricksDataSourceConnection(DataSourceConnection):
+    def __init__(self, name: str, connection_properties: DataSourceConnectionProperties):
+        super().__init__(name, connection_properties)
+
+    def _create_connection(
+        self,
+        config: DatabricksConnectionProperties,
+    ):
+        return sql.connect(
+            user_agent_entry="Soda Core",
+            **config.to_connection_kwargs(),
+        )
+
+    def rollback(self) -> None:
+        # We do not start any transactions, Databricks default is autocommit.
+        pass
+
+    def commit(self) -> None:
+        # We do not start any transactions, Databricks default is autocommit.
+        pass
+
+    def _execute_query_get_result_row_column_name(self, column) -> str:
+        return column[0]

soda_databricks-4.0.5/src/soda_databricks/common/statements/hive_metadata_tables_query.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from soda_core.common.data_source_connection import DataSourceConnection
+from soda_core.common.data_source_results import QueryResult
+from soda_core.common.sql_dialect import SqlDialect
+from soda_core.common.statements.metadata_tables_query import (
+    FullyQualifiedTableName,
+    MetadataTablesQuery,
+)
+from soda_core.common.statements.table_types import FullyQualifiedViewName, TableType
+
+
+class HiveMetadataTablesQuery(MetadataTablesQuery):
+    def __init__(
+        self,
+        sql_dialect: SqlDialect,
+        data_source_connection: DataSourceConnection,
+        prefixes: Optional[list[str]] = None,
+    ):
+        self.sql_dialect = sql_dialect
+        self.data_source_connection: DataSourceConnection = data_source_connection
+        self.prefixes = prefixes
+
+    def execute(
+        self,
+        database_name: Optional[str] = None,
+        schema_name: Optional[str] = None,
+        include_table_name_like_filters: Optional[list[str]] = None,
+        exclude_table_name_like_filters: Optional[list[str]] = None,
+        types_to_return: Optional[
+            list[TableType]
+        ] = None,  # To make sure it's backwards compatible with the old behavior, when we use None it should default to [TableType.TABLE]
+    ) -> list[FullyQualifiedTableName]:
+        if types_to_return is None:
+            types_to_return = [TableType.TABLE]
+        results: list[FullyQualifiedTableName] = []
+        if TableType.TABLE in types_to_return:
+            sql: str = self.build_sql_statement(
+                database_name=database_name, schema_name=schema_name, object_type_to_fetch=TableType.TABLE
+            )
+            query_result: QueryResult = self.data_source_connection.execute_query(sql)
+            results.extend(
+                self.get_results(
+                    query_result,
+                    object_type_to_fetch=TableType.TABLE,
+                    include_table_name_like_filters=include_table_name_like_filters,
+                    exclude_table_name_like_filters=exclude_table_name_like_filters,
+                )
+            )
+        if TableType.VIEW in types_to_return:
+            sql: str = self.build_sql_statement(
+                database_name=database_name, schema_name=schema_name, object_type_to_fetch=TableType.VIEW
+            )
+            query_result: QueryResult = self.data_source_connection.execute_query(sql)
+            results.extend(
+                self.get_results(
+                    query_result,
+                    object_type_to_fetch=TableType.VIEW,
+                    include_table_name_like_filters=include_table_name_like_filters,
+                    exclude_table_name_like_filters=exclude_table_name_like_filters,
+                )
+            )
+        return results
+
+    def build_sql_statement(
+        self,
+        database_name: Optional[str] = None,
+        schema_name: Optional[str] = None,
+        include_table_name_like_filters: Optional[list[str]] = None,
+        exclude_table_name_like_filters: Optional[list[str]] = None,
+        object_type_to_fetch: TableType = TableType.TABLE,
+    ) -> str:  # Return type for this function is a string, not a list (for the super method it is a list!)
+        schema_str = ""
+        if schema_name:
+            schema_str = f" FROM {self.sql_dialect.quote_default(schema_name)}"
+        if object_type_to_fetch == TableType.TABLE:
+            return f"SHOW TABLES{schema_str}"
+        elif object_type_to_fetch == TableType.VIEW:
+            return f"SHOW VIEWS{schema_str}"
+        else:
+            raise ValueError(f"Invalid object type to fetch: {object_type_to_fetch}")
+
+    def get_results(
+        self,
+        query_result: QueryResult,
+        object_type_to_fetch: TableType,
+        include_table_name_like_filters: Optional[list[str]] = None,
+        exclude_table_name_like_filters: Optional[list[str]] = None,
+    ) -> list[FullyQualifiedTableName]:
+        if object_type_to_fetch == TableType.TABLE:
+            names_for_filtering = [table_name for _, table_name, _ in query_result.rows]
+        elif object_type_to_fetch == TableType.VIEW:
+            names_for_filtering = [view_name for _, view_name, *_ in query_result.rows]
+        else:
+            raise ValueError(f"Invalid object type to fetch: {object_type_to_fetch}")
+        filtered_names = self._filter_include_exclude(
+            names_for_filtering, include_table_name_like_filters, exclude_table_name_like_filters
+        )
+
+        if object_type_to_fetch == TableType.TABLE:
+            return [
+                FullyQualifiedTableName(database_name="hive_metastore", schema_name=schema_name, table_name=table_name)
+                for schema_name, table_name, _is_temporary in query_result.rows
+                if table_name in filtered_names
+            ]
+        elif object_type_to_fetch == TableType.VIEW:
+            return [
+                FullyQualifiedViewName(database_name="hive_metastore", schema_name=schema_name, view_name=view_name)
+                for schema_name, view_name, *_ in query_result.rows
+                if view_name in filtered_names
+            ]
+        else:
+            raise ValueError(f"Invalid object type to fetch: {object_type_to_fetch}")
+
+    # Copy from soda-library (v3)
+    @staticmethod
+    def _filter_include_exclude(
+        item_names: list[str], included_items: Optional[list[str]] = None, excluded_items: Optional[list[str]] = None
+    ) -> list[str]:
+        filtered_names = item_names
+        if included_items or excluded_items:
+
+            def matches(name, pattern: str) -> bool:
+                pattern_regex = pattern.replace("%", ".*").lower()
+                is_match = re.fullmatch(pattern_regex, name.lower())
+                return bool(is_match)
+
+            if included_items:
+                filtered_names = [
+                    filtered_name
+                    for filtered_name in filtered_names
+                    if any(matches(filtered_name, included_item) for included_item in included_items)
+                ]
+            if excluded_items:
+                filtered_names = [
+                    filtered_name
+                    for filtered_name in filtered_names
+                    if all(not matches(filtered_name, excluded_item) for excluded_item in excluded_items)
+                ]
+        return filtered_names

soda_databricks-4.0.5/src/soda_databricks/model/data_source/databricks_connection_properties.py

@@ -0,0 +1,41 @@
+from abc import ABC
+from typing import ClassVar, Dict, Optional
+
+from pydantic import Field, SecretStr
+from soda_core.model.data_source.data_source_connection_properties import (
+    DataSourceConnectionProperties,
+)
+
+
+class DatabricksConnectionProperties(DataSourceConnectionProperties, ABC):
+    ...
+
+
+class DatabricksSharedConnectionProperties(DatabricksConnectionProperties, ABC):
+    host: str = Field(
+        ...,
+        description="Databricks workspace hostname (e.g. 'abc.cloud.databricks.com'). If it starts with https:// or http://, it will be removed.",
+    )
+    http_path: str = Field(..., description="HTTP path for the SQL endpoint or cluster")
+    catalog: str = Field(None, description="Default catalog to use")
+    session_configuration: Optional[Dict[str, str]] = Field(None, description="Optional session configuration dict")
+
+    field_mapping: ClassVar[Dict[str, str]] = {
+        "host": "server_hostname",
+    }
+
+    def to_connection_kwargs(self) -> dict:
+        connection_kwargs = super().to_connection_kwargs()
+        server_hostname: str = connection_kwargs["server_hostname"]
+        # Check if the server_hostname starts with https:// or http:// and remove it
+        prefixes = ["https://", "http://"]
+        for prefix in prefixes:
+            if server_hostname.startswith(prefix):
+                server_hostname = server_hostname[len(prefix) :]
+                break  # Stop looking for prefixes once we find one
+        connection_kwargs["server_hostname"] = server_hostname
+        return connection_kwargs
+
+
+class DatabricksTokenAuth(DatabricksSharedConnectionProperties):
+    access_token: SecretStr = Field(..., description="Personal access token")

soda_databricks-4.0.5/src/soda_databricks/model/data_source/databricks_data_source.py

@@ -0,0 +1,23 @@
+import abc
+from typing import Literal
+
+from pydantic import Field, field_validator
+from soda_core.model.data_source.data_source import DataSourceBase
+from soda_databricks.model.data_source.databricks_connection_properties import (
+    DatabricksConnectionProperties,
+    DatabricksTokenAuth,
+)
+
+
+class DatabricksDataSource(DataSourceBase, abc.ABC):
+    type: Literal["databricks"] = Field("databricks")
+    connection_properties: DatabricksConnectionProperties = Field(
+        ..., alias="connection", description="Databricks connection configuration"
+    )
+
+    @field_validator("connection_properties", mode="before")
+    @classmethod
+    def infer_connection_type(cls, value):
+        if "access_token" in value:
+            return DatabricksTokenAuth(**value)
+        raise ValueError("Could not infer Databricks connection type from input")

soda_databricks-4.0.5/src/soda_databricks/test_helpers/databricks_data_source_test_helper.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import os
+
+from helpers.data_source_test_helper import DataSourceTestHelper
+
+
+class DatabricksDataSourceTestHelper(DataSourceTestHelper):
+    def _create_database_name(self) -> str | None:
+        return os.getenv("DATABRICKS_CATALOG", "unity_catalog")
+
+    def _create_data_source_yaml_str(self) -> str:
+        """
+        Called in _create_data_source_impl to initialized self.data_source_impl
+        self.database_name and self.schema_name are available if appropriate for the data source type
+        """
+        return f"""
+            type: databricks
+            name: {self.name}
+            connection:
+                host: {os.getenv("DATABRICKS_HOST")}
+                http_path: {os.getenv("DATABRICKS_HTTP_PATH")}
+                access_token: {os.getenv("DATABRICKS_TOKEN")}
+                catalog: {os.getenv("DATABRICKS_CATALOG", "unity_catalog")}
+        """

soda_databricks-4.0.5/src/soda_databricks.egg-info/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: soda-databricks
+Version: 4.0.5
+Requires-Dist: soda-core==4.0.5
+Requires-Dist: databricks-sql-connector
+Dynamic: requires-dist

soda_databricks-4.0.5/src/soda_databricks.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+setup.py
+src/soda_databricks.egg-info/PKG-INFO
+src/soda_databricks.egg-info/SOURCES.txt
+src/soda_databricks.egg-info/dependency_links.txt
+src/soda_databricks.egg-info/entry_points.txt
+src/soda_databricks.egg-info/requires.txt
+src/soda_databricks.egg-info/top_level.txt
+src/soda_databricks/common/data_sources/databricks_data_source.py
+src/soda_databricks/common/data_sources/databricks_data_source_connection.py
+src/soda_databricks/common/statements/hive_metadata_tables_query.py
+src/soda_databricks/model/data_source/databricks_connection_properties.py
+src/soda_databricks/model/data_source/databricks_data_source.py
+src/soda_databricks/test_helpers/databricks_data_source_test_helper.py

soda_databricks-4.0.5/src/soda_databricks.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

soda_databricks-4.0.5/src/soda_databricks.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[soda.plugins.data_source.databricks]
+DatabricksDataSourceImpl = soda_databricks.common.data_sources.databricks_data_source:DatabricksDataSourceImpl

soda_databricks-4.0.5/src/soda_databricks.egg-info/requires.txt

@@ -0,0 +1,2 @@
+soda-core==4.0.5
+databricks-sql-connector

soda_databricks-4.0.5/src/soda_databricks.egg-info/top_level.txt

@@ -0,0 +1 @@
+soda_databricks