castor-extractor 0.17.2__py3-none-any.whl → 0.17.4__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.17.4 - 2024-07-03
+
+* Sigma: Add `input-table`, `pivot-table` and `viz` in the list of supported **Elements**
+
+## 0.17.3 - 2024-06-24
+
+* Databricks: extract tags for tables and column
+
 ## 0.17.2 - 2024-06-14
 
 * Uploader: support multipart

castor_extractor/visualization/looker/constant.py

@@ -2,6 +2,8 @@
 Request timeout in seconds for Looker API
 """
 
+from looker_sdk.error import SDKError  # type: ignore
+
 DEFAULT_LOOKER_TIMEOUT_SECOND = 120
 KEY_LOOKER_TIMEOUT_SECOND = "CASTOR_LOOKER_TIMEOUT_SECOND"
 
@@ -23,3 +25,9 @@ CLIENT_ID = "CASTOR_LOOKER_CLIENT_ID"
 CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET"  # noqa: S105
 SEARCH_PER_FOLDER = "CASTOR_LOOKER_SEARCH_PER_FOLDER"
 LOG_TO_STDOUT = "CASTOR_LOOKER_LOG_TO_STDOUT"
+
+"""
+Safe mode parameters
+"""
+SAFE_MODE_MAX_ERRORS = 3
+SAFE_MODE_EXCEPTIONS = (SDKError,)

castor_extractor/visualization/sigma/client/client.py

@@ -12,7 +12,13 @@ from .pagination import Pagination
 logger = logging.getLogger()
 
 
-DATA_ELEMENTS: Tuple[str, str] = ("table", "visualization")
+DATA_ELEMENTS: Tuple[str, ...] = (
+    "input-table",
+    "pivot-table",
+    "table",
+    "visualization",
+    "viz",
+)
 _CONTENT_TYPE = "application/x-www-form-urlencoded"
 
 

castor_extractor/warehouse/databricks/__init__.py

@@ -1,5 +1,5 @@
 from .client import DatabricksClient
-from .credentials import DatabricksCredentials, to_credentials
+from .credentials import DatabricksCredentials
 from .extract import (
     DATABRICKS_ASSETS,
     DatabricksExtractionProcessor,

castor_extractor/warehouse/databricks/client.py

@@ -1,10 +1,13 @@
 import logging
+from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date
+from enum import Enum
 from functools import partial
 from typing import Any, Dict, List, Optional, Set, Tuple, cast
 
 import requests
+from databricks import sql  # type: ignore
 from requests import Response
 
 from ...utils import (
@@ -19,8 +22,9 @@ from ...utils.client.api import APIClient
 from ...utils.pager import PagerOnToken
 from ..abstract.time_filter import TimeFilter
 from .credentials import DatabricksCredentials
-from .format import DatabricksFormatter
+from .format import DatabricksFormatter, TagMapping
 from .types import Link, Ostr, OTimestampedLink, TablesColumns, TimestampedLink
+from .utils import build_path, tag_label
 
 logger = logging.getLogger(__name__)
 
@@ -33,9 +37,18 @@ _RETRY_EXCEPTIONS = [
 ]
 _WORKSPACE_ID_HEADER = "X-Databricks-Org-Id"
 
+_INFORMATION_SCHEMA_SQL = "SELECT * FROM system.information_schema"
+
 safe_params = SafeMode((BaseException,), _MAX_NUMBER_OF_LINEAGE_ERRORS)
 
 
+class TagEntity(Enum):
+    """Entities that can be tagged in Databricks"""
+
+    COLUMN = "COLUMN"
+    TABLE = "TABLE"
+
+
 def _day_to_epoch_ms(day: date) -> int:
     return int(at_midnight(day).timestamp() * 1000)
 
@@ -76,12 +89,38 @@ class DatabricksClient(APIClient):
         credentials: DatabricksCredentials,
         db_allowed: Optional[Set[str]] = None,
         db_blocked: Optional[Set[str]] = None,
+        has_table_tags: bool = False,
+        has_column_tags: bool = False,
     ):
         super().__init__(host=credentials.host, token=credentials.token)
+        self._http_path = credentials.http_path
         self._db_allowed = db_allowed
         self._db_blocked = db_blocked
+        self._has_table_tags = has_table_tags
+        self._has_column_tags = has_column_tags
         self.formatter = DatabricksFormatter()
 
+    def execute_sql(
+        self,
+        query: str,
+        params: Optional[dict] = None,
+    ):
+        """
+        Execute a SQL query on Databricks system tables and return the results.
+        https://docs.databricks.com/en/dev-tools/python-sql-connector.html
+
+        /!\ credentials.http_path is required in order to run SQL queries
+        """
+        assert self._http_path, "HTTP_PATH is required to run SQL queries"
+        with sql.connect(
+            server_hostname=self._host,
+            http_path=self._http_path,
+            access_token=self._token,
+        ) as connection:
+            with connection.cursor() as cursor:
+                cursor.execute(query, params)
+                return cursor.fetchall()
+
     @staticmethod
     def name() -> str:
         return "Databricks"
@@ -130,7 +169,12 @@ class DatabricksClient(APIClient):
         """
         return response.json(), response.headers[_WORKSPACE_ID_HEADER]
 
-    def _tables_columns_of_schema(self, schema: dict) -> TablesColumns:
+    def _tables_columns_of_schema(
+        self,
+        schema: dict,
+        table_tags: TagMapping,
+        column_tags: TagMapping,
+    ) -> TablesColumns:
         path = "api/2.1/unity-catalog/tables"
         payload = {
             "catalog_name": schema["database_id"],
@@ -143,7 +187,12 @@ class DatabricksClient(APIClient):
         )
         host = self.build_url(self._host, path="")
         return self.formatter.format_table_column(
-            content.get("tables", []), schema, host, workspace_id
+            raw_tables=content.get("tables", []),
+            schema=schema,
+            host=host,
+            workspace_id=workspace_id,
+            table_tags=table_tags,
+            column_tags=column_tags,
         )
 
     @staticmethod
@@ -156,6 +205,40 @@ class DatabricksClient(APIClient):
             return table
         return {**table, "owner_external_id": owner_external_id}
 
+    def _needs_extraction(self, entity: TagEntity) -> bool:
+        if entity == TagEntity.TABLE:
+            return self._has_table_tags
+        if entity == TagEntity.COLUMN:
+            return self._has_column_tags
+        raise AssertionError(f"Entity not supported: {entity}")
+
+    def _get_tags_mapping(self, entity: TagEntity) -> TagMapping:
+        """
+        Fetch tags of the given entity and build a mapping:
+        { path: list[tags] }
+
+        https://docs.databricks.com/en/sql/language-manual/information-schema/table_tags.html
+        https://docs.databricks.com/en/sql/language-manual/information-schema/column_tags.html
+        """
+        if not self._needs_extraction(entity):
+            # extracting tags require additional credentials (http_path)
+            return dict()
+
+        table = f"{entity.value.lower()}_tags"
+        query = f"{_INFORMATION_SCHEMA_SQL}.{table}"
+        result = self.execute_sql(query)
+        mapping = defaultdict(list)
+        for row in result:
+            dict_row = row.asDict()
+            keys = ["catalog_name", "schema_name", "table_name"]
+            if entity == TagEntity.COLUMN:
+                keys.append("column_name")
+            path = build_path(dict_row, keys)
+            label = tag_label(dict_row)
+            mapping[path].append(label)
+
+        return mapping
+
     @staticmethod
     def _get_user_mapping(users: List[dict]) -> dict:
         return {
@@ -172,8 +255,15 @@ class DatabricksClient(APIClient):
         tables: List[dict] = []
         columns: List[dict] = []
         user_mapping = self._get_user_mapping(users)
+        table_tags = self._get_tags_mapping(TagEntity.TABLE)
+        column_tags = self._get_tags_mapping(TagEntity.COLUMN)
         for schema in schemas:
-            t_to_add, c_to_add = self._tables_columns_of_schema(schema)
+
+            t_to_add, c_to_add = self._tables_columns_of_schema(
+                schema=schema,
+                table_tags=table_tags,
+                column_tags=column_tags,
+            )
             t_with_owner = [
                 self._match_table_with_user(table, user_mapping)
                 for table in t_to_add

castor_extractor/warehouse/databricks/credentials.py

@@ -1,11 +1,10 @@
 from dataclasses import field
+from typing import Optional
 
 from pydantic.dataclasses import dataclass
+from pydantic_settings import SettingsConfigDict
 
-from ...utils import from_env
-
-_HOST = "CASTOR_DATABRICKS_HOST"
-_TOKEN = "CASTOR_DATABRICKS_TOKEN"  # noqa: S105
+DATABRICKS_ENV_PREFIX = "CASTOR_DATABRICKS_"
 
 
 @dataclass
@@ -19,10 +18,10 @@ class DatabricksCredentials:
 
     host: str
     token: str = field(metadata={"sensitive": True})
+    http_path: Optional[str] = field(default=None)
 
-
-def to_credentials(params: dict) -> DatabricksCredentials:
-    """extract Databricks credentials"""
-    host = params.get("host") or from_env(_HOST)
-    token = params.get("token") or from_env(_TOKEN)
-    return DatabricksCredentials(host=host, token=token)
+    model_config = SettingsConfigDict(
+        env_prefix=DATABRICKS_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )

castor_extractor/warehouse/databricks/extract.py

@@ -15,7 +15,7 @@ from ..abstract import (
     common_args,
 )
 from .client import DatabricksClient
-from .credentials import to_credentials
+from .credentials import DatabricksCredentials
 
 DATABRICKS_ASSETS: SupportedAssets = {
     WarehouseAssetGroup.ADDITIONAL_LINEAGE: ADDITIONAL_LINEAGE_ASSETS,
@@ -170,7 +170,7 @@ def extract_all(**kwargs) -> None:
     output_directory, skip_existing = common_args(kwargs)
 
     client = DatabricksClient(
-        credentials=to_credentials(kwargs),
+        credentials=DatabricksCredentials(**kwargs),
         db_allowed=kwargs.get("db_allowed"),
         db_blocked=kwargs.get("db_blocked"),
     )

castor_extractor/warehouse/databricks/format.py

@@ -1,8 +1,9 @@
 import logging
 from datetime import datetime
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from .types import TablesColumns
+from .utils import build_path
 
 logger = logging.getLogger(__name__)
 
@@ -11,6 +12,8 @@ EXCLUDED_SCHEMAS = {"information_schema", "default"}
 
 TABLE_URL_TPL = "{host}explore/data/{catalog_name}/{schema_name}/{table_name}?o={workspace_id}"
 
+TagMapping = Dict[str, List[str]]
+
 
 def _to_datetime_or_none(time_ms: Optional[int]) -> Optional[datetime]:
     """return time in ms as datetime or None"""
@@ -24,6 +27,7 @@ def _table_payload(
     table: dict,
     host: str,
     workspace_id: str,
+    tags: TagMapping,
 ) -> dict:
     """
     Prepares the table payload. This also includes a source link which is built
@@ -36,26 +40,43 @@ def _table_payload(
         table_name=table["name"],
         workspace_id=workspace_id,
     )
+
+    keys = ["catalog_name", "schema_name", "name"]
+    path = build_path(table, keys)
+
     return {
         "description": table.get("comment"),
         "id": table["table_id"],
         "owner_email": table.get("owner"),
         "schema_id": f"{schema['id']}",
         "table_name": table["name"],
-        "tags": [],
+        "tags": tags.get(path, []),
         "type": table.get("table_type"),
         "url": url,
     }
 
 
-def _column_payload(table: dict, column: dict) -> dict:
+def _column_path(table: dict, column: dict) -> str:
+    keys = ["catalog_name", "schema_name", "name"]
+    table_path = build_path(table, keys)
+    column_name = column["name"]
+    return f"{table_path}.{column_name}"
+
+
+def _column_payload(
+    table: dict,
+    column: dict,
+    tags: TagMapping,
+) -> dict:
+    path = _column_path(table, column)
     return {
         "column_name": column["name"],
         "data_type": column["type_name"],
         "description": column.get("comment"),
-        "id": f"`{table['id']}`.`{column['name']}`",
+        "id": f"`{table['table_id']}`.`{column['name']}`",
         "ordinal_position": column["position"],
-        "table_id": table["id"],
+        "table_id": table["table_id"],
+        "tags": tags.get(path, []),
     }
 
 
@@ -97,19 +118,24 @@ class DatabricksFormatter:
 
     @staticmethod
     def format_table_column(
-        raw_tables: List[dict], schema: dict, host: str, workspace_id: str
+        raw_tables: List[dict],
+        schema: dict,
+        host: str,
+        workspace_id: str,
+        table_tags: TagMapping,
+        column_tags: TagMapping,
     ) -> TablesColumns:
         tables = []
         columns = []
         if not raw_tables:
             return [], []
         for table in raw_tables:
-            t = _table_payload(schema, table, host, workspace_id)
+            t = _table_payload(schema, table, host, workspace_id, table_tags)
             tables.append(t)
             if not table.get("columns"):
                 continue
             for column in table["columns"]:
-                c = _column_payload(t, column)
+                c = _column_payload(table, column, column_tags)
                 columns.append(c)
 
         return tables, columns

castor_extractor/warehouse/databricks/format_test.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 from .format import (
     DatabricksFormatter,
+    _column_path,
     _column_payload,
     _table_payload,
     _to_datetime_or_none,
@@ -42,7 +43,12 @@ def test__table_payload():
     host = "https://some.cloud.databricks.net/"
     workspace_id = "123456"
 
-    payload = _table_payload(schema, table, host, workspace_id)
+    tags = {
+        "foo.bar.baz": ["riri", "fifi"],
+        "dummy.path": ["loulou"],
+    }
+
+    payload = _table_payload(schema, table, host, workspace_id, tags)
 
     expected = {
         "description": None,
@@ -50,7 +56,7 @@ def test__table_payload():
         "owner_email": "pot@ato.com",
         "schema_id": "id123",
         "table_name": "baz",
-        "tags": [],
+        "tags": ["riri", "fifi"],
         "type": "MANAGED",
         "url": "https://some.cloud.databricks.net/explore/data/foo/bar/baz?o=123456",
     }
@@ -59,40 +65,59 @@ def test__table_payload():
 
 def test__column_payload():
     table = {
-        "id": "18175cd5-9b9b-4d78-9d28-caaa12c21ce0",
-        "schema_id": "dv_microservices.company_silver",
-        "table_name": "companyrepository_organization_v1",
-        "description": "some description",
-        "tags": [],
-        "type": "TABLE",
+        "catalog_name": "foo",
+        "name": "baz",
+        "owner": "pot@ato.com",
+        "schema_name": "bar",
+        "table_id": "732pot5e-8ato-4c27-b701-9fa51febc192",
+        "table_type": "MANAGED",
     }
     column = {
+        "comment": "some description",
         "name": "Uid",
-        "type_text": "string",
-        "type_name": "STRING",
+        "nullable": True,
         "position": 0,
+        "type_json": '{"name":"Uid","type":"string","nullable":true,"metadata":{}}',
+        "type_name": "STRING",
         "type_precision": 0,
         "type_scale": 0,
-        "type_json": '{"name":"Uid","type":"string","nullable":true,"metadata":{}}',
-        "nullable": True,
-        "comment": "some description",
+        "type_text": "string",
     }
-    payload = _column_payload(table, column)
+    tags = {
+        "foo.bar.baz.Uid": ["riri", "fifi"],
+        "dummy.path": ["loulou"],
+    }
+    payload = _column_payload(table, column, tags)
 
     expected = {
-        "id": "`18175cd5-9b9b-4d78-9d28-caaa12c21ce0`.`Uid`",
         "column_name": "Uid",
-        "table_id": "18175cd5-9b9b-4d78-9d28-caaa12c21ce0",
-        "description": "some description",
         "data_type": "STRING",
+        "description": "some description",
+        "id": "`732pot5e-8ato-4c27-b701-9fa51febc192`.`Uid`",
         "ordinal_position": 0,
+        "table_id": "732pot5e-8ato-4c27-b701-9fa51febc192",
+        "tags": ["riri", "fifi"],
     }
     assert payload == expected
 
     # case where there are spaces in the name
     column["name"] = "column name with spaces"
-    payload = _column_payload(table, column)
+    payload = _column_payload(table, column, tags)
     expected_id = (
-        "`18175cd5-9b9b-4d78-9d28-caaa12c21ce0`.`column name with spaces`"
+        "`732pot5e-8ato-4c27-b701-9fa51febc192`.`column name with spaces`"
     )
     assert payload["id"] == expected_id
+
+
+def test__column_path():
+    table = {
+        "catalog_name": "Jo",
+        "schema_name": "William",
+        "name": "Jack",
+    }
+    column = {
+        "name": "Averell",
+    }
+
+    expected = "Jo.William.Jack.Averell"
+    assert _column_path(table=table, column=column) == expected

castor_extractor/warehouse/databricks/utils.py

@@ -0,0 +1,27 @@
+from typing import Dict, List
+
+
+def build_path(
+    row: Dict,
+    keys: List[str],
+) -> str:
+    """
+    format an asset's path:
+    - picks the given keys from dict
+    - join keys with a dot "."
+    """
+    key_values = [row[key] for key in keys]
+    return ".".join(key_values)
+
+
+def tag_label(row: Dict) -> str:
+    """
+    format the tag's label:
+    - {key:value} when the value is not empty
+    - {key} otherwise
+    """
+    tag_name = row["tag_name"]
+    tag_value = row["tag_value"]
+    if not tag_value:
+        return tag_name
+    return f"{tag_name}:{tag_value}"

castor_extractor/warehouse/databricks/utils_test.py

@@ -0,0 +1,25 @@
+from .utils import build_path, tag_label
+
+
+def test_build_path():
+    row = {
+        "bigflo": "oli",
+        "laurel": "hardy",
+        "dupond": "dupont",
+    }
+    keys = ["laurel", "dupond"]
+    assert build_path(row, keys) == "hardy.dupont"
+
+
+def test_tag_label():
+    row = {
+        "tag_name": "marketplace",
+        "tag_value": "",
+    }
+    assert tag_label(row) == "marketplace"
+
+    row = {
+        "tag_name": "fi",
+        "tag_value": "fou",
+    }
+    assert tag_label(row) == "fi:fou"

{castor_extractor-0.17.2.dist-info → castor_extractor-0.17.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.17.2
+Version: 0.17.4
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: all
 Provides-Extra: bigquery
+Provides-Extra: databricks
 Provides-Extra: dbt
 Provides-Extra: looker
 Provides-Extra: metabase
@@ -29,6 +30,7 @@ Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
 Requires-Dist: cryptography (>=41.0.5) ; extra == "snowflake"
+Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
@@ -39,6 +41,9 @@ Requires-Dist: looker-sdk (>=23.0.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: numpy (<1.25) ; python_version >= "3.8" and python_version < "3.9"
 Requires-Dist: numpy (>=1.26,<2) ; python_version >= "3.12" and python_version < "3.13"
+Requires-Dist: pandas (>=2,<2.2.0) ; python_version >= "3.9" and python_full_version <= "3.11.0"
+Requires-Dist: pandas (>=2.0,<2.1) ; python_version >= "3.8" and python_version < "3.9"
+Requires-Dist: pandas (>=2.1,<2.2.0) ; python_version >= "3.12" and python_version < "3.13"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
 Requires-Dist: pydantic (>=2.6,<3.0)

{castor_extractor-0.17.2.dist-info → castor_extractor-0.17.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=qR-os6GsyPBBkE3MwBCB4nYlMULY-D0vqKLXDcamyMU,11078
+CHANGELOG.md,sha256=zKmrqoaMd1E5ht626viLJ8zXTZdkalHhJyFSOhvx23Y,11267
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=uF6PXm9ocPITlKVSh9afTakHmpLx3TvawLf-CbMP3wM,3578
@@ -120,7 +120,7 @@ castor_extractor/visualization/looker/api/sdk.py,sha256=RiCb-3BAL59iGqLfPEO9490c
 castor_extractor/visualization/looker/api/sdk_test.py,sha256=NHtKZTflPhqzBFHs1TyAQaubgxfzLLwYKFT8rEqR55I,1742
 castor_extractor/visualization/looker/api/utils.py,sha256=TJqq9UBVFtS33VB1zHzT6kU8f9MOyqXsUcr57ZurJy4,588
 castor_extractor/visualization/looker/assets.py,sha256=K08nV6MMIpfF9r91TmCO7_62smHzGRv3gR4aIOootMQ,827
-castor_extractor/visualization/looker/constant.py,sha256=JvmAfbyAg-LQIoI48Slg9k5T0mYggs_76D_6vD-mP88,695
+castor_extractor/visualization/looker/constant.py,sha256=C3IQWs6pMHSumDVSJH-XcZe1I9z0d1zTEJljfwZllQc,840
 castor_extractor/visualization/looker/env.py,sha256=vPqirdeGKm3as2T-tBTjbpulQe8W7-3UE2j-Z57wFXk,1174
 castor_extractor/visualization/looker/extract.py,sha256=Y7ZsiUO3uaAkBX0inhjMcEXX4nyBTo9ewthtLfbQhzU,5132
 castor_extractor/visualization/looker/fields.py,sha256=WmiSehmczWTufCLg4r2Ozq2grUpzxDNvIAHyGuOoGs4,636
@@ -199,7 +199,7 @@ castor_extractor/visualization/salesforce_reporting/extract.py,sha256=5QwZwP27uX
 castor_extractor/visualization/sigma/__init__.py,sha256=m98AEysUsVHQAWT6m5nvrtLMs22SDQH9G78-IcUwBoY,130
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=sFqsbcwilIxu75njtSbnAIsNlPdRgB39SAInNUf-nbQ,90
-castor_extractor/visualization/sigma/client/client.py,sha256=894tBAtKlyXHqRB4-mpyM4BgaE80T58VHbqauAmime8,7112
+castor_extractor/visualization/sigma/client/client.py,sha256=x3f2LMErj_muZ8cLFuVdJbjFyOFt8zVpE-zxT7Ce72Y,7172
 castor_extractor/visualization/sigma/client/client_test.py,sha256=BdxJsmTyhslup4vePbkiaoTTaJ9RDY7UfK4OhBpu1Z4,1592
 castor_extractor/visualization/sigma/client/credentials.py,sha256=k3mtaXrBHQdBRk5J2rzFU0YUuO89uX-FjYU1WLqfmjU,797
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=7yNDL0p9wZDE7CcVOj9Z1eP2tdJE1CUT9vbfSEV2kOY,1157
@@ -276,15 +276,17 @@ castor_extractor/warehouse/bigquery/queries/user.sql,sha256=l-fkNGWJVdZwVhbFZL23
 castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTwcgNSBDD0ZXPgRjlxJjrZDSEH2MU,326
 castor_extractor/warehouse/bigquery/query.py,sha256=hrFfjd5jW2oQnZ6ozlkn-gDe6sCIzu5zSX19T9W6fIk,4162
 castor_extractor/warehouse/bigquery/types.py,sha256=LZVWSmE57lOemNbB5hBRyYmDk9bFAU4nbRaJWALl6N8,140
-castor_extractor/warehouse/databricks/__init__.py,sha256=bTvDxjGQGM2J3hOnVhfNmFP1y8DK0tySiD_EXe5_xWE,200
-castor_extractor/warehouse/databricks/client.py,sha256=sNY-7FDg9nLaqw2zk_aoGAhSGi8KST9QpHAHn46700w,17439
+castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
+castor_extractor/warehouse/databricks/client.py,sha256=pnYb6nl9U33nH6UukYP5piWGYF-m1SP2TYiWiUozM_4,20552
 castor_extractor/warehouse/databricks/client_test.py,sha256=KNp4Hi_CC6GwiW2QDJQQwqALfUebuT9D_qL6FuP_8tY,5246
-castor_extractor/warehouse/databricks/credentials.py,sha256=PpGv5_GP320UQjV_gvaxSpOw58AmqSznmjGhGfe6bdU,655
-castor_extractor/warehouse/databricks/extract.py,sha256=VX-3uo5dZucenrg-wnPur3CxOgpC5H7Ds92TO7OTAjc,7379
-castor_extractor/warehouse/databricks/format.py,sha256=zSO3Cm-vpidzNA07W81I506u-ToQzkjXVwKDmS-tfiE,6088
-castor_extractor/warehouse/databricks/format_test.py,sha256=HZvJjcB7sj7LF1kIxAeaf_KdD3XOKn9nfeQLRT39G3s,2804
+castor_extractor/warehouse/databricks/credentials.py,sha256=iphbVynVTQXMEbJy4QaT5fer-GpOi7QtbAlg8R7-Lj4,598
+castor_extractor/warehouse/databricks/extract.py,sha256=VYygE06f7ngYWVlRa48O6drLIZF-_4IBJdyXTYfxZQU,7395
+castor_extractor/warehouse/databricks/format.py,sha256=p252NFzQN1uZdsu5wpP-bMHK0rBBVzallX3-o92Mvh4,6744
+castor_extractor/warehouse/databricks/format_test.py,sha256=ls0IcOElqp_qecAzNbK0zdca7Pms4seCHimbw8NAoAI,3322
 castor_extractor/warehouse/databricks/test_constants.py,sha256=Hm96yq_ltVAKv7WYhYz637r4Cuj-1cCdyOuxMEe3J-Q,2246
 castor_extractor/warehouse/databricks/types.py,sha256=hD6gC8oiT3QSWEvbtgUOGK_lLzzz36sEauB3lS_wxlE,218
+castor_extractor/warehouse/databricks/utils.py,sha256=RWRViqLaj2K0in5T5F6OLp7HCm554BCh3zi4CJqOEt8,576
+castor_extractor/warehouse/databricks/utils_test.py,sha256=5Qrd_tLNLWrDHX2uQyVUf0vqXJzD44uQGGxDBOkwvUU,503
 castor_extractor/warehouse/mysql/__init__.py,sha256=2KFDogo9GNbApHqw3Vm5t_uNmIRjdp76nmP_WQQMfQY,116
 castor_extractor/warehouse/mysql/client.py,sha256=IwoJvbmE5VZkMCP9yHf6ta3_AQPEuBPrZZ3meefbcJs,974
 castor_extractor/warehouse/mysql/client_test.py,sha256=wRTv-3c5chy_HKj-buasNiYOOCIfynYqbabM4Hxdh5E,1052
@@ -370,8 +372,8 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.17.2.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.17.2.dist-info/METADATA,sha256=V-1vK-HPqqZxUKCRncvmp__Yg4UA3B1Aza7Ac4uVFrA,6582
-castor_extractor-0.17.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.17.2.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
-castor_extractor-0.17.2.dist-info/RECORD,,
+castor_extractor-0.17.4.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.17.4.dist-info/METADATA,sha256=sEitPBWyBHFOf2L7awhHWNo-BFGB-F5SDKXNwyVlMok,6985
+castor_extractor-0.17.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.17.4.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
+castor_extractor-0.17.4.dist-info/RECORD,,