castor-extractor 0.22.0__py3-none-any.whl → 0.22.5__py3-none-any.whl

castor_extractor/warehouse/databricks/lineage_test.py

@@ -1,34 +1,89 @@
-from .lineage import LineageLinks
-from .test_constants import (
-    CLOSER_DATE,
-    OLDER_DATE,
-)
+from .enums import LineageEntity
+from .lineage import LineageProcessor, valid_lineage
+
+_OLDER_DATE = "2025-01-01 00:00:01.0"
+_CLOSER_DATE = "2025-01-01 02:02:02.0"
+
+_TABLE_LINEAGES = [
+    {
+        "source_table_full_name": "a.b.source",
+        "target_table_full_name": "a.b.target",
+        "event_time": _CLOSER_DATE,
+        "other": "more recent stuff",
+    },
+    {
+        "source_table_full_name": "a.b.source",
+        "target_table_full_name": "a.b.target",
+        "event_time": _OLDER_DATE,
+        "other": "stuff that's too old",
+    },
+    {
+        "source_table_full_name": "no target",
+        "target_table_full_name": None,
+        "event_time": _CLOSER_DATE,
+    },
+    {
+        "source_table_full_name": None,
+        "target_table_full_name": "no source",
+        "event_time": _CLOSER_DATE,
+    },
+]
+
+
+_COLUMN_LINEAGES = [
+    {
+        "source_table_full_name": "a.b.source",
+        "source_column_name": "src_col",
+        "target_table_full_name": "a.b.target",
+        "target_column_name": "trgt_col",
+        "event_time": _OLDER_DATE,
+        "other": "old stuff",
+    },
+    {
+        "source_table_full_name": "a.b.source",
+        "source_column_name": "src_col",
+        "target_table_full_name": "a.b.target",
+        "target_column_name": "trgt_col",
+        "event_time": _CLOSER_DATE,
+        "other": "newer stuff",
+    },
+    {
+        "source_table_full_name": "a.b.toto",
+        "source_column_name": "toto_col",
+        "target_table_full_name": "a.b.tata",
+        "target_column_name": "tata_col",
+        "event_time": _OLDER_DATE,
+    },
+    {
+        "source_table_full_name": "a.b.source",
+        "source_column_name": "a.b.source",
+        "target_table_full_name": None,
+        "target_column_name": None,
+        "event_time": _CLOSER_DATE,
+    },
+]
+
+
+def test_valid_lineage():
+    table_links = valid_lineage(_TABLE_LINEAGES, LineageEntity.TABLE)
+
+    assert len(table_links) == 1
+    assert table_links[0]["source_table_full_name"] == "a.b.source"
+    assert table_links[0]["target_table_full_name"] == "a.b.target"
+    assert table_links[0]["event_time"] == _CLOSER_DATE
+    assert table_links[0]["other"] == "more recent stuff"
 
 
 def test_LineageLinks_add():
-    links = LineageLinks()
-    timestamped_link = ("parent", "child", None)
-    expected_key = ("parent", "child")
-
-    links.add(timestamped_link)
-
-    assert expected_key in links.lineage
-    assert links.lineage[expected_key] is None
-
-    # we replace None by an actual timestamp
-    timestamped_link = ("parent", "child", OLDER_DATE)
-    links.add(timestamped_link)
-    assert expected_key in links.lineage
-    assert links.lineage[expected_key] == OLDER_DATE
-
-    # we update with the more recent timestamp
-    timestamped_link = ("parent", "child", CLOSER_DATE)
-    links.add(timestamped_link)
-    assert expected_key in links.lineage
-    assert links.lineage[expected_key] == CLOSER_DATE
-
-    # we keep the more recent timestamp
-    timestamped_link = ("parent", "child", OLDER_DATE)
-    links.add(timestamped_link)
-    assert expected_key in links.lineage
-    assert links.lineage[expected_key] == CLOSER_DATE
+    deduplicated_lineage = LineageProcessor(LineageEntity.COLUMN)
+    for link in _COLUMN_LINEAGES:
+        deduplicated_lineage.add(link)
+
+    lineage = deduplicated_lineage.lineage
+    assert len(lineage) == 2
+    assert ("a.b.source.src_col", "a.b.target.trgt_col") in lineage
+    assert ("a.b.toto.toto_col", "a.b.tata.tata_col") in lineage
+    assert (
+        lineage[("a.b.source.src_col", "a.b.target.trgt_col")]["other"]
+        == "newer stuff"
+    )

castor_extractor/warehouse/databricks/sql_client.py

@@ -1,24 +1,24 @@
 import logging
 from collections import defaultdict
-from enum import Enum
+from datetime import date
 from typing import Optional
 
 from databricks import sql  # type: ignore
 
 from .credentials import DatabricksCredentials
+from .enums import LineageEntity, TagEntity
 from .format import TagMapping
+from .lineage import valid_lineage
 from .utils import build_path, tag_label
 
 logger = logging.getLogger(__name__)
 
 _INFORMATION_SCHEMA_SQL = "SELECT * FROM system.information_schema"
 
-
-class TagEntity(Enum):
-    """Entities that can be tagged in Databricks"""
-
-    COLUMN = "COLUMN"
-    TABLE = "TABLE"
+_LINEAGE_SQL_TPL = """
+SELECT * FROM system.access.{table_name}
+WHERE event_date = :day
+"""
 
 
 class DatabricksSQLClient:
@@ -71,7 +71,6 @@ class DatabricksSQLClient:
         https://docs.databricks.com/en/sql/language-manual/information-schema/column_tags.html
         """
         if not self._needs_extraction(entity):
-            # extracting tags require additional credentials (http_path)
             return dict()
 
         table = f"{entity.value.lower()}_tags"
@@ -88,3 +87,19 @@ class DatabricksSQLClient:
             mapping[path].append(label)
 
         return mapping
+
+    def get_lineage(
+        self, lineage_entity: LineageEntity, day: date
+    ) -> list[dict]:
+        """
+        Fetch {TABLE|COLUMN} lineage of the given day, via system tables
+        https://docs.databricks.com/en/admin/system-tables/lineage.html
+        """
+        table_name = f"{lineage_entity.value.lower()}_lineage"
+        query = _LINEAGE_SQL_TPL.format(table_name=table_name)
+        params = {"day": day}
+        result = self.execute_sql(query, params)
+        data = []
+        for row in result:
+            data.append(row.asDict())
+        return valid_lineage(data, lineage_entity)

castor_extractor/warehouse/databricks/types.py

@@ -1,8 +1 @@
-from typing import Optional
-
-Link = tuple[str, str]
 TablesColumns = tuple[list[dict], list[dict]]
-Ostr = Optional[str]
-TimestampedLink = tuple[str, str, Ostr]
-
-OTimestampedLink = Optional[TimestampedLink]

castor_extractor/warehouse/salesforce/format.py

@@ -4,7 +4,7 @@ from typing import Any
 from ...utils import group_by
 from .constants import SCHEMA_NAME
 
-_HAS_DUPLICATE_KEY = "#has_duplicate"
+_HAS_DUPLICATE_KEY = "#has_duplicate_label"
 
 
 def _clean(raw: str) -> str:
@@ -70,9 +70,15 @@ def _to_table_payload(sobject: dict) -> dict:
     }
 
 
-def _detect_duplicates(sobjects: list[dict]) -> list[dict]:
+def _remove_duplicates(sobjects: list[dict]) -> list[dict]:
+    """only keep one object per QualifiedApiName"""
+    by_name = group_by("QualifiedApiName", sobjects)
+    return [objects[0] for _, objects in by_name.items()]
+
+
+def _detect_duplicate_labels(sobjects: list[dict]) -> list[dict]:
     """
-    enrich the given data with "has_duplicate" flag:
+    enrich the given data with "has_duplicate_label" flag:
     - True when another asset has the same Label in the list
     - False otherwise
     """
@@ -94,7 +100,8 @@ class SalesforceFormatter:
         """
         formats the raw list of sobjects to tables
         """
-        sobjects = _detect_duplicates(sobjects)
+        sobjects = _remove_duplicates(sobjects)
+        sobjects = _detect_duplicate_labels(sobjects)
         for sobject in sobjects:
             yield _to_table_payload(sobject)
 
@@ -102,6 +109,6 @@ class SalesforceFormatter:
     def columns(sobject_fields: dict[str, list[dict]]) -> Iterator[dict]:
         """formats the raw list of sobject fields to columns"""
         for table_name, fields in sobject_fields.items():
-            fields = _detect_duplicates(fields)
+            fields = _detect_duplicate_labels(fields)
             for index, field in enumerate(fields):
                 yield _to_column_payload(field, index, table_name)

castor_extractor/warehouse/salesforce/format_test.py

@@ -1,9 +1,10 @@
 from .format import (
     _HAS_DUPLICATE_KEY,
     SalesforceFormatter,
-    _detect_duplicates,
+    _detect_duplicate_labels,
     _field_description,
     _name,
+    _remove_duplicates,
 )
 
 
@@ -11,9 +12,10 @@ def _tables_sobjects() -> tuple[dict[str, str], ...]:
     """Returns 4 sobjects with 2 sharing the same label"""
     a = {"Label": "a", "QualifiedApiName": "a_one"}
     b = {"Label": "b", "QualifiedApiName": "b"}
-    c = {"Label": "c", "QualifiedApiName": "c"}
+    c = {"Label": "c", "QualifiedApiName": "c_unique_so_doesnt_matter"}
     a_prime = {"Label": "a", "QualifiedApiName": "a_two"}
-    return a, b, c, a_prime
+    b_exact_duplicate = {"Label": "b", "QualifiedApiName": "b"}
+    return a, b, c, a_prime, b_exact_duplicate
 
 
 def _columns_sobjects() -> dict[str, list[dict]]:
@@ -79,14 +81,14 @@ def test__name():
     assert _name(empty_label_sobject) == "empty_label"
 
 
-def test__detect_duplicates():
+def test__detect_duplicate_labels():
     objects = [
         {"Label": "Foo"},
         {"Label": "Bar"},
         {"Label": "Foo"},
     ]
 
-    objects = _detect_duplicates(objects)
+    objects = _detect_duplicate_labels(objects)
     assert objects == [
         {"Label": "Foo", _HAS_DUPLICATE_KEY: True},
         {"Label": "Bar", _HAS_DUPLICATE_KEY: False},
@@ -94,11 +96,25 @@ def test__detect_duplicates():
     ]
 
 
+def test__remove_duplicates():
+    objects = [
+        {"QualifiedApiName": "Foo"},
+        {"QualifiedApiName": "Bar"},
+        {"QualifiedApiName": "Foo"},
+    ]
+
+    objects = _remove_duplicates(objects)
+    assert len(objects) == 2
+    names = {sobject["QualifiedApiName"] for sobject in objects}
+    assert names == {"Foo", "Bar"}
+
+
 def test_salesforce_formatter_tables():
     sobjects = [*_tables_sobjects()]
-    tables = SalesforceFormatter.tables(sobjects)
+    tables = [t for t in SalesforceFormatter.tables(sobjects)]
     expected_names = {"a (a_one)", "a (a_two)", "b", "c"}
     payload_names = {t["table_name"] for t in tables}
+    assert len(tables) == 4  # we only keep one "b"
     assert payload_names == expected_names
 
 

{castor_extractor-0.22.0.dist-info → castor_extractor-0.22.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.22.0
+Version: 0.22.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -19,6 +19,7 @@ Provides-Extra: bigquery
 Provides-Extra: databricks
 Provides-Extra: dbt
 Provides-Extra: looker
+Provides-Extra: lookerstudio
 Provides-Extra: metabase
 Provides-Extra: mysql
 Provides-Extra: postgres
@@ -31,6 +32,7 @@ Provides-Extra: tableau
 Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
 Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
+Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
@@ -205,6 +207,26 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.22.5 - 2025-01-09
+
+* Databricks: validate and deduplicate lineage links
+
+## 0.22.4 - 2025-01-08
+
+* ThoughtSpot: extract answers
+
+## 0.22.3 - 2024-12-10
+
+* Databricks: extract lineage from system tables
+
+## 0.22.2 - 2024-12-06
+
+* Sigma: multithreading to retrieve lineage
+
+## 0.22.1 - 2024-12-05
+
+* Salesforce: deduplicate tables
+
 ## 0.22.0 - 2024-12-04
 
 * Stop supporting python3.8

{castor_extractor-0.22.0.dist-info → castor_extractor-0.22.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=hLifRdD-7Mm2l8gfTHM6y37ld5FLwhMAsmm8FVVQdks,15000
+CHANGELOG.md,sha256=JzTJEZxIMP9F_aePVfIvqLt0OuG0jYcDygsLyfTAV84,15335
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -76,7 +76,7 @@ castor_extractor/uploader/settings.py,sha256=3MvOX-UFRqrLZoiT7wYn9jUGro7NX4RCafY
 castor_extractor/uploader/upload.py,sha256=PSQfkO_7LSE0WBo9Tm_hlS2ONepKeB0cBFdJXySnues,4310
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=otAaySj5aeem6f0CTd0Te6ioJ6uP2J1p348j-SdIwDI,802
-castor_extractor/utils/__init__.py,sha256=jyYquzC2-R-UYl3VTP49ZDHB0IErGogTPMy3GfScbaA,1524
+castor_extractor/utils/__init__.py,sha256=X7WOOgrpGf7Vh8r-7eNGjuC0rKs0g9GTO3d7hZ18gwo,1550
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
 castor_extractor/utils/client/__init__.py,sha256=h5gm8UNNCCkAqhjYK5f6BY7k0cHFOyAvkmlktqwpir0,392
@@ -135,8 +135,8 @@ castor_extractor/utils/salesforce/pagination.py,sha256=wJq0rKLdacFRggyHwB6Fh3K6i
 castor_extractor/utils/store.py,sha256=hnyrFwCsL48e9QrsBns-n8FospujZrkUy1P2YHAh_C0,2067
 castor_extractor/utils/string.py,sha256=IQqNum7CJwuSvDGPbTAmz46YwtYDYgJKeXY7iixdjI4,2370
 castor_extractor/utils/string_test.py,sha256=u3P2tAPhyfCLvD19rH_JcpHhPuWTHUdg0z_N_-Kxwno,2501
-castor_extractor/utils/time.py,sha256=Mv-wTbh1uONXNEd09nb_B8wB5mP8DjDUfPg0S3xmo9Y,1619
-castor_extractor/utils/time_test.py,sha256=pEwpcHI7wGPnfgwrH1DNHEbPz3HEAryNF5yPL7Dqkp8,448
+castor_extractor/utils/time.py,sha256=jmP1QWg4lv21Jp_Oy71lfJ47hjNOSgHiBOFf964RMPU,1732
+castor_extractor/utils/time_test.py,sha256=pH8DSosNlwDYZXZNNjYDcL0WbmZc_c212LEEn88Oqew,647
 castor_extractor/utils/type.py,sha256=Sd8JlEgbGkBUZnRqCUDtREeBkOMTXtlNMyCph90_J0Q,328
 castor_extractor/utils/validation.py,sha256=kQAFtqt3gfy7YqYQ0u-60vyNYUF_96he5QDVUQnZmDo,1896
 castor_extractor/utils/validation_test.py,sha256=aSetitOCkH_K-Wto9ISOVGso5jGfTUOBLm3AZnvavO8,1181
@@ -168,6 +168,17 @@ castor_extractor/visualization/looker/extract.py,sha256=O_hzRftww3Cw1cgijL-K-8gh
 castor_extractor/visualization/looker/fields.py,sha256=7oC7p-3Wp7XHBP_FT_D1wH3kINFRnc_qGVeH1a4UNZY,623
 castor_extractor/visualization/looker/fields_test.py,sha256=7Cwq8Qky6aTZg8nCHp1gmPJtd9pGNB4QeMIRRWdHo5w,782
 castor_extractor/visualization/looker/multithreading.py,sha256=Muuh3usBLqtv3sfHoyPYJ6jJ7V5ajR6N9ZJ_F-bNc60,2608
+castor_extractor/visualization/looker_studio/__init__.py,sha256=p3mTWz7Yk1_m9vYohxCqwxnuE7SUYbU--TH2ezhf734,142
+castor_extractor/visualization/looker_studio/assets.py,sha256=_ir4L2RTmGDb1WetAm6-EZ6W4tPXxi0kNppNBlmy9QE,135
+castor_extractor/visualization/looker_studio/client/__init__.py,sha256=YkQaVDJa-7KSwdOLjtgKJMRiafbGNKC_46YVx0hYZ1Q,129
+castor_extractor/visualization/looker_studio/client/admin_sdk_client.py,sha256=hYKdU6TlWKkXx07r6HsZ4Wbxhasx8DP_jO6iDCjHjgk,3508
+castor_extractor/visualization/looker_studio/client/client.py,sha256=AYdR46NOdn_ITK_wPAASROW0gJjx-iA0Gi43QeuU5BU,1302
+castor_extractor/visualization/looker_studio/client/credentials.py,sha256=yzTaiJQ5cArTnbybUPF6fZZXbX9XQ0SBq-jVI2ECovA,521
+castor_extractor/visualization/looker_studio/client/endpoints.py,sha256=5eY-ffqNDdlDBOOpiF7LpjyHMrzeClJktidCr1pTDUs,669
+castor_extractor/visualization/looker_studio/client/enums.py,sha256=fHgemTaQpnwee8cw1YQVDsVnH--vTyFwT4Px8aVYYHQ,167
+castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py,sha256=Oqu_bGBEqYRR_aitBFyvfCZnx0kSZf4qGEI16tIRnhw,3482
+castor_extractor/visualization/looker_studio/client/pagination.py,sha256=9HQ3Rkdiz2VB6AvYtZ0F-WouiD0pMmdZyAmkv-3wh08,783
+castor_extractor/visualization/looker_studio/client/scopes.py,sha256=824cqqgZuGq4L-rPNoHJe0ibXsxkRwB0CLG_kqw9Q0g,256
 castor_extractor/visualization/metabase/__init__.py,sha256=3E36cmkMyEgBB6Ot5rWk-N75i0G-7k24QTlc-Iol4pM,193
 castor_extractor/visualization/metabase/assets.py,sha256=nu3FwQBU_hdS2DBvgXAwQlEEi76QiNK2tMKEtMyctaY,2874
 castor_extractor/visualization/metabase/client/__init__.py,sha256=KBvaPMofBRV3m_sZAnKNCrJGr-Z88EbpdzEzWPQ_uBk,99
@@ -241,7 +252,7 @@ castor_extractor/visualization/salesforce_reporting/extract.py,sha256=ScStilebLG
 castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
-castor_extractor/visualization/sigma/client/client.py,sha256=nT61lN2yRpKd6jeqwR0NVOAUVpA5KAQyHkEGTl7n00A,6283
+castor_extractor/visualization/sigma/client/client.py,sha256=d9CpE7vRZAPGzck0jFn37LY_6E_Njz9D1sCnFVGJSWk,8006
 castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUGM_bKBAq28Nl7LWSZ6VRsbxrxtDg,1162
 castor_extractor/visualization/sigma/client/pagination.py,sha256=kNEhNq08tTGbypyMjxs0w4uvDtQc_iaWpOZweaa_FsU,690
@@ -295,13 +306,13 @@ castor_extractor/visualization/tableau_revamp/client/rest_fields.py,sha256=3kvaq
 castor_extractor/visualization/tableau_revamp/constants.py,sha256=lHGB50FgVNO2nXeIhkvQKivD8ZFBIjDrflgD5cTXKJw,104
 castor_extractor/visualization/tableau_revamp/extract.py,sha256=HqnBypuNGx_xKk-68WEOy_ucD15LuRF4t2xXf0XKPE0,1370
 castor_extractor/visualization/thoughtspot/__init__.py,sha256=NhTGUk5Kdt54oCjHYoAt0cLBmVLys5lFYiRANL6wCmI,150
-castor_extractor/visualization/thoughtspot/assets.py,sha256=lPRvXk0PKybgLv1AcDVxg-ssf4XLTs0biRqLrqC2TzU,196
+castor_extractor/visualization/thoughtspot/assets.py,sha256=SAQWPKaD2NTSDg7-GSkcRSSEkKSws0MJfOVcHkdeTSg,276
 castor_extractor/visualization/thoughtspot/client/__init__.py,sha256=svrE2rMxR-OXctjPeAHMEPePlfcra-9KDevTMcHunAA,86
-castor_extractor/visualization/thoughtspot/client/client.py,sha256=RHOaJjvlWcSdASXzvlgMbmsSU9oTIixPhH8g0NgyIbc,3719
+castor_extractor/visualization/thoughtspot/client/client.py,sha256=mtwMCPI1-1tyZb1gSYYr-O2QZMTFQwNgillU6ycsOU4,5552
 castor_extractor/visualization/thoughtspot/client/credentials.py,sha256=fp4YHiZy-dstWiLr5c4kFU9SyPK5rd2nCeh8k5sVRpM,462
 castor_extractor/visualization/thoughtspot/client/endpoints.py,sha256=u3FRkmG6j5OIMEeXWZcgRObP8JeC4EutIJEeitNV44c,330
-castor_extractor/visualization/thoughtspot/client/utils.py,sha256=ua7-10HKpFHYRDBVGLJ5hIEfuUA7ryIH9tl0sBjl0MU,883
-castor_extractor/visualization/thoughtspot/client/utils_test.py,sha256=-5ZaEYpQSrIp1-Sx-ViQOLPlv2LoOajEs2mE5YNi_tU,1887
+castor_extractor/visualization/thoughtspot/client/utils.py,sha256=3LgbIWoG1e39VW8rYaV4ot_0EFipziwf3rFAZKxrlEY,1072
+castor_extractor/visualization/thoughtspot/client/utils_test.py,sha256=2XysRU7a58KA2JgNwU2j4GPrN0rkN7Gvk8kQCJlYXVk,2469
 castor_extractor/visualization/thoughtspot/extract.py,sha256=mcXS0jGFpa50td98AVbbTqxchyI5wDCpB-v1o5iRc3g,1354
 castor_extractor/warehouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/warehouse/abstract/__init__.py,sha256=Fdfa026tgOo64MvzVRLHM_F2G-JmcehrF0mh3dHgb7s,419
@@ -329,21 +340,21 @@ castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTw
 castor_extractor/warehouse/bigquery/query.py,sha256=FEekxlkrfAXzsT8Kj1AIqYd5mURB5MlZIkbFVXVqEhU,4762
 castor_extractor/warehouse/bigquery/types.py,sha256=rfKkKA13Et7TM4I0uVaXkLfuaBXkv51bNTp4AO0QSdw,57
 castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
-castor_extractor/warehouse/databricks/api_client.py,sha256=1E3t8uCi3b8xVXLCodwlH5y8FIGmu9otORvA7ZqcGKE,8283
+castor_extractor/warehouse/databricks/api_client.py,sha256=kLcUGSgrfybZUrpt0tE7qe2OoSSN7IK4myyB7c0czOY,6260
 castor_extractor/warehouse/databricks/api_client_test.py,sha256=YTWC-X7L-XAfK5b39TUgTmR1ifv0QrY5tvLNoSbpmjg,466
-castor_extractor/warehouse/databricks/client.py,sha256=K3RafGL_UerFAGmRKK2Cp2IXzalQYqkneQFvgsYdOZY,4993
-castor_extractor/warehouse/databricks/client_test.py,sha256=UKr_D3M8mhqV1oL2_3y_6pEzAFLVE3FHDNZh4omFLK4,2286
-castor_extractor/warehouse/databricks/credentials.py,sha256=iphbVynVTQXMEbJy4QaT5fer-GpOi7QtbAlg8R7-Lj4,598
+castor_extractor/warehouse/databricks/client.py,sha256=H6vcKfos7op5AKSQF9qduG4afx-GZgBdyGE7waS6__o,3292
+castor_extractor/warehouse/databricks/client_test.py,sha256=hOuSPh45z6m9T1hjuqpOayby_q8bYdJVdq5qiwkiXrg,1370
+castor_extractor/warehouse/databricks/credentials.py,sha256=ExtVcl2NpMXTx1Lg8vHQdzQtSEm2aqpg3D1BJrNAUjI,528
 castor_extractor/warehouse/databricks/endpoints.py,sha256=qPoL9CtPFJdwVuW9rJ37nmeMd-nChOBouEVYb4SlaUE,670
-castor_extractor/warehouse/databricks/extract.py,sha256=G_-78-vrvEyn8rcKXXDXlxjad4Ot-Ko4vnhvEcOzjJQ,7389
+castor_extractor/warehouse/databricks/enums.py,sha256=3T6BbVvbWvfWkD23krsYT1x0kKh1qRzNPl6WpcXe300,274
+castor_extractor/warehouse/databricks/extract.py,sha256=Z4VTEIf0QMiua0QGAlJdQ86kxmGAXekQ304aCKme6IY,7358
 castor_extractor/warehouse/databricks/format.py,sha256=FUBMrFFWSa_lX5PtixJCDR3eRYycqeMw0oKHt7AkA4o,6732
 castor_extractor/warehouse/databricks/format_test.py,sha256=ls0IcOElqp_qecAzNbK0zdca7Pms4seCHimbw8NAoAI,3322
-castor_extractor/warehouse/databricks/lineage.py,sha256=RUCcKz19R0dJVab6JUSUbGx4L5Vyb4sVoTAwLbfgjxo,4700
-castor_extractor/warehouse/databricks/lineage_test.py,sha256=EejO4qKH_kJlJSrIap6GvkUi9E55RFvfiySKazAh0_A,1048
+castor_extractor/warehouse/databricks/lineage.py,sha256=jwiRXrgqBAtzQt5EgErYrN8YRyviEEHmyrSbw8TSPq4,2105
+castor_extractor/warehouse/databricks/lineage_test.py,sha256=PyBn1eAoxLm4Bz5M0F4zmaxFX2mXRTM_uug5OKbQPQs,2684
 castor_extractor/warehouse/databricks/pagination.py,sha256=sM1G0sN1pf1TPpI0Y3Oew378UGEKVkMRc2Mlu9tDjLo,545
-castor_extractor/warehouse/databricks/sql_client.py,sha256=KBP0rmMQBWw3jshDfv_NpFW8HqPxGfcBkS4d9T9aXvE,2977
-castor_extractor/warehouse/databricks/test_constants.py,sha256=Hm96yq_ltVAKv7WYhYz637r4Cuj-1cCdyOuxMEe3J-Q,2246
-castor_extractor/warehouse/databricks/types.py,sha256=-qO5y-uI95B666iDhyNM0TL8WlwYC-3Q4xZuolh3PwE,205
+castor_extractor/warehouse/databricks/sql_client.py,sha256=5isGsRL0MW1lu_E_xTyCvSj_rwaJ2nh-kPlhvTvDy_w,3566
+castor_extractor/warehouse/databricks/types.py,sha256=-TFX4jS6_c3wQLOpJTKpLeGS21YIPjKDjISnzeUPdCc,46
 castor_extractor/warehouse/databricks/utils.py,sha256=5CKn6Me1Tus97H_qDEz_5tkhd4ARmwk2qiC3GndjyCc,1969
 castor_extractor/warehouse/databricks/utils_test.py,sha256=_guTuzRWRTZdDY7ils0X1K8jhI9T877MEtw3x_YDg9I,2415
 castor_extractor/warehouse/mysql/__init__.py,sha256=2KFDogo9GNbApHqw3Vm5t_uNmIRjdp76nmP_WQQMfQY,116
@@ -390,8 +401,8 @@ castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29dee
 castor_extractor/warehouse/salesforce/client.py,sha256=067ZyccmIYoY6VwLTSneefOJqUpobtnoEzxJMY2oSPs,3268
 castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
 castor_extractor/warehouse/salesforce/extract.py,sha256=BUQ1ZxGGSq9wWCJfRbKIzIBBeth_YXg8YSV72lbz2lc,3417
-castor_extractor/warehouse/salesforce/format.py,sha256=TUQrxkVEbgs9GDXDI4gsR8LqRmwmVs9Xs-Q5R36ibuQ,3385
-castor_extractor/warehouse/salesforce/format_test.py,sha256=z1Jwo5W74YfBHw_e_DFAJTTCN2ltbdPI294dj4LoGcg,3228
+castor_extractor/warehouse/salesforce/format.py,sha256=M5uGA8aURL_Nt27T8R2tDfbU5ZUM3ECG4fGalEkWkYA,3688
+castor_extractor/warehouse/salesforce/format_test.py,sha256=puTL-Co84jE2SQzKFKGLYU9rey4Ja_Ox8xiKy4iOjeo,3780
 castor_extractor/warehouse/salesforce/pagination.py,sha256=m1S9JRNf6Oe-6dDghYUY5wwTzGzKW5H9pE60PCXMha0,920
 castor_extractor/warehouse/salesforce/soql.py,sha256=XB8ohKwHFfC4Xger7Y84DXLW17IJDye_bZ3FL6DCcOI,1188
 castor_extractor/warehouse/snowflake/__init__.py,sha256=TEGXTyxWp4Tr9gIHb-UFVTRKj6YWmrRtqHruiKSZGiY,174
@@ -425,8 +436,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.22.0.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.22.0.dist-info/METADATA,sha256=qDZUEOjIUsdzWwEQI5MLVA7wTdqgX3LeCcFeP11Zb-4,21885
-castor_extractor-0.22.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.22.0.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
-castor_extractor-0.22.0.dist-info/RECORD,,
+castor_extractor-0.22.5.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.22.5.dist-info/METADATA,sha256=11A9xI9Bd6Uu1Na_AJngfTbkt-ECXjsabWNTppaZsOk,22352
+castor_extractor-0.22.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.22.5.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
+castor_extractor-0.22.5.dist-info/RECORD,,

castor_extractor/warehouse/databricks/test_constants.py

@@ -1,79 +0,0 @@
-OLDER_DATE = "2024-04-18 20:20:20.0"
-CLOSER_DATE = "2024-04-19 20:20:20.0"
-
-MOCK_TABLES_FOR_TABLE_LINEAGE = [
-    {
-        "id": "f51ba2ca-8cc3-4de6-8f8b-730359e8f40f",
-        "schema_id": "dev.silver",
-        "table_name": "analytics",
-    },
-    {
-        "id": "4e140bdc-a67c-4b68-8a07-c684657d8b44",
-        "schema_id": "dev.silver",
-        "table_name": "pre_analytics",
-    },
-    {
-        "id": "7d403198-55ea-4a40-9995-6ee2f4c79dfa",
-        "schema_id": "dev.bronze",
-        "table_name": "analytics",
-    },
-]
-
-_RAW_LINEAGE_DEV_SILVER_ANALYTICS = {
-    "upstreams": [
-        {  # there could be other keys: jobInfos, notebookInfos, queryInfos
-            "tableInfo": {
-                "name": "pre_analytics",
-                "catalog_name": "dev",
-                "schema_name": "silver",
-                "table_type": "PERSISTED_VIEW",  # not used
-                "lineage_timestamp": OLDER_DATE,
-            }
-        },
-        {
-            "tableInfo": {
-                "name": "analytics",
-                "catalog_name": "dev",
-                "schema_name": "bronze",
-                "table_type": "PERSISTED_VIEW",  # not used
-                "lineage_timestamp": CLOSER_DATE,
-            }
-        },
-    ],
-    "downstreams": [],
-}
-_RAW_LINEAGE_DEV_SILVER_PRE_ANALYTICS = {
-    "upstreams": [],
-    "downstreams": [
-        {
-            "tableInfo": {
-                "name": "analytics",
-                "catalog_name": "dev",
-                "schema_name": "silver",
-                "table_type": "PERSISTED_VIEW",  # not used
-                "lineage_timestamp": OLDER_DATE,
-            }
-        },
-    ],
-}
-_RAW_LINEAGE_DEV_BRONZE_ANALYTICS = {
-    "upstreams": [],
-    "downstreams": [
-        {
-            "tableInfo": {
-                "name": "analytics",
-                "catalog_name": "dev",
-                "schema_name": "silver",
-                "table_type": "PERSISTED_VIEW",  # not used
-                "lineage_timestamp": OLDER_DATE,
-            }
-        },
-    ],
-}
-
-# should be in the same order as MOCK_TABLES_FOR_TABLE_LINEAGE
-TABLE_LINEAGE_SIDE_EFFECT: tuple = (
-    _RAW_LINEAGE_DEV_SILVER_ANALYTICS,
-    _RAW_LINEAGE_DEV_SILVER_PRE_ANALYTICS,
-    _RAW_LINEAGE_DEV_BRONZE_ANALYTICS,
-)