castor-extractor 0.19.7__py3-none-any.whl → 0.20.0__py3-none-any.whl

CHANGELOG.md

@@ -1,6 +1,20 @@
 
 # Changelog
 
+## 0.20.0 - 2024-09-23
+
+* Switch to Tableau revamped connector
+
+## 0.19.9 - 2024-09-19
+
+* Databricks: multithreading to retrieve column lineage
+
+## 0.19.8 - 2024-09-18
+
+* Metabase: Handle duplicate dashboards
+* Snowflake: Exclude unnamed tables from extraction
+* Bump dependencies: cryptography, setuptools
+
 ## 0.19.7 - 2024-09-05
 
 * Metabase: Handle compatibility with older version

castor_extractor/commands/extract_tableau.py

@@ -1,7 +1,8 @@
 import logging
 from argparse import ArgumentParser
 
-from castor_extractor.visualization import tableau  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import tableau_revamp  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
@@ -19,29 +20,14 @@ def main():
 
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
+
     parser.add_argument(
-        "-s",
-        "--safe-mode",
-        help="Tableau safe mode",
+        "--with-pulse",
+        dest="with_pulse",
         action="store_true",
+        help="Extract Tableau Pulse assets: Metrics and Subscriptions",
     )
-    parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = tableau.ApiClient(
-        user=args.user,
-        password=args.password,
-        token_name=args.token_name,
-        token=args.token,
-        server_url=args.server_url,
-        site_id=args.site_id,
-        safe_mode=args.safe_mode,
-    )
-
-    client.login()
+    parser.add_argument("-o", "--output", help="Directory to write to")
 
-    tableau.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    tableau_revamp.extract_all(**parse_filled_arguments(parser))

castor_extractor/knowledge/notion/client/client.py

@@ -20,6 +20,8 @@ NOTION_BASE_HEADERS = {
     "User-Agent": CASTOR_NOTION_USER_AGENT,
 }
 
+NOTION_DEFAULT_TIMEOUT_S = 180
+
 
 def _search_filter(asset: str) -> Dict[str, Dict[str, str]]:
     return {"filter": {"value": asset, "property": "object"}}
@@ -45,6 +47,7 @@ class NotionClient(APIClient):
             auth=auth,
             headers=NOTION_BASE_HEADERS,
             safe_mode=safe_mode or NOTION_SAFE_MODE,
+            timeout=NOTION_DEFAULT_TIMEOUT_S,
         )
 
     def users(self) -> Iterator[dict]:

castor_extractor/knowledge/notion/client/pagination.py

@@ -1,14 +1,16 @@
 from typing import Optional
 
+from pydantic import Field
+
 from ....utils import PaginationModel
 
 
 class NotionPagination(PaginationModel):
     """Class to handle paginated results"""
 
-    results: list
-    next_cursor: Optional[str]
-    has_more: bool
+    results: list = Field(default_factory=list)
+    next_cursor: Optional[str] = None
+    has_more: bool = False
 
     def is_last(self) -> bool:
         return not self.has_more

castor_extractor/quality/soda/client/client.py

@@ -1,5 +1,5 @@
 from functools import partial
-from typing import Iterator
+from typing import Callable, Iterator
 
 from ....utils import (
     APIClient,
@@ -11,10 +11,10 @@ from .credentials import SodaCredentials
 from .endpoints import SodaEndpointFactory
 from .pagination import SodaCloudPagination
 
-_CLOUD_API = "https://cloud.soda.io/api/v1/"
 _REQUESTS_PER_MINUTE = 10
 _SECONDS_PER_MINUTE = 60
 _RATE_LIMIT_MS = (_SECONDS_PER_MINUTE // _REQUESTS_PER_MINUTE) + 1
+_CLOUD_PAGE_SIZE = 100
 
 HEADERS = {"Content-Type": "application/json"}
 
@@ -24,16 +24,26 @@ class SodaClient(APIClient):
         cloud_auth = BasicAuth(
             username=credentials.api_key, password=credentials.secret
         )
-        super().__init__(host=_CLOUD_API, auth=cloud_auth, headers=HEADERS)
+        super().__init__(
+            host=credentials.host, auth=cloud_auth, headers=HEADERS
+        )
+
+    @property
+    def _base_request(self) -> Callable:
+        return partial(self._get, params={"size": _CLOUD_PAGE_SIZE})
 
     def datasets(self) -> Iterator[dict]:
-        request = partial(self._get, endpoint=SodaEndpointFactory.datasets())
+        request = partial(
+            self._base_request, endpoint=SodaEndpointFactory.datasets()
+        )
         yield from fetch_all_pages(
             request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
         )
 
     def checks(self) -> Iterator[dict]:
-        request = partial(self._get, endpoint=SodaEndpointFactory.checks())
+        request = partial(
+            self._base_request, endpoint=SodaEndpointFactory.checks()
+        )
         yield from fetch_all_pages(
             request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
         )

castor_extractor/quality/soda/client/credentials.py

@@ -3,6 +3,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 
 SODA_ENV_PREFIX = "CASTOR_SODA_"
 
+_CLOUD_API = "https://cloud.soda.io/api/v1/"
+
 
 class SodaCredentials(BaseSettings):
     """Class to handle Soda rest API permissions"""
@@ -15,3 +17,4 @@ class SodaCredentials(BaseSettings):
 
     api_key: str = Field(repr=False)
     secret: str = Field(repr=False)
+    host: str = Field(default=_CLOUD_API)

castor_extractor/quality/soda/client/pagination.py

@@ -2,7 +2,6 @@ from typing import List
 
 from ....utils import PaginationModel
 
-_CLOUD_PAGE_SIZE = 100
 _CLOUD_FIRST_PAGE = 0
 
 
@@ -19,7 +18,7 @@ class SodaCloudPagination(PaginationModel):
             if self.current_page_payload
             else _CLOUD_FIRST_PAGE
         )
-        return {"page": current_page + 1, "size": _CLOUD_PAGE_SIZE}
+        return {"page": current_page + 1}
 
     def page_results(self) -> list:
         return self.content

castor_extractor/utils/__init__.py

@@ -34,7 +34,7 @@ from .pager import (
     PagerOnIdLogger,
     PagerStopStrategy,
 )
-from .retry import RetryStrategy, retry
+from .retry import RetryStrategy, retry, retry_request
 from .safe import SafeMode, safe_mode
 from .store import AbstractStorage, LocalStorage
 from .string import decode_when_bytes, string_to_tuple

castor_extractor/utils/client/api/client.py

@@ -26,18 +26,18 @@ def _generate_payloads(
     params: Optional[dict],
     data: Optional[dict],
     pagination_params: Optional[dict],
-) -> Tuple[dict, dict]:
+) -> Tuple[Optional[dict], Optional[dict]]:
     _pagination_params = pagination_params or {}
-    params = params or {}
-    data = data or {}
 
     if method == "GET":
+        params = params or {}
         params = {**params, **_pagination_params}
-    elif method == "POST":
+        return data, params
+    if method == "POST":
+        data = data or {}
         data = {**data, **_pagination_params}
-    else:
-        raise ValueError(f"Method {method} is not yet supported")
-    return data, params
+        return data, params
+    raise ValueError(f"Method {method} is not yet supported")
 
 
 class APIClient:

castor_extractor/utils/client/api/pagination.py

@@ -67,7 +67,7 @@ def fetch_all_pages(
     response_payload = request()
     paginated_response = pagination_model(**response_payload)
     while not paginated_response.is_last():
-        logger.info(f"Fetching page number {page_number}")
+        logger.debug(f"Fetching page number {page_number}")
         yield from paginated_response.page_results()
         next_page_parameters = paginated_response.next_page_parameters()
         new_request = partial(request, **next_page_parameters)
@@ -79,5 +79,5 @@ def fetch_all_pages(
         page_number += 1
 
     # send last page's results
-    logger.info(f"Fetching page number {page_number}")
+    logger.debug(f"Fetching page number {page_number}")
     yield from paginated_response.page_results()

castor_extractor/utils/write.py

@@ -35,7 +35,7 @@ def write_json(filename: str, data: Any):
     """
     with open(filename, "w", encoding=ENCODING) as f:
         json.dump(data, f)
-        logger.info(f"Wrote output file: {filename}")
+        logger.info(f"Wrote output file: {filename} ({f.tell()} bytes)")
 
 
 def _current_version() -> str:

castor_extractor/visualization/domo/client/client.py

@@ -45,6 +45,8 @@ _RETRY_EXCEPTIONS = [
 _RETRY_COUNT = 2
 _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 
+_PARENT_FOLDER = "/Dashboards"
+
 logger = logging.getLogger(__name__)
 
 
@@ -177,7 +179,7 @@ class DomoClient:
     def _process_pages(
         self,
         page_tree: List[dict],
-        parent_path: str = "/root",
+        parent_path: str = _PARENT_FOLDER,
     ) -> Iterator[dict]:
         """Recursively fetch pages while building the folder architecture"""
         if not page_tree:

castor_extractor/visualization/metabase/client/api/client.py

@@ -1,6 +1,6 @@
 import logging
 from http import HTTPStatus
-from typing import Any, Dict, Iterator, List, Optional, cast
+from typing import Any, Dict, Iterator, List, Optional, Set, cast
 
 import requests
 from requests import HTTPError
@@ -134,15 +134,22 @@ class ApiClient:
             collection = self._call(f"collection/{_id}/items?models=dashboard")
             if not collection:
                 continue
+
+            seen_dashboard_ids: Set[int] = set()
+
             for dashboard in cast(SerializedAsset, collection):
                 if dashboard.get("model") != "dashboard":
                     # This is to maintain compatibility with older versions
                     # where ?models=dashboard has no effects
                     continue
+
                 dashboard_id = dashboard.get("id")
                 if not dashboard_id:
                     continue
-                yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
+
+                if dashboard_id not in seen_dashboard_ids:
+                    seen_dashboard_ids.add(dashboard_id)
+                    yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
 
     @staticmethod
     def _collection_specifics(collections: SerializedAsset) -> SerializedAsset:

castor_extractor/visualization/tableau_revamp/extract.py

@@ -11,7 +11,7 @@ from ...utils import (
     write_summary,
 )
 from .assets import TableauRevampAsset
-from .client import TableauRevampClient
+from .client import TableauRevampClient, TableauRevampCredentials
 
 logger = logging.getLogger(__name__)
 
@@ -26,16 +26,19 @@ def iterate_all_data(
         yield asset, deep_serialize(data)
 
 
-def extract_all(client: TableauRevampClient, **kwargs: str) -> None:
+def extract_all(**kwargs) -> None:
     """
-    Extract Data from tableau
-    Store data locally in files under the output_directory
-    If errors from Tableau's API are catch store them locally in file under the output_directory
+    Extract Data From tableau and store it locally in files under the
+    output_directory
     """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
-
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    with_pulse = kwargs.get("with_pulse") or False
     timestamp = current_timestamp()
 
+    credentials = TableauRevampCredentials(**kwargs)
+    client = TableauRevampClient(credentials, with_pulse=with_pulse)
+    client.login()
+
     for key, data in iterate_all_data(client):
         filename = get_output_filename(key.value, output_directory, timestamp)
         write_json(filename, data)

castor_extractor/warehouse/databricks/api_client.py

@@ -1,5 +1,6 @@
 import logging
 from functools import partial
+from http import HTTPStatus
 from typing import Iterator, List, Optional, Set, Tuple
 
 import requests
@@ -12,6 +13,7 @@ from ...utils import (
     fetch_all_pages,
     handle_response,
     retry,
+    retry_request,
     safe_mode,
 )
 from ..abstract import TimeFilter
@@ -135,6 +137,10 @@ class DatabricksAPIClient(APIClient):
         max_retries=_RETRY_ATTEMPTS,
         base_ms=_RETRY_BASE_MS,
     )
+    @retry_request(
+        status_codes=(HTTPStatus.TOO_MANY_REQUESTS,),
+        max_retries=_RETRY_ATTEMPTS,
+    )
     def get_single_column_lineage(
         self,
         names: Tuple[str, str],

castor_extractor/warehouse/databricks/client.py

@@ -15,7 +15,8 @@ from .types import TablesColumns, TimestampedLink
 
 logger = logging.getLogger(__name__)
 
-_MAX_THREADS = 10
+_THREADS_COLUMN_LINEAGE = 2
+_THREADS_TABLE_LINEAGE = 10
 
 
 class DatabricksClient:
@@ -99,7 +100,7 @@ class DatabricksClient:
         Wrapper function that retrieves all table lineage
         """
         # retrieve table lineage
-        with ThreadPoolExecutor(max_workers=_MAX_THREADS) as executor:
+        with ThreadPoolExecutor(max_workers=_THREADS_TABLE_LINEAGE) as executor:
             table_paths = [
                 ".".join([table["schema_id"], table["table_name"]])
                 for table in tables
@@ -121,10 +122,15 @@ class DatabricksClient:
         candidate_paths = paths_for_column_lineage(
             tables, columns, table_lineage
         )
+        # retrieve column lineage
+        with ThreadPoolExecutor(
+            max_workers=_THREADS_COLUMN_LINEAGE
+        ) as executor:
+            results = executor.map(
+                self.api_client.get_single_column_lineage, candidate_paths
+            )
         lineages: List[TimestampedLink] = [
-            link
-            for paths in candidate_paths
-            for link in self.api_client.get_single_column_lineage(paths)
+            link for links in results for link in links
         ]
         deduplicated = deduplicate_lineage(lineages)
         return self.formatter.format_lineage(deduplicated)

castor_extractor/warehouse/snowflake/queries/table.sql

@@ -41,6 +41,8 @@ FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id
 WHERE TRUE
+    AND t.table_name IS NOT NULL
+    AND t.table_name != ''
     AND UPPER(t.table_catalog) NOT IN ('SNOWFLAKE', 'UTIL_DB')
     AND (
         t.deleted IS NULL

castor_extractor/warehouse/sqlserver/client.py

@@ -12,8 +12,8 @@ _KEYS = ("user", "password", "host", "port", "database")
 
 
 def _check_key(credentials: dict) -> None:
-    for key in credentials:
-        if key not in _KEYS:
+    for key in _KEYS:
+        if key not in credentials:
             raise KeyError(f"Missing {key} in credentials")
 
 

{castor_extractor-0.19.7.dist-info → castor_extractor-0.20.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.19.7
+Version: 0.20.0
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -29,7 +29,7 @@ Provides-Extra: redshift
 Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
-Requires-Dist: cryptography (>=42.0.0,<43.0.0) ; extra == "snowflake"
+Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
 Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-auth (>=2,<3)
@@ -52,7 +52,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=72,<73)
+Requires-Dist: setuptools (>=74,<75)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -208,6 +208,20 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.20.0 - 2024-09-23
+
+* Switch to Tableau revamped connector
+
+## 0.19.9 - 2024-09-19
+
+* Databricks: multithreading to retrieve column lineage
+
+## 0.19.8 - 2024-09-18
+
+* Metabase: Handle duplicate dashboards
+* Snowflake: Exclude unnamed tables from extraction
+* Bump dependencies: cryptography, setuptools
+
 ## 0.19.7 - 2024-09-05
 
 * Metabase: Handle compatibility with older version

{castor_extractor-0.19.7.dist-info → castor_extractor-0.20.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=cJ5aAA_dulThT10jAw6hHkBGiDyrZFqN7S_78E3q1Qk,13293
+CHANGELOG.md,sha256=_iTzy8VrdNYmYKWXfGfPMIlSenr4M7LCvoE1K0H96co,13601
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -23,7 +23,7 @@ castor_extractor/commands/extract_salesforce_reporting.py,sha256=FdANTNiLkIPdm80
 castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_9bYjUMg7BOW-k,643
 castor_extractor/commands/extract_snowflake.py,sha256=vYiruxRoo--GeMemOGsSE1w9kcKTh_y4E165HtMVzkM,1982
 castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7WJw-sDU6GHl3lzBt_0,1141
-castor_extractor/commands/extract_tableau.py,sha256=u-6UCd-kfXwyhNWYxZusqtgTTYkf4gAJS1vRIYWsAVU,1415
+castor_extractor/commands/extract_tableau.py,sha256=VUb_1Y85EzfF1f9OaCQQt8kFYBdp0u31Mw1Wm2fkxWs,1221
 castor_extractor/commands/file_check.py,sha256=VSD84kpQKf7b0wJOhUgkJQ9n4mK3v52sjMWL7wkNYa0,2667
 castor_extractor/commands/upload.py,sha256=WLDI3zDmK2CjtbxiMWX2mZGjxx8DozfCw6tLE3CAMcE,1833
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
@@ -41,22 +41,22 @@ castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 castor_extractor/knowledge/notion/__init__.py,sha256=ZDmh0eNSxHf1zVPm0aYlKPci-vzOXhAgdsWjS2hdjh4,117
 castor_extractor/knowledge/notion/assets.py,sha256=QHv1-pomt5UeN_prP2L6t_zJ-tDSqB8LgopkGAODYPQ,164
 castor_extractor/knowledge/notion/client/__init__.py,sha256=CDPorBCethuNTEtpjvHGcWnWeVfqkEq-IbakWjDKATw,76
-castor_extractor/knowledge/notion/client/client.py,sha256=PJSM27wrJUxOwOoET6bScO_O8GOdoljxkVQYPFmcXmg,3607
+castor_extractor/knowledge/notion/client/client.py,sha256=UJ2fHfwjCiNEpkTm2inVkPZkmSgfXtBxUgS0fChULy0,3685
 castor_extractor/knowledge/notion/client/client_test.py,sha256=fo3_WgCIUfeF3nMfHTSB6wjxIW_dHp1XZL41tNzBisQ,1976
 castor_extractor/knowledge/notion/client/constants.py,sha256=dZCpSrxFlLbR_cFPJKz4M5wcMVcY4UfWFG0N5S22Fhw,147
 castor_extractor/knowledge/notion/client/credentials.py,sha256=-mBY6IhAb9z2RP1MilzIyS3BW39J__u_M5n9uIuUlxA,398
 castor_extractor/knowledge/notion/client/endpoints.py,sha256=5wUodq7les28CpZkMm5drhry6FFWbBU3V4oeUAn0G-w,405
-castor_extractor/knowledge/notion/client/pagination.py,sha256=pPJZxFHe9ecwS8xHat8tADQohgqerCfTnhMvED3XpT8,445
+castor_extractor/knowledge/notion/client/pagination.py,sha256=yIHov3eoQsqRMYZxHrkBxJBl2uKHDLLn3A8PEBz8J8U,518
 castor_extractor/knowledge/notion/extract.py,sha256=ExG354_5VJKLUp_7QQvIl6HnZqBmOypAquY3tA9rFXc,1717
 castor_extractor/logger.py,sha256=ovf1mBEKwbJBskBXoqHbcAomBrp58mUwSrCWtEMlYPM,1197
 castor_extractor/quality/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/quality/soda/__init__.py,sha256=y2H6f_UqLrqlssKlTVqPwa4ITvRh7na4P6HywrUoTb8,78
 castor_extractor/quality/soda/assets.py,sha256=1mMBFoVOEelmX4dtY0oxzMzHzfwFpGWq6t9UBykJ-80,140
 castor_extractor/quality/soda/client/__init__.py,sha256=LUvHHyyL2t3mk5GORshnef4FgAu7VMqrIQWQx-7kO4I,48
-castor_extractor/quality/soda/client/client.py,sha256=8JEnLJn4-m_km-JyJaq-9RNpqBxwErVDIEgYpRju5QA,1509
-castor_extractor/quality/soda/client/credentials.py,sha256=yVFIYeEv76LogOwP_JEWieRpRFyOtACj7VLhSjyPH5k,426
+castor_extractor/quality/soda/client/client.py,sha256=UEhtN3fpCrVYUGyaO3GXPMqzk8uGwBLyHaaU0hh4zs8,1712
+castor_extractor/quality/soda/client/credentials.py,sha256=R1g7nHpJlQ5hBjtUFN06QjjWAouQtb_V-je7cAXXIA4,514
 castor_extractor/quality/soda/client/endpoints.py,sha256=x3B-XlnDF8NJMuk-81N72_6HA-YZEzA895khLyj0j54,228
-castor_extractor/quality/soda/client/pagination.py,sha256=KgCNdLLX-G9SSnNuTslw5rh7Llwl_4LDmJEuf2ylm_A,604
+castor_extractor/quality/soda/client/pagination.py,sha256=IrAnju5CFipHarTLZRWZflVZ3KHoRVMvaCaNAePHVug,555
 castor_extractor/types.py,sha256=Hd_shbsGAknJLTrAk3SBxZeFPOlbWBXXjIscC9C7CW8,1281
 castor_extractor/uploader/__init__.py,sha256=SSRtwjg-dNoxME-RJy9G1flASiUKAC5bH1htq3CURQg,75
 castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
@@ -65,7 +65,7 @@ castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsX
 castor_extractor/uploader/upload.py,sha256=c86NP4ZxWnz3Hy1iWDYd9qjJSSjZ1bLq3fxVGBIU4Rc,3238
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=Tx_i875L2vJ8btOLV3-L0UMEFiyhH8E5n0XXRyLjO0Y,793
-castor_extractor/utils/__init__.py,sha256=MReSpH6I4AQZ5WTQp752P2sP4wVVZO8MyyglN0K0VKw,1509
+castor_extractor/utils/__init__.py,sha256=jyYquzC2-R-UYl3VTP49ZDHB0IErGogTPMy3GfScbaA,1524
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
 castor_extractor/utils/client/__init__.py,sha256=h5gm8UNNCCkAqhjYK5f6BY7k0cHFOyAvkmlktqwpir0,392
@@ -73,9 +73,9 @@ castor_extractor/utils/client/abstract.py,sha256=aA5Qcb9TwWDSMq8WpXbGkOB20hehwX2
 castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
 castor_extractor/utils/client/api/auth.py,sha256=QDLM5h1zGibLaKyATxLF0gycg01SE92G-Y69f_YBClc,1896
 castor_extractor/utils/client/api/auth_test.py,sha256=NoZYsz7bcCyWBZdMF1TaOuK-s1j09DhTRyM4GSUW_YQ,1311
-castor_extractor/utils/client/api/client.py,sha256=h8cQZ4CSD0LUuNv1VC5qa_p_jFbiOUbYztJnNmpJa2Q,4231
+castor_extractor/utils/client/api/client.py,sha256=0E5GG5Yxk-J5B11YdeIcccYk7jAfuQoWJIz5ljMGYUE,4275
 castor_extractor/utils/client/api/client_test.py,sha256=FM3ZxsLLfMOBn44cXX6FIgnA31-5TTNIyp9D4LBwtXE,1222
-castor_extractor/utils/client/api/pagination.py,sha256=Ne-4dqKrwEAETheBn14KPrGRzxsrkuCO96Z6gkCI-0Y,2437
+castor_extractor/utils/client/api/pagination.py,sha256=Efg3P9ct_U5rtgXijMGV05oQxSzjldEopECWjIFWerM,2439
 castor_extractor/utils/client/api/pagination_test.py,sha256=jCOgXFXrH-jrCxe2dfk80ZksJF-EtmpJPU11BGabsqk,1385
 castor_extractor/utils/client/api/safe_request.py,sha256=SeBteAK8KhBjXldIdyUpkZphf9ktjzbvBM49AXrvD0g,1686
 castor_extractor/utils/client/api/safe_request_test.py,sha256=LqS5FBxs6lLLcTkcgxIoLb6OinxShHXR5y4CWZpwmwg,2005
@@ -129,12 +129,12 @@ castor_extractor/utils/time_test.py,sha256=pEwpcHI7wGPnfgwrH1DNHEbPz3HEAryNF5yPL
 castor_extractor/utils/type.py,sha256=87t32cTctEjX-_BqZLtPLWu-M9OVvw_lFU4DbaQ6V0U,313
 castor_extractor/utils/validation.py,sha256=NNMkdyvMzConslnyCM3gmciEtPPvefW0vAT2gNsMhvE,1909
 castor_extractor/utils/validation_test.py,sha256=aSetitOCkH_K-Wto9ISOVGso5jGfTUOBLm3AZnvavO8,1181
-castor_extractor/utils/write.py,sha256=CbLMz-mkUFduwogERwe69GXXVO65HEDirVm-kDJkRxg,2135
+castor_extractor/utils/write.py,sha256=_7tNpu2p35E3GZnjsC_GWBbviR3pz3xsL7KAUahs8UE,2154
 castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/visualization/domo/__init__.py,sha256=1axOCPm4RpdIyUt9LQEvlMvbOPllW8rk63h6EjVgJ0Y,111
 castor_extractor/visualization/domo/assets.py,sha256=bK1urFR2tnlWkVkkhR32mAKMoKbESNlop-CNGx-65PY,206
 castor_extractor/visualization/domo/client/__init__.py,sha256=Do0fU4B8Hhlhahcv734gnJl_ryCztfTBDea7XNCKfB8,72
-castor_extractor/visualization/domo/client/client.py,sha256=ZeEwudAjjDixYHiA8chTDYn9p8ac3wCbF-6y6Cvfw1g,9607
+castor_extractor/visualization/domo/client/client.py,sha256=qzOzloGIUHyePsMTB_y7s388a0XU2OTd9WQnt387YfY,9646
 castor_extractor/visualization/domo/client/credentials.py,sha256=fyJ4wOmCqMQ1g3jDoOjtChtUES0sHfp3eLAfb4JxZP4,879
 castor_extractor/visualization/domo/client/endpoints.py,sha256=jl5JvE6HZ36g9kpbm4TU501uxLa0mMZyitJ7JTJP-HI,2781
 castor_extractor/visualization/domo/client/pagination.py,sha256=ukVkHVzoH4mfZ29H9YcnC2YrdVolP10wv25J6Q3ehRw,821
@@ -161,7 +161,7 @@ castor_extractor/visualization/metabase/__init__.py,sha256=3E36cmkMyEgBB6Ot5rWk-
 castor_extractor/visualization/metabase/assets.py,sha256=RrrE7H0ezqD-YpNLjDOo4u5fFPUK1PIRxLXNERFqFn8,2906
 castor_extractor/visualization/metabase/client/__init__.py,sha256=KBvaPMofBRV3m_sZAnKNCrJGr-Z88EbpdzEzWPQ_uBk,99
 castor_extractor/visualization/metabase/client/api/__init__.py,sha256=BYSPWHY4KbT-LvenNI0pRonxolzZ5HSl6v3-PbJr-7M,78
-castor_extractor/visualization/metabase/client/api/client.py,sha256=2b8NUBjVev-DcMG-rshCPGbG4673NRgjgseUy46JI5M,6460
+castor_extractor/visualization/metabase/client/api/client.py,sha256=wKtoRAzp5rqpQ6S7QRz8MLdskjc5-dzBUB6u65zanzY,6638
 castor_extractor/visualization/metabase/client/api/client_test.py,sha256=7Lb5yvrvHQmCIOnFzS2D00oR2Zps1SVxGIAukzRVeKg,559
 castor_extractor/visualization/metabase/client/api/credentials.py,sha256=SiHWyWWEDy_Ak4ne0hgXPZCSP6mEuHH26wDhsed_9v8,519
 castor_extractor/visualization/metabase/client/db/__init__.py,sha256=nawDhJ-JGlpM6VMzZZRjf066QXk9kWzZr6l9n6OHTZ0,76
@@ -281,7 +281,7 @@ castor_extractor/visualization/tableau_revamp/client/errors.py,sha256=dTe1shqmWm
 castor_extractor/visualization/tableau_revamp/client/gql_queries.py,sha256=-V3ToD5Gi7nmfVB2OxTOZw8dcOiF7_ciSWjjW2UdvvI,2270
 castor_extractor/visualization/tableau_revamp/client/rest_fields.py,sha256=gx39X1zMfRVpjmFbgvbgbvtlE0QwxOtk8rZFsIqeGRI,978
 castor_extractor/visualization/tableau_revamp/constants.py,sha256=thS935pJyuZkdciM2EFHbIuTqSFYfB3YGCJYJ_Ls294,55
-castor_extractor/visualization/tableau_revamp/extract.py,sha256=o8ZORM5OXtp51plmDNhk7o0egc0Vf2MOBysjJtRucBI,1289
+castor_extractor/visualization/tableau_revamp/extract.py,sha256=BPy38rFjGG6Nh1eDFeCckE4RHaO-bWW2uhXh7wm8mKk,1368
 castor_extractor/warehouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/warehouse/abstract/__init__.py,sha256=Fdfa026tgOo64MvzVRLHM_F2G-JmcehrF0mh3dHgb7s,419
 castor_extractor/warehouse/abstract/asset.py,sha256=Qs7T2Iw7KHgWVT2aAoBfCQ8tB143cUZY-DRUSkpgvGU,2689
@@ -308,9 +308,9 @@ castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTw
 castor_extractor/warehouse/bigquery/query.py,sha256=5Qc8PEa-kQKpTzguj4RNCAwKyvzWt20vAESYNB0lueo,4768
 castor_extractor/warehouse/bigquery/types.py,sha256=DHK3wUaaLyLMp7LP-7QkXTDYpYTZiPtvptAOkpxgp4g,88
 castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
-castor_extractor/warehouse/databricks/api_client.py,sha256=lfS-DMLlTEsEMAEs7TeH8JmLX74l4Ai0k12nuP2fo2Y,8110
+castor_extractor/warehouse/databricks/api_client.py,sha256=z9sIgZ4S3wZzUcVa_mlAW8s70sY8o7v19Kj_r6M0skM,8274
 castor_extractor/warehouse/databricks/api_client_test.py,sha256=YTWC-X7L-XAfK5b39TUgTmR1ifv0QrY5tvLNoSbpmjg,466
-castor_extractor/warehouse/databricks/client.py,sha256=neH-KnyWtxEMApbjIV2uhY7Me3tCpavd1NMnljrT05k,4768
+castor_extractor/warehouse/databricks/client.py,sha256=7MtOvxhxRQOEGi9lmYWbi2W9baNQEDnuF3QTrycdzUw,5004
 castor_extractor/warehouse/databricks/client_test.py,sha256=UKr_D3M8mhqV1oL2_3y_6pEzAFLVE3FHDNZh4omFLK4,2286
 castor_extractor/warehouse/databricks/credentials.py,sha256=iphbVynVTQXMEbJy4QaT5fer-GpOi7QtbAlg8R7-Lj4,598
 castor_extractor/warehouse/databricks/endpoints.py,sha256=qPoL9CtPFJdwVuW9rJ37nmeMd-nChOBouEVYb4SlaUE,670
@@ -387,12 +387,12 @@ castor_extractor/warehouse/snowflake/queries/grant_to_user.sql,sha256=7AalVajU5v
 castor_extractor/warehouse/snowflake/queries/query.sql,sha256=-OYcWUvdPBkpOfezkZaW7hrOdDz3JyoqjNdRm_88Rsk,1779
 castor_extractor/warehouse/snowflake/queries/role.sql,sha256=D0VvGxLZMwug2SvefhAsNR9YIun0fZvcDWkz891xSYM,96
 castor_extractor/warehouse/snowflake/queries/schema.sql,sha256=HCDEw0Nj_GPHBNH3Ik_5BF4rkD5yBfSyeN9UaiFGrI4,730
-castor_extractor/warehouse/snowflake/queries/table.sql,sha256=sSfJ2sNsn-pTVDiHQVJnwzcfq_MVZvddwpVgKuwP_n4,1378
+castor_extractor/warehouse/snowflake/queries/table.sql,sha256=qTwkAJ7-kM8vX03RP16U_5_euWW5ZTQAKuiLPsbj2hs,1438
 castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsKOHHlqCtBMQHOV54yy6RKJaXk,570
 castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
 castor_extractor/warehouse/snowflake/query.py,sha256=yDpG4e23xtjEfAKNSAgL9wx17ChFSlvAbig2mJ5ZEC0,1769
 castor_extractor/warehouse/sqlserver/__init__.py,sha256=PdOuYznmvKAbfWAm8UdN47MfEsd9jqPi_dDi3WEo1KY,116
-castor_extractor/warehouse/sqlserver/client.py,sha256=l6O0OhAKtRqy0oTl9KDYDawgWKc4XCNOA-bVdnMdEjM,1592
+castor_extractor/warehouse/sqlserver/client.py,sha256=8OCWD9Xv2M1pUAdWdcvChU5eLFIW565zyC0xmbtrZf0,1592
 castor_extractor/warehouse/sqlserver/extract.py,sha256=2mBNx9clyrhoiirD635BW-5u6pPoxHyIsB071XoZjho,2087
 castor_extractor/warehouse/sqlserver/queries/.sqlfluff,sha256=yy0KQdz8I_67vnXyX8eeWwOWkxTXvHyVKSVwhURktd8,48
 castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=Szdf8hwcDffRTgtD6zf4ZuIyHIVijFgSDk1rZbKI3g8,2480
@@ -401,18 +401,9 @@ castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=elM9s02I9d9F5E4MH
 castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx_LGkIf7LX6ojTjI8wgJDxm3f0,2542
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
-castor_extractor/warehouse/synapse/__init__.py,sha256=-VyWS6O9gptZ-FTDnUVDi87r_PUA59dPXPjLG1lwzpY,36
-castor_extractor/warehouse/synapse/extract.py,sha256=Tvo6BOXsw2T5Suf4b-aCwJf4GzssijjbpVhulBjazAs,538
-castor_extractor/warehouse/synapse/queries/.sqlfluff,sha256=7eDQrhRQHKUc5zmmNCNG55acRNPMX4mzxylOLf2pM_g,26
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor/warehouse/synapse/queries/database.sql,sha256=TwDKFsFda3LXTUPjAm79M80ZMZj0yYM9Ay7RwFOLh8I,138
-castor_extractor/warehouse/synapse/queries/query.sql,sha256=LKEA3cEdFpVkw-MduM8p_fO2IzshPGcPXRmn06Zqk0Q,1110
-castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSFJV4BChIb_5zEXsEMnmM26BY8,250
-castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
-castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
-castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.19.7.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.19.7.dist-info/METADATA,sha256=C94_SY5XYZLhVttCVc8CHNOIKny9F9mTiDpq_dBt79A,20511
-castor_extractor-0.19.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.19.7.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
-castor_extractor-0.19.7.dist-info/RECORD,,
+castor_extractor-0.20.0.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.20.0.dist-info/METADATA,sha256=u6XUIwUCR1lLswRR_mMI9sm9zXrqIbE05MGVnAPLOBA,20819
+castor_extractor-0.20.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.20.0.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
+castor_extractor-0.20.0.dist-info/RECORD,,

castor_extractor/warehouse/synapse/__init__.py

@@ -1 +0,0 @@
-from .extract import SYNAPSE_ASSETS

castor_extractor/warehouse/synapse/extract.py

@@ -1,21 +0,0 @@
-import logging
-
-from ..abstract import (
-    CATALOG_ASSETS,
-    EXTERNAL_LINEAGE_ASSETS,
-    QUERIES_ASSETS,
-    VIEWS_ASSETS,
-    SupportedAssets,
-    WarehouseAsset,
-    WarehouseAssetGroup,
-)
-
-logger = logging.getLogger(__name__)
-
-SYNAPSE_ASSETS: SupportedAssets = {
-    WarehouseAssetGroup.CATALOG: CATALOG_ASSETS,
-    WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
-    WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
-    WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
-    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
-}

castor_extractor/warehouse/synapse/queries/.sqlfluff

@@ -1,2 +0,0 @@
-[sqlfluff]
-dialect = tsql

castor_extractor/warehouse/synapse/queries/database.sql

@@ -1,6 +0,0 @@
-SELECT
-    d.name AS database_id,
-    d.name AS database_name,
-    NULL AS "comment"
-FROM sys.databases AS d
-    WHERE d.name != 'master'

castor_extractor/warehouse/synapse/queries/query.sql

@@ -1,33 +0,0 @@
-WITH query_execution AS (
-    SELECT
-        plan_id,
-        CASE execution_type_desc
-            WHEN 'Aborted' THEN 1
-            WHEN 'Exception' THEN 1
-            ELSE 0
-        END AS aborted,
-        MIN(CAST(first_execution_time AS DATETIME)) AS start_time,
-        MAX(CAST(last_execution_time AS DATETIME)) AS end_time
-    FROM sys.query_store_runtime_stats
-    WHERE CAST(first_execution_time AS DATE) = CAST(DATEADD(day, -1, getdate()) AS DATE)
-    -- Filtering on hour will be needed when running that query out of notebooks
-    GROUP BY plan_id, execution_type_desc
-)
-
-SELECT
-    q.query_id,
-    qt.query_sql_text AS query_text,
-    DB_NAME() AS database_id,
-    DB_NAME() AS database_name,
-    'SYNAPSE' AS user_name,
-    qp.plan_id AS process_id,
-    qe.aborted AS aborted,
-    qe.start_time AS start_time,
-    qe.end_time AS end_time
-FROM sys.query_store_query AS q
-    JOIN sys.query_store_query_text AS qt
-        ON q.query_text_id = qt.query_text_id
-    JOIN sys.query_store_plan AS qp
-        ON q.query_id = qp.query_id
-    JOIN query_execution AS qe
-        ON qp.plan_id = qe.plan_id

castor_extractor/warehouse/synapse/queries/schema.sql

@@ -1,7 +0,0 @@
-SELECT DISTINCT
-    s.catalog_name AS database_id,
-    s.catalog_name AS database_name,
-    s.schema_name AS schema_name,
-    s.schema_owner AS schema_owner,
-    s.catalog_name + '.' + s.schema_name AS schema_id
-FROM INFORMATION_SCHEMA.SCHEMATA AS s

castor_extractor/warehouse/synapse/queries/table.sql

@@ -1,36 +0,0 @@
-WITH users AS (
-    SELECT DISTINCT
-        uid,
-        name
-    FROM sys.sysusers
-)
-
-SELECT
-    DB_NAME() as database_id,
-    DB_NAME() AS database_name,
-    DB_NAME() + '.' + s.name AS schema_id,
-    s.name AS schema_name,
-    DB_NAME() + '.' + s.name + '.' + o.name AS table_id,
-    o.name AS table_name,
-    CASE UPPER(REPLACE(o.type, ' ', ''))
-        WHEN 'U' THEN 'TABLE'
-        WHEN 'V' THEN 'VIEW'
-        ELSE 'UNKNOWN'
-    END AS table_type,
-    -- Normally schema owner is table owner, but we can
-    -- change table owner with ALTER AUTHORIZATION command
-    -- in this case we have to look up for the owner
-    -- in sys.objects table
-    COALESCE(su.name, pu.name) AS table_owner,
-    COALESCE(su.uid, pu.uid) AS table_owner_id,
-    NULL AS comment
-FROM sys.objects AS o
-    JOIN sys.schemas AS s
-        ON o.schema_id = s.schema_id
-    LEFT JOIN users AS pu
-        ON s.principal_id = pu.uid
-    LEFT JOIN users AS su
-        ON o.principal_id = su.uid
-WHERE
--- SYNAPSE NOTATION: U for TABLE, V for VIEW
-    o.type IN ('U', 'V')

castor_extractor/warehouse/synapse/queries/user.sql

@@ -1,4 +0,0 @@
-SELECT
-    name AS user_name,
-    uid AS user_id
-FROM sys.sysusers

castor_extractor/warehouse/synapse/queries/view_ddl.sql

@@ -1,8 +0,0 @@
-SELECT
-    DB_NAME() AS database_id,
-    DB_NAME() AS database_name,
-    DB_NAME() + '.' + table_schema AS schema_id,
-    table_schema AS schema_name,
-    table_name AS view_name,
-    view_definition AS view_definition
-FROM INFORMATION_SCHEMA.VIEWS