castor-extractor 0.19.8__tar.gz → 0.20.0__tar.gz

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/CHANGELOG.md

@@ -1,6 +1,14 @@
 
 # Changelog
 
+## 0.20.0 - 2024-09-23
+
+* Switch to Tableau revamped connector
+
+## 0.19.9 - 2024-09-19
+
+* Databricks: multithreading to retrieve column lineage
+
 ## 0.19.8 - 2024-09-18
 
 * Metabase: Handle duplicate dashboards

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.19.8
+Version: 0.20.0
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,14 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.20.0 - 2024-09-23
+
+* Switch to Tableau revamped connector
+
+## 0.19.9 - 2024-09-19
+
+* Databricks: multithreading to retrieve column lineage
+
 ## 0.19.8 - 2024-09-18
 
 * Metabase: Handle duplicate dashboards

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/commands/extract_tableau.py

@@ -1,7 +1,8 @@
 import logging
 from argparse import ArgumentParser
 
-from castor_extractor.visualization import tableau  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import tableau_revamp  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
@@ -19,29 +20,14 @@ def main():
 
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
+
     parser.add_argument(
-        "-s",
-        "--safe-mode",
-        help="Tableau safe mode",
+        "--with-pulse",
+        dest="with_pulse",
         action="store_true",
+        help="Extract Tableau Pulse assets: Metrics and Subscriptions",
     )
-    parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = tableau.ApiClient(
-        user=args.user,
-        password=args.password,
-        token_name=args.token_name,
-        token=args.token,
-        server_url=args.server_url,
-        site_id=args.site_id,
-        safe_mode=args.safe_mode,
-    )
-
-    client.login()
+    parser.add_argument("-o", "--output", help="Directory to write to")
 
-    tableau.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    tableau_revamp.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/utils/__init__.py

@@ -34,7 +34,7 @@ from .pager import (
     PagerOnIdLogger,
     PagerStopStrategy,
 )
-from .retry import RetryStrategy, retry
+from .retry import RetryStrategy, retry, retry_request
 from .safe import SafeMode, safe_mode
 from .store import AbstractStorage, LocalStorage
 from .string import decode_when_bytes, string_to_tuple

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/utils/client/api/client.py

@@ -26,18 +26,18 @@ def _generate_payloads(
     params: Optional[dict],
     data: Optional[dict],
     pagination_params: Optional[dict],
-) -> Tuple[dict, dict]:
+) -> Tuple[Optional[dict], Optional[dict]]:
     _pagination_params = pagination_params or {}
-    params = params or {}
-    data = data or {}
 
     if method == "GET":
+        params = params or {}
         params = {**params, **_pagination_params}
-    elif method == "POST":
+        return data, params
+    if method == "POST":
+        data = data or {}
         data = {**data, **_pagination_params}
-    else:
-        raise ValueError(f"Method {method} is not yet supported")
-    return data, params
+        return data, params
+    raise ValueError(f"Method {method} is not yet supported")
 
 
 class APIClient:

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/utils/client/api/pagination.py

@@ -67,7 +67,7 @@ def fetch_all_pages(
     response_payload = request()
     paginated_response = pagination_model(**response_payload)
     while not paginated_response.is_last():
-        logger.info(f"Fetching page number {page_number}")
+        logger.debug(f"Fetching page number {page_number}")
         yield from paginated_response.page_results()
         next_page_parameters = paginated_response.next_page_parameters()
         new_request = partial(request, **next_page_parameters)
@@ -79,5 +79,5 @@ def fetch_all_pages(
         page_number += 1
 
     # send last page's results
-    logger.info(f"Fetching page number {page_number}")
+    logger.debug(f"Fetching page number {page_number}")
     yield from paginated_response.page_results()

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/utils/write.py

@@ -35,7 +35,7 @@ def write_json(filename: str, data: Any):
     """
     with open(filename, "w", encoding=ENCODING) as f:
         json.dump(data, f)
-        logger.info(f"Wrote output file: {filename}")
+        logger.info(f"Wrote output file: {filename} ({f.tell()} bytes)")
 
 
 def _current_version() -> str:

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/visualization/tableau_revamp/extract.py

@@ -11,7 +11,7 @@ from ...utils import (
     write_summary,
 )
 from .assets import TableauRevampAsset
-from .client import TableauRevampClient
+from .client import TableauRevampClient, TableauRevampCredentials
 
 logger = logging.getLogger(__name__)
 
@@ -26,16 +26,19 @@ def iterate_all_data(
         yield asset, deep_serialize(data)
 
 
-def extract_all(client: TableauRevampClient, **kwargs: str) -> None:
+def extract_all(**kwargs) -> None:
     """
-    Extract Data from tableau
-    Store data locally in files under the output_directory
-    If errors from Tableau's API are catch store them locally in file under the output_directory
+    Extract Data From tableau and store it locally in files under the
+    output_directory
     """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
-
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    with_pulse = kwargs.get("with_pulse") or False
     timestamp = current_timestamp()
 
+    credentials = TableauRevampCredentials(**kwargs)
+    client = TableauRevampClient(credentials, with_pulse=with_pulse)
+    client.login()
+
     for key, data in iterate_all_data(client):
         filename = get_output_filename(key.value, output_directory, timestamp)
         write_json(filename, data)

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/warehouse/databricks/api_client.py

@@ -1,5 +1,6 @@
 import logging
 from functools import partial
+from http import HTTPStatus
 from typing import Iterator, List, Optional, Set, Tuple
 
 import requests
@@ -12,6 +13,7 @@ from ...utils import (
     fetch_all_pages,
     handle_response,
     retry,
+    retry_request,
     safe_mode,
 )
 from ..abstract import TimeFilter
@@ -135,6 +137,10 @@ class DatabricksAPIClient(APIClient):
         max_retries=_RETRY_ATTEMPTS,
         base_ms=_RETRY_BASE_MS,
     )
+    @retry_request(
+        status_codes=(HTTPStatus.TOO_MANY_REQUESTS,),
+        max_retries=_RETRY_ATTEMPTS,
+    )
     def get_single_column_lineage(
         self,
         names: Tuple[str, str],

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/warehouse/databricks/client.py

@@ -15,7 +15,8 @@ from .types import TablesColumns, TimestampedLink
 
 logger = logging.getLogger(__name__)
 
-_MAX_THREADS = 10
+_THREADS_COLUMN_LINEAGE = 2
+_THREADS_TABLE_LINEAGE = 10
 
 
 class DatabricksClient:
@@ -99,7 +100,7 @@ class DatabricksClient:
         Wrapper function that retrieves all table lineage
         """
         # retrieve table lineage
-        with ThreadPoolExecutor(max_workers=_MAX_THREADS) as executor:
+        with ThreadPoolExecutor(max_workers=_THREADS_TABLE_LINEAGE) as executor:
             table_paths = [
                 ".".join([table["schema_id"], table["table_name"]])
                 for table in tables
@@ -121,10 +122,15 @@ class DatabricksClient:
         candidate_paths = paths_for_column_lineage(
             tables, columns, table_lineage
         )
+        # retrieve column lineage
+        with ThreadPoolExecutor(
+            max_workers=_THREADS_COLUMN_LINEAGE
+        ) as executor:
+            results = executor.map(
+                self.api_client.get_single_column_lineage, candidate_paths
+            )
         lineages: List[TimestampedLink] = [
-            link
-            for paths in candidate_paths
-            for link in self.api_client.get_single_column_lineage(paths)
+            link for links in results for link in links
         ]
         deduplicated = deduplicate_lineage(lineages)
         return self.formatter.format_lineage(deduplicated)

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/castor_extractor/warehouse/sqlserver/client.py

@@ -12,8 +12,8 @@ _KEYS = ("user", "password", "host", "port", "database")
 
 
 def _check_key(credentials: dict) -> None:
-    for key in credentials:
-        if key not in _KEYS:
+    for key in _KEYS:
+        if key not in credentials:
             raise KeyError(f"Missing {key} in credentials")
 
 

{castor_extractor-0.19.8 → castor_extractor-0.20.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.19.8"
+version = "0.20.0"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"