castor-extractor 0.24.29__tar.gz → 0.24.33__tar.gz

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.24.33 - 2025-07-10
+
+* Tableau - Add an option to skip fields ingestion
+
+## 0.24.32 - 2025-07-02
+
+* Salesforce reporting - extract report's metadata
+
+## 0.24.31 - 2025-07-02
+
+* Looker Studio: add option to list users via a provided JSON file
+
+## 0.24.30 - 2025-06-26
+
+* Sigma: remove retry on timeout, decrease pagination for queries
+
 ## 0.24.29 - 2025-06-24
 
 * Strategy: skip descriptions on ValueErrors

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.29
+Version: 0.24.33
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,22 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.33 - 2025-07-10
+
+* Tableau - Add an option to skip fields ingestion
+
+## 0.24.32 - 2025-07-02
+
+* Salesforce reporting - extract report's metadata
+
+## 0.24.31 - 2025-07-02
+
+* Looker Studio: add option to list users via a provided JSON file
+
+## 0.24.30 - 2025-06-26
+
+* Sigma: remove retry on timeout, decrease pagination for queries
+
 ## 0.24.29 - 2025-06-24
 
 * Strategy: skip descriptions on ValueErrors

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/commands/extract_looker_studio.py

@@ -30,6 +30,14 @@ def main():
         default=False,
         help="Skips the extraction of activity logs",
     )
+    parser.add_argument(
+        "--users-file-path",
+        help=(
+            "Optional path to a JSON file with user email addresses "
+            'as a list of strings (e.g. ["foo@bar.com", "fee@bar.com"]). '
+            "If provided, only extracts assets owned by the specified users."
+        ),
+    )
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/commands/extract_tableau.py

@@ -28,6 +28,13 @@ def main():
         help="Option to avoid extracting Tableau columns, default to False",
     )
 
+    parser.add_argument(
+        "--skip-fields",
+        dest="skip_fields",
+        action="store_true",
+        help="Option to avoid extracting Tableau fields, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",
@@ -41,6 +48,14 @@ def main():
         required=False,
     )
 
+    parser.add_argument(
+        "-ie",
+        "--ignore-errors",
+        action="store_true",
+        dest="ignore_errors",
+        help="Allow partial extraction of Fields and Columns: skip batch in case of Timeout errors",
+    )
+
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     tableau.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/looker_studio/client/client.py

@@ -36,23 +36,40 @@ class LookerStudioClient:
         self,
         credentials: LookerStudioCredentials,
         bigquery_credentials: Optional[dict] = None,
+        user_emails: Optional[list[str]] = None,
     ):
         self.admin_sdk_client = AdminSDKClient(credentials)
         self.looker_studio_client = LookerStudioAPIClient(credentials)
+        self.user_emails = user_emails
 
         self.bigquery_client: Optional[BigQueryClient] = None
         if bigquery_credentials:
             self.bigquery_client = BigQueryClient(bigquery_credentials)
 
-    def _get_assets(self) -> Iterator[dict]:
+    def _list_user_emails(self) -> Iterator[str]:
         """
-        Extracts reports and data sources user by user.
+        Lists user emails either from a provided JSON file or via the Admin SDK API.
+
+        Using all Google Workspace users can be inefficient for large clients -
+        the client might spend hours checking thousands of users for Looker Studio
+        assets when only a handful actually own any. A JSON file allows
+        targeting known owners instead.
         """
-        users = self.admin_sdk_client.list_users()
+        if self.user_emails is not None:
+            yield from self.user_emails
+            return
+
+        for user in self.admin_sdk_client.list_users():
+            yield user[USER_EMAIL_FIELD]
 
-        for user in users:
-            email = user[USER_EMAIL_FIELD]
-            yield from self.looker_studio_client.fetch_user_assets(email)
+    def _get_assets(self) -> Iterator[dict]:
+        """
+        Extracts reports and data sources user by user. The loop is necessary
+        because the Looker Studio API can only retrieve the assets owned by a
+        single user.
+        """
+        for user_email in self._list_user_emails():
+            yield from self.looker_studio_client.fetch_user_assets(user_email)
 
     def _get_source_queries(self) -> Iterator[dict]:
         """

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/looker_studio/extract.py

@@ -70,21 +70,53 @@ def _bigquery_credentials_or_none(params: dict) -> Optional[dict]:
         return cast(dict, json.load(file))
 
 
+def _validate_user_emails(user_emails: list[str]):
+    """
+    Raises an error if the user emails are not in the expected format (list of strings),
+    or if the list is empty.
+    """
+    if not isinstance(user_emails, list):
+        raise TypeError("The users file must be a list")
+
+    if len(user_emails) == 0:
+        raise ValueError("The users file must contain at least one user email")
+
+    if not all(isinstance(email, str) for email in user_emails):
+        raise TypeError("All items in users list must be strings")
+
+
+def _read_optional_user_emails(
+    users_file_path: Optional[str],
+) -> Optional[list[str]]:
+    """Loads the user emails from a file, if it was provided."""
+    if not users_file_path:
+        return None
+
+    with open(users_file_path, "r") as file:
+        user_emails = json.load(file)
+
+    _validate_user_emails(user_emails)
+    return user_emails
+
+
 def extract_all(**kwargs) -> None:
     """
     Extracts data from Looker Studio and stores the output files locally under
     the given output_directory.
     """
+    users_file_path = kwargs.get("users_file_path")
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     credentials = _credentials(kwargs)
     has_view_activity_logs = bool(credentials.has_view_activity_logs)
+    user_emails = _read_optional_user_emails(users_file_path)
 
     bigquery_credentials = _bigquery_credentials_or_none(kwargs)
 
     client = LookerStudioClient(
         credentials=credentials,
         bigquery_credentials=bigquery_credentials,
+        user_emails=user_emails,
     )
     ts = current_timestamp()
 

castor_extractor-0.24.33/castor_extractor/visualization/looker_studio/extract_test.py

@@ -0,0 +1,19 @@
+import pytest
+
+from .extract import _validate_user_emails
+
+
+def test__validate_user_emails():
+    with pytest.raises(TypeError):
+        _validate_user_emails("toto@tata.com")
+
+    with pytest.raises(TypeError):
+        _validate_user_emails({"not": "the", "right": "format"})
+
+    with pytest.raises(ValueError):
+        _validate_user_emails([])
+
+    with pytest.raises(TypeError):
+        _validate_user_emails([1, 2, 3, 4])
+
+    _validate_user_emails(["admin@toto.com", "tata@toto.com"])

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/salesforce_reporting/assets.py

@@ -8,4 +8,5 @@ class SalesforceReportingAsset(ExternalAsset):
     DASHBOARD_COMPONENTS = "dashboard_components"
     FOLDERS = "folders"
     REPORTS = "reports"
+    REPORTS_METADATA = "reports_metadata"
     USERS = "users"

castor_extractor-0.24.33/castor_extractor/visualization/salesforce_reporting/client/rest.py

@@ -0,0 +1,114 @@
+import logging
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional
+
+import requests
+
+from ....utils import build_url
+from ....utils.salesforce import SalesforceBaseClient
+from ..assets import SalesforceReportingAsset
+from .soql import queries
+
+logger = logging.getLogger(__name__)
+
+REQUIRING_URL_ASSETS = (
+    SalesforceReportingAsset.REPORTS,
+    SalesforceReportingAsset.DASHBOARDS,
+    SalesforceReportingAsset.FOLDERS,
+)
+
+_CONCURRENT_THREADS = 50
+
+
+class SalesforceReportingClient(SalesforceBaseClient):
+    """
+    Salesforce Reporting API client
+    """
+
+    def _get_asset_url(
+        self, asset_type: SalesforceReportingAsset, asset: dict
+    ) -> Optional[str]:
+        """
+        Fetch the given Asset + add the corresponding URL.
+        """
+
+        if asset_type == SalesforceReportingAsset.DASHBOARDS:
+            path = f"lightning/r/Dashboard/{asset['Id']}/view"
+            return build_url(self._host, path)
+
+        if asset_type == SalesforceReportingAsset.FOLDERS:
+            path = asset["attributes"]["url"].lstrip("/")
+            return build_url(self._host, path)
+
+        if asset_type == SalesforceReportingAsset.REPORTS:
+            path = f"lightning/r/Report/{asset['Id']}/view"
+            return build_url(self._host, path)
+
+        return None
+
+    def _fetch_and_add_url(
+        self, asset_type: SalesforceReportingAsset
+    ) -> Iterator[dict]:
+        assets = self._query_all(queries[asset_type])
+        for asset in assets:
+            url = self._get_asset_url(asset_type, asset)
+            yield {**asset, "Url": url}
+
+    def _metadata(self, report_id: str) -> Optional[dict]:
+        url = f"services/data/v60.0/analytics/reports/{report_id}/describe"
+        try:
+            metadata = self._get(url, retry_on_timeout=False)
+            # pick only what we need to build the lineage
+            columns = metadata["reportExtendedMetadata"]["detailColumnInfo"]
+            return {
+                "reportId": report_id,
+                "detailColumnInfo": columns or dict(),
+            }
+        except (requests.HTTPError, requests.RequestException) as ex:
+            # Extracting column metadata is used only for lineage purposes
+            # and is non-critical. API errors are common during this step,
+            # so we choose to skip them rather than fail the process. The same
+            # rows consistently fail, and retries have proven ineffective.
+            logger.info(ex)
+            return None
+
+    def _fetch_reports_metadata(self) -> Iterator[dict]:
+        """
+        Use the "describe" endpoint to extract report metadata.
+        Keep only the detailColumnInfo, which is required for building the lineage.
+
+        More info here:
+        https://developer.salesforce.com/docs/atlas.en-us.api_analytics.meta/api_analytics/sforce_analytics_rest_api_getbasic_reportmetadata.htm
+        https://www.notion.so/castordoc/Salesforce-Lineage-216a1c3d458580859888cf4ca2d7fa51?source=copy_link
+        """
+        # The "describe" endpoint requires report_ids. To avoid introducing
+        # task dependencies, we opted to re-extract the reports.
+        # It is fast anyway, since it's running a SQL query
+        reports = self.fetch(SalesforceReportingAsset.REPORTS)
+        report_ids = [report["Id"] for report in reports]
+
+        # Calling "describe" on each report individually can be slow,
+        # especially for accounts with thousands of reports. That's why
+        # we use multithreading here — it significantly improves performance.
+        with ThreadPoolExecutor(max_workers=_CONCURRENT_THREADS) as executor:
+            fetch_results = executor.map(self._metadata, report_ids)
+
+            for metadata in fetch_results:
+                if not metadata:
+                    continue
+                yield metadata
+
+    def fetch(self, asset: SalesforceReportingAsset) -> list[dict]:
+        """
+        Fetch Salesforce Reporting assets
+        """
+        logger.info(f"Starting extraction of {asset}")
+
+        if asset in REQUIRING_URL_ASSETS:
+            return list(self._fetch_and_add_url(asset))
+
+        if asset == SalesforceReportingAsset.REPORTS_METADATA:
+            return list(self._fetch_reports_metadata())
+
+        return list(self._query_all(queries[asset]))

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/sigma/client/client.py

@@ -2,7 +2,7 @@ from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from http import HTTPStatus
-from typing import Callable, Optional
+from typing import Callable, Iterable, Optional
 
 import requests
 from pydantic import BaseModel
@@ -19,7 +19,11 @@ from ....utils import (
 from ..assets import SigmaAsset
 from .credentials import SigmaCredentials
 from .endpoints import SigmaEndpointFactory
-from .pagination import SIGMA_API_LIMIT, SigmaPagination
+from .pagination import (
+    SIGMA_API_LIMIT,
+    SIGMA_QUERIES_PAGINATION_LIMIT,
+    SigmaPagination,
+)
 
 _CONTENT_TYPE = "application/x-www-form-urlencoded"
 
@@ -101,9 +105,27 @@ class SigmaClient(APIClient):
             safe_mode=safe_mode or SIGMA_SAFE_MODE,
         )
 
-    def _get_paginated(self, endpoint: str) -> Callable:
+    def _get_paginated(
+        self,
+        endpoint: str,
+        limit: int = SIGMA_API_LIMIT,
+    ) -> Callable:
+        """
+        Sigma’s API does not experience random timeouts, unlike some other APIs.
+        However, extracting queries from certain workbooks can take a
+        significant amount of time.
+        Previously, when a timeout occurred, the system would retry multiple
+        times — even though we knew it would eventually fail due to the inherent
+        slowness of the operation.
+        These retries only delayed the inevitable failure without adding value.
+        To address this, we've disabled retries on timeout and instead adjusted
+        the page size when extracting queries.
+        """
         return partial(
-            self._get, endpoint=endpoint, params={"limit": SIGMA_API_LIMIT}
+            self._get,
+            retry_on_timeout=False,  # explained in the docstring
+            endpoint=endpoint,
+            params={"limit": limit},
         )
 
     def _get_all_datasets(self) -> Iterator[dict]:
@@ -200,16 +222,34 @@ class SigmaClient(APIClient):
                 "element_id": lineage.context.element_id,
             }
 
+    @staticmethod
+    def _yield_deduplicated_queries(
+        queries: Iterable[dict], workbook_id: str
+    ) -> Iterator[dict]:
+        """
+        Returns unique queries for a workbook. This is necessary because the API
+        unfortunately returns duplicate entries for some workbook elements.
+        """
+        seen_elements = set()
+
+        for query in queries:
+            element_id = query["elementId"]
+            if element_id in seen_elements:
+                continue
+
+            seen_elements.add(element_id)
+            yield {**query, "workbook_id": workbook_id}
+
     def _get_all_queries(self, workbooks: list[dict]) -> Iterator[dict]:
         for workbook in workbooks:
             workbook_id = workbook["workbookId"]
             request = self._get_paginated(
-                SigmaEndpointFactory.queries(workbook_id)
+                SigmaEndpointFactory.queries(workbook_id),
+                limit=SIGMA_QUERIES_PAGINATION_LIMIT,
             )
             queries = fetch_all_pages(request, SigmaPagination)
 
-            for query in queries:
-                yield {**query, "workbook_id": workbook_id}
+            yield from self._yield_deduplicated_queries(queries, workbook_id)
 
     def fetch(
         self,

castor_extractor-0.24.33/castor_extractor/visualization/sigma/client/client_test.py

@@ -0,0 +1,19 @@
+from .client import SigmaClient
+
+
+def test_SigmaClient__yield_deduplicated_queries():
+    workbook_id = "workbook1"
+    mock_queries = [
+        {"elementId": "element1", "name": "Query 1"},
+        {"elementId": "element2", "name": "Query 2"},
+        {"elementId": "element1", "name": "Query 1"},  # Duplicate
+        {"elementId": "element3", "name": "Query 3"},
+    ]
+
+    queries = list(
+        SigmaClient._yield_deduplicated_queries(mock_queries, workbook_id)
+    )
+
+    assert len(queries) == 3
+    for query in queries:
+        assert query["workbook_id"] == workbook_id

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/sigma/client/pagination.py

@@ -6,6 +6,7 @@ from pydantic.alias_generators import to_camel
 from ....utils import PaginationModel
 
 SIGMA_API_LIMIT = 200  # default number of records per page
+SIGMA_QUERIES_PAGINATION_LIMIT = 50
 
 
 class SigmaPagination(PaginationModel):

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/tableau/client/client.py

@@ -122,13 +122,17 @@ class TableauClient:
         credentials: TableauCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
         with_columns: bool = True,
+        with_fields: bool = True,
         with_pulse: bool = False,
         override_page_size: Optional[int] = None,
+        ignore_errors: bool = False,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
         self._with_columns = with_columns
+        self._with_fields = with_fields
         self._with_pulse = with_pulse
+        self._ignore_errors = ignore_errors
 
         self._client_metadata = TableauClientMetadataApi(
             server=self._server,
@@ -221,6 +225,10 @@ class TableauClient:
             logger.info(f"Skipping asset {asset} - deactivated columns")
             return []
 
+        if asset == TableauAsset.FIELD and not self._with_fields:
+            logger.info(f"Skipping asset {asset} - deactivated fields")
+            return []
+
         logger.info(f"Extracting {asset.name}...")
 
         if asset == TableauAsset.DATASOURCE:
@@ -240,4 +248,4 @@ class TableauClient:
             return self._client_rest.fetch(asset)
 
         # other assets can be extracted via Metadata API
-        return self._client_metadata.fetch(asset)
+        return self._client_metadata.fetch(asset, self._ignore_errors)

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/tableau/client/client_metadata_api.py

@@ -2,6 +2,7 @@ import logging
 from collections.abc import Iterator
 from typing import Optional
 
+import requests
 import tableauserverclient as TSC  # type: ignore
 
 from ....utils import SerializedAsset, retry
@@ -12,6 +13,13 @@ from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
 logger = logging.getLogger(__name__)
 
+# These assets are known to be error-prone, so it's acceptable if a few are missed.
+# If errors occur, skip the current batch.
+_SAFE_MODE_ASSETS = (
+    TableauAsset.COLUMN,
+    TableauAsset.FIELD,
+)
+
 # increase the value when extraction is too slow
 # decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: dict[TableauAsset, int] = {
@@ -92,6 +100,7 @@ def gql_query_scroll(
     resource: str,
     fields: str,
     page_size: int,
+    skip_batch: bool,
 ) -> Iterator[SerializedAsset]:
     """
     Iterate over GQL query results, handling pagination and cursor
@@ -119,15 +128,22 @@ def gql_query_scroll(
 
     current_offset = 0
     while True:
-        payload = _call(first=page_size, offset=current_offset)
-        yield payload["nodes"]
+        try:
+            payload = _call(first=page_size, offset=current_offset)
+            yield payload["nodes"]
+
+            current_offset += len(payload["nodes"])
+            total = payload["totalCount"]
+            logger.info(f"Extracted {current_offset}/{total} {resource}")
 
-        current_offset += len(payload["nodes"])
-        total = payload["totalCount"]
-        logger.info(f"Extracted {current_offset}/{total} {resource}")
+            if not payload["pageInfo"]["hasNextPage"]:
+                break
+        except requests.exceptions.ReadTimeout:
+            if not skip_batch:
+                raise
 
-        if not payload["pageInfo"]["hasNextPage"]:
-            break
+            logger.warning("Skipping batch because of TableauServer Timeout")
+            current_offset += page_size
 
 
 def _deduplicate(result_pages: Iterator[SerializedAsset]) -> SerializedAsset:
@@ -177,12 +193,14 @@ class TableauClientMetadataApi:
         resource: str,
         fields: str,
         page_size: int = DEFAULT_PAGE_SIZE,
+        skip_batch: bool = False,
     ) -> SerializedAsset:
         result_pages = gql_query_scroll(
             self._server,
             resource=resource,
             fields=fields,
             page_size=page_size,
+            skip_batch=skip_batch,
         )
         return _deduplicate(result_pages)
 
@@ -193,21 +211,41 @@ class TableauClientMetadataApi:
             or DEFAULT_PAGE_SIZE
         )
 
-    def _fetch_fields(self) -> SerializedAsset:
+    def _fetch_fields(self, skip_batch: bool = False) -> SerializedAsset:
         result: SerializedAsset = []
         page_size = self._page_size(TableauAsset.FIELD)
         for resource, fields in FIELDS_QUERIES:
-            current = self._call(resource, fields, page_size)
+            current = self._call(
+                resource,
+                fields,
+                page_size,
+                skip_batch=skip_batch,
+            )
             result.extend(current)
         return result
 
+    @staticmethod
+    def _should_skip_batch_with_timeout(
+        asset: TableauAsset,
+        ignore_metadata_errors: bool = False,
+    ) -> bool:
+        return asset in _SAFE_MODE_ASSETS and ignore_metadata_errors
+
     def fetch(
         self,
         asset: TableauAsset,
+        ignore_errors: bool = False,
     ) -> SerializedAsset:
+        skip_batch = self._should_skip_batch_with_timeout(asset, ignore_errors)
+
         if asset == TableauAsset.FIELD:
-            return self._fetch_fields()
+            return self._fetch_fields(skip_batch=skip_batch)
 
         page_size = self._page_size(asset)
         resource, fields = GQL_QUERIES[asset]
-        return self._call(resource, fields, page_size)
+        return self._call(
+            resource=resource,
+            fields=fields,
+            page_size=page_size,
+            skip_batch=skip_batch,
+        )

{castor_extractor-0.24.29 → castor_extractor-0.24.33}/castor_extractor/visualization/tableau/extract.py

@@ -33,16 +33,20 @@ def extract_all(**kwargs) -> None:
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     with_columns = not kwargs.get("skip_columns")
+    with_fields = not kwargs.get("skip_fields")
     with_pulse = kwargs.get("with_pulse") or False
     page_size = kwargs.get("page_size")
+    ignore_errors = kwargs.get("ignore_errors") or False
     timestamp = current_timestamp()
 
     credentials = TableauCredentials(**kwargs)
     client = TableauClient(
         credentials,
         with_columns=with_columns,
+        with_fields=with_fields,
         with_pulse=with_pulse,
         override_page_size=page_size,
+        ignore_errors=ignore_errors,
     )
     client.login()