castor-extractor 0.24.32__py3-none-any.whl → 0.24.33__py3-none-any.whl

CHANGELOG.md

@@ -1,9 +1,13 @@
 # Changelog
 
+## 0.24.33 - 2025-07-10
+
+* Tableau - Add an option to skip fields ingestion
+
 ## 0.24.32 - 2025-07-02
 
 * Salesforce reporting - extract report's metadata
-*
+
 ## 0.24.31 - 2025-07-02
 
 * Looker Studio: add option to list users via a provided JSON file

castor_extractor/commands/extract_tableau.py

@@ -28,6 +28,13 @@ def main():
         help="Option to avoid extracting Tableau columns, default to False",
     )
 
+    parser.add_argument(
+        "--skip-fields",
+        dest="skip_fields",
+        action="store_true",
+        help="Option to avoid extracting Tableau fields, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",
@@ -41,6 +48,14 @@ def main():
         required=False,
     )
 
+    parser.add_argument(
+        "-ie",
+        "--ignore-errors",
+        action="store_true",
+        dest="ignore_errors",
+        help="Allow partial extraction of Fields and Columns: skip batch in case of Timeout errors",
+    )
+
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     tableau.extract_all(**parse_filled_arguments(parser))

castor_extractor/visualization/tableau/client/client.py

@@ -122,13 +122,17 @@ class TableauClient:
         credentials: TableauCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
         with_columns: bool = True,
+        with_fields: bool = True,
         with_pulse: bool = False,
         override_page_size: Optional[int] = None,
+        ignore_errors: bool = False,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
         self._with_columns = with_columns
+        self._with_fields = with_fields
         self._with_pulse = with_pulse
+        self._ignore_errors = ignore_errors
 
         self._client_metadata = TableauClientMetadataApi(
             server=self._server,
@@ -221,6 +225,10 @@ class TableauClient:
             logger.info(f"Skipping asset {asset} - deactivated columns")
             return []
 
+        if asset == TableauAsset.FIELD and not self._with_fields:
+            logger.info(f"Skipping asset {asset} - deactivated fields")
+            return []
+
         logger.info(f"Extracting {asset.name}...")
 
         if asset == TableauAsset.DATASOURCE:
@@ -240,4 +248,4 @@ class TableauClient:
             return self._client_rest.fetch(asset)
 
         # other assets can be extracted via Metadata API
-        return self._client_metadata.fetch(asset)
+        return self._client_metadata.fetch(asset, self._ignore_errors)

castor_extractor/visualization/tableau/client/client_metadata_api.py

@@ -2,6 +2,7 @@ import logging
 from collections.abc import Iterator
 from typing import Optional
 
+import requests
 import tableauserverclient as TSC  # type: ignore
 
 from ....utils import SerializedAsset, retry
@@ -12,6 +13,13 @@ from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
 logger = logging.getLogger(__name__)
 
+# These assets are known to be error-prone, so it's acceptable if a few are missed.
+# If errors occur, skip the current batch.
+_SAFE_MODE_ASSETS = (
+    TableauAsset.COLUMN,
+    TableauAsset.FIELD,
+)
+
 # increase the value when extraction is too slow
 # decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: dict[TableauAsset, int] = {
@@ -92,6 +100,7 @@ def gql_query_scroll(
     resource: str,
     fields: str,
     page_size: int,
+    skip_batch: bool,
 ) -> Iterator[SerializedAsset]:
     """
     Iterate over GQL query results, handling pagination and cursor
@@ -119,15 +128,22 @@ def gql_query_scroll(
 
     current_offset = 0
     while True:
-        payload = _call(first=page_size, offset=current_offset)
-        yield payload["nodes"]
+        try:
+            payload = _call(first=page_size, offset=current_offset)
+            yield payload["nodes"]
+
+            current_offset += len(payload["nodes"])
+            total = payload["totalCount"]
+            logger.info(f"Extracted {current_offset}/{total} {resource}")
 
-        current_offset += len(payload["nodes"])
-        total = payload["totalCount"]
-        logger.info(f"Extracted {current_offset}/{total} {resource}")
+            if not payload["pageInfo"]["hasNextPage"]:
+                break
+        except requests.exceptions.ReadTimeout:
+            if not skip_batch:
+                raise
 
-        if not payload["pageInfo"]["hasNextPage"]:
-            break
+            logger.warning("Skipping batch because of TableauServer Timeout")
+            current_offset += page_size
 
 
 def _deduplicate(result_pages: Iterator[SerializedAsset]) -> SerializedAsset:
@@ -177,12 +193,14 @@ class TableauClientMetadataApi:
         resource: str,
         fields: str,
         page_size: int = DEFAULT_PAGE_SIZE,
+        skip_batch: bool = False,
     ) -> SerializedAsset:
         result_pages = gql_query_scroll(
             self._server,
             resource=resource,
             fields=fields,
             page_size=page_size,
+            skip_batch=skip_batch,
         )
         return _deduplicate(result_pages)
 
@@ -193,21 +211,41 @@ class TableauClientMetadataApi:
             or DEFAULT_PAGE_SIZE
         )
 
-    def _fetch_fields(self) -> SerializedAsset:
+    def _fetch_fields(self, skip_batch: bool = False) -> SerializedAsset:
         result: SerializedAsset = []
         page_size = self._page_size(TableauAsset.FIELD)
         for resource, fields in FIELDS_QUERIES:
-            current = self._call(resource, fields, page_size)
+            current = self._call(
+                resource,
+                fields,
+                page_size,
+                skip_batch=skip_batch,
+            )
             result.extend(current)
         return result
 
+    @staticmethod
+    def _should_skip_batch_with_timeout(
+        asset: TableauAsset,
+        ignore_metadata_errors: bool = False,
+    ) -> bool:
+        return asset in _SAFE_MODE_ASSETS and ignore_metadata_errors
+
     def fetch(
         self,
         asset: TableauAsset,
+        ignore_errors: bool = False,
     ) -> SerializedAsset:
+        skip_batch = self._should_skip_batch_with_timeout(asset, ignore_errors)
+
         if asset == TableauAsset.FIELD:
-            return self._fetch_fields()
+            return self._fetch_fields(skip_batch=skip_batch)
 
         page_size = self._page_size(asset)
         resource, fields = GQL_QUERIES[asset]
-        return self._call(resource, fields, page_size)
+        return self._call(
+            resource=resource,
+            fields=fields,
+            page_size=page_size,
+            skip_batch=skip_batch,
+        )

castor_extractor/visualization/tableau/extract.py

@@ -33,16 +33,20 @@ def extract_all(**kwargs) -> None:
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     with_columns = not kwargs.get("skip_columns")
+    with_fields = not kwargs.get("skip_fields")
     with_pulse = kwargs.get("with_pulse") or False
     page_size = kwargs.get("page_size")
+    ignore_errors = kwargs.get("ignore_errors") or False
     timestamp = current_timestamp()
 
     credentials = TableauCredentials(**kwargs)
     client = TableauClient(
         credentials,
         with_columns=with_columns,
+        with_fields=with_fields,
         with_pulse=with_pulse,
         override_page_size=page_size,
+        ignore_errors=ignore_errors,
     )
     client.login()
 

{castor_extractor-0.24.32.dist-info → castor_extractor-0.24.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.32
+Version: 0.24.33
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,10 +215,14 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.33 - 2025-07-10
+
+* Tableau - Add an option to skip fields ingestion
+
 ## 0.24.32 - 2025-07-02
 
 * Salesforce reporting - extract report's metadata
-*
+
 ## 0.24.31 - 2025-07-02
 
 * Looker Studio: add option to list users via a provided JSON file

{castor_extractor-0.24.32.dist-info → castor_extractor-0.24.33.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=lQxSt8IlqUEvw7ldjh6EV3ifzbvsiqfztK5dHGx_Y8g,18703
+CHANGELOG.md,sha256=jKQMJGiDeDEZG-753wDrtfOoOYa5Db5Liy0AsATdsuc,18779
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -26,7 +26,7 @@ castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_
 castor_extractor/commands/extract_snowflake.py,sha256=GwlrRxwEBjHqGs_3bs5vM9fzmv61_iwvBr1KcIgFgWM,2161
 castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7WJw-sDU6GHl3lzBt_0,1141
 castor_extractor/commands/extract_strategy.py,sha256=Q-pUymatPrBFGXobhyUPzFph0-t774-XOpjdCFF1dYo,821
-castor_extractor/commands/extract_tableau.py,sha256=ngujGYohWOqOK1qjIP1Hh951jr0KNKNSeOyoaOnO450,1558
+castor_extractor/commands/extract_tableau.py,sha256=LNtI29LbVk1vp4RNrn89GmdW6R_7QBYunRmkowDhbco,1982
 castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=TJx76Ymd0QCECmq35zRJMkPE8DJtSInB28MuSXWk8Ao,2644
 castor_extractor/commands/upload.py,sha256=rLXp7gQ8zb1kLbho4FT87q8eJd8Gvo_TkyIynAaQ-4s,1342
@@ -289,8 +289,8 @@ castor_extractor/visualization/strategy/extract.py,sha256=2fBuvS2xiOGXRpxXnZsE_C
 castor_extractor/visualization/tableau/__init__.py,sha256=eFI_1hjdkxyUiAYiy3szwyuwn3yJ5C_KbpBU0ySJDcQ,138
 castor_extractor/visualization/tableau/assets.py,sha256=HbCRd8VCj1WBEeqg9jwnygnT7xOFJ6PQD7Lq7sV-XR0,635
 castor_extractor/visualization/tableau/client/__init__.py,sha256=P8RKFKOC63WkH5hdEytJOwHS9vzQ8GXreLfXZetmMP8,78
-castor_extractor/visualization/tableau/client/client.py,sha256=iJ3Y-vwPvmPyAUTs1PqFJEZelPGiLvsiwXpTI3b5THc,7867
-castor_extractor/visualization/tableau/client/client_metadata_api.py,sha256=ryRq4_qUok8vvWGhj5CNWXtwR2JlUsu1qjsov2KhQTE,6286
+castor_extractor/visualization/tableau/client/client.py,sha256=QV-GFS4nEq976JLji57pIfsw2ZZaGTvfCFqy6_HOWMg,8204
+castor_extractor/visualization/tableau/client/client_metadata_api.py,sha256=eAq9rjrB_2ZCQy9NwREHBOTXZffWdkwtwhzswm1pEfk,7449
 castor_extractor/visualization/tableau/client/client_metadata_api_test.py,sha256=rikyQKDLFYHLJhHJTF3LwWhKJ80svtTsYp5n7n9oTU8,2665
 castor_extractor/visualization/tableau/client/client_rest_api.py,sha256=x4dNw4PPJdalTlGowwkANwqiS2ZhGxzpQytkHq3KbpY,3988
 castor_extractor/visualization/tableau/client/client_tsc.py,sha256=VI_PJyd1ty3HSYXHHQjshmG2ziowIbrwJRonRPCHbks,1820
@@ -299,7 +299,7 @@ castor_extractor/visualization/tableau/client/errors.py,sha256=ecT8Tit5VtzrOBB9y
 castor_extractor/visualization/tableau/client/gql_queries.py,sha256=XJAfhpMZ5S7-AhfpOaoHMHCAdil-l5e5xB-CH4NC38M,2177
 castor_extractor/visualization/tableau/client/rest_fields.py,sha256=ZKYYuMxg9PXhczVXaD4rXNk7dYyWJ1_bVM8FLEXju7s,888
 castor_extractor/visualization/tableau/constants.py,sha256=lHGB50FgVNO2nXeIhkvQKivD8ZFBIjDrflgD5cTXKJw,104
-castor_extractor/visualization/tableau/extract.py,sha256=hGVr1BZVsHlIgNXOFusRN2YwUUhXvF3reOeN8g1CTEo,1508
+castor_extractor/visualization/tableau/extract.py,sha256=9mSHFJ2DGlW-cDYiRZlJafAgj4_ObACxO0l9vBBfjUw,1683
 castor_extractor/visualization/thoughtspot/__init__.py,sha256=NhTGUk5Kdt54oCjHYoAt0cLBmVLys5lFYiRANL6wCmI,150
 castor_extractor/visualization/thoughtspot/assets.py,sha256=SAQWPKaD2NTSDg7-GSkcRSSEkKSws0MJfOVcHkdeTSg,276
 castor_extractor/visualization/thoughtspot/client/__init__.py,sha256=svrE2rMxR-OXctjPeAHMEPePlfcra-9KDevTMcHunAA,86
@@ -430,8 +430,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.32.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.32.dist-info/METADATA,sha256=vsfvzg3F_c34Ek6G9oQ5LRVpGafwrxIJdXnNcJO4_n8,26156
-castor_extractor-0.24.32.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.32.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.32.dist-info/RECORD,,
+castor_extractor-0.24.33.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.33.dist-info/METADATA,sha256=vCEpwDM8sngoUEfrGtRPSjtCjTw6zxJGiJrnmj4eq_Y,26232
+castor_extractor-0.24.33.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.33.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.33.dist-info/RECORD,,