castor-extractor 0.21.7__tar.gz → 0.21.9__tar.gz

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/CHANGELOG.md

@@ -1,6 +1,10 @@
 
 # Changelog
 
+## 0.21.9 - 2024-12-04
+
+* Tableau: fix handling of timeout retry
+
 ## 0.21.8 - 2024-11-26
 
 * Redshift: improve deduplication of columns

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.7
+Version: 0.21.9
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,10 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.9 - 2024-12-04
+
+* Tableau: fix handling of timeout retry
+
 ## 0.21.8 - 2024-11-26
 
 * Redshift: improve deduplication of columns

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/castor_extractor/utils/client/api/client.py

@@ -19,6 +19,7 @@ HttpMethod = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]
 
 DEFAULT_TIMEOUT = 60
 RETRY_ON_EXPIRED_TOKEN = 1
+RETRY_ON_GATEWAY_TIMEOUT = 3
 
 
 def _generate_payloads(
@@ -102,6 +103,10 @@ class APIClient:
             timeout=self._timeout,
         )
 
+    @retry_request(
+        status_codes=(HTTPStatus.GATEWAY_TIMEOUT,),
+        max_retries=RETRY_ON_GATEWAY_TIMEOUT,
+    )
     @retry_request(
         status_codes=(HTTPStatus.UNAUTHORIZED,),
         max_retries=RETRY_ON_EXPIRED_TOKEN,

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/castor_extractor/visualization/sigma/client/client.py

@@ -28,6 +28,7 @@ _DATA_ELEMENTS: Tuple[str, ...] = (
 )
 
 _AUTH_TIMEOUT_S = 60
+_SIGMA_TIMEOUT = 120
 
 _SIGMA_HEADERS = {
     "Content-Type": _CONTENT_TYPE,
@@ -75,6 +76,7 @@ class SigmaClient(APIClient):
             host=credentials.host,
             auth=auth,
             headers=_SIGMA_HEADERS,
+            timeout=_SIGMA_TIMEOUT,
             safe_mode=safe_mode or SIGMA_SAFE_MODE,
         )
 

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/castor_extractor/visualization/sigma/client/pagination.py

@@ -5,7 +5,7 @@ from pydantic.alias_generators import to_camel
 
 from ....utils import PaginationModel
 
-SIGMA_API_LIMIT = 500  # default number of records per page
+SIGMA_API_LIMIT = 200  # default number of records per page
 
 
 class SigmaPagination(PaginationModel):

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/castor_extractor/visualization/tableau_revamp/client/client.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 import tableauserverclient as TSC  # type: ignore
 
@@ -121,12 +122,16 @@ class TableauRevampClient:
         credentials: TableauRevampCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
         with_pulse: bool = False,
+        override_page_size: Optional[int] = None,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
         self._with_pulse = with_pulse
 
-        self._client_metadata = TableauClientMetadataApi(server=self._server)
+        self._client_metadata = TableauClientMetadataApi(
+            server=self._server,
+            override_page_size=override_page_size,
+        )
         self._client_rest = TableauClientRestApi(server=self._server)
         self._client_tsc = TableauClientTSC(server=self._server)
 

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py

@@ -2,10 +2,10 @@ from typing import Dict, Iterator, Optional
 
 import tableauserverclient as TSC  # type: ignore
 
-from ....utils import SerializedAsset
+from ....utils import SerializedAsset, retry
 from ..assets import TableauRevampAsset
 from ..constants import DEFAULT_PAGE_SIZE
-from .errors import TableauApiError
+from .errors import TableauApiError, TableauApiTimeout
 from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
 # increase the value when extraction is too slow
@@ -20,21 +20,58 @@ _CUSTOM_PAGE_SIZE: Dict[TableauRevampAsset, int] = {
     TableauRevampAsset.TABLE: 50,
 }
 
+_TIMEOUT_MESSAGE = (
+    "Execution canceled because timeout of 30000 millis was reached"
+)
+
+_RETRY_BASE_MS = 10_000
+_RETRY_COUNT = 4
+
+
+def _check_errors(answer: dict) -> None:
+    """
+    handle errors in graphql response:
+    - return None when there's no errors in the answer
+    - TableauApiTimeout if any of the errors is a timeout
+    - TableauApiError (generic) otherwise
+    """
+    if "errors" not in answer:
+        return
+
+    errors = answer["errors"]
+
+    for error in errors:
+        if error.get("message") == _TIMEOUT_MESSAGE:
+            # we need specific handling for timeout issues (retry strategy)
+            raise TableauApiTimeout(errors)
+
+    raise TableauApiError(answer["errors"])
+
 
 def gql_query_scroll(
     server,
     query: str,
     resource: str,
 ) -> Iterator[SerializedAsset]:
-    """Iterate over GQL query results, handling pagination and cursor"""
+    """
+    Iterate over GQL query results, handling pagination and cursor
 
+    We have a retry strategy when timeout issues arise.
+    It's a known issue on Tableau side, still waiting for their fix:
+    https://issues.salesforce.com/issue/a028c00000zKahoAAC/undefined
+    """
+
+    @retry(
+        exceptions=(TableauApiTimeout,),
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _call(cursor: Optional[str]) -> dict:
         # If cursor is defined it must be quoted else use null token
         token = "null" if cursor is None else f'"{cursor}"'
         query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
         answer = server.metadata.query(query_)
-        if "errors" in answer:
-            raise TableauApiError(answer["errors"])
+        _check_errors(answer)
         return answer["data"][f"{resource}Connection"]
 
     cursor = None
@@ -58,8 +95,10 @@ class TableauClientMetadataApi:
     def __init__(
         self,
         server: TSC.Server,
+        override_page_size: Optional[int] = None,
     ):
         self._server = server
+        self._forced_page_size = override_page_size
 
     def _call(
         self,
@@ -75,9 +114,16 @@ class TableauClientMetadataApi:
         result_pages = gql_query_scroll(self._server, query, resource)
         return [asset for page in result_pages for asset in page]
 
+    def _page_size(self, asset: TableauRevampAsset) -> int:
+        return (
+            self._forced_page_size
+            or _CUSTOM_PAGE_SIZE.get(asset)
+            or DEFAULT_PAGE_SIZE
+        )
+
     def _fetch_fields(self) -> SerializedAsset:
         result: SerializedAsset = []
-        page_size = _CUSTOM_PAGE_SIZE[TableauRevampAsset.FIELD]
+        page_size = self._page_size(TableauRevampAsset.FIELD)
         for resource, fields in FIELDS_QUERIES:
             current = self._call(resource, fields, page_size)
             result.extend(current)
@@ -90,6 +136,6 @@ class TableauClientMetadataApi:
         if asset == TableauRevampAsset.FIELD:
             return self._fetch_fields()
 
-        page_size = _CUSTOM_PAGE_SIZE.get(asset) or DEFAULT_PAGE_SIZE
+        page_size = self._page_size(asset)
         resource, fields = GQL_QUERIES[asset]
         return self._call(resource, fields, page_size)

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/castor_extractor/visualization/tableau_revamp/client/errors.py

@@ -1,3 +1,8 @@
 class TableauApiError(ValueError):
     def __init__(self, error: str):
         super().__init__(f"Tableau API returned the following error: {error}")
+
+
+class TableauApiTimeout(ValueError):
+    def __init__(self, error: str):
+        super().__init__(f"Tableau API returned a timeout error: {error}")

{castor_extractor-0.21.7 → castor_extractor-0.21.9}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.21.7"
+version = "0.21.9"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"