castor-extractor 0.21.7__py3-none-any.whl → 0.21.9__py3-none-any.whl

CHANGELOG.md

@@ -1,6 +1,10 @@
 
 # Changelog
 
+## 0.21.9 - 2024-12-04
+
+* Tableau: fix handling of timeout retry
+
 ## 0.21.8 - 2024-11-26
 
 * Redshift: improve deduplication of columns

castor_extractor/utils/client/api/client.py

@@ -19,6 +19,7 @@ HttpMethod = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]
 
 DEFAULT_TIMEOUT = 60
 RETRY_ON_EXPIRED_TOKEN = 1
+RETRY_ON_GATEWAY_TIMEOUT = 3
 
 
 def _generate_payloads(
@@ -102,6 +103,10 @@ class APIClient:
             timeout=self._timeout,
         )
 
+    @retry_request(
+        status_codes=(HTTPStatus.GATEWAY_TIMEOUT,),
+        max_retries=RETRY_ON_GATEWAY_TIMEOUT,
+    )
     @retry_request(
         status_codes=(HTTPStatus.UNAUTHORIZED,),
         max_retries=RETRY_ON_EXPIRED_TOKEN,

castor_extractor/visualization/sigma/client/client.py

@@ -28,6 +28,7 @@ _DATA_ELEMENTS: Tuple[str, ...] = (
 )
 
 _AUTH_TIMEOUT_S = 60
+_SIGMA_TIMEOUT = 120
 
 _SIGMA_HEADERS = {
     "Content-Type": _CONTENT_TYPE,
@@ -75,6 +76,7 @@ class SigmaClient(APIClient):
             host=credentials.host,
             auth=auth,
             headers=_SIGMA_HEADERS,
+            timeout=_SIGMA_TIMEOUT,
             safe_mode=safe_mode or SIGMA_SAFE_MODE,
         )
 

castor_extractor/visualization/sigma/client/pagination.py

@@ -5,7 +5,7 @@ from pydantic.alias_generators import to_camel
 
 from ....utils import PaginationModel
 
-SIGMA_API_LIMIT = 500  # default number of records per page
+SIGMA_API_LIMIT = 200  # default number of records per page
 
 
 class SigmaPagination(PaginationModel):

castor_extractor/visualization/tableau_revamp/client/client.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 import tableauserverclient as TSC  # type: ignore
 
@@ -121,12 +122,16 @@ class TableauRevampClient:
         credentials: TableauRevampCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
         with_pulse: bool = False,
+        override_page_size: Optional[int] = None,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
         self._with_pulse = with_pulse
 
-        self._client_metadata = TableauClientMetadataApi(server=self._server)
+        self._client_metadata = TableauClientMetadataApi(
+            server=self._server,
+            override_page_size=override_page_size,
+        )
         self._client_rest = TableauClientRestApi(server=self._server)
         self._client_tsc = TableauClientTSC(server=self._server)
 

castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py

@@ -2,10 +2,10 @@ from typing import Dict, Iterator, Optional
 
 import tableauserverclient as TSC  # type: ignore
 
-from ....utils import SerializedAsset
+from ....utils import SerializedAsset, retry
 from ..assets import TableauRevampAsset
 from ..constants import DEFAULT_PAGE_SIZE
-from .errors import TableauApiError
+from .errors import TableauApiError, TableauApiTimeout
 from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
 # increase the value when extraction is too slow
@@ -20,21 +20,58 @@ _CUSTOM_PAGE_SIZE: Dict[TableauRevampAsset, int] = {
     TableauRevampAsset.TABLE: 50,
 }
 
+_TIMEOUT_MESSAGE = (
+    "Execution canceled because timeout of 30000 millis was reached"
+)
+
+_RETRY_BASE_MS = 10_000
+_RETRY_COUNT = 4
+
+
+def _check_errors(answer: dict) -> None:
+    """
+    handle errors in graphql response:
+    - return None when there's no errors in the answer
+    - TableauApiTimeout if any of the errors is a timeout
+    - TableauApiError (generic) otherwise
+    """
+    if "errors" not in answer:
+        return
+
+    errors = answer["errors"]
+
+    for error in errors:
+        if error.get("message") == _TIMEOUT_MESSAGE:
+            # we need specific handling for timeout issues (retry strategy)
+            raise TableauApiTimeout(errors)
+
+    raise TableauApiError(answer["errors"])
+
 
 def gql_query_scroll(
     server,
     query: str,
     resource: str,
 ) -> Iterator[SerializedAsset]:
-    """Iterate over GQL query results, handling pagination and cursor"""
+    """
+    Iterate over GQL query results, handling pagination and cursor
 
+    We have a retry strategy when timeout issues arise.
+    It's a known issue on Tableau side, still waiting for their fix:
+    https://issues.salesforce.com/issue/a028c00000zKahoAAC/undefined
+    """
+
+    @retry(
+        exceptions=(TableauApiTimeout,),
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _call(cursor: Optional[str]) -> dict:
         # If cursor is defined it must be quoted else use null token
         token = "null" if cursor is None else f'"{cursor}"'
         query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
         answer = server.metadata.query(query_)
-        if "errors" in answer:
-            raise TableauApiError(answer["errors"])
+        _check_errors(answer)
         return answer["data"][f"{resource}Connection"]
 
     cursor = None
@@ -58,8 +95,10 @@ class TableauClientMetadataApi:
     def __init__(
         self,
         server: TSC.Server,
+        override_page_size: Optional[int] = None,
     ):
         self._server = server
+        self._forced_page_size = override_page_size
 
     def _call(
         self,
@@ -75,9 +114,16 @@ class TableauClientMetadataApi:
         result_pages = gql_query_scroll(self._server, query, resource)
         return [asset for page in result_pages for asset in page]
 
+    def _page_size(self, asset: TableauRevampAsset) -> int:
+        return (
+            self._forced_page_size
+            or _CUSTOM_PAGE_SIZE.get(asset)
+            or DEFAULT_PAGE_SIZE
+        )
+
     def _fetch_fields(self) -> SerializedAsset:
         result: SerializedAsset = []
-        page_size = _CUSTOM_PAGE_SIZE[TableauRevampAsset.FIELD]
+        page_size = self._page_size(TableauRevampAsset.FIELD)
         for resource, fields in FIELDS_QUERIES:
             current = self._call(resource, fields, page_size)
             result.extend(current)
@@ -90,6 +136,6 @@ class TableauClientMetadataApi:
         if asset == TableauRevampAsset.FIELD:
             return self._fetch_fields()
 
-        page_size = _CUSTOM_PAGE_SIZE.get(asset) or DEFAULT_PAGE_SIZE
+        page_size = self._page_size(asset)
         resource, fields = GQL_QUERIES[asset]
         return self._call(resource, fields, page_size)

castor_extractor/visualization/tableau_revamp/client/errors.py

@@ -1,3 +1,8 @@
 class TableauApiError(ValueError):
     def __init__(self, error: str):
         super().__init__(f"Tableau API returned the following error: {error}")
+
+
+class TableauApiTimeout(ValueError):
+    def __init__(self, error: str):
+        super().__init__(f"Tableau API returned a timeout error: {error}")

{castor_extractor-0.21.7.dist-info → castor_extractor-0.21.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.7
+Version: 0.21.9
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,10 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.9 - 2024-12-04
+
+* Tableau: fix handling of timeout retry
+
 ## 0.21.8 - 2024-11-26
 
 * Redshift: improve deduplication of columns

{castor_extractor-0.21.7.dist-info → castor_extractor-0.21.9.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=HIo_bRiTbT5GZ6Hugs5Y1gHIHSsfh1VDVWXhoC_SXWM,14881
+CHANGELOG.md,sha256=TiUeWCSqrrEdYSnHUjkzy8CH0kVXRfjNROxBnUFcEQY,14947
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -84,7 +84,7 @@ castor_extractor/utils/client/abstract.py,sha256=aA5Qcb9TwWDSMq8WpXbGkOB20hehwX2
 castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
 castor_extractor/utils/client/api/auth.py,sha256=QDLM5h1zGibLaKyATxLF0gycg01SE92G-Y69f_YBClc,1896
 castor_extractor/utils/client/api/auth_test.py,sha256=NoZYsz7bcCyWBZdMF1TaOuK-s1j09DhTRyM4GSUW_YQ,1311
-castor_extractor/utils/client/api/client.py,sha256=_XkOMbkit0_YYJ2tY1rYrEuoPTUP826CJPnI9OLgmuQ,4434
+castor_extractor/utils/client/api/client.py,sha256=NXNAbL4-2H2x0NPwmaYz_NKVsSVjuDw61OJUJgoaJSA,4587
 castor_extractor/utils/client/api/client_test.py,sha256=FM3ZxsLLfMOBn44cXX6FIgnA31-5TTNIyp9D4LBwtXE,1222
 castor_extractor/utils/client/api/pagination.py,sha256=Efg3P9ct_U5rtgXijMGV05oQxSzjldEopECWjIFWerM,2439
 castor_extractor/utils/client/api/pagination_test.py,sha256=jCOgXFXrH-jrCxe2dfk80ZksJF-EtmpJPU11BGabsqk,1385
@@ -241,10 +241,10 @@ castor_extractor/visualization/salesforce_reporting/extract.py,sha256=RMhlf7NeYi
 castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
-castor_extractor/visualization/sigma/client/client.py,sha256=Gi8GzP_xxjDZDmNRK4pXJUvSPzZCCJpn6ClUTML5amE,6218
+castor_extractor/visualization/sigma/client/client.py,sha256=ngu4-fN7vquvmqGLVZGgBGhvJYR5srbDVCxhaCvwuHk,6275
 castor_extractor/visualization/sigma/client/credentials.py,sha256=0f2VeDFaNbaZOTN0fPdmx3s-nxxFe35yBF9D_2B2lGI,718
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUGM_bKBAq28Nl7LWSZ6VRsbxrxtDg,1162
-castor_extractor/visualization/sigma/client/pagination.py,sha256=gHBewkYoJaR0vADzU_VPsNkcAa-6KAC6hplpDL5gT8U,690
+castor_extractor/visualization/sigma/client/pagination.py,sha256=kNEhNq08tTGbypyMjxs0w4uvDtQc_iaWpOZweaa_FsU,690
 castor_extractor/visualization/sigma/extract.py,sha256=pnArK5-F6DZcO0f3wp3_km_Od0f18Qmw9yTmynJ-2TU,2278
 castor_extractor/visualization/tableau/__init__.py,sha256=hDohrWjkorrX01JMc154aa9vi3ZqBKmA1lkfQtMFfYE,114
 castor_extractor/visualization/tableau/assets.py,sha256=mfBUzcBCLyiU9gnTB_6rvtiB5yXSDU99nezhGC__HQo,1270
@@ -284,12 +284,12 @@ castor_extractor/visualization/tableau/usage.py,sha256=LlFwlbEr-EnYUJjKZha99CRCR
 castor_extractor/visualization/tableau_revamp/__init__.py,sha256=a3DGjQhaz17gBqW-E84TAgupKbqLC40y5Ajo1yn-ot4,156
 castor_extractor/visualization/tableau_revamp/assets.py,sha256=8sJsK6Qixao6xVmVaO1usvs16SjNub9sIx7o-adYV14,659
 castor_extractor/visualization/tableau_revamp/client/__init__.py,sha256=wmS9uLtUiqNYVloi0-DgD8d2qzu3RVZEAtWiaDp6G_M,90
-castor_extractor/visualization/tableau_revamp/client/client.py,sha256=oaxvPsCccAcTWooXmDQNcJ6RFUVsCUzl6HxaHIwh5kU,7564
-castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py,sha256=yNnGR3Tk32TUmaDejaz5fkw2p9DtmMeCv5rsZNOHUfY,3047
+castor_extractor/visualization/tableau_revamp/client/client.py,sha256=Ju89lMDiLOZ2LjxylcFm5429WElxGxjc52bMIWoKCDA,7716
+castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py,sha256=gmWM5kn6iSOXd9G46kvOOrbs5tn3X9mCWUIRkFMNov8,4332
 castor_extractor/visualization/tableau_revamp/client/client_rest_api.py,sha256=HZ6KdNJ6sqhWElfpYlUwbZDJXxoKkNc3p-YAKExzDxM,4030
 castor_extractor/visualization/tableau_revamp/client/client_tsc.py,sha256=BBwIOqK2zU66udFRmLGmB_3J1ILGhVOY5Hq4nmsonF0,1853
 castor_extractor/visualization/tableau_revamp/client/credentials.py,sha256=qA-EaX-4rbQRsn8v4zWh5Kh784ndHLjJaoZwnkQgCyo,1905
-castor_extractor/visualization/tableau_revamp/client/errors.py,sha256=dTe1shqmWmAXpDpCz-E24m8dGYjt6rvIGV9qQb4jnvI,150
+castor_extractor/visualization/tableau_revamp/client/errors.py,sha256=ecT8Tit5VtzrOBB9ykblA0nvd75j5-_QDFupjV48zJQ,300
 castor_extractor/visualization/tableau_revamp/client/gql_queries.py,sha256=-V3ToD5Gi7nmfVB2OxTOZw8dcOiF7_ciSWjjW2UdvvI,2270
 castor_extractor/visualization/tableau_revamp/client/rest_fields.py,sha256=gx39X1zMfRVpjmFbgvbgbvtlE0QwxOtk8rZFsIqeGRI,978
 castor_extractor/visualization/tableau_revamp/constants.py,sha256=lHGB50FgVNO2nXeIhkvQKivD8ZFBIjDrflgD5cTXKJw,104
@@ -425,8 +425,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.21.7.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.21.7.dist-info/METADATA,sha256=-vrBbF2Ipl1Ar3SwUthFTrDdvjod8IXzEzP2Sb-TuWg,22109
-castor_extractor-0.21.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.21.7.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
-castor_extractor-0.21.7.dist-info/RECORD,,
+castor_extractor-0.21.9.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.21.9.dist-info/METADATA,sha256=_ZFrW5NZYKjKn8CYc5SzxuosZcfUfVAiQQpjW5eI5TQ,22175
+castor_extractor-0.21.9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.21.9.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
+castor_extractor-0.21.9.dist-info/RECORD,,