castor-extractor 0.24.4__py3-none-any.whl → 0.24.7__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.24.7 - 2025-04-07
+
+* Tableau - switch from `cursor` to `offset` pagination to mitigate timeout issues
+
+## 0.24.6 - 2025-04-03
+
+* Domo - extract cards metadata by batch to prevent from hitting URL max length
+
+## 0.24.5 - 2025-04-02
+
+* bump dependencies: google-cloud-storage
+
 ## 0.24.4 - 2025-03-19
 
 * Snowflake:

castor_extractor/utils/__init__.py

@@ -1,4 +1,5 @@
 from .argument_parser import parse_filled_arguments
+from .batch import batch_of_length
 from .client import (
     AbstractSourceClient,
     APIClient,

castor_extractor/utils/batch.py

@@ -0,0 +1,16 @@
+from typing import Iterator, List, TypeVar
+
+T = TypeVar("T")
+
+
+def batch_of_length(
+    elements: List[T],
+    batch_size: int,
+) -> Iterator[List[T]]:
+    """
+    Split the given elements into smaller chunks
+    """
+    assert batch_size > 1, "batch size must be greater or equal to 1"
+    element_count = len(elements)
+    for index in range(0, element_count, batch_size):
+        yield elements[index : min((index + batch_size), element_count)]

castor_extractor/utils/batch_test.py

@@ -0,0 +1,27 @@
+import pytest
+
+from .batch import batch_of_length
+
+
+def test_batch_of_length():
+    elements = ["a", "b", "c", "d", "e", "f", "g", "h"]
+    result = list(batch_of_length(elements, 3))
+    assert result == [
+        ["a", "b", "c"],
+        ["d", "e", "f"],
+        ["g", "h"],
+    ]
+
+    result = list(batch_of_length(elements, 1000))
+    assert result == [
+        elements,
+    ]
+
+    result = list(batch_of_length(elements, 7))
+    assert result == [
+        ["a", "b", "c", "d", "e", "f", "g"],
+        ["h"],
+    ]
+
+    with pytest.raises(AssertionError):
+        list(batch_of_length(elements, -12))

castor_extractor/visualization/domo/client/client.py

@@ -9,6 +9,7 @@ import requests
 from ....utils import (
     RequestSafeMode,
     at_midnight,
+    batch_of_length,
     current_date,
     empty_iterator,
     handle_response,
@@ -48,6 +49,8 @@ _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 
 _PARENT_FOLDER = "/Dashboards"
 
+_CARDS_BATCH_SIZE = 100
+
 logger = logging.getLogger(__name__)
 
 
@@ -156,16 +159,19 @@ class DomoClient:
 
         return all_results
 
+    def _cards_metadata(self, card_ids: list[int]) -> Iterator[dict]:
+        # batch to avoid hitting the URL max length
+        for batch_card_ids in batch_of_length(card_ids, _CARDS_BATCH_SIZE):
+            endpoint = self._endpoint_factory.cards_metadata(batch_card_ids)
+            yield from self._get_element(endpoint)
+
     def _datasources(self, card_ids: list[int]) -> RawData:
         """Yields all distinct datasources associated to the given cards"""
         if not card_ids:
             return empty_iterator()
 
-        endpoint = self._endpoint_factory.cards_metadata(card_ids)
-        cards_metadata = self._get_element(endpoint)
-
         processed: set[str] = set()
-        for card in cards_metadata:
+        for card in self._cards_metadata(card_ids):
             for datasource in card["datasources"]:
                 id_ = datasource["dataSourceId"]
                 if id_ in processed:

castor_extractor/visualization/tableau/client/client_metadata_api.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Iterator
 from typing import Optional
 
@@ -9,15 +10,14 @@ from ..constants import DEFAULT_PAGE_SIZE
 from .errors import TableauApiError, TableauApiTimeout
 from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
+logger = logging.getLogger(__name__)
+
 # increase the value when extraction is too slow
 # decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: dict[TableauAsset, int] = {
-    # for some clients, extraction of columns tend to hit the node limit
-    # https://community.tableau.com/s/question/0D54T00000YuK60SAF/metadata-query-nodelimitexceeded-error
-    # the workaround is to reduce pagination
-    TableauAsset.COLUMN: 50,
     # fields are light but volumes are bigger
     TableauAsset.FIELD: 1000,
+    # tables are sometimes heavy
     TableauAsset.TABLE: 50,
 }
 
@@ -51,8 +51,9 @@ def _check_errors(answer: dict) -> None:
 
 def gql_query_scroll(
     server,
-    query: str,
     resource: str,
+    fields: str,
+    page_size: int,
 ) -> Iterator[SerializedAsset]:
     """
     Iterate over GQL query results, handling pagination and cursor
@@ -67,23 +68,27 @@ def gql_query_scroll(
         max_retries=_RETRY_COUNT,
         base_ms=_RETRY_BASE_MS,
     )
-    def _call(cursor: Optional[str]) -> dict:
-        # If cursor is defined it must be quoted else use null token
-        token = "null" if cursor is None else f'"{cursor}"'
-        query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
-        answer = server.metadata.query(query_)
+    def _call(first: int, offset: int) -> dict:
+        query = QUERY_TEMPLATE.format(
+            resource=resource,
+            fields=fields,
+            first=first,
+            offset=offset,
+        )
+        answer = server.metadata.query(query)
         _check_errors(answer)
         return answer["data"][f"{resource}Connection"]
 
-    cursor = None
+    current_offset = 0
     while True:
-        payload = _call(cursor)
+        payload = _call(first=page_size, offset=current_offset)
         yield payload["nodes"]
 
-        page_info = payload["pageInfo"]
-        if page_info["hasNextPage"]:
-            cursor = page_info["endCursor"]
-        else:
+        current_offset += len(payload["nodes"])
+        total = payload["totalCount"]
+        logger.info(f"Extracted {current_offset}/{total} {resource}")
+
+        if not payload["pageInfo"]["hasNextPage"]:
             break
 
 
@@ -107,12 +112,12 @@ class TableauClientMetadataApi:
         fields: str,
         page_size: int = DEFAULT_PAGE_SIZE,
     ) -> SerializedAsset:
-        query = QUERY_TEMPLATE.format(
+        result_pages = gql_query_scroll(
+            self._server,
             resource=resource,
             fields=fields,
             page_size=page_size,
         )
-        result_pages = gql_query_scroll(self._server, query, resource)
         return [asset for page in result_pages for asset in page]
 
     def _page_size(self, asset: TableauAsset) -> int:

castor_extractor/visualization/tableau/client/gql_queries.py

@@ -2,7 +2,7 @@ from ..assets import TableauAsset
 
 QUERY_TEMPLATE = """
 {{
-  {resource}Connection(first: {page_size}, after: AFTER_TOKEN_SIGNAL) {{
+  {resource}Connection(first: {first}, offset: {offset}) {{
     nodes {{ {fields}
     }}
     pageInfo {{

{castor_extractor-0.24.4.dist-info → castor_extractor-0.24.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.4
+Version: 0.24.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -35,7 +35,7 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=2,<3)
+Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -51,7 +51,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=75.6)
+Requires-Dist: setuptools (>=78.1)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -210,6 +210,18 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.24.7 - 2025-04-07
+
+* Tableau - switch from `cursor` to `offset` pagination to mitigate timeout issues
+
+## 0.24.6 - 2025-04-03
+
+* Domo - extract cards metadata by batch to prevent from hitting URL max length
+
+## 0.24.5 - 2025-04-02
+
+* bump dependencies: google-cloud-storage
+
 ## 0.24.4 - 2025-03-19
 
 * Snowflake:

{castor_extractor-0.24.4.dist-info → castor_extractor-0.24.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=1Y5FmmQDspwZaOhKjnJosP2sNd898LeTOmVIMTBt9Bw,16387
+CHANGELOG.md,sha256=UWuENqrKnLu244f4Of6dtZ59XZ7jrLWkcQni3MqXPBg,16667
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -77,9 +77,11 @@ castor_extractor/uploader/settings.py,sha256=3MvOX-UFRqrLZoiT7wYn9jUGro7NX4RCafY
 castor_extractor/uploader/upload.py,sha256=PSQfkO_7LSE0WBo9Tm_hlS2ONepKeB0cBFdJXySnues,4310
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=otAaySj5aeem6f0CTd0Te6ioJ6uP2J1p348j-SdIwDI,802
-castor_extractor/utils/__init__.py,sha256=X7WOOgrpGf7Vh8r-7eNGjuC0rKs0g9GTO3d7hZ18gwo,1550
+castor_extractor/utils/__init__.py,sha256=KQkr_CmxWG0Vpu7CaqjbJkffUeEWcyeA9Cbm394Hygk,1585
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
+castor_extractor/utils/batch.py,sha256=SFlLmJgVjV2nVhIrjVIEp8wJ9du4dKKHq8YVYubnwQQ,448
+castor_extractor/utils/batch_test.py,sha256=84JYXOxiTkZFAceVh0mzN6VtKxcqoFPbxkZfIDyLGlg,606
 castor_extractor/utils/client/__init__.py,sha256=h5gm8UNNCCkAqhjYK5f6BY7k0cHFOyAvkmlktqwpir0,392
 castor_extractor/utils/client/abstract.py,sha256=CWF7_afNpEZ3jor-22wXbKIvM20ukHkaDy_uknKz8B0,2075
 castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
@@ -146,7 +148,7 @@ castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 castor_extractor/visualization/domo/__init__.py,sha256=1axOCPm4RpdIyUt9LQEvlMvbOPllW8rk63h6EjVgJ0Y,111
 castor_extractor/visualization/domo/assets.py,sha256=bK1urFR2tnlWkVkkhR32mAKMoKbESNlop-CNGx-65PY,206
 castor_extractor/visualization/domo/client/__init__.py,sha256=Do0fU4B8Hhlhahcv734gnJl_ryCztfTBDea7XNCKfB8,72
-castor_extractor/visualization/domo/client/client.py,sha256=vOMBY5dY6N3v55YJPdh9aoiddXnuLnGoFHLE5BeUKSg,9662
+castor_extractor/visualization/domo/client/client.py,sha256=bgzXWUm-UnTIwgyJKaJkoHzQpDYwWCGCe97MsMFw6ng,9930
 castor_extractor/visualization/domo/client/credentials.py,sha256=4gnsk4Tpt3ggdUYbvyNPJEXeCyTy12s-X24P5hFdULg,873
 castor_extractor/visualization/domo/client/endpoints.py,sha256=eIE9oeZ_cmJSWWDuyxh6JaAOs3y5bTJQQ265HYgpulE,2775
 castor_extractor/visualization/domo/client/pagination.py,sha256=ukVkHVzoH4mfZ29H9YcnC2YrdVolP10wv25J6Q3ehRw,821
@@ -264,12 +266,12 @@ castor_extractor/visualization/tableau/__init__.py,sha256=eFI_1hjdkxyUiAYiy3szwy
 castor_extractor/visualization/tableau/assets.py,sha256=HbCRd8VCj1WBEeqg9jwnygnT7xOFJ6PQD7Lq7sV-XR0,635
 castor_extractor/visualization/tableau/client/__init__.py,sha256=P8RKFKOC63WkH5hdEytJOwHS9vzQ8GXreLfXZetmMP8,78
 castor_extractor/visualization/tableau/client/client.py,sha256=zzqhzIqKyJygo4ZNGk6cZh0e6Z9R1W5T0P9un52KC1M,7626
-castor_extractor/visualization/tableau/client/client_metadata_api.py,sha256=fIBsSbRTypBABsCoigO2dkKsw4Eu3GrsEPTDfjY8A80,4303
+castor_extractor/visualization/tableau/client/client_metadata_api.py,sha256=VHNV1Q0EVKuiFKm1yKSx4tIuPGww4Mlw3yui2DgKe7I,4196
 castor_extractor/visualization/tableau/client/client_rest_api.py,sha256=x4dNw4PPJdalTlGowwkANwqiS2ZhGxzpQytkHq3KbpY,3988
 castor_extractor/visualization/tableau/client/client_tsc.py,sha256=VI_PJyd1ty3HSYXHHQjshmG2ziowIbrwJRonRPCHbks,1820
 castor_extractor/visualization/tableau/client/credentials.py,sha256=uQICIgeXmLZfOroTgZt7PuKNKTyqQllRGSTcOmIfrKU,1893
 castor_extractor/visualization/tableau/client/errors.py,sha256=ecT8Tit5VtzrOBB9ykblA0nvd75j5-_QDFupjV48zJQ,300
-castor_extractor/visualization/tableau/client/gql_queries.py,sha256=NISarYh33Ij7DhYxqjTdv681AHYpbft8kPwVUQbAZ7U,2190
+castor_extractor/visualization/tableau/client/gql_queries.py,sha256=XJAfhpMZ5S7-AhfpOaoHMHCAdil-l5e5xB-CH4NC38M,2177
 castor_extractor/visualization/tableau/client/rest_fields.py,sha256=ZKYYuMxg9PXhczVXaD4rXNk7dYyWJ1_bVM8FLEXju7s,888
 castor_extractor/visualization/tableau/constants.py,sha256=lHGB50FgVNO2nXeIhkvQKivD8ZFBIjDrflgD5cTXKJw,104
 castor_extractor/visualization/tableau/extract.py,sha256=FnjmmUdNA9MEf3S5Tw37x6ZXxVsK8R3YnVk1UVYbaZk,1423
@@ -403,8 +405,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.4.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.4.dist-info/METADATA,sha256=eY2TPP3IDq9an2JJzoZcN-_rG5DJIGzbJOqEtGBhzd4,23543
-castor_extractor-0.24.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.4.dist-info/entry_points.txt,sha256=FQNShG4w4nRO95_bZnagh7FQ2oiZ-40bdt8ZdTW1-uI,1731
-castor_extractor-0.24.4.dist-info/RECORD,,
+castor_extractor-0.24.7.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.7.dist-info/METADATA,sha256=qWp3OBv1FO123RJqz2YKTEd12WzhKoDmcxVZLhvzn6M,23831
+castor_extractor-0.24.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.7.dist-info/entry_points.txt,sha256=FQNShG4w4nRO95_bZnagh7FQ2oiZ-40bdt8ZdTW1-uI,1731
+castor_extractor-0.24.7.dist-info/RECORD,,