castor-extractor 0.10.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.10.1 - 2023-12-04
+
+* Domo: fix pagination
+
 ## 0.10.0 - 2023-11-28
 
 * Looker : extract all Looker Explores, even if unused in Dashboards

castor_extractor/visualization/domo/client/client.py

@@ -128,7 +128,10 @@ class DomoClient:
         while pagination.needs_increment:
             results = self._get_many(
                 endpoint=endpoint,
-                params={"offset": pagination.offset},
+                params={
+                    "offset": pagination.offset,
+                    "limit": pagination.per_page,
+                },
             )
             all_results.extend(results)
             number_of_items = len(results)

castor_extractor/visualization/domo/client/pagination.py

@@ -1,23 +1,28 @@
 from dataclasses import dataclass
+from typing import Optional
 
-PER_PAGE = 50
+PER_PAGE = 50  # maximum value accepted by DOMO is 50
 
 
 @dataclass
 class Pagination:
     """Handles pagination within DOMO Api"""
 
-    number_results: int = PER_PAGE  # max init
+    number_results: Optional[int] = None
     offset: int = 0
     per_page: int = PER_PAGE
     should_stop: bool = False
 
     @property
     def needs_increment(self) -> bool:
+        if self.number_results is None:
+            return True  # first iteration
+
         if (self.number_results < self.per_page) or self.should_stop:
             return False
+
         return True
 
     def increment_offset(self, number_results: int) -> None:
-        self.offset += self.per_page
+        self.offset += number_results
         self.number_results = number_results

castor_extractor/visualization/domo/client/pagination_test.py

@@ -0,0 +1,22 @@
+from .pagination import Pagination
+
+
+def test_pagination():
+    per_page = 20
+
+    pagination = Pagination(per_page=per_page)
+
+    assert pagination.number_results is None
+    assert pagination.offset == 0
+
+    pagination.increment_offset(per_page)
+    assert pagination.offset == per_page
+    assert pagination.needs_increment
+
+    pagination.increment_offset(per_page)
+    assert pagination.offset == per_page * 2
+    assert pagination.needs_increment
+
+    pagination.increment_offset(5)
+    assert pagination.offset == per_page * 2 + 5
+    assert not pagination.needs_increment

castor_extractor/warehouse/bigquery/client.py

@@ -1,8 +1,12 @@
 from typing import List, Optional, Set, Tuple
 
+from google.api_core.exceptions import Forbidden
+from google.api_core.page_iterator import Iterator as PageIterator
 from google.cloud.bigquery import Client as GoogleCloudClient  # type: ignore
+from google.cloud.bigquery.dataset import Dataset  # type: ignore
 from google.oauth2.service_account import Credentials  # type: ignore
 
+from ...utils import retry
 from ..abstract import SqlalchemyClient
 
 BIGQUERY_URI = "bigquery://"
@@ -10,6 +14,9 @@ BIGQUERY_URI = "bigquery://"
 CREDENTIALS_INFO_KEY = "credentials_info"
 PROJECT_ID_KEY = "project_id"
 
+_RETRY_NUMBER = 1
+_RETRY_BASE_MS = 60_000
+
 
 class BigQueryClient(SqlalchemyClient):
     """Connect to BigQuery and run SQL queries"""
@@ -25,6 +32,9 @@ class BigQueryClient(SqlalchemyClient):
         self._db_allowed = db_allowed
         self._db_blocked = db_blocked
         self._dataset_blocked = dataset_blocked
+        self.client = self._client()
+        self._projects: List[str] | None = None
+        self._datasets: List[Dataset] | None = None
 
     @staticmethod
     def name() -> str:
@@ -51,7 +61,7 @@ class BigQueryClient(SqlalchemyClient):
     def _build_uri(self, credentials: dict) -> str:
         return BIGQUERY_URI
 
-    def _google_cloud_client(self) -> GoogleCloudClient:
+    def _client(self) -> GoogleCloudClient:
         assert (
             CREDENTIALS_INFO_KEY in self._options
         ), "Missing BigQuery credentials in engine's options"
@@ -61,25 +71,47 @@ class BigQueryClient(SqlalchemyClient):
             credentials=Credentials.from_service_account_info(credentials),
         )
 
-    def _list_datasets(self) -> List:
-        client = self._google_cloud_client()
-        return [
-            dataset
-            for project_id in self.get_projects()
-            for dataset in client.list_datasets(project_id)
-            if self._keep_dataset(dataset.dataset_id)
-        ]
+    def _list_datasets(self) -> List[Dataset]:
+        """
+        Returns datasets available for the given GCP client
+        Cache the result in self._datasets to reduce number of API calls
+        """
+        if self._datasets is None:
+            self._datasets = [
+                dataset
+                for project_id in self.get_projects()
+                for dataset in self.client.list_datasets(project_id)
+                if self._keep_dataset(dataset.dataset_id)
+            ]
+        return self._datasets
+
+    @retry((Forbidden,), count=_RETRY_NUMBER, base_ms=_RETRY_BASE_MS)
+    def _list_projects(self) -> PageIterator:
+        """
+        Note: Calling list_projects from GoogleCloudClient causes some
+        ```
+        google.api_core.exceptions.Forbidden: 403 GET https://bigquery.googleapis.com/bigquery/v2/projects?prettyPrint=false
+        Quota exceeded: Your user exceeded quota for concurrent project.lists requests.
+        ````
+
+        This function aims to isolate the call with a custom retry strategy.
+        Note that google allows a retry parameter on client.list_projects but
+        that looks way too complex to customize.
+        """
+        return self.client.list_projects()
 
     def get_projects(self) -> List[str]:
         """
         Returns distinct project_id available for the given GCP client
+        Cache the result in self._projects to reduce number of API calls
         """
-        client = self._google_cloud_client()
-        return [
-            p.project_id
-            for p in client.list_projects()
-            if self._keep_project(p.project_id)
-        ]
+        if self._projects is None:
+            self._projects = [
+                p.project_id
+                for p in self._list_projects()
+                if self._keep_project(p.project_id)
+            ]
+        return self._projects
 
     def get_regions(self) -> Set[Tuple[str, str]]:
         """

castor_extractor/warehouse/bigquery/client_test.py

@@ -26,8 +26,11 @@ class MockBigQueryClient(BigQueryClient):
         self._db_allowed = ["project_2", "project_1"]
         self._dataset_blocked = ["hidden_dataset"]
         self._db_blocked = ["hidden_project"]
+        self._projects = None
+        self._datasets = None
+        self.client = self._client()
 
-    def _google_cloud_client(self) -> Mock:
+    def _client(self) -> Mock:
         fake_client = Mock()
         fake_client.list_projects = Mock(
             return_value=[

{castor_extractor-0.10.0.dist-info → castor_extractor-0.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.10.0
+Version: 0.10.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.10.0.dist-info → castor_extractor-0.10.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=-SDIu5XMRtiE4Pp7-F8aCJteadjcjuxDwdvRIjKefBw,7349
+CHANGELOG.md,sha256=YHDD-wLJh1LKqODJfg07WPSIj6pPmkI8F0za9PzKFiw,7397
 Dockerfile,sha256=TC6hFjG3mvnt1nkw2EpaS42hRYaGA2YIPKgWhVSKTWc,303
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=EL6JpZxvaQFOYv5WFuSjZvSk9Hcpsf7alMlUC5IPFjA,3423
@@ -82,11 +82,12 @@ castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 castor_extractor/visualization/domo/__init__.py,sha256=_mAYVfoVLizfLGF_f6ZiwBhdPpvoJY_diySf33dt3Jo,127
 castor_extractor/visualization/domo/assets.py,sha256=JI45W7n5z_opbuRrCwgzQJuN-VikuRrilUj5g5lX7Hk,184
 castor_extractor/visualization/domo/client/__init__.py,sha256=UDszV3IXNC9Wp_j55NZ-6ey2INo0TYtAg2QNIJOjglE,88
-castor_extractor/visualization/domo/client/client.py,sha256=QbbvkCgCzNg1SH0W5oE2NWaIBfQypAn07dV2BaLQX-o,8994
+castor_extractor/visualization/domo/client/client.py,sha256=MN-qleOpSVawZRlVL8EiglEOMj0cdvMqEbgYYBfz86w,9083
 castor_extractor/visualization/domo/client/client_test.py,sha256=5Z_C2B0fs60aGnMF78llBnRkzehH4tZqRPXPkccLvBM,583
 castor_extractor/visualization/domo/client/credentials.py,sha256=CksQ9W9X6IGjTlYN0okwGAmURMRJKAjctxODAvAJUAo,1148
 castor_extractor/visualization/domo/client/endpoints.py,sha256=-B7mRKJ44Bg0hb3E5dQXvCVK6qHzizdeSQXsSwEJEIY,1812
-castor_extractor/visualization/domo/client/pagination.py,sha256=BT9ZIb-GYGWEQZpK_80aAYKQ-xrr7VlfsFbZv9rcejQ,565
+castor_extractor/visualization/domo/client/pagination.py,sha256=E3WMK9Uw-u5qt9LCUzwKdKh9oSzyFEC0GgnRMFgxgrs,713
+castor_extractor/visualization/domo/client/pagination_test.py,sha256=nV4yZWfus13QFCr-tlBUgwva21VqfpF6P-0ks_Awwis,581
 castor_extractor/visualization/domo/constants.py,sha256=AriJZPrCY5Z3HRUANrMu-4U0b7hQK_jRDcxiB-hbrQ4,233
 castor_extractor/visualization/domo/extract.py,sha256=GWWRfPEMt4SgzBGFaTcoOabsoOqLRFIEFAtgXwb8LDI,2567
 castor_extractor/visualization/looker/__init__.py,sha256=Xu5bJ3743kaP8szMMp2NXCgvM1EdOQgtic4utUlO9Cc,145
@@ -221,8 +222,8 @@ castor_extractor/warehouse/abstract/query.py,sha256=GAgeISCmAdrkTKzFGO79hQDf6SA6
 castor_extractor/warehouse/abstract/time_filter.py,sha256=bggIONfMmUxffkA6TwM3BsjfS2l9WFxPq8krfsau5pw,935
 castor_extractor/warehouse/abstract/time_filter_test.py,sha256=PIkegB7KOKBdpc6zIvmyl_CeQyADeFDplyQ8HTNU5LA,448
 castor_extractor/warehouse/bigquery/__init__.py,sha256=cQTw-nkKg3CQemgufknJHPssvUWCPURTMVb6q5hiIcs,125
-castor_extractor/warehouse/bigquery/client.py,sha256=EHwTIoKR6P-W-_rGT5mYGXHdia8d7XjlxwstTVNQYmY,3062
-castor_extractor/warehouse/bigquery/client_test.py,sha256=ws7JMYW_-7Ewo7XLHLibuOkemfTN87goZwYc-QufCMY,1430
+castor_extractor/warehouse/bigquery/client.py,sha256=hxz8nYx8O60Ve48hDdQ0vEJEEpRgXdMXos7kJj4fI3w,4483
+castor_extractor/warehouse/bigquery/client_test.py,sha256=Ym8e4d--0YQwiVcNUnXLx0X-X6ZznwNMBMbMaDS5oEA,1514
 castor_extractor/warehouse/bigquery/extract.py,sha256=jo_9sxsCFl0ZaL1VdQ9JZ5iEEGJQVm_ogJhfpmVgT3k,2810
 castor_extractor/warehouse/bigquery/queries/.sqlfluff,sha256=ce8UDW2k39v6RBVxgKqjOHHYMoGN9S9f7BCZNHHhox8,30
 castor_extractor/warehouse/bigquery/queries/column.sql,sha256=NxdTnHwomHTEGSc-UoXFKUwg59I9XAOwrSau7JUqGQE,1815
@@ -289,7 +290,7 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.10.0.dist-info/METADATA,sha256=XnrJ7dly9vVplF9ksw1thIrfr3Y215iEPCJC4nfs_-A,5880
-castor_extractor-0.10.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-castor_extractor-0.10.0.dist-info/entry_points.txt,sha256=cvLvgE8Yi10sIiafUVL86XZPMUUyu9x11CF5PshAyiw,1045
-castor_extractor-0.10.0.dist-info/RECORD,,
+castor_extractor-0.10.1.dist-info/METADATA,sha256=5lkBM0b76v9IbuKEBQ9lnq5vCeuP0Aqa0JyizNnbMc8,5880
+castor_extractor-0.10.1.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+castor_extractor-0.10.1.dist-info/entry_points.txt,sha256=cvLvgE8Yi10sIiafUVL86XZPMUUyu9x11CF5PshAyiw,1045
+castor_extractor-0.10.1.dist-info/RECORD,,