castor-extractor 0.9.2__tar.gz → 0.10.1__tar.gz

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.10.1 - 2023-12-04
+
+* Domo: fix pagination
+
+## 0.10.0 - 2023-11-28
+
+* Looker : extract all Looker Explores, even if unused in Dashboards
+
 ## 0.9.2 - 2023-11-23
 
 * Looker : remove deprecated all_looks parameter

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.9.2
+Version: 0.10.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/visualization/domo/client/client.py

@@ -128,7 +128,10 @@ class DomoClient:
         while pagination.needs_increment:
             results = self._get_many(
                 endpoint=endpoint,
-                params={"offset": pagination.offset},
+                params={
+                    "offset": pagination.offset,
+                    "limit": pagination.per_page,
+                },
             )
             all_results.extend(results)
             number_of_items = len(results)

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/visualization/domo/client/pagination.py

@@ -1,23 +1,28 @@
 from dataclasses import dataclass
+from typing import Optional
 
-PER_PAGE = 50
+PER_PAGE = 50  # maximum value accepted by DOMO is 50
 
 
 @dataclass
 class Pagination:
     """Handles pagination within DOMO Api"""
 
-    number_results: int = PER_PAGE  # max init
+    number_results: Optional[int] = None
     offset: int = 0
     per_page: int = PER_PAGE
     should_stop: bool = False
 
     @property
     def needs_increment(self) -> bool:
+        if self.number_results is None:
+            return True  # first iteration
+
         if (self.number_results < self.per_page) or self.should_stop:
             return False
+
         return True
 
     def increment_offset(self, number_results: int) -> None:
-        self.offset += self.per_page
+        self.offset += number_results
         self.number_results = number_results

castor_extractor-0.10.1/castor_extractor/visualization/domo/client/pagination_test.py

@@ -0,0 +1,22 @@
+from .pagination import Pagination
+
+
+def test_pagination():
+    per_page = 20
+
+    pagination = Pagination(per_page=per_page)
+
+    assert pagination.number_results is None
+    assert pagination.offset == 0
+
+    pagination.increment_offset(per_page)
+    assert pagination.offset == per_page
+    assert pagination.needs_increment
+
+    pagination.increment_offset(per_page)
+    assert pagination.offset == per_page * 2
+    assert pagination.needs_increment
+
+    pagination.increment_offset(5)
+    assert pagination.offset == per_page * 2 + 5
+    assert not pagination.needs_increment

castor_extractor-0.10.1/castor_extractor/visualization/looker/__init__.py

@@ -0,0 +1,3 @@
+from .api import ApiClient, Credentials, lookml_explore_names
+from .assets import LookerAsset
+from .extract import extract_all, iterate_all_data

castor_extractor-0.10.1/castor_extractor/visualization/looker/api/__init__.py

@@ -0,0 +1,3 @@
+from .client import ApiClient
+from .sdk import Credentials
+from .utils import lookml_explore_names

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/visualization/looker/api/client.py

@@ -191,7 +191,7 @@ class ApiClient:
     def explores(
         self,
         explore_names=Iterator[Tuple[str, str]],
-    ) -> List[LookmlModelExplore]:
+    ) -> Iterator[LookmlModelExplore]:
         """Iterates explores of the given Looker account for the provided model/explore names"""
 
         @safe_mode(self._safe_mode)
@@ -202,11 +202,10 @@ class ApiClient:
             self._on_api_call()
             return explore
 
-        explores = [
-            _call(model_name, explore_name)
-            for model_name, explore_name in explore_names
-        ]
-        return list(filter(None, explores))
+        for lookml_model_name, lookml_explore_name_ in explore_names:
+            explore_ = _call(lookml_model_name, lookml_explore_name_)
+            if explore_ is not None:
+                yield explore_
 
     def connections(self) -> List[DBConnection]:
         """Lists databases connections of the given Looker account"""

castor_extractor-0.10.1/castor_extractor/visualization/looker/api/utils.py

@@ -0,0 +1,24 @@
+from typing import Iterable, Set, Tuple
+
+from .sdk import LookmlModel
+
+
+def lookml_explore_names(
+    lookmls: Iterable[LookmlModel],
+) -> Set[Tuple[str, str]]:
+    """
+    Explores from the lookml models
+    Only valid explores are yielded: with all infos
+    """
+    model_explores = (
+        (model, explore)
+        for model in lookmls
+        for explore in model.explores or []
+    )
+
+    return {
+        (model.name, explore.name)
+        for model, explore in model_explores
+        # accept hidden resources
+        if model.name and explore.name
+    }

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/visualization/looker/extract.py

@@ -11,12 +11,8 @@ from ...utils import (
     write_json,
     write_summary,
 )
-from .api import (
-    ApiClient,
-    Credentials,
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-)
+from .api import ApiClient, Credentials, lookml_explore_names
+from .api.sdk import LookmlModel
 from .assets import LookerAsset
 from .multithreading import MultithreadingFetcher
 from .parameters import get_parameters
@@ -24,6 +20,15 @@ from .parameters import get_parameters
 logger = logging.getLogger(__name__)
 
 
+def _extract_explores_by_name(
+    lookmls: Iterable[LookmlModel], client: ApiClient
+) -> Iterable[dict]:
+    explore_names = lookml_explore_names(lookmls)
+    explores = client.explores(explore_names)
+    for explore in explores:
+        yield deep_serialize(explore)  # type: ignore
+
+
 def _safe_mode(directory: str) -> SafeMode:
     add_logging_file_handler(directory)
     return SafeMode((Exception,), float("inf"))
@@ -79,10 +84,8 @@ def iterate_all_data(
     if search_per_folder:
         dashboards_stream = fetcher.fetch_assets(LookerAsset.DASHBOARDS)
         yield LookerAsset.DASHBOARDS, StreamableList(dashboards_stream)
-        dashboard_explore_names_ = fetcher.explores
     else:
         dashboards = client.dashboards()
-        dashboard_explore_names_ = dashboard_explore_names(dashboards)
         yield LookerAsset.DASHBOARDS, deep_serialize(dashboards)
 
     logger.info("Extracting lookml models from Looker API")
@@ -90,10 +93,8 @@ def iterate_all_data(
     yield LookerAsset.LOOKML_MODELS, deep_serialize(lookmls)
 
     logger.info("Extracting explores from Looker API")
-    explore_names = explore_names_associated_to_dashboards(
-        lookmls, dashboard_explore_names_
-    )
-    yield LookerAsset.EXPLORES, deep_serialize(client.explores(explore_names))
+    explores = _extract_explores_by_name(lookmls, client)
+    yield LookerAsset.EXPLORES, StreamableList(explores)
     del lookmls
 
     logger.info("Extracting connections from Looker API")

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/visualization/looker/multithreading.py

@@ -2,13 +2,13 @@ import logging
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import Iterable, List, Set, Tuple
+from typing import Iterable, List, Set
 
 from tqdm import tqdm  # type: ignore
 
 from ...utils import RetryStrategy, deep_serialize, retry
-from . import ApiClient, dashboard_explore_names
-from .api.sdk import Dashboard, SDKError
+from . import ApiClient
+from .api.sdk import SDKError
 from .assets import LookerAsset
 
 logger = logging.getLogger(__name__)
@@ -54,16 +54,6 @@ class MultithreadingFetcher:
         self._thread_pool_size = thread_pool_size
         self._log_to_stdout = log_to_stdout
 
-        self.explores: Set[Tuple[str, str]] = set()
-
-    def _save_explore_names(self, dashboards_per_folder: Iterable[Dashboard]):
-        """
-        Since dashboards are streamed right to the file, we need to keep
-        the relevant information to extract Explores later.
-        """
-        explores = dashboard_explore_names(dashboards_per_folder)
-        self.explores.update(explores)
-
     def _progress_bar(self, fetch_results: Iterable, total: int) -> tqdm:
         """Create a tqdm progress bar with the appropriate logs destination"""
         file = sys.stderr
@@ -73,7 +63,7 @@ class MultithreadingFetcher:
 
         return tqdm(fetch_results, total=total, file=file)
 
-    def fetch_assets(self, asset: LookerAsset):
+    def fetch_assets(self, asset: LookerAsset) -> Iterable[dict]:
         """
         Yields serialized Looks or Dashboards with a request per folder ID.
         Requests are parallelised.
@@ -88,12 +78,10 @@ class MultithreadingFetcher:
             fetch_results = executor.map(_fetch, self._folder_ids)
 
             for results in self._progress_bar(fetch_results, total_folders):
-                if asset == LookerAsset.DASHBOARDS:
-                    self._save_explore_names(results)
-
                 for result in results:
                     if not result:
                         continue
+
                     total_assets_count += len(result)
                     yield deep_serialize(result)
 

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/warehouse/bigquery/client.py

@@ -1,8 +1,12 @@
 from typing import List, Optional, Set, Tuple
 
+from google.api_core.exceptions import Forbidden
+from google.api_core.page_iterator import Iterator as PageIterator
 from google.cloud.bigquery import Client as GoogleCloudClient  # type: ignore
+from google.cloud.bigquery.dataset import Dataset  # type: ignore
 from google.oauth2.service_account import Credentials  # type: ignore
 
+from ...utils import retry
 from ..abstract import SqlalchemyClient
 
 BIGQUERY_URI = "bigquery://"
@@ -10,6 +14,9 @@ BIGQUERY_URI = "bigquery://"
 CREDENTIALS_INFO_KEY = "credentials_info"
 PROJECT_ID_KEY = "project_id"
 
+_RETRY_NUMBER = 1
+_RETRY_BASE_MS = 60_000
+
 
 class BigQueryClient(SqlalchemyClient):
     """Connect to BigQuery and run SQL queries"""
@@ -25,6 +32,9 @@ class BigQueryClient(SqlalchemyClient):
         self._db_allowed = db_allowed
         self._db_blocked = db_blocked
         self._dataset_blocked = dataset_blocked
+        self.client = self._client()
+        self._projects: List[str] | None = None
+        self._datasets: List[Dataset] | None = None
 
     @staticmethod
     def name() -> str:
@@ -51,7 +61,7 @@ class BigQueryClient(SqlalchemyClient):
     def _build_uri(self, credentials: dict) -> str:
         return BIGQUERY_URI
 
-    def _google_cloud_client(self) -> GoogleCloudClient:
+    def _client(self) -> GoogleCloudClient:
         assert (
             CREDENTIALS_INFO_KEY in self._options
         ), "Missing BigQuery credentials in engine's options"
@@ -61,25 +71,47 @@ class BigQueryClient(SqlalchemyClient):
             credentials=Credentials.from_service_account_info(credentials),
         )
 
-    def _list_datasets(self) -> List:
-        client = self._google_cloud_client()
-        return [
-            dataset
-            for project_id in self.get_projects()
-            for dataset in client.list_datasets(project_id)
-            if self._keep_dataset(dataset.dataset_id)
-        ]
+    def _list_datasets(self) -> List[Dataset]:
+        """
+        Returns datasets available for the given GCP client
+        Cache the result in self._datasets to reduce number of API calls
+        """
+        if self._datasets is None:
+            self._datasets = [
+                dataset
+                for project_id in self.get_projects()
+                for dataset in self.client.list_datasets(project_id)
+                if self._keep_dataset(dataset.dataset_id)
+            ]
+        return self._datasets
+
+    @retry((Forbidden,), count=_RETRY_NUMBER, base_ms=_RETRY_BASE_MS)
+    def _list_projects(self) -> PageIterator:
+        """
+        Note: Calling list_projects from GoogleCloudClient causes some
+        ```
+        google.api_core.exceptions.Forbidden: 403 GET https://bigquery.googleapis.com/bigquery/v2/projects?prettyPrint=false
+        Quota exceeded: Your user exceeded quota for concurrent project.lists requests.
+        ````
+
+        This function aims to isolate the call with a custom retry strategy.
+        Note that google allows a retry parameter on client.list_projects but
+        that looks way too complex to customize.
+        """
+        return self.client.list_projects()
 
     def get_projects(self) -> List[str]:
         """
         Returns distinct project_id available for the given GCP client
+        Cache the result in self._projects to reduce number of API calls
         """
-        client = self._google_cloud_client()
-        return [
-            p.project_id
-            for p in client.list_projects()
-            if self._keep_project(p.project_id)
-        ]
+        if self._projects is None:
+            self._projects = [
+                p.project_id
+                for p in self._list_projects()
+                if self._keep_project(p.project_id)
+            ]
+        return self._projects
 
     def get_regions(self) -> Set[Tuple[str, str]]:
         """

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/castor_extractor/warehouse/bigquery/client_test.py

@@ -26,8 +26,11 @@ class MockBigQueryClient(BigQueryClient):
         self._db_allowed = ["project_2", "project_1"]
         self._dataset_blocked = ["hidden_dataset"]
         self._db_blocked = ["hidden_project"]
+        self._projects = None
+        self._datasets = None
+        self.client = self._client()
 
-    def _google_cloud_client(self) -> Mock:
+    def _client(self) -> Mock:
         fake_client = Mock()
         fake_client.list_projects = Mock(
             return_value=[

{castor_extractor-0.9.2 → castor_extractor-0.10.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.9.2"
+version = "0.10.1"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.9.2/castor_extractor/visualization/looker/__init__.py

@@ -1,9 +0,0 @@
-from .api import (
-    ApiClient,
-    Credentials,
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-    lookml_explore_names,
-)
-from .assets import LookerAsset
-from .extract import extract_all, iterate_all_data

castor_extractor-0.9.2/castor_extractor/visualization/looker/api/__init__.py

@@ -1,7 +0,0 @@
-from .client import ApiClient
-from .sdk import Credentials
-from .utils import (
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-    lookml_explore_names,
-)

castor_extractor-0.9.2/castor_extractor/visualization/looker/api/utils.py

@@ -1,49 +0,0 @@
-from typing import Iterable, Set, Tuple
-
-from .sdk import Dashboard, LookmlModel
-
-
-def lookml_explore_names(
-    lookmls: Iterable[LookmlModel],
-) -> Set[Tuple[str, str]]:
-    """
-    Explores from the lookml models
-    Only valid explores are yielded: with all infos
-    """
-    model_explores = (
-        (model, explore)
-        for model in lookmls
-        for explore in model.explores or []
-    )
-
-    return {
-        (model.name, explore.name)
-        for model, explore in model_explores
-        # accept hidden resources
-        if model.name and explore.name
-    }
-
-
-def dashboard_explore_names(
-    dashboards: Iterable[Dashboard],
-) -> Set[Tuple[str, str]]:
-    """Explores that appear in dashboards"""
-    elements = (
-        element
-        for dashboard in dashboards
-        for element in dashboard.dashboard_elements or []
-    )
-
-    return {
-        (element.query.model, element.query.view)
-        for element in elements
-        if element.query and element.query.model and element.query.view
-    }
-
-
-def explore_names_associated_to_dashboards(
-    lookmls: Iterable[LookmlModel],
-    dashboard_explore_names_: Set[Tuple[str, str]],
-):
-    """Retrieve only explores that are associated to a looker dashboard"""
-    return lookml_explore_names(lookmls).intersection(dashboard_explore_names_)