castor-extractor 0.9.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.10.0 - 2023-11-28
+
+* Looker : extract all Looker Explores, even if unused in Dashboards
+
 ## 0.9.2 - 2023-11-23
 
 * Looker : remove deprecated all_looks parameter

castor_extractor/visualization/looker/__init__.py

@@ -1,9 +1,3 @@
-from .api import (
-    ApiClient,
-    Credentials,
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-    lookml_explore_names,
-)
+from .api import ApiClient, Credentials, lookml_explore_names
 from .assets import LookerAsset
 from .extract import extract_all, iterate_all_data

castor_extractor/visualization/looker/api/__init__.py

@@ -1,7 +1,3 @@
 from .client import ApiClient
 from .sdk import Credentials
-from .utils import (
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-    lookml_explore_names,
-)
+from .utils import lookml_explore_names

castor_extractor/visualization/looker/api/client.py

@@ -191,7 +191,7 @@ class ApiClient:
     def explores(
         self,
         explore_names=Iterator[Tuple[str, str]],
-    ) -> List[LookmlModelExplore]:
+    ) -> Iterator[LookmlModelExplore]:
         """Iterates explores of the given Looker account for the provided model/explore names"""
 
         @safe_mode(self._safe_mode)
@@ -202,11 +202,10 @@ class ApiClient:
             self._on_api_call()
             return explore
 
-        explores = [
-            _call(model_name, explore_name)
-            for model_name, explore_name in explore_names
-        ]
-        return list(filter(None, explores))
+        for lookml_model_name, lookml_explore_name_ in explore_names:
+            explore_ = _call(lookml_model_name, lookml_explore_name_)
+            if explore_ is not None:
+                yield explore_
 
     def connections(self) -> List[DBConnection]:
         """Lists databases connections of the given Looker account"""

castor_extractor/visualization/looker/api/utils.py

@@ -1,6 +1,6 @@
 from typing import Iterable, Set, Tuple
 
-from .sdk import Dashboard, LookmlModel
+from .sdk import LookmlModel
 
 
 def lookml_explore_names(
@@ -22,28 +22,3 @@ def lookml_explore_names(
         # accept hidden resources
         if model.name and explore.name
     }
-
-
-def dashboard_explore_names(
-    dashboards: Iterable[Dashboard],
-) -> Set[Tuple[str, str]]:
-    """Explores that appear in dashboards"""
-    elements = (
-        element
-        for dashboard in dashboards
-        for element in dashboard.dashboard_elements or []
-    )
-
-    return {
-        (element.query.model, element.query.view)
-        for element in elements
-        if element.query and element.query.model and element.query.view
-    }
-
-
-def explore_names_associated_to_dashboards(
-    lookmls: Iterable[LookmlModel],
-    dashboard_explore_names_: Set[Tuple[str, str]],
-):
-    """Retrieve only explores that are associated to a looker dashboard"""
-    return lookml_explore_names(lookmls).intersection(dashboard_explore_names_)

castor_extractor/visualization/looker/extract.py

@@ -11,12 +11,8 @@ from ...utils import (
     write_json,
     write_summary,
 )
-from .api import (
-    ApiClient,
-    Credentials,
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-)
+from .api import ApiClient, Credentials, lookml_explore_names
+from .api.sdk import LookmlModel
 from .assets import LookerAsset
 from .multithreading import MultithreadingFetcher
 from .parameters import get_parameters
@@ -24,6 +20,15 @@ from .parameters import get_parameters
 logger = logging.getLogger(__name__)
 
 
+def _extract_explores_by_name(
+    lookmls: Iterable[LookmlModel], client: ApiClient
+) -> Iterable[dict]:
+    explore_names = lookml_explore_names(lookmls)
+    explores = client.explores(explore_names)
+    for explore in explores:
+        yield deep_serialize(explore)  # type: ignore
+
+
 def _safe_mode(directory: str) -> SafeMode:
     add_logging_file_handler(directory)
     return SafeMode((Exception,), float("inf"))
@@ -79,10 +84,8 @@ def iterate_all_data(
     if search_per_folder:
         dashboards_stream = fetcher.fetch_assets(LookerAsset.DASHBOARDS)
         yield LookerAsset.DASHBOARDS, StreamableList(dashboards_stream)
-        dashboard_explore_names_ = fetcher.explores
     else:
         dashboards = client.dashboards()
-        dashboard_explore_names_ = dashboard_explore_names(dashboards)
         yield LookerAsset.DASHBOARDS, deep_serialize(dashboards)
 
     logger.info("Extracting lookml models from Looker API")
@@ -90,10 +93,8 @@ def iterate_all_data(
     yield LookerAsset.LOOKML_MODELS, deep_serialize(lookmls)
 
     logger.info("Extracting explores from Looker API")
-    explore_names = explore_names_associated_to_dashboards(
-        lookmls, dashboard_explore_names_
-    )
-    yield LookerAsset.EXPLORES, deep_serialize(client.explores(explore_names))
+    explores = _extract_explores_by_name(lookmls, client)
+    yield LookerAsset.EXPLORES, StreamableList(explores)
     del lookmls
 
     logger.info("Extracting connections from Looker API")

castor_extractor/visualization/looker/multithreading.py

@@ -2,13 +2,13 @@ import logging
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import Iterable, List, Set, Tuple
+from typing import Iterable, List, Set
 
 from tqdm import tqdm  # type: ignore
 
 from ...utils import RetryStrategy, deep_serialize, retry
-from . import ApiClient, dashboard_explore_names
-from .api.sdk import Dashboard, SDKError
+from . import ApiClient
+from .api.sdk import SDKError
 from .assets import LookerAsset
 
 logger = logging.getLogger(__name__)
@@ -54,16 +54,6 @@ class MultithreadingFetcher:
         self._thread_pool_size = thread_pool_size
         self._log_to_stdout = log_to_stdout
 
-        self.explores: Set[Tuple[str, str]] = set()
-
-    def _save_explore_names(self, dashboards_per_folder: Iterable[Dashboard]):
-        """
-        Since dashboards are streamed right to the file, we need to keep
-        the relevant information to extract Explores later.
-        """
-        explores = dashboard_explore_names(dashboards_per_folder)
-        self.explores.update(explores)
-
     def _progress_bar(self, fetch_results: Iterable, total: int) -> tqdm:
         """Create a tqdm progress bar with the appropriate logs destination"""
         file = sys.stderr
@@ -73,7 +63,7 @@ class MultithreadingFetcher:
 
         return tqdm(fetch_results, total=total, file=file)
 
-    def fetch_assets(self, asset: LookerAsset):
+    def fetch_assets(self, asset: LookerAsset) -> Iterable[dict]:
         """
         Yields serialized Looks or Dashboards with a request per folder ID.
         Requests are parallelised.
@@ -88,12 +78,10 @@ class MultithreadingFetcher:
             fetch_results = executor.map(_fetch, self._folder_ids)
 
             for results in self._progress_bar(fetch_results, total_folders):
-                if asset == LookerAsset.DASHBOARDS:
-                    self._save_explore_names(results)
-
                 for result in results:
                     if not result:
                         continue
+
                     total_assets_count += len(result)
                     yield deep_serialize(result)
 

{castor_extractor-0.9.2.dist-info → castor_extractor-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.9.2
+Version: 0.10.0
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.9.2.dist-info → castor_extractor-0.10.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=t0iqGADIz1dmWKroLXNSYADYg84v9eM08zBKEWItKvk,7255
+CHANGELOG.md,sha256=-SDIu5XMRtiE4Pp7-F8aCJteadjcjuxDwdvRIjKefBw,7349
 Dockerfile,sha256=TC6hFjG3mvnt1nkw2EpaS42hRYaGA2YIPKgWhVSKTWc,303
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=EL6JpZxvaQFOYv5WFuSjZvSk9Hcpsf7alMlUC5IPFjA,3423
@@ -89,21 +89,21 @@ castor_extractor/visualization/domo/client/endpoints.py,sha256=-B7mRKJ44Bg0hb3E5
 castor_extractor/visualization/domo/client/pagination.py,sha256=BT9ZIb-GYGWEQZpK_80aAYKQ-xrr7VlfsFbZv9rcejQ,565
 castor_extractor/visualization/domo/constants.py,sha256=AriJZPrCY5Z3HRUANrMu-4U0b7hQK_jRDcxiB-hbrQ4,233
 castor_extractor/visualization/domo/extract.py,sha256=GWWRfPEMt4SgzBGFaTcoOabsoOqLRFIEFAtgXwb8LDI,2567
-castor_extractor/visualization/looker/__init__.py,sha256=nUPT7ojaYE56Xk9LCCquRfuCepBIdbe-GwGbZNO7oXo,235
-castor_extractor/visualization/looker/api/__init__.py,sha256=eMtzoMAE7ZnvfwJJAOh9Cooztj4Juzc40wSEHhIYXw4,181
-castor_extractor/visualization/looker/api/client.py,sha256=KcdvbWWE-y1_vYixspxSkBHNoI_oX8Aeie5Tpz4EXj4,9547
+castor_extractor/visualization/looker/__init__.py,sha256=Xu5bJ3743kaP8szMMp2NXCgvM1EdOQgtic4utUlO9Cc,145
+castor_extractor/visualization/looker/api/__init__.py,sha256=rN03VMucxIqc0yfd17dIe3ZNFpcg5CA09epn1fKJg90,99
+castor_extractor/visualization/looker/api/client.py,sha256=R-hnVK1TYaCPYaJ1wvpVUwa_AqQPu2RAcZz0kOK4l58,9582
 castor_extractor/visualization/looker/api/client_test.py,sha256=wsi20-neBXHaahDqf4nwCp8Ew5fRFCmVHG3OqrePKFs,1868
 castor_extractor/visualization/looker/api/constants.py,sha256=pZpq09tqcGi2Vh8orXxn9eil8ewfPUOLKfVuqgV2W-A,4126
 castor_extractor/visualization/looker/api/sdk.py,sha256=hSNcRsCoFae3zmjWFGsMrhQCIP57TcMJ2SorMPYJwn4,3553
 castor_extractor/visualization/looker/api/sdk_test.py,sha256=NHtKZTflPhqzBFHs1TyAQaubgxfzLLwYKFT8rEqR55I,1742
-castor_extractor/visualization/looker/api/utils.py,sha256=9xDKekJU6FNl_X5clO2NkDxPm7Q9t4ogJC7-dbh7HXo,1315
+castor_extractor/visualization/looker/api/utils.py,sha256=NpP90CA-SwdUjHhaWFBsKpJz0Z9BXgDOahIqfc3R9tk,565
 castor_extractor/visualization/looker/assets.py,sha256=4EwCI9VwIYRPrIjHU-ZnPY3Mf3cRykWACiYXanr5jFI,442
 castor_extractor/visualization/looker/constant.py,sha256=0tX6KOGdc9O6FiHcoj08j-QIZyBI2Mhh8UQc_XahDaM,694
 castor_extractor/visualization/looker/env.py,sha256=vPqirdeGKm3as2T-tBTjbpulQe8W7-3UE2j-Z57wFXk,1174
-castor_extractor/visualization/looker/extract.py,sha256=a0FahT3zi6izSQTKAE8xSX7vovsy1XUKGS7yQ2sBWUo,5051
+castor_extractor/visualization/looker/extract.py,sha256=vOIP8Hoxv05MiRa-l79YKOCHahuNiSW9uSKjwQQQQKs,5112
 castor_extractor/visualization/looker/fields.py,sha256=WmiSehmczWTufCLg4r2Ozq2grUpzxDNvIAHyGuOoGs4,636
 castor_extractor/visualization/looker/fields_test.py,sha256=7Cwq8Qky6aTZg8nCHp1gmPJtd9pGNB4QeMIRRWdHo5w,782
-castor_extractor/visualization/looker/multithreading.py,sha256=SvyUiXXXbYeo0xtiqjVUdyM-btLeS-qnBjVEQvUjF5w,3122
+castor_extractor/visualization/looker/multithreading.py,sha256=BB_oUkPo3LH96f1pPaywQ5Y8r2T7vo4hdAQl7y0bpPw,2596
 castor_extractor/visualization/looker/parameters.py,sha256=Nk2hfrg3L9twU-51Q7Wdp9uaxy8M2_juEebWoLfIMPc,2427
 castor_extractor/visualization/metabase/__init__.py,sha256=hSIoVgPzhQh-9H8XRUzga4EZSOYejGdH-qY_hBNGbyw,125
 castor_extractor/visualization/metabase/assets.py,sha256=XeP8UzbqsrleBtdv2nJ2LtWC_p3TAL7g0zlrcjt82TM,2814
@@ -289,7 +289,7 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.9.2.dist-info/METADATA,sha256=X4jmbMqEyEiFECwvadiwxz0M-9ifkn1TQZG2chp6A6w,5879
-castor_extractor-0.9.2.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-castor_extractor-0.9.2.dist-info/entry_points.txt,sha256=cvLvgE8Yi10sIiafUVL86XZPMUUyu9x11CF5PshAyiw,1045
-castor_extractor-0.9.2.dist-info/RECORD,,
+castor_extractor-0.10.0.dist-info/METADATA,sha256=XnrJ7dly9vVplF9ksw1thIrfr3Y215iEPCJC4nfs_-A,5880
+castor_extractor-0.10.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+castor_extractor-0.10.0.dist-info/entry_points.txt,sha256=cvLvgE8Yi10sIiafUVL86XZPMUUyu9x11CF5PshAyiw,1045
+castor_extractor-0.10.0.dist-info/RECORD,,