castor-extractor 0.9.0__tar.gz → 0.10.0__tar.gz

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.10.0 - 2023-11-28
+
+* Looker : extract all Looker Explores, even if unused in Dashboards
+
+## 0.9.2 - 2023-11-23
+
+* Looker : remove deprecated all_looks parameter
+
+## 0.9.1 - 2023-11-22
+
+* Redshift : filter out queries exceeding 65535 char
+
 ## 0.9.0 - 2023-11-20
 
 * Snowflake : Add key-pair authentication

castor_extractor-0.10.0/Dockerfile

@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:1.5
+
+FROM --platform=linux/amd64 python:3.10-slim
+
+ARG EXTRA
+ENV EXTRA=${EXTRA}
+
+RUN pip install --upgrade pip && \
+    pip install castor-extractor[${EXTRA}]
+
+ENV CASTOR_OUTPUT_DIRECTORY=/data
+
+VOLUME [ "/data" ]
+
+ENTRYPOINT castor-extract-${EXTRA}
+CMD [ "--log-to-stdout" ]

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.9.0
+Version: 0.10.0
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/commands/extract_looker.py

@@ -27,13 +27,7 @@ def main():
         action="store_true",
     )
 
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        "--all-looks",
-        help="Use all_looks endpoint instead of the paginated search_looks",
-        action="store_true",
-    )
-    group.add_argument(
+    parser.add_argument(
         "--search-per-folder",
         help="Fetches Looks and Dashboards per folder",
         action="store_true",
@@ -42,7 +36,6 @@ def main():
     args = parser.parse_args()
 
     looker.extract_all(
-        all_looks=args.all_looks,
         base_url=args.base_url,
         client_id=args.username,
         client_secret=args.password,

castor_extractor-0.10.0/castor_extractor/visualization/looker/__init__.py

@@ -0,0 +1,3 @@
+from .api import ApiClient, Credentials, lookml_explore_names
+from .assets import LookerAsset
+from .extract import extract_all, iterate_all_data

castor_extractor-0.10.0/castor_extractor/visualization/looker/api/__init__.py

@@ -0,0 +1,3 @@
+from .client import ApiClient
+from .sdk import Credentials
+from .utils import lookml_explore_names

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/visualization/looker/api/client.py

@@ -125,7 +125,7 @@ class ApiClient:
 
         return Pager(_search, logger=self._logger).all(per_page=self.per_page)
 
-    def _search_looks(self, folder_id: Optional[str] = None) -> List[Look]:
+    def looks(self, folder_id: Optional[str] = None) -> List[Look]:
         """
         Fetch looks via `search_looks` using pagination. The optional folder_id
         allows restricting the search to the given folder.
@@ -156,22 +156,6 @@ class ApiClient:
         # No pagination : see https://community.looker.com/looker-api-77/api-paging-limits-14598
         return list(self._sdk.all_looks(fields=format_fields(LOOK_FIELDS)))
 
-    def looks(
-        self,
-        all_looks_endpoint: Optional[bool] = False,
-        folder_id: Optional[str] = None,
-    ) -> List[Look]:
-        """Lists looks of the given Looker account
-
-        By default, uses the endpoint `search_looks` that allows pagination and
-        filtering on a folder.
-        If `all_looks_endpoint` parameter is set to True, it uses `all_looks` endpoint.
-        """
-        if all_looks_endpoint:
-            return self._all_looks()
-
-        return self._search_looks(folder_id)
-
     def users(self) -> List[User]:
         """Lists users of the given Looker account"""
 
@@ -207,7 +191,7 @@ class ApiClient:
     def explores(
         self,
         explore_names=Iterator[Tuple[str, str]],
-    ) -> List[LookmlModelExplore]:
+    ) -> Iterator[LookmlModelExplore]:
         """Iterates explores of the given Looker account for the provided model/explore names"""
 
         @safe_mode(self._safe_mode)
@@ -218,11 +202,10 @@ class ApiClient:
             self._on_api_call()
             return explore
 
-        explores = [
-            _call(model_name, explore_name)
-            for model_name, explore_name in explore_names
-        ]
-        return list(filter(None, explores))
+        for lookml_model_name, lookml_explore_name_ in explore_names:
+            explore_ = _call(lookml_model_name, lookml_explore_name_)
+            if explore_ is not None:
+                yield explore_
 
     def connections(self) -> List[DBConnection]:
         """Lists databases connections of the given Looker account"""

castor_extractor-0.10.0/castor_extractor/visualization/looker/api/utils.py

@@ -0,0 +1,24 @@
+from typing import Iterable, Set, Tuple
+
+from .sdk import LookmlModel
+
+
+def lookml_explore_names(
+    lookmls: Iterable[LookmlModel],
+) -> Set[Tuple[str, str]]:
+    """
+    Explores from the lookml models
+    Only valid explores are yielded: with all infos
+    """
+    model_explores = (
+        (model, explore)
+        for model in lookmls
+        for explore in model.explores or []
+    )
+
+    return {
+        (model.name, explore.name)
+        for model, explore in model_explores
+        # accept hidden resources
+        if model.name and explore.name
+    }

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/visualization/looker/constant.py

@@ -20,6 +20,5 @@ KEY_LOOKER_THREAD_POOL_SIZE = "CASTOR_LOOKER_THREAD_POOL_SIZE"
 BASE_URL = "CASTOR_LOOKER_BASE_URL"
 CLIENT_ID = "CASTOR_LOOKER_CLIENT_ID"
 CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET"  # noqa: S105
-ALL_LOOKS = "CASTOR_LOOKER_ALL_LOOKS"
 SEARCH_PER_FOLDER = "CASTOR_LOOKER_SEARCH_PER_FOLDER"
 LOG_TO_STDOUT = "CASTOR_LOOKER_LOG_TO_STDOUT"

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/visualization/looker/extract.py

@@ -11,12 +11,8 @@ from ...utils import (
     write_json,
     write_summary,
 )
-from .api import (
-    ApiClient,
-    Credentials,
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-)
+from .api import ApiClient, Credentials, lookml_explore_names
+from .api.sdk import LookmlModel
 from .assets import LookerAsset
 from .multithreading import MultithreadingFetcher
 from .parameters import get_parameters
@@ -24,6 +20,15 @@ from .parameters import get_parameters
 logger = logging.getLogger(__name__)
 
 
+def _extract_explores_by_name(
+    lookmls: Iterable[LookmlModel], client: ApiClient
+) -> Iterable[dict]:
+    explore_names = lookml_explore_names(lookmls)
+    explores = client.explores(explore_names)
+    for explore in explores:
+        yield deep_serialize(explore)  # type: ignore
+
+
 def _safe_mode(directory: str) -> SafeMode:
     add_logging_file_handler(directory)
     return SafeMode((Exception,), float("inf"))
@@ -47,7 +52,6 @@ def _client(
 
 def iterate_all_data(
     client: ApiClient,
-    all_looks: bool,
     search_per_folder: bool,
     thread_pool_size: int,
     log_to_stdout: bool,
@@ -74,16 +78,14 @@ def iterate_all_data(
         looks_stream = fetcher.fetch_assets(LookerAsset.LOOKS)
         yield LookerAsset.LOOKS, StreamableList(looks_stream)
     else:
-        yield LookerAsset.LOOKS, deep_serialize(client.looks(all_looks))
+        yield LookerAsset.LOOKS, deep_serialize(client.looks())
 
     logger.info("Extracting dashboards from Looker API")
     if search_per_folder:
         dashboards_stream = fetcher.fetch_assets(LookerAsset.DASHBOARDS)
         yield LookerAsset.DASHBOARDS, StreamableList(dashboards_stream)
-        dashboard_explore_names_ = fetcher.explores
     else:
         dashboards = client.dashboards()
-        dashboard_explore_names_ = dashboard_explore_names(dashboards)
         yield LookerAsset.DASHBOARDS, deep_serialize(dashboards)
 
     logger.info("Extracting lookml models from Looker API")
@@ -91,10 +93,8 @@ def iterate_all_data(
     yield LookerAsset.LOOKML_MODELS, deep_serialize(lookmls)
 
     logger.info("Extracting explores from Looker API")
-    explore_names = explore_names_associated_to_dashboards(
-        lookmls, dashboard_explore_names_
-    )
-    yield LookerAsset.EXPLORES, deep_serialize(client.explores(explore_names))
+    explores = _extract_explores_by_name(lookmls, client)
+    yield LookerAsset.EXPLORES, StreamableList(explores)
     del lookmls
 
     logger.info("Extracting connections from Looker API")
@@ -144,7 +144,6 @@ def extract_all(**kwargs) -> None:
 
     data = iterate_all_data(
         client=client,
-        all_looks=parameters.all_looks,
         search_per_folder=parameters.search_per_folder,
         thread_pool_size=parameters.thread_pool_size,
         log_to_stdout=parameters.log_to_stdout,

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/visualization/looker/multithreading.py

@@ -2,13 +2,13 @@ import logging
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import Iterable, List, Set, Tuple
+from typing import Iterable, List, Set
 
 from tqdm import tqdm  # type: ignore
 
 from ...utils import RetryStrategy, deep_serialize, retry
-from . import ApiClient, dashboard_explore_names
-from .api.sdk import Dashboard, SDKError
+from . import ApiClient
+from .api.sdk import SDKError
 from .assets import LookerAsset
 
 logger = logging.getLogger(__name__)
@@ -54,16 +54,6 @@ class MultithreadingFetcher:
         self._thread_pool_size = thread_pool_size
         self._log_to_stdout = log_to_stdout
 
-        self.explores: Set[Tuple[str, str]] = set()
-
-    def _save_explore_names(self, dashboards_per_folder: Iterable[Dashboard]):
-        """
-        Since dashboards are streamed right to the file, we need to keep
-        the relevant information to extract Explores later.
-        """
-        explores = dashboard_explore_names(dashboards_per_folder)
-        self.explores.update(explores)
-
     def _progress_bar(self, fetch_results: Iterable, total: int) -> tqdm:
         """Create a tqdm progress bar with the appropriate logs destination"""
         file = sys.stderr
@@ -73,7 +63,7 @@ class MultithreadingFetcher:
 
         return tqdm(fetch_results, total=total, file=file)
 
-    def fetch_assets(self, asset: LookerAsset):
+    def fetch_assets(self, asset: LookerAsset) -> Iterable[dict]:
         """
         Yields serialized Looks or Dashboards with a request per folder ID.
         Requests are parallelised.
@@ -88,12 +78,10 @@ class MultithreadingFetcher:
             fetch_results = executor.map(_fetch, self._folder_ids)
 
             for results in self._progress_bar(fetch_results, total_folders):
-                if asset == LookerAsset.DASHBOARDS:
-                    self._save_explore_names(results)
-
                 for result in results:
                     if not result:
                         continue
+
                     total_assets_count += len(result)
                     yield deep_serialize(result)
 

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/visualization/looker/parameters.py

@@ -2,7 +2,6 @@ from typing import NamedTuple, Optional
 
 from ...utils import OUTPUT_DIR, from_env, validate_baseurl
 from .constant import (
-    ALL_LOOKS,
     BASE_URL,
     CLIENT_ID,
     CLIENT_SECRET,
@@ -31,7 +30,6 @@ def _bool_env_variable(key: str) -> bool:
 class Parameters(NamedTuple):
     """Parameters for Looker extraction"""
 
-    all_looks: bool
     base_url: str
     client_id: str
     client_secret: str
@@ -63,13 +61,11 @@ def get_parameters(**kwargs) -> Parameters:
     search_per_folder = kwargs.get("search_per_folder") or _bool_env_variable(
         SEARCH_PER_FOLDER,
     )
-    all_looks = kwargs.get("all_looks") or _bool_env_variable(ALL_LOOKS)
     log_to_stdout = kwargs.get("log_to_stdout") or _bool_env_variable(
         LOG_TO_STDOUT
     )
 
     return Parameters(
-        all_looks=all_looks,
         base_url=base_url,
         client_id=client_id,
         client_secret=client_secret,

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/castor_extractor/warehouse/redshift/queries/query.sql

@@ -14,16 +14,26 @@ queries_deduplicated AS (
         AND EXTRACT('hour' FROM q.starttime) BETWEEN p.hour_min AND p.hour_max
 ),
 
-raw_query_text AS
-(
+query AS (
     SELECT
         q.query,
-        LISTAGG(qt.text, '') WITHIN GROUP (ORDER BY qt.sequence) AS agg_text
+        qt.text,
+        qt.sequence,
+        COUNT(*) OVER(PARTITION BY q.query) AS sequence_count
     FROM queries_deduplicated AS q
         INNER JOIN pg_catalog.stl_querytext AS qt ON q.query = qt.query
+),
+
+raw_query_text AS
+(
+    SELECT
+        q.query,
+        LISTAGG(q.text, '') WITHIN GROUP (ORDER BY q.sequence) AS agg_text
+    FROM query AS q
     WHERE TRUE
-        -- LISTAGG raises an error when total length >= 64K
-        AND qt.sequence < (65535 / 200)
+    -- LISTAGG raises an error when total length >= 65535
+    -- each sequence contains 200 char max
+        AND q.sequence_count < (65535 / 200)
     GROUP BY q.query
 ),
 
@@ -59,7 +69,7 @@ read_query AS (
         AND EXTRACT('hour' FROM q.starttime) BETWEEN p.hour_min AND p.hour_max
 ),
 
--- the DDL part is sensible to any change of JOIN and AGGREGATIOn: test in the field prior to merging
+-- the DDL part is sensible to any change of JOIN and AGGREGATION: test in the field prior to merging
 ddl_query AS (
     SELECT
         (q.xid || '-' || q.query_part_rank)::VARCHAR(256) AS query_id,

{castor_extractor-0.9.0 → castor_extractor-0.10.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.9.0"
+version = "0.10.0"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -14,7 +14,7 @@ classifiers = [
 readme = "README.md"
 homepage = "https://www.castordoc.com/"
 documentation = "https://docs.castordoc.com/castor-package/castor-extractor"
-include = ["README.md", "CHANGELOG.md", "LICENCE", "*/*/*/*.sql"]
+include = ["README.md", "CHANGELOG.md", "LICENCE", "*/*/*/*.sql", "Dockerfile", "DockerfileUsage.md"]
 
 
 [tool.poetry.dependencies]

castor_extractor-0.9.0/castor_extractor/visualization/looker/__init__.py

@@ -1,9 +0,0 @@
-from .api import (
-    ApiClient,
-    Credentials,
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-    lookml_explore_names,
-)
-from .assets import LookerAsset
-from .extract import extract_all, iterate_all_data

castor_extractor-0.9.0/castor_extractor/visualization/looker/api/__init__.py

@@ -1,7 +0,0 @@
-from .client import ApiClient
-from .sdk import Credentials
-from .utils import (
-    dashboard_explore_names,
-    explore_names_associated_to_dashboards,
-    lookml_explore_names,
-)

castor_extractor-0.9.0/castor_extractor/visualization/looker/api/utils.py

@@ -1,49 +0,0 @@
-from typing import Iterable, Set, Tuple
-
-from .sdk import Dashboard, LookmlModel
-
-
-def lookml_explore_names(
-    lookmls: Iterable[LookmlModel],
-) -> Set[Tuple[str, str]]:
-    """
-    Explores from the lookml models
-    Only valid explores are yielded: with all infos
-    """
-    model_explores = (
-        (model, explore)
-        for model in lookmls
-        for explore in model.explores or []
-    )
-
-    return {
-        (model.name, explore.name)
-        for model, explore in model_explores
-        # accept hidden resources
-        if model.name and explore.name
-    }
-
-
-def dashboard_explore_names(
-    dashboards: Iterable[Dashboard],
-) -> Set[Tuple[str, str]]:
-    """Explores that appear in dashboards"""
-    elements = (
-        element
-        for dashboard in dashboards
-        for element in dashboard.dashboard_elements or []
-    )
-
-    return {
-        (element.query.model, element.query.view)
-        for element in elements
-        if element.query and element.query.model and element.query.view
-    }
-
-
-def explore_names_associated_to_dashboards(
-    lookmls: Iterable[LookmlModel],
-    dashboard_explore_names_: Set[Tuple[str, str]],
-):
-    """Retrieve only explores that are associated to a looker dashboard"""
-    return lookml_explore_names(lookmls).intersection(dashboard_explore_names_)