castor-extractor 0.17.4__py3-none-any.whl → 0.18.5__py3-none-any.whl

castor_extractor/visualization/domo/client/client.py

@@ -1,11 +1,21 @@
 import logging
 from datetime import datetime, timedelta
 from http import HTTPStatus
-from typing import Iterator, List, Optional, Set, Tuple
+from typing import Iterator, List, Optional, Set
 
 import requests
 
-from ....utils import at_midnight, current_date, past_date, retry, timestamp_ms
+from ....utils import (
+    RequestSafeMode,
+    ResponseJson,
+    at_midnight,
+    current_date,
+    empty_iterator,
+    handle_response,
+    past_date,
+    retry,
+    timestamp_ms,
+)
 from ..assets import DomoAsset
 from .credentials import DomoCredentials
 from .endpoints import Endpoint, EndpointFactory
@@ -17,11 +27,17 @@ DOMO_PUBLIC_URL = "https://api.domo.com"
 DEFAULT_TIMEOUT = 120
 TOKEN_EXPIRATION_SECONDS = timedelta(seconds=3000)  # auth token lasts 1 hour
 
+
+# Safe Mode
+VOLUME_IGNORED = 10
 IGNORED_ERROR_CODES = (
     HTTPStatus.BAD_REQUEST,
     HTTPStatus.NOT_FOUND,
 )
-ERROR_TPL = "Request failed with status code {status_code} and reason {reason}"
+DOMO_SAFE_MODE = RequestSafeMode(
+    max_errors=VOLUME_IGNORED,
+    status_codes=IGNORED_ERROR_CODES,
+)
 
 _RETRY_EXCEPTIONS = [
     requests.exceptions.ConnectTimeout,
@@ -33,40 +49,17 @@ _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 logger = logging.getLogger(__name__)
 
 
-def _handle_response(response: requests.Response) -> requests.Response:
-    response.raise_for_status()
-    return response
-
-
-def _ignore_or_raise(
-    error: requests.RequestException,
-    ignore_error_codes: Optional[Tuple[int, ...]],
-) -> dict:
-    """
-    Raises the error unless the response status code is in the ignored error
-    codes list.
-    """
-    if not ignore_error_codes:
-        raise error
-
-    response = error.response
-    if response is None:
-        raise error
-
-    if response.status_code in ignore_error_codes:
-        logger.warning(error)
-        return {}
-
-    raise error
-
-
 class DomoClient:
     """
     Connect to Domo API and fetch main assets.
     https://developer.domo.com/portal/8ba9aedad3679-ap-is#platform-oauth-apis
     """
 
-    def __init__(self, credentials: DomoCredentials):
+    def __init__(
+        self,
+        credentials: DomoCredentials,
+        safe_mode: Optional[RequestSafeMode] = None,
+    ):
         self._authentication = credentials.authentication
         self._bearer_headers: Optional[dict] = None
         self._session = requests.session()
@@ -76,6 +69,7 @@ class DomoClient:
         self._timeout = DEFAULT_TIMEOUT
         self.base_url = credentials.base_url
         self.cloud_id = credentials.cloud_id
+        self.safe_mode = safe_mode or DOMO_SAFE_MODE
 
     def _token_expired(self) -> bool:
         token_lifetime = datetime.now() - self._token_creation_time
@@ -95,7 +89,8 @@ class DomoClient:
             auth=basic_authentication,
             timeout=self._timeout,
         )
-        result = _handle_response(response).json()
+        response.raise_for_status()
+        result = response.json()
 
         bearer_token = result["access_token"]
         self._bearer_headers = {"authorization": f"Bearer {bearer_token}"}
@@ -113,7 +108,7 @@ class DomoClient:
         endpoint: Endpoint,
         params: Optional[dict] = None,
         asset_id: Optional[str] = None,
-    ) -> requests.Response:
+    ) -> ResponseJson:
         params = params if params else {}
         is_private = endpoint.is_private
         headers = self._private_headers if is_private else self._bearer_auth()
@@ -125,27 +120,16 @@ class DomoClient:
             timeout=self._timeout,
         )
 
-        if response.status_code != HTTPStatus.OK:
-            logger.warning(
-                ERROR_TPL.format(
-                    status_code=response.status_code,
-                    reason=response.reason,
-                )
-            )
-        return response
+        return handle_response(response, self.safe_mode)
 
     def _get_element(
         self,
         endpoint: Endpoint,
         params: Optional[dict] = None,
         asset_id: Optional[str] = None,
-        ignore_error_codes: Optional[Tuple[int, ...]] = None,
     ) -> dict:
         """Used when the response only contains one element"""
-        try:
-            return self._get(endpoint, params, asset_id).json()
-        except requests.RequestException as error:
-            return _ignore_or_raise(error, ignore_error_codes)
+        return self._get(endpoint, params, asset_id)
 
     def _get_many(
         self,
@@ -154,7 +138,7 @@ class DomoClient:
         asset_id: Optional[str] = None,
     ) -> List[dict]:
         """Used when the response contains multiple elements"""
-        return self._get(endpoint, params, asset_id).json()
+        return self._get(endpoint, params, asset_id)
 
     def _get_paginated(self, endpoint: Endpoint) -> List[dict]:
         """Used when the response is paginated and need iterations"""
@@ -172,10 +156,7 @@ class DomoClient:
 
     def _datasources(self, page_id: str) -> RawData:
         endpoint = self._endpoint_factory.page_content(page_id)
-        page_content = self._get_element(
-            endpoint,
-            ignore_error_codes=IGNORED_ERROR_CODES,
-        )
+        page_content = self._get_element(endpoint)
         processed: set[str] = set()
         for card in page_content.get("cards", []):
             for datasource in card["datasources"]:
@@ -196,7 +177,7 @@ class DomoClient:
     ) -> Iterator[dict]:
         """Recursively fetch pages while building the folder architecture"""
         if not page_tree:
-            return []
+            return empty_iterator()
 
         for page in page_tree:
             page_id = page.get("id")
@@ -211,6 +192,10 @@ class DomoClient:
                 self._endpoint_factory.pages,
                 asset_id=page_id,
             )
+
+            if not detail:
+                continue
+
             datasources = self._datasources(page_id)
             yield {
                 **detail,

castor_extractor/visualization/domo/client/credentials.py

@@ -1,39 +1,26 @@
-from dataclasses import dataclass
-from enum import Enum
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional
 
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from requests.auth import HTTPBasicAuth
 
+DOMO_ENV_PREFIX = "CASTOR_DOMO_"
 
-class CredentialsKey(Enum):
-    """Value enum object for the credentials"""
 
-    API_TOKEN = "api_token"  # noqa: S105
-    BASE_URL = "base_url"
-    CLIENT_ID = "client_id"
-    CLOUD_ID = "cloud_id"
-    DEVELOPER_TOKEN = "developer_token"  # noqa: S105
-
-
-CLIENT_ALLOWED_KEYS: Tuple[str, ...] = tuple(c.value for c in CredentialsKey)
-
-
-@dataclass
-class DomoCredentials:
+class DomoCredentials(BaseSettings):
     """Class to handle Domo rest API permissions"""
 
-    api_token: str
+    model_config = SettingsConfigDict(
+        env_prefix=DOMO_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    api_token: str = Field(repr=False)
     base_url: str
     client_id: str
-    developer_token: str
-    cloud_id: Optional[str] = None
-
-    @classmethod
-    def from_secret(cls, secret: dict) -> "DomoCredentials":
-        credentials = {
-            k: v for k, v in secret.items() if k in CLIENT_ALLOWED_KEYS
-        }
-        return cls(**credentials)
+    cloud_id: Optional[str] = Field(validation_alias="CLOUD_ID", default=None)
+    developer_token: str = Field(repr=False)
 
     @property
     def authentication(self) -> HTTPBasicAuth:

castor_extractor/visualization/domo/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -12,7 +12,6 @@ from ...utils import (
 )
 from .assets import DomoAsset
 from .client import DomoClient, DomoCredentials
-from .constants import API_TOKEN, BASE_URL, CLIENT_ID, CLOUD_ID, DEVELOPER_TOKEN
 
 logger = logging.getLogger(__name__)
 
@@ -43,33 +42,13 @@ def iterate_all_data(
     yield DomoAsset.DATAFLOWS, list(deep_serialize(dataflows))
 
 
-def extract_all(
-    api_token: Optional[str] = None,
-    base_url: Optional[str] = None,
-    client_id: Optional[str] = None,
-    cloud_id: Optional[str] = None,
-    developer_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Domo API
     Store the output files locally under the given output_directory
     """
-
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    _client_id = client_id or from_env(CLIENT_ID)
-    _base_url = base_url or from_env(BASE_URL)
-    _api_token = api_token or from_env(API_TOKEN)
-    _developer_token = developer_token or from_env(DEVELOPER_TOKEN)
-    _cloud_id = cloud_id or from_env(CLOUD_ID)
-
-    credentials = DomoCredentials(
-        base_url=_base_url,
-        client_id=_client_id,
-        api_token=_api_token,
-        developer_token=_developer_token,
-        cloud_id=_cloud_id,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = DomoCredentials(**kwargs)
     client = DomoClient(credentials=credentials)
 
     ts = current_timestamp()
@@ -78,4 +57,4 @@ def extract_all(
         filename = get_output_filename(key.name.lower(), _output_directory, ts)
         write_json(filename, data)
 
-    write_summary(_output_directory, ts, base_url=_base_url)
+    write_summary(_output_directory, ts, base_url=credentials.base_url)

castor_extractor/visualization/looker/__init__.py

@@ -1,3 +1,8 @@
-from .api import ApiClient, Credentials, lookml_explore_names
+from .api import (
+    ApiClient,
+    ExtractionParameters,
+    LookerCredentials,
+    lookml_explore_names,
+)
 from .assets import LookerAsset
 from .extract import extract_all, iterate_all_data

castor_extractor/visualization/looker/api/__init__.py

@@ -1,3 +1,4 @@
 from .client import ApiClient
-from .sdk import Credentials
+from .credentials import LookerCredentials
+from .extraction_parameters import ExtractionParameters
 from .utils import lookml_explore_names

castor_extractor/visualization/looker/api/client.py

@@ -22,7 +22,7 @@ from looker_sdk.sdk.api40.models import (
 from looker_sdk.sdk.constants import sdk_version
 
 from ....utils import Pager, PagerLogger, SafeMode, past_date, safe_mode
-from ..env import page_size
+from ..constants import DEFAULT_LOOKER_PAGE_SIZE
 from ..fields import format_fields
 from .constants import (
     CONNECTION_FIELDS,
@@ -39,7 +39,8 @@ from .constants import (
     USER_FIELDS,
     USERS_ATTRIBUTES_FIELDS,
 )
-from .sdk import CastorApiSettings, Credentials, has_admin_permissions
+from .credentials import LookerCredentials
+from .sdk import CastorApiSettings, has_admin_permissions
 
 logger = logging.getLogger(__name__)
 
@@ -78,9 +79,10 @@ class ApiClient:
 
     def __init__(
         self,
-        credentials: Credentials,
+        credentials: LookerCredentials,
         on_api_call: OnApiCall = lambda: None,
         safe_mode: Optional[SafeMode] = None,
+        page_size: int = DEFAULT_LOOKER_PAGE_SIZE,
     ):
         settings = CastorApiSettings(
             credentials=credentials, sdk_version=sdk_version
@@ -92,7 +94,7 @@ class ApiClient:
             self._sdk = sdk
         self._on_api_call = on_api_call
         self._logger = ApiPagerLogger(on_api_call)
-        self.per_page = page_size()
+        self.per_page = page_size
         self._safe_mode = safe_mode
 
     def folders(self) -> List[Folder]:

castor_extractor/visualization/looker/api/client_test.py

@@ -2,9 +2,11 @@ import datetime
 from unittest.mock import patch
 
 import pytest
-from castor_extractor.visualization.looker import (  # type: ignore
+from castor_extractor.visualization.looker.api.client import (  # type: ignore
     ApiClient,
-    Credentials,
+)
+from castor_extractor.visualization.looker.api.credentials import (  # type: ignore
+    LookerCredentials,
 )
 from dateutil.utils import today
 from freezegun import freeze_time
@@ -13,7 +15,7 @@ from .client import _mondays
 
 
 def _credentials():
-    return Credentials(  # noqa: S106
+    return LookerCredentials(  # noqa: S106
         base_url="base_url",
         client_id="client_id",
         client_secret="secret",

castor_extractor/visualization/looker/api/credentials.py

@@ -0,0 +1,33 @@
+from looker_sdk.rtl.api_settings import SettingsConfig
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from ..constants import DEFAULT_LOOKER_TIMEOUT_SECOND, LOOKER_ENV_PREFIX
+
+KEY_LOOKER_TIMEOUT_SECOND = f"{LOOKER_ENV_PREFIX}TIMEOUT_SECOND"
+
+
+class LookerCredentials(BaseSettings):
+    """ValueObject for the credentials"""
+
+    model_config = SettingsConfigDict(
+        env_prefix=LOOKER_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    base_url: str
+    client_id: str
+    client_secret: str = Field(repr=False)
+    timeout: int = Field(
+        validation_alias=KEY_LOOKER_TIMEOUT_SECOND,
+        default=DEFAULT_LOOKER_TIMEOUT_SECOND,
+    )
+
+    def to_settings_config(self) -> SettingsConfig:
+        return SettingsConfig(
+            base_url=self.base_url,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            timeout=str(self.timeout),
+        )

castor_extractor/visualization/looker/api/extraction_parameters.py

@@ -0,0 +1,38 @@
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from ..constants import (
+    DEFAULT_LOOKER_PAGE_SIZE,
+    DEFAULT_LOOKER_THREAD_POOL_SIZE,
+    LOOKER_ENV_PREFIX,
+    MAX_THREAD_POOL_SIZE,
+    MIN_THREAD_POOL_SIZE,
+)
+
+
+class ExtractionParameters(BaseSettings):
+    """
+    Class holding all the parameters needed for the extraction of
+    Looker metadata
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix=LOOKER_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    is_safe_mode: bool = False
+    log_to_stdout: bool
+    output_directory: str
+    search_per_folder: bool
+    page_size: int = Field(default=DEFAULT_LOOKER_PAGE_SIZE)
+    thread_pool_size: int = Field(default=DEFAULT_LOOKER_THREAD_POOL_SIZE)
+
+    @field_validator("thread_pool_size", mode="before")
+    @classmethod
+    def _check_thread_pool_size(cls, thread_pool_size: int) -> int:
+        thread_pool_size = thread_pool_size or DEFAULT_LOOKER_THREAD_POOL_SIZE
+        if MIN_THREAD_POOL_SIZE <= thread_pool_size <= MAX_THREAD_POOL_SIZE:
+            return thread_pool_size
+        raise ValueError("Thread pool size must be between 1 and 200 inclusive")

castor_extractor/visualization/looker/api/sdk.py

@@ -8,33 +8,7 @@ from looker_sdk.rtl import transport
 from looker_sdk.rtl.api_settings import ApiSettings, SettingsConfig
 from looker_sdk.sdk.api40 import methods as methods40
 
-from ..env import timeout_second
-
-
-class Credentials:
-    """ValueObject for the credentials"""
-
-    def __init__(
-        self,
-        *,
-        base_url: str,
-        client_id: str,
-        client_secret: str,
-        timeout: Optional[int] = None,
-        **_kwargs,
-    ):
-        self.base_url = base_url
-        self.client_id = client_id
-        self.client_secret = client_secret
-        self.timeout: int = timeout or timeout_second()
-
-    def to_settings_config(self) -> SettingsConfig:
-        return SettingsConfig(
-            base_url=self.base_url,
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            timeout=str(self.timeout),
-        )
+from .credentials import LookerCredentials
 
 
 def has_admin_permissions(sdk_: methods40.Looker40SDK) -> bool:
@@ -54,7 +28,7 @@ class CastorApiSettings(ApiSettings):
     """SDK settings with initialisation using a credential object instead of a path to a .ini file"""
 
     def __init__(
-        self, credentials: Credentials, sdk_version: Optional[str] = ""
+        self, credentials: LookerCredentials, sdk_version: Optional[str] = ""
     ):
         """Configure using a config dict"""
         self.config = credentials.to_settings_config()

castor_extractor/visualization/looker/constant.py

@@ -1,33 +1,8 @@
 """
-Request timeout in seconds for Looker API
-"""
-
-from looker_sdk.error import SDKError  # type: ignore
-
-DEFAULT_LOOKER_TIMEOUT_SECOND = 120
-KEY_LOOKER_TIMEOUT_SECOND = "CASTOR_LOOKER_TIMEOUT_SECOND"
-
-"""
-Number of items per page when requesting Looker API
-"""
-DEFAULT_LOOKER_PAGE_SIZE = 500
-KEY_LOOKER_PAGE_SIZE = "CASTOR_LOOKER_PAGE_SIZE"
-
-"""
-Maximum concurrent threads to run when fetching
+Safe mode parameters
 """
-DEFAULT_LOOKER_THREAD_POOL_SIZE = 20
-KEY_LOOKER_THREAD_POOL_SIZE = "CASTOR_LOOKER_THREAD_POOL_SIZE"
 
-# env variables
-BASE_URL = "CASTOR_LOOKER_BASE_URL"
-CLIENT_ID = "CASTOR_LOOKER_CLIENT_ID"
-CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET"  # noqa: S105
-SEARCH_PER_FOLDER = "CASTOR_LOOKER_SEARCH_PER_FOLDER"
-LOG_TO_STDOUT = "CASTOR_LOOKER_LOG_TO_STDOUT"
+from looker_sdk.error import SDKError
 
-"""
-Safe mode parameters
-"""
 SAFE_MODE_MAX_ERRORS = 3
 SAFE_MODE_EXCEPTIONS = (SDKError,)

castor_extractor/visualization/looker/constants.py

@@ -0,0 +1,17 @@
+"""
+Request timeout in seconds for Looker API
+"""
+
+LOOKER_ENV_PREFIX = "CASTOR_LOOKER_"
+
+DEFAULT_LOOKER_TIMEOUT_SECOND = 120
+"""
+Number of items per page when requesting Looker API
+"""
+DEFAULT_LOOKER_PAGE_SIZE = 500
+"""
+Maximum concurrent threads to run when fetching
+"""
+DEFAULT_LOOKER_THREAD_POOL_SIZE = 20
+MIN_THREAD_POOL_SIZE = 1
+MAX_THREAD_POOL_SIZE = 200

castor_extractor/visualization/looker/extract.py

@@ -13,10 +13,14 @@ from ...utils import (
     write_json,
     write_summary,
 )
-from .api import ApiClient, Credentials, lookml_explore_names
+from .api import (
+    ApiClient,
+    ExtractionParameters,
+    LookerCredentials,
+    lookml_explore_names,
+)
 from .assets import LookerAsset
 from .multithreading import MultithreadingFetcher
-from .parameters import get_parameters
 
 logger = logging.getLogger(__name__)
 
@@ -30,25 +34,23 @@ def _extract_explores_by_name(
         yield deep_serialize(explore)  # type: ignore
 
 
-def _safe_mode(directory: str) -> SafeMode:
-    add_logging_file_handler(directory)
+def _safe_mode(
+    extraction_parameters: ExtractionParameters,
+) -> Optional[SafeMode]:
+    if extraction_parameters.is_safe_mode:
+        return None
+    add_logging_file_handler(extraction_parameters.output_directory)
     return SafeMode((Exception,), float("inf"))
 
 
 def _client(
-    base_url: str,
-    client_id: str,
-    client_secret: str,
-    timeout: Optional[int],
+    credentials: LookerCredentials,
     safe_mode: Optional[SafeMode],
+    page_size: int,
 ) -> ApiClient:
-    credentials = Credentials(
-        base_url=base_url,
-        client_id=client_id,
-        client_secret=client_secret,
-        timeout=timeout,
+    return ApiClient(
+        credentials=credentials, safe_mode=safe_mode, page_size=page_size
     )
-    return ApiClient(credentials=credentials, safe_mode=safe_mode)
 
 
 def iterate_all_data(
@@ -56,7 +58,7 @@ def iterate_all_data(
     search_per_folder: bool,
     thread_pool_size: int,
     log_to_stdout: bool,
-) -> Union[StreamableList, Iterable[Tuple[LookerAsset, list]]]:
+) -> Iterable[Union[StreamableList, Tuple[LookerAsset, list]]]:
     """Iterate over the extracted Data From looker"""
 
     logger.info("Extracting users from Looker API")
@@ -124,33 +126,31 @@ def extract_all(**kwargs) -> None:
     Extract Data From looker and store it locally in files under the
     output_directory
     """
-    parameters = get_parameters(**kwargs)
-    output_directory = parameters.output_directory
-    base_url = parameters.base_url
+    extraction_parameters = ExtractionParameters(**kwargs)
+    output_directory = extraction_parameters.output_directory
+
+    credentials = LookerCredentials(**kwargs)
 
-    if parameters.log_to_stdout:
+    if extraction_parameters.log_to_stdout:
         set_stream_handler_to_stdout()
 
-    is_safe_mode = parameters.is_safe_mode
-    safe_mode = _safe_mode(output_directory) if is_safe_mode else None
+    safe_mode = _safe_mode(extraction_parameters)
     client = _client(
-        base_url=base_url,
-        client_id=parameters.client_id,
-        client_secret=parameters.client_secret,
-        timeout=parameters.timeout,
+        credentials=credentials,
         safe_mode=safe_mode,
+        page_size=extraction_parameters.page_size,
     )
 
     ts = current_timestamp()
 
     data = iterate_all_data(
         client=client,
-        search_per_folder=parameters.search_per_folder,
-        thread_pool_size=parameters.thread_pool_size,
-        log_to_stdout=parameters.log_to_stdout,
+        search_per_folder=extraction_parameters.search_per_folder,
+        thread_pool_size=extraction_parameters.thread_pool_size,
+        log_to_stdout=extraction_parameters.log_to_stdout,
     )
     for asset, data in data:
         filename = get_output_filename(asset.value, output_directory, ts)
         write_json(filename, data)
 
-    write_summary(output_directory, ts, base_url=base_url)
+    write_summary(output_directory, ts, base_url=credentials.base_url)

castor_extractor/visualization/metabase/__init__.py

@@ -1,3 +1,8 @@
 from .assets import MetabaseAsset
-from .client import ApiClient, DbClient
+from .client import (
+    ApiClient,
+    DbClient,
+    MetabaseApiCredentials,
+    MetabaseDbCredentials,
+)
 from .extract import extract_all, iterate_all_data

castor_extractor/visualization/metabase/client/__init__.py

@@ -1,2 +1,2 @@
-from .api import ApiClient
-from .db import DbClient
+from .api import ApiClient, MetabaseApiCredentials
+from .db import DbClient, MetabaseDbCredentials

castor_extractor/visualization/metabase/client/api/__init__.py

@@ -1 +1,2 @@
 from .client import ApiClient
+from .credentials import MetabaseApiCredentials