castor-extractor 0.17.3__py3-none-any.whl → 0.18.2__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.18.2 - 2024-07-08
+
+* Added StatusCode handling to SafeMode
+
+## 0.18.1 - 2024-07-04
+
+* Bump dependencies: numpy, setuptools, tableauserverclient
+
+## 0.18.0 - 2024-07-03
+
+* Dashboarding technologies : Reworked credentials using Pydantic
+
+## 0.17.5 - 2024-07-03
+
+* Snowflake, Synapse, Redshift: Remove default_value from the extracted column
+
+## 0.17.4 - 2024-07-03
+
+* Sigma: Add `input-table`, `pivot-table` and `viz` in the list of supported **Elements**
+
 ## 0.17.3 - 2024-06-24
 
 * Databricks: extract tags for tables and column

castor_extractor/commands/extract_metabase_api.py

@@ -17,11 +17,12 @@ def main():
 
     args = parser.parse_args()
 
-    client = metabase.ApiClient(
+    credentials = metabase.MetabaseApiCredentials(
         base_url=args.base_url,
         user=args.username,
         password=args.password,
     )
+    client = metabase.ApiClient(credentials)
 
     metabase.extract_all(
         client,

castor_extractor/commands/extract_metabase_db.py

@@ -35,7 +35,7 @@ def main():
 
     args = parser.parse_args()
 
-    client = metabase.DbClient(
+    credentials = metabase.MetabaseDbCredentials(
         host=args.host,
         port=args.port,
         database=args.database,
@@ -46,6 +46,8 @@ def main():
         require_ssl=args.require_ssl,
     )
 
+    client = metabase.DbClient(credentials)
+
     metabase.extract_all(
         client,
         output_directory=args.output,

castor_extractor/commands/extract_mode.py

@@ -25,8 +25,7 @@ def main():
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     args = parser.parse_args()
-
-    client = mode.Client(
+    credentials = mode.Credentials(
         host=args.host,
         workspace=args.workspace,
         token=args.token,
@@ -34,6 +33,6 @@ def main():
     )
 
     mode.extract_all(
-        client,
+        credentials,
         output_directory=args.output,
     )

castor_extractor/utils/__init__.py

@@ -5,7 +5,7 @@ from .client import (
     SqlalchemyClient,
     uri_encode,
 )
-from .collection import group_by, mapping_from_rows
+from .collection import empty_iterator, group_by, mapping_from_rows
 from .constants import OUTPUT_DIR
 from .deprecate import deprecate_python
 from .env import from_env
@@ -23,6 +23,7 @@ from .pager import (
 )
 from .retry import RetryStrategy, retry
 from .safe import SafeMode, safe_mode
+from .safe_request import RequestSafeMode, ResponseJson, handle_response
 from .store import AbstractStorage, LocalStorage
 from .string import decode_when_bytes, string_to_tuple
 from .time import (

castor_extractor/utils/collection.py

@@ -44,3 +44,11 @@ def mapping_from_rows(rows: List[Dict], key: Any, value: Any) -> Dict:
         mapping[mapping_key] = mapping_value
 
     return mapping
+
+
+def empty_iterator():
+    """
+    Utils to return empty iterator, mainly used for viz transformers
+    Remark: missing return type is on purpose, it breaks the typing
+    """
+    return iter([])

castor_extractor/utils/safe_request.py

@@ -0,0 +1,57 @@
+import logging
+from typing import List, Tuple, Union
+
+from requests import HTTPError, Response
+
+logger = logging.getLogger(__name__)
+
+ResponseJson = Union[dict, List[dict]]
+
+
+class RequestSafeMode:
+    """
+    RequestSafeMode class to parameterize what should be done if response
+    raises due to the status code.
+
+    Attributes:
+        self.status_codes: tuple of status codes that will be caught
+        self.errors_caught : list of errors caught
+    """
+
+    def __init__(
+        self,
+        max_errors: Union[int, float] = 0,
+        status_codes: Tuple[int, ...] = (),
+    ):
+        self.max_errors = max_errors
+        self.status_codes: List[int] = list(status_codes)
+        self.status_codes_caught: List[int] = []
+
+    def catch_response(self, exception: HTTPError, status_code: int):
+        if int(status_code) not in self.status_codes:
+            raise exception
+
+        self.status_codes_caught.append(int(status_code))
+
+    @property
+    def should_raise(self) -> bool:
+        return len(self.status_codes_caught) > self.max_errors
+
+
+def handle_response(
+    response: Response, safe_mode: RequestSafeMode
+) -> ResponseJson:
+    """
+    Util to handle a HTTP Response based on the response status code and the
+    safe mode used
+    """
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        safe_mode.catch_response(e, response.status_code)
+        if safe_mode.should_raise:
+            raise e
+        logger.error(f"Safe mode : skip request with error {e}")
+        logger.debug(e, exc_info=True)
+        return {}
+    return response.json()

castor_extractor/utils/safe_request_test.py

@@ -0,0 +1,77 @@
+import io
+from http import HTTPStatus
+
+import pytest
+from requests import HTTPError, Response
+
+from .safe_request import RequestSafeMode, handle_response
+
+
+def mock_response(status_code: int):
+    response = Response()
+    response.status_code = status_code
+    response.raw = io.BytesIO(b'[{"data": "working"}]')
+    return response
+
+
+def test_http_error_with_no_safe_mode():
+    safe_params = RequestSafeMode()  # Caught
+
+    with pytest.raises(HTTPError):
+        handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+
+def test_http_error_with_no_status_code():
+    safe_params = RequestSafeMode(2)  # Caught
+
+    with pytest.raises(HTTPError):
+        handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+
+def test_http_error_with_status_code():
+    safe_params = RequestSafeMode(2, (HTTPStatus.FORBIDDEN,))  # Caught
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+    assert call() == {}
+    assert call() == {}
+
+    with pytest.raises(HTTPError):
+        call()
+
+
+def test_http_error_with_multiple_status_code():
+    safe_params = RequestSafeMode(
+        2, (HTTPStatus.NOT_FOUND, HTTPStatus.FORBIDDEN)
+    )  # Caught
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+    def call_2():
+        return handle_response(mock_response(HTTPStatus.NOT_FOUND), safe_params)
+
+    assert call() == {}
+    assert call_2() == {}
+    with pytest.raises(HTTPError):  # 3 failed calls > retries
+        call()
+
+
+def test_http_error_with_wrong_status_code():
+    safe_params = RequestSafeMode(2, (HTTPStatus.NOT_FOUND,))  # Wrong Status
+
+    def call():
+        handle_response(mock_response(HTTPStatus.BAD_REQUEST), safe_params)
+
+    with pytest.raises(HTTPError):
+        call()
+
+
+def test_http_error_with_return():
+    safe_params = RequestSafeMode(2, (HTTPStatus.NOT_FOUND,))  # Wrong Status
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.OK), safe_params)
+
+    assert call() == [{"data": "working"}]

castor_extractor/utils/salesforce/__init__.py

@@ -1,3 +1,2 @@
 from .client import SalesforceBaseClient
-from .constants import Keys
-from .credentials import SalesforceCredentials, to_credentials
+from .credentials import SalesforceCredentials

castor_extractor/utils/salesforce/constants.py

@@ -1,13 +1,2 @@
 DEFAULT_API_VERSION = 59.0
 DEFAULT_PAGINATION_LIMIT = 100
-
-
-class Keys:
-    """Salesforce's credentials keys"""
-
-    USERNAME = "username"
-    PASSWORD = "password"  # noqa: S105
-    CLIENT_ID = "client_id"
-    CLIENT_SECRET = "client_secret"  # noqa: S105
-    SECURITY_TOKEN = "security_token"  # noqa: S105
-    BASE_URL = "base_url"

castor_extractor/utils/salesforce/credentials.py

@@ -1,36 +1,33 @@
 from typing import Dict
 
-from ...utils import from_env
-from .constants import Keys
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
-_USERNAME = "CASTOR_SALESFORCE_USERNAME"
-_PASSWORD = "CASTOR_SALESFORCE_PASSWORD"  # noqa: S105
-_SECURITY_TOKEN = "CASTOR_SALESFORCE_SECURITY_TOKEN"  # noqa: S105
-_CLIENT_ID = "CASTOR_SALESFORCE_CLIENT_ID"
-_CLIENT_SECRET = "CASTOR_SALESFORCE_CLIENT_SECRET"  # noqa: S105
-_BASE_URL = "CASTOR_SALESFORCE_BASE_URL"
+CASTOR_ENV_PREFIX = "CASTOR_SALESFORCE_"
 
 
-class SalesforceCredentials:
+class SalesforceCredentials(BaseSettings):
     """
     Class to handle Salesforce rest API permissions
     """
 
-    def __init__(
-        self,
-        *,
-        username: str,
-        password: str,
-        security_token: str,
-        client_id: str,
-        client_secret: str,
-        base_url: str,
-    ):
-        self.username = username
-        self.password = password + security_token
-        self.client_id = client_id
-        self.client_secret = client_secret
-        self.base_url = base_url
+    model_config = SettingsConfigDict(
+        env_prefix=CASTOR_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    base_url: str
+    client_id: str
+    client_secret: str = Field(repr=False)
+    password: str = Field(repr=False)
+    security_token: str = Field(repr=False)
+    username: str
+
+    @property
+    def password_token(self) -> str:
+        """Generates the password for authentication"""
+        return self.password + self.security_token
 
     def token_request_payload(self) -> Dict[str, str]:
         """
@@ -41,25 +38,5 @@ class SalesforceCredentials:
             "client_id": self.client_id,
             "client_secret": self.client_secret,
             "username": self.username,
-            "password": self.password,
+            "password": self.password_token,
         }
-
-
-def to_credentials(params: dict) -> SalesforceCredentials:
-    """extract Salesforce credentials"""
-    username = params.get(Keys.USERNAME) or from_env(_USERNAME)
-    password = params.get(Keys.PASSWORD) or from_env(_PASSWORD)
-    security_token = params.get(Keys.SECURITY_TOKEN) or from_env(
-        _SECURITY_TOKEN
-    )
-    client_id = params.get(Keys.CLIENT_ID) or from_env(_CLIENT_ID)
-    client_secret = params.get(Keys.CLIENT_SECRET) or from_env(_CLIENT_SECRET)
-    base_url = params.get(Keys.BASE_URL) or from_env(_BASE_URL)
-    return SalesforceCredentials(
-        username=username,
-        password=password,
-        client_id=client_id,
-        client_secret=client_secret,
-        security_token=security_token,
-        base_url=base_url,
-    )

castor_extractor/visualization/domo/__init__.py

@@ -1,3 +1,3 @@
 from .assets import DomoAsset
-from .client import CredentialsKey, DomoClient, DomoCredentials
+from .client import DomoClient, DomoCredentials
 from .extract import extract_all

castor_extractor/visualization/domo/client/__init__.py

@@ -1,2 +1,2 @@
 from .client import DomoClient
-from .credentials import CredentialsKey, DomoCredentials
+from .credentials import DomoCredentials

castor_extractor/visualization/domo/client/client.py

@@ -1,11 +1,21 @@
 import logging
 from datetime import datetime, timedelta
 from http import HTTPStatus
-from typing import Iterator, List, Optional, Set, Tuple
+from typing import Iterator, List, Optional, Set
 
 import requests
 
-from ....utils import at_midnight, current_date, past_date, retry, timestamp_ms
+from ....utils import (
+    RequestSafeMode,
+    ResponseJson,
+    at_midnight,
+    current_date,
+    empty_iterator,
+    handle_response,
+    past_date,
+    retry,
+    timestamp_ms,
+)
 from ..assets import DomoAsset
 from .credentials import DomoCredentials
 from .endpoints import Endpoint, EndpointFactory
@@ -17,11 +27,17 @@ DOMO_PUBLIC_URL = "https://api.domo.com"
 DEFAULT_TIMEOUT = 120
 TOKEN_EXPIRATION_SECONDS = timedelta(seconds=3000)  # auth token lasts 1 hour
 
+
+# Safe Mode
+VOLUME_IGNORED = 10
 IGNORED_ERROR_CODES = (
     HTTPStatus.BAD_REQUEST,
     HTTPStatus.NOT_FOUND,
 )
-ERROR_TPL = "Request failed with status code {status_code} and reason {reason}"
+DOMO_SAFE_MODE = RequestSafeMode(
+    max_errors=VOLUME_IGNORED,
+    status_codes=IGNORED_ERROR_CODES,
+)
 
 _RETRY_EXCEPTIONS = [
     requests.exceptions.ConnectTimeout,
@@ -33,40 +49,17 @@ _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 logger = logging.getLogger(__name__)
 
 
-def _handle_response(response: requests.Response) -> requests.Response:
-    response.raise_for_status()
-    return response
-
-
-def _ignore_or_raise(
-    error: requests.RequestException,
-    ignore_error_codes: Optional[Tuple[int, ...]],
-) -> dict:
-    """
-    Raises the error unless the response status code is in the ignored error
-    codes list.
-    """
-    if not ignore_error_codes:
-        raise error
-
-    response = error.response
-    if response is None:
-        raise error
-
-    if response.status_code in ignore_error_codes:
-        logger.warning(error)
-        return {}
-
-    raise error
-
-
 class DomoClient:
     """
     Connect to Domo API and fetch main assets.
     https://developer.domo.com/portal/8ba9aedad3679-ap-is#platform-oauth-apis
     """
 
-    def __init__(self, credentials: DomoCredentials):
+    def __init__(
+        self,
+        credentials: DomoCredentials,
+        safe_mode: Optional[RequestSafeMode] = None,
+    ):
         self._authentication = credentials.authentication
         self._bearer_headers: Optional[dict] = None
         self._session = requests.session()
@@ -76,6 +69,7 @@ class DomoClient:
         self._timeout = DEFAULT_TIMEOUT
         self.base_url = credentials.base_url
         self.cloud_id = credentials.cloud_id
+        self.safe_mode = safe_mode or DOMO_SAFE_MODE
 
     def _token_expired(self) -> bool:
         token_lifetime = datetime.now() - self._token_creation_time
@@ -95,7 +89,8 @@ class DomoClient:
             auth=basic_authentication,
             timeout=self._timeout,
         )
-        result = _handle_response(response).json()
+        response.raise_for_status()
+        result = response.json()
 
         bearer_token = result["access_token"]
         self._bearer_headers = {"authorization": f"Bearer {bearer_token}"}
@@ -113,7 +108,7 @@ class DomoClient:
         endpoint: Endpoint,
         params: Optional[dict] = None,
         asset_id: Optional[str] = None,
-    ) -> requests.Response:
+    ) -> ResponseJson:
         params = params if params else {}
         is_private = endpoint.is_private
         headers = self._private_headers if is_private else self._bearer_auth()
@@ -125,27 +120,16 @@ class DomoClient:
             timeout=self._timeout,
         )
 
-        if response.status_code != HTTPStatus.OK:
-            logger.warning(
-                ERROR_TPL.format(
-                    status_code=response.status_code,
-                    reason=response.reason,
-                )
-            )
-        return response
+        return handle_response(response, self.safe_mode)
 
     def _get_element(
         self,
         endpoint: Endpoint,
         params: Optional[dict] = None,
         asset_id: Optional[str] = None,
-        ignore_error_codes: Optional[Tuple[int, ...]] = None,
     ) -> dict:
         """Used when the response only contains one element"""
-        try:
-            return self._get(endpoint, params, asset_id).json()
-        except requests.RequestException as error:
-            return _ignore_or_raise(error, ignore_error_codes)
+        return self._get(endpoint, params, asset_id)
 
     def _get_many(
         self,
@@ -154,7 +138,7 @@ class DomoClient:
         asset_id: Optional[str] = None,
     ) -> List[dict]:
         """Used when the response contains multiple elements"""
-        return self._get(endpoint, params, asset_id).json()
+        return self._get(endpoint, params, asset_id)
 
     def _get_paginated(self, endpoint: Endpoint) -> List[dict]:
         """Used when the response is paginated and need iterations"""
@@ -172,10 +156,7 @@ class DomoClient:
 
     def _datasources(self, page_id: str) -> RawData:
         endpoint = self._endpoint_factory.page_content(page_id)
-        page_content = self._get_element(
-            endpoint,
-            ignore_error_codes=IGNORED_ERROR_CODES,
-        )
+        page_content = self._get_element(endpoint)
         processed: set[str] = set()
         for card in page_content.get("cards", []):
             for datasource in card["datasources"]:
@@ -196,7 +177,7 @@ class DomoClient:
     ) -> Iterator[dict]:
         """Recursively fetch pages while building the folder architecture"""
         if not page_tree:
-            return []
+            return empty_iterator()
 
         for page in page_tree:
             page_id = page.get("id")
@@ -211,6 +192,10 @@ class DomoClient:
                 self._endpoint_factory.pages,
                 asset_id=page_id,
             )
+
+            if not detail:
+                continue
+
             datasources = self._datasources(page_id)
             yield {
                 **detail,

castor_extractor/visualization/domo/client/credentials.py

@@ -1,39 +1,26 @@
-from dataclasses import dataclass
-from enum import Enum
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional
 
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from requests.auth import HTTPBasicAuth
 
+DOMO_ENV_PREFIX = "CASTOR_DOMO_"
 
-class CredentialsKey(Enum):
-    """Value enum object for the credentials"""
 
-    API_TOKEN = "api_token"  # noqa: S105
-    BASE_URL = "base_url"
-    CLIENT_ID = "client_id"
-    CLOUD_ID = "cloud_id"
-    DEVELOPER_TOKEN = "developer_token"  # noqa: S105
-
-
-CLIENT_ALLOWED_KEYS: Tuple[str, ...] = tuple(c.value for c in CredentialsKey)
-
-
-@dataclass
-class DomoCredentials:
+class DomoCredentials(BaseSettings):
     """Class to handle Domo rest API permissions"""
 
-    api_token: str
+    model_config = SettingsConfigDict(
+        env_prefix=DOMO_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    api_token: str = Field(repr=False)
     base_url: str
     client_id: str
-    developer_token: str
-    cloud_id: Optional[str] = None
-
-    @classmethod
-    def from_secret(cls, secret: dict) -> "DomoCredentials":
-        credentials = {
-            k: v for k, v in secret.items() if k in CLIENT_ALLOWED_KEYS
-        }
-        return cls(**credentials)
+    cloud_id: Optional[str] = Field(validation_alias="CLOUD_ID", default=None)
+    developer_token: str = Field(repr=False)
 
     @property
     def authentication(self) -> HTTPBasicAuth:

castor_extractor/visualization/domo/extract.py

@@ -12,7 +12,6 @@ from ...utils import (
 )
 from .assets import DomoAsset
 from .client import DomoClient, DomoCredentials
-from .constants import API_TOKEN, BASE_URL, CLIENT_ID, CLOUD_ID, DEVELOPER_TOKEN
 
 logger = logging.getLogger(__name__)
 
@@ -57,18 +56,13 @@ def extract_all(
     """
 
     _output_directory = output_directory or from_env(OUTPUT_DIR)
-    _client_id = client_id or from_env(CLIENT_ID)
-    _base_url = base_url or from_env(BASE_URL)
-    _api_token = api_token or from_env(API_TOKEN)
-    _developer_token = developer_token or from_env(DEVELOPER_TOKEN)
-    _cloud_id = cloud_id or from_env(CLOUD_ID)
 
     credentials = DomoCredentials(
-        base_url=_base_url,
-        client_id=_client_id,
-        api_token=_api_token,
-        developer_token=_developer_token,
-        cloud_id=_cloud_id,
+        base_url=base_url,
+        client_id=client_id,
+        api_token=api_token,
+        developer_token=developer_token,
+        cloud_id=cloud_id,
     )
     client = DomoClient(credentials=credentials)
 
@@ -78,4 +72,4 @@ def extract_all(
         filename = get_output_filename(key.name.lower(), _output_directory, ts)
         write_json(filename, data)
 
-    write_summary(_output_directory, ts, base_url=_base_url)
+    write_summary(_output_directory, ts, base_url=credentials.base_url)

castor_extractor/visualization/looker/__init__.py

@@ -1,3 +1,8 @@
-from .api import ApiClient, Credentials, lookml_explore_names
+from .api import (
+    ApiClient,
+    ExtractionParameters,
+    LookerCredentials,
+    lookml_explore_names,
+)
 from .assets import LookerAsset
 from .extract import extract_all, iterate_all_data

castor_extractor/visualization/looker/api/__init__.py

@@ -1,3 +1,4 @@
 from .client import ApiClient
-from .sdk import Credentials
+from .credentials import LookerCredentials
+from .extraction_parameters import ExtractionParameters
 from .utils import lookml_explore_names