castor-extractor 0.17.4__py3-none-any.whl → 0.18.5__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 0.18.5 - 2024-07-17
+
+* Salesforce: extract DeveloperName and tooling url
+
+## 0.18.4 - 2024-07-16
+
+* Fix environment variables assignments for credentials
+
+## 0.18.3 - 2024-07-16
+
+* bump dependencies (minor and patches)
+
+## 0.18.2 - 2024-07-08
+
+* Added StatusCode handling to SafeMode
+
+## 0.18.1 - 2024-07-04
+
+* Bump dependencies: numpy, setuptools, tableauserverclient
+
+## 0.18.0 - 2024-07-03
+
+* Dashboarding technologies : Reworked credentials using Pydantic
+
+## 0.17.5 - 2024-07-03
+
+* Snowflake, Synapse, Redshift: Remove default_value from the extracted column
+
 ## 0.17.4 - 2024-07-03
 
 * Sigma: Add `input-table`, `pivot-table` and `viz` in the list of supported **Elements**

DockerfileUsage.md

@@ -0,0 +1,21 @@
+# Dockerfile usage for CastorDoc package
+
+## How To
+
+- The Dockerfile is present on the pypi package
+- For building it you should use this command `docker build -t castor-extractor-looker --build-arg EXTRA=looker .` with replacing looker one or several of: [bigquery,looker,metabase,powerbi,qlik,redshift,snowflake,tableau]
+- For running it you should do `docker run -v <local-path>:/data --env-file <castor-extract-looker.env> castor-extractor-looker` where `</local-path>` have to be replaced and `<castor-extract-looker.env>` have to be set.
+- Extracted datas would be available on `<local-path>`. The path should exists
+- `<castor-extract-looker.env>` would contain env vars for credentials, url...
+
+#### example
+
+```bash
+docker run -v /logs:/data --env-file /config/castor-extract-looker.env castor-extractor-looker
+```
+
+## Limitation
+
+- This docker image is for a specific techno
+- This docker image is based on python 3.11
+- This docker image use the latest castor-extractor package version

castor_extractor/commands/extract_domo.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import domo  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -38,13 +39,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    domo.extract_all(
-        api_token=args.api_token,
-        base_url=args.base_url,
-        client_id=args.client_id,
-        cloud_id=args.cloud_id,
-        developer_token=args.developer_token,
-        output_directory=args.output,
-    )
+    domo.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_looker.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import looker  # type: ignore
 
 
@@ -33,16 +34,4 @@ def main():
         action="store_true",
     )
 
-    args = parser.parse_args()
-
-    looker.extract_all(
-        base_url=args.base_url,
-        client_id=args.username,
-        client_secret=args.password,
-        log_to_stdout=args.log_to_stdout,
-        output_directory=args.output,
-        safe_mode=args.safe_mode,
-        search_per_folder=args.search_per_folder,
-        thread_pool_size=args.thread_pool_size,
-        timeout=args.timeout,
-    )
+    looker.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_metabase_api.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -15,15 +16,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
+    args = parse_filled_arguments(parser)
 
-    client = metabase.ApiClient(
-        base_url=args.base_url,
-        user=args.username,
-        password=args.password,
-    )
+    credentials = metabase.MetabaseApiCredentials(**args)
+    client = metabase.ApiClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

castor_extractor/commands/extract_metabase_db.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -33,20 +34,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = metabase.DbClient(
-        host=args.host,
-        port=args.port,
-        database=args.database,
-        schema=args.schema,
-        user=args.username,
-        password=args.password,
-        encryption_secret_key=args.encryption_secret_key,
-        require_ssl=args.require_ssl,
-    )
+    args = parse_filled_arguments(parser)
+    credentials = metabase.MetabaseDbCredentials(**args)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    client = metabase.DbClient(credentials)
+
+    metabase.extract_all(client, **args)

castor_extractor/commands/extract_mode.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import mode  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -24,16 +25,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = mode.Client(
-        host=args.host,
-        workspace=args.workspace,
-        token=args.token,
-        secret=args.secret,
-    )
-
-    mode.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    mode.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_powerbi.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import powerbi  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -20,11 +21,4 @@ def main():
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    powerbi.extract_all(
-        tenant_id=args.tenant_id,
-        client_id=args.client_id,
-        secret=args.secret,
-        scopes=args.scopes,
-        output_directory=args.output,
-    )
+    powerbi.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_qlik.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import qlik  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -22,10 +23,4 @@ def main():
         "missing rights on some assets.",
     )
 
-    args = parser.parse_args()
-    qlik.extract_all(
-        base_url=args.base_url,
-        api_key=args.api_key,
-        output_directory=args.output,
-        except_http_error_statuses=args.except_http_error_statuses,
-    )
+    qlik.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_salesforce.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.warehouse import salesforce  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -29,15 +30,5 @@ def main():
     )
     parser.set_defaults(skip_existing=False)
 
-    args = parser.parse_args()
-
-    salesforce.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-        skip_existing=args.skip_existing,
-    )
+    args = parse_filled_arguments(parser)
+    salesforce.extract_all(output_directory=args.get("output"), **args)

castor_extractor/commands/extract_salesforce_reporting.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import salesforce_reporting  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -21,13 +22,4 @@ def main():
     parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    salesforce_reporting.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-    )
+    salesforce_reporting.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_sigma.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import sigma  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -14,10 +15,4 @@ def main():
     parser.add_argument("-a", "--api-token", help="Generated API key")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    sigma.extract_all(
-        host=args.host,
-        client_id=args.client_id,
-        api_token=args.api_token,
-        output_directory=args.output,
-    )
+    sigma.extract_all(**parse_filled_arguments(parser))

castor_extractor/utils/__init__.py

@@ -1,3 +1,4 @@
+from .argument_parser import parse_filled_arguments
 from .client import (
     AbstractSourceClient,
     ExtractionQuery,
@@ -5,7 +6,7 @@ from .client import (
     SqlalchemyClient,
     uri_encode,
 )
-from .collection import group_by, mapping_from_rows
+from .collection import empty_iterator, group_by, mapping_from_rows
 from .constants import OUTPUT_DIR
 from .deprecate import deprecate_python
 from .env import from_env
@@ -23,6 +24,7 @@ from .pager import (
 )
 from .retry import RetryStrategy, retry
 from .safe import SafeMode, safe_mode
+from .safe_request import RequestSafeMode, ResponseJson, handle_response
 from .store import AbstractStorage, LocalStorage
 from .string import decode_when_bytes, string_to_tuple
 from .time import (

castor_extractor/utils/argument_parser.py

@@ -0,0 +1,7 @@
+from argparse import ArgumentParser
+
+
+def parse_filled_arguments(parser: ArgumentParser) -> dict:
+    """Parse arguments and remove all those with None values"""
+    parsed_arguments = vars(parser.parse_args())
+    return {k: v for k, v in parsed_arguments.items() if v is not None}

castor_extractor/utils/argument_parser_test.py

@@ -0,0 +1,25 @@
+from argparse import Namespace
+
+from .argument_parser import parse_filled_arguments
+
+
+class MockArgumentParser:
+
+    def __init__(self):
+        self.attributes = {}
+
+    def add_argument(self, name, value):
+        self.attributes[name] = value
+
+    def parse_args(self) -> Namespace:
+        return Namespace(**self.attributes)
+
+
+def test_parse_filled_arguments():
+    parser = MockArgumentParser()
+    parser.add_argument("filled", "value")
+    parser.add_argument("unfilled", None)
+    parser.add_argument("empty_str", "")
+
+    expected = {"filled": "value", "empty_str": ""}
+    assert parse_filled_arguments(parser) == expected

castor_extractor/utils/collection.py

@@ -44,3 +44,11 @@ def mapping_from_rows(rows: List[Dict], key: Any, value: Any) -> Dict:
         mapping[mapping_key] = mapping_value
 
     return mapping
+
+
+def empty_iterator():
+    """
+    Utils to return empty iterator, mainly used for viz transformers
+    Remark: missing return type is on purpose, it breaks the typing
+    """
+    return iter([])

castor_extractor/utils/safe_request.py

@@ -0,0 +1,57 @@
+import logging
+from typing import List, Tuple, Union
+
+from requests import HTTPError, Response
+
+logger = logging.getLogger(__name__)
+
+ResponseJson = Union[dict, List[dict]]
+
+
+class RequestSafeMode:
+    """
+    RequestSafeMode class to parameterize what should be done if response
+    raises due to the status code.
+
+    Attributes:
+        self.status_codes: tuple of status codes that will be caught
+        self.errors_caught : list of errors caught
+    """
+
+    def __init__(
+        self,
+        max_errors: Union[int, float] = 0,
+        status_codes: Tuple[int, ...] = (),
+    ):
+        self.max_errors = max_errors
+        self.status_codes: List[int] = list(status_codes)
+        self.status_codes_caught: List[int] = []
+
+    def catch_response(self, exception: HTTPError, status_code: int):
+        if int(status_code) not in self.status_codes:
+            raise exception
+
+        self.status_codes_caught.append(int(status_code))
+
+    @property
+    def should_raise(self) -> bool:
+        return len(self.status_codes_caught) > self.max_errors
+
+
+def handle_response(
+    response: Response, safe_mode: RequestSafeMode
+) -> ResponseJson:
+    """
+    Util to handle a HTTP Response based on the response status code and the
+    safe mode used
+    """
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        safe_mode.catch_response(e, response.status_code)
+        if safe_mode.should_raise:
+            raise e
+        logger.error(f"Safe mode : skip request with error {e}")
+        logger.debug(e, exc_info=True)
+        return {}
+    return response.json()

castor_extractor/utils/safe_request_test.py

@@ -0,0 +1,77 @@
+import io
+from http import HTTPStatus
+
+import pytest
+from requests import HTTPError, Response
+
+from .safe_request import RequestSafeMode, handle_response
+
+
+def mock_response(status_code: int):
+    response = Response()
+    response.status_code = status_code
+    response.raw = io.BytesIO(b'[{"data": "working"}]')
+    return response
+
+
+def test_http_error_with_no_safe_mode():
+    safe_params = RequestSafeMode()  # Caught
+
+    with pytest.raises(HTTPError):
+        handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+
+def test_http_error_with_no_status_code():
+    safe_params = RequestSafeMode(2)  # Caught
+
+    with pytest.raises(HTTPError):
+        handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+
+def test_http_error_with_status_code():
+    safe_params = RequestSafeMode(2, (HTTPStatus.FORBIDDEN,))  # Caught
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+    assert call() == {}
+    assert call() == {}
+
+    with pytest.raises(HTTPError):
+        call()
+
+
+def test_http_error_with_multiple_status_code():
+    safe_params = RequestSafeMode(
+        2, (HTTPStatus.NOT_FOUND, HTTPStatus.FORBIDDEN)
+    )  # Caught
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+    def call_2():
+        return handle_response(mock_response(HTTPStatus.NOT_FOUND), safe_params)
+
+    assert call() == {}
+    assert call_2() == {}
+    with pytest.raises(HTTPError):  # 3 failed calls > retries
+        call()
+
+
+def test_http_error_with_wrong_status_code():
+    safe_params = RequestSafeMode(2, (HTTPStatus.NOT_FOUND,))  # Wrong Status
+
+    def call():
+        handle_response(mock_response(HTTPStatus.BAD_REQUEST), safe_params)
+
+    with pytest.raises(HTTPError):
+        call()
+
+
+def test_http_error_with_return():
+    safe_params = RequestSafeMode(2, (HTTPStatus.NOT_FOUND,))  # Wrong Status
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.OK), safe_params)
+
+    assert call() == [{"data": "working"}]

castor_extractor/utils/salesforce/__init__.py

@@ -1,3 +1,2 @@
 from .client import SalesforceBaseClient
-from .constants import Keys
-from .credentials import SalesforceCredentials, to_credentials
+from .credentials import SalesforceCredentials

castor_extractor/utils/salesforce/constants.py

@@ -1,13 +1,2 @@
 DEFAULT_API_VERSION = 59.0
 DEFAULT_PAGINATION_LIMIT = 100
-
-
-class Keys:
-    """Salesforce's credentials keys"""
-
-    USERNAME = "username"
-    PASSWORD = "password"  # noqa: S105
-    CLIENT_ID = "client_id"
-    CLIENT_SECRET = "client_secret"  # noqa: S105
-    SECURITY_TOKEN = "security_token"  # noqa: S105
-    BASE_URL = "base_url"

castor_extractor/utils/salesforce/credentials.py

@@ -1,36 +1,33 @@
 from typing import Dict
 
-from ...utils import from_env
-from .constants import Keys
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
-_USERNAME = "CASTOR_SALESFORCE_USERNAME"
-_PASSWORD = "CASTOR_SALESFORCE_PASSWORD"  # noqa: S105
-_SECURITY_TOKEN = "CASTOR_SALESFORCE_SECURITY_TOKEN"  # noqa: S105
-_CLIENT_ID = "CASTOR_SALESFORCE_CLIENT_ID"
-_CLIENT_SECRET = "CASTOR_SALESFORCE_CLIENT_SECRET"  # noqa: S105
-_BASE_URL = "CASTOR_SALESFORCE_BASE_URL"
+CASTOR_ENV_PREFIX = "CASTOR_SALESFORCE_"
 
 
-class SalesforceCredentials:
+class SalesforceCredentials(BaseSettings):
     """
     Class to handle Salesforce rest API permissions
     """
 
-    def __init__(
-        self,
-        *,
-        username: str,
-        password: str,
-        security_token: str,
-        client_id: str,
-        client_secret: str,
-        base_url: str,
-    ):
-        self.username = username
-        self.password = password + security_token
-        self.client_id = client_id
-        self.client_secret = client_secret
-        self.base_url = base_url
+    model_config = SettingsConfigDict(
+        env_prefix=CASTOR_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    base_url: str
+    client_id: str
+    client_secret: str = Field(repr=False)
+    password: str = Field(repr=False)
+    security_token: str = Field(repr=False)
+    username: str
+
+    @property
+    def password_token(self) -> str:
+        """Generates the password for authentication"""
+        return self.password + self.security_token
 
     def token_request_payload(self) -> Dict[str, str]:
         """
@@ -41,25 +38,5 @@ class SalesforceCredentials:
             "client_id": self.client_id,
             "client_secret": self.client_secret,
             "username": self.username,
-            "password": self.password,
+            "password": self.password_token,
         }
-
-
-def to_credentials(params: dict) -> SalesforceCredentials:
-    """extract Salesforce credentials"""
-    username = params.get(Keys.USERNAME) or from_env(_USERNAME)
-    password = params.get(Keys.PASSWORD) or from_env(_PASSWORD)
-    security_token = params.get(Keys.SECURITY_TOKEN) or from_env(
-        _SECURITY_TOKEN
-    )
-    client_id = params.get(Keys.CLIENT_ID) or from_env(_CLIENT_ID)
-    client_secret = params.get(Keys.CLIENT_SECRET) or from_env(_CLIENT_SECRET)
-    base_url = params.get(Keys.BASE_URL) or from_env(_BASE_URL)
-    return SalesforceCredentials(
-        username=username,
-        password=password,
-        client_id=client_id,
-        client_secret=client_secret,
-        security_token=security_token,
-        base_url=base_url,
-    )

castor_extractor/visualization/domo/__init__.py

@@ -1,3 +1,3 @@
 from .assets import DomoAsset
-from .client import CredentialsKey, DomoClient, DomoCredentials
+from .client import DomoClient, DomoCredentials
 from .extract import extract_all

castor_extractor/visualization/domo/client/__init__.py

@@ -1,2 +1,2 @@
 from .client import DomoClient
-from .credentials import CredentialsKey, DomoCredentials
+from .credentials import DomoCredentials