castor-extractor 0.17.4__tar.gz → 0.18.5__tar.gz

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 0.18.5 - 2024-07-17
+
+* Salesforce: extract DeveloperName and tooling url
+
+## 0.18.4 - 2024-07-16
+
+* Fix environment variables assignments for credentials
+
+## 0.18.3 - 2024-07-16
+
+* bump dependencies (minor and patches)
+
+## 0.18.2 - 2024-07-08
+
+* Added StatusCode handling to SafeMode
+
+## 0.18.1 - 2024-07-04
+
+* Bump dependencies: numpy, setuptools, tableauserverclient
+
+## 0.18.0 - 2024-07-03
+
+* Dashboarding technologies : Reworked credentials using Pydantic
+
+## 0.17.5 - 2024-07-03
+
+* Snowflake, Synapse, Redshift: Remove default_value from the extracted column
+
 ## 0.17.4 - 2024-07-03
 
 * Sigma: Add `input-table`, `pivot-table` and `viz` in the list of supported **Elements**

castor_extractor-0.18.5/DockerfileUsage.md

@@ -0,0 +1,21 @@
+# Dockerfile usage for CastorDoc package
+
+## How To
+
+- The Dockerfile is present on the pypi package
+- For building it you should use this command `docker build -t castor-extractor-looker --build-arg EXTRA=looker .` with replacing looker one or several of: [bigquery,looker,metabase,powerbi,qlik,redshift,snowflake,tableau]
+- For running it you should do `docker run -v <local-path>:/data --env-file <castor-extract-looker.env> castor-extractor-looker` where `</local-path>` have to be replaced and `<castor-extract-looker.env>` have to be set.
+- Extracted datas would be available on `<local-path>`. The path should exists
+- `<castor-extract-looker.env>` would contain env vars for credentials, url...
+
+#### example
+
+```bash
+docker run -v /logs:/data --env-file /config/castor-extract-looker.env castor-extractor-looker
+```
+
+## Limitation
+
+- This docker image is for a specific techno
+- This docker image is based on python 3.11
+- This docker image use the latest castor-extractor package version

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.17.4
+Version: 0.18.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,13 +37,13 @@ Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
-Requires-Dist: looker-sdk (>=23.0.0) ; extra == "looker" or extra == "all"
+Requires-Dist: looker-sdk (>=23.0.0,<24.0.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
-Requires-Dist: numpy (<1.25) ; python_version >= "3.8" and python_version < "3.9"
-Requires-Dist: numpy (>=1.26,<2) ; python_version >= "3.12" and python_version < "3.13"
-Requires-Dist: pandas (>=2,<2.2.0) ; python_version >= "3.9" and python_full_version <= "3.11.0"
-Requires-Dist: pandas (>=2.0,<2.1) ; python_version >= "3.8" and python_version < "3.9"
-Requires-Dist: pandas (>=2.1,<2.2.0) ; python_version >= "3.12" and python_version < "3.13"
+Requires-Dist: numpy (<1.25) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "bigquery" or extra == "databricks" or extra == "all")
+Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
+Requires-Dist: numpy (>=1.26) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "bigquery" or extra == "databricks" or extra == "all")
+Requires-Dist: pandas (<2.1) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "databricks" or extra == "all")
+Requires-Dist: pandas (>=2.1) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "databricks" or extra == "all")
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
 Requires-Dist: pydantic (>=2.6,<3.0)
@@ -52,13 +52,13 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=69,<70)
+Requires-Dist: setuptools (>=70,<71)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
 Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigquery" or extra == "all"
 Requires-Dist: sqlalchemy-redshift (>=0.8.14,<0.9.0) ; extra == "redshift" or extra == "all"
-Requires-Dist: tableauserverclient (==0.17.0) ; extra == "tableau" or extra == "all"
+Requires-Dist: tableauserverclient (==0.25.0) ; extra == "tableau" or extra == "all"
 Requires-Dist: tqdm (>=4.0.0,<5.0.0)
 Requires-Dist: typing-extensions (>=4,<5)
 Requires-Dist: websocket-client (>=1,<2) ; extra == "qlik" or extra == "all"

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_domo.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import domo  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -38,13 +39,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    domo.extract_all(
-        api_token=args.api_token,
-        base_url=args.base_url,
-        client_id=args.client_id,
-        cloud_id=args.cloud_id,
-        developer_token=args.developer_token,
-        output_directory=args.output,
-    )
+    domo.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_looker.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import looker  # type: ignore
 
 
@@ -33,16 +34,4 @@ def main():
         action="store_true",
     )
 
-    args = parser.parse_args()
-
-    looker.extract_all(
-        base_url=args.base_url,
-        client_id=args.username,
-        client_secret=args.password,
-        log_to_stdout=args.log_to_stdout,
-        output_directory=args.output,
-        safe_mode=args.safe_mode,
-        search_per_folder=args.search_per_folder,
-        thread_pool_size=args.thread_pool_size,
-        timeout=args.timeout,
-    )
+    looker.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_metabase_api.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -15,15 +16,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
+    args = parse_filled_arguments(parser)
 
-    client = metabase.ApiClient(
-        base_url=args.base_url,
-        user=args.username,
-        password=args.password,
-    )
+    credentials = metabase.MetabaseApiCredentials(**args)
+    client = metabase.ApiClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_metabase_db.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -33,20 +34,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = metabase.DbClient(
-        host=args.host,
-        port=args.port,
-        database=args.database,
-        schema=args.schema,
-        user=args.username,
-        password=args.password,
-        encryption_secret_key=args.encryption_secret_key,
-        require_ssl=args.require_ssl,
-    )
+    args = parse_filled_arguments(parser)
+    credentials = metabase.MetabaseDbCredentials(**args)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    client = metabase.DbClient(credentials)
+
+    metabase.extract_all(client, **args)

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_mode.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import mode  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -24,16 +25,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = mode.Client(
-        host=args.host,
-        workspace=args.workspace,
-        token=args.token,
-        secret=args.secret,
-    )
-
-    mode.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    mode.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_powerbi.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import powerbi  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -20,11 +21,4 @@ def main():
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    powerbi.extract_all(
-        tenant_id=args.tenant_id,
-        client_id=args.client_id,
-        secret=args.secret,
-        scopes=args.scopes,
-        output_directory=args.output,
-    )
+    powerbi.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_qlik.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import qlik  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -22,10 +23,4 @@ def main():
         "missing rights on some assets.",
     )
 
-    args = parser.parse_args()
-    qlik.extract_all(
-        base_url=args.base_url,
-        api_key=args.api_key,
-        output_directory=args.output,
-        except_http_error_statuses=args.except_http_error_statuses,
-    )
+    qlik.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_salesforce.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.warehouse import salesforce  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -29,15 +30,5 @@ def main():
     )
     parser.set_defaults(skip_existing=False)
 
-    args = parser.parse_args()
-
-    salesforce.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-        skip_existing=args.skip_existing,
-    )
+    args = parse_filled_arguments(parser)
+    salesforce.extract_all(output_directory=args.get("output"), **args)

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_salesforce_reporting.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import salesforce_reporting  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -21,13 +22,4 @@ def main():
     parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    salesforce_reporting.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-    )
+    salesforce_reporting.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/commands/extract_sigma.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import sigma  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -14,10 +15,4 @@ def main():
     parser.add_argument("-a", "--api-token", help="Generated API key")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    sigma.extract_all(
-        host=args.host,
-        client_id=args.client_id,
-        api_token=args.api_token,
-        output_directory=args.output,
-    )
+    sigma.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/utils/__init__.py

@@ -1,3 +1,4 @@
+from .argument_parser import parse_filled_arguments
 from .client import (
     AbstractSourceClient,
     ExtractionQuery,
@@ -5,7 +6,7 @@ from .client import (
     SqlalchemyClient,
     uri_encode,
 )
-from .collection import group_by, mapping_from_rows
+from .collection import empty_iterator, group_by, mapping_from_rows
 from .constants import OUTPUT_DIR
 from .deprecate import deprecate_python
 from .env import from_env
@@ -23,6 +24,7 @@ from .pager import (
 )
 from .retry import RetryStrategy, retry
 from .safe import SafeMode, safe_mode
+from .safe_request import RequestSafeMode, ResponseJson, handle_response
 from .store import AbstractStorage, LocalStorage
 from .string import decode_when_bytes, string_to_tuple
 from .time import (

castor_extractor-0.18.5/castor_extractor/utils/argument_parser.py

@@ -0,0 +1,7 @@
+from argparse import ArgumentParser
+
+
+def parse_filled_arguments(parser: ArgumentParser) -> dict:
+    """Parse arguments and remove all those with None values"""
+    parsed_arguments = vars(parser.parse_args())
+    return {k: v for k, v in parsed_arguments.items() if v is not None}

castor_extractor-0.18.5/castor_extractor/utils/argument_parser_test.py

@@ -0,0 +1,25 @@
+from argparse import Namespace
+
+from .argument_parser import parse_filled_arguments
+
+
+class MockArgumentParser:
+
+    def __init__(self):
+        self.attributes = {}
+
+    def add_argument(self, name, value):
+        self.attributes[name] = value
+
+    def parse_args(self) -> Namespace:
+        return Namespace(**self.attributes)
+
+
+def test_parse_filled_arguments():
+    parser = MockArgumentParser()
+    parser.add_argument("filled", "value")
+    parser.add_argument("unfilled", None)
+    parser.add_argument("empty_str", "")
+
+    expected = {"filled": "value", "empty_str": ""}
+    assert parse_filled_arguments(parser) == expected

{castor_extractor-0.17.4 → castor_extractor-0.18.5}/castor_extractor/utils/collection.py

@@ -44,3 +44,11 @@ def mapping_from_rows(rows: List[Dict], key: Any, value: Any) -> Dict:
         mapping[mapping_key] = mapping_value
 
     return mapping
+
+
+def empty_iterator():
+    """
+    Utils to return empty iterator, mainly used for viz transformers
+    Remark: missing return type is on purpose, it breaks the typing
+    """
+    return iter([])

castor_extractor-0.18.5/castor_extractor/utils/safe_request.py

@@ -0,0 +1,57 @@
+import logging
+from typing import List, Tuple, Union
+
+from requests import HTTPError, Response
+
+logger = logging.getLogger(__name__)
+
+ResponseJson = Union[dict, List[dict]]
+
+
+class RequestSafeMode:
+    """
+    RequestSafeMode class to parameterize what should be done if response
+    raises due to the status code.
+
+    Attributes:
+        self.status_codes: tuple of status codes that will be caught
+        self.errors_caught : list of errors caught
+    """
+
+    def __init__(
+        self,
+        max_errors: Union[int, float] = 0,
+        status_codes: Tuple[int, ...] = (),
+    ):
+        self.max_errors = max_errors
+        self.status_codes: List[int] = list(status_codes)
+        self.status_codes_caught: List[int] = []
+
+    def catch_response(self, exception: HTTPError, status_code: int):
+        if int(status_code) not in self.status_codes:
+            raise exception
+
+        self.status_codes_caught.append(int(status_code))
+
+    @property
+    def should_raise(self) -> bool:
+        return len(self.status_codes_caught) > self.max_errors
+
+
+def handle_response(
+    response: Response, safe_mode: RequestSafeMode
+) -> ResponseJson:
+    """
+    Util to handle a HTTP Response based on the response status code and the
+    safe mode used
+    """
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        safe_mode.catch_response(e, response.status_code)
+        if safe_mode.should_raise:
+            raise e
+        logger.error(f"Safe mode : skip request with error {e}")
+        logger.debug(e, exc_info=True)
+        return {}
+    return response.json()

castor_extractor-0.18.5/castor_extractor/utils/safe_request_test.py

@@ -0,0 +1,77 @@
+import io
+from http import HTTPStatus
+
+import pytest
+from requests import HTTPError, Response
+
+from .safe_request import RequestSafeMode, handle_response
+
+
+def mock_response(status_code: int):
+    response = Response()
+    response.status_code = status_code
+    response.raw = io.BytesIO(b'[{"data": "working"}]')
+    return response
+
+
+def test_http_error_with_no_safe_mode():
+    safe_params = RequestSafeMode()  # Caught
+
+    with pytest.raises(HTTPError):
+        handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+
+def test_http_error_with_no_status_code():
+    safe_params = RequestSafeMode(2)  # Caught
+
+    with pytest.raises(HTTPError):
+        handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+
+def test_http_error_with_status_code():
+    safe_params = RequestSafeMode(2, (HTTPStatus.FORBIDDEN,))  # Caught
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+    assert call() == {}
+    assert call() == {}
+
+    with pytest.raises(HTTPError):
+        call()
+
+
+def test_http_error_with_multiple_status_code():
+    safe_params = RequestSafeMode(
+        2, (HTTPStatus.NOT_FOUND, HTTPStatus.FORBIDDEN)
+    )  # Caught
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.FORBIDDEN), safe_params)
+
+    def call_2():
+        return handle_response(mock_response(HTTPStatus.NOT_FOUND), safe_params)
+
+    assert call() == {}
+    assert call_2() == {}
+    with pytest.raises(HTTPError):  # 3 failed calls > retries
+        call()
+
+
+def test_http_error_with_wrong_status_code():
+    safe_params = RequestSafeMode(2, (HTTPStatus.NOT_FOUND,))  # Wrong Status
+
+    def call():
+        handle_response(mock_response(HTTPStatus.BAD_REQUEST), safe_params)
+
+    with pytest.raises(HTTPError):
+        call()
+
+
+def test_http_error_with_return():
+    safe_params = RequestSafeMode(2, (HTTPStatus.NOT_FOUND,))  # Wrong Status
+
+    def call():
+        return handle_response(mock_response(HTTPStatus.OK), safe_params)
+
+    assert call() == [{"data": "working"}]

castor_extractor-0.18.5/castor_extractor/utils/salesforce/__init__.py

@@ -0,0 +1,2 @@
+from .client import SalesforceBaseClient
+from .credentials import SalesforceCredentials

castor_extractor-0.18.5/castor_extractor/utils/salesforce/constants.py

@@ -0,0 +1,2 @@
+DEFAULT_API_VERSION = 59.0
+DEFAULT_PAGINATION_LIMIT = 100

castor_extractor-0.18.5/castor_extractor/utils/salesforce/credentials.py

@@ -0,0 +1,42 @@
+from typing import Dict
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+CASTOR_ENV_PREFIX = "CASTOR_SALESFORCE_"
+
+
+class SalesforceCredentials(BaseSettings):
+    """
+    Class to handle Salesforce rest API permissions
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix=CASTOR_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    base_url: str
+    client_id: str
+    client_secret: str = Field(repr=False)
+    password: str = Field(repr=False)
+    security_token: str = Field(repr=False)
+    username: str
+
+    @property
+    def password_token(self) -> str:
+        """Generates the password for authentication"""
+        return self.password + self.security_token
+
+    def token_request_payload(self) -> Dict[str, str]:
+        """
+        Params to post to the API in order to retrieve the authentication token
+        """
+        return {
+            "grant_type": "password",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+            "username": self.username,
+            "password": self.password_token,
+        }

castor_extractor-0.18.5/castor_extractor/visualization/domo/__init__.py

@@ -0,0 +1,3 @@
+from .assets import DomoAsset
+from .client import DomoClient, DomoCredentials
+from .extract import extract_all

castor_extractor-0.18.5/castor_extractor/visualization/domo/client/__init__.py

@@ -0,0 +1,2 @@
+from .client import DomoClient
+from .credentials import DomoCredentials