castor-extractor 0.24.21__tar.gz → 0.24.25__tar.gz

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.24.25 - 2025-06-12
+
+* DBT: Fix API base url
+
+## 0.24.24 - 2025-06-06
+
+* Power BI: handle rate limit issues when extracting pages
+
+## 0.24.23 - 2025-06-05
+
+* Salesforce: print response's error message when authentication fails
+
+## 0.24.22 - 2025-05-27
+
+* Add retry for `Request.Timeout` on **ApiClient**
+
 ## 0.24.21 - 2025-05-26
 
 * Looker Studio: add option to skip the extraction of view activity logs

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.21
+Version: 0.24.25
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,22 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.25 - 2025-06-12
+
+* DBT: Fix API base url
+
+## 0.24.24 - 2025-06-06
+
+* Power BI: handle rate limit issues when extracting pages
+
+## 0.24.23 - 2025-06-05
+
+* Salesforce: print response's error message when authentication fails
+
+## 0.24.22 - 2025-05-27
+
+* Add retry for `Request.Timeout` on **ApiClient**
+
 ## 0.24.21 - 2025-05-26
 
 * Looker Studio: add option to skip the extraction of view activity logs

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/transformation/coalesce/client/client.py

@@ -1,14 +1,14 @@
+import logging
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-import requests
+from requests import ConnectionError
 
 from ....utils import (
     APIClient,
     BearerAuth,
     RequestSafeMode,
     SerializedAsset,
-    retry,
 )
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
@@ -20,9 +20,8 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
-_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
-_RETRY_COUNT = 2
-_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
+
+logger = logging.getLogger(__name__)
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -75,11 +74,6 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
-    @retry(
-        exceptions=_RETRY_EXCEPTIONS,
-        max_retries=_RETRY_COUNT,
-        base_ms=_RETRY_BASE_MS,
-    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id
@@ -91,8 +85,14 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         nodes: list[dict] = []
         for node in result["data"]:
-            details = self._node_details(environment_id, node["id"])
-            nodes.append({**node, **details})
+            try:
+                details = self._node_details(environment_id, node["id"])
+                nodes.append({**node, **details})
+            except ConnectionError as e:
+                node_id = node["id"]
+                message = f"ConnectionError, environment: {environment_id}, node: {node_id}"
+                logger.warning(message)
+                raise e
         return nodes
 
     def _fetch_all_nodes(self) -> SerializedAsset:

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/transformation/dbt/client.py

@@ -8,12 +8,13 @@ from typing import Literal, Optional
 import requests
 from dateutil.parser import parse
 
+from ...utils.url import add_path
 from .credentials import DbtCredentials
 
 logger = logging.getLogger(__name__)
 
 
-_URL_SUFFIX = "/api/v2/accounts"
+_URL_SUFFIX = "/api/v2/accounts/"
 
 _DATA_KEY = "data"
 _SUCCESSFUL_RUN_STATUS = 10
@@ -52,7 +53,7 @@ class DbtClient:
         self._credentials = credentials
         self._account_url = _account_url(self._credentials.host)
         self._session = requests.Session()
-        self._account_id: int = self._infer_account_id()
+        self._account_id: str = self._infer_account_id()
 
     def _headers(self, content_type: ContentType) -> dict:
         return {
@@ -88,16 +89,16 @@ class DbtClient:
                 return result[_DATA_KEY]
             return result
 
-    def _infer_account_id(self) -> int:
+    def _infer_account_id(self) -> str:
         result = self._call(url=self._account_url)
-        return result[0]["id"]
+        return str(result[0]["id"])
 
     def list_job_identifiers(self) -> set[int]:
         """
         Return the IDs of all non-deleted jobs for this account
         https://docs.getdbt.com/dbt-cloud/api-v2-legacy#tag/Jobs/operation/listJobsForAccount
         """
-        url = f"{self._account_url}/{self._account_id}/jobs/"
+        url = add_path(self._account_url, self._account_id, "jobs", "/")
         jobs = self._call(url)
         return {job["id"] for job in jobs if not _is_deleted(job)}
 
@@ -110,7 +111,7 @@ class DbtClient:
         Extract the last successful run id, optionally filtered on a given datetime range
         https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/listRunsForAccount
         """
-        url = f"{self._account_url}/{self._account_id}/runs/"
+        url = add_path(self._account_url, self._account_id, "runs", "/")
 
         params = {
             "job_definition_id": job_id or self._credentials.job_id,
@@ -142,7 +143,14 @@ class DbtClient:
         Fetch dbt manifest or run results
         https://docs.getdbt.com/dbt-cloud/api-v2-legacy#tag/Runs/operation/getArtifactsByRunId
         """
-        url = f"{self._account_url}/{self._account_id}/runs/{run_id}/artifacts/{artifact}"
+        url = add_path(
+            self._account_url,
+            self._account_id,
+            "runs",
+            str(run_id),
+            "artifacts",
+            artifact,
+        )
         logger.info(
             f"Extracting {artifact} from run id {run_id} with url {url}"
         )

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/transformation/dbt/client_test.py

@@ -46,7 +46,7 @@ def test_DbtClient_last_run():
     mock_response_default_job = [{"id": 1, "finished_at": _OLD_DATE_STR}]
     mock_response_job_42 = [{"id": 2, "finished_at": _RECENT_DATE_STR}]
 
-    with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
+    with patch(infer_path, return_value="40"), patch(call_path) as mocked_call:
         credentials = DbtCredentials(token="some-token", job_id=default_job_id)
 
         dbt_client = DbtClient(credentials=credentials)
@@ -85,7 +85,7 @@ def test_DbtClient_list_job_identifiers():
         {"id": 395, "state": 1},
     ]
 
-    with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
+    with patch(infer_path, return_value="40"), patch(call_path) as mocked_call:
         mocked_call.return_value = jobs
         credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
@@ -100,7 +100,7 @@ def test_DbtClient_fetch_artifacts():
     run_id = 12345
     url = "https://cloud.getdbt.com/api/v2/accounts/40/runs/{}/artifacts/{}"
 
-    with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
+    with patch(infer_path, return_value="40"), patch(call_path) as mocked_call:
         credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
@@ -123,7 +123,7 @@ def test_DbtClient_fetch_artifacts():
 
 def test___account_url():
     base_url = "https://cloud.getdbt.com"
-    assert _account_url(base_url) == "https://cloud.getdbt.com/api/v2/accounts"
+    assert _account_url(base_url) == "https://cloud.getdbt.com/api/v2/accounts/"
 
     base_url = "https://emea.dbt.com/"
-    assert _account_url(base_url) == "https://emea.dbt.com/api/v2/accounts"
+    assert _account_url(base_url) == "https://emea.dbt.com/api/v2/accounts/"

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/utils/__init__.py

@@ -52,6 +52,7 @@ from .time import (
     yesterday,
 )
 from .type import Callback, Getter, JsonType, SerializedAsset
+from .url import add_path as add_path_to_url, url_from
 from .validation import clean_path, validate_baseurl
 from .write import (
     get_output_filename,

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/utils/client/api/client.py

@@ -5,7 +5,7 @@ from typing import Callable, Literal, Optional
 import requests
 from requests import Response
 
-from ...retry import retry_request
+from ...retry import retry, retry_request
 from .auth import Auth
 from .safe_request import RequestSafeMode, handle_response
 from .utils import build_url
@@ -21,6 +21,10 @@ DEFAULT_TIMEOUT = 60
 RETRY_ON_EXPIRED_TOKEN = 1
 RETRY_ON_GATEWAY_TIMEOUT = 3
 
+_TIMEOUT_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_TIMEOUT_RETRY_COUNT = 2
+_TIMEOUT_RETRY_EXCEPTIONS = (requests.exceptions.Timeout,)
+
 
 def _generate_payloads(
     method: HttpMethod,
@@ -81,6 +85,7 @@ class APIClient:
         params: Optional[dict] = None,
         data: Optional[dict] = None,
         pagination_params: Optional[dict] = None,
+        retry_on_timeout: bool = True,
     ) -> Response:
         headers = headers or {}
 
@@ -93,7 +98,17 @@ class APIClient:
 
         url = build_url(self._host, endpoint)
 
-        return requests.request(
+        if retry_on_timeout:
+            retry_wrapper = retry(
+                exceptions=_TIMEOUT_RETRY_EXCEPTIONS,
+                max_retries=_TIMEOUT_RETRY_COUNT,
+                base_ms=_TIMEOUT_RETRY_BASE_MS,
+            )
+            request_fn = retry_wrapper(requests.request)
+        else:
+            request_fn = requests.request
+
+        return request_fn(
             method=method,
             url=url,
             auth=self._auth,
@@ -119,6 +134,7 @@ class APIClient:
         params: Optional[dict] = None,
         data: Optional[dict] = None,
         pagination_params: Optional[dict] = None,
+        retry_on_timeout: bool = True,
     ):
         response = self._call(
             method="GET",
@@ -127,6 +143,7 @@ class APIClient:
             data=data,
             pagination_params=pagination_params,
             headers=headers,
+            retry_on_timeout=retry_on_timeout,
         )
         if response.status_code == HTTPStatus.UNAUTHORIZED:
             self._auth.refresh_token()

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/utils/retry.py

@@ -26,6 +26,15 @@ class RetryStrategy(Enum):
 DEFAULT_STRATEGY = RetryStrategy.CONSTANT
 
 
+def _warning(callable: Callable, exception: BaseException) -> None:
+    exception_type = type(exception)
+    exception_path = f"{exception_type.__module__}.{exception_type.__name__}"
+    callable_path = f"{callable.__module__}.{callable.__name__}"
+
+    msg = f"Exception '{exception_path}' occurred within `{callable_path}`"
+    logger.warning(msg)
+
+
 class Retry(BaseModel):
     """
     This class checks if the retry conditions are met, and if so, how long to
@@ -96,7 +105,7 @@ def retry(
             try:
                 return None, callable(*args, **kwargs)
             except exceptions_ as err:
-                logger.warning(f"Exception within {callable.__name__}")
+                _warning(callable, err)
                 return err, None
 
         def _func(*args, **kwargs) -> Any:
@@ -139,7 +148,7 @@ def retry_request(
                 status_code = err.response.status_code
                 if status_code not in exceptions_:
                     raise err
-                logger.warning(f"Exception within {callable.__name__}")
+                _warning(callable, err)
                 return err, None
 
         def _func(*args, **kwargs) -> Any:

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/utils/salesforce/client.py

@@ -1,9 +1,11 @@
 import logging
 from collections.abc import Iterator
 from functools import partial
+from http import HTTPStatus
 from typing import Optional
 
 import requests
+from requests import HTTPError, Response
 
 from ...utils import (
     APIClient,
@@ -21,6 +23,21 @@ logger = logging.getLogger(__name__)
 SALESFORCE_TIMEOUT_S = 120
 
 
+class SalesforceBadRequestError(HTTPError):
+    """
+    Custom Exception to print the response's text when an error occurs
+    during Salesforce's authentication.
+    """
+
+    def __init__(self, response: Response):
+        text = response.text
+        message = (
+            f"{response.status_code} Client Error: {response.reason} for url: {response.url}"
+            f"\nResponse text: {text}"
+        )
+        super().__init__(message, response=response)
+
+
 class SalesforceAuth(BearerAuth):
     _AUTH_ENDPOINT = "services/oauth2/token"
 
@@ -29,8 +46,16 @@ class SalesforceAuth(BearerAuth):
         self._token_payload = credentials.token_request_payload()
 
     def fetch_token(self) -> Optional[str]:
+        """
+        Fetches the access token from Salesforce using the provided credentials.
+        A custom Exception is raised if the request fails with a 400 status code.
+        """
         url = build_url(self._host, self._AUTH_ENDPOINT)
         response = requests.post(url, "POST", params=self._token_payload)
+
+        if response.status_code == HTTPStatus.BAD_REQUEST:
+            raise SalesforceBadRequestError(response)
+
         handled_response = handle_response(response)
         return handled_response["access_token"]
 

castor_extractor-0.24.25/castor_extractor/utils/url.py

@@ -0,0 +1,48 @@
+from urllib.parse import urlsplit, urlunsplit
+
+
+def url_from(
+    scheme: str = "",
+    netloc: str = "",
+    path: str = "",
+    query: str = "",
+    fragment: str = "",
+) -> str:
+    """Constructs an url from part"""
+    return urlunsplit((scheme, netloc, path, query, fragment))
+
+
+def add_path(base_url: str, *paths: str) -> str:
+    """Adds a path from a base_url."""
+
+    if not is_valid(base_url):
+        raise ValueError(f"Invalid base_url: {base_url}")
+    base_url = _format_base_url(base_url)
+    split = urlsplit(base_url)
+
+    return url_from(
+        split.scheme,
+        split.netloc,
+        "/".join([split.path] + [p.strip("/") for p in paths]),
+        split.query,
+        split.fragment,
+    )
+
+
+def _format_base_url(url: str) -> str:
+    """Remove trailing slash in base url, if applicable."""
+    if url.endswith("/"):
+        return url[:-1]
+    return url
+
+
+def is_valid(
+    url: str,
+    valid_schemes: tuple[str, ...] = ("http", "https"),
+) -> bool:
+    """
+    Simple url validation that ensures the scheme and that there is an hostname.
+    Malformatted url can pass this check such as http://http://toto.com
+    """
+    split = urlsplit(url)
+    return split.scheme in valid_schemes and bool(split.netloc)

castor_extractor-0.24.25/castor_extractor/utils/url_test.py

@@ -0,0 +1,55 @@
+from pytest import raises
+
+from ..utils.url import (
+    add_path,
+    is_valid,
+    url_from,
+)
+
+
+def test_add_path():
+    base = "https://test.com"
+
+    # simple
+    assert add_path(base, "toto") == f"{base}/toto"
+
+    # multiple parts
+    assert add_path(base, "to", "ta") == f"{base}/to/ta"
+
+    # multiple parts with slash
+    assert add_path(base, "a/b", "/c/d") == f"{base}/a/b/c/d"
+
+    # base with path
+    assert add_path(f"{base}/my/path", "/1/2/", "3") == f"{base}/my/path/1/2/3"
+
+    # base with query string and fragment
+    assert add_path(f"{base}?q=2#frag", "1/2") == f"{base}/1/2?q=2#frag"
+
+    # bad base url
+    with raises(ValueError):
+        add_path("toto", "toto")
+
+    # trailing slash
+    base = "https://test.com/"
+
+    # multiple parts with slash
+    assert add_path(base, "a/b", "/c/d") == "https://test.com/a/b/c/d"
+
+
+def test_url_is_valid():
+    # valid
+    assert is_valid("https://google.com")
+    assert is_valid("http://user:pass@test.com:444/my/path?my=query#fragment")
+    assert is_valid("ftp://hello.com", valid_schemes=("ftp",))
+
+    # invalid
+    assert not is_valid("hello.com")
+    assert not is_valid("ftp://hello.com")
+    assert not is_valid("http://")
+
+
+def test_url_from():
+    assert url_from() == ""
+    assert url_from("http") == "http://"
+    assert url_from("https", "google.com") == "https://google.com"
+    assert url_from(netloc="te.st", query="q=3") == "//te.st?q=3"

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/visualization/powerbi/client/client.py

@@ -2,6 +2,7 @@ import logging
 from collections.abc import Iterator
 from datetime import date
 from functools import partial
+from http import HTTPStatus
 from time import sleep
 from typing import Optional, Union
 
@@ -11,6 +12,7 @@ from requests import HTTPError
 from ....utils import (
     APIClient,
     fetch_all_pages,
+    retry_request,
 )
 from ..assets import PowerBiAsset
 from .authentication import PowerBiBearerAuth
@@ -27,6 +29,9 @@ METADATA_BATCH_SIZE = 100
 POWERBI_SCAN_STATUS_DONE = "Succeeded"
 POWERBI_SCAN_SLEEP_S = 1
 
+MAX_RETRY_PAGES = 1
+RETRY_PAGES_TIMEOUT_MS = 35 * 1000  # 35 seconds
+
 logger = logging.getLogger(__name__)
 
 
@@ -71,6 +76,23 @@ class PowerbiClient(APIClient):
         """
         yield from self._get(self.endpoint_factory.dashboards())[Keys.VALUE]
 
+    @retry_request(
+        status_codes=(HTTPStatus.TOO_MANY_REQUESTS,),
+        max_retries=MAX_RETRY_PAGES,
+        base_ms=RETRY_PAGES_TIMEOUT_MS,
+    )
+    def _pages(self, report_id: str) -> Iterator[dict]:
+        """
+        Extracts the pages of a report.
+        This endpoint is very flaky and frequently returns 400 and 404 errors.
+        After around 50 requests, it hits the rate limit and returns 429 Too Many Requests,
+        which is why we retry it after a short delay.
+        Timeouts are also common; we must skip them because the extraction task
+        might take too long otherwise.
+        """
+        pages_endpoint = self.endpoint_factory.pages(report_id)
+        return self._get(pages_endpoint, retry_on_timeout=False)[Keys.VALUE]
+
     def _reports(self) -> Iterator[dict]:
         """
         Returns a list of reports for the organization.
@@ -83,8 +105,7 @@ class PowerbiClient(APIClient):
             report_id = report.get(Keys.ID)
 
             try:
-                pages_endpoint = self.endpoint_factory.pages(report_id)
-                pages = self._get(pages_endpoint)[Keys.VALUE]
+                pages = self._pages(report_id)
                 report["pages"] = pages
             except (requests.HTTPError, requests.exceptions.Timeout) as e:
                 logger.debug(e)

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/castor_extractor/visualization/powerbi/client/client_test.py

@@ -85,7 +85,7 @@ def test__reports(power_bi_client):
         reports = list(power_bi_client._reports())
         calls = [
             call(ENDPOINT_FACTORY.reports()),
-            call(ENDPOINT_FACTORY.pages("1")),
+            call(ENDPOINT_FACTORY.pages("1"), retry_on_timeout=False),
         ]
         mocked_get.assert_has_calls(calls)
         assert reports == [

{castor_extractor-0.24.21 → castor_extractor-0.24.25}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.21"
+version = "0.24.25"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"