castor-extractor 0.24.20__tar.gz → 0.24.22__tar.gz

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.24.22 - 2025-05-27
+
+* Add retry for `Request.Timeout` on **ApiClient**
+
+## 0.24.21 - 2025-05-26
+
+* Looker Studio: add option to skip the extraction of view activity logs
+
 ## 0.24.20 - 2025-05-19
 
 * Powerbi: allow custom api base and login url

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.20
+Version: 0.24.22
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,14 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.22 - 2025-05-27
+
+* Add retry for `Request.Timeout` on **ApiClient**
+
+## 0.24.21 - 2025-05-26
+
+* Looker Studio: add option to skip the extraction of view activity logs
+
 ## 0.24.20 - 2025-05-19
 
 * Powerbi: allow custom api base and login url

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/commands/extract_looker_studio.py

@@ -24,6 +24,12 @@ def main():
             "This can be the same file path as for Looker Studio."
         ),
     )
+    parser.add_argument(
+        "--skip-view-activity-logs",
+        action="store_true",
+        default=False,
+        help="Skips the extraction of activity logs",
+    )
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/transformation/coalesce/client/client.py

@@ -1,14 +1,11 @@
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-import requests
-
 from ....utils import (
     APIClient,
     BearerAuth,
     RequestSafeMode,
     SerializedAsset,
-    retry,
 )
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
@@ -20,9 +17,6 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
-_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
-_RETRY_COUNT = 2
-_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -75,11 +69,6 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
-    @retry(
-        exceptions=_RETRY_EXCEPTIONS,
-        max_retries=_RETRY_COUNT,
-        base_ms=_RETRY_BASE_MS,
-    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/utils/client/api/client.py

@@ -5,7 +5,7 @@ from typing import Callable, Literal, Optional
 import requests
 from requests import Response
 
-from ...retry import retry_request
+from ...retry import retry, retry_request
 from .auth import Auth
 from .safe_request import RequestSafeMode, handle_response
 from .utils import build_url
@@ -21,6 +21,10 @@ DEFAULT_TIMEOUT = 60
 RETRY_ON_EXPIRED_TOKEN = 1
 RETRY_ON_GATEWAY_TIMEOUT = 3
 
+_TIMEOUT_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_TIMEOUT_RETRY_COUNT = 2
+_TIMEOUT_RETRY_EXCEPTIONS = (requests.exceptions.Timeout,)
+
 
 def _generate_payloads(
     method: HttpMethod,
@@ -72,6 +76,11 @@ class APIClient:
         self._auth = auth
         self._safe_mode = safe_mode
 
+    @retry(
+        exceptions=_TIMEOUT_RETRY_EXCEPTIONS,
+        max_retries=_TIMEOUT_RETRY_COUNT,
+        base_ms=_TIMEOUT_RETRY_BASE_MS,
+    )
     def _call(
         self,
         method: HttpMethod,

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/utils/retry.py

@@ -26,6 +26,15 @@ class RetryStrategy(Enum):
 DEFAULT_STRATEGY = RetryStrategy.CONSTANT
 
 
+def _warning(callable: Callable, exception: BaseException) -> None:
+    exception_type = type(exception)
+    exception_path = f"{exception_type.__module__}.{exception_type.__name__}"
+    callable_path = f"{callable.__module__}.{callable.__name__}"
+
+    msg = f"Exception '{exception_path}' occurred within `{callable_path}`"
+    logger.warning(msg)
+
+
 class Retry(BaseModel):
     """
     This class checks if the retry conditions are met, and if so, how long to
@@ -96,7 +105,7 @@ def retry(
             try:
                 return None, callable(*args, **kwargs)
             except exceptions_ as err:
-                logger.warning(f"Exception within {callable.__name__}")
+                _warning(callable, err)
                 return err, None
 
         def _func(*args, **kwargs) -> Any:
@@ -139,7 +148,7 @@ def retry_request(
                 status_code = err.response.status_code
                 if status_code not in exceptions_:
                     raise err
-                logger.warning(f"Exception within {callable.__name__}")
+                _warning(callable, err)
                 return err, None
 
         def _func(*args, **kwargs) -> Any:

castor_extractor-0.24.22/castor_extractor/visualization/looker_studio/assets.py

@@ -0,0 +1,11 @@
+from ...types import ExternalAsset, classproperty
+
+
+class LookerStudioAsset(ExternalAsset):
+    ASSETS = "assets"
+    SOURCE_QUERIES = "source_queries"
+    VIEW_ACTIVITY = "view_activity"
+
+    @classproperty
+    def optional(cls) -> set["LookerStudioAsset"]:
+        return {LookerStudioAsset.VIEW_ACTIVITY}

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/visualization/looker_studio/client/admin_sdk_client.py

@@ -12,7 +12,6 @@ from ....utils import (
 )
 from .credentials import LookerStudioCredentials
 from .pagination import LookerStudioPagination
-from .scopes import SCOPES
 
 USER_EMAIL_FIELD = "primaryEmail"
 
@@ -26,7 +25,7 @@ class AdminSDKClient:
     def __init__(self, credentials: LookerStudioCredentials):
         self._credentials = Credentials.from_service_account_info(
             credentials.model_dump(),
-            scopes=SCOPES,
+            scopes=credentials.scopes,
             subject=credentials.admin_email,  # impersonates an admin
         )
         self.directory_api = discovery.build(

castor_extractor-0.24.22/castor_extractor/visualization/looker_studio/client/credentials.py

@@ -0,0 +1,46 @@
+from typing import Optional
+
+from pydantic import BaseModel, SecretStr, field_serializer
+
+SCOPES_NO_ACTIVITY: tuple[str, ...] = (
+    "https://www.googleapis.com/auth/datastudio",
+    "https://www.googleapis.com/auth/userinfo.profile",
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+)
+
+DEFAULT_SCOPES: tuple[str, ...] = (
+    *SCOPES_NO_ACTIVITY,
+    "https://www.googleapis.com/auth/admin.reports.audit.readonly",
+)
+
+
+class LookerStudioCredentials(BaseModel):
+    """
+    Looker Studio Credentials match the Service Account credentials JSON
+    but with an additional admin_email field.
+    """
+
+    admin_email: str
+    auth_provider_x509_cert_url: str
+    auth_uri: str
+    client_email: str
+    client_id: str
+    client_x509_cert_url: str
+    private_key: SecretStr
+    private_key_id: str
+    project_id: str
+    token_uri: str
+    type: str
+
+    has_view_activity_logs: Optional[bool] = True
+    scopes: Optional[tuple] = DEFAULT_SCOPES
+
+    def model_post_init(self, __context):
+        """Set scopes based on has_view_activity_logs after initialization"""
+        if self.has_view_activity_logs is False:
+            self.scopes = SCOPES_NO_ACTIVITY
+
+    @field_serializer("private_key")
+    def dump_secret(self, pk):
+        """When using model_dump, show private_key value"""
+        return pk.get_secret_value()

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py

@@ -16,7 +16,6 @@ from .credentials import LookerStudioCredentials
 from .endpoints import LookerStudioAPIEndpoint
 from .enums import LookerStudioAssetType
 from .pagination import LookerStudioPagination
-from .scopes import SCOPES
 
 
 @contextmanager
@@ -47,7 +46,8 @@ class LookerStudioAPIAuth(BearerAuth):
         that user and make requests on that user's behalf.
         """
         self._credentials = Credentials.from_service_account_info(
-            credentials.model_dump(), scopes=SCOPES
+            credentials.model_dump(),
+            scopes=credentials.scopes,
         )
         if subject:
             self._credentials = self._credentials.with_subject(subject)

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/castor_extractor/visualization/looker_studio/extract.py

@@ -23,8 +23,14 @@ LOOKER_STUDIO_ADMIN_EMAIL = "CASTOR_LOOKER_STUDIO_ADMIN_EMAIL"
 
 def iterate_all_data(
     client: LookerStudioClient,
+    has_view_activity_logs: bool = True,
 ) -> Iterable[tuple[LookerStudioAsset, Union[list, dict]]]:
-    for asset in LookerStudioAsset:
+    assets_to_extract = LookerStudioAsset.mandatory
+
+    if has_view_activity_logs:
+        assets_to_extract.add(LookerStudioAsset.VIEW_ACTIVITY)
+
+    for asset in assets_to_extract:
         logger.info(f"Extracting {asset.name} from API")
         data = list(deep_serialize(client.fetch(asset)))
         yield asset, data
@@ -45,6 +51,8 @@ def _credentials(params: dict) -> LookerStudioCredentials:
         LOOKER_STUDIO_ADMIN_EMAIL
     )
     credentials["admin_email"] = admin_email
+    has_view_activity_logs = not params["skip_view_activity_logs"]
+    credentials["has_view_activity_logs"] = has_view_activity_logs
     return LookerStudioCredentials(**credentials)
 
 
@@ -68,7 +76,10 @@ def extract_all(**kwargs) -> None:
     the given output_directory.
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+
     credentials = _credentials(kwargs)
+    has_view_activity_logs = bool(credentials.has_view_activity_logs)
+
     bigquery_credentials = _bigquery_credentials_or_none(kwargs)
 
     client = LookerStudioClient(
@@ -77,7 +88,7 @@ def extract_all(**kwargs) -> None:
     )
     ts = current_timestamp()
 
-    for key, data in iterate_all_data(client):
+    for key, data in iterate_all_data(client, has_view_activity_logs):
         filename = get_output_filename(key.name.lower(), output_directory, ts)
         write_json(filename, data)
 

{castor_extractor-0.24.20 → castor_extractor-0.24.22}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.20"
+version = "0.24.22"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.24.20/castor_extractor/visualization/looker_studio/assets.py

@@ -1,7 +0,0 @@
-from ...types import ExternalAsset
-
-
-class LookerStudioAsset(ExternalAsset):
-    ASSETS = "assets"
-    SOURCE_QUERIES = "source_queries"
-    VIEW_ACTIVITY = "view_activity"

castor_extractor-0.24.20/castor_extractor/visualization/looker_studio/client/credentials.py

@@ -1,25 +0,0 @@
-from pydantic import BaseModel, SecretStr, field_serializer
-
-
-class LookerStudioCredentials(BaseModel):
-    """
-    Looker Studio Credentials match the Service Account credentials JSON
-    but with an additional admin_email field.
-    """
-
-    admin_email: str
-    auth_provider_x509_cert_url: str
-    auth_uri: str
-    client_email: str
-    client_id: str
-    client_x509_cert_url: str
-    private_key: SecretStr
-    private_key_id: str
-    project_id: str
-    token_uri: str
-    type: str
-
-    @field_serializer("private_key")
-    def dump_secret(self, pk):
-        """When using model_dump, show private_key value"""
-        return pk.get_secret_value()

castor_extractor-0.24.20/castor_extractor/visualization/looker_studio/client/scopes.py

@@ -1,6 +0,0 @@
-SCOPES = (
-    "https://www.googleapis.com/auth/datastudio",
-    "https://www.googleapis.com/auth/userinfo.profile",
-    "https://www.googleapis.com/auth/admin.reports.audit.readonly",
-    "https://www.googleapis.com/auth/admin.directory.user.readonly",
-)