castor-extractor 0.24.20__py3-none-any.whl → 0.24.22__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.24.22 - 2025-05-27
+
+* Add retry for `Request.Timeout` on **ApiClient**
+
+## 0.24.21 - 2025-05-26
+
+* Looker Studio: add option to skip the extraction of view activity logs
+
 ## 0.24.20 - 2025-05-19
 
 * Powerbi: allow custom api base and login url

castor_extractor/commands/extract_looker_studio.py

@@ -24,6 +24,12 @@ def main():
             "This can be the same file path as for Looker Studio."
         ),
     )
+    parser.add_argument(
+        "--skip-view-activity-logs",
+        action="store_true",
+        default=False,
+        help="Skips the extraction of activity logs",
+    )
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 

castor_extractor/transformation/coalesce/client/client.py

@@ -1,14 +1,11 @@
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-import requests
-
 from ....utils import (
     APIClient,
     BearerAuth,
     RequestSafeMode,
     SerializedAsset,
-    retry,
 )
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
@@ -20,9 +17,6 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
-_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
-_RETRY_COUNT = 2
-_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -75,11 +69,6 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
-    @retry(
-        exceptions=_RETRY_EXCEPTIONS,
-        max_retries=_RETRY_COUNT,
-        base_ms=_RETRY_BASE_MS,
-    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id

castor_extractor/utils/client/api/client.py

@@ -5,7 +5,7 @@ from typing import Callable, Literal, Optional
 import requests
 from requests import Response
 
-from ...retry import retry_request
+from ...retry import retry, retry_request
 from .auth import Auth
 from .safe_request import RequestSafeMode, handle_response
 from .utils import build_url
@@ -21,6 +21,10 @@ DEFAULT_TIMEOUT = 60
 RETRY_ON_EXPIRED_TOKEN = 1
 RETRY_ON_GATEWAY_TIMEOUT = 3
 
+_TIMEOUT_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_TIMEOUT_RETRY_COUNT = 2
+_TIMEOUT_RETRY_EXCEPTIONS = (requests.exceptions.Timeout,)
+
 
 def _generate_payloads(
     method: HttpMethod,
@@ -72,6 +76,11 @@ class APIClient:
         self._auth = auth
         self._safe_mode = safe_mode
 
+    @retry(
+        exceptions=_TIMEOUT_RETRY_EXCEPTIONS,
+        max_retries=_TIMEOUT_RETRY_COUNT,
+        base_ms=_TIMEOUT_RETRY_BASE_MS,
+    )
     def _call(
         self,
         method: HttpMethod,

castor_extractor/utils/retry.py

@@ -26,6 +26,15 @@ class RetryStrategy(Enum):
 DEFAULT_STRATEGY = RetryStrategy.CONSTANT
 
 
+def _warning(callable: Callable, exception: BaseException) -> None:
+    exception_type = type(exception)
+    exception_path = f"{exception_type.__module__}.{exception_type.__name__}"
+    callable_path = f"{callable.__module__}.{callable.__name__}"
+
+    msg = f"Exception '{exception_path}' occurred within `{callable_path}`"
+    logger.warning(msg)
+
+
 class Retry(BaseModel):
     """
     This class checks if the retry conditions are met, and if so, how long to
@@ -96,7 +105,7 @@ def retry(
             try:
                 return None, callable(*args, **kwargs)
             except exceptions_ as err:
-                logger.warning(f"Exception within {callable.__name__}")
+                _warning(callable, err)
                 return err, None
 
         def _func(*args, **kwargs) -> Any:
@@ -139,7 +148,7 @@ def retry_request(
                 status_code = err.response.status_code
                 if status_code not in exceptions_:
                     raise err
-                logger.warning(f"Exception within {callable.__name__}")
+                _warning(callable, err)
                 return err, None
 
         def _func(*args, **kwargs) -> Any:

castor_extractor/visualization/looker_studio/assets.py

@@ -1,7 +1,11 @@
-from ...types import ExternalAsset
+from ...types import ExternalAsset, classproperty
 
 
 class LookerStudioAsset(ExternalAsset):
     ASSETS = "assets"
     SOURCE_QUERIES = "source_queries"
     VIEW_ACTIVITY = "view_activity"
+
+    @classproperty
+    def optional(cls) -> set["LookerStudioAsset"]:
+        return {LookerStudioAsset.VIEW_ACTIVITY}

castor_extractor/visualization/looker_studio/client/admin_sdk_client.py

@@ -12,7 +12,6 @@ from ....utils import (
 )
 from .credentials import LookerStudioCredentials
 from .pagination import LookerStudioPagination
-from .scopes import SCOPES
 
 USER_EMAIL_FIELD = "primaryEmail"
 
@@ -26,7 +25,7 @@ class AdminSDKClient:
     def __init__(self, credentials: LookerStudioCredentials):
         self._credentials = Credentials.from_service_account_info(
             credentials.model_dump(),
-            scopes=SCOPES,
+            scopes=credentials.scopes,
             subject=credentials.admin_email,  # impersonates an admin
         )
         self.directory_api = discovery.build(

castor_extractor/visualization/looker_studio/client/credentials.py

@@ -1,5 +1,18 @@
+from typing import Optional
+
 from pydantic import BaseModel, SecretStr, field_serializer
 
+SCOPES_NO_ACTIVITY: tuple[str, ...] = (
+    "https://www.googleapis.com/auth/datastudio",
+    "https://www.googleapis.com/auth/userinfo.profile",
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+)
+
+DEFAULT_SCOPES: tuple[str, ...] = (
+    *SCOPES_NO_ACTIVITY,
+    "https://www.googleapis.com/auth/admin.reports.audit.readonly",
+)
+
 
 class LookerStudioCredentials(BaseModel):
     """
@@ -19,6 +32,14 @@ class LookerStudioCredentials(BaseModel):
     token_uri: str
     type: str
 
+    has_view_activity_logs: Optional[bool] = True
+    scopes: Optional[tuple] = DEFAULT_SCOPES
+
+    def model_post_init(self, __context):
+        """Set scopes based on has_view_activity_logs after initialization"""
+        if self.has_view_activity_logs is False:
+            self.scopes = SCOPES_NO_ACTIVITY
+
     @field_serializer("private_key")
     def dump_secret(self, pk):
         """When using model_dump, show private_key value"""

castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py

@@ -16,7 +16,6 @@ from .credentials import LookerStudioCredentials
 from .endpoints import LookerStudioAPIEndpoint
 from .enums import LookerStudioAssetType
 from .pagination import LookerStudioPagination
-from .scopes import SCOPES
 
 
 @contextmanager
@@ -47,7 +46,8 @@ class LookerStudioAPIAuth(BearerAuth):
         that user and make requests on that user's behalf.
         """
         self._credentials = Credentials.from_service_account_info(
-            credentials.model_dump(), scopes=SCOPES
+            credentials.model_dump(),
+            scopes=credentials.scopes,
         )
         if subject:
             self._credentials = self._credentials.with_subject(subject)

castor_extractor/visualization/looker_studio/extract.py

@@ -23,8 +23,14 @@ LOOKER_STUDIO_ADMIN_EMAIL = "CASTOR_LOOKER_STUDIO_ADMIN_EMAIL"
 
 def iterate_all_data(
     client: LookerStudioClient,
+    has_view_activity_logs: bool = True,
 ) -> Iterable[tuple[LookerStudioAsset, Union[list, dict]]]:
-    for asset in LookerStudioAsset:
+    assets_to_extract = LookerStudioAsset.mandatory
+
+    if has_view_activity_logs:
+        assets_to_extract.add(LookerStudioAsset.VIEW_ACTIVITY)
+
+    for asset in assets_to_extract:
         logger.info(f"Extracting {asset.name} from API")
         data = list(deep_serialize(client.fetch(asset)))
         yield asset, data
@@ -45,6 +51,8 @@ def _credentials(params: dict) -> LookerStudioCredentials:
         LOOKER_STUDIO_ADMIN_EMAIL
     )
     credentials["admin_email"] = admin_email
+    has_view_activity_logs = not params["skip_view_activity_logs"]
+    credentials["has_view_activity_logs"] = has_view_activity_logs
     return LookerStudioCredentials(**credentials)
 
 
@@ -68,7 +76,10 @@ def extract_all(**kwargs) -> None:
     the given output_directory.
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+
     credentials = _credentials(kwargs)
+    has_view_activity_logs = bool(credentials.has_view_activity_logs)
+
     bigquery_credentials = _bigquery_credentials_or_none(kwargs)
 
     client = LookerStudioClient(
@@ -77,7 +88,7 @@ def extract_all(**kwargs) -> None:
     )
     ts = current_timestamp()
 
-    for key, data in iterate_all_data(client):
+    for key, data in iterate_all_data(client, has_view_activity_logs):
         filename = get_output_filename(key.name.lower(), output_directory, ts)
         write_json(filename, data)
 

{castor_extractor-0.24.20.dist-info → castor_extractor-0.24.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.20
+Version: 0.24.22
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,14 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.22 - 2025-05-27
+
+* Add retry for `Request.Timeout` on **ApiClient**
+
+## 0.24.21 - 2025-05-26
+
+* Looker Studio: add option to skip the extraction of view activity logs
+
 ## 0.24.20 - 2025-05-19
 
 * Powerbi: allow custom api base and login url

{castor_extractor-0.24.20.dist-info → castor_extractor-0.24.22.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=NxSwszCNlJ8oD2ffivq3g75DK436mue9WwilR0r6bE4,17760
+CHANGELOG.md,sha256=8EyATqcpMtcJ3LhQgQadRhcDiEULwOXYw40137Y5jgs,17936
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -10,7 +10,7 @@ castor_extractor/commands/extract_confluence.py,sha256=blYcnDqywXNKRQ1aZAD9FclhL
 castor_extractor/commands/extract_databricks.py,sha256=SVKyoa-BBUQAM6HRHf1Wdg9-tpICic2yyvXQwHcNBhA,1264
 castor_extractor/commands/extract_domo.py,sha256=jvAawUsUTHrwCn_koK6StmQr4n_b5GyvJi6uu6WS0SM,1061
 castor_extractor/commands/extract_looker.py,sha256=cySLiolLCgrREJ9d0kMrJ7P8K3efHTBTzShalWVfI3A,1214
-castor_extractor/commands/extract_looker_studio.py,sha256=e79gbyTtCexRz5pg_Pp55GWkXJZWjm6NvVclmvcR0lM,916
+castor_extractor/commands/extract_looker_studio.py,sha256=M7wx8XZScLizCI2vq80aj88vYrdiHChiCiebrrChlZY,1090
 castor_extractor/commands/extract_metabase_api.py,sha256=NXctea4GT_1iRDitY92nV3TKSqhjEUwYSxwPJMRS3iw,786
 castor_extractor/commands/extract_metabase_db.py,sha256=tYIhTPPgj1mN-07LyWcL6e-YoGp7HCWda58-5Ukyg_I,1255
 castor_extractor/commands/extract_mode.py,sha256=Q4iO-VAKMg4zFPejhAO-foZibL5Ht3jsnhWKwJ0oqUU,823
@@ -76,7 +76,7 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 castor_extractor/transformation/coalesce/__init__.py,sha256=CW_qdtEfwgJRsCyBlk5hNlxwEO-VV6mBXZvkRbND_J8,112
 castor_extractor/transformation/coalesce/assets.py,sha256=pzccYPP66c9PAnVroemx7-6MeRHw7Ft1OlTC6jIamAA,363
 castor_extractor/transformation/coalesce/client/__init__.py,sha256=VRmVpH29rOghtDQnCN7dAdA0dI0Lxseu4BC8rnwM9dU,80
-castor_extractor/transformation/coalesce/client/client.py,sha256=-fFxWtDPPruNmDPc6FXft_6RwRKEee4JM-13d90fms0,6442
+castor_extractor/transformation/coalesce/client/client.py,sha256=o-BlVqdzcQ2RhrmShsNFXVhsmJF4i0pP6DmiWGAE2dY,6172
 castor_extractor/transformation/coalesce/client/credentials.py,sha256=jbJxjbdPspf-dzYKfeb7oqL_8TXd1nvkJrjAcdAnLPc,548
 castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9qr_50SEYV_-heQE4BwED9oNMgYsL-w,1272
 castor_extractor/transformation/coalesce/client/type.py,sha256=oiiVP9NL0ijTXyQmaB8aJVYckc7m-m8ZgMyNIAduUKE,43
@@ -106,7 +106,7 @@ castor_extractor/utils/client/abstract.py,sha256=CWF7_afNpEZ3jor-22wXbKIvM20ukHk
 castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
 castor_extractor/utils/client/api/auth.py,sha256=lq0K3UEl1vwIIa_vKTdlpIQPdE5K1-5DXmCwO4dKzng,1890
 castor_extractor/utils/client/api/auth_test.py,sha256=LlyXytnatg6ZzR4Zkvzk0BH99FYhHX7qn_nyr2MSnDI,1305
-castor_extractor/utils/client/api/client.py,sha256=d3I-RbKgy7dqONKj2hAuRxTr9i9U82RklIuRjnVppeM,4574
+castor_extractor/utils/client/api/client.py,sha256=En1V9eglhB71q5jZwi5XhNZanLb7IUKfd4AT-sSexJk,4866
 castor_extractor/utils/client/api/client_test.py,sha256=FM3ZxsLLfMOBn44cXX6FIgnA31-5TTNIyp9D4LBwtXE,1222
 castor_extractor/utils/client/api/pagination.py,sha256=ph5TYqPiyFGgygsIhCATAHPIQ9UJNZyiTcqlyRdGEno,2460
 castor_extractor/utils/client/api/pagination_test.py,sha256=jCOgXFXrH-jrCxe2dfk80ZksJF-EtmpJPU11BGabsqk,1385
@@ -138,7 +138,7 @@ castor_extractor/utils/pager/pager.py,sha256=93Rw7jCz6GnqrS4HfYfKYV2xgEx2esl1qC9
 castor_extractor/utils/pager/pager_on_id.py,sha256=jBvmlEhkJ-sODkNyz1KyyXHobLsNhC4AwhOwYvLyB4E,1967
 castor_extractor/utils/pager/pager_on_id_test.py,sha256=eDGrIYPGffuKPUATgu5fiXIwPKdSwEXGgTtfMiHqoj0,1601
 castor_extractor/utils/pager/pager_test.py,sha256=PQOXQwQD2wOP0xzZfNTuLxcn3Bpa4FCASVklH71GO_s,1699
-castor_extractor/utils/retry.py,sha256=LvfBsE_R6g-U72hkN531rJDMj42zwkUqgIrrjlxctJs,4733
+castor_extractor/utils/retry.py,sha256=xRlAxHRnjmjh2sDUuuUSS-s38pokoAvSgSKjdgWGqbc,5020
 castor_extractor/utils/retry_test.py,sha256=j_6IJStBomEhxmGpIY9IIlESgMxhcDpmIKj24unLqlA,2892
 castor_extractor/utils/safe.py,sha256=gvIMRIoggdVeYMl222IYqXnHVDninDklFMlAHt-WldA,1948
 castor_extractor/utils/safe_test.py,sha256=IHN1Z761tYMFslYC-2HAfkXmFPh4LYSqNLs4QZwykjk,2160
@@ -186,18 +186,17 @@ castor_extractor/visualization/looker/fields.py,sha256=7oC7p-3Wp7XHBP_FT_D1wH3kI
 castor_extractor/visualization/looker/fields_test.py,sha256=7Cwq8Qky6aTZg8nCHp1gmPJtd9pGNB4QeMIRRWdHo5w,782
 castor_extractor/visualization/looker/multithreading.py,sha256=Muuh3usBLqtv3sfHoyPYJ6jJ7V5ajR6N9ZJ_F-bNc60,2608
 castor_extractor/visualization/looker_studio/__init__.py,sha256=GccG-GJXoNhjXFPkw-rHHZ0SXVQTFKjqkMIYHVeu3T4,175
-castor_extractor/visualization/looker_studio/assets.py,sha256=lFIqr8EB6eK-Mf80R_x2qAscCyX7ZUcOcHVef1CM9B0,173
+castor_extractor/visualization/looker_studio/assets.py,sha256=rI73rbVrfwkkepqZr0zPouP2lPUfJxSi21RKtOTHtAA,308
 castor_extractor/visualization/looker_studio/client/__init__.py,sha256=YkQaVDJa-7KSwdOLjtgKJMRiafbGNKC_46YVx0hYZ1Q,129
-castor_extractor/visualization/looker_studio/client/admin_sdk_client.py,sha256=hYKdU6TlWKkXx07r6HsZ4Wbxhasx8DP_jO6iDCjHjgk,3508
+castor_extractor/visualization/looker_studio/client/admin_sdk_client.py,sha256=HIeyT9JTW1TPwVzD2Q-VfJ99jMP80Z-4CznKAnTnp2w,3493
 castor_extractor/visualization/looker_studio/client/client.py,sha256=6sTfLRUhuxhkqDjC2ZBEaw6YnR6ze8-_VW2rc1u9Ksk,3191
-castor_extractor/visualization/looker_studio/client/credentials.py,sha256=QImJPh8VctkrGt65UiU5hM12JI4WdCMSUFt88aiOoLw,657
+castor_extractor/visualization/looker_studio/client/credentials.py,sha256=F4ISI8Ua_HJsMuGhYql28o3hKYR4sL_uzkrUkRiekRo,1347
 castor_extractor/visualization/looker_studio/client/endpoints.py,sha256=5eY-ffqNDdlDBOOpiF7LpjyHMrzeClJktidCr1pTDUs,669
 castor_extractor/visualization/looker_studio/client/enums.py,sha256=fHgemTaQpnwee8cw1YQVDsVnH--vTyFwT4Px8aVYYHQ,167
-castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py,sha256=oySC6rsppj67RSifxwSCw4bFrz1Irx6IFJhX7tc_v1E,4087
+castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py,sha256=Phq378VEaFLD-nyP2_A1wge6HUP45jSthhlNjD7aqSg,4085
 castor_extractor/visualization/looker_studio/client/pagination.py,sha256=9HQ3Rkdiz2VB6AvYtZ0F-WouiD0pMmdZyAmkv-3wh08,783
 castor_extractor/visualization/looker_studio/client/queries/query.sql,sha256=Ub4rdrJ5WTPWKI-eVmXrNMv0Ktmti4b-93zZBr0xEB0,1426
-castor_extractor/visualization/looker_studio/client/scopes.py,sha256=824cqqgZuGq4L-rPNoHJe0ibXsxkRwB0CLG_kqw9Q0g,256
-castor_extractor/visualization/looker_studio/extract.py,sha256=cHyroNZ1fKoBTvIbEebnKDrU3xpkcEgIPJy75ljCL70,2607
+castor_extractor/visualization/looker_studio/extract.py,sha256=uNpvg4wtFflmpkqXFfo_9Nm12AEKXBOCKKajIggySho,3026
 castor_extractor/visualization/metabase/__init__.py,sha256=3E36cmkMyEgBB6Ot5rWk-N75i0G-7k24QTlc-Iol4pM,193
 castor_extractor/visualization/metabase/assets.py,sha256=nu3FwQBU_hdS2DBvgXAwQlEEi76QiNK2tMKEtMyctaY,2874
 castor_extractor/visualization/metabase/client/__init__.py,sha256=KBvaPMofBRV3m_sZAnKNCrJGr-Z88EbpdzEzWPQ_uBk,99
@@ -426,8 +425,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.20.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.20.dist-info/METADATA,sha256=DvgjbhUmiYXzV9e3MXsEGuSfgOog38LrEBNmqFIAcyI,25213
-castor_extractor-0.24.20.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.20.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.20.dist-info/RECORD,,
+castor_extractor-0.24.22.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.22.dist-info/METADATA,sha256=VTqwYHqBEng4wSZ5N88tT1z6Y2OdS9uETv4qIE3CZVw,25389
+castor_extractor-0.24.22.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.22.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.22.dist-info/RECORD,,

castor_extractor/visualization/looker_studio/client/scopes.py

@@ -1,6 +0,0 @@
-SCOPES = (
-    "https://www.googleapis.com/auth/datastudio",
-    "https://www.googleapis.com/auth/userinfo.profile",
-    "https://www.googleapis.com/auth/admin.reports.audit.readonly",
-    "https://www.googleapis.com/auth/admin.directory.user.readonly",
-)