castor-extractor 0.25.12__tar.gz → 0.25.14__tar.gz

{castor_extractor-0.25.12 → castor_extractor-0.25.14}/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.25.14 - 2025-10-28
+
+* Removing unused dependency (google-cloud-storage)
+
+## 0.25.13 - 2025-10-16
+
+* Coalesce:
+  * reduce pagination
+  * retry once on ReadTimeout or ConnectionTimeout
+
 ## 0.25.12 - 2025-10-16
 
 * Add a deprecation warning for Python versions below 3.10

{castor_extractor-0.25.12 → castor_extractor-0.25.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.25.12
+Version: 0.25.14
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,7 +37,6 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -216,6 +215,16 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.25.14 - 2025-10-28
+
+* Removing unused dependency (google-cloud-storage)
+
+## 0.25.13 - 2025-10-16
+
+* Coalesce:
+  * reduce pagination
+  * retry once on ReadTimeout or ConnectionTimeout
+
 ## 0.25.12 - 2025-10-16
 
 * Add a deprecation warning for Python versions below 3.10

{castor_extractor-0.25.12 → castor_extractor-0.25.14}/castor_extractor/transformation/coalesce/client/client.py

@@ -3,6 +3,8 @@ from functools import partial
 from http import HTTPStatus
 from typing import Callable, Optional
 
+from requests.exceptions import ConnectTimeout, ReadTimeout
+
 from ....exceptions import TestConnectionException
 from ....utils import (
     APIClient,
@@ -10,6 +12,7 @@ from ....utils import (
     RequestSafeMode,
     SerializedAsset,
     fetch_all_pages,
+    retry,
 )
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
@@ -20,10 +23,12 @@ from .pagination import CoalescePagination
 
 logger = logging.getLogger(__name__)
 
-
-COALESCE_PAGE_SIZE = 300
+# increase pagination when extraction is too slow
+# reduce pagination when hitting timeouts
+COALESCE_PAGE_SIZE = 200
 COALESCE_PAGE_SIZE_RUN_RESULTS = 1_000
 
+# raising this value might end up in remote server disconnection
 COALESCE_TIMEOUT_SECONDS = 90
 
 _MAX_ERRORS = 200
@@ -33,6 +38,10 @@ COALESCE_SAFE_MODE = RequestSafeMode(
     max_errors=_MAX_ERRORS,
 )
 
+RETRY_COUNT = 1
+RETRY_BASE_MS = 5 * 60 * 1000  # 5 minutes
+RETRY_EXCEPTIONS = [ConnectTimeout, ReadTimeout]
+
 
 def _run_result_payload(
     environment_id: str,
@@ -107,6 +116,11 @@ class CoalesceClient(APIClient):
         result = fetch_all_pages(request, CoalescePagination)
         return list(result)
 
+    @retry(
+        exceptions=RETRY_EXCEPTIONS,
+        max_retries=RETRY_COUNT,
+        base_ms=RETRY_BASE_MS,
+    )
     def _fetch_env_nodes(self, environment_id: int) -> SerializedAsset:
         endpoint = CoalesceEndpointFactory.nodes(environment_id=environment_id)
         request = self._get_paginated(

{castor_extractor-0.25.12 → castor_extractor-0.25.14}/castor_extractor/warehouse/snowflake/queries/table.sql

@@ -36,7 +36,8 @@ SELECT
     t.created,
     t.last_altered,
     t.deleted,
-    ta.tags
+    ta.tags,
+    t.is_dynamic
 FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id

{castor_extractor-0.25.12 → castor_extractor-0.25.14}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.25.12"
+version = "0.25.14"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -32,7 +32,6 @@ google-api-core = ">=2.1.1, <3.0.0"
 google-api-python-client = {version = "^2.121.0", optional = true}
 google-auth = "^2"
 google-cloud-core = ">=2.1.0, <3.0.0"
-google-cloud-storage = "^3.1.0"
 google-resumable-media = "^2.0.3"
 googleapis-common-protos = "^1.53.0"
 looker-sdk = { version = "^25.0.0", optional = true }