castor-extractor 0.25.12__py3-none-any.whl → 0.25.14__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.25.14 - 2025-10-28
+
+* Removing unused dependency (google-cloud-storage)
+
+## 0.25.13 - 2025-10-16
+
+* Coalesce:
+  * reduce pagination
+  * retry once on ReadTimeout or ConnectionTimeout
+
 ## 0.25.12 - 2025-10-16
 
 * Add a deprecation warning for Python versions below 3.10

castor_extractor/transformation/coalesce/client/client.py

@@ -3,6 +3,8 @@ from functools import partial
 from http import HTTPStatus
 from typing import Callable, Optional
 
+from requests.exceptions import ConnectTimeout, ReadTimeout
+
 from ....exceptions import TestConnectionException
 from ....utils import (
     APIClient,
@@ -10,6 +12,7 @@ from ....utils import (
     RequestSafeMode,
     SerializedAsset,
     fetch_all_pages,
+    retry,
 )
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
@@ -20,10 +23,12 @@ from .pagination import CoalescePagination
 
 logger = logging.getLogger(__name__)
 
-
-COALESCE_PAGE_SIZE = 300
+# increase pagination when extraction is too slow
+# reduce pagination when hitting timeouts
+COALESCE_PAGE_SIZE = 200
 COALESCE_PAGE_SIZE_RUN_RESULTS = 1_000
 
+# raising this value might end up in remote server disconnection
 COALESCE_TIMEOUT_SECONDS = 90
 
 _MAX_ERRORS = 200
@@ -33,6 +38,10 @@ COALESCE_SAFE_MODE = RequestSafeMode(
     max_errors=_MAX_ERRORS,
 )
 
+RETRY_COUNT = 1
+RETRY_BASE_MS = 5 * 60 * 1000  # 5 minutes
+RETRY_EXCEPTIONS = [ConnectTimeout, ReadTimeout]
+
 
 def _run_result_payload(
     environment_id: str,
@@ -107,6 +116,11 @@ class CoalesceClient(APIClient):
         result = fetch_all_pages(request, CoalescePagination)
         return list(result)
 
+    @retry(
+        exceptions=RETRY_EXCEPTIONS,
+        max_retries=RETRY_COUNT,
+        base_ms=RETRY_BASE_MS,
+    )
     def _fetch_env_nodes(self, environment_id: int) -> SerializedAsset:
         endpoint = CoalesceEndpointFactory.nodes(environment_id=environment_id)
         request = self._get_paginated(

castor_extractor/warehouse/snowflake/queries/table.sql

@@ -36,7 +36,8 @@ SELECT
     t.created,
     t.last_altered,
     t.deleted,
-    ta.tags
+    ta.tags,
+    t.is_dynamic
 FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id

{castor_extractor-0.25.12.dist-info → castor_extractor-0.25.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.25.12
+Version: 0.25.14
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,7 +37,6 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -216,6 +215,16 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.25.14 - 2025-10-28
+
+* Removing unused dependency (google-cloud-storage)
+
+## 0.25.13 - 2025-10-16
+
+* Coalesce:
+  * reduce pagination
+  * retry once on ReadTimeout or ConnectionTimeout
+
 ## 0.25.12 - 2025-10-16
 
 * Add a deprecation warning for Python versions below 3.10

{castor_extractor-0.25.12.dist-info → castor_extractor-0.25.14.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=mF8dWRAF5Bnd2HZCRn6ZkVk-rjPmEWdvs9nNwYCxAVM,22057
+CHANGELOG.md,sha256=2pAjf3JtSe5AR2zTewGuKOHPXZebBAZ_vNdR_S14C1s,22246
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -78,7 +78,7 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 castor_extractor/transformation/coalesce/__init__.py,sha256=CW_qdtEfwgJRsCyBlk5hNlxwEO-VV6mBXZvkRbND_J8,112
 castor_extractor/transformation/coalesce/assets.py,sha256=pzccYPP66c9PAnVroemx7-6MeRHw7Ft1OlTC6jIamAA,363
 castor_extractor/transformation/coalesce/client/__init__.py,sha256=VRmVpH29rOghtDQnCN7dAdA0dI0Lxseu4BC8rnwM9dU,80
-castor_extractor/transformation/coalesce/client/client.py,sha256=qBg0Qrs-YgkypgF51oK08ryY8XRtwvm1iAY7pFSn2CQ,6184
+castor_extractor/transformation/coalesce/client/client.py,sha256=me9U6ovT4qR3AxTv9PN6fxG9UvPAdf7S9dvFDKVgFLY,6640
 castor_extractor/transformation/coalesce/client/credentials.py,sha256=jbJxjbdPspf-dzYKfeb7oqL_8TXd1nvkJrjAcdAnLPc,548
 castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9qr_50SEYV_-heQE4BwED9oNMgYsL-w,1272
 castor_extractor/transformation/coalesce/client/pagination.py,sha256=zynyWCMEzUQ7HA1Q5AP4BAOmxRQI6NA5jCPEo0lHn44,705
@@ -429,7 +429,7 @@ castor_extractor/warehouse/snowflake/queries/database.sql,sha256=yxHzV2oQfW0bo2d
 castor_extractor/warehouse/snowflake/queries/function.sql,sha256=8LRh0ybhd-RldJ8UZspWUm3yv52evq11O2uqIO4KqeQ,372
 castor_extractor/warehouse/snowflake/queries/query.sql,sha256=w4T6-TgwUozDgaF3Fk-qex7bDdEIHLkkB5XEe2VJXZQ,1992
 castor_extractor/warehouse/snowflake/queries/schema.sql,sha256=iLn6_y5rn63KigjE4GEAMp8ZuZZofhMXYGb8saPDGUc,776
-castor_extractor/warehouse/snowflake/queries/table.sql,sha256=CbSLfJAylyyyD3mkGPSLLE7BHrGjlY499kzO9RN0e4Y,1473
+castor_extractor/warehouse/snowflake/queries/table.sql,sha256=ovZwQ5NqJB_IFggKfdeuMh2eCiptn4z-Ek6_1CFd1nU,1491
 castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsKOHHlqCtBMQHOV54yy6RKJaXk,570
 castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
 castor_extractor/warehouse/snowflake/query.py,sha256=C2LTdPwBzMQ_zMncg0Kq4_WkoY7K9as5tvxBDrIOlwI,1763
@@ -446,8 +446,8 @@ castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=MAlnTis43E3Amu1e1Oz
 castor_extractor/warehouse/sqlserver/queries/view_ddl.sql,sha256=9rynvx6MWg3iZzrWPB7haZfVKEPkxulzryE2g19x804,315
 castor_extractor/warehouse/sqlserver/query.py,sha256=gr5lnZSUm-wSYuVnJlg6fc7jXWirbL-sCiQN9RnAiPQ,1789
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.25.12.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.25.12.dist-info/METADATA,sha256=8CbY2v74BOKMCVvYm8-P7rPKSE5i9gyX2Qd-Ct_K0rY,29560
-castor_extractor-0.25.12.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.25.12.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
-castor_extractor-0.25.12.dist-info/RECORD,,
+castor_extractor-0.25.14.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.25.14.dist-info/METADATA,sha256=DTV_oWepOjhPFLkQ3XXf06lXcmmUaWCDSHz-PD68Hm0,29696
+castor_extractor-0.25.14.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.25.14.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
+castor_extractor-0.25.14.dist-info/RECORD,,