castor-extractor 0.25.10__py3-none-any.whl → 0.25.14__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.25.14 - 2025-10-28
+
+* Removing unused dependency (google-cloud-storage)
+
+## 0.25.13 - 2025-10-16
+
+* Coalesce:
+  * reduce pagination
+  * retry once on ReadTimeout or ConnectionTimeout
+
+## 0.25.12 - 2025-10-16
+
+* Add a deprecation warning for Python versions below 3.10
+
+## 0.25.11 - 2025-10-15
+
+* Sigma: catch ReadTimeouts during queries extraction
+
 ## 0.25.10 - 2025-10-09
 
 * Fix import

castor_extractor/commands/__init__.py

@@ -0,0 +1,3 @@
+from castor_extractor.utils import deprecate_python  # type: ignore
+
+deprecate_python(min_version_supported=(3, 10))

castor_extractor/exceptions.py

@@ -3,3 +3,8 @@ class NoDatabaseProvidedException(Exception):
         super().__init__("""No database eligible for extraction.
 If you are using the db_allow/db_block options, please make sure to use the correct case.
         """)
+
+
+class TestConnectionException(Exception):
+    def __init__(self, caused_by: Exception):
+        super().__init__(f"Connection could not be verified: {caused_by}")

castor_extractor/transformation/coalesce/client/client.py

@@ -3,12 +3,16 @@ from functools import partial
 from http import HTTPStatus
 from typing import Callable, Optional
 
+from requests.exceptions import ConnectTimeout, ReadTimeout
+
+from ....exceptions import TestConnectionException
 from ....utils import (
     APIClient,
     BearerAuth,
     RequestSafeMode,
     SerializedAsset,
     fetch_all_pages,
+    retry,
 )
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
@@ -19,10 +23,12 @@ from .pagination import CoalescePagination
 
 logger = logging.getLogger(__name__)
 
-
-COALESCE_PAGE_SIZE = 300
+# increase pagination when extraction is too slow
+# reduce pagination when hitting timeouts
+COALESCE_PAGE_SIZE = 200
 COALESCE_PAGE_SIZE_RUN_RESULTS = 1_000
 
+# raising this value might end up in remote server disconnection
 COALESCE_TIMEOUT_SECONDS = 90
 
 _MAX_ERRORS = 200
@@ -32,6 +38,10 @@ COALESCE_SAFE_MODE = RequestSafeMode(
     max_errors=_MAX_ERRORS,
 )
 
+RETRY_COUNT = 1
+RETRY_BASE_MS = 5 * 60 * 1000  # 5 minutes
+RETRY_EXCEPTIONS = [ConnectTimeout, ReadTimeout]
+
 
 def _run_result_payload(
     environment_id: str,
@@ -64,6 +74,14 @@ class CoalesceBearerAuth(BearerAuth):
 class CoalesceClient(APIClient):
     """REST API client to extract data from Coalesce"""
 
+    def test_connection(self) -> None:
+        endpoint = CoalesceEndpointFactory.environments()
+        request = self._get_paginated(endpoint=endpoint)
+        try:
+            next(fetch_all_pages(request, CoalescePagination))
+        except Exception as e:
+            raise TestConnectionException(e)
+
     def __init__(
         self,
         credentials: CoalesceCredentials,
@@ -98,6 +116,11 @@ class CoalesceClient(APIClient):
         result = fetch_all_pages(request, CoalescePagination)
         return list(result)
 
+    @retry(
+        exceptions=RETRY_EXCEPTIONS,
+        max_retries=RETRY_COUNT,
+        base_ms=RETRY_BASE_MS,
+    )
     def _fetch_env_nodes(self, environment_id: int) -> SerializedAsset:
         endpoint = CoalesceEndpointFactory.nodes(environment_id=environment_id)
         request = self._get_paginated(

castor_extractor/visualization/sigma/client/client.py

@@ -192,7 +192,12 @@ class SigmaClient(APIClient):
             )
             queries = fetch_all_pages(request, SigmaPagination)
 
-            yield from self._yield_deduplicated_queries(queries, workbook_id)
+            try:
+                yield from self._yield_deduplicated_queries(
+                    queries, workbook_id
+                )
+            except ReadTimeout:
+                logger.warning(f"ReadTimeout for workbook {workbook_id}")
 
     def _get_all_datamodel_sources(
         self, datamodels: list[dict]

castor_extractor/warehouse/snowflake/queries/table.sql

@@ -36,7 +36,8 @@ SELECT
     t.created,
     t.last_altered,
     t.deleted,
-    ta.tags
+    ta.tags,
+    t.is_dynamic
 FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id

{castor_extractor-0.25.10.dist-info → castor_extractor-0.25.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.25.10
+Version: 0.25.14
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,7 +37,6 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -216,6 +215,24 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.25.14 - 2025-10-28
+
+* Removing unused dependency (google-cloud-storage)
+
+## 0.25.13 - 2025-10-16
+
+* Coalesce:
+  * reduce pagination
+  * retry once on ReadTimeout or ConnectionTimeout
+
+## 0.25.12 - 2025-10-16
+
+* Add a deprecation warning for Python versions below 3.10
+
+## 0.25.11 - 2025-10-15
+
+* Sigma: catch ReadTimeouts during queries extraction
+
 ## 0.25.10 - 2025-10-09
 
 * Fix import

{castor_extractor-0.25.10.dist-info → castor_extractor-0.25.14.dist-info}/RECORD

@@ -1,10 +1,10 @@
-CHANGELOG.md,sha256=-GLL2qrB_fkho3FplO7T7Fd6WtPbwEaA-9nfEhXWjtg,21892
+CHANGELOG.md,sha256=2pAjf3JtSe5AR2zTewGuKOHPXZebBAZ_vNdR_S14C1s,22246
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=C6hTyZO60T7z7xwHbspHlii384Jn02k0Rycxu3bCX0o,3866
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-castor_extractor/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+castor_extractor/commands/__init__.py,sha256=3q18pP2jt0slwgrMkja-8vNq_GNk9-PZW6GPfqGQgxk,117
 castor_extractor/commands/extract_bigquery.py,sha256=dU4OiYO1V0n32orvZnMh1_xtFKF_VxHNXcVsH3otY-g,1269
 castor_extractor/commands/extract_confluence.py,sha256=blYcnDqywXNKRQ1aZAD9FclhLlO7x8Y_tb0lgl85v0w,1641
 castor_extractor/commands/extract_count.py,sha256=cITp-2UmPYjbcICvYZzxE9oWieI8NbTH1DcWxLAZxJ4,611
@@ -31,7 +31,7 @@ castor_extractor/commands/extract_tableau.py,sha256=cfH-b0Hq9LGrQSJv02Yr_4d6oNqh
 castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=TJx76Ymd0QCECmq35zRJMkPE8DJtSInB28MuSXWk8Ao,2644
 castor_extractor/commands/upload.py,sha256=sqpEF_qqCNvT_niIrM6jPhzLaFVjtYwpc2iZw540F20,1633
-castor_extractor/exceptions.py,sha256=WypgqL8AxYL1viYUCSY4528pk5TRcqVNPvyLMMMDWGw,238
+castor_extractor/exceptions.py,sha256=fhG5San5XzUr02XXEmCWs1_I_YS_cUji-E_G8_VYCsY,403
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
 castor_extractor/file_checker/column.py,sha256=6bJhcW1snYwgHKkqlS0Ak7XLHZr4YBwO46JCIlnQNKg,3086
 castor_extractor/file_checker/column_test.py,sha256=1j8PxvmvmJgpd-mk30iMYOme32ovPSIn4yCXywFoXrg,1935
@@ -78,7 +78,7 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 castor_extractor/transformation/coalesce/__init__.py,sha256=CW_qdtEfwgJRsCyBlk5hNlxwEO-VV6mBXZvkRbND_J8,112
 castor_extractor/transformation/coalesce/assets.py,sha256=pzccYPP66c9PAnVroemx7-6MeRHw7Ft1OlTC6jIamAA,363
 castor_extractor/transformation/coalesce/client/__init__.py,sha256=VRmVpH29rOghtDQnCN7dAdA0dI0Lxseu4BC8rnwM9dU,80
-castor_extractor/transformation/coalesce/client/client.py,sha256=Fm4ozdjosV77AvLVUvOkdMuH2GhepH-AU4xyu7WEgMM,5826
+castor_extractor/transformation/coalesce/client/client.py,sha256=me9U6ovT4qR3AxTv9PN6fxG9UvPAdf7S9dvFDKVgFLY,6640
 castor_extractor/transformation/coalesce/client/credentials.py,sha256=jbJxjbdPspf-dzYKfeb7oqL_8TXd1nvkJrjAcdAnLPc,548
 castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9qr_50SEYV_-heQE4BwED9oNMgYsL-w,1272
 castor_extractor/transformation/coalesce/client/pagination.py,sha256=zynyWCMEzUQ7HA1Q5AP4BAOmxRQI6NA5jCPEo0lHn44,705
@@ -288,7 +288,7 @@ castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13
 castor_extractor/visualization/sigma/assets.py,sha256=iVZqi7XtNgSOVXy0jgeHZonVOeXi7jyikor8ztbECBc,398
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
 castor_extractor/visualization/sigma/client/authentication.py,sha256=gHukrpfboIjZc_O9CcuDtrl6U-StH0J73VY2J74Bm9o,2279
-castor_extractor/visualization/sigma/client/client.py,sha256=bYlOwtb6ftASb9onUSAHCdEiyfsY5H_KHWkQB4y2Y-o,9063
+castor_extractor/visualization/sigma/client/client.py,sha256=l2VlNktvtinRPR9RIIw8LlHRH-hhoycQlMG5HrQiIbc,9228
 castor_extractor/visualization/sigma/client/client_test.py,sha256=ae0ZOvKutCm44jnrJ-0_A5Y6ZGyDkMf9Ml3eEP8dNkY,581
 castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=by9VIFml2whlzQT66f2m56RYBsqPrWdAmIP4JkTaBV4,1799
@@ -429,7 +429,7 @@ castor_extractor/warehouse/snowflake/queries/database.sql,sha256=yxHzV2oQfW0bo2d
 castor_extractor/warehouse/snowflake/queries/function.sql,sha256=8LRh0ybhd-RldJ8UZspWUm3yv52evq11O2uqIO4KqeQ,372
 castor_extractor/warehouse/snowflake/queries/query.sql,sha256=w4T6-TgwUozDgaF3Fk-qex7bDdEIHLkkB5XEe2VJXZQ,1992
 castor_extractor/warehouse/snowflake/queries/schema.sql,sha256=iLn6_y5rn63KigjE4GEAMp8ZuZZofhMXYGb8saPDGUc,776
-castor_extractor/warehouse/snowflake/queries/table.sql,sha256=CbSLfJAylyyyD3mkGPSLLE7BHrGjlY499kzO9RN0e4Y,1473
+castor_extractor/warehouse/snowflake/queries/table.sql,sha256=ovZwQ5NqJB_IFggKfdeuMh2eCiptn4z-Ek6_1CFd1nU,1491
 castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsKOHHlqCtBMQHOV54yy6RKJaXk,570
 castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
 castor_extractor/warehouse/snowflake/query.py,sha256=C2LTdPwBzMQ_zMncg0Kq4_WkoY7K9as5tvxBDrIOlwI,1763
@@ -446,8 +446,8 @@ castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=MAlnTis43E3Amu1e1Oz
 castor_extractor/warehouse/sqlserver/queries/view_ddl.sql,sha256=9rynvx6MWg3iZzrWPB7haZfVKEPkxulzryE2g19x804,315
 castor_extractor/warehouse/sqlserver/query.py,sha256=gr5lnZSUm-wSYuVnJlg6fc7jXWirbL-sCiQN9RnAiPQ,1789
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.25.10.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.25.10.dist-info/METADATA,sha256=G36bUjs5FkXG0aihDl5OmmTf_bFN01Js5Xlbp9T7crA,29395
-castor_extractor-0.25.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.25.10.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
-castor_extractor-0.25.10.dist-info/RECORD,,
+castor_extractor-0.25.14.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.25.14.dist-info/METADATA,sha256=DTV_oWepOjhPFLkQ3XXf06lXcmmUaWCDSHz-PD68Hm0,29696
+castor_extractor-0.25.14.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.25.14.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
+castor_extractor-0.25.14.dist-info/RECORD,,