castor-extractor 0.24.40__py3-none-any.whl → 0.24.43__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.24.43 - 2025-08-20
+
+* SQLServer:
+  * raise error when no database is left after filtering
+  * use uppercase INFORMATION_SCHEMA for case-sensitive database compatibility
+
+## 0.24.42 - 2025-08-19
+
+* Strategy: exclude COLUMNS from extraction
+
+## 0.24.41 - 2025-08-19
+
+* Sigma: retry on 429 errors when fetching connection paths
+
 ## 0.24.40 - 2025-08-18
 
 * SQLServer: fix database allowlist/blocklist filtering

castor_extractor/visualization/sigma/client/sources_transformer.py

@@ -1,6 +1,8 @@
 import logging
+from http import HTTPStatus
 from typing import TYPE_CHECKING, Callable, Iterator
 
+from ....utils import retry_request
 from .endpoints import SigmaEndpointFactory
 
 if TYPE_CHECKING:
@@ -8,6 +10,9 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+SIGMA_CONNECTION_PATH_MAX_RETRY = 1
+SIGMA_CONNECTION_PATH_SLEEP_MS = 30_000  # 30 seconds
+
 
 class SigmaSourcesTransformer:
     """Retrieves asset sources and enhances them with additional information."""
@@ -15,6 +20,17 @@ class SigmaSourcesTransformer:
     def __init__(self, api_client: "SigmaClient"):
         self.api_client = api_client
 
+    @retry_request(
+        status_codes=(HTTPStatus.TOO_MANY_REQUESTS,),
+        max_retries=SIGMA_CONNECTION_PATH_MAX_RETRY,
+        base_ms=SIGMA_CONNECTION_PATH_SLEEP_MS,
+    )
+    def _get_connection_path(self, table_id: str) -> dict:
+        """Retrieves the connection path for a given table id"""
+        return self.api_client._get(
+            endpoint=SigmaEndpointFactory.connection_path(table_id)
+        )
+
     def _map_table_id_to_connection_path(
         self, all_sources: list
     ) -> dict[str, dict]:
@@ -29,9 +45,7 @@ class SigmaSourcesTransformer:
         }
 
         return {
-            table_id: self.api_client._get(
-                endpoint=SigmaEndpointFactory.connection_path(table_id)
-            )
+            table_id: self._get_connection_path(table_id)
             for table_id in unique_table_ids
         }
 

castor_extractor/visualization/strategy/extract.py

@@ -22,7 +22,7 @@ def iterate_all_data(
 ) -> Iterable[tuple[str, Union[list, dict]]]:
     """Iterate over the extracted data from Strategy"""
 
-    for asset in StrategyAsset:
+    for asset in StrategyAsset.mandatory:
         logger.info(f"Extracting {asset.value.upper()} from REST API")
         data = client.fetch(asset)
         yield asset.name.lower(), list(deep_serialize(data))

castor_extractor/warehouse/databricks/api_client.py

@@ -81,7 +81,13 @@ class DatabricksAPIClient(APIClient):
     def databases(self) -> list[dict]:
         content = self._get(DatabricksEndpointFactory.databases())
         _databases = self.formatter.format_database(content.get("catalogs", []))
-        return [d for d in _databases if self._keep_catalog(d["database_name"])]
+        filtered_databases = [
+            d for d in _databases if self._keep_catalog(d["database_name"])
+        ]
+        logger.info(
+            f"Available databases: {[d['database_name'] for d in filtered_databases]}"
+        )
+        return filtered_databases
 
     def _schemas_of_database(self, database: dict) -> list[dict]:
         payload = {"catalog_name": database["database_name"]}

castor_extractor/warehouse/sqlserver/extract.py

@@ -57,6 +57,8 @@ def extract_all(**kwargs) -> None:
         blocked=kwargs.get("db_blocked"),
     )
 
+    logger.info(f"Available databases: {databases}\n")
+
     query_builder = MSSQLQueryBuilder(
         databases=databases,
     )

castor_extractor/warehouse/sqlserver/queries/column.sql

@@ -87,7 +87,7 @@ columns AS (
         i.comment,
         column_id = CONCAT(i.table_id, '.', c.column_name)
     FROM
-        {database}.information_schema.columns AS c
+        {database}.INFORMATION_SCHEMA.COLUMNS AS c
     LEFT JOIN column_ids AS i
         ON
             (

castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -8,7 +8,7 @@ WITH ids AS (
     SELECT DISTINCT
         table_catalog,
         table_schema
-    FROM {database}.information_schema.tables
+    FROM {database}.INFORMATION_SCHEMA.TABLES
 )
 
 SELECT

castor_extractor/warehouse/sqlserver/queries/table.sql

@@ -90,7 +90,7 @@ meta AS (
         t.table_name,
         t.table_type
     FROM
-        {database}.information_schema.tables AS t
+        {database}.INFORMATION_SCHEMA.TABLES AS t
     LEFT JOIN {database}.sys.databases AS db
         ON t.table_catalog COLLATE DATABASE_DEFAULT = db.name COLLATE DATABASE_DEFAULT
 )

castor_extractor/warehouse/sqlserver/query.py

@@ -11,6 +11,11 @@ from ..abstract import (
 logger = logging.getLogger(__name__)
 
 
+_NO_DATABASE_ERROR_MSG = (
+    "No databases eligible for extraction. "
+    "If you are using the db_allow/db_block options, please make sure to use the correct case."
+)
+
 _DATABASE_REQUIRED = (
     WarehouseAsset.SCHEMA,
     WarehouseAsset.TABLE,
@@ -29,6 +34,8 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
         time_filter: Optional[TimeFilter] = None,
     ):
         super().__init__(time_filter=time_filter)
+        if not databases:
+            raise ValueError(_NO_DATABASE_ERROR_MSG)
         self._databases = databases
 
     @staticmethod

{castor_extractor-0.24.40.dist-info → castor_extractor-0.24.43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.40
+Version: 0.24.43
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,20 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.43 - 2025-08-20
+
+* SQLServer:
+  * raise error when no database is left after filtering
+  * use uppercase INFORMATION_SCHEMA for case-sensitive database compatibility
+
+## 0.24.42 - 2025-08-19
+
+* Strategy: exclude COLUMNS from extraction
+
+## 0.24.41 - 2025-08-19
+
+* Sigma: retry on 429 errors when fetching connection paths
+
 ## 0.24.40 - 2025-08-18
 
 * SQLServer: fix database allowlist/blocklist filtering

{castor_extractor-0.24.40.dist-info → castor_extractor-0.24.43.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=tgZkN-SNTMCro37DG0nW91MaD6ZnHM9VWWZG2-7TP68,19406
+CHANGELOG.md,sha256=BP52t2sRBsw161mUbZUgLka80iFSbvhcYPju_bgXLL0,19737
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -277,7 +277,7 @@ castor_extractor/visualization/sigma/client/client_test.py,sha256=ae0ZOvKutCm44j
 castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=i7KTKnl2Os6752CdtJl0vPSC_Z6JxmacodV_saOnce0,1662
 castor_extractor/visualization/sigma/client/pagination.py,sha256=2bFA7GiBUUasFtHJKA90516d283p7Pg50-4zw6Fwt8I,726
-castor_extractor/visualization/sigma/client/sources_transformer.py,sha256=mRupzxjtjDqELIouHF0egBkgslDmn5Y4uqO_sbUGCNs,3244
+castor_extractor/visualization/sigma/client/sources_transformer.py,sha256=n-5mZWSvzfTwpM5VP_bwlcxcaAwCKEEbpMCG_1KRVP4,3748
 castor_extractor/visualization/sigma/client/sources_transformer_test.py,sha256=06yUHXyv65amXLKXhix6K3kkVc1kpBqSjIYcxbyMI4Y,2766
 castor_extractor/visualization/sigma/extract.py,sha256=poTh70Xm2D6BwbdGApLkjXy6-t4iZnOoMB5DPfaTLEI,2929
 castor_extractor/visualization/strategy/__init__.py,sha256=HOMv4JxqF5ZmViWi-pDE-PSXJRLTdXal_jtpHG_rlR8,123
@@ -286,7 +286,7 @@ castor_extractor/visualization/strategy/client/__init__.py,sha256=XWP0yF5j6JefDJ
 castor_extractor/visualization/strategy/client/client.py,sha256=6DJO0Fh67FXxmwY5h_X9cu5sEq3GhM19b9hwn_fvhSE,9460
 castor_extractor/visualization/strategy/client/credentials.py,sha256=urFfNxWX1JG6wwFMYImufQzHa5g-sgjdlVGzi63owwg,1113
 castor_extractor/visualization/strategy/client/properties.py,sha256=66oBm8Kz6HEQW_jNR5_fAI_O921R2F5yH2Ff3zjtJOk,4500
-castor_extractor/visualization/strategy/extract.py,sha256=2fBuvS2xiOGXRpxXnZsE_C3en6t1-BlM5TbusjHyEkg,1166
+castor_extractor/visualization/strategy/extract.py,sha256=9Ec48BwiA-5nykhcQiqkSiL_RB034_Ci-ck89YK5Nic,1176
 castor_extractor/visualization/tableau/__init__.py,sha256=eFI_1hjdkxyUiAYiy3szwyuwn3yJ5C_KbpBU0ySJDcQ,138
 castor_extractor/visualization/tableau/assets.py,sha256=HbCRd8VCj1WBEeqg9jwnygnT7xOFJ6PQD7Lq7sV-XR0,635
 castor_extractor/visualization/tableau/client/__init__.py,sha256=P8RKFKOC63WkH5hdEytJOwHS9vzQ8GXreLfXZetmMP8,78
@@ -335,7 +335,7 @@ castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTw
 castor_extractor/warehouse/bigquery/query.py,sha256=FEekxlkrfAXzsT8Kj1AIqYd5mURB5MlZIkbFVXVqEhU,4762
 castor_extractor/warehouse/bigquery/types.py,sha256=rfKkKA13Et7TM4I0uVaXkLfuaBXkv51bNTp4AO0QSdw,57
 castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
-castor_extractor/warehouse/databricks/api_client.py,sha256=kLcUGSgrfybZUrpt0tE7qe2OoSSN7IK4myyB7c0czOY,6260
+castor_extractor/warehouse/databricks/api_client.py,sha256=qBf2T5R1l5a1_kmTbrTYW8VNsz4h_L3zCpUuboNEunQ,6448
 castor_extractor/warehouse/databricks/api_client_test.py,sha256=YTWC-X7L-XAfK5b39TUgTmR1ifv0QrY5tvLNoSbpmjg,466
 castor_extractor/warehouse/databricks/client.py,sha256=LzpeVQIOYi_QTfdOHbK6SB4SgxhZ7p9TNxh0Iwfz850,3307
 castor_extractor/warehouse/databricks/client_test.py,sha256=dqEdEAt-6e8CtQ7M2L5vDYkn4JvOjqyqZSFEpQ55WRc,1432
@@ -422,17 +422,17 @@ castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3
 castor_extractor/warehouse/snowflake/query.py,sha256=C2LTdPwBzMQ_zMncg0Kq4_WkoY7K9as5tvxBDrIOlwI,1763
 castor_extractor/warehouse/sqlserver/__init__.py,sha256=PdOuYznmvKAbfWAm8UdN47MfEsd9jqPi_dDi3WEo1KY,116
 castor_extractor/warehouse/sqlserver/client.py,sha256=Bjfpw96IKAQfWPiU5SZYEDfetwfkqZrnKbQYoStcnZc,2007
-castor_extractor/warehouse/sqlserver/extract.py,sha256=GbOlSq8JR6HaJZunkfiRxaSt0pbgazQjF8GpgqWWIcU,2294
+castor_extractor/warehouse/sqlserver/extract.py,sha256=C8MubArUUq5r8UyalXzmQycqgQOv_0DYKXmxDYeGpHg,2350
 castor_extractor/warehouse/sqlserver/queries/.sqlfluff,sha256=yy0KQdz8I_67vnXyX8eeWwOWkxTXvHyVKSVwhURktd8,48
-castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=eRILCgdygYRvtfSdxaswIiIYKW-PiJXW2qi3yHtrfns,2913
+castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=MpIpUGP-oRJEDv6baO34RKg0KtZqK9CHH7e818-_qvY,2913
 castor_extractor/warehouse/sqlserver/queries/database.sql,sha256=4dPeBCn85MEOXr1f-DPXxiI3RvvoE_1n8lsbTs26E0I,150
-castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=Zp4G86FJ_Be8Zqvdlu7K8DqmsUL62kxbwaUk5asZ0V4,881
-castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=YwFhHc6rGbszqQt7Izh7EngVwrrBoEZ9kniuWXNtGco,2837
+castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=sMOrMC2UCqTzRMHiQqQFWozaujrefLBSkwMrpbo-rO4,881
+castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=liLvzp6NyrW8vF7TrP5_4Asmw2cyMDu8FCvhEV6Z1jc,2837
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
-castor_extractor/warehouse/sqlserver/query.py,sha256=7sW8cK3JzxPt6faTJ7e4lk9tE4fo_AeCymI-LqsSols,1276
+castor_extractor/warehouse/sqlserver/query.py,sha256=QcGJ3lYbWpGCx-qMytTiEnTqGeEsUZrBLIdg_9jSm8A,1526
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.40.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.40.dist-info/METADATA,sha256=ONg1SCc3gcrOJqBE92EtyfQctf-hRxI_u2VUbBpvgVA,26859
-castor_extractor-0.24.40.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.40.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.40.dist-info/RECORD,,
+castor_extractor-0.24.43.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.43.dist-info/METADATA,sha256=05SXgWB7i36oBtLtNbik3yDxf2vliZuYuVYvXHP7bqg,27190
+castor_extractor-0.24.43.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.43.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.43.dist-info/RECORD,,