castor-extractor 0.25.2__tar.gz → 0.25.6__tar.gz

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.25.6 - 2025-10-06
+
+* PowerBi: Support additional authentication methods
+
+## 0.25.5 - 2025-10-06
+
+* Snowflake: remove extraction of roles and grants
+
+## 0.25.4 - 2025-10-02
+
+* SqlServer: switch database to support clients running their instance on Azure
+
+## 0.25.3 - 2025-10-01
+
+* Sigma: sleep between sources requests to avoid rate limit errors
+
 ## 0.25.2 - 2025-09-30
 
 * PowerBi: Support auth with private_key

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.25.2
+Version: 0.25.6
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -216,6 +216,22 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.25.6 - 2025-10-06
+
+* PowerBi: Support additional authentication methods
+
+## 0.25.5 - 2025-10-06
+
+* Snowflake: remove extraction of roles and grants
+
+## 0.25.4 - 2025-10-02
+
+* SqlServer: switch database to support clients running their instance on Azure
+
+## 0.25.3 - 2025-10-01
+
+* Sigma: sleep between sources requests to avoid rate limit errors
+
 ## 0.25.2 - 2025-09-30
 
 * PowerBi: Support auth with private_key

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/commands/extract_powerbi.py

@@ -21,8 +21,12 @@ def main():
     auth_group.add_argument(
         "-cert",
         "--certificate",
-        help="file path to json certificate file with"
-        "keys: private_key, thumbprint, public_certificate",
+        help=(
+            "Path to certificate file for authentication. "
+            "Accepts: X.509 certificates (.pem, .crt), "
+            "PKCS#12 files (.p12, .pfx), or JSON files containing "
+            "certificate data or custom authentication secrets."
+        ),
     )
     parser.add_argument(
         "-sc",

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/utils/client/abstract.py

@@ -26,6 +26,17 @@ class AbstractSourceClient(ABC):
     def execute(self, query: ExtractionQuery) -> Iterator[dict]:
         pass
 
+    def set_database(self, database: Optional[str]) -> None:
+        """
+        Set the active database for this client.
+
+        Some source technologies require establishing a new connection
+        or using a different connection URI when switching databases.
+
+        Default behaviour is to do nothing.
+        """
+        pass
+
 
 class SqlalchemyClient(AbstractSourceClient, ABC):
     def __init__(self, credentials: dict):

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/utils/client/api/pagination.py

@@ -58,7 +58,7 @@ class PaginationModel(BaseModel):
 def fetch_all_pages(
     request: Callable,
     pagination_model: type[PaginationModel],
-    rate_limit: Optional[int] = None,
+    rate_limit: Optional[float] = None,
 ) -> Iterator:
     """
     Method to return all results of a Paginated API based on the

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/visualization/powerbi/client/credentials.py

@@ -11,9 +11,17 @@ REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
 
 
 class PowerbiCertificate(BaseModel):
+    """
+    Supports all dict credentials formats supported by PowerBI
+    https://learn.microsoft.com/en-us/python/api/msal/msal.application.confidentialclientapplication
+    """
+
+    client_assertion: Optional[str] = None
+    passphrase: Optional[str] = None
+    private_key: Optional[str] = None
+    private_key_pfx_path: Optional[str] = None
     public_certificate: Optional[str] = None
-    private_key: str
-    thumbprint: str
+    thumbprint: Optional[str] = None
 
 
 class PowerbiCredentials(BaseSettings):

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/visualization/sigma/client/sources_transformer.py

@@ -1,5 +1,6 @@
 import logging
 from http import HTTPStatus
+from time import sleep
 from typing import TYPE_CHECKING, Callable, Iterator
 
 from ....utils import fetch_all_pages, retry_request
@@ -13,6 +14,7 @@ logger = logging.getLogger(__name__)
 
 SIGMA_CONNECTION_PATH_MAX_RETRY = 1
 SIGMA_CONNECTION_PATH_SLEEP_MS = 30_000  # 30 seconds
+SIGMA_SOURCES_SLEEP_MS = 2 / 1000  # 0.2 seconds
 
 
 class SigmaSourcesTransformer:
@@ -84,6 +86,29 @@ class SigmaSourcesTransformer:
                 "sources": enhanced_sources,
             }
 
+    def _fetch_asset_sources(
+        self,
+        asset_id: str,
+        endpoint: Callable[[str], str],
+        with_pagination: bool,
+    ) -> dict:
+        """Fetches sources for a single asset."""
+        endpoint_url = endpoint(asset_id)
+
+        if with_pagination:
+            request = self.api_client._get_paginated(endpoint=endpoint_url)
+            pages_generator = fetch_all_pages(
+                request=request,
+                pagination_model=SigmaTokenPagination,
+                rate_limit=SIGMA_SOURCES_SLEEP_MS,
+            )
+            sources = list(pages_generator)
+        else:
+            sources = self.api_client._get(endpoint=endpoint_url)
+            sleep(SIGMA_SOURCES_SLEEP_MS)
+
+        return {"asset_id": asset_id, "sources": sources}
+
     def _get_all_sources(
         self,
         endpoint: Callable[[str], str],
@@ -91,16 +116,10 @@ class SigmaSourcesTransformer:
         with_pagination: bool = False,
     ) -> Iterator[dict]:
         """Returns transformed sources for the given assets"""
-        all_sources = []
-
-        for asset_id in asset_ids:
-            endpoint_url = endpoint(asset_id)
-            if with_pagination:
-                request = self.api_client._get_paginated(endpoint=endpoint_url)
-                sources = list(fetch_all_pages(request, SigmaTokenPagination))
-            else:
-                sources = self.api_client._get(endpoint=endpoint_url)
-            all_sources.append({"asset_id": asset_id, "sources": sources})
+        all_sources = [
+            self._fetch_asset_sources(asset_id, endpoint, with_pagination)
+            for asset_id in asset_ids
+        ]
 
         table_to_path = self._map_table_id_to_connection_path(all_sources)
 

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/warehouse/abstract/extract.py

@@ -51,6 +51,7 @@ class SQLExtractionProcessor:
     def _fetch(self, query: ExtractionQuery) -> Iterator[dict]:
         default: Callable[[], Iterator] = lambda: iter(())  # type: ignore
         decorator = safe_mode(self._safe_mode, default)
+        self._client.set_database(query.database)
         decorated_execute = decorator(self._client.execute)
         return decorated_execute(query)
 

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/warehouse/abstract/query.py

@@ -21,11 +21,22 @@ class AssetNotSupportedError(NotImplementedError):
 
 
 class ExtractionQuery:
-    """contains both statement and parameters for the query"""
+    """
+    Contains useful context to run the query:
+    - the sql statement itself
+    - parameters { ... }
+    - optionally, the target database (can be used to change the engine's URI)
+    """
 
-    def __init__(self, statement: str, params: dict):
+    def __init__(
+        self,
+        statement: str,
+        params: dict,
+        database: Optional[str] = None,
+    ):
         self.statement = statement
         self.params = params
+        self.database = database
 
 
 class AbstractQueryBuilder(ABC):

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/warehouse/snowflake/extract.py

@@ -24,12 +24,7 @@ SNOWFLAKE_ASSETS: SupportedAssets = {
     WarehouseAssetGroup.FUNCTION: FUNCTIONS_ASSETS,
     WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
     WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
-    WarehouseAssetGroup.ROLE: (
-        WarehouseAsset.GRANT_TO_ROLE,
-        WarehouseAsset.GRANT_TO_USER,
-        WarehouseAsset.ROLE,
-        WarehouseAsset.USER,
-    ),
+    WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
     WarehouseAssetGroup.SNOWFLAKE_LINEAGE: (WarehouseAsset.COLUMN_LINEAGE,),
     WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
 }

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/warehouse/sqlserver/client.py

@@ -1,7 +1,8 @@
 import logging
 from collections.abc import Iterator
+from typing import Optional
 
-from sqlalchemy import text
+from sqlalchemy import create_engine, text
 
 from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode
 
@@ -63,6 +64,18 @@ class MSSQLClient(SqlalchemyClient):
         finally:
             self.close()
 
+    def set_database(self, database: Optional[str]) -> None:
+        """
+        To support SQL Server running on Azure, we must change the engine's
+        URI for each database-scoped query
+        https://chatgpt.com/share/68de93c3-9550-8001-9d54-c5da86faa43c
+        """
+        if not database:
+            return
+        self.close()
+        database_uri = f"{self._uri}/{database}"
+        self._engine = create_engine(database_uri, **self._options)
+
     def get_databases(self) -> list[str]:
         result = self.execute(
             ExtractionQuery("SELECT name FROM sys.databases", {})

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/castor_extractor/warehouse/sqlserver/query.py

@@ -40,10 +40,14 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
         self._databases = databases
 
     @staticmethod
-    def _format(query: ExtractionQuery, values: dict) -> ExtractionQuery:
+    def _format(
+        query: ExtractionQuery,
+        database: str,
+    ) -> ExtractionQuery:
         return ExtractionQuery(
-            statement=query.statement.format(**values),
+            statement=query.statement.format(database=database),
             params=query.params,
+            database=database,
         )
 
     def build(self, asset: WarehouseAsset) -> list[ExtractionQuery]:
@@ -58,7 +62,4 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
         logger.info(
             f"\tWill run queries with following database params: {self._databases}",
         )
-        return [
-            self._format(query, {"database": database})
-            for database in self._databases
-        ]
+        return [self._format(query, database) for database in self._databases]

{castor_extractor-0.25.2 → castor_extractor-0.25.6}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.25.2"
+version = "0.25.6"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.25.2/castor_extractor/warehouse/snowflake/queries/grant_to_role.sql

@@ -1,14 +0,0 @@
-SELECT
-    created_on,
-    modified_on,
-    privilege,
-    granted_on,
-    name,
-    table_catalog AS "database",
-    table_schema AS "schema",
-    granted_to,
-    grantee_name,
-    grant_option,
-    granted_by,
-    deleted_on
-FROM snowflake.account_usage.grants_to_roles

castor_extractor-0.25.2/castor_extractor/warehouse/snowflake/queries/grant_to_user.sql

@@ -1,8 +0,0 @@
-SELECT
-    created_on,
-    deleted_on,
-    role,
-    granted_to,
-    grantee_name,
-    granted_by
-FROM snowflake.account_usage.grants_to_users

castor_extractor-0.25.2/castor_extractor/warehouse/snowflake/queries/role.sql

@@ -1,6 +0,0 @@
-SELECT
-    created_on,
-    deleted_on,
-    name,
-    comment
-FROM snowflake.account_usage.roles