castor-extractor 0.24.49__tar.gz → 0.24.54__tar.gz

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.24.54 - 2025-09-18
+
+* SqlServer: fix typo in the extraction query of schemas
+
+## 0.24.53 - 2025-09-18
+
+* Fix CSV field size to support running on Windows
+
+## 0.24.52 - 2025-09-18
+
+* SqlServer : improve extraction of users and technical owners
+
+## 0.24.51 - 2025-09-17
+
+* PowerBI : don't store sensitive data in activity events
+
+## 0.24.50 - 2025-09-15
+
+* SqlServer: support multiple databases in queries
+
 ## 0.24.49 - 2025-09-12
 
 * Tableau: add option to bypass ssl certificate verification

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.49
+Version: 0.24.54
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,26 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.54 - 2025-09-18
+
+* SqlServer: fix typo in the extraction query of schemas
+
+## 0.24.53 - 2025-09-18
+
+* Fix CSV field size to support running on Windows
+
+## 0.24.52 - 2025-09-18
+
+* SqlServer : improve extraction of users and technical owners
+
+## 0.24.51 - 2025-09-17
+
+* PowerBI : don't store sensitive data in activity events
+
+## 0.24.50 - 2025-09-15
+
+* SqlServer: support multiple databases in queries
+
 ## 0.24.49 - 2025-09-12
 
 * Tableau: add option to bypass ssl certificate verification

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/utils/formatter.py

@@ -1,6 +1,7 @@
 """convert files to csv"""
 
 import csv
+import ctypes
 import json
 import logging
 import re
@@ -21,6 +22,8 @@ CSV_OPTIONS: CsvOptions = {
     "quotechar": '"',
 }
 
+CSV_FIELD_SIZE_MB = 100
+
 ScalarValue = Union[int, float, None, str]
 
 logger = logging.getLogger(__name__)
@@ -132,6 +135,31 @@ class Formatter(ABC):
         pass
 
 
+def _set_csv_field_size_limit(target_limit_mb: int) -> None:
+    """
+    Safely set the maximum CSV field size limit across platforms.
+
+    This function wraps `csv.field_size_limit()` to avoid `OverflowError` on
+    Windows, where the maximum C long is only 32 bits (2^31 - 1). On Linux and
+    macOS, the C long is typically 64 bits, allowing much larger values.
+
+    The requested limit is specified in megabytes and converted to bytes.
+
+    It is then clamped to the maximum value supported by:
+      - the current platform's C long
+      - Python's `sys.maxsize`
+      - the requested target limit
+    """
+    target_limit_bytes = target_limit_mb * 1024**2
+
+    # max value of C long for the current platform
+    platform_c_long = (1 << (8 * ctypes.sizeof(ctypes.c_long) - 1)) - 1
+
+    limit_bytes = min(target_limit_bytes, sys.maxsize, platform_c_long)
+
+    csv.field_size_limit(limit_bytes)
+
+
 class CsvFormatter(Formatter):
     """
     Serialize/Deserialize CSV
@@ -141,9 +169,7 @@ class CsvFormatter(Formatter):
         return "csv"
 
     # increase the size limit (some fields are very large)
-    # Limit to 100 MB. Value must be smaller than the C long maximum value.
-    size_limit = min(sys.maxsize, 100 * 1024**3)
-    csv.field_size_limit(size_limit)
+    _set_csv_field_size_limit(target_limit_mb=CSV_FIELD_SIZE_MB)
 
     @staticmethod
     def serialize(buffer: IO[str], data: Iterable[dict]) -> bool:

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/visualization/powerbi/client/client.py

@@ -33,6 +33,8 @@ POWERBI_SCAN_TIMEOUT_S = 60
 MAX_RETRY_PAGES = 1
 RETRY_PAGES_TIMEOUT_MS = 35 * 1000  # 35 seconds
 
+KEYS_TO_HIDE = ("ClientIP", "UserAgent")
+
 logger = logging.getLogger(__name__)
 
 
@@ -61,7 +63,11 @@ class PowerbiClient(APIClient):
             self._get,
             endpoint=self.endpoint_factory.activity_events(day),
         )
-        yield from fetch_all_pages(request, PowerBiPagination)
+        for event in fetch_all_pages(request, PowerBiPagination):
+            for key in KEYS_TO_HIDE:
+                if key in event:
+                    del event[key]
+            yield event
 
     def _datasets(self) -> Iterator[dict]:
         """

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/visualization/powerbi/client/client_test.py

@@ -142,17 +142,44 @@ def test__activity_events(power_bi_client):
     day = date.today()
     mocked_get_results = [
         {
-            Keys.ACTIVITY_EVENT_ENTITIES: ["foo", "bar"],
+            Keys.ACTIVITY_EVENT_ENTITIES: [
+                {
+                    "id": "foo",
+                    "name": "Foo",
+                    "ClientIP": "1.1.1.1",
+                    "UserAgent": "Mozilla/5.0",
+                },
+                {
+                    "id": "bar",
+                    "name": "Bar",
+                    "ClientIP": "1.1.1.2",
+                    "UserAgent": "Mozilla/5.0",
+                },
+            ],
             Keys.LAST_RESULT_SET: False,
             Keys.CONTINUATION_URI: "https://next-call-1",
         },
         {
-            Keys.ACTIVITY_EVENT_ENTITIES: ["baz"],
+            Keys.ACTIVITY_EVENT_ENTITIES: [
+                {
+                    "id": "baz",
+                    "name": "Baz",
+                    "ClientIP": "1.1.1.3",
+                    "UserAgent": "Mozilla/5.0",
+                }
+            ],
             Keys.LAST_RESULT_SET: False,
             Keys.CONTINUATION_URI: "https://next-call-2",
         },
         {
-            Keys.ACTIVITY_EVENT_ENTITIES: ["biz"],
+            Keys.ACTIVITY_EVENT_ENTITIES: [
+                {
+                    "id": "biz",
+                    "name": "Biz",
+                    "ClientIP": "1.1.1.4",
+                    "UserAgent": "Mozilla/5.0",
+                }
+            ],
             Keys.LAST_RESULT_SET: True,
             Keys.CONTINUATION_URI: None,
         },
@@ -162,7 +189,12 @@ def test__activity_events(power_bi_client):
         mocked_get.side_effect = mocked_get_results
 
         result = list(power_bi_client._activity_events(day=day))
-        assert result == ["foo", "bar", "baz", "biz"]
+        assert result == [
+            {"id": "foo", "name": "Foo"},
+            {"id": "bar", "name": "Bar"},
+            {"id": "baz", "name": "Baz"},
+            {"id": "biz", "name": "Biz"},
+        ]
 
         expected_calls = [
             call(endpoint=ENDPOINT_FACTORY.activity_events(day=day)),

castor_extractor-0.24.54/castor_extractor/visualization/sigma/client/authentication.py

@@ -0,0 +1,68 @@
+import logging
+import threading
+import time
+from typing import Optional
+
+import requests
+
+from ....utils import (
+    BearerAuth,
+    build_url,
+    handle_response,
+)
+from .endpoints import (
+    SigmaEndpointFactory,
+)
+
+logger = logging.getLogger(__name__)
+
+
+_AUTH_TIMEOUT_S = 60
+_REFRESH_BUFFER_S = 300
+
+
+class SigmaBearerAuth(BearerAuth):
+    def __init__(self, host: str, token_payload: dict[str, str]):
+        auth_endpoint = SigmaEndpointFactory.authentication()
+        self.authentication_url = build_url(host, auth_endpoint)
+        self.token_payload = token_payload
+        self._token_expires_at: Optional[float] = None
+        self._token_lock = threading.Lock()
+
+    def fetch_token(self) -> str:
+        """Returns the token and sets its expiration time."""
+        token_api_path = self.authentication_url
+        token_response = requests.post(
+            token_api_path, data=self.token_payload, timeout=_AUTH_TIMEOUT_S
+        )
+        response_data = handle_response(token_response)
+        expires_in_seconds = int(response_data["expires_in"])
+        self._token_expires_at = time.time() + expires_in_seconds
+        return response_data["access_token"]
+
+    def _is_token_expired_or_expiring_soon(self) -> bool:
+        """
+        Returns True if the token is expired or will expire soon (within buffer time)
+        """
+        if self._token_expires_at is None:
+            return False
+
+        return time.time() >= (self._token_expires_at - _REFRESH_BUFFER_S)
+
+    def _needs_refresh(self, force_refresh: bool = False) -> bool:
+        """Returns True if the token needs to be refreshed."""
+        is_expired = self._is_token_expired_or_expiring_soon()
+        return not self._token or force_refresh or is_expired
+
+    def _fetch_token(self, force_refresh: bool = False) -> Optional[str]:
+        """Returns the API token, refreshing it if needed (thread-safe)."""
+        if not self._needs_refresh(force_refresh):
+            return f"Bearer {self._token}"
+
+        with self._token_lock:
+            if not self._needs_refresh(force_refresh):
+                return f"Bearer {self._token}"
+
+            logger.info("Refreshing authentication token...")
+            self._token = self.fetch_token()
+            return f"Bearer {self._token}"

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/visualization/sigma/client/client.py

@@ -5,19 +5,16 @@ from functools import partial
 from http import HTTPStatus
 from typing import Callable, Iterable, Optional
 
-import requests
 from pydantic import BaseModel
 
 from ....utils import (
     APIClient,
-    BearerAuth,
     RequestSafeMode,
-    build_url,
     fetch_all_pages,
-    handle_response,
     retry,
 )
 from ..assets import SigmaAsset
+from .authentication import SigmaBearerAuth
 from .credentials import SigmaCredentials
 from .endpoints import SigmaEndpointFactory
 from .pagination import (
@@ -39,7 +36,6 @@ _DATA_ELEMENTS: tuple[str, ...] = (
     "viz",
 )
 
-_AUTH_TIMEOUT_S = 60
 _SIGMA_TIMEOUT_S = 300
 
 _SIGMA_HEADERS = {
@@ -85,20 +81,6 @@ class Lineage(BaseModel):
     context: LineageContext
 
 
-class SigmaBearerAuth(BearerAuth):
-    def __init__(self, host: str, token_payload: dict[str, str]):
-        auth_endpoint = SigmaEndpointFactory.authentication()
-        self.authentication_url = build_url(host, auth_endpoint)
-        self.token_payload = token_payload
-
-    def fetch_token(self):
-        token_api_path = self.authentication_url
-        token_response = requests.post(
-            token_api_path, data=self.token_payload, timeout=_AUTH_TIMEOUT_S
-        )
-        return handle_response(token_response)["access_token"]
-
-
 class SigmaClient(APIClient):
     def __init__(
         self,
@@ -240,8 +222,6 @@ class SigmaClient(APIClient):
 
         lineage_context = self._lineage_context(elements)
 
-        self._auth.refresh_token()
-
         with ThreadPoolExecutor(max_workers=_THREADS_LINEAGE) as executor:
             results = executor.map(self._get_lineage, lineage_context)
 

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/warehouse/sqlserver/client.py

@@ -7,13 +7,20 @@ from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode
 
 logger = logging.getLogger(__name__)
 
-SERVER_URI = "{user}:{password}@{host}:{port}/{database}"
+SERVER_URI = "{user}:{password}@{host}:{port}"
 MSSQL_URI = f"mssql+pymssql://{SERVER_URI}"
 DEFAULT_PORT = 1433
 
-_KEYS = ("user", "password", "host", "port", "database")
+_KEYS = ("user", "password", "host", "port")
 
-_SYSTEM_DATABASES = ("master", "model", "msdb", "tempdb", "DBAdmin")
+_SYSTEM_DATABASES = (
+    "DBAdmin",
+    "dbaTools",
+    "master",
+    "model",
+    "msdb",
+    "tempdb",
+)
 
 
 def _check_key(credentials: dict) -> None:
@@ -39,7 +46,6 @@ class MSSQLClient(SqlalchemyClient):
             password=uri_encode(credentials["password"]),
             host=credentials["host"],
             port=credentials.get("port") or DEFAULT_PORT,
-            database=credentials["database"],
         )
         return uri
 
@@ -67,12 +73,6 @@ class MSSQLClient(SqlalchemyClient):
             if row["name"] not in _SYSTEM_DATABASES
         ]
 
-    def _current_database(self) -> str:
-        result = self.execute(
-            ExtractionQuery("SELECT DB_NAME() AS database_name", {})
-        )
-        return next(result)["database_name"]
-
     def _has_access(self, name: str, object_type: str, permission: str) -> bool:
         query_text = f"""
             SELECT
@@ -83,45 +83,45 @@ class MSSQLClient(SqlalchemyClient):
         result = next(self.execute(query))
         return result["has_permission"] == 1
 
-    def _has_table_read_access(self, table_name: str) -> bool:
+    def _has_table_read_access(self, database: str, table_name: str) -> bool:
         """
         Check whether we have READ access to the given table
         """
         return self._has_access(
-            name=table_name,
+            name=f"[{database}].{table_name}",
             object_type="OBJECT",
             permission="SELECT",
         )
 
-    def _has_view_database_state(self) -> bool:
+    def _has_view_database_state(self, database: str) -> bool:
         """
         Check whether we have VIEW DATABASE STATE permissions, which
         is necessary to fetch data from the Query Store
         """
         return self._has_access(
-            name=self._current_database(),
+            name=database,
             object_type="DATABASE",
             permission="VIEW DATABASE STATE",
         )
 
-    def _has_query_store(self) -> bool:
+    def _has_query_store(self, database: str) -> bool:
         """
-        Checks whether the Query Store is activated on this instance.
+        Checks whether the Query Store is activated on this database.
         This is required to extract the SQL queries history.
         https://learn.microsoft.com/en-us/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store?view=sql-server-ver17"""
-        sql = """
+        sql = f"""
         SELECT
             desired_state
         FROM
-            sys.database_query_store_options
+            [{database}].sys.database_query_store_options
         """
         query = ExtractionQuery(sql, {})
         # 2 = READ_WRITE, which means the Query Store is activated
         return next(self.execute(query))["desired_state"] == 2
 
-    def has_queries_permissions(self) -> bool:
+    def has_queries_permissions(self, database: str) -> bool:
         """
-        Verify that we habe the required permissions to extract
+        Verify that we have the required permissions to extract
         query history and view object definitions (DDL).
 
         This check ensures:
@@ -144,16 +144,16 @@ class MSSQLClient(SqlalchemyClient):
 
         has_permissions = True
         for table in tables:
-            if not self._has_table_read_access(table):
-                logger.info(f"Missing READ permissions for table {table}")
+            if not self._has_table_read_access(database, table):
+                logger.info(f"Missing READ permission on {database}.{table}")
                 has_permissions = False
 
-        if not self._has_view_database_state():
-            logger.info("Missing permissions: VIEW DATABASE STATE")
+        if not self._has_view_database_state(database):
+            logger.info(f"Missing VIEW DATABASE STATE in database {database}")
             has_permissions = False
 
-        if not self._has_query_store():
-            logger.info("Missing permissions: Query Store is not activated")
+        if not self._has_query_store(database):
+            logger.info(f"Query Store is not activated in database {database}")
             has_permissions = False
 
         return has_permissions

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/warehouse/sqlserver/queries/query.sql

@@ -10,15 +10,15 @@ SELECT
         DATEADD(MICROSECOND, last_duration % 1000000, rs.last_execution_time)
     ) AS end_time
 FROM
-    sys.query_store_runtime_stats AS rs
+    [{database}].sys.query_store_runtime_stats AS rs
 INNER JOIN
-    sys.query_store_plan p
+    [{database}].sys.query_store_plan p
     ON rs.plan_id = p.plan_id
 INNER JOIN
-    sys.query_store_query q
+    [{database}].sys.query_store_query q
     ON p.query_id = q.query_id
 INNER JOIN
-    sys.query_store_query_text qt
+    [{database}].sys.query_store_query_text qt
     ON q.query_text_id = qt.query_text_id
 WHERE
     CAST(rs.last_execution_time AS DATE) = :day

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -17,11 +17,11 @@ SELECT
     schema_name = s.name,
     schema_id = CAST(d.database_id AS VARCHAR(10)) + '_' + CAST(s.schema_id AS VARCHAR(10)),
     schema_owner = u.name,
-    schema_owner_id = u.uid
+    schema_owner_id = u.name
 FROM [{database}].sys.schemas AS s
 INNER JOIN ids AS i
     ON s.name = i.table_schema
-LEFT JOIN [{database}].sys.sysusers AS u
-    ON s.principal_id = u.uid
+LEFT JOIN [{database}].sys.database_principals AS u
+    ON s.principal_id = u.principal_id
 LEFT JOIN [{database}].sys.databases AS d
     ON i.table_catalog COLLATE DATABASE_DEFAULT = d.name COLLATE DATABASE_DEFAULT

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/warehouse/sqlserver/queries/table.sql

@@ -8,7 +8,7 @@ WITH extended_tables AS (
     SELECT
         table_id = object_id,
         table_name = name,
-        table_owner_id = principal_id,
+        principal_id,
         schema_id
     FROM
         [{database}].sys.tables
@@ -18,7 +18,7 @@ WITH extended_tables AS (
     SELECT
         table_id = object_id,
         table_name = name,
-        table_owner_id = principal_id,
+        principal_id,
         schema_id
     FROM
         [{database}].sys.views
@@ -28,7 +28,7 @@ WITH extended_tables AS (
     SELECT
         table_id = object_id,
         table_name = name,
-        table_owner_id = principal_id,
+        principal_id,
         schema_id
     FROM
         [{database}].sys.external_tables
@@ -65,15 +65,15 @@ table_ids AS (
         table_name,
         schema_name = ss.name,
         schema_id = ss.schema_id,
-        table_owner_id,
+        table_owner_id = u.name,
         table_owner = u.name,
         row_count,
         comment = CONVERT(varchar(1024), ep.value)
     FROM extended_tables_with_row_count AS et
     LEFT JOIN [{database}].sys.schemas AS ss
         ON et.schema_id = ss.schema_id
-    LEFT JOIN [{database}].sys.sysusers AS u
-        ON et.table_owner_id = u.uid
+    LEFT JOIN [{database}].sys.database_principals AS u
+        ON et.principal_id = u.principal_id
     LEFT JOIN [{database}].sys.extended_properties AS ep
         ON (
             et.table_id = ep.major_id

castor_extractor-0.24.54/castor_extractor/warehouse/sqlserver/queries/user.sql

@@ -0,0 +1,4 @@
+SELECT
+    user_name = u.name,
+    user_id = u.name
+FROM [{database}].sys.database_principals AS u

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/warehouse/sqlserver/queries/view_ddl.sql

@@ -4,10 +4,10 @@ SELECT
     s.name AS schema_name,
     DB_NAME() AS database_name
 FROM
-    sys.views v
+    [{database}].sys.views v
 INNER JOIN
-    sys.schemas s
+    [{database}].sys.schemas s
     ON v.schema_id = s.schema_id
 INNER JOIN
-    sys.sql_modules m
+    [{database}].sys.sql_modules m
     ON v.object_id = m.object_id

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/castor_extractor/warehouse/sqlserver/query.py

@@ -16,10 +16,8 @@ _NO_DATABASE_ERROR_MSG = (
     "If you are using the db_allow/db_block options, please make sure to use the correct case."
 )
 
-_DATABASE_REQUIRED = (
-    WarehouseAsset.SCHEMA,
-    WarehouseAsset.TABLE,
-    WarehouseAsset.COLUMN,
+_SQLSERVER_DUPLICATED_ASSETS: tuple[WarehouseAsset, ...] = (
+    WarehouseAsset.USER,
 )
 
 
@@ -33,7 +31,10 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
         databases: list[str],
         time_filter: Optional[TimeFilter] = None,
     ):
-        super().__init__(time_filter=time_filter)
+        super().__init__(
+            time_filter=time_filter,
+            duplicated=_SQLSERVER_DUPLICATED_ASSETS,
+        )
         if not databases:
             raise ValueError(_NO_DATABASE_ERROR_MSG)
         self._databases = databases
@@ -48,7 +49,10 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
     def build(self, asset: WarehouseAsset) -> list[ExtractionQuery]:
         query = self.build_default(asset)
 
-        if asset not in _DATABASE_REQUIRED:
+        if asset == WarehouseAsset.DATABASE:
+            # database.sql does not include a {database} placeholder.
+            # Indeed, databases are extracted at the server level
+            # (not scoped to a specific database).
             return [query]
 
         logger.info(

{castor_extractor-0.24.49 → castor_extractor-0.24.54}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.49"
+version = "0.24.54"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.24.49/castor_extractor/warehouse/sqlserver/queries/user.sql

@@ -1,4 +0,0 @@
-SELECT
-    user_name = u.name,
-    user_id = u.principal_id
-FROM sys.database_principals AS u