castor-extractor 0.24.43__tar.gz → 0.24.48__tar.gz

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.24.48 - 2025-09-09
+
+* SqlServer: handle hyphens in database name
+
+## 0.24.47 - 2025-09-08
+
+* SqlServer: extract SQL queries and lineage
+
+## 0.24.46 - 2025-09-03
+
+* Sigma: Added `HTTPStatus.FORBIDDEN` to the list of ignored errors
+
+## 0.24.45 - 2025-08-27
+
+* Sigma: Increasing pagination limit for Sigma extraction
+
+## 0.24.44 - 2025-08-22
+
+* Coalesce: do not skip nodes raising a 500 Server Error
+
 ## 0.24.43 - 2025-08-20
 
 * SQLServer:

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.43
+Version: 0.24.48
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,26 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.48 - 2025-09-09
+
+* SqlServer: handle hyphens in database name
+
+## 0.24.47 - 2025-09-08
+
+* SqlServer: extract SQL queries and lineage
+
+## 0.24.46 - 2025-09-03
+
+* Sigma: Added `HTTPStatus.FORBIDDEN` to the list of ignored errors
+
+## 0.24.45 - 2025-08-27
+
+* Sigma: Increasing pagination limit for Sigma extraction
+
+## 0.24.44 - 2025-08-22
+
+* Coalesce: do not skip nodes raising a 500 Server Error
+
 ## 0.24.43 - 2025-08-20
 
 * SQLServer:

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/commands/extract_sqlserver.py

@@ -15,6 +15,14 @@ def main():
     parser.add_argument("-u", "--user", help="MSSQL User")
     parser.add_argument("-p", "--password", help="MSSQL Password")
 
+    parser.add_argument(
+        "-s",
+        "--skip-queries",
+        dest="skip_queries",
+        action="store_true",
+        help="Skip the extraction of SQL queries",
+    )
+
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     parser.add_argument(

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/transformation/coalesce/client/client.py

@@ -3,8 +3,6 @@ from functools import partial
 from http import HTTPStatus
 from typing import Callable, Optional
 
-from pydantic import ValidationError
-
 from ....utils import (
     APIClient,
     BearerAuth,
@@ -123,13 +121,7 @@ class CoalesceClient(APIClient):
         for index, env in enumerate(environments):
             env_id = env["id"]
             logger.info(f"Fetching nodes for env #{env_id} - {index}/{total}")
-            try:
-                nodes.extend(self._fetch_env_nodes(env_id))
-            except ValidationError as e:
-                # 500 Server Error: Internal Server Error on Coalesce API
-                logger.warning(
-                    f"Skipping nodes for {env_id} due to the following Error: {e}"
-                )
+            nodes.extend(self._fetch_env_nodes(env_id))
             logger.info(f"{len(nodes)} nodes extracted so far")
         return nodes
 

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/visualization/sigma/client/client.py

@@ -53,6 +53,7 @@ _IGNORED_ERROR_CODES = (
     HTTPStatus.INTERNAL_SERVER_ERROR,
     HTTPStatus.CONFLICT,
     HTTPStatus.NOT_FOUND,
+    HTTPStatus.FORBIDDEN,
 )
 SIGMA_SAFE_MODE = RequestSafeMode(
     max_errors=_VOLUME_IGNORED,

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/visualization/sigma/client/pagination.py

@@ -5,7 +5,7 @@ from pydantic.alias_generators import to_camel
 
 from ....utils import PaginationModel
 
-SIGMA_API_LIMIT = 200  # default number of records per page
+SIGMA_API_LIMIT = 1_000  # default number of records per page
 SIGMA_QUERIES_PAGINATION_LIMIT = 50
 
 

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/warehouse/abstract/asset.py

@@ -64,6 +64,11 @@ FUNCTIONS_ASSETS = (WarehouseAsset.FUNCTION,)
 QUERIES_ASSETS = (WarehouseAsset.QUERY,)
 VIEWS_ASSETS = (WarehouseAsset.VIEW_DDL,)
 
+QUERIES_ASSET_GROUPS = (
+    WarehouseAssetGroup.QUERY,
+    WarehouseAssetGroup.VIEW_DDL,
+)
+
 EXTERNAL_LINEAGE_ASSETS = (
     WarehouseAsset.EXTERNAL_COLUMN_LINEAGE,
     WarehouseAsset.EXTERNAL_TABLE_LINEAGE,

castor_extractor-0.24.48/castor_extractor/warehouse/sqlserver/client.py

@@ -0,0 +1,159 @@
+import logging
+from collections.abc import Iterator
+
+from sqlalchemy import text
+
+from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode
+
+logger = logging.getLogger(__name__)
+
+SERVER_URI = "{user}:{password}@{host}:{port}/{database}"
+MSSQL_URI = f"mssql+pymssql://{SERVER_URI}"
+DEFAULT_PORT = 1433
+
+_KEYS = ("user", "password", "host", "port", "database")
+
+_SYSTEM_DATABASES = ("master", "model", "msdb", "tempdb", "DBAdmin")
+
+
+def _check_key(credentials: dict) -> None:
+    for key in _KEYS:
+        if key not in credentials:
+            raise KeyError(f"Missing {key} in credentials")
+
+
+class MSSQLClient(SqlalchemyClient):
+    """Microsoft Server SQL client"""
+
+    @staticmethod
+    def name() -> str:
+        return "MSSQL"
+
+    def _engine_options(self, credentials: dict) -> dict:
+        return {}
+
+    def _build_uri(self, credentials: dict) -> str:
+        _check_key(credentials)
+        uri = MSSQL_URI.format(
+            user=credentials["user"],
+            password=uri_encode(credentials["password"]),
+            host=credentials["host"],
+            port=credentials.get("port") or DEFAULT_PORT,
+            database=credentials["database"],
+        )
+        return uri
+
+    def execute(self, query: ExtractionQuery) -> Iterator[dict]:
+        """
+        Re-implements the SQLAlchemyClient execute function to ensure we consume
+        the cursor before calling connection.close() as it wipes out the data
+        otherwise
+        """
+        connection = self.connect()
+        try:
+            proxy = connection.execute(text(query.statement), query.params)
+            results = list(self._process_result(proxy))
+            yield from results
+        finally:
+            self.close()
+
+    def get_databases(self) -> list[str]:
+        result = self.execute(
+            ExtractionQuery("SELECT name FROM sys.databases", {})
+        )
+        return [
+            row["name"]
+            for row in result
+            if row["name"] not in _SYSTEM_DATABASES
+        ]
+
+    def _current_database(self) -> str:
+        result = self.execute(
+            ExtractionQuery("SELECT DB_NAME() AS database_name", {})
+        )
+        return next(result)["database_name"]
+
+    def _has_access(self, name: str, object_type: str, permission: str) -> bool:
+        query_text = f"""
+            SELECT
+            HAS_PERMS_BY_NAME('{name}', '{object_type}', '{permission}')
+            AS has_permission
+        """
+        query = ExtractionQuery(query_text, dict())
+        result = next(self.execute(query))
+        return result["has_permission"] == 1
+
+    def _has_table_read_access(self, table_name: str) -> bool:
+        """
+        Check whether we have READ access to the given table
+        """
+        return self._has_access(
+            name=table_name,
+            object_type="OBJECT",
+            permission="SELECT",
+        )
+
+    def _has_view_database_state(self) -> bool:
+        """
+        Check whether we have VIEW DATABASE STATE permissions, which
+        is necessary to fetch data from the Query Store
+        """
+        return self._has_access(
+            name=self._current_database(),
+            object_type="DATABASE",
+            permission="VIEW DATABASE STATE",
+        )
+
+    def _has_query_store(self) -> bool:
+        """
+        Checks whether the Query Store is activated on this instance.
+        This is required to extract the SQL queries history.
+        https://learn.microsoft.com/en-us/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store?view=sql-server-ver17"""
+        sql = """
+        SELECT
+            desired_state
+        FROM
+            sys.database_query_store_options
+        """
+        query = ExtractionQuery(sql, {})
+        # 2 = READ_WRITE, which means the Query Store is activated
+        return next(self.execute(query))["desired_state"] == 2
+
+    def has_queries_permissions(self) -> bool:
+        """
+        Verify that we habe the required permissions to extract
+        query history and view object definitions (DDL).
+
+        This check ensures:
+        - Query Store is enabled on the database.
+        - We have the VIEW DATABASE STATE permissions
+        - We have read access to the relevant system tables.
+        """
+
+        tables = (
+            # SQL queries
+            "sys.query_store_plan",
+            "sys.query_store_query",
+            "sys.query_store_query_text",
+            "sys.query_store_runtime_stats",
+            # views DDL
+            "sys.schemas",
+            "sys.sql_modules",
+            "sys.views",
+        )
+
+        has_permissions = True
+        for table in tables:
+            if not self._has_table_read_access(table):
+                logger.info(f"Missing READ permissions for table {table}")
+                has_permissions = False
+
+        if not self._has_view_database_state():
+            logger.info("Missing permissions: VIEW DATABASE STATE")
+            has_permissions = False
+
+        if not self._has_query_store():
+            logger.info("Missing permissions: Query Store is not activated")
+            has_permissions = False
+
+        return has_permissions

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/warehouse/sqlserver/extract.py

@@ -4,6 +4,8 @@ from ...utils import LocalStorage, filter_items, from_env, write_summary
 from ..abstract import (
     CATALOG_ASSETS,
     EXTERNAL_LINEAGE_ASSETS,
+    QUERIES_ASSETS,
+    VIEWS_ASSETS,
     SQLExtractionProcessor,
     SupportedAssets,
     WarehouseAsset,
@@ -19,8 +21,10 @@ logger = logging.getLogger(__name__)
 
 MSSQL_ASSETS: SupportedAssets = {
     WarehouseAssetGroup.CATALOG: CATALOG_ASSETS,
-    WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
     WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
+    WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
+    WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
+    WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
 }
 
 
@@ -71,7 +75,10 @@ def extract_all(**kwargs) -> None:
         storage=storage,
     )
 
+    skip_queries = kwargs.get("skip_queries") or False
     for group in extractable_asset_groups(MSSQL_ASSETS):
+        if group == WarehouseAssetGroup.QUERY and skip_queries:
+            continue
         for asset in group:
             logger.info(f"Extracting `{asset.value.upper()}` ...")
             location = extractor.extract(asset, skip_existing)

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/warehouse/sqlserver/queries/column.sql

@@ -11,7 +11,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        {database}.sys.tables
+        [{database}].sys.tables
 
     UNION
 
@@ -21,7 +21,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        {database}.sys.views
+        [{database}].sys.views
 
     UNION
 
@@ -31,7 +31,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        {database}.sys.external_tables
+        [{database}].sys.external_tables
 ),
 /*
 `sys.columns` contains, among others:
@@ -54,11 +54,11 @@ column_ids AS (
         schema_name = ss.name,
         schema_id = ss.schema_id,
         comment = CONVERT(varchar(1024), ep.value)
-    FROM {database}.sys.columns AS sc
+    FROM [{database}].sys.columns AS sc
     LEFT JOIN extended_tables AS et ON sc.object_id = et.table_id
-    LEFT JOIN {database}.sys.schemas AS ss ON et.schema_id = ss.schema_id
-    LEFT JOIN {database}.sys.databases AS sd ON sd.name = '{database}'
-    LEFT JOIN {database}.sys.extended_properties AS ep
+    LEFT JOIN [{database}].sys.schemas AS ss ON et.schema_id = ss.schema_id
+    LEFT JOIN [{database}].sys.databases AS sd ON sd.name = '{database}'
+    LEFT JOIN [{database}].sys.extended_properties AS ep
         ON
             sc.object_id = ep.major_id
             AND sc.column_id = ep.minor_id
@@ -87,7 +87,7 @@ columns AS (
         i.comment,
         column_id = CONCAT(i.table_id, '.', c.column_name)
     FROM
-        {database}.INFORMATION_SCHEMA.COLUMNS AS c
+        [{database}].INFORMATION_SCHEMA.COLUMNS AS c
     LEFT JOIN column_ids AS i
         ON
             (

castor_extractor-0.24.48/castor_extractor/warehouse/sqlserver/queries/query.sql

@@ -0,0 +1,25 @@
+SELECT
+    q.query_id,
+    qt.query_sql_text as query_text,
+    rs.count_executions,
+    rs.last_duration as duration,
+    rs.last_execution_time as start_time,
+    'unknown-user' as user_name,
+    'unknown-user' as user_id,
+    DATEADD(SECOND, last_duration / 1000000,
+        DATEADD(MICROSECOND, last_duration % 1000000, rs.last_execution_time)
+    ) AS end_time
+FROM
+    sys.query_store_runtime_stats AS rs
+INNER JOIN
+    sys.query_store_plan p
+    ON rs.plan_id = p.plan_id
+INNER JOIN
+    sys.query_store_query q
+    ON p.query_id = q.query_id
+INNER JOIN
+    sys.query_store_query_text qt
+    ON q.query_text_id = qt.query_text_id
+WHERE
+    CAST(rs.last_execution_time AS DATE) = :day
+    AND DATEPART(HOUR, rs.last_execution_time) BETWEEN :hour_min AND :hour_max

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -8,7 +8,7 @@ WITH ids AS (
     SELECT DISTINCT
         table_catalog,
         table_schema
-    FROM {database}.INFORMATION_SCHEMA.TABLES
+    FROM [{database}].INFORMATION_SCHEMA.TABLES
 )
 
 SELECT
@@ -18,10 +18,10 @@ SELECT
     schema_id = CAST(d.database_id AS VARCHAR(10)) + '_' + CAST(s.schema_id AS VARCHAR(10)),
     schema_owner = u.name,
     schema_owner_id = u.uid
-FROM {database}.sys.schemas AS s
+FROM [{database}].sys.schemas AS s
 INNER JOIN ids AS i
     ON s.name = i.table_schema
-LEFT JOIN {database}.sys.sysusers AS u
+LEFT JOIN [{database}].sys.sysusers AS u
     ON s.principal_id = u.uid
-LEFT JOIN {database}.sys.databases AS d
+LEFT JOIN [{database}].sys.databases AS d
     ON i.table_catalog COLLATE DATABASE_DEFAULT = d.name COLLATE DATABASE_DEFAULT

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/castor_extractor/warehouse/sqlserver/queries/table.sql

@@ -11,7 +11,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        {database}.sys.tables
+        [{database}].sys.tables
 
     UNION
 
@@ -21,7 +21,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        {database}.sys.views
+        [{database}].sys.views
 
     UNION
 
@@ -31,14 +31,14 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        {database}.sys.external_tables
+        [{database}].sys.external_tables
 ),
 -- Get the row count per table
 partitions AS (
     SELECT
         object_id,
         row_count = SUM(rows)
-    FROM {database}.sys.partitions
+    FROM [{database}].sys.partitions
     GROUP BY object_id
 ),
 -- Append row count to table properties
@@ -70,11 +70,11 @@ table_ids AS (
         row_count,
         comment = CONVERT(varchar(1024), ep.value)
     FROM extended_tables_with_row_count AS et
-    LEFT JOIN {database}.sys.schemas AS ss
+    LEFT JOIN [{database}].sys.schemas AS ss
         ON et.schema_id = ss.schema_id
-    LEFT JOIN {database}.sys.sysusers AS u
+    LEFT JOIN [{database}].sys.sysusers AS u
         ON et.table_owner_id = u.uid
-    LEFT JOIN {database}.sys.extended_properties AS ep
+    LEFT JOIN [{database}].sys.extended_properties AS ep
         ON (
             et.table_id = ep.major_id
             AND ep.minor_id = 0
@@ -90,8 +90,8 @@ meta AS (
         t.table_name,
         t.table_type
     FROM
-        {database}.INFORMATION_SCHEMA.TABLES AS t
-    LEFT JOIN {database}.sys.databases AS db
+        [{database}].INFORMATION_SCHEMA.TABLES AS t
+    LEFT JOIN [{database}].sys.databases AS db
         ON t.table_catalog COLLATE DATABASE_DEFAULT = db.name COLLATE DATABASE_DEFAULT
 )
 

castor_extractor-0.24.48/castor_extractor/warehouse/sqlserver/queries/view_ddl.sql

@@ -0,0 +1,13 @@
+SELECT
+    v.name AS view_name,
+    m.definition AS view_definition,
+    s.name AS schema_name,
+    DB_NAME() AS database_name
+FROM
+    sys.views v
+INNER JOIN
+    sys.schemas s
+    ON v.schema_id = s.schema_id
+INNER JOIN
+    sys.sql_modules m
+    ON v.object_id = m.object_id

{castor_extractor-0.24.43 → castor_extractor-0.24.48}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.43"
+version = "0.24.48"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.24.43/castor_extractor/warehouse/sqlserver/client.py

@@ -1,68 +0,0 @@
-import logging
-from collections.abc import Iterator
-
-from sqlalchemy import text
-
-from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode
-
-logger = logging.getLogger(__name__)
-
-SERVER_URI = "{user}:{password}@{host}:{port}/{database}"
-MSSQL_URI = f"mssql+pymssql://{SERVER_URI}"
-DEFAULT_PORT = 1433
-
-_KEYS = ("user", "password", "host", "port", "database")
-
-_SYSTEM_DATABASES = ("master", "model", "msdb", "tempdb", "DBAdmin")
-
-
-def _check_key(credentials: dict) -> None:
-    for key in _KEYS:
-        if key not in credentials:
-            raise KeyError(f"Missing {key} in credentials")
-
-
-class MSSQLClient(SqlalchemyClient):
-    """Microsoft Server SQL client"""
-
-    @staticmethod
-    def name() -> str:
-        return "MSSQL"
-
-    def _engine_options(self, credentials: dict) -> dict:
-        return {}
-
-    def _build_uri(self, credentials: dict) -> str:
-        _check_key(credentials)
-        uri = MSSQL_URI.format(
-            user=credentials["user"],
-            password=uri_encode(credentials["password"]),
-            host=credentials["host"],
-            port=credentials.get("port") or DEFAULT_PORT,
-            database=credentials["database"],
-        )
-        return uri
-
-    def execute(self, query: ExtractionQuery) -> Iterator[dict]:
-        """
-        Re-implements the SQLAlchemyClient execute function to ensure we consume
-        the cursor before calling connection.close() as it wipes out the data
-        otherwise
-        """
-        connection = self.connect()
-        try:
-            proxy = connection.execute(text(query.statement), query.params)
-            results = list(self._process_result(proxy))
-            yield from results
-        finally:
-            self.close()
-
-    def get_databases(self) -> list[str]:
-        result = self.execute(
-            ExtractionQuery("SELECT name FROM sys.databases", {})
-        )
-        return [
-            row["name"]
-            for row in result
-            if row["name"] not in _SYSTEM_DATABASES
-        ]