castor-extractor 0.24.38__tar.gz → 0.24.42__tar.gz

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.24.42 - 2025-08-19
+
+* Strategy: exclude COLUMNS from extraction
+
+## 0.24.41 - 2025-08-19
+
+* Sigma: retry on 429 errors when fetching connection paths
+
+## 0.24.40 - 2025-08-18
+
+* SQLServer: fix database allowlist/blocklist filtering
+
+## 0.24.39 - 2025-08-18
+
+* Databricks:
+  * Fix vanishing owner ID column for tables
+  * Deduplicate lineage with SQL to reduce memory use
+
 ## 0.24.38 - 2025-08-07
 
 * Uploader: Support US and EU zones

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.38
+Version: 0.24.42
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,24 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.42 - 2025-08-19
+
+* Strategy: exclude COLUMNS from extraction
+
+## 0.24.41 - 2025-08-19
+
+* Sigma: retry on 429 errors when fetching connection paths
+
+## 0.24.40 - 2025-08-18
+
+* SQLServer: fix database allowlist/blocklist filtering
+
+## 0.24.39 - 2025-08-18
+
+* Databricks:
+  * Fix vanishing owner ID column for tables
+  * Deduplicate lineage with SQL to reduce memory use
+
 ## 0.24.38 - 2025-08-07
 
 * Uploader: Support US and EU zones

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/visualization/powerbi/client/client.py

@@ -28,6 +28,7 @@ POWERBI_DEFAULT_TIMEOUT_S = 30
 METADATA_BATCH_SIZE = 100
 POWERBI_SCAN_STATUS_DONE = "Succeeded"
 POWERBI_SCAN_SLEEP_S = 1
+POWERBI_SCAN_TIMEOUT_S = 60
 
 MAX_RETRY_PAGES = 1
 RETRY_PAGES_TIMEOUT_MS = 35 * 1000  # 35 seconds
@@ -142,7 +143,7 @@ class PowerbiClient(APIClient):
         endpoint = self.endpoint_factory.metadata_scan_status(scan_id)
         total_waiting_time_s = 0
 
-        while total_waiting_time_s < POWERBI_DEFAULT_TIMEOUT_S:
+        while total_waiting_time_s < POWERBI_SCAN_TIMEOUT_S:
             try:
                 result = self._get(endpoint)
             except HTTPError as e:

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/visualization/sigma/client/sources_transformer.py

@@ -1,6 +1,8 @@
 import logging
+from http import HTTPStatus
 from typing import TYPE_CHECKING, Callable, Iterator
 
+from ....utils import retry_request
 from .endpoints import SigmaEndpointFactory
 
 if TYPE_CHECKING:
@@ -8,6 +10,9 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+SIGMA_CONNECTION_PATH_MAX_RETRY = 1
+SIGMA_CONNECTION_PATH_SLEEP_MS = 30_000  # 30 seconds
+
 
 class SigmaSourcesTransformer:
     """Retrieves asset sources and enhances them with additional information."""
@@ -15,6 +20,17 @@ class SigmaSourcesTransformer:
     def __init__(self, api_client: "SigmaClient"):
         self.api_client = api_client
 
+    @retry_request(
+        status_codes=(HTTPStatus.TOO_MANY_REQUESTS,),
+        max_retries=SIGMA_CONNECTION_PATH_MAX_RETRY,
+        base_ms=SIGMA_CONNECTION_PATH_SLEEP_MS,
+    )
+    def _get_connection_path(self, table_id: str) -> dict:
+        """Retrieves the connection path for a given table id"""
+        return self.api_client._get(
+            endpoint=SigmaEndpointFactory.connection_path(table_id)
+        )
+
     def _map_table_id_to_connection_path(
         self, all_sources: list
     ) -> dict[str, dict]:
@@ -29,9 +45,7 @@ class SigmaSourcesTransformer:
         }
 
         return {
-            table_id: self.api_client._get(
-                endpoint=SigmaEndpointFactory.connection_path(table_id)
-            )
+            table_id: self._get_connection_path(table_id)
             for table_id in unique_table_ids
         }
 

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/visualization/strategy/extract.py

@@ -22,7 +22,7 @@ def iterate_all_data(
 ) -> Iterable[tuple[str, Union[list, dict]]]:
     """Iterate over the extracted data from Strategy"""
 
-    for asset in StrategyAsset:
+    for asset in StrategyAsset.mandatory:
         logger.info(f"Extracting {asset.value.upper()} from REST API")
         data = client.fetch(asset)
         yield asset.name.lower(), list(deep_serialize(data))

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/databricks/client.py

@@ -46,12 +46,11 @@ class DatabricksClient:
 
     @staticmethod
     def _match_table_with_user(table: dict, user_mapping: dict) -> dict:
+        """Matches the table's owner email to an ID, or None if not found."""
         table_owner_email = table.get("owner_email")
-        if not table_owner_email:
-            return table
-        owner_external_id = user_mapping.get(table_owner_email)
-        if not owner_external_id:
-            return table
+        owner_external_id = (
+            user_mapping.get(table_owner_email) if table_owner_email else None
+        )
         return {**table, "owner_external_id": owner_external_id}
 
     @staticmethod

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/databricks/client_test.py

@@ -36,5 +36,6 @@ def test_DatabricksClient__match_table_with_user():
     assert table_with_owner == {**table, "owner_external_id": 3}
 
     table_without_owner = {"id": 1, "owner_email": None}
+    expected = {"id": 1, "owner_email": None, "owner_external_id": None}
     actual = client._match_table_with_user(table_without_owner, user_mapping)
-    assert actual == table_without_owner
+    assert actual == expected

castor_extractor-0.24.42/castor_extractor/warehouse/databricks/queries/column_lineage.sql

@@ -0,0 +1,25 @@
+/*
+Selects all column lineage events for the given day.
+This excludes self-lineage and deduplicates (parent, child) pairs to keep only the most recent lineage event.
+
+Passing parameters is not always supported, so the query must be Python-formatted to set the date.
+*/
+WITH deduplicated_lineage AS (
+    SELECT *,
+           ROW_NUMBER() OVER (
+               PARTITION BY source_table_full_name, source_column_name, target_table_full_name, target_column_name
+               ORDER BY event_time DESC
+           ) AS rank
+    FROM system.access.column_lineage
+    WHERE
+        TRUE
+        AND event_date = DATE('{day}')
+        AND source_table_full_name IS NOT NULL
+        AND source_column_name IS NOT NULL
+        AND target_table_full_name IS NOT NULL
+        AND target_column_name IS NOT NULL
+        AND CONCAT(source_table_full_name, '.', source_column_name) != CONCAT(target_table_full_name, '.', target_column_name)
+)
+SELECT *
+FROM deduplicated_lineage
+WHERE rank = 1

castor_extractor-0.24.42/castor_extractor/warehouse/databricks/queries/table_lineage.sql

@@ -0,0 +1,23 @@
+/*
+Selects all table lineage events for the given day.
+This excludes self-lineage and deduplicates (parent, child) pairs to keep only the most recent lineage event.
+
+Passing parameters is not always supported, so the query must be Python-formatted to set the date.
+*/
+WITH deduplicated_lineage AS (
+    SELECT *,
+           ROW_NUMBER() OVER (
+               PARTITION BY source_table_full_name, target_table_full_name
+               ORDER BY event_time DESC
+           ) AS rank
+    FROM system.access.table_lineage
+    WHERE
+        TRUE
+        AND event_date = DATE('{day}')
+        AND source_table_full_name IS NOT NULL
+        AND target_table_full_name IS NOT NULL
+        AND source_table_full_name != target_table_full_name
+)
+SELECT *
+FROM deduplicated_lineage
+WHERE rank = 1

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/databricks/sql_client.py

@@ -4,20 +4,25 @@ from datetime import date
 
 from databricks import sql  # type: ignore
 
+from ...utils import load_file
 from .credentials import DatabricksCredentials
 from .enums import LineageEntity, TagEntity
 from .format import TagMapping
-from .lineage import valid_lineage
 from .utils import build_path, tag_label
 
 logger = logging.getLogger(__name__)
 
 _INFORMATION_SCHEMA_SQL = "SELECT * FROM system.information_schema"
 
-_LINEAGE_SQL_TPL = """
-SELECT * FROM system.access.{table_name}
-WHERE event_date = DATE('{day}')
-"""
+_LINEAGE_SQL_PATHS = {
+    LineageEntity.COLUMN: "queries/column_lineage.sql",
+    LineageEntity.TABLE: "queries/table_lineage.sql",
+}
+
+
+def _load_lineage_query(lineage_entity: LineageEntity) -> str:
+    filename = _LINEAGE_SQL_PATHS[lineage_entity]
+    return load_file(filename, __file__)
 
 
 class DatabricksSQLClient:
@@ -95,13 +100,11 @@ class DatabricksSQLClient:
         Unfortunately, passing parameters is not always supported. We have to
         format the query beforehand and pass it as plain text for execution.
         """
-        table_name = f"{lineage_entity.value.lower()}_lineage"
-        query = _LINEAGE_SQL_TPL.format(
-            table_name=table_name,
-            day=day,
-        )
+        query_template = _load_lineage_query(lineage_entity)
+        query = query_template.format(day=day)
+
         result = self.execute_sql(query)
         data = []
         for row in result:
             data.append(row.asDict())
-        return valid_lineage(data, lineage_entity)
+        return data

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/sqlserver/extract.py

@@ -52,7 +52,9 @@ def extract_all(**kwargs) -> None:
     client = MSSQLClient(credentials=_credentials(kwargs))
 
     databases = filter_items(
-        client.get_databases(), kwargs.get("allowed"), kwargs.get("blocked")
+        items=client.get_databases(),
+        allowed=kwargs.get("db_allowed"),
+        blocked=kwargs.get("db_blocked"),
     )
 
     query_builder = MSSQLQueryBuilder(

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/sqlserver/queries/column.sql

@@ -91,9 +91,9 @@ columns AS (
     LEFT JOIN column_ids AS i
         ON
             (
-                c.table_name = i.table_name
-                AND c.table_schema = i.schema_name
-                AND c.column_name = i.column_name
+                c.table_name COLLATE DATABASE_DEFAULT = i.table_name COLLATE DATABASE_DEFAULT
+                AND c.table_schema COLLATE DATABASE_DEFAULT = i.schema_name COLLATE DATABASE_DEFAULT
+                AND c.column_name COLLATE DATABASE_DEFAULT = i.column_name COLLATE DATABASE_DEFAULT
             )
 )
 

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -1,4 +1,9 @@
--- Fetch database information
+/*
+Fetch database information
+
+Collation is a set of rules that defines how text data is stored and compared, and it can differ between databases.
+The "COLLATE DATABASE_DEFAULT" is to ensure that text is compared with the same collation.
+ */
 WITH ids AS (
     SELECT DISTINCT
         table_catalog,
@@ -19,4 +24,4 @@ INNER JOIN ids AS i
 LEFT JOIN {database}.sys.sysusers AS u
     ON s.principal_id = u.uid
 LEFT JOIN {database}.sys.databases AS d
-    ON i.table_catalog = d.name
+    ON i.table_catalog COLLATE DATABASE_DEFAULT = d.name COLLATE DATABASE_DEFAULT

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/castor_extractor/warehouse/sqlserver/queries/table.sql

@@ -92,7 +92,7 @@ meta AS (
     FROM
         {database}.information_schema.tables AS t
     LEFT JOIN {database}.sys.databases AS db
-        ON t.table_catalog = db.name
+        ON t.table_catalog COLLATE DATABASE_DEFAULT = db.name COLLATE DATABASE_DEFAULT
 )
 
 SELECT

{castor_extractor-0.24.38 → castor_extractor-0.24.42}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.38"
+version = "0.24.42"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.24.38/castor_extractor/warehouse/databricks/lineage.py

@@ -1,69 +0,0 @@
-from typing import Iterable, Optional
-
-from .enums import LineageEntity
-
-
-class LineageProcessor:
-    """
-    helper class that handles lineage deduplication and filtering
-    """
-
-    def __init__(self, lineage_entity: LineageEntity):
-        self.lineage_entity = lineage_entity
-
-        self.lineage: dict[tuple[str, str], dict] = dict()
-
-    def _parent_path(self, link) -> Optional[str]:
-        if self.lineage_entity == LineageEntity.TABLE:
-            return link["source_table_full_name"]
-
-        source_table = link["source_table_full_name"]
-        source_column = link["source_column_name"]
-        if not (source_table and source_column):
-            return None
-
-        return f"{source_table}.{source_column}"
-
-    def _child_path(self, link) -> Optional[str]:
-        if self.lineage_entity == LineageEntity.TABLE:
-            return link["target_table_full_name"]
-
-        target_table = link["target_table_full_name"]
-        target_column = link["target_column_name"]
-        if not (target_table and target_column):
-            return None
-
-        return f"{target_table}.{target_column}"
-
-    def add(self, link: dict) -> None:
-        """
-        If the parent and child paths are valid, keeps the most recent lineage
-        link in the `self.lineage` map.
-        """
-        parent = self._parent_path(link)
-        child = self._child_path(link)
-        timestamp = link["event_time"]
-
-        if not (parent and child and parent != child):
-            return
-
-        key = (parent, child)
-        if key in self.lineage and self.lineage[key]["event_time"] > timestamp:
-            return
-
-        self.lineage[key] = link
-
-
-def valid_lineage(
-    lineage: Iterable[dict], lineage_entity: LineageEntity
-) -> list[dict]:
-    """
-    Filters out self-lineage or lineage with a missing source or target path,
-    then deduplicates by picking the link with the most recent event timestmap.
-    """
-    deduplicated_lineage = LineageProcessor(lineage_entity)
-
-    for link in lineage:
-        deduplicated_lineage.add(link)
-
-    return list(deduplicated_lineage.lineage.values())

castor_extractor-0.24.38/castor_extractor/warehouse/databricks/lineage_test.py

@@ -1,89 +0,0 @@
-from .enums import LineageEntity
-from .lineage import LineageProcessor, valid_lineage
-
-_OLDER_DATE = "2025-01-01 00:00:01.0"
-_CLOSER_DATE = "2025-01-01 02:02:02.0"
-
-_TABLE_LINEAGES = [
-    {
-        "source_table_full_name": "a.b.source",
-        "target_table_full_name": "a.b.target",
-        "event_time": _CLOSER_DATE,
-        "other": "more recent stuff",
-    },
-    {
-        "source_table_full_name": "a.b.source",
-        "target_table_full_name": "a.b.target",
-        "event_time": _OLDER_DATE,
-        "other": "stuff that's too old",
-    },
-    {
-        "source_table_full_name": "no target",
-        "target_table_full_name": None,
-        "event_time": _CLOSER_DATE,
-    },
-    {
-        "source_table_full_name": None,
-        "target_table_full_name": "no source",
-        "event_time": _CLOSER_DATE,
-    },
-]
-
-
-_COLUMN_LINEAGES = [
-    {
-        "source_table_full_name": "a.b.source",
-        "source_column_name": "src_col",
-        "target_table_full_name": "a.b.target",
-        "target_column_name": "trgt_col",
-        "event_time": _OLDER_DATE,
-        "other": "old stuff",
-    },
-    {
-        "source_table_full_name": "a.b.source",
-        "source_column_name": "src_col",
-        "target_table_full_name": "a.b.target",
-        "target_column_name": "trgt_col",
-        "event_time": _CLOSER_DATE,
-        "other": "newer stuff",
-    },
-    {
-        "source_table_full_name": "a.b.toto",
-        "source_column_name": "toto_col",
-        "target_table_full_name": "a.b.tata",
-        "target_column_name": "tata_col",
-        "event_time": _OLDER_DATE,
-    },
-    {
-        "source_table_full_name": "a.b.source",
-        "source_column_name": "a.b.source",
-        "target_table_full_name": None,
-        "target_column_name": None,
-        "event_time": _CLOSER_DATE,
-    },
-]
-
-
-def test_valid_lineage():
-    table_links = valid_lineage(_TABLE_LINEAGES, LineageEntity.TABLE)
-
-    assert len(table_links) == 1
-    assert table_links[0]["source_table_full_name"] == "a.b.source"
-    assert table_links[0]["target_table_full_name"] == "a.b.target"
-    assert table_links[0]["event_time"] == _CLOSER_DATE
-    assert table_links[0]["other"] == "more recent stuff"
-
-
-def test_LineageLinks_add():
-    deduplicated_lineage = LineageProcessor(LineageEntity.COLUMN)
-    for link in _COLUMN_LINEAGES:
-        deduplicated_lineage.add(link)
-
-    lineage = deduplicated_lineage.lineage
-    assert len(lineage) == 2
-    assert ("a.b.source.src_col", "a.b.target.trgt_col") in lineage
-    assert ("a.b.toto.toto_col", "a.b.tata.tata_col") in lineage
-    assert (
-        lineage[("a.b.source.src_col", "a.b.target.trgt_col")]["other"]
-        == "newer stuff"
-    )