castor-extractor 0.24.32__tar.gz → 0.24.34__tar.gz

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/CHANGELOG.md

@@ -1,12 +1,20 @@
 # Changelog
 
+## 0.24.34 - 2025-07-02
+
+* SQLServer: multiple databases
+
+## 0.24.33 - 2025-07-10
+
+* Tableau - Add an option to skip fields ingestion
+
 ## 0.24.32 - 2025-07-02
 
 * Salesforce reporting - extract report's metadata
-*
+
 ## 0.24.31 - 2025-07-02
 
-* Looker Studio: add option to list users via a provided JSON file
+* Looker Studio: add an option to list users via a provided JSON file
 
 ## 0.24.30 - 2025-06-26
 

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.32
+Version: 0.24.34
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,13 +215,21 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.34 - 2025-07-02
+
+* SQLServer: multiple databases
+
+## 0.24.33 - 2025-07-10
+
+* Tableau - Add an option to skip fields ingestion
+
 ## 0.24.32 - 2025-07-02
 
 * Salesforce reporting - extract report's metadata
-*
+
 ## 0.24.31 - 2025-07-02
 
-* Looker Studio: add option to list users via a provided JSON file
+* Looker Studio: add an option to list users via a provided JSON file
 
 ## 0.24.30 - 2025-06-26
 

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/commands/extract_sqlserver.py

@@ -23,6 +23,16 @@ def main():
         action="store_true",
         help="Skips files already extracted instead of replacing them",
     )
+    parser.add_argument(
+        "--db-allowed",
+        nargs="*",
+        help="List of databases that should be extracted",
+    )
+    parser.add_argument(
+        "--db-blocked",
+        nargs="*",
+        help="List of databases that should not be extracted",
+    )
     parser.set_defaults(skip_existing=False)
 
     args = parser.parse_args()
@@ -35,4 +45,6 @@ def main():
         password=args.password,
         output_directory=args.output,
         skip_existing=args.skip_existing,
+        db_allowed=args.db_allowed,
+        db_blocked=args.db_blocked,
     )

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/commands/extract_tableau.py

@@ -28,6 +28,13 @@ def main():
         help="Option to avoid extracting Tableau columns, default to False",
     )
 
+    parser.add_argument(
+        "--skip-fields",
+        dest="skip_fields",
+        action="store_true",
+        help="Option to avoid extracting Tableau fields, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",
@@ -41,6 +48,14 @@ def main():
         required=False,
     )
 
+    parser.add_argument(
+        "-ie",
+        "--ignore-errors",
+        action="store_true",
+        dest="ignore_errors",
+        help="Allow partial extraction of Fields and Columns: skip batch in case of Timeout errors",
+    )
+
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     tableau.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/utils/__init__.py

@@ -19,7 +19,13 @@ from .client import (
     handle_response,
     uri_encode,
 )
-from .collection import deduplicate, empty_iterator, group_by, mapping_from_rows
+from .collection import (
+    deduplicate,
+    empty_iterator,
+    filter_items,
+    group_by,
+    mapping_from_rows,
+)
 from .constants import OUTPUT_DIR
 from .deprecate import deprecate_python
 from .env import from_env

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/utils/collection.py

@@ -2,6 +2,8 @@ from collections import defaultdict
 from collections.abc import Iterable, Sequence
 from typing import (
     Any,
+    List,
+    Optional,
     TypeVar,
 )
 
@@ -80,3 +82,27 @@ def deduplicate(
         deduplicated.append(element)
 
     return deduplicated
+
+
+def filter_items(
+    items: Iterable[T],
+    allowed: Optional[Iterable[T]] = None,
+    blocked: Optional[Iterable[T]] = None,
+) -> List[T]:
+    """
+    Filters `items` by excluding any in `blocked` or including only those in `allowed`.
+    If both `allowed` and `blocked` are None, returns all items.
+    If both are provided, raise an error.
+    """
+    items = list(items)
+
+    if allowed and blocked:
+        raise AttributeError(
+            "Only one of `allowed` and `blocked` can be provided"
+        )
+    if blocked:
+        return [item for item in items if item not in blocked]
+    if allowed:
+        return [item for item in items if item in allowed]
+
+    return items

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/utils/collection_test.py

@@ -1,4 +1,6 @@
-from .collection import deduplicate, mapping_from_rows
+import pytest
+
+from .collection import deduplicate, filter_items, mapping_from_rows
 
 
 def test__mapping_from_rows__basic_mapping():
@@ -72,3 +74,31 @@ def test_deduplicate():
         {"id": "2", "name": "duplicate"},
     ]
     assert deduplicate("id", elements) == [e1, e2, e3]
+
+
+def test_sqlserver_databases():
+    databases = [
+        "prod",
+        "staging",
+        "test",
+    ]
+
+    # 1. No allowed or blocked: should return all
+    result1 = filter_items(databases)
+    assert result1 == [
+        "prod",
+        "staging",
+        "test",
+    ]
+
+    # 2. Block "prod": only staging and test should remain
+    result2 = filter_items(databases, blocked=["prod"])
+    assert result2 == ["staging", "test"]
+
+    # 3. Only allow "staging" and "test"
+    result3 = filter_items(databases, allowed=["staging", "test"])
+    assert result3 == ["staging", "test"]
+
+    # 4. allowed and blocked, should raise
+    with pytest.raises(AttributeError):
+        filter_items(databases, blocked=["prod"], allowed=["staging", "test"])

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/visualization/looker_studio/client/queries/query.sql

@@ -29,6 +29,7 @@ WITH ranked_by_datasource AS (
         `{project}.region-{region}.INFORMATION_SCHEMA.JOBS_BY_PROJECT`
     WHERE
         job_type = 'QUERY'
+        AND ARRAY_LENGTH(referenced_tables) > 0
         AND EXISTS (
             SELECT
                 1

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/visualization/tableau/client/client.py

@@ -122,13 +122,17 @@ class TableauClient:
         credentials: TableauCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
         with_columns: bool = True,
+        with_fields: bool = True,
         with_pulse: bool = False,
         override_page_size: Optional[int] = None,
+        ignore_errors: bool = False,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
         self._with_columns = with_columns
+        self._with_fields = with_fields
         self._with_pulse = with_pulse
+        self._ignore_errors = ignore_errors
 
         self._client_metadata = TableauClientMetadataApi(
             server=self._server,
@@ -221,6 +225,10 @@ class TableauClient:
             logger.info(f"Skipping asset {asset} - deactivated columns")
             return []
 
+        if asset == TableauAsset.FIELD and not self._with_fields:
+            logger.info(f"Skipping asset {asset} - deactivated fields")
+            return []
+
         logger.info(f"Extracting {asset.name}...")
 
         if asset == TableauAsset.DATASOURCE:
@@ -240,4 +248,4 @@ class TableauClient:
             return self._client_rest.fetch(asset)
 
         # other assets can be extracted via Metadata API
-        return self._client_metadata.fetch(asset)
+        return self._client_metadata.fetch(asset, self._ignore_errors)

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/visualization/tableau/client/client_metadata_api.py

@@ -2,6 +2,7 @@ import logging
 from collections.abc import Iterator
 from typing import Optional
 
+import requests
 import tableauserverclient as TSC  # type: ignore
 
 from ....utils import SerializedAsset, retry
@@ -12,6 +13,13 @@ from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
 logger = logging.getLogger(__name__)
 
+# These assets are known to be error-prone, so it's acceptable if a few are missed.
+# If errors occur, skip the current batch.
+_SAFE_MODE_ASSETS = (
+    TableauAsset.COLUMN,
+    TableauAsset.FIELD,
+)
+
 # increase the value when extraction is too slow
 # decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: dict[TableauAsset, int] = {
@@ -92,6 +100,7 @@ def gql_query_scroll(
     resource: str,
     fields: str,
     page_size: int,
+    skip_batch: bool,
 ) -> Iterator[SerializedAsset]:
     """
     Iterate over GQL query results, handling pagination and cursor
@@ -119,15 +128,22 @@ def gql_query_scroll(
 
     current_offset = 0
     while True:
-        payload = _call(first=page_size, offset=current_offset)
-        yield payload["nodes"]
+        try:
+            payload = _call(first=page_size, offset=current_offset)
+            yield payload["nodes"]
+
+            current_offset += len(payload["nodes"])
+            total = payload["totalCount"]
+            logger.info(f"Extracted {current_offset}/{total} {resource}")
 
-        current_offset += len(payload["nodes"])
-        total = payload["totalCount"]
-        logger.info(f"Extracted {current_offset}/{total} {resource}")
+            if not payload["pageInfo"]["hasNextPage"]:
+                break
+        except requests.exceptions.ReadTimeout:
+            if not skip_batch:
+                raise
 
-        if not payload["pageInfo"]["hasNextPage"]:
-            break
+            logger.warning("Skipping batch because of TableauServer Timeout")
+            current_offset += page_size
 
 
 def _deduplicate(result_pages: Iterator[SerializedAsset]) -> SerializedAsset:
@@ -177,12 +193,14 @@ class TableauClientMetadataApi:
         resource: str,
         fields: str,
         page_size: int = DEFAULT_PAGE_SIZE,
+        skip_batch: bool = False,
     ) -> SerializedAsset:
         result_pages = gql_query_scroll(
             self._server,
             resource=resource,
             fields=fields,
             page_size=page_size,
+            skip_batch=skip_batch,
         )
         return _deduplicate(result_pages)
 
@@ -193,21 +211,41 @@ class TableauClientMetadataApi:
             or DEFAULT_PAGE_SIZE
         )
 
-    def _fetch_fields(self) -> SerializedAsset:
+    def _fetch_fields(self, skip_batch: bool = False) -> SerializedAsset:
         result: SerializedAsset = []
         page_size = self._page_size(TableauAsset.FIELD)
         for resource, fields in FIELDS_QUERIES:
-            current = self._call(resource, fields, page_size)
+            current = self._call(
+                resource,
+                fields,
+                page_size,
+                skip_batch=skip_batch,
+            )
             result.extend(current)
         return result
 
+    @staticmethod
+    def _should_skip_batch_with_timeout(
+        asset: TableauAsset,
+        ignore_metadata_errors: bool = False,
+    ) -> bool:
+        return asset in _SAFE_MODE_ASSETS and ignore_metadata_errors
+
     def fetch(
         self,
         asset: TableauAsset,
+        ignore_errors: bool = False,
     ) -> SerializedAsset:
+        skip_batch = self._should_skip_batch_with_timeout(asset, ignore_errors)
+
         if asset == TableauAsset.FIELD:
-            return self._fetch_fields()
+            return self._fetch_fields(skip_batch=skip_batch)
 
         page_size = self._page_size(asset)
         resource, fields = GQL_QUERIES[asset]
-        return self._call(resource, fields, page_size)
+        return self._call(
+            resource=resource,
+            fields=fields,
+            page_size=page_size,
+            skip_batch=skip_batch,
+        )

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/visualization/tableau/extract.py

@@ -33,16 +33,20 @@ def extract_all(**kwargs) -> None:
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     with_columns = not kwargs.get("skip_columns")
+    with_fields = not kwargs.get("skip_fields")
     with_pulse = kwargs.get("with_pulse") or False
     page_size = kwargs.get("page_size")
+    ignore_errors = kwargs.get("ignore_errors") or False
     timestamp = current_timestamp()
 
     credentials = TableauCredentials(**kwargs)
     client = TableauClient(
         credentials,
         with_columns=with_columns,
+        with_fields=with_fields,
         with_pulse=with_pulse,
         override_page_size=page_size,
+        ignore_errors=ignore_errors,
     )
     client.login()
 

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/warehouse/sqlserver/client.py

@@ -1,15 +1,20 @@
+import logging
 from collections.abc import Iterator
 
 from sqlalchemy import text
 
 from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode
 
+logger = logging.getLogger(__name__)
+
 SERVER_URI = "{user}:{password}@{host}:{port}/{database}"
 MSSQL_URI = f"mssql+pymssql://{SERVER_URI}"
 DEFAULT_PORT = 1433
 
 _KEYS = ("user", "password", "host", "port", "database")
 
+_SYSTEM_DATABASES = ("master", "model", "msdb", "tempdb", "DBAdmin")
+
 
 def _check_key(credentials: dict) -> None:
     for key in _KEYS:
@@ -51,3 +56,13 @@ class MSSQLClient(SqlalchemyClient):
             yield from results
         finally:
             self.close()
+
+    def get_databases(self) -> list[str]:
+        result = self.execute(
+            ExtractionQuery("SELECT name FROM sys.databases", {})
+        )
+        return [
+            row["name"]
+            for row in result
+            if row["name"] not in _SYSTEM_DATABASES
+        ]

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/warehouse/sqlserver/extract.py

@@ -1,6 +1,6 @@
 import logging
 
-from ...utils import LocalStorage, from_env, write_summary
+from ...utils import LocalStorage, filter_items, from_env, write_summary
 from ..abstract import (
     CATALOG_ASSETS,
     EXTERNAL_LINEAGE_ASSETS,
@@ -51,7 +51,13 @@ def extract_all(**kwargs) -> None:
 
     client = MSSQLClient(credentials=_credentials(kwargs))
 
-    query_builder = MSSQLQueryBuilder()
+    databases = filter_items(
+        client.get_databases(), kwargs.get("allowed"), kwargs.get("blocked")
+    )
+
+    query_builder = MSSQLQueryBuilder(
+        databases=databases,
+    )
 
     storage = LocalStorage(directory=output_directory)
 

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/warehouse/sqlserver/queries/column.sql

@@ -11,7 +11,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        sys.tables
+        {database}.sys.tables
 
     UNION
 
@@ -21,7 +21,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        sys.views
+        {database}.sys.views
 
     UNION
 
@@ -31,7 +31,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        sys.external_tables
+        {database}.sys.external_tables
 ),
 /*
 `sys.columns` contains, among others:
@@ -54,11 +54,11 @@ column_ids AS (
         schema_name = ss.name,
         schema_id = ss.schema_id,
         comment = CONVERT(varchar(1024), ep.value)
-    FROM sys.columns AS sc
+    FROM {database}.sys.columns AS sc
     LEFT JOIN extended_tables AS et ON sc.object_id = et.table_id
-    LEFT JOIN sys.schemas AS ss ON et.schema_id = ss.schema_id
-    LEFT JOIN sys.databases AS sd ON sd.name = DB_NAME()
-    LEFT JOIN sys.extended_properties AS ep
+    LEFT JOIN {database}.sys.schemas AS ss ON et.schema_id = ss.schema_id
+    LEFT JOIN {database}.sys.databases AS sd ON sd.name = '{database}'
+    LEFT JOIN {database}.sys.extended_properties AS ep
         ON
             sc.object_id = ep.major_id
             AND sc.column_id = ep.minor_id
@@ -70,9 +70,9 @@ columns AS (
         i.database_name,
         i.database_id,
         schema_name = c.table_schema,
-        i.schema_id,
+        schema_id = CAST(i.database_id AS VARCHAR(10)) + '_' + CAST(i.schema_id AS VARCHAR(10)),
         table_name = c.table_name,
-        i.table_id,
+        table_id = CAST(i.database_id AS VARCHAR(10)) + '_' + CAST(i.schema_id AS VARCHAR(10)) + '_' + CAST(i.table_id AS VARCHAR(10)),
         c.column_name,
         c.data_type,
         c.ordinal_position,
@@ -87,7 +87,7 @@ columns AS (
         i.comment,
         column_id = CONCAT(i.table_id, '.', c.column_name)
     FROM
-        information_schema.columns AS c
+        {database}.information_schema.columns AS c
     LEFT JOIN column_ids AS i
         ON
             (

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/warehouse/sqlserver/queries/database.sql

@@ -2,4 +2,4 @@ SELECT
     db.database_id,
     database_name = db.name
 FROM sys.databases AS db
-WHERE db.name = DB_NAME()
+WHERE db.name NOT IN ('master', 'model', 'msdb', 'tempdb', 'DBAdmin');

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -3,21 +3,20 @@ WITH ids AS (
     SELECT DISTINCT
         table_catalog,
         table_schema
-    FROM information_schema.tables
-    WHERE table_catalog = DB_NAME()
+    FROM {database}.information_schema.tables
 )
 
 SELECT
     d.database_id,
     database_name = i.table_catalog,
     schema_name = s.name,
-    s.schema_id,
+    schema_id = CAST(d.database_id AS VARCHAR(10)) + '_' + CAST(s.schema_id AS VARCHAR(10)),
     schema_owner = u.name,
     schema_owner_id = u.uid
-FROM sys.schemas AS s
+FROM {database}.sys.schemas AS s
 INNER JOIN ids AS i
     ON s.name = i.table_schema
-LEFT JOIN sys.sysusers AS u
+LEFT JOIN {database}.sys.sysusers AS u
     ON s.principal_id = u.uid
-LEFT JOIN sys.databases AS d
+LEFT JOIN {database}.sys.databases AS d
     ON i.table_catalog = d.name

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/castor_extractor/warehouse/sqlserver/queries/table.sql

@@ -11,7 +11,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        sys.tables
+        {database}.sys.tables
 
     UNION
 
@@ -21,7 +21,7 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        sys.views
+        {database}.sys.views
 
     UNION
 
@@ -31,14 +31,14 @@ WITH extended_tables AS (
         table_owner_id = principal_id,
         schema_id
     FROM
-        sys.external_tables
+        {database}.sys.external_tables
 ),
 -- Get the row count per table
 partitions AS (
     SELECT
         object_id,
         row_count = SUM(rows)
-    FROM sys.partitions
+    FROM {database}.sys.partitions
     GROUP BY object_id
 ),
 -- Append row count to table properties
@@ -69,13 +69,12 @@ table_ids AS (
         table_owner = u.name,
         row_count,
         comment = CONVERT(varchar(1024), ep.value)
-    FROM
-        extended_tables_with_row_count AS et
-    LEFT JOIN sys.schemas AS ss
+    FROM extended_tables_with_row_count AS et
+    LEFT JOIN {database}.sys.schemas AS ss
         ON et.schema_id = ss.schema_id
-    LEFT JOIN sys.sysusers AS u
+    LEFT JOIN {database}.sys.sysusers AS u
         ON et.table_owner_id = u.uid
-    LEFT JOIN sys.extended_properties AS ep
+    LEFT JOIN {database}.sys.extended_properties AS ep
         ON (
             et.table_id = ep.major_id
             AND ep.minor_id = 0
@@ -91,19 +90,18 @@ meta AS (
         t.table_name,
         t.table_type
     FROM
-        information_schema.tables AS t
-    LEFT JOIN sys.databases AS db
+        {database}.information_schema.tables AS t
+    LEFT JOIN {database}.sys.databases AS db
         ON t.table_catalog = db.name
-    WHERE t.table_catalog = db_name()
 )
 
 SELECT
     m.database_name,
     m.database_id,
     m.schema_name,
-    i.schema_id,
+    schema_id = CAST(m.database_id AS VARCHAR(10)) + '_' + CAST(i.schema_id AS VARCHAR(10)),
     m.table_name,
-    i.table_id,
+    table_id = CAST(m.database_id AS VARCHAR(10)) + '_' + CAST(i.schema_id AS VARCHAR(10)) + '_' + CAST(i.table_id AS VARCHAR(10)),
     m.table_type,
     i.table_owner,
     i.table_owner_id,

castor_extractor-0.24.34/castor_extractor/warehouse/sqlserver/query.py

@@ -0,0 +1,53 @@
+import logging
+from typing import Optional
+
+from ..abstract import (
+    AbstractQueryBuilder,
+    ExtractionQuery,
+    TimeFilter,
+    WarehouseAsset,
+)
+
+logger = logging.getLogger(__name__)
+
+
+_DATABASE_REQUIRED = (
+    WarehouseAsset.SCHEMA,
+    WarehouseAsset.TABLE,
+    WarehouseAsset.COLUMN,
+)
+
+
+class MSSQLQueryBuilder(AbstractQueryBuilder):
+    """
+    Builds queries to extract assets from SQL Server.
+    """
+
+    def __init__(
+        self,
+        databases: list[str],
+        time_filter: Optional[TimeFilter] = None,
+    ):
+        super().__init__(time_filter=time_filter)
+        self._databases = databases
+
+    @staticmethod
+    def _format(query: ExtractionQuery, values: dict) -> ExtractionQuery:
+        return ExtractionQuery(
+            statement=query.statement.format(**values),
+            params=query.params,
+        )
+
+    def build(self, asset: WarehouseAsset) -> list[ExtractionQuery]:
+        query = self.build_default(asset)
+
+        if asset not in _DATABASE_REQUIRED:
+            return [query]
+
+        logger.info(
+            f"\tWill run queries with following database params: {self._databases}",
+        )
+        return [
+            self._format(query, {"database": database})
+            for database in self._databases
+        ]

{castor_extractor-0.24.32 → castor_extractor-0.24.34}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.32"
+version = "0.24.34"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"