castor-extractor 0.25.14__tar.gz → 0.26.9__tar.gz

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/CHANGELOG.md

@@ -1,5 +1,106 @@
 # Changelog
 
+## 0.26.9 - 2026-01-21
+
+* SqlServer: support default database
+
+## 0.26.8 - 2026-01-14
+
+* Qlik: ignore OBJECT_NOT_FOUND errors
+
+## 0.26.7 - 2026-01-13
+
+* Coalesce: introduce dynamic pagination
+
+## 0.26.6 - 2025-12-30
+
+* Snowflake: add setting `apply_db_filter_to_queries` (for future use)
+
+## 0.26.5 - 2025-12-23
+
+* Looker: extract dashboard's slug
+
+## 0.26.4 - 2025-12-18
+
+* Snowflake: extend query_blocked to column-lineage extraction
+
+## 0.26.3 - 2025-12-17
+
+* Tableau: catch error relationship-service-war
+
+## 0.26.2 - 2025-12-16
+
+* Notion: deduplicate Users
+
+## 0.26.1 - 2025-12-15
+
+* Power BI: pages extraction conditioned on workspace access
+
+## 0.26.0 - 2025-12-09
+
+* Removing support for python3.9
+
+## 0.25.29 - 2025-12-11
+
+* Tableau: dynamic pagination and safe mode for all types of assets
+
+## 0.25.28 - 2025-12-08
+
+* Tableau: fix retry exponential time
+
+## 0.25.27 - 2025-12-04
+
+* Sigma: retry on 503 errors
+
+## 0.25.26 - 2025-12-03
+
+* Strategy: deduplicate users on ID
+
+## 0.25.25 - 2025-12-02
+
+* Tableau: better catching of timeouts
+
+## 0.25.24 - 2025-11-27
+
+* Tableau: extraction of tables and columns without "Extracts"
+
+## 0.25.23 - 2025-11-24
+
+* Looker Studio: add option to extract source queries only
+
+## 0.25.22 - 2025-11-19
+
+* SQL Server: improved error log
+* Sigma : handle service errors in queries and elements extraction
+
+## 0.25.21 - 2025-11-05
+
+* Qlik: Extracting sheets
+
+## 0.25.20 - 2025-11-14
+
+* BigQuery: skip Omni locations when extracting SQL queries
+
+## 0.25.19 - 2025-11-14
+
+* Redshift: support extraction of queries longer than 65,535 characters
+
+## 0.25.18 - 2025-11-13
+
+* Power BI: expand groups into individual users
+
+## 0.25.17 - 2025-11-04
+
+* Snowflake: add query_block to exclude specific sql patterns
+
+## 0.25.16 - 2025-11-03
+
+* Snowflake: remove duplicated queries, based on query_hash
+
+## 0.25.15 - 2025-10-30
+
+* Databricks: Add `http_path` mandatory field for databricks credentials
+
 ## 0.25.14 - 2025-10-28
 
 * Removing unused dependency (google-cloud-storage)

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/PKG-INFO

@@ -1,19 +1,19 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.25.14
+Version: 0.26.9
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
 Author: Castor
 Author-email: support@castordoc.com
-Requires-Python: >=3.9,<3.13
+Requires-Python: >=3.10,<3.14
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: all
 Provides-Extra: bigquery
 Provides-Extra: count
@@ -61,7 +61,6 @@ Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigq
 Requires-Dist: sqlalchemy-redshift (>=0.8.14,<0.9.0) ; extra == "redshift" or extra == "all"
 Requires-Dist: tableauserverclient (>=0.25.0,<0.26.0) ; extra == "tableau" or extra == "all"
 Requires-Dist: tqdm (>=4.0.0,<5.0.0)
-Requires-Dist: typing-extensions (>=4,<5)
 Requires-Dist: websocket-client (>=1,<2) ; extra == "qlik" or extra == "all"
 Project-URL: Documentation, https://docs.castordoc.com/castor-package/castor-extractor
 Description-Content-Type: text/markdown
@@ -102,11 +101,9 @@ It also embeds utilities to help you push your metadata to Castor:
 
 ## Installation
 
-Requirements: **python3.9+**
+Requirements: **python3.10+**
 <img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
 
-**Note:** The Strategy command requires **python3.10+**. All other modules work with python3.9+.
-
 ### Create castor-env
 
 We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
@@ -128,7 +125,7 @@ eval "$(pyenv init --path)"
 eval "$(pyenv virtualenv-init -)"
 ```
 
-- [optional] Install python 3.9+ if needed
+- [optional] Install python 3.10+ if needed
 
 ```bash
 pyenv versions # check your local python installations
@@ -215,6 +212,107 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.26.9 - 2026-01-21
+
+* SqlServer: support default database
+
+## 0.26.8 - 2026-01-14
+
+* Qlik: ignore OBJECT_NOT_FOUND errors
+
+## 0.26.7 - 2026-01-13
+
+* Coalesce: introduce dynamic pagination
+
+## 0.26.6 - 2025-12-30
+
+* Snowflake: add setting `apply_db_filter_to_queries` (for future use)
+
+## 0.26.5 - 2025-12-23
+
+* Looker: extract dashboard's slug
+
+## 0.26.4 - 2025-12-18
+
+* Snowflake: extend query_blocked to column-lineage extraction
+
+## 0.26.3 - 2025-12-17
+
+* Tableau: catch error relationship-service-war
+
+## 0.26.2 - 2025-12-16
+
+* Notion: deduplicate Users
+
+## 0.26.1 - 2025-12-15
+
+* Power BI: pages extraction conditioned on workspace access
+
+## 0.26.0 - 2025-12-09
+
+* Removing support for python3.9
+
+## 0.25.29 - 2025-12-11
+
+* Tableau: dynamic pagination and safe mode for all types of assets
+
+## 0.25.28 - 2025-12-08
+
+* Tableau: fix retry exponential time
+
+## 0.25.27 - 2025-12-04
+
+* Sigma: retry on 503 errors
+
+## 0.25.26 - 2025-12-03
+
+* Strategy: deduplicate users on ID
+
+## 0.25.25 - 2025-12-02
+
+* Tableau: better catching of timeouts
+
+## 0.25.24 - 2025-11-27
+
+* Tableau: extraction of tables and columns without "Extracts"
+
+## 0.25.23 - 2025-11-24
+
+* Looker Studio: add option to extract source queries only
+
+## 0.25.22 - 2025-11-19
+
+* SQL Server: improved error log
+* Sigma : handle service errors in queries and elements extraction
+
+## 0.25.21 - 2025-11-05
+
+* Qlik: Extracting sheets
+
+## 0.25.20 - 2025-11-14
+
+* BigQuery: skip Omni locations when extracting SQL queries
+
+## 0.25.19 - 2025-11-14
+
+* Redshift: support extraction of queries longer than 65,535 characters
+
+## 0.25.18 - 2025-11-13
+
+* Power BI: expand groups into individual users
+
+## 0.25.17 - 2025-11-04
+
+* Snowflake: add query_block to exclude specific sql patterns
+
+## 0.25.16 - 2025-11-03
+
+* Snowflake: remove duplicated queries, based on query_hash
+
+## 0.25.15 - 2025-10-30
+
+* Databricks: Add `http_path` mandatory field for databricks credentials
+
 ## 0.25.14 - 2025-10-28
 
 * Removing unused dependency (google-cloud-storage)

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/README.md

@@ -34,11 +34,9 @@ It also embeds utilities to help you push your metadata to Castor:
 
 ## Installation
 
-Requirements: **python3.9+**
+Requirements: **python3.10+**
 <img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
 
-**Note:** The Strategy command requires **python3.10+**. All other modules work with python3.9+.
-
 ### Create castor-env
 
 We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
@@ -60,7 +58,7 @@ eval "$(pyenv init --path)"
 eval "$(pyenv virtualenv-init -)"
 ```
 
-- [optional] Install python 3.9+ if needed
+- [optional] Install python 3.10+ if needed
 
 ```bash
 pyenv versions # check your local python installations

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/commands/extract_databricks.py

@@ -21,6 +21,9 @@ def main():
         help="List of databricks catalogs that should not be extracted",
     )
     parser.add_argument("-t", "--token", help="Databricks access token")
+    parser.add_argument(
+        "-p", "--http-path", dest="http_path", help="Databricks http path"
+    )
     parser.add_argument("-o", "--output", help="Directory to write to")
     parser.add_argument(
         "--skip-existing",
@@ -38,5 +41,6 @@ def main():
         host=args.host,
         output_directory=args.output,
         token=args.token,
+        http_path=args.http_path,
         skip_existing=args.skip_existing,
     )

castor_extractor-0.26.9/castor_extractor/commands/extract_looker_studio.py

@@ -0,0 +1,67 @@
+from argparse import ArgumentParser
+
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import looker_studio  # type: ignore
+
+
+def main():
+    parser = ArgumentParser(
+        description="Extract data from Looker Studio and related BigQuery source queries"
+    )
+    parser.add_argument("-o", "--output", help="Directory to write to")
+
+    mode_group = parser.add_argument_group("Execution mode")
+    mode_group.add_argument(
+        "--source-queries-only",
+        action="store_true",
+        default=False,
+        help="If selected, only extracts BigQuery source queries (bypasses Looker Studio extraction)",
+    )
+    mode_group.add_argument(
+        "--skip-view-activity-logs",
+        action="store_true",
+        default=False,
+        help="Skip extraction of activity logs (use if credentials lack required scopes)",
+    )
+
+    looker_group = parser.add_argument_group("Looker Studio extraction")
+    looker_group.add_argument(
+        "-c",
+        "--credentials",
+        help="File path to Service Account credentials with Looker Studio access",
+    )
+    looker_group.add_argument(
+        "-a",
+        "--admin-email",
+        help="Email of a Google Workspace user with admin access",
+    )
+    looker_group.add_argument(
+        "--users-file-path",
+        help=(
+            "Optional path to a JSON file with user email addresses "
+            'as a list of strings (e.g. ["foo@bar.com", "fee@bar.com"]). '
+            "If provided, only extracts assets owned by the specified users."
+        ),
+    )
+
+    bigquery_group = parser.add_argument_group("Source queries extraction")
+    bigquery_group.add_argument(
+        "-b",
+        "--bigquery-credentials",
+        help=(
+            "File path to Service Account credentials with BigQuery access. "
+            "Can point to the same file as Looker Studio credentials."
+        ),
+    )
+    bigquery_group.add_argument(
+        "--db-allowed",
+        nargs="*",
+        help="Optional list of GCP projects to allow for source queries extraction",
+    )
+    bigquery_group.add_argument(
+        "--db-blocked",
+        nargs="*",
+        help="Optional list of GCP projects to block from source queries extraction",
+    )
+
+    looker_studio.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/commands/extract_powerbi.py

@@ -31,11 +31,16 @@ def main():
     parser.add_argument(
         "-sc",
         "--scopes",
-        help="PowerBi scopes, optional",
+        help="Optional scopes for the Power BI REST API and the Microsoft Graph API",
         nargs="*",
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
     parser.add_argument("-l", "--login_url", help="Login url (Optional)")
-    parser.add_argument("-a", "--api_base", help="REST API base (Optional)")
+    parser.add_argument(
+        "-a", "--api_base", help="Power BI REST API base (Optional)"
+    )
+    parser.add_argument(
+        "-g", "--graph_api_base", help="Microsoft Graph API base (Optional)"
+    )
 
     powerbi.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/commands/extract_qlik.py

@@ -22,5 +22,11 @@ def main():
         "as warning instead. Helpful to continue script execution when "
         "missing rights on some assets.",
     )
+    parser.add_argument(
+        "-s",
+        "--include-sheets",
+        action="store_true",
+        help="Also extract sheets",
+    )
 
     qlik.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/commands/extract_snowflake.py

@@ -35,6 +35,11 @@ def main():
         nargs="*",
         help="List of databases that should not be extracted",
     )
+    parser.add_argument(
+        "--query-blocked",
+        nargs="*",
+        help="List of query patterns that should not be extracted. SQL wildcards are supported: % and _",
+    )
     parser.add_argument(
         "--fetch-transient",
         action="store_true",
@@ -67,6 +72,7 @@ def main():
         role=args.role,
         db_allowed=args.db_allowed,
         db_blocked=args.db_blocked,
+        query_blocked=args.query_blocked,
         output_directory=args.output,
         skip_existing=args.skip_existing,
         fetch_transient=args.fetch_transient,

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/commands/extract_sqlserver.py

@@ -40,6 +40,10 @@ def main():
         nargs="*",
         help="List of databases that should not be extracted",
     )
+    parser.add_argument(
+        "--default-db",
+        help="Optional database used only when the SQL Server login cannot connect via its default database (e.g. Azure SQL). Not required for most setups.",
+    )
     parser.set_defaults(skip_existing=False)
 
     args = parser.parse_args()
@@ -51,6 +55,8 @@ def main():
         password=args.password,
         output_directory=args.output,
         skip_existing=args.skip_existing,
+        skip_queries=args.skip_queries,
         db_allowed=args.db_allowed,
         db_blocked=args.db_blocked,
+        default_db=args.default_db,
     )

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/commands/extract_tableau.py

@@ -4,6 +4,8 @@ from argparse import ArgumentParser
 from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import tableau  # type: ignore
 
+logger = logging.getLogger(__name__)
+
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
 
@@ -53,8 +55,14 @@ def main():
         "--ignore-errors",
         action="store_true",
         dest="ignore_errors",
-        help="Allow partial extraction of Fields and Columns: skip batch in case of Timeout errors",
+        help="(DEPRECATED) This option no longer has any effect and will be removed in a future version.",
     )
+    args = parser.parse_args()
+    if args.ignore_errors:
+        logger.warning(
+            "--ignore-errors is deprecated and has no more effect. It will be removed in a future version."
+        )
+
     parser.add_argument(
         "--ignore-ssl",
         action="store_true",

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/file_checker/column.py

@@ -1,4 +1,4 @@
-from typing import Callable, Optional
+from collections.abc import Callable
 
 from dateutil.parser import parse
 
@@ -29,8 +29,8 @@ class ColumnChecker:
         data_type: DataType = DataType.STRING,
         is_mandatory: bool = True,
         is_unique: bool = False,
-        foreign: Optional[set[str]] = None,
-        enum_values: Optional[set[str]] = None,
+        foreign: set[str] | None = None,
+        enum_values: set[str] | None = None,
     ):
         self.data_type = data_type
         self.is_mandatory = is_mandatory
@@ -79,7 +79,7 @@ class ColumnChecker:
             return True
         return value in self.foreign
 
-    def check(self, value: Optional[str]) -> Optional[Issue]:
+    def check(self, value: str | None) -> Issue | None:
         """
         Do all checks to validate that the given value respect the column rules.
         Return an issue if something's wrong.

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/knowledge/confluence/client/client.py

@@ -1,7 +1,6 @@
 import logging
-from collections.abc import Iterator
+from collections.abc import Iterable, Iterator
 from functools import partial
-from typing import Iterable, Optional
 
 from ....utils import (
     APIClient,
@@ -31,8 +30,8 @@ class ConfluenceClient(APIClient):
         credentials: ConfluenceCredentials,
         include_archived_spaces: bool = False,
         include_personal_spaces: bool = False,
-        space_ids_allowed: Optional[set[str]] = None,
-        space_ids_blocked: Optional[set[str]] = None,
+        space_ids_allowed: set[str] | None = None,
+        space_ids_blocked: set[str] | None = None,
     ):
         self.account_id = credentials.account_id
         auth = BasicAuth(
@@ -150,8 +149,8 @@ class ConfluenceClient(APIClient):
         self,
         asset: ConfluenceAsset,
         *,
-        folder_ids: Optional[Iterator[str]] = None,
-        database_ids: Optional[Iterator[str]] = None,
+        folder_ids: Iterator[str] | None = None,
+        database_ids: Iterator[str] | None = None,
     ) -> Iterator[dict]:
         """Returns the needed metadata for the queried asset"""
         if asset == ConfluenceAsset.FOLDERS:

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/knowledge/confluence/client/pagination.py

@@ -1,5 +1,3 @@
-from typing import Optional, Union
-
 from pydantic import AliasPath, Field
 
 from ....utils import (
@@ -14,7 +12,7 @@ class ConfluencePagination(PaginationModel):
     fetch_by: FetchNextPageBy = FetchNextPageBy.URL
 
     results: list = Field(default_factory=list)
-    next_url: Optional[str] = Field(
+    next_url: str | None = Field(
         validation_alias=AliasPath("_links", "next"),
         default=None,
     )
@@ -23,7 +21,7 @@ class ConfluencePagination(PaginationModel):
         """Stopping condition for the pagination"""
         return self.next_url is None
 
-    def next_page_payload(self) -> Union[str, None]:
+    def next_page_payload(self) -> str | None:
         """Payload enabling to generate the request for the next page"""
         return self.next_url
 

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/knowledge/confluence/extract.py

@@ -1,6 +1,5 @@
 import logging
 from collections.abc import Iterable, Iterator
-from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -20,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: ConfluenceClient,
-) -> Iterable[tuple[ConfluenceAsset, Union[list, Iterator, dict]]]:
+) -> Iterable[tuple[ConfluenceAsset, list | Iterator | dict]]:
     """Iterate over the extracted data from Confluence"""
 
     logger.info("Extracting USERS from API")

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/knowledge/notion/client/client.py

@@ -1,9 +1,14 @@
 from collections.abc import Iterator
 from functools import partial
 from http import HTTPStatus
-from typing import Optional
 
-from ....utils import APIClient, BearerAuth, RequestSafeMode, fetch_all_pages
+from ....utils import (
+    APIClient,
+    BearerAuth,
+    RequestSafeMode,
+    deduplicate,
+    fetch_all_pages,
+)
 from ..assets import NotionAsset
 from .constants import CASTOR_NOTION_USER_AGENT, NOTION_BASE_URL, NOTION_VERSION
 from .credentials import NotionCredentials
@@ -40,7 +45,7 @@ class NotionClient(APIClient):
     def __init__(
         self,
         credentials: NotionCredentials,
-        safe_mode: Optional[RequestSafeMode] = None,
+        safe_mode: RequestSafeMode | None = None,
     ):
         auth = NotionAuth(token=credentials.token)
         super().__init__(
@@ -52,8 +57,12 @@ class NotionClient(APIClient):
         )
 
     def users(self) -> Iterator[dict]:
+        """
+        Yield all Notion users, deduplicated by user ID.
+        """
         request = partial(self._get, endpoint=NotionEndpointFactory.users())
-        yield from fetch_all_pages(request, NotionPagination)
+        all_users = fetch_all_pages(request, NotionPagination)
+        return deduplicate("id", all_users)
 
     def _page_listing(self) -> Iterator[dict]:
         request = partial(

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/knowledge/notion/client/pagination.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 
 from ....utils import PaginationModel
@@ -9,7 +7,7 @@ class NotionPagination(PaginationModel):
     """Class to handle paginated results"""
 
     results: list = Field(default_factory=list)
-    next_cursor: Optional[str] = None
+    next_cursor: str | None = None
     has_more: bool = False
 
     def is_last(self) -> bool:

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/knowledge/notion/extract.py

@@ -1,6 +1,5 @@
 import logging
 from collections.abc import Iterable, Iterator
-from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -19,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: NotionClient,
-) -> Iterable[tuple[NotionAsset, Union[list, Iterator, dict]]]:
+) -> Iterable[tuple[NotionAsset, list | Iterator | dict]]:
     """Iterate over the extracted data from Notion"""
 
     logger.info("Extracting USERS from API")

{castor_extractor-0.25.14 → castor_extractor-0.26.9}/castor_extractor/quality/soda/client/client.py

@@ -1,6 +1,5 @@
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from functools import partial
-from typing import Callable
 
 from ....utils import (
     APIClient,