castor-extractor 0.24.10__tar.gz → 0.24.15__tar.gz

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
+## 0.24.13 - 2025-05-05
+
+* Rollback cloud-storage version as it's not compatible with Keboola
+
+## 0.24.12 - 2025-05-05
+
+* Redshift - fix query definition of materialized views
+
+## 0.24.11 - 2025-05-05
+
+* add support for Strategy (formerly MicroStrategy)
+
 ## 0.24.10 - 2025-04-30
 
 * Tableau - skip warnings instead of raising an error

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.10
+Version: 0.24.15
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -28,6 +28,7 @@ Provides-Extra: qlik
 Provides-Extra: redshift
 Provides-Extra: snowflake
 Provides-Extra: sqlserver
+Provides-Extra: strategy
 Provides-Extra: tableau
 Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
 Requires-Dist: databricks-sql-connector (==3.6.0) ; extra == "databricks" or extra == "all"
@@ -35,11 +36,12 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
+Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
+Requires-Dist: mstrio-py (>=11.5.3.101,<12.0.0.0) ; (python_version >= "3.10") and (extra == "strategy" or extra == "all")
 Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
 Requires-Dist: numpy (>=1.26) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "bigquery" or extra == "databricks" or extra == "all")
 Requires-Dist: pandas (>=2.1) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "databricks" or extra == "all")
@@ -103,6 +105,8 @@ It also embeds utilities to help you push your metadata to Castor:
 Requirements: **python3.9+**
 <img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
 
+**Note:** The Strategy command requires **python3.10+**. All other modules work with python3.9+.
+
 ### Create castor-env
 
 We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
@@ -181,6 +185,7 @@ pip install castor-extractor[postgres]
 pip install castor-extractor[redshift]
 pip install castor-extractor[snowflake]
 pip install castor-extractor[sqlserver]
+pip install castor-extractor[strategy]
 pip install castor-extractor[tableau]
 ```
 
@@ -204,12 +209,32 @@ export CASTOR_OUTPUT_DIRECTORY="/tmp/castor"
 
 ## Contact
 
-For any questions or bug report, contact us at [support@castordoc.com](mailto:support@castordoc.com)
+For any questions or bug report, contact us at [support@coalesce.io](mailto:support@coalesce.io)
 
-[Castor](https://castordoc.com) helps you find, understand, use your data assets
+[Catalog from Coalesce](https://castordoc.com) helps you find, understand, use your data assets
 
 # Changelog
 
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
+## 0.24.13 - 2025-05-05
+
+* Rollback cloud-storage version as it's not compatible with Keboola
+
+## 0.24.12 - 2025-05-05
+
+* Redshift - fix query definition of materialized views
+
+## 0.24.11 - 2025-05-05
+
+* add support for Strategy (formerly MicroStrategy)
+
 ## 0.24.10 - 2025-04-30
 
 * Tableau - skip warnings instead of raising an error

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/README.md

@@ -37,6 +37,8 @@ It also embeds utilities to help you push your metadata to Castor:
 Requirements: **python3.9+**
 <img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
 
+**Note:** The Strategy command requires **python3.10+**. All other modules work with python3.9+.
+
 ### Create castor-env
 
 We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
@@ -115,6 +117,7 @@ pip install castor-extractor[postgres]
 pip install castor-extractor[redshift]
 pip install castor-extractor[snowflake]
 pip install castor-extractor[sqlserver]
+pip install castor-extractor[strategy]
 pip install castor-extractor[tableau]
 ```
 
@@ -138,6 +141,6 @@ export CASTOR_OUTPUT_DIRECTORY="/tmp/castor"
 
 ## Contact
 
-For any questions or bug report, contact us at [support@castordoc.com](mailto:support@castordoc.com)
+For any questions or bug report, contact us at [support@coalesce.io](mailto:support@coalesce.io)
 
-[Castor](https://castordoc.com) helps you find, understand, use your data assets
+[Catalog from Coalesce](https://castordoc.com) helps you find, understand, use your data assets

castor_extractor-0.24.15/castor_extractor/commands/extract_confluence.py

@@ -0,0 +1,48 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.knowledge import confluence  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-a", "--account_id", help="Confluence account id")
+    parser.add_argument("-b", "--base_url", help="Confluence account base url")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-t", "--token", help="Confluence API token")
+    parser.add_argument("-u", "--username", help="Confluence username")
+
+    parser.add_argument(
+        "--include-archived-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from archived spaces (Optional)",
+    )
+    parser.add_argument(
+        "--include-personal-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from personal spaces (Optional)",
+    )
+    parser.add_argument(
+        "--space-ids-allowed",
+        type=str,
+        nargs="+",
+        help=(
+            "List of Confluence space IDs allowed for extraction (Optional). "
+            "Only pages from these Spaces will be extracted. "
+            "This overrides any other filtering (archived, personal, etc.)"
+        ),
+    )
+    parser.add_argument(
+        "--space-ids-blocked",
+        type=str,
+        nargs="+",
+        help="List of Confluence space IDs to exclude fom the extraction (Optional)",
+    )
+
+    confluence.extract_all(**parse_filled_arguments(parser))

castor_extractor-0.24.15/castor_extractor/commands/extract_strategy.py

@@ -0,0 +1,26 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import strategy  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-u", "--username", help="Strategy username")
+    parser.add_argument("-p", "--password", help="Strategy password")
+    parser.add_argument("-b", "--base-url", help="Strategy instance URL")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+
+    parser.add_argument(
+        "-i",
+        "--project-ids",
+        nargs="*",
+        help="Optional list of project IDs",
+        default=None,
+    )
+
+    strategy.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/castor_extractor/commands/extract_tableau.py

@@ -21,6 +21,13 @@ def main():
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
 
+    parser.add_argument(
+        "--skip-columns",
+        dest="skip_columns",
+        action="store_true",
+        help="Option to avoid extracting Tableau columns, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",

castor_extractor-0.24.15/castor_extractor/knowledge/confluence/client/client.py

@@ -0,0 +1,118 @@
+from collections.abc import Iterator
+from functools import partial
+from typing import Optional
+
+from ....utils import (
+    APIClient,
+    BasicAuth,
+    fetch_all_pages,
+)
+from ..assets import (
+    ConfluenceAsset,
+)
+from .credentials import ConfluenceCredentials
+from .endpoints import ConfluenceEndpointFactory
+from .pagination import ConfluencePagination
+
+_HEADERS = {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+}
+_STATUS_ARCHIVED = "archived"
+_TYPE_PERSONAL = "personal"
+
+
+class ConfluenceClient(APIClient):
+    def __init__(
+        self,
+        credentials: ConfluenceCredentials,
+        include_archived_spaces: bool = False,
+        include_personal_spaces: bool = False,
+        space_ids_allowed: Optional[set[str]] = None,
+        space_ids_blocked: Optional[set[str]] = None,
+    ):
+        self.account_id = credentials.account_id
+        auth = BasicAuth(
+            username=credentials.username, password=credentials.token
+        )
+        super().__init__(
+            auth=auth,
+            host=credentials.base_url,
+            headers=_HEADERS,
+        )
+
+        self.include_archived_spaces = include_archived_spaces
+        self.include_personal_spaces = include_personal_spaces
+        self.space_ids_allowed = space_ids_allowed or set()
+        self.space_ids_blocked = space_ids_blocked or set()
+
+    def pages(self):
+        """Extracts all pages from all relevant Spaces."""
+        for space in self.spaces():
+            space_id = space["id"]
+            request = partial(
+                self._get,
+                endpoint=ConfluenceEndpointFactory.pages(space_id),
+            )
+            yield from fetch_all_pages(request, ConfluencePagination)
+
+    def spaces(self) -> Iterator[dict]:
+        """
+        Returns the spaces meeting the conditions defined by the settings.
+
+        If `space_ids_allowed` is not empty, only matching spaces are returned.
+
+        Otherwise, all spaces are filtered by excluding the following:
+          * The space is in the blocked list
+          * The space is personal (type=personal) and skip_personal_spaces is True
+          * The space is archived (status=archived) and skip_archived_spaces is True
+        """
+        request = partial(
+            self._get,
+            endpoint=ConfluenceEndpointFactory.spaces(),
+        )
+        spaces = list(fetch_all_pages(request, ConfluencePagination))
+
+        if self.space_ids_allowed:
+            yield from (
+                space
+                for space in spaces
+                if space["id"] in self.space_ids_allowed
+            )
+            return
+
+        for space in spaces:
+            space_id = space["id"]
+            type_ = space["type"]
+            status = space["status"]
+
+            if space_id in self.space_ids_blocked:
+                continue
+
+            if status == _STATUS_ARCHIVED and not self.include_archived_spaces:
+                continue
+
+            if type_ == _TYPE_PERSONAL and not self.include_personal_spaces:
+                continue
+
+            yield space
+
+    def users(self):
+        request_body = {"accountIds": [self.account_id]}
+        request = partial(
+            self._post,
+            endpoint=ConfluenceEndpointFactory.users(),
+            data=request_body,
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)
+
+    def fetch(self, asset: ConfluenceAsset) -> Iterator[dict]:
+        """Returns the needed metadata for the queried asset"""
+        if asset == ConfluenceAsset.PAGES:
+            yield from self.pages()
+
+        elif asset == ConfluenceAsset.USERS:
+            yield from self.users()
+
+        else:
+            raise ValueError(f"This asset {asset} is unknown")

castor_extractor-0.24.15/castor_extractor/knowledge/confluence/client/client_test.py

@@ -0,0 +1,91 @@
+from unittest.mock import MagicMock, patch
+
+from .client import ConfluenceClient
+
+
+def test_ConfluenceClient_filtered_spaces_with_allowlist():
+    both_blocked_and_allowed_space_id = "789"
+    archived_space_id = "934"
+    random_space = "1000"
+
+    spaces = [
+        # Both blocked and allowed space. "Allowed" setting takes precedence.
+        {
+            "id": both_blocked_and_allowed_space_id,
+            "type": "global",
+            "status": "current",
+        },
+        {"id": archived_space_id, "type": "global", "status": "archived"},
+        {"id": random_space, "type": "global", "status": "current"},
+    ]
+    # the "allowed" list overrides everything else
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=True,
+        space_ids_allowed={
+            both_blocked_and_allowed_space_id,
+        },
+        space_ids_blocked={
+            both_blocked_and_allowed_space_id,
+        },
+    )
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        assert len(filtered_spaces) == 1
+        filtered_space_ids = {space["id"] for space in filtered_spaces}
+        assert set(filtered_space_ids) == {both_blocked_and_allowed_space_id}
+
+
+def test_ConfluenceClient_filtered_spaces():
+    blocked_id = "42"
+    personal_id = "666"
+    archived_id = "934"
+    random_id = "1000"
+
+    # test the other settings : allow personal spaces & block space "42"
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=False,
+        include_personal_spaces=True,
+        space_ids_blocked={blocked_id},
+    )
+
+    spaces = [
+        # Blocked space, to be skipped
+        {"id": blocked_id, "type": "global", "status": "current"},
+        # Archived space, to be skipped
+        {"id": archived_id, "type": "collaboration", "status": "archived"},
+        # Personal space, to be included
+        {"id": personal_id, "type": "personal", "status": "current"},
+        # Valid space
+        {"id": random_id, "type": "knowledge_base", "status": "current"},
+    ]
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        filtered_space_ids = [space["id"] for space in filtered_spaces]
+
+        # no duplicates
+        assert len(filtered_space_ids) == len(set(filtered_space_ids))
+        assert set(filtered_space_ids) == {personal_id, random_id}

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/castor_extractor/knowledge/confluence/client/endpoints.py

@@ -6,15 +6,24 @@ class ConfluenceEndpointFactory:
 
     API = "wiki/api/v2/"
     PAGES = "pages"
+    SPACES = "spaces"
     USERS = "users-bulk"
 
     @classmethod
-    def pages(cls) -> str:
+    def pages(cls, space_id: str) -> str:
         """
-        Endpoint to fetch all pages.
-        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
+        Endpoint to fetch all pages in the given space.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-spaces-id-pages-get
         """
-        return f"{cls.API}{cls.PAGES}?body-format=atlas_doc_format"
+        return f"{cls.API}{cls.SPACES}/{space_id}/{cls.PAGES}?body-format=atlas_doc_format"
+
+    @classmethod
+    def spaces(cls) -> str:
+        """
+        Endpoint to fetch all spaces.
+        https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-space/#api-spaces-get
+        """
+        return f"{cls.API}{cls.SPACES}"
 
     @classmethod
     def users(cls) -> str:

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/castor_extractor/knowledge/confluence/extract.py

@@ -42,7 +42,13 @@ def extract_all(**kwargs) -> None:
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     credentials = ConfluenceCredentials(**kwargs)
-    client = ConfluenceClient(credentials=credentials)
+    client = ConfluenceClient(
+        credentials=credentials,
+        include_archived_spaces=kwargs.get("include_archived_spaces") or False,
+        include_personal_spaces=kwargs.get("include_personal_spaces") or False,
+        space_ids_allowed=kwargs.get("space_ids_allowed"),
+        space_ids_blocked=kwargs.get("space_ids_blocked"),
+    )
 
     ts = current_timestamp()
 

{castor_extractor-0.24.10 → castor_extractor-0.24.15}/castor_extractor/transformation/coalesce/client/client.py

@@ -1,7 +1,15 @@
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-from ....utils import APIClient, BearerAuth, RequestSafeMode, SerializedAsset
+import requests
+
+from ....utils import (
+    APIClient,
+    BearerAuth,
+    RequestSafeMode,
+    SerializedAsset,
+    retry,
+)
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
 from .endpoint import (
@@ -12,6 +20,9 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
+_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_RETRY_COUNT = 2
+_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -64,6 +75,11 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
+    @retry(
+        exceptions=_RETRY_EXCEPTIONS,
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id

castor_extractor-0.24.15/castor_extractor/visualization/strategy/__init__.py

@@ -0,0 +1,3 @@
+from .assets import StrategyAsset
+from .client import StrategyClient, StrategyCredentials
+from .extract import extract_all

castor_extractor-0.24.15/castor_extractor/visualization/strategy/assets.py

@@ -0,0 +1,14 @@
+from ...types import ExternalAsset
+
+
+class StrategyAsset(ExternalAsset):
+    """Strategy assets that can be extracted"""
+
+    ATTRIBUTE = "attribute"
+    CUBE = "cube"
+    DASHBOARD = "dashboard"
+    DOCUMENT = "document"
+    FACT = "fact"
+    METRIC = "metric"
+    REPORT = "report"
+    USER = "user"

castor_extractor-0.24.15/castor_extractor/visualization/strategy/client/__init__.py

@@ -0,0 +1,2 @@
+from .client import StrategyClient
+from .credentials import StrategyCredentials