castor-extractor 0.24.13__tar.gz → 0.24.20__tar.gz

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 0.24.20 - 2025-05-19
+
+* Powerbi: allow custom api base and login url
+
+## 0.24.19 - 2025-05-14
+
+* Confluence: extract databases
+
+## 0.24.18 - 2025-05-13
+
+* Improve folder organisation for transformation tools
+
+## 0.24.17 - 2025-05-13
+
+* Strategy: fix dashboard URL format
+
+## 0.24.16 - 2025-05-12
+
+* Confluence: extract folders to complete the page hierarchy
+
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
 ## 0.24.13 - 2025-05-05
 
 * Rollback cloud-storage version as it's not compatible with Keboola

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.13
+Version: 0.24.20
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,34 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.20 - 2025-05-19
+
+* Powerbi: allow custom api base and login url
+
+## 0.24.19 - 2025-05-14
+
+* Confluence: extract databases
+
+## 0.24.18 - 2025-05-13
+
+* Improve folder organisation for transformation tools
+
+## 0.24.17 - 2025-05-13
+
+* Strategy: fix dashboard URL format
+
+## 0.24.16 - 2025-05-12
+
+* Confluence: extract folders to complete the page hierarchy
+
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
 ## 0.24.13 - 2025-05-05
 
 * Rollback cloud-storage version as it's not compatible with Keboola

castor_extractor-0.24.20/castor_extractor/commands/extract_confluence.py

@@ -0,0 +1,48 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.knowledge import confluence  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-a", "--account_id", help="Confluence account id")
+    parser.add_argument("-b", "--base_url", help="Confluence account base url")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-t", "--token", help="Confluence API token")
+    parser.add_argument("-u", "--username", help="Confluence username")
+
+    parser.add_argument(
+        "--include-archived-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from archived spaces (Optional)",
+    )
+    parser.add_argument(
+        "--include-personal-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from personal spaces (Optional)",
+    )
+    parser.add_argument(
+        "--space-ids-allowed",
+        type=str,
+        nargs="+",
+        help=(
+            "List of Confluence space IDs allowed for extraction (Optional). "
+            "Only pages from these Spaces will be extracted. "
+            "This overrides any other filtering (archived, personal, etc.)"
+        ),
+    )
+    parser.add_argument(
+        "--space-ids-blocked",
+        type=str,
+        nargs="+",
+        help="List of Confluence space IDs to exclude fom the extraction (Optional)",
+    )
+
+    confluence.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/commands/extract_powerbi.py

@@ -20,5 +20,7 @@ def main():
         nargs="*",
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-l", "--login_url", help="Login url (Optional)")
+    parser.add_argument("-a", "--api_base", help="REST API base (Optional)")
 
     powerbi.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/commands/extract_tableau.py

@@ -21,6 +21,13 @@ def main():
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
 
+    parser.add_argument(
+        "--skip-columns",
+        dest="skip_columns",
+        action="store_true",
+        help="Option to avoid extracting Tableau columns, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/knowledge/confluence/assets.py

@@ -4,5 +4,7 @@ from ...types import ExternalAsset
 class ConfluenceAsset(ExternalAsset):
     """Confluence assets"""
 
+    DATABASES = "databases"
+    FOLDERS = "folders"
     PAGES = "pages"
     USERS = "users"

castor_extractor-0.24.20/castor_extractor/knowledge/confluence/client/client.py

@@ -0,0 +1,172 @@
+import logging
+from collections.abc import Iterator
+from functools import partial
+from typing import Iterable, Optional
+
+from ....utils import (
+    APIClient,
+    BasicAuth,
+    fetch_all_pages,
+)
+from ..assets import (
+    ConfluenceAsset,
+)
+from .credentials import ConfluenceCredentials
+from .endpoints import ConfluenceEndpointFactory
+from .pagination import ConfluencePagination
+
+logger = logging.getLogger(__name__)
+
+_HEADERS = {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+}
+_STATUS_ARCHIVED = "archived"
+_TYPE_PERSONAL = "personal"
+
+
+class ConfluenceClient(APIClient):
+    def __init__(
+        self,
+        credentials: ConfluenceCredentials,
+        include_archived_spaces: bool = False,
+        include_personal_spaces: bool = False,
+        space_ids_allowed: Optional[set[str]] = None,
+        space_ids_blocked: Optional[set[str]] = None,
+    ):
+        self.account_id = credentials.account_id
+        auth = BasicAuth(
+            username=credentials.username, password=credentials.token
+        )
+        super().__init__(
+            auth=auth,
+            host=credentials.base_url,
+            headers=_HEADERS,
+        )
+
+        self.include_archived_spaces = include_archived_spaces
+        self.include_personal_spaces = include_personal_spaces
+        self.space_ids_allowed = space_ids_allowed or set()
+        self.space_ids_blocked = space_ids_blocked or set()
+
+    def databases(self, database_ids: Iterable[str]) -> Iterator[dict]:
+        """
+        Extracts all given databases
+        """
+        for _id in database_ids:
+            database = self._get(
+                endpoint=ConfluenceEndpointFactory.database(_id),
+            )
+            yield database
+
+    def folders(self, folder_ids: Iterable[str]) -> Iterator[dict]:
+        """
+        Extracts all given folders and their parent folders.
+        """
+        candidate_ids = set(folder_ids)
+        seen = set()
+
+        while candidate_ids:
+            folder_id = candidate_ids.pop()
+            if folder_id in seen:
+                continue
+
+            seen.add(folder_id)
+            folder = self._get(
+                endpoint=ConfluenceEndpointFactory.folder(folder_id),
+            )
+            yield folder
+
+            parent_type = folder.get("parentType")
+            if parent_type == "folder":
+                folder_id = folder["parentId"]
+                if folder_id not in seen:
+                    candidate_ids.add(folder["parentId"])
+
+            if not parent_type:
+                logger.info(f"folder with unknown parent: {folder_id}")
+
+    def pages(self):
+        """Extracts all pages from all relevant Spaces."""
+        for space in self.spaces():
+            space_id = space["id"]
+            request = partial(
+                self._get,
+                endpoint=ConfluenceEndpointFactory.pages(space_id),
+            )
+            yield from fetch_all_pages(request, ConfluencePagination)
+
+    def spaces(self) -> Iterator[dict]:
+        """
+        Returns the spaces meeting the conditions defined by the settings.
+
+        If `space_ids_allowed` is not empty, only matching spaces are returned.
+
+        Otherwise, all spaces are filtered by excluding the following:
+          * The space is in the blocked list
+          * The space is personal (type=personal) and skip_personal_spaces is True
+          * The space is archived (status=archived) and skip_archived_spaces is True
+        """
+        request = partial(
+            self._get,
+            endpoint=ConfluenceEndpointFactory.spaces(),
+        )
+        spaces = list(fetch_all_pages(request, ConfluencePagination))
+
+        if self.space_ids_allowed:
+            yield from (
+                space
+                for space in spaces
+                if space["id"] in self.space_ids_allowed
+            )
+            return
+
+        for space in spaces:
+            space_id = space["id"]
+            type_ = space["type"]
+            status = space["status"]
+
+            if space_id in self.space_ids_blocked:
+                continue
+
+            if status == _STATUS_ARCHIVED and not self.include_archived_spaces:
+                continue
+
+            if type_ == _TYPE_PERSONAL and not self.include_personal_spaces:
+                continue
+
+            yield space
+
+    def users(self):
+        request_body = {"accountIds": [self.account_id]}
+        request = partial(
+            self._post,
+            endpoint=ConfluenceEndpointFactory.users(),
+            data=request_body,
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)
+
+    def fetch(
+        self,
+        asset: ConfluenceAsset,
+        *,
+        folder_ids: Optional[Iterator[str]] = None,
+        database_ids: Optional[Iterator[str]] = None,
+    ) -> Iterator[dict]:
+        """Returns the needed metadata for the queried asset"""
+        if asset == ConfluenceAsset.FOLDERS:
+            assert folder_ids is not None
+            yield from self.folders(folder_ids)
+
+        elif asset == ConfluenceAsset.DATABASES:
+            assert database_ids is not None
+            yield from self.databases(database_ids)
+
+        elif asset == ConfluenceAsset.PAGES:
+            yield from self.pages()
+
+        elif asset == ConfluenceAsset.USERS:
+            yield from self.users()
+
+        else:
+            raise ValueError(f"This asset {asset} is unknown")

castor_extractor-0.24.20/castor_extractor/knowledge/confluence/client/client_test.py

@@ -0,0 +1,121 @@
+from unittest.mock import MagicMock, patch
+
+from .client import ConfluenceClient
+
+
+def test_ConfluenceClient_folders():
+    """
+    Folder 1 -> Page A -> Folder 2 -> Folder 3 -> Folder 4 -> Page B
+             -> Page C -> Folder 5 -> Page D
+
+    After extracting the pages, we should have all IDs of folders that are
+    immediate parents of pages. We still need to look out for nested folders.
+    """
+    folder_ids = {"1", "3", "4", "5"}
+    mock_responses = {
+        "1": {"id": "1", "parentType": None, "parentId": None},
+        "2": {"id": "2", "parentType": "page", "parentId": "A"},
+        "3": {"id": "3", "parentType": "folder", "parentId": "2"},
+        "4": {"id": "4", "parentType": "folder", "parentId": "3"},
+        "5": {"id": "5", "parentType": "page", "parentId": "C"},
+    }
+
+    def mock_get(endpoint):
+        folder_id = endpoint.split("/")[-1]
+        return mock_responses[folder_id]
+
+    client = ConfluenceClient(credentials=MagicMock())
+
+    with patch.object(client, "_get", side_effect=mock_get):
+        result = list(client.folders(folder_ids))
+
+    assert len(result) == 5
+    assert {folder["id"] for folder in result} == set(mock_responses.keys())
+
+
+def test_ConfluenceClient_filtered_spaces_with_allowlist():
+    both_blocked_and_allowed_space_id = "789"
+    archived_space_id = "934"
+    random_space = "1000"
+
+    spaces = [
+        # Both blocked and allowed space. "Allowed" setting takes precedence.
+        {
+            "id": both_blocked_and_allowed_space_id,
+            "type": "global",
+            "status": "current",
+        },
+        {"id": archived_space_id, "type": "global", "status": "archived"},
+        {"id": random_space, "type": "global", "status": "current"},
+    ]
+    # the "allowed" list overrides everything else
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=True,
+        space_ids_allowed={
+            both_blocked_and_allowed_space_id,
+        },
+        space_ids_blocked={
+            both_blocked_and_allowed_space_id,
+        },
+    )
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        assert len(filtered_spaces) == 1
+        filtered_space_ids = {space["id"] for space in filtered_spaces}
+        assert set(filtered_space_ids) == {both_blocked_and_allowed_space_id}
+
+
+def test_ConfluenceClient_filtered_spaces():
+    blocked_id = "42"
+    personal_id = "666"
+    archived_id = "934"
+    random_id = "1000"
+
+    # test the other settings : allow personal spaces & block space "42"
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=False,
+        include_personal_spaces=True,
+        space_ids_blocked={blocked_id},
+    )
+
+    spaces = [
+        # Blocked space, to be skipped
+        {"id": blocked_id, "type": "global", "status": "current"},
+        # Archived space, to be skipped
+        {"id": archived_id, "type": "collaboration", "status": "archived"},
+        # Personal space, to be included
+        {"id": personal_id, "type": "personal", "status": "current"},
+        # Valid space
+        {"id": random_id, "type": "knowledge_base", "status": "current"},
+    ]
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        filtered_space_ids = [space["id"] for space in filtered_spaces]
+
+        # no duplicates
+        assert len(filtered_space_ids) == len(set(filtered_space_ids))
+        assert set(filtered_space_ids) == {personal_id, random_id}

castor_extractor-0.24.20/castor_extractor/knowledge/confluence/client/endpoints.py

@@ -0,0 +1,52 @@
+class ConfluenceEndpointFactory:
+    """
+    Confluence rest api v2 endpoint factory.
+    https://developer.atlassian.com/cloud/confluence/rest/v2/intro/#about
+    """
+
+    API = "wiki/api/v2/"
+    DATABASE = "databases"
+    FOLDERS = "folders"
+    PAGES = "pages"
+    SPACES = "spaces"
+    USERS = "users-bulk"
+
+    @classmethod
+    def database(cls, database_id: str) -> str:
+        """
+        Endpoint to fetch a database by id.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-database/#api-databases-id-get
+        """
+        return f"{cls.API}{cls.DATABASE}/{database_id}"
+
+    @classmethod
+    def folder(cls, folder_id: str) -> str:
+        """
+        Endpoint to fetch a folder by id.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-folder/#api-folders-id-get
+        """
+        return f"{cls.API}{cls.FOLDERS}/{folder_id}"
+
+    @classmethod
+    def pages(cls, space_id: str) -> str:
+        """
+        Endpoint to fetch all pages in the given space.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-spaces-id-pages-get
+        """
+        return f"{cls.API}{cls.SPACES}/{space_id}/{cls.PAGES}?body-format=atlas_doc_format"
+
+    @classmethod
+    def spaces(cls) -> str:
+        """
+        Endpoint to fetch all spaces.
+        https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-space/#api-spaces-get
+        """
+        return f"{cls.API}{cls.SPACES}"
+
+    @classmethod
+    def users(cls) -> str:
+        """
+        Endpoint to fetch all user.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-user/#api-users-bulk-post
+        """
+        return f"{cls.API}{cls.USERS}"

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/knowledge/confluence/extract.py

@@ -13,6 +13,7 @@ from ...utils import (
 )
 from .assets import ConfluenceAsset
 from .client import ConfluenceClient, ConfluenceCredentials
+from .utils import pages_to_database_ids, pages_to_folder_ids
 
 logger = logging.getLogger(__name__)
 
@@ -32,6 +33,16 @@ def iterate_all_data(
     yield ConfluenceAsset.PAGES, pages
     logger.info(f"Extracted {len(pages)} pages from API")
 
+    folder_ids = pages_to_folder_ids(pages)
+    logger.info("Extracting FOLDERS from API")
+    folders = list(deep_serialize(client.folders(folder_ids)))
+    yield ConfluenceAsset.FOLDERS, folders
+
+    database_ids = pages_to_database_ids(pages)
+    logger.info("Extracting DATABASES from API")
+    databases = list(deep_serialize(client.databases(database_ids)))
+    yield ConfluenceAsset.DATABASES, databases
+
 
 def extract_all(**kwargs) -> None:
     """
@@ -42,7 +53,13 @@ def extract_all(**kwargs) -> None:
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     credentials = ConfluenceCredentials(**kwargs)
-    client = ConfluenceClient(credentials=credentials)
+    client = ConfluenceClient(
+        credentials=credentials,
+        include_archived_spaces=kwargs.get("include_archived_spaces") or False,
+        include_personal_spaces=kwargs.get("include_personal_spaces") or False,
+        space_ids_allowed=kwargs.get("space_ids_allowed"),
+        space_ids_blocked=kwargs.get("space_ids_blocked"),
+    )
 
     ts = current_timestamp()
 

castor_extractor-0.24.20/castor_extractor/knowledge/confluence/utils.py

@@ -0,0 +1,12 @@
+def pages_to_folder_ids(pages: list[dict]) -> set:
+    """Returns all unique folder parents."""
+    return {
+        page["parentId"] for page in pages if page["parentType"] == "folder"
+    }
+
+
+def pages_to_database_ids(pages: list[dict]) -> set:
+    """Returns all unique database parents."""
+    return {
+        page["parentId"] for page in pages if page["parentType"] == "database"
+    }

castor_extractor-0.24.20/castor_extractor/knowledge/confluence/utils_test.py

@@ -0,0 +1,30 @@
+from .utils import pages_to_database_ids, pages_to_folder_ids
+
+
+def test_pages_to_folder_ids():
+    """Test the pages_to_folder_ids function."""
+    pages = [
+        {"id": "9", "parentId": None, "parentType": None},
+        {"id": "8", "parentId": "2", "parentType": "folder"},
+        {"id": "7", "parentId": "9", "parentType": "page"},
+        {"id": "6", "parentId": "4", "parentType": "folder"},
+        {"id": "5", "parentId": "4", "parentType": "folder"},
+    ]
+    expected = {"2", "4"}
+    result = pages_to_folder_ids(pages)
+    assert result == expected
+
+
+def test_pages_to_database_id():
+    """Test the pages_to_database_id function."""
+    pages = [
+        {"id": "1", "parentId": "db1", "parentType": "database"},
+        {"id": "2", "parentId": "db2", "parentType": "database"},
+        {"id": "3", "parentId": "4", "parentType": "folder"},
+        {"id": "4", "parentId": None, "parentType": None},
+        {"id": "5", "parentId": "db1", "parentType": "database"},
+        {"id": "6", "parentId": "9", "parentType": "page"},
+    ]
+    expected = {"db1", "db2"}
+    result = pages_to_database_ids(pages)
+    assert result == expected

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/transformation/coalesce/client/client.py

@@ -1,7 +1,15 @@
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-from ....utils import APIClient, BearerAuth, RequestSafeMode, SerializedAsset
+import requests
+
+from ....utils import (
+    APIClient,
+    BearerAuth,
+    RequestSafeMode,
+    SerializedAsset,
+    retry,
+)
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
 from .endpoint import (
@@ -12,6 +20,9 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
+_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_RETRY_COUNT = 2
+_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -64,6 +75,11 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
+    @retry(
+        exceptions=_RETRY_EXCEPTIONS,
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id

{castor_extractor-0.24.13/castor_extractor/utils → castor_extractor-0.24.20/castor_extractor/transformation}/dbt/client_test.py

@@ -9,9 +9,7 @@ from dateutil.tz import tzutc
 from .client import ContentType, DbtClient, DbtRun, _account_url  # type: ignore
 from .credentials import DbtCredentials
 
-_DBT_CLIENT_PATH = (
-    "source.packages.extractor.castor_extractor.utils.dbt.client.DbtClient"
-)
+_DBT_CLIENT_PATH = "source.packages.extractor.castor_extractor.transformation.dbt.client.DbtClient"
 _OLD_DATE = datetime(2023, 7, 10, 12, 6, 23, 109171, tzinfo=tzutc())
 _OLD_DATE_STR = "2023-07-10 12:06:23.109171+00:00"
 _RECENT_DATE = datetime(2023, 10, 6, 5, 9, 31, 731991, tzinfo=tzutc())

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/utils/__init__.py

@@ -52,7 +52,7 @@ from .time import (
     yesterday,
 )
 from .type import Callback, Getter, JsonType, SerializedAsset
-from .validation import validate_baseurl
+from .validation import clean_path, validate_baseurl
 from .write import (
     get_output_filename,
     get_summary_filename,

{castor_extractor-0.24.13 → castor_extractor-0.24.20}/castor_extractor/utils/validation.py

@@ -14,7 +14,7 @@ def _preprocess_url(base_url: str) -> str:
     return base_url.strip()
 
 
-def _clean_path(path: str) -> str:
+def clean_path(path: str) -> str:
     return path.rstrip("/")
 
 
@@ -44,7 +44,7 @@ def _urlsplit(base_url: str) -> tuple[str, str, str, str, str, str]:
     url = urlsplit(base_url)
 
     hostname, port = _get_hostname_port(url.netloc)
-    path = _clean_path(url.path)
+    path = clean_path(url.path)
 
     return url.scheme, hostname, path, port, url.query, url.fragment
 

castor_extractor-0.24.20/castor_extractor/visualization/powerbi/__init__.py

@@ -0,0 +1,9 @@
+from .assets import PowerBiAsset
+from .client import (
+    CLIENT_APP_BASE,
+    DEFAULT_SCOPE,
+    REST_API_BASE_PATH,
+    PowerbiClient,
+    PowerbiCredentials,
+)
+from .extract import extract_all