castor-extractor 0.24.13__tar.gz → 0.24.15__tar.gz

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
 ## 0.24.13 - 2025-05-05
 
 * Rollback cloud-storage version as it's not compatible with Keboola

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.13
+Version: 0.24.15
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,14 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
 ## 0.24.13 - 2025-05-05
 
 * Rollback cloud-storage version as it's not compatible with Keboola

castor_extractor-0.24.15/castor_extractor/commands/extract_confluence.py

@@ -0,0 +1,48 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.knowledge import confluence  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-a", "--account_id", help="Confluence account id")
+    parser.add_argument("-b", "--base_url", help="Confluence account base url")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-t", "--token", help="Confluence API token")
+    parser.add_argument("-u", "--username", help="Confluence username")
+
+    parser.add_argument(
+        "--include-archived-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from archived spaces (Optional)",
+    )
+    parser.add_argument(
+        "--include-personal-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from personal spaces (Optional)",
+    )
+    parser.add_argument(
+        "--space-ids-allowed",
+        type=str,
+        nargs="+",
+        help=(
+            "List of Confluence space IDs allowed for extraction (Optional). "
+            "Only pages from these Spaces will be extracted. "
+            "This overrides any other filtering (archived, personal, etc.)"
+        ),
+    )
+    parser.add_argument(
+        "--space-ids-blocked",
+        type=str,
+        nargs="+",
+        help="List of Confluence space IDs to exclude fom the extraction (Optional)",
+    )
+
+    confluence.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/castor_extractor/commands/extract_tableau.py

@@ -21,6 +21,13 @@ def main():
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
 
+    parser.add_argument(
+        "--skip-columns",
+        dest="skip_columns",
+        action="store_true",
+        help="Option to avoid extracting Tableau columns, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",

castor_extractor-0.24.15/castor_extractor/knowledge/confluence/client/client.py

@@ -0,0 +1,118 @@
+from collections.abc import Iterator
+from functools import partial
+from typing import Optional
+
+from ....utils import (
+    APIClient,
+    BasicAuth,
+    fetch_all_pages,
+)
+from ..assets import (
+    ConfluenceAsset,
+)
+from .credentials import ConfluenceCredentials
+from .endpoints import ConfluenceEndpointFactory
+from .pagination import ConfluencePagination
+
+_HEADERS = {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+}
+_STATUS_ARCHIVED = "archived"
+_TYPE_PERSONAL = "personal"
+
+
+class ConfluenceClient(APIClient):
+    def __init__(
+        self,
+        credentials: ConfluenceCredentials,
+        include_archived_spaces: bool = False,
+        include_personal_spaces: bool = False,
+        space_ids_allowed: Optional[set[str]] = None,
+        space_ids_blocked: Optional[set[str]] = None,
+    ):
+        self.account_id = credentials.account_id
+        auth = BasicAuth(
+            username=credentials.username, password=credentials.token
+        )
+        super().__init__(
+            auth=auth,
+            host=credentials.base_url,
+            headers=_HEADERS,
+        )
+
+        self.include_archived_spaces = include_archived_spaces
+        self.include_personal_spaces = include_personal_spaces
+        self.space_ids_allowed = space_ids_allowed or set()
+        self.space_ids_blocked = space_ids_blocked or set()
+
+    def pages(self):
+        """Extracts all pages from all relevant Spaces."""
+        for space in self.spaces():
+            space_id = space["id"]
+            request = partial(
+                self._get,
+                endpoint=ConfluenceEndpointFactory.pages(space_id),
+            )
+            yield from fetch_all_pages(request, ConfluencePagination)
+
+    def spaces(self) -> Iterator[dict]:
+        """
+        Returns the spaces meeting the conditions defined by the settings.
+
+        If `space_ids_allowed` is not empty, only matching spaces are returned.
+
+        Otherwise, all spaces are filtered by excluding the following:
+          * The space is in the blocked list
+          * The space is personal (type=personal) and skip_personal_spaces is True
+          * The space is archived (status=archived) and skip_archived_spaces is True
+        """
+        request = partial(
+            self._get,
+            endpoint=ConfluenceEndpointFactory.spaces(),
+        )
+        spaces = list(fetch_all_pages(request, ConfluencePagination))
+
+        if self.space_ids_allowed:
+            yield from (
+                space
+                for space in spaces
+                if space["id"] in self.space_ids_allowed
+            )
+            return
+
+        for space in spaces:
+            space_id = space["id"]
+            type_ = space["type"]
+            status = space["status"]
+
+            if space_id in self.space_ids_blocked:
+                continue
+
+            if status == _STATUS_ARCHIVED and not self.include_archived_spaces:
+                continue
+
+            if type_ == _TYPE_PERSONAL and not self.include_personal_spaces:
+                continue
+
+            yield space
+
+    def users(self):
+        request_body = {"accountIds": [self.account_id]}
+        request = partial(
+            self._post,
+            endpoint=ConfluenceEndpointFactory.users(),
+            data=request_body,
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)
+
+    def fetch(self, asset: ConfluenceAsset) -> Iterator[dict]:
+        """Returns the needed metadata for the queried asset"""
+        if asset == ConfluenceAsset.PAGES:
+            yield from self.pages()
+
+        elif asset == ConfluenceAsset.USERS:
+            yield from self.users()
+
+        else:
+            raise ValueError(f"This asset {asset} is unknown")

castor_extractor-0.24.15/castor_extractor/knowledge/confluence/client/client_test.py

@@ -0,0 +1,91 @@
+from unittest.mock import MagicMock, patch
+
+from .client import ConfluenceClient
+
+
+def test_ConfluenceClient_filtered_spaces_with_allowlist():
+    both_blocked_and_allowed_space_id = "789"
+    archived_space_id = "934"
+    random_space = "1000"
+
+    spaces = [
+        # Both blocked and allowed space. "Allowed" setting takes precedence.
+        {
+            "id": both_blocked_and_allowed_space_id,
+            "type": "global",
+            "status": "current",
+        },
+        {"id": archived_space_id, "type": "global", "status": "archived"},
+        {"id": random_space, "type": "global", "status": "current"},
+    ]
+    # the "allowed" list overrides everything else
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=True,
+        space_ids_allowed={
+            both_blocked_and_allowed_space_id,
+        },
+        space_ids_blocked={
+            both_blocked_and_allowed_space_id,
+        },
+    )
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        assert len(filtered_spaces) == 1
+        filtered_space_ids = {space["id"] for space in filtered_spaces}
+        assert set(filtered_space_ids) == {both_blocked_and_allowed_space_id}
+
+
+def test_ConfluenceClient_filtered_spaces():
+    blocked_id = "42"
+    personal_id = "666"
+    archived_id = "934"
+    random_id = "1000"
+
+    # test the other settings : allow personal spaces & block space "42"
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=False,
+        include_personal_spaces=True,
+        space_ids_blocked={blocked_id},
+    )
+
+    spaces = [
+        # Blocked space, to be skipped
+        {"id": blocked_id, "type": "global", "status": "current"},
+        # Archived space, to be skipped
+        {"id": archived_id, "type": "collaboration", "status": "archived"},
+        # Personal space, to be included
+        {"id": personal_id, "type": "personal", "status": "current"},
+        # Valid space
+        {"id": random_id, "type": "knowledge_base", "status": "current"},
+    ]
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        filtered_space_ids = [space["id"] for space in filtered_spaces]
+
+        # no duplicates
+        assert len(filtered_space_ids) == len(set(filtered_space_ids))
+        assert set(filtered_space_ids) == {personal_id, random_id}

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/castor_extractor/knowledge/confluence/client/endpoints.py

@@ -6,15 +6,24 @@ class ConfluenceEndpointFactory:
 
     API = "wiki/api/v2/"
     PAGES = "pages"
+    SPACES = "spaces"
     USERS = "users-bulk"
 
     @classmethod
-    def pages(cls) -> str:
+    def pages(cls, space_id: str) -> str:
         """
-        Endpoint to fetch all pages.
-        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
+        Endpoint to fetch all pages in the given space.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-spaces-id-pages-get
         """
-        return f"{cls.API}{cls.PAGES}?body-format=atlas_doc_format"
+        return f"{cls.API}{cls.SPACES}/{space_id}/{cls.PAGES}?body-format=atlas_doc_format"
+
+    @classmethod
+    def spaces(cls) -> str:
+        """
+        Endpoint to fetch all spaces.
+        https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-space/#api-spaces-get
+        """
+        return f"{cls.API}{cls.SPACES}"
 
     @classmethod
     def users(cls) -> str:

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/castor_extractor/knowledge/confluence/extract.py

@@ -42,7 +42,13 @@ def extract_all(**kwargs) -> None:
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     credentials = ConfluenceCredentials(**kwargs)
-    client = ConfluenceClient(credentials=credentials)
+    client = ConfluenceClient(
+        credentials=credentials,
+        include_archived_spaces=kwargs.get("include_archived_spaces") or False,
+        include_personal_spaces=kwargs.get("include_personal_spaces") or False,
+        space_ids_allowed=kwargs.get("space_ids_allowed"),
+        space_ids_blocked=kwargs.get("space_ids_blocked"),
+    )
 
     ts = current_timestamp()
 

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/castor_extractor/transformation/coalesce/client/client.py

@@ -1,7 +1,15 @@
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-from ....utils import APIClient, BearerAuth, RequestSafeMode, SerializedAsset
+import requests
+
+from ....utils import (
+    APIClient,
+    BearerAuth,
+    RequestSafeMode,
+    SerializedAsset,
+    retry,
+)
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
 from .endpoint import (
@@ -12,6 +20,9 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
+_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_RETRY_COUNT = 2
+_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -64,6 +75,11 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
+    @retry(
+        exceptions=_RETRY_EXCEPTIONS,
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/castor_extractor/visualization/tableau/client/client.py

@@ -121,11 +121,13 @@ class TableauClient:
         self,
         credentials: TableauCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
+        with_columns: bool = True,
         with_pulse: bool = False,
         override_page_size: Optional[int] = None,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
+        self._with_columns = with_columns
         self._with_pulse = with_pulse
 
         self._client_metadata = TableauClientMetadataApi(
@@ -215,6 +217,10 @@ class TableauClient:
             logger.info(f"Skipping asset {asset} - Tableau Pulse de-activated")
             return []
 
+        if asset == TableauAsset.COLUMN and not self._with_columns:
+            logger.info(f"Skipping asset {asset} - deactivated columns")
+            return []
+
         logger.info(f"Extracting {asset.name}...")
 
         if asset == TableauAsset.DATASOURCE:

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/castor_extractor/visualization/tableau/extract.py

@@ -32,6 +32,7 @@ def extract_all(**kwargs) -> None:
     output_directory
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    with_columns = not kwargs.get("skip_columns")
     with_pulse = kwargs.get("with_pulse") or False
     page_size = kwargs.get("page_size")
     timestamp = current_timestamp()
@@ -39,6 +40,7 @@ def extract_all(**kwargs) -> None:
     credentials = TableauCredentials(**kwargs)
     client = TableauClient(
         credentials,
+        with_columns=with_columns,
         with_pulse=with_pulse,
         override_page_size=page_size,
     )

{castor_extractor-0.24.13 → castor_extractor-0.24.15}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.13"
+version = "0.24.15"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.24.13/castor_extractor/commands/extract_confluence.py

@@ -1,19 +0,0 @@
-import logging
-from argparse import ArgumentParser
-
-from castor_extractor.knowledge import confluence  # type: ignore
-from castor_extractor.utils import parse_filled_arguments  # type: ignore
-
-logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
-
-
-def main():
-    parser = ArgumentParser()
-
-    parser.add_argument("-a", "--account_id", help="Confluence account id")
-    parser.add_argument("-b", "--base_url", help="Confluence account base url")
-    parser.add_argument("-o", "--output", help="Directory to write to")
-    parser.add_argument("-t", "--token", help="Confluence API token")
-    parser.add_argument("-u", "--username", help="Confluence username")
-
-    confluence.extract_all(**parse_filled_arguments(parser))

castor_extractor-0.24.13/castor_extractor/knowledge/confluence/client/client.py

@@ -1,62 +0,0 @@
-from collections.abc import Iterator
-from functools import partial
-
-from ....utils import (
-    APIClient,
-    BasicAuth,
-    fetch_all_pages,
-)
-from ..assets import (
-    ConfluenceAsset,
-)
-from .credentials import ConfluenceCredentials
-from .endpoints import ConfluenceEndpointFactory
-from .pagination import ConfluencePagination
-
-_HEADERS = {
-    "Accept": "application/json",
-    "Content-Type": "application/json",
-}
-
-
-class ConfluenceClient(APIClient):
-    def __init__(
-        self,
-        credentials: ConfluenceCredentials,
-    ):
-        self.account_id = credentials.account_id
-        auth = BasicAuth(
-            username=credentials.username, password=credentials.token
-        )
-        super().__init__(
-            auth=auth,
-            host=credentials.base_url,
-            headers=_HEADERS,
-        )
-
-    def pages(self):
-        request = partial(
-            self._get,
-            endpoint=ConfluenceEndpointFactory.pages(),
-        )
-        yield from fetch_all_pages(request, ConfluencePagination)
-
-    def users(self):
-        request_body = {"accountIds": [self.account_id]}
-        request = partial(
-            self._post,
-            endpoint=ConfluenceEndpointFactory.users(),
-            data=request_body,
-        )
-        yield from fetch_all_pages(request, ConfluencePagination)
-
-    def fetch(self, asset: ConfluenceAsset) -> Iterator[dict]:
-        """Returns the needed metadata for the queried asset"""
-        if asset == ConfluenceAsset.PAGES:
-            yield from self.pages()
-
-        elif asset == ConfluenceAsset.USERS:
-            yield from self.users()
-
-        else:
-            raise ValueError(f"This asset {asset} is unknown")