castor-extractor 0.24.13__py3-none-any.whl → 0.24.20__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 0.24.20 - 2025-05-19
+
+* Powerbi: allow custom api base and login url
+
+## 0.24.19 - 2025-05-14
+
+* Confluence: extract databases
+
+## 0.24.18 - 2025-05-13
+
+* Improve folder organisation for transformation tools
+
+## 0.24.17 - 2025-05-13
+
+* Strategy: fix dashboard URL format
+
+## 0.24.16 - 2025-05-12
+
+* Confluence: extract folders to complete the page hierarchy
+
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
 ## 0.24.13 - 2025-05-05
 
 * Rollback cloud-storage version as it's not compatible with Keboola

castor_extractor/commands/extract_confluence.py

@@ -16,4 +16,33 @@ def main():
     parser.add_argument("-t", "--token", help="Confluence API token")
     parser.add_argument("-u", "--username", help="Confluence username")
 
+    parser.add_argument(
+        "--include-archived-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from archived spaces (Optional)",
+    )
+    parser.add_argument(
+        "--include-personal-spaces",
+        action="store_true",
+        default=False,
+        help="Include pages from personal spaces (Optional)",
+    )
+    parser.add_argument(
+        "--space-ids-allowed",
+        type=str,
+        nargs="+",
+        help=(
+            "List of Confluence space IDs allowed for extraction (Optional). "
+            "Only pages from these Spaces will be extracted. "
+            "This overrides any other filtering (archived, personal, etc.)"
+        ),
+    )
+    parser.add_argument(
+        "--space-ids-blocked",
+        type=str,
+        nargs="+",
+        help="List of Confluence space IDs to exclude fom the extraction (Optional)",
+    )
+
     confluence.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_powerbi.py

@@ -20,5 +20,7 @@ def main():
         nargs="*",
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-l", "--login_url", help="Login url (Optional)")
+    parser.add_argument("-a", "--api_base", help="REST API base (Optional)")
 
     powerbi.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_tableau.py

@@ -21,6 +21,13 @@ def main():
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
 
+    parser.add_argument(
+        "--skip-columns",
+        dest="skip_columns",
+        action="store_true",
+        help="Option to avoid extracting Tableau columns, default to False",
+    )
+
     parser.add_argument(
         "--with-pulse",
         dest="with_pulse",

castor_extractor/knowledge/confluence/assets.py

@@ -4,5 +4,7 @@ from ...types import ExternalAsset
 class ConfluenceAsset(ExternalAsset):
     """Confluence assets"""
 
+    DATABASES = "databases"
+    FOLDERS = "folders"
     PAGES = "pages"
     USERS = "users"

castor_extractor/knowledge/confluence/client/client.py

@@ -1,5 +1,7 @@
+import logging
 from collections.abc import Iterator
 from functools import partial
+from typing import Iterable, Optional
 
 from ....utils import (
     APIClient,
@@ -13,16 +15,24 @@ from .credentials import ConfluenceCredentials
 from .endpoints import ConfluenceEndpointFactory
 from .pagination import ConfluencePagination
 
+logger = logging.getLogger(__name__)
+
 _HEADERS = {
     "Accept": "application/json",
     "Content-Type": "application/json",
 }
+_STATUS_ARCHIVED = "archived"
+_TYPE_PERSONAL = "personal"
 
 
 class ConfluenceClient(APIClient):
     def __init__(
         self,
         credentials: ConfluenceCredentials,
+        include_archived_spaces: bool = False,
+        include_personal_spaces: bool = False,
+        space_ids_allowed: Optional[set[str]] = None,
+        space_ids_blocked: Optional[set[str]] = None,
     ):
         self.account_id = credentials.account_id
         auth = BasicAuth(
@@ -34,12 +44,98 @@ class ConfluenceClient(APIClient):
             headers=_HEADERS,
         )
 
+        self.include_archived_spaces = include_archived_spaces
+        self.include_personal_spaces = include_personal_spaces
+        self.space_ids_allowed = space_ids_allowed or set()
+        self.space_ids_blocked = space_ids_blocked or set()
+
+    def databases(self, database_ids: Iterable[str]) -> Iterator[dict]:
+        """
+        Extracts all given databases
+        """
+        for _id in database_ids:
+            database = self._get(
+                endpoint=ConfluenceEndpointFactory.database(_id),
+            )
+            yield database
+
+    def folders(self, folder_ids: Iterable[str]) -> Iterator[dict]:
+        """
+        Extracts all given folders and their parent folders.
+        """
+        candidate_ids = set(folder_ids)
+        seen = set()
+
+        while candidate_ids:
+            folder_id = candidate_ids.pop()
+            if folder_id in seen:
+                continue
+
+            seen.add(folder_id)
+            folder = self._get(
+                endpoint=ConfluenceEndpointFactory.folder(folder_id),
+            )
+            yield folder
+
+            parent_type = folder.get("parentType")
+            if parent_type == "folder":
+                folder_id = folder["parentId"]
+                if folder_id not in seen:
+                    candidate_ids.add(folder["parentId"])
+
+            if not parent_type:
+                logger.info(f"folder with unknown parent: {folder_id}")
+
     def pages(self):
+        """Extracts all pages from all relevant Spaces."""
+        for space in self.spaces():
+            space_id = space["id"]
+            request = partial(
+                self._get,
+                endpoint=ConfluenceEndpointFactory.pages(space_id),
+            )
+            yield from fetch_all_pages(request, ConfluencePagination)
+
+    def spaces(self) -> Iterator[dict]:
+        """
+        Returns the spaces meeting the conditions defined by the settings.
+
+        If `space_ids_allowed` is not empty, only matching spaces are returned.
+
+        Otherwise, all spaces are filtered by excluding the following:
+          * The space is in the blocked list
+          * The space is personal (type=personal) and skip_personal_spaces is True
+          * The space is archived (status=archived) and skip_archived_spaces is True
+        """
         request = partial(
             self._get,
-            endpoint=ConfluenceEndpointFactory.pages(),
+            endpoint=ConfluenceEndpointFactory.spaces(),
         )
-        yield from fetch_all_pages(request, ConfluencePagination)
+        spaces = list(fetch_all_pages(request, ConfluencePagination))
+
+        if self.space_ids_allowed:
+            yield from (
+                space
+                for space in spaces
+                if space["id"] in self.space_ids_allowed
+            )
+            return
+
+        for space in spaces:
+            space_id = space["id"]
+            type_ = space["type"]
+            status = space["status"]
+
+            if space_id in self.space_ids_blocked:
+                continue
+
+            if status == _STATUS_ARCHIVED and not self.include_archived_spaces:
+                continue
+
+            if type_ == _TYPE_PERSONAL and not self.include_personal_spaces:
+                continue
+
+            yield space
 
     def users(self):
         request_body = {"accountIds": [self.account_id]}
@@ -50,9 +146,23 @@ class ConfluenceClient(APIClient):
         )
         yield from fetch_all_pages(request, ConfluencePagination)
 
-    def fetch(self, asset: ConfluenceAsset) -> Iterator[dict]:
+    def fetch(
+        self,
+        asset: ConfluenceAsset,
+        *,
+        folder_ids: Optional[Iterator[str]] = None,
+        database_ids: Optional[Iterator[str]] = None,
+    ) -> Iterator[dict]:
         """Returns the needed metadata for the queried asset"""
-        if asset == ConfluenceAsset.PAGES:
+        if asset == ConfluenceAsset.FOLDERS:
+            assert folder_ids is not None
+            yield from self.folders(folder_ids)
+
+        elif asset == ConfluenceAsset.DATABASES:
+            assert database_ids is not None
+            yield from self.databases(database_ids)
+
+        elif asset == ConfluenceAsset.PAGES:
             yield from self.pages()
 
         elif asset == ConfluenceAsset.USERS:

castor_extractor/knowledge/confluence/client/client_test.py

@@ -0,0 +1,121 @@
+from unittest.mock import MagicMock, patch
+
+from .client import ConfluenceClient
+
+
+def test_ConfluenceClient_folders():
+    """
+    Folder 1 -> Page A -> Folder 2 -> Folder 3 -> Folder 4 -> Page B
+             -> Page C -> Folder 5 -> Page D
+
+    After extracting the pages, we should have all IDs of folders that are
+    immediate parents of pages. We still need to look out for nested folders.
+    """
+    folder_ids = {"1", "3", "4", "5"}
+    mock_responses = {
+        "1": {"id": "1", "parentType": None, "parentId": None},
+        "2": {"id": "2", "parentType": "page", "parentId": "A"},
+        "3": {"id": "3", "parentType": "folder", "parentId": "2"},
+        "4": {"id": "4", "parentType": "folder", "parentId": "3"},
+        "5": {"id": "5", "parentType": "page", "parentId": "C"},
+    }
+
+    def mock_get(endpoint):
+        folder_id = endpoint.split("/")[-1]
+        return mock_responses[folder_id]
+
+    client = ConfluenceClient(credentials=MagicMock())
+
+    with patch.object(client, "_get", side_effect=mock_get):
+        result = list(client.folders(folder_ids))
+
+    assert len(result) == 5
+    assert {folder["id"] for folder in result} == set(mock_responses.keys())
+
+
+def test_ConfluenceClient_filtered_spaces_with_allowlist():
+    both_blocked_and_allowed_space_id = "789"
+    archived_space_id = "934"
+    random_space = "1000"
+
+    spaces = [
+        # Both blocked and allowed space. "Allowed" setting takes precedence.
+        {
+            "id": both_blocked_and_allowed_space_id,
+            "type": "global",
+            "status": "current",
+        },
+        {"id": archived_space_id, "type": "global", "status": "archived"},
+        {"id": random_space, "type": "global", "status": "current"},
+    ]
+    # the "allowed" list overrides everything else
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=True,
+        space_ids_allowed={
+            both_blocked_and_allowed_space_id,
+        },
+        space_ids_blocked={
+            both_blocked_and_allowed_space_id,
+        },
+    )
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        assert len(filtered_spaces) == 1
+        filtered_space_ids = {space["id"] for space in filtered_spaces}
+        assert set(filtered_space_ids) == {both_blocked_and_allowed_space_id}
+
+
+def test_ConfluenceClient_filtered_spaces():
+    blocked_id = "42"
+    personal_id = "666"
+    archived_id = "934"
+    random_id = "1000"
+
+    # test the other settings : allow personal spaces & block space "42"
+    client = ConfluenceClient(
+        credentials=MagicMock(),
+        include_archived_spaces=False,
+        include_personal_spaces=True,
+        space_ids_blocked={blocked_id},
+    )
+
+    spaces = [
+        # Blocked space, to be skipped
+        {"id": blocked_id, "type": "global", "status": "current"},
+        # Archived space, to be skipped
+        {"id": archived_id, "type": "collaboration", "status": "archived"},
+        # Personal space, to be included
+        {"id": personal_id, "type": "personal", "status": "current"},
+        # Valid space
+        {"id": random_id, "type": "knowledge_base", "status": "current"},
+    ]
+
+    with (
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.ConfluenceClient._get"
+        ),
+        patch(
+            "source.packages.extractor.castor_extractor.knowledge.confluence.client.client.fetch_all_pages"
+        ) as mock_fetch_all_pages,
+    ):
+        mock_fetch_all_pages.return_value = spaces
+
+        filtered_spaces = list(client.spaces())
+
+        filtered_space_ids = [space["id"] for space in filtered_spaces]
+
+        # no duplicates
+        assert len(filtered_space_ids) == len(set(filtered_space_ids))
+        assert set(filtered_space_ids) == {personal_id, random_id}

castor_extractor/knowledge/confluence/client/endpoints.py

@@ -5,16 +5,43 @@ class ConfluenceEndpointFactory:
     """
 
     API = "wiki/api/v2/"
+    DATABASE = "databases"
+    FOLDERS = "folders"
     PAGES = "pages"
+    SPACES = "spaces"
     USERS = "users-bulk"
 
     @classmethod
-    def pages(cls) -> str:
+    def database(cls, database_id: str) -> str:
         """
-        Endpoint to fetch all pages.
-        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
+        Endpoint to fetch a database by id.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-database/#api-databases-id-get
         """
-        return f"{cls.API}{cls.PAGES}?body-format=atlas_doc_format"
+        return f"{cls.API}{cls.DATABASE}/{database_id}"
+
+    @classmethod
+    def folder(cls, folder_id: str) -> str:
+        """
+        Endpoint to fetch a folder by id.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-folder/#api-folders-id-get
+        """
+        return f"{cls.API}{cls.FOLDERS}/{folder_id}"
+
+    @classmethod
+    def pages(cls, space_id: str) -> str:
+        """
+        Endpoint to fetch all pages in the given space.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-spaces-id-pages-get
+        """
+        return f"{cls.API}{cls.SPACES}/{space_id}/{cls.PAGES}?body-format=atlas_doc_format"
+
+    @classmethod
+    def spaces(cls) -> str:
+        """
+        Endpoint to fetch all spaces.
+        https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-space/#api-spaces-get
+        """
+        return f"{cls.API}{cls.SPACES}"
 
     @classmethod
     def users(cls) -> str:

castor_extractor/knowledge/confluence/extract.py

@@ -13,6 +13,7 @@ from ...utils import (
 )
 from .assets import ConfluenceAsset
 from .client import ConfluenceClient, ConfluenceCredentials
+from .utils import pages_to_database_ids, pages_to_folder_ids
 
 logger = logging.getLogger(__name__)
 
@@ -32,6 +33,16 @@ def iterate_all_data(
     yield ConfluenceAsset.PAGES, pages
     logger.info(f"Extracted {len(pages)} pages from API")
 
+    folder_ids = pages_to_folder_ids(pages)
+    logger.info("Extracting FOLDERS from API")
+    folders = list(deep_serialize(client.folders(folder_ids)))
+    yield ConfluenceAsset.FOLDERS, folders
+
+    database_ids = pages_to_database_ids(pages)
+    logger.info("Extracting DATABASES from API")
+    databases = list(deep_serialize(client.databases(database_ids)))
+    yield ConfluenceAsset.DATABASES, databases
+
 
 def extract_all(**kwargs) -> None:
     """
@@ -42,7 +53,13 @@ def extract_all(**kwargs) -> None:
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     credentials = ConfluenceCredentials(**kwargs)
-    client = ConfluenceClient(credentials=credentials)
+    client = ConfluenceClient(
+        credentials=credentials,
+        include_archived_spaces=kwargs.get("include_archived_spaces") or False,
+        include_personal_spaces=kwargs.get("include_personal_spaces") or False,
+        space_ids_allowed=kwargs.get("space_ids_allowed"),
+        space_ids_blocked=kwargs.get("space_ids_blocked"),
+    )
 
     ts = current_timestamp()
 

castor_extractor/knowledge/confluence/utils.py

@@ -0,0 +1,12 @@
+def pages_to_folder_ids(pages: list[dict]) -> set:
+    """Returns all unique folder parents."""
+    return {
+        page["parentId"] for page in pages if page["parentType"] == "folder"
+    }
+
+
+def pages_to_database_ids(pages: list[dict]) -> set:
+    """Returns all unique database parents."""
+    return {
+        page["parentId"] for page in pages if page["parentType"] == "database"
+    }

castor_extractor/knowledge/confluence/utils_test.py

@@ -0,0 +1,30 @@
+from .utils import pages_to_database_ids, pages_to_folder_ids
+
+
+def test_pages_to_folder_ids():
+    """Test the pages_to_folder_ids function."""
+    pages = [
+        {"id": "9", "parentId": None, "parentType": None},
+        {"id": "8", "parentId": "2", "parentType": "folder"},
+        {"id": "7", "parentId": "9", "parentType": "page"},
+        {"id": "6", "parentId": "4", "parentType": "folder"},
+        {"id": "5", "parentId": "4", "parentType": "folder"},
+    ]
+    expected = {"2", "4"}
+    result = pages_to_folder_ids(pages)
+    assert result == expected
+
+
+def test_pages_to_database_id():
+    """Test the pages_to_database_id function."""
+    pages = [
+        {"id": "1", "parentId": "db1", "parentType": "database"},
+        {"id": "2", "parentId": "db2", "parentType": "database"},
+        {"id": "3", "parentId": "4", "parentType": "folder"},
+        {"id": "4", "parentId": None, "parentType": None},
+        {"id": "5", "parentId": "db1", "parentType": "database"},
+        {"id": "6", "parentId": "9", "parentType": "page"},
+    ]
+    expected = {"db1", "db2"}
+    result = pages_to_database_ids(pages)
+    assert result == expected

castor_extractor/transformation/coalesce/client/client.py

@@ -1,7 +1,15 @@
 from http import HTTPStatus
 from typing import Iterator, Optional
 
-from ....utils import APIClient, BearerAuth, RequestSafeMode, SerializedAsset
+import requests
+
+from ....utils import (
+    APIClient,
+    BearerAuth,
+    RequestSafeMode,
+    SerializedAsset,
+    retry,
+)
 from ..assets import CoalesceAsset, CoalesceQualityAsset
 from .credentials import CoalesceCredentials
 from .endpoint import (
@@ -12,6 +20,9 @@ from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
 _MAX_ERRORS = 50
+_RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
+_RETRY_COUNT = 2
+_RETRY_EXCEPTIONS = [requests.exceptions.ConnectTimeout]
 
 
 def _run_result_payload(result: dict, query_result: dict) -> dict:
@@ -64,6 +75,11 @@ class CoalesceClient(APIClient):
         result = self._get(endpoint=endpoint)
         return result["data"]
 
+    @retry(
+        exceptions=_RETRY_EXCEPTIONS,
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _node_details(self, environment_id: int, node_id: str) -> dict:
         endpoint = CoalesceEndpointFactory.nodes(
             environment_id=environment_id, node_id=node_id

castor_extractor/{utils → transformation}/dbt/client_test.py

@@ -9,9 +9,7 @@ from dateutil.tz import tzutc
 from .client import ContentType, DbtClient, DbtRun, _account_url  # type: ignore
 from .credentials import DbtCredentials
 
-_DBT_CLIENT_PATH = (
-    "source.packages.extractor.castor_extractor.utils.dbt.client.DbtClient"
-)
+_DBT_CLIENT_PATH = "source.packages.extractor.castor_extractor.transformation.dbt.client.DbtClient"
 _OLD_DATE = datetime(2023, 7, 10, 12, 6, 23, 109171, tzinfo=tzutc())
 _OLD_DATE_STR = "2023-07-10 12:06:23.109171+00:00"
 _RECENT_DATE = datetime(2023, 10, 6, 5, 9, 31, 731991, tzinfo=tzutc())

castor_extractor/utils/__init__.py

@@ -52,7 +52,7 @@ from .time import (
     yesterday,
 )
 from .type import Callback, Getter, JsonType, SerializedAsset
-from .validation import validate_baseurl
+from .validation import clean_path, validate_baseurl
 from .write import (
     get_output_filename,
     get_summary_filename,

castor_extractor/utils/validation.py

@@ -14,7 +14,7 @@ def _preprocess_url(base_url: str) -> str:
     return base_url.strip()
 
 
-def _clean_path(path: str) -> str:
+def clean_path(path: str) -> str:
     return path.rstrip("/")
 
 
@@ -44,7 +44,7 @@ def _urlsplit(base_url: str) -> tuple[str, str, str, str, str, str]:
     url = urlsplit(base_url)
 
     hostname, port = _get_hostname_port(url.netloc)
-    path = _clean_path(url.path)
+    path = clean_path(url.path)
 
     return url.scheme, hostname, path, port, url.query, url.fragment
 

castor_extractor/visualization/powerbi/__init__.py

@@ -1,3 +1,9 @@
 from .assets import PowerBiAsset
-from .client import DEFAULT_SCOPE, PowerbiClient, PowerbiCredentials
+from .client import (
+    CLIENT_APP_BASE,
+    DEFAULT_SCOPE,
+    REST_API_BASE_PATH,
+    PowerbiClient,
+    PowerbiCredentials,
+)
 from .extract import extract_all

castor_extractor/visualization/powerbi/client/__init__.py

@@ -1,2 +1,7 @@
 from .client import PowerbiClient
-from .credentials import DEFAULT_SCOPE, PowerbiCredentials
+from .credentials import (
+    CLIENT_APP_BASE,
+    DEFAULT_SCOPE,
+    REST_API_BASE_PATH,
+    PowerbiCredentials,
+)

castor_extractor/visualization/powerbi/client/authentication.py

@@ -9,7 +9,11 @@ from .endpoints import PowerBiEndpointFactory
 class PowerBiBearerAuth(BearerAuth):
     def __init__(self, credentials: PowerbiCredentials):
         self.credentials = credentials
-        authority = PowerBiEndpointFactory.authority(self.credentials.tenant_id)
+        endpoint_factory = PowerBiEndpointFactory(
+            login_url=self.credentials.login_url,
+            api_base=self.credentials.api_base,
+        )
+        authority = endpoint_factory.authority(self.credentials.tenant_id)
         self.app = msal.ConfidentialClientApplication(
             client_id=self.credentials.client_id,
             authority=authority,

castor_extractor/visualization/powerbi/client/client.py

@@ -40,6 +40,10 @@ class PowerbiClient(APIClient):
             auth=auth,
             timeout=POWERBI_DEFAULT_TIMEOUT_S,
         )
+        self.endpoint_factory = PowerBiEndpointFactory(
+            login_url=credentials.login_url,
+            api_base=credentials.api_base,
+        )
 
     def _activity_events(self, day: Optional[date] = None) -> Iterator[dict]:
         """
@@ -49,7 +53,7 @@ class PowerbiClient(APIClient):
         """
         request = partial(
             self._get,
-            endpoint=PowerBiEndpointFactory.activity_events(day),
+            endpoint=self.endpoint_factory.activity_events(day),
         )
         yield from fetch_all_pages(request, PowerBiPagination)
 
@@ -58,28 +62,28 @@ class PowerbiClient(APIClient):
         Returns a list of datasets for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/datasets-get-datasets-as-admin
         """
-        yield from self._get(PowerBiEndpointFactory.datasets())[Keys.VALUE]
+        yield from self._get(self.endpoint_factory.datasets())[Keys.VALUE]
 
     def _dashboards(self) -> Iterator[dict]:
         """
         Returns a list of dashboards for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/dashboards-get-dashboards-as-admin
         """
-        yield from self._get(PowerBiEndpointFactory.dashboards())[Keys.VALUE]
+        yield from self._get(self.endpoint_factory.dashboards())[Keys.VALUE]
 
     def _reports(self) -> Iterator[dict]:
         """
         Returns a list of reports for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/reports-get-reports-as-admin
         """
-        reports_endpoint = PowerBiEndpointFactory.reports()
+        reports_endpoint = self.endpoint_factory.reports()
         reports = self._get(reports_endpoint)[Keys.VALUE]
 
         for report in reports:
             report_id = report.get(Keys.ID)
 
             try:
-                pages_endpoint = PowerBiEndpointFactory.pages(report_id)
+                pages_endpoint = self.endpoint_factory.pages(report_id)
                 pages = self._get(pages_endpoint)[Keys.VALUE]
                 report["pages"] = pages
             except (requests.HTTPError, requests.exceptions.Timeout) as e:
@@ -99,14 +103,14 @@ class PowerbiClient(APIClient):
         }
 
         response = self._get(
-            PowerBiEndpointFactory.workspace_ids(),
+            self.endpoint_factory.workspace_ids(),
             params=params,
         )
 
         return [x[Keys.ID] for x in response]
 
     def _get_scan_result(self, scan_id: int) -> Iterator[dict]:
-        endpoint = PowerBiEndpointFactory.metadata_scan_result(scan_id)
+        endpoint = self.endpoint_factory.metadata_scan_result(scan_id)
         yield from self._get(endpoint)[Keys.WORKSPACES]
 
     def _wait_for_scan_result(self, scan_id: int) -> bool:
@@ -114,7 +118,7 @@ class PowerbiClient(APIClient):
         Periodically checks the status of the metadata scan until the results
         are ready.
         """
-        endpoint = PowerBiEndpointFactory.metadata_scan_status(scan_id)
+        endpoint = self.endpoint_factory.metadata_scan_status(scan_id)
         total_waiting_time_s = 0
 
         while total_waiting_time_s < POWERBI_DEFAULT_TIMEOUT_S:
@@ -152,7 +156,7 @@ class PowerbiClient(APIClient):
         }
         request_body = {"workspaces": workspaces_ids}
         scan_id = self._post(
-            PowerBiEndpointFactory.metadata_create_scan(),
+            self.endpoint_factory.metadata_create_scan(),
             params=params,
             data=request_body,
         )

castor_extractor/visualization/powerbi/client/client_test.py

@@ -6,13 +6,18 @@ import pytest
 from .authentication import msal
 from .client import PowerbiClient
 from .constants import Keys
-from .credentials import PowerbiCredentials
+from .credentials import CLIENT_APP_BASE, REST_API_BASE_PATH, PowerbiCredentials
 from .endpoints import PowerBiEndpointFactory
 
 FAKE_TENANT_ID = "IamFake"
 FAKE_CLIENT_ID = "MeTwo"
 FAKE_SECRET = "MeThree"
 
+ENDPOINT_FACTORY = PowerBiEndpointFactory(
+    login_url=CLIENT_APP_BASE,
+    api_base=REST_API_BASE_PATH + "/",  # superfluous "/" to test resiliency
+)
+
 
 @pytest.fixture
 def mock_msal():
@@ -55,7 +60,7 @@ def test__datasets(power_bi_client):
     with patch.object(power_bi_client, "_get") as mocked_get:
         mocked_get.return_value = {"value": [{"id": 1, "type": "dataset"}]}
         datasets = list(power_bi_client._datasets())
-        mocked_get.assert_called_with(PowerBiEndpointFactory.datasets())
+        mocked_get.assert_called_with(ENDPOINT_FACTORY.datasets())
         assert datasets == [{"id": 1, "type": "dataset"}]
 
 
@@ -63,7 +68,7 @@ def test__dashboards(power_bi_client):
     with patch.object(power_bi_client, "_get") as mocked_get:
         mocked_get.return_value = {"value": [{"id": 1, "type": "dashboard"}]}
         dashboards = list(power_bi_client._dashboards())
-        mocked_get.assert_called_with(PowerBiEndpointFactory.dashboards())
+        mocked_get.assert_called_with(ENDPOINT_FACTORY.dashboards())
         assert dashboards == [{"id": 1, "type": "dashboard"}]
 
 
@@ -79,8 +84,8 @@ def test__reports(power_bi_client):
         ]
         reports = list(power_bi_client._reports())
         calls = [
-            call(PowerBiEndpointFactory.reports()),
-            call(PowerBiEndpointFactory.pages("1")),
+            call(ENDPOINT_FACTORY.reports()),
+            call(ENDPOINT_FACTORY.pages("1")),
         ]
         mocked_get.assert_has_calls(calls)
         assert reports == [
@@ -107,7 +112,7 @@ def test__workspace_ids(power_bi_client):
         }
 
         mocked_get.assert_called_with(
-            PowerBiEndpointFactory.workspace_ids(),
+            ENDPOINT_FACTORY.workspace_ids(),
             params=params,
         )
 
@@ -160,7 +165,7 @@ def test__activity_events(power_bi_client):
         assert result == ["foo", "bar", "baz", "biz"]
 
         expected_calls = [
-            call(endpoint=PowerBiEndpointFactory.activity_events(day=day)),
+            call(endpoint=ENDPOINT_FACTORY.activity_events(day=day)),
             call(endpoint="https://next-call-1"),
             call(endpoint="https://next-call-2"),
         ]

castor_extractor/visualization/powerbi/client/credentials.py

@@ -6,6 +6,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 DEFAULT_SCOPE = "https://analysis.windows.net/powerbi/api/.default"
 POWERBI_ENV_PREFIX = "CASTOR_POWERBI_"
 
+CLIENT_APP_BASE = "https://login.microsoftonline.com"
+REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
+
 
 class PowerbiCredentials(BaseSettings):
     """Class to handle PowerBI rest API permissions"""
@@ -19,9 +22,21 @@ class PowerbiCredentials(BaseSettings):
     client_id: str
     tenant_id: str
     secret: str = Field(repr=False)
+    api_base: str = REST_API_BASE_PATH
+    login_url: str = CLIENT_APP_BASE
     scopes: list[str] = [DEFAULT_SCOPE]
 
     @field_validator("scopes", mode="before")
     @classmethod
     def _check_scopes(cls, scopes: Optional[list[str]]) -> list[str]:
         return scopes if scopes is not None else [DEFAULT_SCOPE]
+
+    @field_validator("login_url", mode="before")
+    @classmethod
+    def _check_login_url(cls, login_url: Optional[str]) -> str:
+        return login_url if login_url is not None else CLIENT_APP_BASE
+
+    @field_validator("api_base", mode="before")
+    @classmethod
+    def _check_api_base(cls, api_base: Optional[str]) -> str:
+        return api_base if api_base is not None else REST_API_BASE_PATH

castor_extractor/visualization/powerbi/client/endpoints.py

@@ -1,10 +1,7 @@
 from datetime import date, datetime
 from typing import Optional
 
-from ....utils import at_midnight, format_date, yesterday
-
-_CLIENT_APP_BASE = "https://login.microsoftonline.com"
-_REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
+from ....utils import at_midnight, clean_path, format_date, yesterday
 
 
 def _time_filter(day: Optional[date]) -> tuple[datetime, datetime]:
@@ -15,51 +12,44 @@ def _time_filter(day: Optional[date]) -> tuple[datetime, datetime]:
 
 
 class PowerBiEndpointFactory:
-    @classmethod
-    def activity_events(cls, day: Optional[date]) -> str:
+    def __init__(self, login_url: str, api_base: str):
+        self.app_base = clean_path(login_url)
+        self.rest_api_base = clean_path(api_base)
+
+    def activity_events(self, day: Optional[date]) -> str:
         start, end = _time_filter(day)
-        url = f"{_REST_API_BASE_PATH}/admin/activityevents"
+        url = f"{self.rest_api_base}/admin/activityevents"
         url += "?$filter=Activity eq 'viewreport'"
         url += f"&startDateTime='{format_date(start)}'"
         url += f"&endDateTime='{format_date(end)}'"
         return url
 
-    @classmethod
-    def authority(cls, tenant_id: str) -> str:
-        return f"{_CLIENT_APP_BASE}/{tenant_id}"
+    def authority(self, tenant_id: str) -> str:
+        return f"{self.app_base}/{tenant_id}"
 
-    @classmethod
-    def dashboards(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/dashboards"
+    def dashboards(self) -> str:
+        return f"{self.rest_api_base}/admin/dashboards"
 
-    @classmethod
-    def datasets(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/datasets"
+    def datasets(self) -> str:
+        return f"{self.rest_api_base}/admin/datasets"
 
-    @classmethod
-    def groups(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/groups"
+    def groups(self) -> str:
+        return f"{self.rest_api_base}/admin/groups"
 
-    @classmethod
-    def metadata_create_scan(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/getInfo"
+    def metadata_create_scan(self) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/getInfo"
 
-    @classmethod
-    def metadata_scan_result(cls, scan_id: int) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/scanResult/{scan_id}"
+    def metadata_scan_result(self, scan_id: int) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/scanResult/{scan_id}"
 
-    @classmethod
-    def metadata_scan_status(cls, scan_id: int) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/scanStatus/{scan_id}"
+    def metadata_scan_status(self, scan_id: int) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/scanStatus/{scan_id}"
 
-    @classmethod
-    def pages(cls, report_id: str) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/reports/{report_id}/pages"
+    def pages(self, report_id: str) -> str:
+        return f"{self.rest_api_base}/admin/reports/{report_id}/pages"
 
-    @classmethod
-    def reports(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/reports"
+    def reports(self) -> str:
+        return f"{self.rest_api_base}/admin/reports"
 
-    @classmethod
-    def workspace_ids(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/modified"
+    def workspace_ids(self) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/modified"

castor_extractor/visualization/strategy/client/client.py

@@ -34,9 +34,7 @@ _BATCH_SIZE: int = 100
 
 
 class URLTemplates(Enum):
-    DASHBOARD = (
-        "https://{hostname}/MicroStrategyLibrarySTD/app/{project_id}/{id_}"
-    )
+    DASHBOARD = "https://{hostname}/MicroStrategyLibrary/app/{project_id}/{id_}"
     DOCUMENT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?documentID={id_}&projectID={project_id}"
     REPORT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?reportID={id_}&projectID={project_id}"
     FOLDER = "https://{hostname}/MicroStrategy/servlet/mstrWeb?folderID={id_}&projectID={project_id}"

castor_extractor/visualization/tableau/client/client.py

@@ -121,11 +121,13 @@ class TableauClient:
         self,
         credentials: TableauCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
+        with_columns: bool = True,
         with_pulse: bool = False,
         override_page_size: Optional[int] = None,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
+        self._with_columns = with_columns
         self._with_pulse = with_pulse
 
         self._client_metadata = TableauClientMetadataApi(
@@ -215,6 +217,10 @@ class TableauClient:
             logger.info(f"Skipping asset {asset} - Tableau Pulse de-activated")
             return []
 
+        if asset == TableauAsset.COLUMN and not self._with_columns:
+            logger.info(f"Skipping asset {asset} - deactivated columns")
+            return []
+
         logger.info(f"Extracting {asset.name}...")
 
         if asset == TableauAsset.DATASOURCE:

castor_extractor/visualization/tableau/extract.py

@@ -32,6 +32,7 @@ def extract_all(**kwargs) -> None:
     output_directory
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    with_columns = not kwargs.get("skip_columns")
     with_pulse = kwargs.get("with_pulse") or False
     page_size = kwargs.get("page_size")
     timestamp = current_timestamp()
@@ -39,6 +40,7 @@ def extract_all(**kwargs) -> None:
     credentials = TableauCredentials(**kwargs)
     client = TableauClient(
         credentials,
+        with_columns=with_columns,
         with_pulse=with_pulse,
         override_page_size=page_size,
     )

{castor_extractor-0.24.13.dist-info → castor_extractor-0.24.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.13
+Version: 0.24.20
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,34 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.20 - 2025-05-19
+
+* Powerbi: allow custom api base and login url
+
+## 0.24.19 - 2025-05-14
+
+* Confluence: extract databases
+
+## 0.24.18 - 2025-05-13
+
+* Improve folder organisation for transformation tools
+
+## 0.24.17 - 2025-05-13
+
+* Strategy: fix dashboard URL format
+
+## 0.24.16 - 2025-05-12
+
+* Confluence: extract folders to complete the page hierarchy
+
+## 0.24.15 - 2025-05-12
+
+* Tableau: Add argument to skip columns extraction
+
+## 0.24.14 - 2025-05-06
+
+* Confluence: extract pages per space to allow additional filtering. by default, pages from archived or personal spaces are not extracted.
+
 ## 0.24.13 - 2025-05-05
 
 * Rollback cloud-storage version as it's not compatible with Keboola

{castor_extractor-0.24.13.dist-info → castor_extractor-0.24.20.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=u3BUqDPLtQ7K1RytlxEmtcSy4kJG1qhPtmrimODTYZU,17156
+CHANGELOG.md,sha256=NxSwszCNlJ8oD2ffivq3g75DK436mue9WwilR0r6bE4,17760
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -6,7 +6,7 @@ README.md,sha256=C6hTyZO60T7z7xwHbspHlii384Jn02k0Rycxu3bCX0o,3866
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/commands/extract_bigquery.py,sha256=dU4OiYO1V0n32orvZnMh1_xtFKF_VxHNXcVsH3otY-g,1269
-castor_extractor/commands/extract_confluence.py,sha256=xQjC0VZdz8jFHnugqQ0fQGjzGq34v49cqcP4ONeVYNs,748
+castor_extractor/commands/extract_confluence.py,sha256=blYcnDqywXNKRQ1aZAD9FclhLlO7x8Y_tb0lgl85v0w,1641
 castor_extractor/commands/extract_databricks.py,sha256=SVKyoa-BBUQAM6HRHf1Wdg9-tpICic2yyvXQwHcNBhA,1264
 castor_extractor/commands/extract_domo.py,sha256=jvAawUsUTHrwCn_koK6StmQr4n_b5GyvJi6uu6WS0SM,1061
 castor_extractor/commands/extract_looker.py,sha256=cySLiolLCgrREJ9d0kMrJ7P8K3efHTBTzShalWVfI3A,1214
@@ -17,7 +17,7 @@ castor_extractor/commands/extract_mode.py,sha256=Q4iO-VAKMg4zFPejhAO-foZibL5Ht3j
 castor_extractor/commands/extract_mysql.py,sha256=7AH5qMzeLTsENCOeJwtesrWg8Vo8MCEq8fx2YT74Mcw,1034
 castor_extractor/commands/extract_notion.py,sha256=uaxcF3_bT7D_-JxnIW0F7VVDphI_ZgOfQQxZzoLXo_M,504
 castor_extractor/commands/extract_postgres.py,sha256=pX0RnCPi4nw6QQ6wiAuZ_Xt3ZbDuMUG9aQKuqFgJtAU,1154
-castor_extractor/commands/extract_powerbi.py,sha256=f0G5w61KXExJ6Sw39_mJIwqQNpLorE5-LKmZXlUqvKI,783
+castor_extractor/commands/extract_powerbi.py,sha256=RKkw9H2ZsbJ4xLE84bmNFUgYUjlrLmSXahQSVrQr_Bc,934
 castor_extractor/commands/extract_qlik.py,sha256=VBe_xFKh_nR0QSFFIncAaC8yDqBeMa6VunBAga7AeGg,891
 castor_extractor/commands/extract_redshift.py,sha256=zRBg2D_ft4GLdPSdmetRcgQVAA80DXtdRSYsQhAWIik,1334
 castor_extractor/commands/extract_salesforce.py,sha256=3j3YTmMkPAwocR-B1ozJQai0UIZPtpmAyWj-hHvdWn4,1226
@@ -26,7 +26,7 @@ castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_
 castor_extractor/commands/extract_snowflake.py,sha256=GwlrRxwEBjHqGs_3bs5vM9fzmv61_iwvBr1KcIgFgWM,2161
 castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7WJw-sDU6GHl3lzBt_0,1141
 castor_extractor/commands/extract_strategy.py,sha256=Q-pUymatPrBFGXobhyUPzFph0-t774-XOpjdCFF1dYo,821
-castor_extractor/commands/extract_tableau.py,sha256=xXlLKLN8Eu_a8Kt2F4E-C5D-gq8SUmvoxJcdR_thKKY,1365
+castor_extractor/commands/extract_tableau.py,sha256=ngujGYohWOqOK1qjIP1Hh951jr0KNKNSeOyoaOnO450,1558
 castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=TJx76Ymd0QCECmq35zRJMkPE8DJtSInB28MuSXWk8Ao,2644
 castor_extractor/commands/upload.py,sha256=rLXp7gQ8zb1kLbho4FT87q8eJd8Gvo_TkyIynAaQ-4s,1342
@@ -43,13 +43,16 @@ castor_extractor/file_checker/templates/__init__.py,sha256=StVLm4ZGyGVmPzarxEaDR
 castor_extractor/file_checker/templates/generic_warehouse.py,sha256=S5qFIkbfihdWz16y4HSiTfDH1SmGl40u3kZ706sFBVI,2939
 castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/knowledge/confluence/__init__.py,sha256=pRT615pMDlB7Ifs09erVn2EdpZHgkvX5selemWU3VPE,129
-castor_extractor/knowledge/confluence/assets.py,sha256=zv2G2LB8H0fKDbVJ4kHrAjbqehXI_K-wgd_ghSXGFvs,144
+castor_extractor/knowledge/confluence/assets.py,sha256=ZRmRnJpbujL8LMQD3FrcFME-pEQM7G1lCZ1abTsx5OU,196
 castor_extractor/knowledge/confluence/client/__init__.py,sha256=ALAzo0JEhxFzH2FnIO6HmtkAGS2_bGY8KXXMcTGV3aE,84
-castor_extractor/knowledge/confluence/client/client.py,sha256=F8A_ckZ4ojJC8BnAXeAIHUC2oOQMBWnTfqQwJbAyTns,1689
+castor_extractor/knowledge/confluence/client/client.py,sha256=Ysl4KDo4ISXuLvUwEZrowGiaUHPC5cli7zN5Vtte_Jc,5392
+castor_extractor/knowledge/confluence/client/client_test.py,sha256=O4YarFiZbm2z_H0zPzPi-awRhsGEY1iOwwHHTv9gCVA,4177
 castor_extractor/knowledge/confluence/client/credentials.py,sha256=tqUMw-SVoAi4o6I6OeGk4MeDiIPU3-ihhaomXv4CQ64,419
-castor_extractor/knowledge/confluence/client/endpoints.py,sha256=eWMKtjDUPGoKR8Nqq18JTIoEq913GNo1Klm9RduIOHM,765
+castor_extractor/knowledge/confluence/client/endpoints.py,sha256=q5JCybVPtSs4rEXmS5IdkJyFQZsx2ff838mJyxryEFo,1742
 castor_extractor/knowledge/confluence/client/pagination.py,sha256=ty4meiMEujDVSiQyOJTibd-ReYyDyGezdFuk7EAGtMA,862
-castor_extractor/knowledge/confluence/extract.py,sha256=6pA68CmYNC50qCJ7NgZMW0jD4ev0a_ltI5kSyBqSC0U,1565
+castor_extractor/knowledge/confluence/extract.py,sha256=Ba1IDDFkZUYJ2HDWNcXNJ1fzFYOTcE1sTDgqFAK9-fA,2332
+castor_extractor/knowledge/confluence/utils.py,sha256=-lcpWY8oacDdg537s5-LsY8B3L7ZKGXT6GCd3yvVw-A,392
+castor_extractor/knowledge/confluence/utils_test.py,sha256=NYlbKUSRjuZ8FaRMqhKNQCW3KmMAhqIVaMAHNNpobsk,1152
 castor_extractor/knowledge/notion/__init__.py,sha256=ZDmh0eNSxHf1zVPm0aYlKPci-vzOXhAgdsWjS2hdjh4,117
 castor_extractor/knowledge/notion/assets.py,sha256=QHv1-pomt5UeN_prP2L6t_zJ-tDSqB8LgopkGAODYPQ,164
 castor_extractor/knowledge/notion/client/__init__.py,sha256=CDPorBCethuNTEtpjvHGcWnWeVfqkEq-IbakWjDKATw,76
@@ -73,12 +76,17 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 castor_extractor/transformation/coalesce/__init__.py,sha256=CW_qdtEfwgJRsCyBlk5hNlxwEO-VV6mBXZvkRbND_J8,112
 castor_extractor/transformation/coalesce/assets.py,sha256=pzccYPP66c9PAnVroemx7-6MeRHw7Ft1OlTC6jIamAA,363
 castor_extractor/transformation/coalesce/client/__init__.py,sha256=VRmVpH29rOghtDQnCN7dAdA0dI0Lxseu4BC8rnwM9dU,80
-castor_extractor/transformation/coalesce/client/client.py,sha256=yrPzIk-6VN4MDHwti3Yxy3PCfHmxE6znjuehl_-dYTg,6151
+castor_extractor/transformation/coalesce/client/client.py,sha256=-fFxWtDPPruNmDPc6FXft_6RwRKEee4JM-13d90fms0,6442
 castor_extractor/transformation/coalesce/client/credentials.py,sha256=jbJxjbdPspf-dzYKfeb7oqL_8TXd1nvkJrjAcdAnLPc,548
 castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9qr_50SEYV_-heQE4BwED9oNMgYsL-w,1272
 castor_extractor/transformation/coalesce/client/type.py,sha256=oiiVP9NL0ijTXyQmaB8aJVYckc7m-m8ZgMyNIAduUKE,43
 castor_extractor/transformation/coalesce/client/utils.py,sha256=jbxh3OCbYm3fKZD1QfqX5zm1ZD_jFIrpUQsX8paRP7g,1627
 castor_extractor/transformation/coalesce/client/utils_test.py,sha256=Q00Y1n0Q_sZ0LFnYn98yDGFumBsifzVJSc7_3PSBMfI,1543
+castor_extractor/transformation/dbt/__init__.py,sha256=LHQROlMqYWCc7tcmhdjXtROFpJqUvCg9jPC8avHgD4I,107
+castor_extractor/transformation/dbt/assets.py,sha256=JY1nKEGySZ84wNoe7dnizwAYw2q0t8NVaIfqhB2rSw0,148
+castor_extractor/transformation/dbt/client.py,sha256=KSlMHaesDOTYjTKs_ZFSYG1Udxjj5jn0m5zaCf5AZZk,5526
+castor_extractor/transformation/dbt/client_test.py,sha256=YK86romOhTRFqzA2Gs262mDUfmvz0cmhiN3jC9nB8P8,4527
+castor_extractor/transformation/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
 castor_extractor/types.py,sha256=nHel2hv6NoHmdpOX_heEfO2-DnZPoYA2x0eJdbFvT0s,1276
 castor_extractor/uploader/__init__.py,sha256=A4bq_SrEtKAsl0r_D_duSTvL5WIQjVfsMy7tDx9IKg0,87
 castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
@@ -88,7 +96,7 @@ castor_extractor/uploader/settings.py,sha256=3MvOX-UFRqrLZoiT7wYn9jUGro7NX4RCafY
 castor_extractor/uploader/upload.py,sha256=PSQfkO_7LSE0WBo9Tm_hlS2ONepKeB0cBFdJXySnues,4310
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=otAaySj5aeem6f0CTd0Te6ioJ6uP2J1p348j-SdIwDI,802
-castor_extractor/utils/__init__.py,sha256=KQkr_CmxWG0Vpu7CaqjbJkffUeEWcyeA9Cbm394Hygk,1585
+castor_extractor/utils/__init__.py,sha256=ybzci46Myi9LABZGBK2qXOiGd00llcpmBpvAp5LEVHc,1597
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
 castor_extractor/utils/batch.py,sha256=SFlLmJgVjV2nVhIrjVIEp8wJ9du4dKKHq8YVYubnwQQ,448
@@ -113,11 +121,6 @@ castor_extractor/utils/client/uri_test.py,sha256=1XKF6qSseCeD4G4ckaNO07JXfGbt7XU
 castor_extractor/utils/collection.py,sha256=FiIJWZZ865oqNjtTm40gQ13R9zh--W2W5YsMBZJf2bk,2334
 castor_extractor/utils/collection_test.py,sha256=XJAGo0Veg0H8wZRCESIkU2t8bXxTNET0BdosomO3-Ls,2104
 castor_extractor/utils/constants.py,sha256=qBQprS9U66mS-RIBXiLujdTSV3WvGv40Bc0khP4Abdk,39
-castor_extractor/utils/dbt/__init__.py,sha256=LHQROlMqYWCc7tcmhdjXtROFpJqUvCg9jPC8avHgD4I,107
-castor_extractor/utils/dbt/assets.py,sha256=JY1nKEGySZ84wNoe7dnizwAYw2q0t8NVaIfqhB2rSw0,148
-castor_extractor/utils/dbt/client.py,sha256=KSlMHaesDOTYjTKs_ZFSYG1Udxjj5jn0m5zaCf5AZZk,5526
-castor_extractor/utils/dbt/client_test.py,sha256=9mHhFLyQ-NrjyRKy0kCEp4hgMj2um7HrhTd452oyRbM,4526
-castor_extractor/utils/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
 castor_extractor/utils/deprecate.py,sha256=aBIN2QqZUx5CBNZMFfOUhi8QqtPqRcJtmrN6xqfm-y8,805
 castor_extractor/utils/env.py,sha256=TqdtB50U8LE0993WhhEhpy89TJrHbjtIKjvg6KQ-5q0,596
 castor_extractor/utils/files.py,sha256=qKbfu5FRjsQdKnRmaJNd5EdX_F6gf5C5tV8LdoYKxs0,1527
@@ -152,7 +155,7 @@ castor_extractor/utils/string_test.py,sha256=u3P2tAPhyfCLvD19rH_JcpHhPuWTHUdg0z_
 castor_extractor/utils/time.py,sha256=jmP1QWg4lv21Jp_Oy71lfJ47hjNOSgHiBOFf964RMPU,1732
 castor_extractor/utils/time_test.py,sha256=pH8DSosNlwDYZXZNNjYDcL0WbmZc_c212LEEn88Oqew,647
 castor_extractor/utils/type.py,sha256=Sd8JlEgbGkBUZnRqCUDtREeBkOMTXtlNMyCph90_J0Q,328
-castor_extractor/utils/validation.py,sha256=kQAFtqt3gfy7YqYQ0u-60vyNYUF_96he5QDVUQnZmDo,1896
+castor_extractor/utils/validation.py,sha256=dRvC9SoFVecVZuLQNN3URq37yX2sBSW3-NxIxkcol5o,1894
 castor_extractor/utils/validation_test.py,sha256=A7P6VmI0kYX2aGIeEN12y7LsY7Kpm8pE4bdVFhbBAMw,1184
 castor_extractor/utils/write.py,sha256=Z_RYm47XeHiUPPUMYMuAjQrVZ18CAkL3daQHQG1XPlM,2148
 castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -230,16 +233,16 @@ castor_extractor/visualization/mode/client/constants.py,sha256=_Si5AF6VnpoSfnNNg
 castor_extractor/visualization/mode/client/credentials.py,sha256=ptIpCCpoNt06yYaWQgl3Xu78_jVMoqsqWAGqQXVFZlo,606
 castor_extractor/visualization/mode/errors.py,sha256=SKpFT2AiLOuWx2VRLyO7jbAiKcGDFXXrsebpNEKtr0E,1495
 castor_extractor/visualization/mode/extract.py,sha256=PmLWWjUwplQh3TNMemiGwyFdxMcKVMvumZPxSMLJAwk,1625
-castor_extractor/visualization/powerbi/__init__.py,sha256=AJnmfdmm2mGaInWJkUfZxRqrI7dBkTUSebpow05g5zo,135
+castor_extractor/visualization/powerbi/__init__.py,sha256=hoZ73ngLhMc9edqxO9PUIE3FABQlvcfY2W8fuc6DEjY,197
 castor_extractor/visualization/powerbi/assets.py,sha256=IB_XKwgdN1pZYGZ4RfeHrLjflianTzWf_6tg-4CIwu0,742
-castor_extractor/visualization/powerbi/client/__init__.py,sha256=8Bzhd9Z0ebVg2gDchXCOPa80Yqlq_9oCjbGi8u1M6J0,93
-castor_extractor/visualization/powerbi/client/authentication.py,sha256=fz0v9qxeADwA1jiS9UzAQN5mA5kmZT53onlcWon2RGw,892
-castor_extractor/visualization/powerbi/client/client.py,sha256=Y_rwkyPO3GbTAtaOClqnGVEK5iWmREIuDKDJx0LefHs,7203
-castor_extractor/visualization/powerbi/client/client_test.py,sha256=6NtpcKZCxBWyJO3phnVgE70Wmunb6tWsdXikkReJ02E,5539
+castor_extractor/visualization/powerbi/client/__init__.py,sha256=UPIhMaCCdNxhiLdkItC0IPFE_AMi-SgqI_ahwjB9utI,151
+castor_extractor/visualization/powerbi/client/authentication.py,sha256=cTohunKr1nUDfvxB0sejJSyfE2BdCtwT1WMPecWlbyU,1045
+castor_extractor/visualization/powerbi/client/client.py,sha256=MbqqUF4yadjbAQ_I0iwOmdqR0qC1L3yfbDGgIZqZ0hQ,7348
+castor_extractor/visualization/powerbi/client/client_test.py,sha256=MhQfg6kj__zpARXfv9-VrJXt1fXj6Eri91y8KA9Sn9E,5694
 castor_extractor/visualization/powerbi/client/constants.py,sha256=88R_aGachNNUZh6OSH2fkDwZtY4KTStzKm_g7HNCqqo,387
-castor_extractor/visualization/powerbi/client/credentials.py,sha256=sVi4ecJP8ydfrGRKKdJML-wxxZjxUshQtyqTUaJYq_g,795
+castor_extractor/visualization/powerbi/client/credentials.py,sha256=OVWdhZSNODzTdLysY-sbpBZ3uUkLokeayQZnbJAqt2I,1386
 castor_extractor/visualization/powerbi/client/credentials_test.py,sha256=TzFqxsWVQ3sXR_n0bJsexK9Uz7ceXCEPVqDGWTJzW60,993
-castor_extractor/visualization/powerbi/client/endpoints.py,sha256=DrAFpYHhp9Z7fxebdy_Ir6LFfFluHRBJId4tVCYTUVs,2051
+castor_extractor/visualization/powerbi/client/endpoints.py,sha256=38ZETzSSnNq3vA9O6nLZQ8T1BVE01R9CjMC03-PRXsM,1911
 castor_extractor/visualization/powerbi/client/pagination.py,sha256=OZMjoDQPRGMoWd9QcKKrPh3aErJR20SHlrTqY_siLkk,755
 castor_extractor/visualization/powerbi/extract.py,sha256=Z5KbqMhMnqjWcnzged2G1-Gf6GYWJobTL9_TpAdgb8o,1309
 castor_extractor/visualization/qlik/__init__.py,sha256=u6lIfm_WOykBwt6SlaB7C0Dtx37XBliUbM5oWv26gC8,177
@@ -276,13 +279,13 @@ castor_extractor/visualization/sigma/extract.py,sha256=XIT1qsj6g6dgBWP8HPfj_medZ
 castor_extractor/visualization/strategy/__init__.py,sha256=HOMv4JxqF5ZmViWi-pDE-PSXJRLTdXal_jtpHG_rlR8,123
 castor_extractor/visualization/strategy/assets.py,sha256=tqB3GOtp-r7IOnYO8UxZgrldoSMImJnv5KeIwDFxg68,302
 castor_extractor/visualization/strategy/client/__init__.py,sha256=XWP0yF5j6JefDJkDfX-RSJn3HF2ceQ0Yx1PLCfB3BBo,80
-castor_extractor/visualization/strategy/client/client.py,sha256=F7taX0jSQpM8R3GOGeUQ7U_bJKkoHTwAc9oyc3ZDxbM,10261
+castor_extractor/visualization/strategy/client/client.py,sha256=_K7JkatG0DYtbQOJULTNYKHWuBZ11KMR_rQjx8LiR5c,10242
 castor_extractor/visualization/strategy/client/credentials.py,sha256=urFfNxWX1JG6wwFMYImufQzHa5g-sgjdlVGzi63owwg,1113
 castor_extractor/visualization/strategy/extract.py,sha256=2fBuvS2xiOGXRpxXnZsE_C3en6t1-BlM5TbusjHyEkg,1166
 castor_extractor/visualization/tableau/__init__.py,sha256=eFI_1hjdkxyUiAYiy3szwyuwn3yJ5C_KbpBU0ySJDcQ,138
 castor_extractor/visualization/tableau/assets.py,sha256=HbCRd8VCj1WBEeqg9jwnygnT7xOFJ6PQD7Lq7sV-XR0,635
 castor_extractor/visualization/tableau/client/__init__.py,sha256=P8RKFKOC63WkH5hdEytJOwHS9vzQ8GXreLfXZetmMP8,78
-castor_extractor/visualization/tableau/client/client.py,sha256=zzqhzIqKyJygo4ZNGk6cZh0e6Z9R1W5T0P9un52KC1M,7626
+castor_extractor/visualization/tableau/client/client.py,sha256=iJ3Y-vwPvmPyAUTs1PqFJEZelPGiLvsiwXpTI3b5THc,7867
 castor_extractor/visualization/tableau/client/client_metadata_api.py,sha256=ryRq4_qUok8vvWGhj5CNWXtwR2JlUsu1qjsov2KhQTE,6286
 castor_extractor/visualization/tableau/client/client_metadata_api_test.py,sha256=rikyQKDLFYHLJhHJTF3LwWhKJ80svtTsYp5n7n9oTU8,2665
 castor_extractor/visualization/tableau/client/client_rest_api.py,sha256=x4dNw4PPJdalTlGowwkANwqiS2ZhGxzpQytkHq3KbpY,3988
@@ -292,7 +295,7 @@ castor_extractor/visualization/tableau/client/errors.py,sha256=ecT8Tit5VtzrOBB9y
 castor_extractor/visualization/tableau/client/gql_queries.py,sha256=XJAfhpMZ5S7-AhfpOaoHMHCAdil-l5e5xB-CH4NC38M,2177
 castor_extractor/visualization/tableau/client/rest_fields.py,sha256=ZKYYuMxg9PXhczVXaD4rXNk7dYyWJ1_bVM8FLEXju7s,888
 castor_extractor/visualization/tableau/constants.py,sha256=lHGB50FgVNO2nXeIhkvQKivD8ZFBIjDrflgD5cTXKJw,104
-castor_extractor/visualization/tableau/extract.py,sha256=FnjmmUdNA9MEf3S5Tw37x6ZXxVsK8R3YnVk1UVYbaZk,1423
+castor_extractor/visualization/tableau/extract.py,sha256=hGVr1BZVsHlIgNXOFusRN2YwUUhXvF3reOeN8g1CTEo,1508
 castor_extractor/visualization/thoughtspot/__init__.py,sha256=NhTGUk5Kdt54oCjHYoAt0cLBmVLys5lFYiRANL6wCmI,150
 castor_extractor/visualization/thoughtspot/assets.py,sha256=SAQWPKaD2NTSDg7-GSkcRSSEkKSws0MJfOVcHkdeTSg,276
 castor_extractor/visualization/thoughtspot/client/__init__.py,sha256=svrE2rMxR-OXctjPeAHMEPePlfcra-9KDevTMcHunAA,86
@@ -423,8 +426,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.13.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.13.dist-info/METADATA,sha256=TBjkOLzvfC6wrL6Myv0N7IkcNXzkQDTqdd8QdbNpbxU,24609
-castor_extractor-0.24.13.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.13.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.13.dist-info/RECORD,,
+castor_extractor-0.24.20.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.20.dist-info/METADATA,sha256=DvgjbhUmiYXzV9e3MXsEGuSfgOog38LrEBNmqFIAcyI,25213
+castor_extractor-0.24.20.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.20.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.20.dist-info/RECORD,,