castor-extractor 0.24.15__py3-none-any.whl → 0.24.20__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.24.20 - 2025-05-19
+
+* Powerbi: allow custom api base and login url
+
+## 0.24.19 - 2025-05-14
+
+* Confluence: extract databases
+
+## 0.24.18 - 2025-05-13
+
+* Improve folder organisation for transformation tools
+
+## 0.24.17 - 2025-05-13
+
+* Strategy: fix dashboard URL format
+
+## 0.24.16 - 2025-05-12
+
+* Confluence: extract folders to complete the page hierarchy
+
 ## 0.24.15 - 2025-05-12
 
 * Tableau: Add argument to skip columns extraction

castor_extractor/commands/extract_powerbi.py

@@ -20,5 +20,7 @@ def main():
         nargs="*",
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-l", "--login_url", help="Login url (Optional)")
+    parser.add_argument("-a", "--api_base", help="REST API base (Optional)")
 
     powerbi.extract_all(**parse_filled_arguments(parser))

castor_extractor/knowledge/confluence/assets.py

@@ -4,5 +4,7 @@ from ...types import ExternalAsset
 class ConfluenceAsset(ExternalAsset):
     """Confluence assets"""
 
+    DATABASES = "databases"
+    FOLDERS = "folders"
     PAGES = "pages"
     USERS = "users"

castor_extractor/knowledge/confluence/client/client.py

@@ -1,6 +1,7 @@
+import logging
 from collections.abc import Iterator
 from functools import partial
-from typing import Optional
+from typing import Iterable, Optional
 
 from ....utils import (
     APIClient,
@@ -14,6 +15,8 @@ from .credentials import ConfluenceCredentials
 from .endpoints import ConfluenceEndpointFactory
 from .pagination import ConfluencePagination
 
+logger = logging.getLogger(__name__)
+
 _HEADERS = {
     "Accept": "application/json",
     "Content-Type": "application/json",
@@ -46,6 +49,43 @@ class ConfluenceClient(APIClient):
         self.space_ids_allowed = space_ids_allowed or set()
         self.space_ids_blocked = space_ids_blocked or set()
 
+    def databases(self, database_ids: Iterable[str]) -> Iterator[dict]:
+        """
+        Extracts all given databases
+        """
+        for _id in database_ids:
+            database = self._get(
+                endpoint=ConfluenceEndpointFactory.database(_id),
+            )
+            yield database
+
+    def folders(self, folder_ids: Iterable[str]) -> Iterator[dict]:
+        """
+        Extracts all given folders and their parent folders.
+        """
+        candidate_ids = set(folder_ids)
+        seen = set()
+
+        while candidate_ids:
+            folder_id = candidate_ids.pop()
+            if folder_id in seen:
+                continue
+
+            seen.add(folder_id)
+            folder = self._get(
+                endpoint=ConfluenceEndpointFactory.folder(folder_id),
+            )
+            yield folder
+
+            parent_type = folder.get("parentType")
+            if parent_type == "folder":
+                folder_id = folder["parentId"]
+                if folder_id not in seen:
+                    candidate_ids.add(folder["parentId"])
+
+            if not parent_type:
+                logger.info(f"folder with unknown parent: {folder_id}")
+
     def pages(self):
         """Extracts all pages from all relevant Spaces."""
         for space in self.spaces():
@@ -106,9 +146,23 @@ class ConfluenceClient(APIClient):
         )
         yield from fetch_all_pages(request, ConfluencePagination)
 
-    def fetch(self, asset: ConfluenceAsset) -> Iterator[dict]:
+    def fetch(
+        self,
+        asset: ConfluenceAsset,
+        *,
+        folder_ids: Optional[Iterator[str]] = None,
+        database_ids: Optional[Iterator[str]] = None,
+    ) -> Iterator[dict]:
         """Returns the needed metadata for the queried asset"""
-        if asset == ConfluenceAsset.PAGES:
+        if asset == ConfluenceAsset.FOLDERS:
+            assert folder_ids is not None
+            yield from self.folders(folder_ids)
+
+        elif asset == ConfluenceAsset.DATABASES:
+            assert database_ids is not None
+            yield from self.databases(database_ids)
+
+        elif asset == ConfluenceAsset.PAGES:
             yield from self.pages()
 
         elif asset == ConfluenceAsset.USERS:

castor_extractor/knowledge/confluence/client/client_test.py

@@ -3,6 +3,36 @@ from unittest.mock import MagicMock, patch
 from .client import ConfluenceClient
 
 
+def test_ConfluenceClient_folders():
+    """
+    Folder 1 -> Page A -> Folder 2 -> Folder 3 -> Folder 4 -> Page B
+             -> Page C -> Folder 5 -> Page D
+
+    After extracting the pages, we should have all IDs of folders that are
+    immediate parents of pages. We still need to look out for nested folders.
+    """
+    folder_ids = {"1", "3", "4", "5"}
+    mock_responses = {
+        "1": {"id": "1", "parentType": None, "parentId": None},
+        "2": {"id": "2", "parentType": "page", "parentId": "A"},
+        "3": {"id": "3", "parentType": "folder", "parentId": "2"},
+        "4": {"id": "4", "parentType": "folder", "parentId": "3"},
+        "5": {"id": "5", "parentType": "page", "parentId": "C"},
+    }
+
+    def mock_get(endpoint):
+        folder_id = endpoint.split("/")[-1]
+        return mock_responses[folder_id]
+
+    client = ConfluenceClient(credentials=MagicMock())
+
+    with patch.object(client, "_get", side_effect=mock_get):
+        result = list(client.folders(folder_ids))
+
+    assert len(result) == 5
+    assert {folder["id"] for folder in result} == set(mock_responses.keys())
+
+
 def test_ConfluenceClient_filtered_spaces_with_allowlist():
     both_blocked_and_allowed_space_id = "789"
     archived_space_id = "934"

castor_extractor/knowledge/confluence/client/endpoints.py

@@ -5,10 +5,28 @@ class ConfluenceEndpointFactory:
     """
 
     API = "wiki/api/v2/"
+    DATABASE = "databases"
+    FOLDERS = "folders"
     PAGES = "pages"
     SPACES = "spaces"
     USERS = "users-bulk"
 
+    @classmethod
+    def database(cls, database_id: str) -> str:
+        """
+        Endpoint to fetch a database by id.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-database/#api-databases-id-get
+        """
+        return f"{cls.API}{cls.DATABASE}/{database_id}"
+
+    @classmethod
+    def folder(cls, folder_id: str) -> str:
+        """
+        Endpoint to fetch a folder by id.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-folder/#api-folders-id-get
+        """
+        return f"{cls.API}{cls.FOLDERS}/{folder_id}"
+
     @classmethod
     def pages(cls, space_id: str) -> str:
         """

castor_extractor/knowledge/confluence/extract.py

@@ -13,6 +13,7 @@ from ...utils import (
 )
 from .assets import ConfluenceAsset
 from .client import ConfluenceClient, ConfluenceCredentials
+from .utils import pages_to_database_ids, pages_to_folder_ids
 
 logger = logging.getLogger(__name__)
 
@@ -32,6 +33,16 @@ def iterate_all_data(
     yield ConfluenceAsset.PAGES, pages
     logger.info(f"Extracted {len(pages)} pages from API")
 
+    folder_ids = pages_to_folder_ids(pages)
+    logger.info("Extracting FOLDERS from API")
+    folders = list(deep_serialize(client.folders(folder_ids)))
+    yield ConfluenceAsset.FOLDERS, folders
+
+    database_ids = pages_to_database_ids(pages)
+    logger.info("Extracting DATABASES from API")
+    databases = list(deep_serialize(client.databases(database_ids)))
+    yield ConfluenceAsset.DATABASES, databases
+
 
 def extract_all(**kwargs) -> None:
     """

castor_extractor/knowledge/confluence/utils.py

@@ -0,0 +1,12 @@
+def pages_to_folder_ids(pages: list[dict]) -> set:
+    """Returns all unique folder parents."""
+    return {
+        page["parentId"] for page in pages if page["parentType"] == "folder"
+    }
+
+
+def pages_to_database_ids(pages: list[dict]) -> set:
+    """Returns all unique database parents."""
+    return {
+        page["parentId"] for page in pages if page["parentType"] == "database"
+    }

castor_extractor/knowledge/confluence/utils_test.py

@@ -0,0 +1,30 @@
+from .utils import pages_to_database_ids, pages_to_folder_ids
+
+
+def test_pages_to_folder_ids():
+    """Test the pages_to_folder_ids function."""
+    pages = [
+        {"id": "9", "parentId": None, "parentType": None},
+        {"id": "8", "parentId": "2", "parentType": "folder"},
+        {"id": "7", "parentId": "9", "parentType": "page"},
+        {"id": "6", "parentId": "4", "parentType": "folder"},
+        {"id": "5", "parentId": "4", "parentType": "folder"},
+    ]
+    expected = {"2", "4"}
+    result = pages_to_folder_ids(pages)
+    assert result == expected
+
+
+def test_pages_to_database_id():
+    """Test the pages_to_database_id function."""
+    pages = [
+        {"id": "1", "parentId": "db1", "parentType": "database"},
+        {"id": "2", "parentId": "db2", "parentType": "database"},
+        {"id": "3", "parentId": "4", "parentType": "folder"},
+        {"id": "4", "parentId": None, "parentType": None},
+        {"id": "5", "parentId": "db1", "parentType": "database"},
+        {"id": "6", "parentId": "9", "parentType": "page"},
+    ]
+    expected = {"db1", "db2"}
+    result = pages_to_database_ids(pages)
+    assert result == expected

castor_extractor/{utils → transformation}/dbt/client_test.py

@@ -9,9 +9,7 @@ from dateutil.tz import tzutc
 from .client import ContentType, DbtClient, DbtRun, _account_url  # type: ignore
 from .credentials import DbtCredentials
 
-_DBT_CLIENT_PATH = (
-    "source.packages.extractor.castor_extractor.utils.dbt.client.DbtClient"
-)
+_DBT_CLIENT_PATH = "source.packages.extractor.castor_extractor.transformation.dbt.client.DbtClient"
 _OLD_DATE = datetime(2023, 7, 10, 12, 6, 23, 109171, tzinfo=tzutc())
 _OLD_DATE_STR = "2023-07-10 12:06:23.109171+00:00"
 _RECENT_DATE = datetime(2023, 10, 6, 5, 9, 31, 731991, tzinfo=tzutc())

castor_extractor/utils/__init__.py

@@ -52,7 +52,7 @@ from .time import (
     yesterday,
 )
 from .type import Callback, Getter, JsonType, SerializedAsset
-from .validation import validate_baseurl
+from .validation import clean_path, validate_baseurl
 from .write import (
     get_output_filename,
     get_summary_filename,

castor_extractor/utils/validation.py

@@ -14,7 +14,7 @@ def _preprocess_url(base_url: str) -> str:
     return base_url.strip()
 
 
-def _clean_path(path: str) -> str:
+def clean_path(path: str) -> str:
     return path.rstrip("/")
 
 
@@ -44,7 +44,7 @@ def _urlsplit(base_url: str) -> tuple[str, str, str, str, str, str]:
     url = urlsplit(base_url)
 
     hostname, port = _get_hostname_port(url.netloc)
-    path = _clean_path(url.path)
+    path = clean_path(url.path)
 
     return url.scheme, hostname, path, port, url.query, url.fragment
 

castor_extractor/visualization/powerbi/__init__.py

@@ -1,3 +1,9 @@
 from .assets import PowerBiAsset
-from .client import DEFAULT_SCOPE, PowerbiClient, PowerbiCredentials
+from .client import (
+    CLIENT_APP_BASE,
+    DEFAULT_SCOPE,
+    REST_API_BASE_PATH,
+    PowerbiClient,
+    PowerbiCredentials,
+)
 from .extract import extract_all

castor_extractor/visualization/powerbi/client/__init__.py

@@ -1,2 +1,7 @@
 from .client import PowerbiClient
-from .credentials import DEFAULT_SCOPE, PowerbiCredentials
+from .credentials import (
+    CLIENT_APP_BASE,
+    DEFAULT_SCOPE,
+    REST_API_BASE_PATH,
+    PowerbiCredentials,
+)

castor_extractor/visualization/powerbi/client/authentication.py

@@ -9,7 +9,11 @@ from .endpoints import PowerBiEndpointFactory
 class PowerBiBearerAuth(BearerAuth):
     def __init__(self, credentials: PowerbiCredentials):
         self.credentials = credentials
-        authority = PowerBiEndpointFactory.authority(self.credentials.tenant_id)
+        endpoint_factory = PowerBiEndpointFactory(
+            login_url=self.credentials.login_url,
+            api_base=self.credentials.api_base,
+        )
+        authority = endpoint_factory.authority(self.credentials.tenant_id)
         self.app = msal.ConfidentialClientApplication(
             client_id=self.credentials.client_id,
             authority=authority,

castor_extractor/visualization/powerbi/client/client.py

@@ -40,6 +40,10 @@ class PowerbiClient(APIClient):
             auth=auth,
             timeout=POWERBI_DEFAULT_TIMEOUT_S,
         )
+        self.endpoint_factory = PowerBiEndpointFactory(
+            login_url=credentials.login_url,
+            api_base=credentials.api_base,
+        )
 
     def _activity_events(self, day: Optional[date] = None) -> Iterator[dict]:
         """
@@ -49,7 +53,7 @@ class PowerbiClient(APIClient):
         """
         request = partial(
             self._get,
-            endpoint=PowerBiEndpointFactory.activity_events(day),
+            endpoint=self.endpoint_factory.activity_events(day),
         )
         yield from fetch_all_pages(request, PowerBiPagination)
 
@@ -58,28 +62,28 @@ class PowerbiClient(APIClient):
         Returns a list of datasets for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/datasets-get-datasets-as-admin
         """
-        yield from self._get(PowerBiEndpointFactory.datasets())[Keys.VALUE]
+        yield from self._get(self.endpoint_factory.datasets())[Keys.VALUE]
 
     def _dashboards(self) -> Iterator[dict]:
         """
         Returns a list of dashboards for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/dashboards-get-dashboards-as-admin
         """
-        yield from self._get(PowerBiEndpointFactory.dashboards())[Keys.VALUE]
+        yield from self._get(self.endpoint_factory.dashboards())[Keys.VALUE]
 
     def _reports(self) -> Iterator[dict]:
         """
         Returns a list of reports for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/reports-get-reports-as-admin
         """
-        reports_endpoint = PowerBiEndpointFactory.reports()
+        reports_endpoint = self.endpoint_factory.reports()
         reports = self._get(reports_endpoint)[Keys.VALUE]
 
         for report in reports:
             report_id = report.get(Keys.ID)
 
             try:
-                pages_endpoint = PowerBiEndpointFactory.pages(report_id)
+                pages_endpoint = self.endpoint_factory.pages(report_id)
                 pages = self._get(pages_endpoint)[Keys.VALUE]
                 report["pages"] = pages
             except (requests.HTTPError, requests.exceptions.Timeout) as e:
@@ -99,14 +103,14 @@ class PowerbiClient(APIClient):
         }
 
         response = self._get(
-            PowerBiEndpointFactory.workspace_ids(),
+            self.endpoint_factory.workspace_ids(),
             params=params,
         )
 
         return [x[Keys.ID] for x in response]
 
     def _get_scan_result(self, scan_id: int) -> Iterator[dict]:
-        endpoint = PowerBiEndpointFactory.metadata_scan_result(scan_id)
+        endpoint = self.endpoint_factory.metadata_scan_result(scan_id)
         yield from self._get(endpoint)[Keys.WORKSPACES]
 
     def _wait_for_scan_result(self, scan_id: int) -> bool:
@@ -114,7 +118,7 @@ class PowerbiClient(APIClient):
         Periodically checks the status of the metadata scan until the results
         are ready.
         """
-        endpoint = PowerBiEndpointFactory.metadata_scan_status(scan_id)
+        endpoint = self.endpoint_factory.metadata_scan_status(scan_id)
         total_waiting_time_s = 0
 
         while total_waiting_time_s < POWERBI_DEFAULT_TIMEOUT_S:
@@ -152,7 +156,7 @@ class PowerbiClient(APIClient):
         }
         request_body = {"workspaces": workspaces_ids}
         scan_id = self._post(
-            PowerBiEndpointFactory.metadata_create_scan(),
+            self.endpoint_factory.metadata_create_scan(),
             params=params,
             data=request_body,
         )

castor_extractor/visualization/powerbi/client/client_test.py

@@ -6,13 +6,18 @@ import pytest
 from .authentication import msal
 from .client import PowerbiClient
 from .constants import Keys
-from .credentials import PowerbiCredentials
+from .credentials import CLIENT_APP_BASE, REST_API_BASE_PATH, PowerbiCredentials
 from .endpoints import PowerBiEndpointFactory
 
 FAKE_TENANT_ID = "IamFake"
 FAKE_CLIENT_ID = "MeTwo"
 FAKE_SECRET = "MeThree"
 
+ENDPOINT_FACTORY = PowerBiEndpointFactory(
+    login_url=CLIENT_APP_BASE,
+    api_base=REST_API_BASE_PATH + "/",  # superfluous "/" to test resiliency
+)
+
 
 @pytest.fixture
 def mock_msal():
@@ -55,7 +60,7 @@ def test__datasets(power_bi_client):
     with patch.object(power_bi_client, "_get") as mocked_get:
         mocked_get.return_value = {"value": [{"id": 1, "type": "dataset"}]}
         datasets = list(power_bi_client._datasets())
-        mocked_get.assert_called_with(PowerBiEndpointFactory.datasets())
+        mocked_get.assert_called_with(ENDPOINT_FACTORY.datasets())
         assert datasets == [{"id": 1, "type": "dataset"}]
 
 
@@ -63,7 +68,7 @@ def test__dashboards(power_bi_client):
     with patch.object(power_bi_client, "_get") as mocked_get:
         mocked_get.return_value = {"value": [{"id": 1, "type": "dashboard"}]}
         dashboards = list(power_bi_client._dashboards())
-        mocked_get.assert_called_with(PowerBiEndpointFactory.dashboards())
+        mocked_get.assert_called_with(ENDPOINT_FACTORY.dashboards())
         assert dashboards == [{"id": 1, "type": "dashboard"}]
 
 
@@ -79,8 +84,8 @@ def test__reports(power_bi_client):
         ]
         reports = list(power_bi_client._reports())
         calls = [
-            call(PowerBiEndpointFactory.reports()),
-            call(PowerBiEndpointFactory.pages("1")),
+            call(ENDPOINT_FACTORY.reports()),
+            call(ENDPOINT_FACTORY.pages("1")),
         ]
         mocked_get.assert_has_calls(calls)
         assert reports == [
@@ -107,7 +112,7 @@ def test__workspace_ids(power_bi_client):
         }
 
         mocked_get.assert_called_with(
-            PowerBiEndpointFactory.workspace_ids(),
+            ENDPOINT_FACTORY.workspace_ids(),
             params=params,
         )
 
@@ -160,7 +165,7 @@ def test__activity_events(power_bi_client):
         assert result == ["foo", "bar", "baz", "biz"]
 
         expected_calls = [
-            call(endpoint=PowerBiEndpointFactory.activity_events(day=day)),
+            call(endpoint=ENDPOINT_FACTORY.activity_events(day=day)),
             call(endpoint="https://next-call-1"),
             call(endpoint="https://next-call-2"),
         ]

castor_extractor/visualization/powerbi/client/credentials.py

@@ -6,6 +6,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 DEFAULT_SCOPE = "https://analysis.windows.net/powerbi/api/.default"
 POWERBI_ENV_PREFIX = "CASTOR_POWERBI_"
 
+CLIENT_APP_BASE = "https://login.microsoftonline.com"
+REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
+
 
 class PowerbiCredentials(BaseSettings):
     """Class to handle PowerBI rest API permissions"""
@@ -19,9 +22,21 @@ class PowerbiCredentials(BaseSettings):
     client_id: str
     tenant_id: str
     secret: str = Field(repr=False)
+    api_base: str = REST_API_BASE_PATH
+    login_url: str = CLIENT_APP_BASE
     scopes: list[str] = [DEFAULT_SCOPE]
 
     @field_validator("scopes", mode="before")
     @classmethod
     def _check_scopes(cls, scopes: Optional[list[str]]) -> list[str]:
         return scopes if scopes is not None else [DEFAULT_SCOPE]
+
+    @field_validator("login_url", mode="before")
+    @classmethod
+    def _check_login_url(cls, login_url: Optional[str]) -> str:
+        return login_url if login_url is not None else CLIENT_APP_BASE
+
+    @field_validator("api_base", mode="before")
+    @classmethod
+    def _check_api_base(cls, api_base: Optional[str]) -> str:
+        return api_base if api_base is not None else REST_API_BASE_PATH

castor_extractor/visualization/powerbi/client/endpoints.py

@@ -1,10 +1,7 @@
 from datetime import date, datetime
 from typing import Optional
 
-from ....utils import at_midnight, format_date, yesterday
-
-_CLIENT_APP_BASE = "https://login.microsoftonline.com"
-_REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
+from ....utils import at_midnight, clean_path, format_date, yesterday
 
 
 def _time_filter(day: Optional[date]) -> tuple[datetime, datetime]:
@@ -15,51 +12,44 @@ def _time_filter(day: Optional[date]) -> tuple[datetime, datetime]:
 
 
 class PowerBiEndpointFactory:
-    @classmethod
-    def activity_events(cls, day: Optional[date]) -> str:
+    def __init__(self, login_url: str, api_base: str):
+        self.app_base = clean_path(login_url)
+        self.rest_api_base = clean_path(api_base)
+
+    def activity_events(self, day: Optional[date]) -> str:
         start, end = _time_filter(day)
-        url = f"{_REST_API_BASE_PATH}/admin/activityevents"
+        url = f"{self.rest_api_base}/admin/activityevents"
         url += "?$filter=Activity eq 'viewreport'"
         url += f"&startDateTime='{format_date(start)}'"
         url += f"&endDateTime='{format_date(end)}'"
         return url
 
-    @classmethod
-    def authority(cls, tenant_id: str) -> str:
-        return f"{_CLIENT_APP_BASE}/{tenant_id}"
+    def authority(self, tenant_id: str) -> str:
+        return f"{self.app_base}/{tenant_id}"
 
-    @classmethod
-    def dashboards(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/dashboards"
+    def dashboards(self) -> str:
+        return f"{self.rest_api_base}/admin/dashboards"
 
-    @classmethod
-    def datasets(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/datasets"
+    def datasets(self) -> str:
+        return f"{self.rest_api_base}/admin/datasets"
 
-    @classmethod
-    def groups(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/groups"
+    def groups(self) -> str:
+        return f"{self.rest_api_base}/admin/groups"
 
-    @classmethod
-    def metadata_create_scan(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/getInfo"
+    def metadata_create_scan(self) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/getInfo"
 
-    @classmethod
-    def metadata_scan_result(cls, scan_id: int) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/scanResult/{scan_id}"
+    def metadata_scan_result(self, scan_id: int) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/scanResult/{scan_id}"
 
-    @classmethod
-    def metadata_scan_status(cls, scan_id: int) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/scanStatus/{scan_id}"
+    def metadata_scan_status(self, scan_id: int) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/scanStatus/{scan_id}"
 
-    @classmethod
-    def pages(cls, report_id: str) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/reports/{report_id}/pages"
+    def pages(self, report_id: str) -> str:
+        return f"{self.rest_api_base}/admin/reports/{report_id}/pages"
 
-    @classmethod
-    def reports(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/reports"
+    def reports(self) -> str:
+        return f"{self.rest_api_base}/admin/reports"
 
-    @classmethod
-    def workspace_ids(cls) -> str:
-        return f"{_REST_API_BASE_PATH}/admin/workspaces/modified"
+    def workspace_ids(self) -> str:
+        return f"{self.rest_api_base}/admin/workspaces/modified"

castor_extractor/visualization/strategy/client/client.py

@@ -34,9 +34,7 @@ _BATCH_SIZE: int = 100
 
 
 class URLTemplates(Enum):
-    DASHBOARD = (
-        "https://{hostname}/MicroStrategyLibrarySTD/app/{project_id}/{id_}"
-    )
+    DASHBOARD = "https://{hostname}/MicroStrategyLibrary/app/{project_id}/{id_}"
     DOCUMENT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?documentID={id_}&projectID={project_id}"
     REPORT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?reportID={id_}&projectID={project_id}"
     FOLDER = "https://{hostname}/MicroStrategy/servlet/mstrWeb?folderID={id_}&projectID={project_id}"

{castor_extractor-0.24.15.dist-info → castor_extractor-0.24.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.15
+Version: 0.24.20
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,26 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.20 - 2025-05-19
+
+* Powerbi: allow custom api base and login url
+
+## 0.24.19 - 2025-05-14
+
+* Confluence: extract databases
+
+## 0.24.18 - 2025-05-13
+
+* Improve folder organisation for transformation tools
+
+## 0.24.17 - 2025-05-13
+
+* Strategy: fix dashboard URL format
+
+## 0.24.16 - 2025-05-12
+
+* Confluence: extract folders to complete the page hierarchy
+
 ## 0.24.15 - 2025-05-12
 
 * Tableau: Add argument to skip columns extraction

{castor_extractor-0.24.15.dist-info → castor_extractor-0.24.20.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=6XUz09FfUZSXFfROVG7BliBupfyr5eeBy0J3cQZVvys,17398
+CHANGELOG.md,sha256=NxSwszCNlJ8oD2ffivq3g75DK436mue9WwilR0r6bE4,17760
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -17,7 +17,7 @@ castor_extractor/commands/extract_mode.py,sha256=Q4iO-VAKMg4zFPejhAO-foZibL5Ht3j
 castor_extractor/commands/extract_mysql.py,sha256=7AH5qMzeLTsENCOeJwtesrWg8Vo8MCEq8fx2YT74Mcw,1034
 castor_extractor/commands/extract_notion.py,sha256=uaxcF3_bT7D_-JxnIW0F7VVDphI_ZgOfQQxZzoLXo_M,504
 castor_extractor/commands/extract_postgres.py,sha256=pX0RnCPi4nw6QQ6wiAuZ_Xt3ZbDuMUG9aQKuqFgJtAU,1154
-castor_extractor/commands/extract_powerbi.py,sha256=f0G5w61KXExJ6Sw39_mJIwqQNpLorE5-LKmZXlUqvKI,783
+castor_extractor/commands/extract_powerbi.py,sha256=RKkw9H2ZsbJ4xLE84bmNFUgYUjlrLmSXahQSVrQr_Bc,934
 castor_extractor/commands/extract_qlik.py,sha256=VBe_xFKh_nR0QSFFIncAaC8yDqBeMa6VunBAga7AeGg,891
 castor_extractor/commands/extract_redshift.py,sha256=zRBg2D_ft4GLdPSdmetRcgQVAA80DXtdRSYsQhAWIik,1334
 castor_extractor/commands/extract_salesforce.py,sha256=3j3YTmMkPAwocR-B1ozJQai0UIZPtpmAyWj-hHvdWn4,1226
@@ -43,14 +43,16 @@ castor_extractor/file_checker/templates/__init__.py,sha256=StVLm4ZGyGVmPzarxEaDR
 castor_extractor/file_checker/templates/generic_warehouse.py,sha256=S5qFIkbfihdWz16y4HSiTfDH1SmGl40u3kZ706sFBVI,2939
 castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/knowledge/confluence/__init__.py,sha256=pRT615pMDlB7Ifs09erVn2EdpZHgkvX5selemWU3VPE,129
-castor_extractor/knowledge/confluence/assets.py,sha256=zv2G2LB8H0fKDbVJ4kHrAjbqehXI_K-wgd_ghSXGFvs,144
+castor_extractor/knowledge/confluence/assets.py,sha256=ZRmRnJpbujL8LMQD3FrcFME-pEQM7G1lCZ1abTsx5OU,196
 castor_extractor/knowledge/confluence/client/__init__.py,sha256=ALAzo0JEhxFzH2FnIO6HmtkAGS2_bGY8KXXMcTGV3aE,84
-castor_extractor/knowledge/confluence/client/client.py,sha256=ihdagtAEgIcO5MmX5-coGEJkUg7_Tw1_7Vl50NDorhE,3731
-castor_extractor/knowledge/confluence/client/client_test.py,sha256=LTT49ORl0DPTdDpKdREUErnwIA40xPy2C3uwdkVS1I0,3071
+castor_extractor/knowledge/confluence/client/client.py,sha256=Ysl4KDo4ISXuLvUwEZrowGiaUHPC5cli7zN5Vtte_Jc,5392
+castor_extractor/knowledge/confluence/client/client_test.py,sha256=O4YarFiZbm2z_H0zPzPi-awRhsGEY1iOwwHHTv9gCVA,4177
 castor_extractor/knowledge/confluence/client/credentials.py,sha256=tqUMw-SVoAi4o6I6OeGk4MeDiIPU3-ihhaomXv4CQ64,419
-castor_extractor/knowledge/confluence/client/endpoints.py,sha256=ClBzE8a5zqA4ngAecc8vMv9QJCdbtcv3GKuorZ8kOdA,1100
+castor_extractor/knowledge/confluence/client/endpoints.py,sha256=q5JCybVPtSs4rEXmS5IdkJyFQZsx2ff838mJyxryEFo,1742
 castor_extractor/knowledge/confluence/client/pagination.py,sha256=ty4meiMEujDVSiQyOJTibd-ReYyDyGezdFuk7EAGtMA,862
-castor_extractor/knowledge/confluence/extract.py,sha256=mOAs5uvjM0LZJzrD36uFAt_nsiqQ48kKTDFIKru3LSo,1858
+castor_extractor/knowledge/confluence/extract.py,sha256=Ba1IDDFkZUYJ2HDWNcXNJ1fzFYOTcE1sTDgqFAK9-fA,2332
+castor_extractor/knowledge/confluence/utils.py,sha256=-lcpWY8oacDdg537s5-LsY8B3L7ZKGXT6GCd3yvVw-A,392
+castor_extractor/knowledge/confluence/utils_test.py,sha256=NYlbKUSRjuZ8FaRMqhKNQCW3KmMAhqIVaMAHNNpobsk,1152
 castor_extractor/knowledge/notion/__init__.py,sha256=ZDmh0eNSxHf1zVPm0aYlKPci-vzOXhAgdsWjS2hdjh4,117
 castor_extractor/knowledge/notion/assets.py,sha256=QHv1-pomt5UeN_prP2L6t_zJ-tDSqB8LgopkGAODYPQ,164
 castor_extractor/knowledge/notion/client/__init__.py,sha256=CDPorBCethuNTEtpjvHGcWnWeVfqkEq-IbakWjDKATw,76
@@ -80,6 +82,11 @@ castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9
 castor_extractor/transformation/coalesce/client/type.py,sha256=oiiVP9NL0ijTXyQmaB8aJVYckc7m-m8ZgMyNIAduUKE,43
 castor_extractor/transformation/coalesce/client/utils.py,sha256=jbxh3OCbYm3fKZD1QfqX5zm1ZD_jFIrpUQsX8paRP7g,1627
 castor_extractor/transformation/coalesce/client/utils_test.py,sha256=Q00Y1n0Q_sZ0LFnYn98yDGFumBsifzVJSc7_3PSBMfI,1543
+castor_extractor/transformation/dbt/__init__.py,sha256=LHQROlMqYWCc7tcmhdjXtROFpJqUvCg9jPC8avHgD4I,107
+castor_extractor/transformation/dbt/assets.py,sha256=JY1nKEGySZ84wNoe7dnizwAYw2q0t8NVaIfqhB2rSw0,148
+castor_extractor/transformation/dbt/client.py,sha256=KSlMHaesDOTYjTKs_ZFSYG1Udxjj5jn0m5zaCf5AZZk,5526
+castor_extractor/transformation/dbt/client_test.py,sha256=YK86romOhTRFqzA2Gs262mDUfmvz0cmhiN3jC9nB8P8,4527
+castor_extractor/transformation/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
 castor_extractor/types.py,sha256=nHel2hv6NoHmdpOX_heEfO2-DnZPoYA2x0eJdbFvT0s,1276
 castor_extractor/uploader/__init__.py,sha256=A4bq_SrEtKAsl0r_D_duSTvL5WIQjVfsMy7tDx9IKg0,87
 castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
@@ -89,7 +96,7 @@ castor_extractor/uploader/settings.py,sha256=3MvOX-UFRqrLZoiT7wYn9jUGro7NX4RCafY
 castor_extractor/uploader/upload.py,sha256=PSQfkO_7LSE0WBo9Tm_hlS2ONepKeB0cBFdJXySnues,4310
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=otAaySj5aeem6f0CTd0Te6ioJ6uP2J1p348j-SdIwDI,802
-castor_extractor/utils/__init__.py,sha256=KQkr_CmxWG0Vpu7CaqjbJkffUeEWcyeA9Cbm394Hygk,1585
+castor_extractor/utils/__init__.py,sha256=ybzci46Myi9LABZGBK2qXOiGd00llcpmBpvAp5LEVHc,1597
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
 castor_extractor/utils/batch.py,sha256=SFlLmJgVjV2nVhIrjVIEp8wJ9du4dKKHq8YVYubnwQQ,448
@@ -114,11 +121,6 @@ castor_extractor/utils/client/uri_test.py,sha256=1XKF6qSseCeD4G4ckaNO07JXfGbt7XU
 castor_extractor/utils/collection.py,sha256=FiIJWZZ865oqNjtTm40gQ13R9zh--W2W5YsMBZJf2bk,2334
 castor_extractor/utils/collection_test.py,sha256=XJAGo0Veg0H8wZRCESIkU2t8bXxTNET0BdosomO3-Ls,2104
 castor_extractor/utils/constants.py,sha256=qBQprS9U66mS-RIBXiLujdTSV3WvGv40Bc0khP4Abdk,39
-castor_extractor/utils/dbt/__init__.py,sha256=LHQROlMqYWCc7tcmhdjXtROFpJqUvCg9jPC8avHgD4I,107
-castor_extractor/utils/dbt/assets.py,sha256=JY1nKEGySZ84wNoe7dnizwAYw2q0t8NVaIfqhB2rSw0,148
-castor_extractor/utils/dbt/client.py,sha256=KSlMHaesDOTYjTKs_ZFSYG1Udxjj5jn0m5zaCf5AZZk,5526
-castor_extractor/utils/dbt/client_test.py,sha256=9mHhFLyQ-NrjyRKy0kCEp4hgMj2um7HrhTd452oyRbM,4526
-castor_extractor/utils/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
 castor_extractor/utils/deprecate.py,sha256=aBIN2QqZUx5CBNZMFfOUhi8QqtPqRcJtmrN6xqfm-y8,805
 castor_extractor/utils/env.py,sha256=TqdtB50U8LE0993WhhEhpy89TJrHbjtIKjvg6KQ-5q0,596
 castor_extractor/utils/files.py,sha256=qKbfu5FRjsQdKnRmaJNd5EdX_F6gf5C5tV8LdoYKxs0,1527
@@ -153,7 +155,7 @@ castor_extractor/utils/string_test.py,sha256=u3P2tAPhyfCLvD19rH_JcpHhPuWTHUdg0z_
 castor_extractor/utils/time.py,sha256=jmP1QWg4lv21Jp_Oy71lfJ47hjNOSgHiBOFf964RMPU,1732
 castor_extractor/utils/time_test.py,sha256=pH8DSosNlwDYZXZNNjYDcL0WbmZc_c212LEEn88Oqew,647
 castor_extractor/utils/type.py,sha256=Sd8JlEgbGkBUZnRqCUDtREeBkOMTXtlNMyCph90_J0Q,328
-castor_extractor/utils/validation.py,sha256=kQAFtqt3gfy7YqYQ0u-60vyNYUF_96he5QDVUQnZmDo,1896
+castor_extractor/utils/validation.py,sha256=dRvC9SoFVecVZuLQNN3URq37yX2sBSW3-NxIxkcol5o,1894
 castor_extractor/utils/validation_test.py,sha256=A7P6VmI0kYX2aGIeEN12y7LsY7Kpm8pE4bdVFhbBAMw,1184
 castor_extractor/utils/write.py,sha256=Z_RYm47XeHiUPPUMYMuAjQrVZ18CAkL3daQHQG1XPlM,2148
 castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -231,16 +233,16 @@ castor_extractor/visualization/mode/client/constants.py,sha256=_Si5AF6VnpoSfnNNg
 castor_extractor/visualization/mode/client/credentials.py,sha256=ptIpCCpoNt06yYaWQgl3Xu78_jVMoqsqWAGqQXVFZlo,606
 castor_extractor/visualization/mode/errors.py,sha256=SKpFT2AiLOuWx2VRLyO7jbAiKcGDFXXrsebpNEKtr0E,1495
 castor_extractor/visualization/mode/extract.py,sha256=PmLWWjUwplQh3TNMemiGwyFdxMcKVMvumZPxSMLJAwk,1625
-castor_extractor/visualization/powerbi/__init__.py,sha256=AJnmfdmm2mGaInWJkUfZxRqrI7dBkTUSebpow05g5zo,135
+castor_extractor/visualization/powerbi/__init__.py,sha256=hoZ73ngLhMc9edqxO9PUIE3FABQlvcfY2W8fuc6DEjY,197
 castor_extractor/visualization/powerbi/assets.py,sha256=IB_XKwgdN1pZYGZ4RfeHrLjflianTzWf_6tg-4CIwu0,742
-castor_extractor/visualization/powerbi/client/__init__.py,sha256=8Bzhd9Z0ebVg2gDchXCOPa80Yqlq_9oCjbGi8u1M6J0,93
-castor_extractor/visualization/powerbi/client/authentication.py,sha256=fz0v9qxeADwA1jiS9UzAQN5mA5kmZT53onlcWon2RGw,892
-castor_extractor/visualization/powerbi/client/client.py,sha256=Y_rwkyPO3GbTAtaOClqnGVEK5iWmREIuDKDJx0LefHs,7203
-castor_extractor/visualization/powerbi/client/client_test.py,sha256=6NtpcKZCxBWyJO3phnVgE70Wmunb6tWsdXikkReJ02E,5539
+castor_extractor/visualization/powerbi/client/__init__.py,sha256=UPIhMaCCdNxhiLdkItC0IPFE_AMi-SgqI_ahwjB9utI,151
+castor_extractor/visualization/powerbi/client/authentication.py,sha256=cTohunKr1nUDfvxB0sejJSyfE2BdCtwT1WMPecWlbyU,1045
+castor_extractor/visualization/powerbi/client/client.py,sha256=MbqqUF4yadjbAQ_I0iwOmdqR0qC1L3yfbDGgIZqZ0hQ,7348
+castor_extractor/visualization/powerbi/client/client_test.py,sha256=MhQfg6kj__zpARXfv9-VrJXt1fXj6Eri91y8KA9Sn9E,5694
 castor_extractor/visualization/powerbi/client/constants.py,sha256=88R_aGachNNUZh6OSH2fkDwZtY4KTStzKm_g7HNCqqo,387
-castor_extractor/visualization/powerbi/client/credentials.py,sha256=sVi4ecJP8ydfrGRKKdJML-wxxZjxUshQtyqTUaJYq_g,795
+castor_extractor/visualization/powerbi/client/credentials.py,sha256=OVWdhZSNODzTdLysY-sbpBZ3uUkLokeayQZnbJAqt2I,1386
 castor_extractor/visualization/powerbi/client/credentials_test.py,sha256=TzFqxsWVQ3sXR_n0bJsexK9Uz7ceXCEPVqDGWTJzW60,993
-castor_extractor/visualization/powerbi/client/endpoints.py,sha256=DrAFpYHhp9Z7fxebdy_Ir6LFfFluHRBJId4tVCYTUVs,2051
+castor_extractor/visualization/powerbi/client/endpoints.py,sha256=38ZETzSSnNq3vA9O6nLZQ8T1BVE01R9CjMC03-PRXsM,1911
 castor_extractor/visualization/powerbi/client/pagination.py,sha256=OZMjoDQPRGMoWd9QcKKrPh3aErJR20SHlrTqY_siLkk,755
 castor_extractor/visualization/powerbi/extract.py,sha256=Z5KbqMhMnqjWcnzged2G1-Gf6GYWJobTL9_TpAdgb8o,1309
 castor_extractor/visualization/qlik/__init__.py,sha256=u6lIfm_WOykBwt6SlaB7C0Dtx37XBliUbM5oWv26gC8,177
@@ -277,7 +279,7 @@ castor_extractor/visualization/sigma/extract.py,sha256=XIT1qsj6g6dgBWP8HPfj_medZ
 castor_extractor/visualization/strategy/__init__.py,sha256=HOMv4JxqF5ZmViWi-pDE-PSXJRLTdXal_jtpHG_rlR8,123
 castor_extractor/visualization/strategy/assets.py,sha256=tqB3GOtp-r7IOnYO8UxZgrldoSMImJnv5KeIwDFxg68,302
 castor_extractor/visualization/strategy/client/__init__.py,sha256=XWP0yF5j6JefDJkDfX-RSJn3HF2ceQ0Yx1PLCfB3BBo,80
-castor_extractor/visualization/strategy/client/client.py,sha256=F7taX0jSQpM8R3GOGeUQ7U_bJKkoHTwAc9oyc3ZDxbM,10261
+castor_extractor/visualization/strategy/client/client.py,sha256=_K7JkatG0DYtbQOJULTNYKHWuBZ11KMR_rQjx8LiR5c,10242
 castor_extractor/visualization/strategy/client/credentials.py,sha256=urFfNxWX1JG6wwFMYImufQzHa5g-sgjdlVGzi63owwg,1113
 castor_extractor/visualization/strategy/extract.py,sha256=2fBuvS2xiOGXRpxXnZsE_C3en6t1-BlM5TbusjHyEkg,1166
 castor_extractor/visualization/tableau/__init__.py,sha256=eFI_1hjdkxyUiAYiy3szwyuwn3yJ5C_KbpBU0ySJDcQ,138
@@ -424,8 +426,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.15.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.15.dist-info/METADATA,sha256=ktAgO-d5jJmInoD_VCLwIT522Qy31paP3Smh_TGa6MI,24851
-castor_extractor-0.24.15.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.15.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.15.dist-info/RECORD,,
+castor_extractor-0.24.20.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.20.dist-info/METADATA,sha256=DvgjbhUmiYXzV9e3MXsEGuSfgOog38LrEBNmqFIAcyI,25213
+castor_extractor-0.24.20.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.20.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.20.dist-info/RECORD,,