gooddata-pipelines 1.47.1.dev1__py3-none-any.whl

gooddata_pipelines/api/gooddata_sdk.py

@@ -0,0 +1,374 @@
+# (C) 2025 GoodData Corporation
+
+"""Interaction with GoodData Cloud via the Gooddata Python SDK."""
+
+from pathlib import Path
+from typing import Callable
+
+from gooddata_sdk.catalog.permission.declarative_model.permission import (
+    CatalogDeclarativeWorkspacePermissions,
+)
+from gooddata_sdk.catalog.user.entity_model.user import CatalogUser
+from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.workspace import (
+    CatalogDeclarativeWorkspaceDataFilters,
+)
+from gooddata_sdk.catalog.workspace.entity_model.user_data_filter import (
+    CatalogUserDataFilter,
+)
+from gooddata_sdk.catalog.workspace.entity_model.workspace import (
+    CatalogWorkspace,
+)
+from gooddata_sdk.sdk import GoodDataSdk
+
+from gooddata_pipelines.api.utils import raise_with_context
+
+
+def apply_to_all_methods(decorator: Callable) -> Callable:
+    def decorate(cls: type) -> type:
+        for attr in cls.__dict__:
+            if callable(getattr(cls, attr)) and not attr.startswith("__"):
+                setattr(cls, attr, decorator(getattr(cls, attr)))
+        return cls
+
+    return decorate
+
+
+@apply_to_all_methods(raise_with_context())
+class SdkMethods:
+    """
+    Class to intaract with GoodData Cloud via the Gooddata Python SDK.
+    """
+
+    _sdk: GoodDataSdk
+
+    def get_organization_id(self) -> str:
+        return self._sdk.catalog_organization.organization_id
+
+    def check_workspace_exists(self, workspace_id: str) -> bool:
+        try:
+            self._sdk.catalog_workspace.get_workspace(workspace_id)
+            return True
+        except Exception:
+            return False
+
+    def get_workspace(self, workspace_id: str, **_: str) -> CatalogWorkspace:
+        """
+        Calls GoodData Python SDK to retrieve a workspace by its ID.
+
+        Args:
+            workspace_id (str): The ID of the workspace to retrieve.
+        Returns:
+            CatalogWorkspace: The workspace object retrieved from the SDK.
+        Raises:
+            GoodDataApiException: If the workspace cannot be retrieved, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_workspace.get_workspace(workspace_id)
+
+    def delete_panther_workspace(self, workspace_id: str) -> None:
+        """
+        Calls GoodData Python SDK to delete a workspace by its ID.
+
+        Args:
+            workspace_id (str): The ID of the workspace to delete.
+        Raises:
+            GoodDataApiException: If the workspace cannot be deleted, an exception
+            is raised with additional context information.
+        """
+        self._sdk.catalog_workspace.delete_workspace(workspace_id)
+
+    def create_or_update_panther_workspace(
+        self,
+        workspace_id: str,
+        workspace_name: str,
+        parent_id: str | None,
+        **_: str,
+    ) -> None:
+        """
+        Calls GoodData Python SDK to create or update a workspace with the given ID,
+        name, and parent ID.
+
+        Args:
+            workspace_id (str): The ID of the workspace to create or update.
+            workspace_name (str): The name of the workspace.
+            parent_id (str | None): The ID of the parent workspace, if any.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the workspace cannot be created or updated,
+            an exception is raised with additional context information.
+        """
+        return self._sdk.catalog_workspace.create_or_update(
+            CatalogWorkspace(
+                workspace_id=workspace_id,
+                name=workspace_name,
+                parent_id=parent_id,
+            )
+        )
+
+    def get_panther_children_workspaces(
+        self, parent_workspace_ids: set[str]
+    ) -> list[CatalogWorkspace]:
+        """
+        Calls GoodData Python SDK to retrieve all workspaces in domain and filters the
+        result by the set of parent workspace IDs.
+
+        Args:
+            parent_workspace_ids (set[str]): A set of parent workspace IDs to filter
+                child workspaces.
+        Returns:
+            list[CatalogWorkspace]: List of child workspaces in the parent workspace.
+        """
+        all_workspaces: list[CatalogWorkspace] = self.list_workspaces()
+
+        children: list[CatalogWorkspace] = [
+            workspace
+            for workspace in all_workspaces
+            if workspace.parent_id in parent_workspace_ids
+        ]
+
+        return children
+
+    def list_workspaces(self) -> list[CatalogWorkspace]:
+        """Retrieves all workspaces in the GoodData Cloud domain.
+
+        Returns:
+            list[CatalogWorkspace]: A list of all workspaces in the domain.
+        Raises:
+            GoodDataApiException: If the workspaces cannot be retrieved, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_workspace.list_workspaces()
+
+    def get_declarative_permissions(
+        self, workspace_id: str
+    ) -> CatalogDeclarativeWorkspacePermissions:
+        """
+        Retrieves the declarative permissions for a given workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace for which to retrieve
+                permissions.
+        Returns:
+            CatalogDeclarativeWorkspacePermissions: The declarative permissions
+                for the workspace.
+        Raises:
+            GoodDataApiException: If the permissions cannot be retrieved, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_permission.get_declarative_permissions(
+            workspace_id
+        )
+
+    def put_declarative_permissions(
+        self,
+        workspace_id: str,
+        ws_permissions: CatalogDeclarativeWorkspacePermissions,
+    ) -> None:
+        """
+        Updates the declarative permissions for a given workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace for which to update
+                permissions.
+            ws_permissions (CatalogDeclarativeWorkspacePermissions): The new
+                declarative permissions to set for the workspace.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the permissions cannot be updated, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_permission.put_declarative_permissions(
+            workspace_id, ws_permissions
+        )
+
+    def get_user(self, user_id: str, **_: str) -> CatalogUser:
+        """
+        Calls GoodData Python SDK to retrieve a user by its ID.
+
+        Args:
+            user_id (str): The ID of the user to retrieve.
+        Returns:
+            CatalogUser: The user object retrieved from the SDK.
+        Raises:
+            GoodDataApiException: If the user cannot be retrieved, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_user.get_user(user_id)
+
+    def create_or_update_user(self, user: CatalogUser, **_: str) -> None:
+        """
+        Calls GoodData Python SDK to create or update a user.
+
+        Args:
+            user (CatalogUser): The user object to create or update.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the user cannot be created or updated,
+            an exception is raised with additional context information.
+        """
+        return self._sdk.catalog_user.create_or_update_user(user)
+
+    def delete_user(self, user_id: str, **_: str) -> None:
+        """
+        Calls GoodData Python SDK to delete a user by its ID.
+
+        Args:
+            user_id (str): The ID of the user to delete.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the user cannot be deleted, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_user.delete_user(user_id)
+
+    def get_user_group(self, user_group_id: str, **_: str) -> CatalogUserGroup:
+        """
+        Calls GoodData Python SDK to retrieve a user group by its ID.
+
+        Args:
+            user_group_id (str): The ID of the user group to retrieve.
+        Returns:
+            CatalogUserGroup: The user group object retrieved from the SDK.
+        Raises:
+            GoodDataApiException: If the user group cannot be retrieved, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_user.get_user_group(user_group_id)
+
+    def list_user_groups(self) -> list[CatalogUserGroup]:
+        """
+        Calls GoodData Python SDK to retrieve all user groups.
+
+        Returns:
+            list[CatalogUserGroup]: A list of all user groups in the domain.
+        Raises:
+            GoodDataApiException: If the user groups cannot be retrieved, an
+            exception is raised with additional context information.
+        """
+        return self._sdk.catalog_user.list_user_groups()
+
+    def list_users(self) -> list[CatalogUser]:
+        """Calls GoodData Python SDK to retrieve all users.
+
+        Returns:
+            list[CatalogUser]: A list of all users in the domain.
+        """
+        return self._sdk.catalog_user.list_users()
+
+    def create_or_update_user_group(
+        self, catalog_user_group: CatalogUserGroup, **_: str
+    ) -> None:
+        """Calls GoodData Python SDK to create or update a user group.
+
+        Args:
+            catalog_user_group (CatalogUserGroup): The user group object to create or update.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the user group cannot be created or updated,
+            an exception is raised with additional context information.
+        """
+        return self._sdk.catalog_user.create_or_update_user_group(
+            catalog_user_group
+        )
+
+    def delete_user_group(self, user_group_id: str) -> None:
+        """Calls GoodData Python SDK to delete a user group by its ID.
+
+        Args:
+            user_group_id (str): The ID of the user group to delete.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the user group cannot be deleted, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_user.delete_user_group(user_group_id)
+
+    def get_declarative_workspace_data_filters(
+        self,
+    ) -> CatalogDeclarativeWorkspaceDataFilters:
+        """Retrieves the declarative workspace data filters.
+
+        Returns:
+            CatalogDeclarativeWorkspaceDataFilters: The declarative workspace data filters.
+        Raises:
+            GoodDataApiException: If the declarative workspace data filters cannot be retrieved,
+            an exception is raised with additional context information.
+        """
+        return (
+            self._sdk.catalog_workspace.get_declarative_workspace_data_filters()
+        )
+
+    def list_user_data_filters(
+        self, workspace_id: str
+    ) -> list[CatalogUserDataFilter]:
+        """Lists all user data filters for a given workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace for which to list user data filters.
+        Returns:
+            list[CatalogUserDataFilter]: A list of user data filters for the specified workspace.
+        Raises:
+            GoodDataApiException: If the user data filters cannot be listed, an exception
+            is raised with additional context information.
+        """
+        return self._sdk.catalog_workspace.list_user_data_filters(workspace_id)
+
+    def delete_user_data_filter(
+        self, workspace_id: str, user_data_filter_id: str
+    ) -> None:
+        """Deletes a user data filter by its ID in the specified workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace containing the user data filter.
+            user_data_filter_id (str): The ID of the user data filter to delete.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the user data filter cannot be deleted, an exception
+            is raised with additional context information.
+        """
+        self._sdk.catalog_workspace.delete_user_data_filter(
+            workspace_id, user_data_filter_id
+        )
+
+    def create_or_update_user_data_filter(
+        self, workspace_id: str, user_data_filter: CatalogUserDataFilter
+    ) -> None:
+        """Creates or updates a user data filter in the specified workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace where the user data filter
+                should be created or updated.
+            user_data_filter (CatalogUserDataFilter): The user data filter object to create or update.
+        Returns:
+            None
+        Raises:
+            GoodDataApiException: If the user data filter cannot be created or updated,
+            an exception is raised with additional context information.
+        """
+        self._sdk.catalog_workspace.create_or_update_user_data_filter(
+            workspace_id, user_data_filter
+        )
+
+    def store_declarative_workspace(
+        self, workspace_id: str, export_path: Path
+    ) -> None:
+        """Stores the declarative workspace in the specified export path."""
+        self._sdk.catalog_workspace.store_declarative_workspace(
+            workspace_id, export_path
+        )
+
+    def store_declarative_filter_views(
+        self, workspace_id: str, export_path: Path
+    ) -> None:
+        """Stores the declarative filter views in the specified export path."""
+        self._sdk.catalog_workspace.store_declarative_filter_views(
+            workspace_id, export_path
+        )

gooddata_pipelines/api/utils.py

@@ -0,0 +1,43 @@
+# (C) 2025 GoodData Corporation
+
+"""Utility functions for GoodData Cloud API interactions."""
+
+from typing import Any, Callable
+
+from gooddata_api_client import ApiException  # type: ignore
+
+from gooddata_pipelines.api.exceptions import GoodDataApiException
+
+
+def raise_with_context(**context_kwargs: str) -> Callable:
+    """
+    Decorator to catch exceptions raised by SDK methods and raise a GoodDataApiException
+    with additional context information.
+
+    Args:
+        context_kwargs (dict): Additional context information to include in the
+            GoodDataApiException.
+    """
+
+    def decorator(fn: Callable) -> Callable:
+        def wrapper(*method_args: Any, **method_kwargs: Any) -> Callable:
+            try:
+                return fn(*method_args, **method_kwargs)
+            except Exception as e:
+                # Process known exceptions
+                if isinstance(e, ApiException):
+                    context_kwargs["http_status"] = f"{e.status} {e.reason}"
+                    exception_content = e.body
+                else:
+                    exception_content = str(e)
+
+                # Format the exception message: "{exception_type}: {exception_content}"
+                message = f"{type(e).__name__}: {exception_content}"
+
+                raise GoodDataApiException(
+                    message, **context_kwargs, **method_kwargs
+                )
+
+        return wrapper
+
+    return decorator

gooddata_pipelines/backup_and_restore/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation

gooddata_pipelines/backup_and_restore/backup_input_processor.py

@@ -0,0 +1,195 @@
+# (C) 2025 GoodData Corporation
+
+from dataclasses import dataclass
+
+import requests
+
+from gooddata_pipelines.api import GoodDataApi
+from gooddata_pipelines.api.gooddata_api import API_VERSION
+from gooddata_pipelines.backup_and_restore.csv_reader import CSVReader
+from gooddata_pipelines.backup_and_restore.models.input_type import InputType
+from gooddata_pipelines.backup_and_restore.models.workspace_response import (
+    Workspace,
+    WorkspaceResponse,
+)
+from gooddata_pipelines.logger import LogObserver
+
+
+class BackupInputProcessor:
+    """Class to handle the input CSV and prepare the actual input for the backup.
+
+    Based on the InputType value, this class will determine which approach to take
+    in getting the IDs of workspaces to backup. It will then call appropriate
+    GoodData Cloud endpoints to get the IDs and return them as a list.
+    """
+
+    _api: GoodDataApi
+    base_workspace_endpoint: str
+    hierarchy_endpoint: str
+    all_workspaces_endpoint: str
+
+    def __init__(self, api: GoodDataApi, page_size: int) -> None:
+        self._api = api
+        self.page_size = page_size
+        self.logger = LogObserver()
+        self.csv_reader = CSVReader()
+
+        self.set_endpoints()
+
+    def set_endpoints(self) -> None:
+        """Sets the hierarchy endpoint for the API client."""
+        self.base_workspace_endpoint = "/api/v1/entities/workspaces"
+        self.hierarchy_endpoint = (
+            f"{self.base_workspace_endpoint}?"
+            + "filter=parent.id=={parent_id}"
+            + f"&include=parent&page=0&size={self.page_size}&sort=name,asc&metaInclude=page,hierarchy"
+        )
+        self.all_workspaces_endpoint = f"{self.base_workspace_endpoint}?page=0&size={self.page_size}&sort=name,asc&metaInclude=page"
+
+    @dataclass
+    class _ProcessDataOutput:
+        workspace_ids: list[str]
+        sub_parents: list[str] | None = None
+
+    def fetch_page(self, url: str) -> WorkspaceResponse:
+        """Fetch a page of workspaces."""
+
+        # Separate the API path from the URL so that it can be fed to the api class
+        endpoint: str = url.split(f"api/{API_VERSION}")[1]
+        response: requests.Response = self._api._get(endpoint)
+        if response.ok:
+            return WorkspaceResponse(**response.json())
+        else:
+            raise RuntimeError(
+                f"Failed to fetch data from the API. URL: {endpoint}"
+            )
+
+    @staticmethod
+    def process_data(data: list[Workspace]) -> _ProcessDataOutput:
+        """Extract children and sub-parents from workspace data."""
+        children: list[str] = []
+        sub_parents: list[str] = []
+
+        for workspace in data:
+            # append child workspace IDs
+            children.append(workspace.id)
+
+            # if hierarchy is present and has children, append child workspace ID to sub_parents
+            if workspace.meta and workspace.meta.hierarchy:
+                if workspace.meta.hierarchy.children_count > 0:
+                    sub_parents.append(workspace.id)
+        return BackupInputProcessor._ProcessDataOutput(children, sub_parents)
+
+    def log_paging_progress(self, response: WorkspaceResponse) -> None:
+        """Log the progress of paging through API responses if paginatino data is present"""
+        current_page: int | None
+        total_pages: int | None
+
+        if response.meta.page:
+            current_page = response.meta.page.number + 1
+            total_pages = response.meta.page.total_pages
+        else:
+            current_page = None
+            total_pages = None
+
+        if current_page and total_pages:
+            self.logger.info(f"Fetched page: {current_page} of {total_pages}")
+
+    def _paginate(
+        self, url: str | None
+    ) -> list["BackupInputProcessor._ProcessDataOutput"]:
+        """Paginates through the API responses and returns a list of workspace IDs."""
+        result: list[BackupInputProcessor._ProcessDataOutput] = []
+        while url:
+            response: WorkspaceResponse = self.fetch_page(url)
+            self.log_paging_progress(response)
+            result.append(self.process_data(response.data))
+            url = response.links.next
+
+        return result
+
+    def get_hierarchy(self, parent_id: str) -> list[str]:
+        """Returns a list of workspace IDs in the hierarchy."""
+        self.logger.info(f"Fetching children of {parent_id}")
+        url = self.hierarchy_endpoint.format(parent_id=parent_id)
+
+        all_children, sub_parents = [], []
+
+        results: list[BackupInputProcessor._ProcessDataOutput] = self._paginate(
+            url
+        )
+
+        for result in results:
+            all_children.extend(result.workspace_ids)
+            if result.sub_parents:
+                sub_parents.extend(result.sub_parents)
+
+        for subparent in sub_parents:
+            all_children += self.get_hierarchy(subparent)
+
+        if not all_children:
+            self.logger.warning(
+                f"No child workspaces found for parent workspace ID: {parent_id}"
+            )
+
+        return all_children
+
+    def get_all_workspaces(self) -> list[str]:
+        """Returns a list of all workspace IDs in the organization."""
+        # TODO: can be optimized - requests can be sent asynchronously.
+        # Use the total number of pages to calculate the number of requests
+        # to be sent. Use semaphore or otherwise limit the number of concurrent
+        # requests to avoid putting too much load on the server.
+        self.logger.info("Fetching all workspaces")
+        url = self.all_workspaces_endpoint
+
+        all_workspaces: list[str] = []
+
+        results: list[BackupInputProcessor._ProcessDataOutput] = self._paginate(
+            url
+        )
+
+        for result in results:
+            all_workspaces.extend(result.workspace_ids)
+
+        if not all_workspaces:
+            self.logger.warning("No workspaces found in the organization.")
+
+        return all_workspaces
+
+    def get_ids_to_backup(
+        self, input_type: InputType, path_to_csv: str | None = None
+    ) -> list[str]:
+        """Returns the list of workspace IDs to back up based on the input type."""
+
+        if input_type in (InputType.LIST_OF_WORKSPACES, InputType.HIERARCHY):
+            if path_to_csv is None:
+                raise ValueError(
+                    f"Path to CSV is required for this input type: {input_type.value}"
+                )
+
+            # If we're backing up based on the list, simply read it from the CSV
+            if input_type == InputType.LIST_OF_WORKSPACES:
+                return self.csv_reader.read_backup_csv(path_to_csv)
+            else:
+                # For hierarchy backup, we read the CSV and treat it as a list of
+                # parent workspace IDs. Then we retrieve the children of each parent,
+                # including their children, and so on. The parent workspaces are
+                # also included in the backup.
+                list_of_parents = self.csv_reader.read_backup_csv(path_to_csv)
+                list_of_children: list[str] = []
+
+                for parent in list_of_parents:
+                    list_of_children.extend(self.get_hierarchy(parent))
+
+                return list_of_parents + list_of_children
+
+        # If we're backing up the entire organization, we simply get all workspaces
+        elif input_type == InputType.ORGANIZATION:
+            list_of_workspaces = self.get_all_workspaces()
+            return list_of_workspaces
+
+        else:
+            # This path should be unreachable as long as the conditions above
+            # exhaustively check all values of InputType Enum.
+            raise ValueError(f"Unsupported input type: {input_type.value}")