gooddata-pipelines 1.47.1.dev1__py3-none-any.whl

gooddata_pipelines/provisioning/entities/workspaces/workspace.py

@@ -0,0 +1,263 @@
+# (C) 2025 GoodData Corporation
+"""Module for provisioning workspaces in GoodData Cloud."""
+
+from typing import Literal
+
+from gooddata_sdk.catalog.workspace.entity_model.workspace import (
+    CatalogWorkspace,
+)
+
+from gooddata_pipelines.api.exceptions import GoodDataApiException
+from gooddata_pipelines.provisioning.entities.workspaces.models import (
+    WorkspaceDataMaps,
+    WorkspaceFullLoad,
+    WorkspaceIncrementalLoad,
+)
+from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_filters import (
+    WorkspaceDataFilterManager,
+)
+from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_parser import (
+    WorkspaceDataParser,
+)
+from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_validator import (
+    WorkspaceDataValidator,
+)
+from gooddata_pipelines.provisioning.provisioning import Provisioning
+from gooddata_pipelines.provisioning.utils.context_objects import (
+    WorkspaceContext,
+)
+from gooddata_pipelines.provisioning.utils.exceptions import WorkspaceException
+
+
+class WorkspaceProvisioner(
+    Provisioning[WorkspaceFullLoad, WorkspaceIncrementalLoad]
+):
+    source_group_full: list[WorkspaceFullLoad]
+    source_group_incremental: list[WorkspaceIncrementalLoad]
+
+    def __init__(self, *args: str, **kwargs: str) -> None:
+        """Creates an instance of the WorkspaceProvisioner.
+
+        Calls the superclass constructor and initializes the validator, parser,
+        and maps for workspace data.
+        """
+        super().__init__(*args, **kwargs)
+        self.validator: WorkspaceDataValidator = WorkspaceDataValidator(
+            self._api
+        )
+        self.parser: WorkspaceDataParser = WorkspaceDataParser()
+        self.maps: WorkspaceDataMaps = WorkspaceDataMaps()
+
+    def _find_workspaces_to_update(
+        self,
+        source_group: list[WorkspaceFullLoad],
+        panther_group: list[CatalogWorkspace],
+        ids_in_both_systems: set[str],
+    ) -> set[str]:
+        """
+        Inspects existing Panther workspaces and compares them to workspaces from
+        the source database. If the ID exists in both systems but the workspace
+        name in GoodData Cloud is different from the source, the workspace will
+        be updated. The rest of the workspaces will be ignored.
+        """
+        existing_workspaces: dict[str, CatalogWorkspace] = {
+            workspace.id: workspace for workspace in panther_group
+        }
+
+        ids_to_update: set[str] = set()
+
+        for source_workspace in source_group:
+            source_id = source_workspace.workspace_id
+            source_name = source_workspace.workspace_name
+
+            if source_id not in ids_in_both_systems:
+                continue
+
+            if existing_workspaces.get(source_id):
+                panther_name = existing_workspaces[source_id].name
+            else:
+                continue
+
+            if source_name == panther_name:
+                continue
+
+            ids_to_update.add(source_id)
+
+        return ids_to_update
+
+    def _create_or_update_panther_workspaces(
+        self,
+        workspace_ids_to_create: set[str],
+        workspace_ids_to_update: set[str],
+        child_to_parent_map: dict[str, str],
+        workspace_id_to_wdf_map: dict[str, dict[str, list[str]]],
+    ) -> None:
+        action: Literal["CREATE", "UPDATE"]
+
+        for source_workspace in self.source_group_full:
+            if source_workspace.workspace_id in workspace_ids_to_update:
+                action = "UPDATE"
+            elif source_workspace.workspace_id in workspace_ids_to_create:
+                action = "CREATE"
+            else:
+                continue
+
+            context: WorkspaceContext = WorkspaceContext(
+                workspace_id=source_workspace.workspace_id,
+                workspace_name=source_workspace.workspace_name,
+                wdf_id=source_workspace.workspace_data_filter_id,
+                wdf_values=source_workspace.workspace_data_filter_values,
+            )
+
+            parent_workspace_id: str = child_to_parent_map[context.workspace_id]
+
+            try:
+                self._api.create_or_update_panther_workspace(
+                    workspace_id=context.workspace_id,
+                    workspace_name=str(context.workspace_name),
+                    parent_id=parent_workspace_id,
+                )
+                self.logger.info(
+                    f"{action.title()}d workspace: {context.workspace_id}"
+                )
+
+            except GoodDataApiException as e:
+                combined_context = {**context.__dict__, **e.__dict__}
+                self.logger.error(
+                    f"Failed to {action.title()} workspace: {context.workspace_id}. "
+                    + f"Error: {e} Context: {combined_context}"
+                )
+
+            # If child workspace has WDF settings, apply them
+            child_wdfs: dict[str, list[str]] = workspace_id_to_wdf_map.get(
+                context.workspace_id, {}
+            )
+            if child_wdfs:
+                self.wdf_manager.check_wdf_settings(
+                    context,
+                )
+
+    def delete_panther_workspaces(
+        self, ids_to_delete: set[str], workspace_id_to_name_map: dict[str, str]
+    ) -> None:
+        for workspace_id in ids_to_delete:
+            workspace_context: WorkspaceContext = WorkspaceContext(
+                workspace_id=workspace_id,
+                workspace_name=workspace_id_to_name_map.get(workspace_id),
+            )
+            try:
+                self._api.delete_panther_workspace(workspace_id)
+                self.logger.info(
+                    f"Deleted workspace: {workspace_context.workspace_id}"
+                )
+
+            except GoodDataApiException as e:
+                exception_context = {**workspace_context.__dict__, **e.__dict__}
+                self.logger.error(
+                    f"Failed to delete workspace: {workspace_context.workspace_id}. "
+                    + f"Error: {e} Context: {exception_context}"
+                )
+
+    def verify_workspace_provisioning(
+        self,
+        source_group: list[WorkspaceFullLoad],
+        parent_workspace_ids: set[str],
+    ) -> None:
+        """Verifies that upstream content is equal to the source data."""
+        source_ids_names: set[tuple[str, str]] = {
+            (item.workspace_id, item.workspace_name) for item in source_group
+        }
+
+        panther_workspaces: list[CatalogWorkspace] = (
+            self._api.get_panther_children_workspaces(parent_workspace_ids)
+        )
+
+        panther_ids_names: set[tuple[str, str]] = {
+            (workspace.workspace_id, workspace.name)
+            for workspace in panther_workspaces
+        }
+
+        diff: set[tuple[str, str]] = source_ids_names.symmetric_difference(
+            panther_ids_names
+        )
+
+        if diff:
+            raise WorkspaceException(
+                "Provisioning failed. The source and Panther workspaces do not "
+                + f"match. Difference: {diff}"
+            )
+
+    def _provision_full_load(self) -> None:
+        """Full load workspace provisioning."""
+
+        # Validate the source data.
+        self.validator.validate_source_data(self.source_group_full)
+
+        # Set the maps based on the source data.
+        self.maps = self.parser.set_maps_based_on_source(
+            self.maps, self.source_group_full
+        )
+
+        # Get upstream children of all parent workspaces.
+        self.upstream_group: list[CatalogWorkspace] = (
+            self._api.get_panther_children_workspaces(self.maps.parent_ids)
+        )
+
+        # Set maps that require upstream data.
+        self.maps = self.parser.set_maps_with_upstream_data(
+            self.maps, self.source_group_full, self.upstream_group
+        )
+
+        # Create an instance of WDF manager with the created maps.
+        self.wdf_manager = WorkspaceDataFilterManager(self._api, self.maps)
+
+        # Sort the ids to groups based on provisioning logic.
+        id_groups = self._create_groups(
+            self.maps.source_ids, self.maps.upstream_ids
+        )
+
+        # Find out which workspaces to update.
+        self.ids_to_update: set[str] = self._find_workspaces_to_update(
+            self.source_group_full,
+            self.upstream_group,
+            id_groups.ids_in_both_systems,
+        )
+
+        # Delete the workspaces that are not in the source.
+        self.delete_panther_workspaces(
+            id_groups.ids_to_delete, self.maps.workspace_id_to_name_map
+        )
+
+        # Create or update selected workspaces.
+        self._create_or_update_panther_workspaces(
+            id_groups.ids_to_create,
+            self.ids_to_update,
+            self.maps.child_to_parent_id_map,
+            self.maps.workspace_id_to_wdf_map,
+        )
+
+        # Check WDF settings of ignored workspaces.
+        ignored_workspace_ids: set[str] = self.maps.source_ids.difference(
+            id_groups.ids_to_create.union(self.ids_to_update).union(
+                id_groups.ids_to_delete
+            )
+        )
+
+        for ignored_workspace_id in ignored_workspace_ids:
+            ignored_workspace_context: WorkspaceContext = WorkspaceContext(
+                workspace_id=ignored_workspace_id,
+                workspace_name=self.maps.workspace_id_to_name_map.get(
+                    ignored_workspace_id
+                ),
+            )
+            self.wdf_manager.check_wdf_settings(ignored_workspace_context)
+
+        # Verify the provisioning by queries to GoodData Cloud.
+        self.verify_workspace_provisioning(
+            self.source_group_full, self.maps.parent_ids
+        )
+
+    def _provision_incremental_load(self) -> None:
+        """Incremental workspace provisioning."""
+
+        raise NotImplementedError("Not implemented yet.")

gooddata_pipelines/provisioning/entities/workspaces/workspace_data_filters.py

@@ -0,0 +1,286 @@
+# (C) 2025 GoodData Corporation
+
+"""Module for managing workspace data filter settings in GoodData Cloud."""
+
+import json
+from typing import Any
+from uuid import uuid4
+
+from requests import Response
+
+from gooddata_pipelines.api import GoodDataApi
+from gooddata_pipelines.logger.logger import LogObserver
+from gooddata_pipelines.provisioning.entities.workspaces.models import (
+    WDFSetting,
+    WorkspaceDataMaps,
+)
+from gooddata_pipelines.provisioning.utils.context_objects import (
+    WorkspaceContext,
+)
+from gooddata_pipelines.provisioning.utils.exceptions import WorkspaceException
+
+
+class WorkspaceDataFilterManager:
+    """
+    Helper class to manage workspace data filter settings. Note that Workspace
+    Data Filters themselves are not managed here. The Workspace Data Filter
+    Setting object represents the relationship of values in a WDF column and
+    a specific workspace.
+    """
+
+    def __init__(self, api: GoodDataApi, maps: WorkspaceDataMaps) -> None:
+        self.api: GoodDataApi = api
+        self.maps: WorkspaceDataMaps = maps
+        self.logger: LogObserver = LogObserver()
+
+    @staticmethod
+    def _create_wdf_setting_dict(
+        wdf_setting_id: str, wdf_id: str, wdf_values: list[str]
+    ) -> dict[str, Any]:
+        """Loads a JSON template of a WDF setting and fills it with the given values."""
+        values = [str(value) for value in wdf_values]
+
+        import os
+
+        wdf_setting_path = os.path.join(
+            os.path.dirname(__file__), "../../assets/wdf_setting.json"
+        )
+        with open(os.path.abspath(wdf_setting_path)) as file:
+            wdf_setting: dict[str, Any] = json.load(file)
+
+        wdf_setting["data"]["attributes"]["filterValues"] = values
+        wdf_setting["data"]["id"] = wdf_setting_id
+        wdf_setting["data"]["relationships"]["workspaceDataFilter"]["data"][
+            "id"
+        ] = wdf_id
+
+        return wdf_setting
+
+    def _get_wdf_settings_for_workspace(
+        self, workspace_id: str
+    ) -> list[WDFSetting]:
+        """Gets all workspace data filter settings for a given workspace."""
+        wdf_settings_response: Response = (
+            self.api.get_workspace_data_filter_settings(workspace_id)
+        )
+
+        if not wdf_settings_response.ok:
+            raise WorkspaceException(
+                f"Failed to get WDF settings: {wdf_settings_response.text}",
+                workspace_id=workspace_id,
+                http_status=str(wdf_settings_response.status_code),
+            )
+
+        raw_wdf_settings: dict[str, Any] = wdf_settings_response.json()
+
+        data: list[dict[str, Any]] = raw_wdf_settings["data"]
+
+        wdf_settings: list[WDFSetting] = [
+            WDFSetting(**wdf_setting) for wdf_setting in data
+        ]
+
+        return wdf_settings
+
+    @staticmethod
+    def _get_actual_wdf_setting_id_and_values(
+        actual_wdf_settings: list[WDFSetting], actual_wdf_id: str
+    ) -> tuple[str, list[str]]:
+        """Returns WDF setting ID and values for given WDF ID."""
+        for actual_wdf_setting in actual_wdf_settings:
+            if (
+                actual_wdf_setting.relationships.workspaceDataFilter["data"].id
+                == actual_wdf_id
+            ):
+                actual_wdf_setting_id = actual_wdf_setting.id
+                actual_wdf_values = actual_wdf_setting.attributes.filterValues
+
+                return actual_wdf_setting_id, actual_wdf_values
+
+        raise WorkspaceException(
+            "Could not find WDF setting for WDF in actual WDF settings.",
+            wdf_id=actual_wdf_id,
+        )
+
+    def _delete_redundant_wdf_setting(
+        self,
+        workspace_context: WorkspaceContext,
+        actual_wdf_id: str,
+        actual_wdf_settings: list[WDFSetting],
+    ) -> None:
+        """Deletes a WDF setting."""
+        actual_wdf_setting_id, actual_wdf_values = (
+            self._get_actual_wdf_setting_id_and_values(
+                actual_wdf_settings, actual_wdf_id
+            )
+        )
+        # Update context with actual values
+        workspace_context.wdf_id = actual_wdf_id
+        workspace_context.wdf_values = actual_wdf_values
+
+        # If there is a WDF setting for a WDF that should not be associated with
+        # the workspace, then delete the setting
+        delete_response: Response = (
+            self.api.delete_workspace_data_filter_setting(
+                workspace_context.workspace_id,
+                actual_wdf_setting_id,
+            )
+        )
+        if delete_response.ok:
+            self.logger.info(
+                f"Deleted WDF setting for WDF {workspace_context.wdf_id} in "
+                + f"workspace {workspace_context.workspace_id}"
+            )
+        else:
+            raise WorkspaceException(
+                f"Failed to delete WDF setting: {delete_response.text}",
+                delete_response,
+                workspace_context,
+            )
+
+    def _post_wdf_setting(
+        self,
+        workspace_context: WorkspaceContext,
+    ) -> None:
+        """Posts a WDF setting to Panther."""
+        wdf_setting = self._create_wdf_setting_dict(
+            str(uuid4()),
+            str(workspace_context.wdf_id),
+            workspace_context.wdf_values
+            if workspace_context.wdf_values
+            else [],
+        )
+        post_response: Response = self.api.post_workspace_data_filter_setting(
+            workspace_context.workspace_id,
+            wdf_setting,
+        )
+        if post_response.ok:
+            self.logger.info(
+                f"Created WDF setting for WDF {workspace_context.wdf_id} in workspace {workspace_context.workspace_id}"
+            )
+        else:
+            raise WorkspaceException(
+                f"Failed to create WDF setting: {post_response.text}",
+                post_response,
+                workspace_context,
+            )
+
+    def _put_wdf_setting(
+        self,
+        workspace_context: WorkspaceContext,
+        actual_wdf_settings: list[WDFSetting],
+    ) -> None:
+        # get Panther WDF setting ID
+        actual_wdf_setting_id, _ = self._get_actual_wdf_setting_id_and_values(
+            actual_wdf_settings, str(workspace_context.wdf_id)
+        )
+
+        wdf_setting = self._create_wdf_setting_dict(
+            actual_wdf_setting_id,
+            str(workspace_context.wdf_id),
+            workspace_context.wdf_values
+            if workspace_context.wdf_values
+            else [],
+        )
+
+        put_response: Response = self.api.put_workspace_data_filter_setting(
+            workspace_context.workspace_id,
+            wdf_setting,
+        )
+        if put_response.ok:
+            self.logger.info(
+                f"Updated WDF setting for WDF {workspace_context.wdf_id} in workspace {workspace_context.workspace_id}"
+            )
+        else:
+            raise WorkspaceException(
+                f"Failed to update WDF setting: {put_response.text}",
+                put_response,
+                workspace_context,
+            )
+
+    def _compare_wdf_settings(
+        self,
+        workspace_context: WorkspaceContext,
+        source_wdf_config: dict[str, list[str]],
+        upstream_wdf_settings: list[WDFSetting],
+    ) -> None:
+        """
+        Compares WDF settings as extracted from the source with the actual WDF
+        settings in Panther. We do not know the WDF setting IDs from the outset,
+        which is why we need to check the WDF IDs and then the settings values
+        in a roundabout way. I.e., we know that a WDF should have some setting
+        with an unknown ID, but certain values. In this case, we don't care about
+        the setting ID, but need to make sure that the workspace has the correct
+        values for the WDF.
+        """
+        upstream_wdf_ids: set[str] = {
+            upstream_wdf_setting.relationships.workspaceDataFilter["data"].id
+            for upstream_wdf_setting in upstream_wdf_settings
+        }
+
+        source_wdf_ids: set[str] = set(source_wdf_config.keys())
+
+        # Create map of upstream WDF_ID : WDF values
+        upstream_wdf_ids_and_values: dict[str, list[str]] = {}
+        for upstream_wdf_setting in upstream_wdf_settings:
+            upstream_wdf_ids_and_values[
+                upstream_wdf_setting.relationships.workspaceDataFilter[
+                    "data"
+                ].id
+            ] = upstream_wdf_setting.attributes.filterValues
+
+        # Iterate through source WDF settings
+        for source_wdf_id in source_wdf_ids:
+            # Update WDF information in context to make sure we have the correct
+            # data -> there can be multiple WDFs per workspace
+            source_values: list[str] = source_wdf_config[source_wdf_id]
+            workspace_context.wdf_id = source_wdf_id
+            workspace_context.wdf_values = source_values
+
+            # Post WDF setting if missing for a WDF, when it should be there
+            if source_wdf_id not in upstream_wdf_ids:
+                self._post_wdf_setting(workspace_context)
+
+            # If settings exist for a WDF that should be there, then compare values
+            elif source_wdf_id in upstream_wdf_ids:
+                actual_values: list[str] = upstream_wdf_ids_and_values[
+                    source_wdf_id
+                ]
+
+                # If values are different, then update the WDF settings
+                if set(source_values) != set(actual_values):
+                    self._put_wdf_setting(
+                        workspace_context,
+                        upstream_wdf_settings,
+                    )
+
+        # Go through Panther WDF settings and check if there are any that should
+        # not be there. Delete them if so.
+        for actual_wdf_id in upstream_wdf_ids:
+            if actual_wdf_id not in source_wdf_ids:
+                self._delete_redundant_wdf_setting(
+                    workspace_context,
+                    actual_wdf_id,
+                    upstream_wdf_settings,
+                )
+
+    def check_wdf_settings(
+        self,
+        workspace_context: WorkspaceContext,
+    ) -> None:
+        """
+        Checks WDF settings for a given workspace.
+        Creates WDF settings defined in source if they are missing in Panther.
+        Updates WDF setting values if they are different in source and Panther.
+        Deletes WDF settings from Panther if they are not defined in source.
+        """
+        actual_wdf_settings: list[WDFSetting] = (
+            self._get_wdf_settings_for_workspace(workspace_context.workspace_id)
+        )
+
+        source_wdf_config: dict[str, list[str]] = (
+            self.maps.workspace_id_to_wdf_map[workspace_context.workspace_id]
+        )
+
+        self._compare_wdf_settings(
+            workspace_context, source_wdf_config, actual_wdf_settings
+        )

gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py

@@ -0,0 +1,123 @@
+# (C) 2025 GoodData Corporation
+
+"""Module for parsing and processing workspace data in GoodData Cloud."""
+
+from gooddata_sdk.catalog.workspace.entity_model.workspace import (
+    CatalogWorkspace,
+)
+
+from gooddata_pipelines.provisioning.entities.workspaces.models import (
+    WorkspaceDataMaps,
+    WorkspaceFullLoad,
+)
+
+
+class WorkspaceDataParser:
+    """Helper class to process workspace data retrieved from Panther and source DB."""
+
+    @staticmethod
+    def _get_id_to_name_map(
+        source_group: list[WorkspaceFullLoad],
+        upstream_group: list[CatalogWorkspace],
+    ) -> dict[str, str]:
+        """Creates a map of workspace IDs to their names for all known workspaces."""
+        source_map: dict[str, str] = {
+            workspace.workspace_id: workspace.workspace_name
+            for workspace in source_group
+        }
+        upstream_map: dict[str, str] = {
+            item.workspace_id: item.name for item in upstream_group
+        }
+
+        return {**upstream_map, **source_map}
+
+    @staticmethod
+    def _get_child_to_parent_map(
+        source_group: list[WorkspaceFullLoad],
+    ) -> dict[str, str]:
+        """Creates a map of child workspace IDs to their parent workspace IDs."""
+        child_to_parent_map: dict[str, str] = {
+            workspace.workspace_id: workspace.parent_id
+            for workspace in source_group
+        }
+
+        return child_to_parent_map
+
+    @staticmethod
+    def _get_set_of_ids_from_source(
+        source_group: list[WorkspaceFullLoad], column_name: str
+    ) -> set[str]:
+        """Creates a set of unique parent workspace IDs."""
+        set_of_ids: set[str] = {
+            getattr(workspace, column_name)
+            for workspace in source_group
+            if getattr(workspace, column_name)
+        }
+        return set_of_ids
+
+    @staticmethod
+    def get_set_of_upstream_workspace_ids(
+        upstream_group: list[CatalogWorkspace],
+    ) -> set[str]:
+        """Creates a set of unique upstream workspace IDs."""
+        set_of_ids: set[str] = {item.workspace_id for item in upstream_group}
+        return set_of_ids
+
+    def _get_child_to_wdfs_map(
+        self, source_group: list[WorkspaceFullLoad]
+    ) -> dict[str, dict[str, list[str]]]:
+        """Creates a map of child workspace IDs to their WDF IDs."""
+        # TODO: Use objects or a more transparent data structure instead of this.
+        child_to_wdf_map: dict[str, dict[str, list[str]]] = {}
+
+        # For each child, get its possible WDF IDs and values for each id
+        for workspace in source_group:
+            child_id: str = workspace.workspace_id
+            wdf_id: str | None = workspace.workspace_data_filter_id
+            wdf_values: list[str] | None = (
+                workspace.workspace_data_filter_values
+            )
+
+            if wdf_values and wdf_id:
+                if not child_to_wdf_map.get(child_id):
+                    child_to_wdf_map[child_id] = {}
+                child_to_wdf_map[child_id][wdf_id] = wdf_values
+
+        return child_to_wdf_map
+
+    def set_maps_based_on_source(
+        self,
+        map_object: WorkspaceDataMaps,
+        source_group: list[WorkspaceFullLoad],
+    ) -> WorkspaceDataMaps:
+        """Creates maps which are dependent on the source group only."""
+        map_object.child_to_parent_id_map = self._get_child_to_parent_map(
+            source_group
+        )
+        map_object.workspace_id_to_wdf_map = self._get_child_to_wdfs_map(
+            source_group
+        )
+        map_object.parent_ids = self._get_set_of_ids_from_source(
+            source_group, "parent_id"
+        )
+        map_object.source_ids = self._get_set_of_ids_from_source(
+            source_group, "workspace_id"
+        )
+
+        return map_object
+
+    def set_maps_with_upstream_data(
+        self,
+        map_object: WorkspaceDataMaps,
+        source_group: list[WorkspaceFullLoad],
+        upstream_group: list[CatalogWorkspace],
+    ) -> WorkspaceDataMaps:
+        """Creates maps which are dependent on both the source group and upstream group."""
+        map_object.workspace_id_to_name_map = self._get_id_to_name_map(
+            source_group, upstream_group
+        )
+        map_object.upstream_ids = self.get_set_of_upstream_workspace_ids(
+            upstream_group
+        )
+
+        return map_object