gooddata-pipelines 1.49.1.dev2__tar.gz → 1.50.0__tar.gz

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.49.1.dev2
+Version: 1.50.0
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License: MIT
@@ -8,7 +8,7 @@ License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.49.1.dev2
+Requires-Dist: gooddata-sdk~=1.50.0
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/TODO.md

@@ -10,7 +10,7 @@ A list of outstanding tasks, features, or technical debt to be addressed in this
 
 - [ ] Integrate with GoodDataApiClient
 - [ ] Consider replacing the SdkMethods wrapper with direct calls to the SDK methods
-- [ ] Consider using orjson library instead of json
+- [ ] Consider using orjson library instead of json to load test data
 - [ ] Cleanup custom exceptions
 - [ ] Improve test coverage. Write missing unit tests for legacy code (e.g., user data filters)
 

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/__init__.py

@@ -19,6 +19,7 @@ from .provisioning.entities.user_data_filters.user_data_filters import (
     UserDataFilterProvisioner,
 )
 from .provisioning.entities.users.models.permissions import (
+    EntityType,
     PermissionFullLoad,
     PermissionIncrementalLoad,
 )
@@ -33,7 +34,10 @@ from .provisioning.entities.users.models.users import (
 from .provisioning.entities.users.permissions import PermissionProvisioner
 from .provisioning.entities.users.user_groups import UserGroupProvisioner
 from .provisioning.entities.users.users import UserProvisioner
-from .provisioning.entities.workspaces.models import WorkspaceFullLoad
+from .provisioning.entities.workspaces.models import (
+    WorkspaceFullLoad,
+    WorkspaceIncrementalLoad,
+)
 from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner
 
 __all__ = [
@@ -52,8 +56,10 @@ __all__ = [
     "UserGroupFullLoad",
     "UserProvisioner",
     "UserGroupProvisioner",
+    "WorkspaceIncrementalLoad",
     "PermissionProvisioner",
     "UserDataFilterProvisioner",
     "UserDataFilterFullLoad",
+    "EntityType",
     "__version__",
 ]

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/api/gooddata_api.py

@@ -7,9 +7,6 @@ from typing import Any
 
 import requests
 
-# TODO: Limit the use of "typing.Any". Improve readability by using either models
-#  or typed dicts.
-
 TIMEOUT = 60
 REQUEST_PAGE_SIZE = 250
 API_VERSION = "v1"
@@ -55,42 +52,6 @@ class ApiMethods:
         """
         return f"{self.base_url}{endpoint}"
 
-    def get_custom_application_setting(
-        self, workspace_id: str, setting_id: str
-    ) -> requests.Response:
-        """Gets a custom application setting.
-
-        Args:
-            workspace_id (str): The ID of the workspace.
-            setting_id (str): The ID of the custom application setting.
-        Returns:
-            requests.Response: The response from the server containing the
-                custom application setting.
-        """
-        url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}"
-        return self._get(url)
-
-    def put_custom_application_setting(
-        self, workspace_id: str, setting_id: str, data: dict[str, Any]
-    ) -> requests.Response:
-        url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}"
-        return self._put(url, data, self.headers)
-
-    def post_custom_application_setting(
-        self, workspace_id: str, data: dict[str, Any]
-    ) -> requests.Response:
-        """Creates a custom application setting for a given workspace.
-
-        Args:
-            workspace_id (str): The ID of the workspace.
-            data (dict[str, Any]): The data for the custom application setting.
-        Returns:
-            requests.Response: The response from the server containing the
-                created custom application setting.
-        """
-        url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/"
-        return self._post(url, data, self.headers)
-
     def get_all_workspace_data_filters(
         self, workspace_id: str
     ) -> requests.Response:
@@ -201,21 +162,6 @@ class ApiMethods:
             endpoint,
         )
 
-    def post_workspace_data_filter(
-        self, workspace_id: str, data: dict[str, Any]
-    ) -> requests.Response:
-        """Creates a workspace data filter for a given workspace.
-
-        Args:
-            workspace_id (str): The ID of the workspace.
-            data (dict[str, Any]): The data for the workspace data filter.
-        Returns:
-            requests.Response: The response from the server containing the
-                created workspace data filter.
-        """
-        endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilters"
-        return self._post(endpoint, data, self.headers)
-
     def get_user_data_filters(self, workspace_id: str) -> requests.Response:
         """Gets the user data filters for a given workspace."""
         endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters"

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -376,7 +376,9 @@ class BackupManager:
                     raise
 
     def backup_workspaces(
-        self, path_to_csv: str | None, workspace_ids: list[str] | None
+        self,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> None:
         """Runs the backup process for a list of workspace IDs.
 
@@ -391,7 +393,9 @@ class BackupManager:
         self._backup(InputType.LIST_OF_WORKSPACES, path_to_csv, workspace_ids)
 
     def backup_hierarchies(
-        self, path_to_csv: str | None, workspace_ids: list[str] | None
+        self,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> None:
         """Runs the backup process for a list of hierarchies.
 

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/provisioning/entities/users/models/permissions.py

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
-from abc import abstractmethod
+
 from enum import Enum
-from typing import Any, Iterator, TypeAlias, TypeVar
+from typing import Iterator, TypeAlias
 
 import attrs
 from gooddata_sdk.catalog.identifier import CatalogAssigneeIdentifier
@@ -14,85 +14,29 @@ from pydantic import BaseModel
 from gooddata_pipelines.provisioning.utils.exceptions import BaseUserException
 
 TargetsPermissionDict: TypeAlias = dict[str, dict[str, bool]]
-ConstructorType = TypeVar("ConstructorType", bound="ConstructorMixin")
 
 
-class PermissionType(str, Enum):
+class EntityType(str, Enum):
     # NOTE: Start using StrEnum with Python 3.11
     user = "user"
     user_group = "userGroup"
 
 
-class ConstructorMixin:
-    @staticmethod
-    def _get_id_and_type(
-        permission: dict[str, Any],
-    ) -> tuple[str, PermissionType]:
-        user_id: str | None = permission.get("user_id")
-        user_group_id: str | None = permission.get("ug_id")
-        if user_id and user_group_id:
-            raise ValueError("Only one of user_id or ug_id must be present")
-        elif user_id:
-            return user_id, PermissionType.user
-        elif user_group_id:
-            return user_group_id, PermissionType.user_group
-        else:
-            raise ValueError("Either user_id or ug_id must be present")
-
-    @classmethod
-    def from_list_of_dicts(
-        cls: type[ConstructorType], data: list[dict[str, Any]]
-    ) -> list[ConstructorType]:
-        """Creates a list of instances from list of dicts."""
-        # NOTE: We can use typing.Self for the return type in Python 3.11
-        permissions = []
-        for permission in data:
-            permissions.append(cls.from_dict(permission))
-        return permissions
-
-    @classmethod
-    @abstractmethod
-    def from_dict(cls, data: dict[str, Any]) -> Any:
-        """Construction form a dictionary to be implemented by subclasses."""
-        pass
-
-
-class PermissionIncrementalLoad(BaseModel, ConstructorMixin):
+class BasePermission(BaseModel):
     permission: str
     workspace_id: str
-    id_: str
-    type_: PermissionType
-    is_active: bool
+    entity_id: str
+    entity_type: EntityType
 
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> "PermissionIncrementalLoad":
-        """Returns an instance of PermissionIncrementalLoad from a dictionary."""
-        id_, target_type = cls._get_id_and_type(data)
-        return cls(
-            permission=data["ws_permissions"],
-            workspace_id=data["ws_id"],
-            id_=id_,
-            type_=target_type,
-            is_active=data["is_active"],
-        )
 
+class PermissionFullLoad(BasePermission):
+    """Input validator for full load of workspace permissions provisioning."""
 
-class PermissionFullLoad(BaseModel, ConstructorMixin):
-    permission: str
-    workspace_id: str
-    id_: str
-    type_: PermissionType
 
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> "PermissionFullLoad":
-        """Returns an instance of PermissionFullLoad from a dictionary."""
-        id_, target_type = cls._get_id_and_type(data)
-        return cls(
-            permission=data["ws_permissions"],
-            workspace_id=data["ws_id"],
-            id_=id_,
-            type_=target_type,
-        )
+class PermissionIncrementalLoad(BasePermission):
+    """Input validator for incremental load of workspace permissions provisioning."""
+
+    is_active: bool
 
 
 @attrs.define
@@ -117,7 +61,7 @@ class PermissionDeclaration:
                 permission.assignee.id,
             )
 
-            if permission_type == PermissionType.user.value:
+            if permission_type == EntityType.user.value:
                 target_dict = users
             else:
                 target_dict = user_groups
@@ -170,7 +114,7 @@ class PermissionDeclaration:
 
         for user_id, permissions in self.users.items():
             assignee = CatalogAssigneeIdentifier(
-                id=user_id, type=PermissionType.user.value
+                id=user_id, type=EntityType.user.value
             )
             for declaration in self._permissions_for_target(
                 permissions, assignee
@@ -179,7 +123,7 @@ class PermissionDeclaration:
 
         for ug_id, permissions in self.user_groups.items():
             assignee = CatalogAssigneeIdentifier(
-                id=ug_id, type=PermissionType.user_group.value
+                id=ug_id, type=EntityType.user_group.value
             )
             for declaration in self._permissions_for_target(
                 permissions, assignee
@@ -200,15 +144,15 @@ class PermissionDeclaration:
         """
         target_dict = (
             self.users
-            if permission.type_ == PermissionType.user
+            if permission.entity_type == EntityType.user
             else self.user_groups
         )
 
-        if permission.id_ not in target_dict:
-            target_dict[permission.id_] = {}
+        if permission.entity_id not in target_dict:
+            target_dict[permission.entity_id] = {}
 
         is_active = permission.is_active
-        target_permissions = target_dict[permission.id_]
+        target_permissions = target_dict[permission.entity_id]
         permission_value = permission.permission
 
         if permission_value not in target_permissions:
@@ -233,14 +177,14 @@ class PermissionDeclaration:
         """
         target_dict = (
             self.users
-            if permission.type_ == PermissionType.user
+            if permission.entity_type == EntityType.user
             else self.user_groups
         )
 
-        if permission.id_ not in target_dict:
-            target_dict[permission.id_] = {}
+        if permission.entity_id not in target_dict:
+            target_dict[permission.entity_id] = {}
 
-        target_permissions = target_dict[permission.id_]
+        target_permissions = target_dict[permission.entity_id]
         permission_value = permission.permission
 
         if permission_value not in target_permissions:

gooddata_pipelines-1.50.0/gooddata_pipelines/provisioning/entities/users/models/user_groups.py

@@ -0,0 +1,37 @@
+# (C) 2025 GoodData Corporation
+
+from pydantic import BaseModel, Field, ValidationInfo, field_validator
+
+
+class UserGroupBase(BaseModel):
+    user_group_id: str
+    user_group_name: str
+    parent_user_groups: list[str] = Field(default_factory=list)
+
+    @field_validator("user_group_name", mode="before")
+    @classmethod
+    def validate_user_group_name(
+        cls, v: str | None, info: ValidationInfo
+    ) -> str:
+        """If user_group_name is None or empty, default to user_group_id."""
+        if not v:  # handles None and empty string
+            return info.data.get("user_group_id", "")
+        return v
+
+    @field_validator("parent_user_groups", mode="before")
+    @classmethod
+    def validate_parent_user_groups(cls, v: list[str] | None) -> list[str]:
+        """If parent_user_groups is None or empty, default to empty list."""
+        if not v:
+            return []
+        return v
+
+
+class UserGroupFullLoad(UserGroupBase):
+    """Input validator for full load of user group provisioning."""
+
+
+class UserGroupIncrementalLoad(UserGroupBase):
+    """Input validator for incremental load of user group provisioning."""
+
+    is_active: bool

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/provisioning/entities/users/models/users.py

@@ -5,10 +5,8 @@ from typing import Any
 from gooddata_sdk.catalog.user.entity_model.user import CatalogUser
 from pydantic import BaseModel
 
-from gooddata_pipelines.provisioning.utils.utils import SplitMixin
 
-
-class BaseUser(BaseModel, SplitMixin):
+class BaseUser(BaseModel):
     """Base class containing shared user fields and functionality."""
 
     user_id: str
@@ -18,21 +16,6 @@ class BaseUser(BaseModel, SplitMixin):
     auth_id: str | None
     user_groups: list[str]
 
-    @classmethod
-    def _create_from_dict_data(
-        cls, user_data: dict[str, Any], delimiter: str = ","
-    ) -> dict[str, Any]:
-        """Helper method to extract common data from dict."""
-        user_groups = cls.split(user_data["user_groups"], delimiter=delimiter)
-        return {
-            "user_id": user_data["user_id"],
-            "firstname": user_data["firstname"],
-            "lastname": user_data["lastname"],
-            "email": user_data["email"],
-            "auth_id": user_data["auth_id"],
-            "user_groups": user_groups,
-        }
-
     @classmethod
     def _create_from_sdk_data(cls, obj: CatalogUser) -> dict[str, Any]:
         """Helper method to extract common data from SDK object."""
@@ -68,47 +51,24 @@ class BaseUser(BaseModel, SplitMixin):
         )
 
 
-class UserIncrementalLoad(BaseUser):
-    """User model for incremental load operations with active status tracking."""
-
-    is_active: bool
-
-    @classmethod
-    def from_list_of_dicts(
-        cls, data: list[dict[str, Any]], delimiter: str = ","
-    ) -> list["UserIncrementalLoad"]:
-        """Creates a list of User objects from list of dicts."""
-        converted_users = []
-        for user in data:
-            base_data = cls._create_from_dict_data(user, delimiter)
-            base_data["is_active"] = user["is_active"]
-            converted_users.append(cls(**base_data))
-        return converted_users
+class UserFullLoad(BaseUser):
+    """Input validator for full load of user provisioning."""
 
     @classmethod
-    def from_sdk_obj(cls, obj: CatalogUser) -> "UserIncrementalLoad":
+    def from_sdk_obj(cls, obj: CatalogUser) -> "UserFullLoad":
         """Creates GDUserTarget from CatalogUser SDK object."""
         base_data = cls._create_from_sdk_data(obj)
-        base_data["is_active"] = True
         return cls(**base_data)
 
 
-class UserFullLoad(BaseUser):
-    """User model for full load operations."""
+class UserIncrementalLoad(BaseUser):
+    """Input validator for incremental load of user provisioning."""
 
-    @classmethod
-    def from_list_of_dicts(
-        cls, data: list[dict[str, Any]], delimiter: str = ","
-    ) -> list["UserFullLoad"]:
-        """Creates a list of User objects from list of dicts."""
-        converted_users = []
-        for user in data:
-            base_data = cls._create_from_dict_data(user, delimiter)
-            converted_users.append(cls(**base_data))
-        return converted_users
+    is_active: bool
 
     @classmethod
-    def from_sdk_obj(cls, obj: CatalogUser) -> "UserFullLoad":
+    def from_sdk_obj(cls, obj: CatalogUser) -> "UserIncrementalLoad":
         """Creates GDUserTarget from CatalogUser SDK object."""
         base_data = cls._create_from_sdk_data(obj)
+        base_data["is_active"] = True
         return cls(**base_data)

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/provisioning/entities/users/permissions.py

@@ -6,10 +6,10 @@ from typing import TypeVar
 
 from gooddata_pipelines.api.exceptions import GoodDataApiException
 from gooddata_pipelines.provisioning.entities.users.models.permissions import (
+    EntityType,
     PermissionDeclaration,
     PermissionFullLoad,
     PermissionIncrementalLoad,
-    PermissionType,
     TargetsPermissionDict,
     WSPermissionsDeclarations,
 )
@@ -28,12 +28,18 @@ class PermissionProvisioner(
     """Provisioning class for user permissions in GoodData workspaces.
 
     This class handles the provisioning of user permissions based on the provided
-    source data.
+    source data. Use the `full_load` or `incremental_load`
+    methods to run the provisioning.
     """
 
     source_group_incremental: list[PermissionIncrementalLoad]
     source_group_full: list[PermissionFullLoad]
 
+    FULL_LOAD_TYPE: type[PermissionFullLoad] = PermissionFullLoad
+    INCREMENTAL_LOAD_TYPE: type[PermissionIncrementalLoad] = (
+        PermissionIncrementalLoad
+    )
+
     def _get_ws_declaration(self, ws_id: str) -> PermissionDeclaration:
         users: TargetsPermissionDict = {}
         user_groups: TargetsPermissionDict = {}
@@ -47,7 +53,7 @@ class PermissionProvisioner(
             )
             target_dict = (
                 users
-                if permission_type == PermissionType.user.value
+                if permission_type == EntityType.user.value
                 else user_groups
             )
 
@@ -105,11 +111,13 @@ class PermissionProvisioner(
         self, permission: PermissionFullLoad | PermissionIncrementalLoad
     ) -> None:
         """Validates if the permission is correctly defined."""
-        if permission.type_ == PermissionType.user:
-            self._api.get_user(permission.id_, error_message="User not found")
+        if permission.entity_type == EntityType.user:
+            self._api.get_user(
+                permission.entity_id, error_message="User not found"
+            )
         else:
             self._api.get_user_group(
-                permission.id_, error_message="User group not found"
+                permission.entity_id, error_message="User group not found"
             )
 
         self._api.get_workspace(

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/provisioning/entities/users/user_groups.py

@@ -21,13 +21,19 @@ class UserGroupProvisioner(
     """Provisioning class for user groups in GoodData workspaces.
 
     This class handles the creation, update, and deletion of user groups
-    based on the provided source data.
+    based on the provided source data. Use the `full_load` or `incremental_load`
+    methods to run the provisioning.
     """
 
     source_group_incremental: list[UserGroupIncrementalLoad]
     source_group_full: list[UserGroupFullLoad]
     upstream_user_groups: list[CatalogUserGroup]
 
+    FULL_LOAD_TYPE: type[UserGroupFullLoad] = UserGroupFullLoad
+    INCREMENTAL_LOAD_TYPE: type[UserGroupIncrementalLoad] = (
+        UserGroupIncrementalLoad
+    )
+
     @staticmethod
     def _is_changed(
         group: UserGroupModel, existing_group: CatalogUserGroup

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/provisioning/entities/users/users.py

@@ -30,6 +30,9 @@ class UserProvisioner(Provisioning[UserFullLoad, UserIncrementalLoad]):
     source_group_incremental: list[UserIncrementalLoad]
     source_group_full: list[UserFullLoad]
 
+    FULL_LOAD_TYPE: type[UserFullLoad] = UserFullLoad
+    INCREMENTAL_LOAD_TYPE: type[UserIncrementalLoad] = UserIncrementalLoad
+
     def __init__(self, host: str, token: str) -> None:
         super().__init__(host, token)
         self.upstream_user_cache: dict[UserId, UserModel] = {}

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.0}/gooddata_pipelines/provisioning/entities/workspaces/models.py

@@ -1,29 +1,27 @@
 # (C) 2025 GoodData Corporation
 """Module containing models related to workspace provisioning in GoodData Cloud."""
 
-from dataclasses import dataclass, field
 from typing import Literal
 
+import attrs
 from pydantic import BaseModel, ConfigDict
 
 
-@dataclass
+@attrs.define
 class WorkspaceDataMaps:
     """Dataclass to hold various mappings related to workspace data."""
 
-    child_to_parent_id_map: dict[str, str] = field(default_factory=dict)
-    workspace_id_to_wdf_map: dict[str, dict[str, list[str]]] = field(
-        default_factory=dict
+    child_to_parent_id_map: dict[str, str] = attrs.field(factory=dict)
+    workspace_id_to_wdf_map: dict[str, dict[str, list[str]]] = attrs.field(
+        factory=dict
     )
-    parent_ids: set[str] = field(default_factory=set)
-    source_ids: set[str] = field(default_factory=set)
-    workspace_id_to_name_map: dict[str, str] = field(default_factory=dict)
-    upstream_ids: set[str] = field(default_factory=set)
+    parent_ids: set[str] = attrs.field(factory=set)
+    source_ids: set[str] = attrs.field(factory=set)
+    workspace_id_to_name_map: dict[str, str] = attrs.field(factory=dict)
+    upstream_ids: set[str] = attrs.field(factory=set)
 
 
-class WorkspaceFullLoad(BaseModel):
-    """Model representing input for provisioning of workspaces in GoodData Cloud."""
-
+class WorkspaceBase(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True)
 
     parent_id: str
@@ -33,10 +31,13 @@ class WorkspaceFullLoad(BaseModel):
     workspace_data_filter_values: list[str] | None = None
 
 
-class WorkspaceIncrementalLoad(WorkspaceFullLoad):
-    """Model representing input for incremental provisioning of workspaces in GoodData Cloud."""
+class WorkspaceFullLoad(WorkspaceBase):
+    """Input validator for full load of workspace provisioning."""
+
+
+class WorkspaceIncrementalLoad(WorkspaceBase):
+    """Input validator for incremental load of workspace provisioning."""
 
-    # TODO: double check that the model loads the data correctly, write a test
     is_active: bool