gooddata-pipelines 1.49.1.dev2__tar.gz → 1.50.1.dev1__tar.gz

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.49.1.dev2
+Version: 1.50.1.dev1
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License: MIT
@@ -8,7 +8,7 @@ License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.49.1.dev2
+Requires-Dist: gooddata-sdk~=1.50.1.dev1
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/TODO.md

@@ -10,7 +10,7 @@ A list of outstanding tasks, features, or technical debt to be addressed in this
 
 - [ ] Integrate with GoodDataApiClient
 - [ ] Consider replacing the SdkMethods wrapper with direct calls to the SDK methods
-- [ ] Consider using orjson library instead of json
+- [ ] Consider using orjson library instead of json to load test data
 - [ ] Cleanup custom exceptions
 - [ ] Improve test coverage. Write missing unit tests for legacy code (e.g., user data filters)
 

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/gooddata_pipelines/__init__.py

@@ -19,6 +19,7 @@ from .provisioning.entities.user_data_filters.user_data_filters import (
     UserDataFilterProvisioner,
 )
 from .provisioning.entities.users.models.permissions import (
+    EntityType,
     PermissionFullLoad,
     PermissionIncrementalLoad,
 )
@@ -33,7 +34,10 @@ from .provisioning.entities.users.models.users import (
 from .provisioning.entities.users.permissions import PermissionProvisioner
 from .provisioning.entities.users.user_groups import UserGroupProvisioner
 from .provisioning.entities.users.users import UserProvisioner
-from .provisioning.entities.workspaces.models import WorkspaceFullLoad
+from .provisioning.entities.workspaces.models import (
+    WorkspaceFullLoad,
+    WorkspaceIncrementalLoad,
+)
 from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner
 
 __all__ = [
@@ -52,8 +56,10 @@ __all__ = [
     "UserGroupFullLoad",
     "UserProvisioner",
     "UserGroupProvisioner",
+    "WorkspaceIncrementalLoad",
     "PermissionProvisioner",
     "UserDataFilterProvisioner",
     "UserDataFilterFullLoad",
+    "EntityType",
     "__version__",
 ]

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/gooddata_pipelines/api/gooddata_api.py

@@ -7,9 +7,6 @@ from typing import Any
 
 import requests
 
-# TODO: Limit the use of "typing.Any". Improve readability by using either models
-#  or typed dicts.
-
 TIMEOUT = 60
 REQUEST_PAGE_SIZE = 250
 API_VERSION = "v1"
@@ -55,42 +52,6 @@ class ApiMethods:
         """
         return f"{self.base_url}{endpoint}"
 
-    def get_custom_application_setting(
-        self, workspace_id: str, setting_id: str
-    ) -> requests.Response:
-        """Gets a custom application setting.
-
-        Args:
-            workspace_id (str): The ID of the workspace.
-            setting_id (str): The ID of the custom application setting.
-        Returns:
-            requests.Response: The response from the server containing the
-                custom application setting.
-        """
-        url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}"
-        return self._get(url)
-
-    def put_custom_application_setting(
-        self, workspace_id: str, setting_id: str, data: dict[str, Any]
-    ) -> requests.Response:
-        url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}"
-        return self._put(url, data, self.headers)
-
-    def post_custom_application_setting(
-        self, workspace_id: str, data: dict[str, Any]
-    ) -> requests.Response:
-        """Creates a custom application setting for a given workspace.
-
-        Args:
-            workspace_id (str): The ID of the workspace.
-            data (dict[str, Any]): The data for the custom application setting.
-        Returns:
-            requests.Response: The response from the server containing the
-                created custom application setting.
-        """
-        url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/"
-        return self._post(url, data, self.headers)
-
     def get_all_workspace_data_filters(
         self, workspace_id: str
     ) -> requests.Response:
@@ -201,21 +162,6 @@ class ApiMethods:
             endpoint,
         )
 
-    def post_workspace_data_filter(
-        self, workspace_id: str, data: dict[str, Any]
-    ) -> requests.Response:
-        """Creates a workspace data filter for a given workspace.
-
-        Args:
-            workspace_id (str): The ID of the workspace.
-            data (dict[str, Any]): The data for the workspace data filter.
-        Returns:
-            requests.Response: The response from the server containing the
-                created workspace data filter.
-        """
-        endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilters"
-        return self._post(endpoint, data, self.headers)
-
     def get_user_data_filters(self, workspace_id: str) -> requests.Response:
         """Gets the user data filters for a given workspace."""
         endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters"

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -4,10 +4,8 @@ import json
 import os
 import shutil
 import tempfile
-import threading
 import time
 import traceback
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Type
@@ -39,6 +37,7 @@ from gooddata_pipelines.backup_and_restore.storage.s3_storage import (
     S3Storage,
 )
 from gooddata_pipelines.logger import LogObserver
+from gooddata_pipelines.utils.rate_limiter import RateLimiter
 
 
 @dataclass
@@ -60,6 +59,10 @@ class BackupManager:
 
         self.loader = BackupInputProcessor(self._api, self.config.api_page_size)
 
+        self._api_rate_limiter = RateLimiter(
+            calls_per_second=self.config.api_calls_per_second,
+        )
+
     @classmethod
     def create(
         cls: Type["BackupManager"],
@@ -95,11 +98,12 @@ class BackupManager:
 
     def get_user_data_filters(self, ws_id: str) -> dict:
         """Returns the user data filters for the specified workspace."""
-        response: requests.Response = self._api.get_user_data_filters(ws_id)
-        if response.ok:
-            return response.json()
-        else:
-            raise RuntimeError(f"{response.status_code}: {response.text}")
+        with self._api_rate_limiter:
+            response: requests.Response = self._api.get_user_data_filters(ws_id)
+            if response.ok:
+                return response.json()
+            else:
+                raise RuntimeError(f"{response.status_code}: {response.text}")
 
     def _store_user_data_filters(
         self,
@@ -144,14 +148,17 @@ class BackupManager:
 
     def _get_automations_from_api(self, workspace_id: str) -> Any:
         """Returns automations for the workspace as JSON."""
-        response: requests.Response = self._api.get_automations(workspace_id)
-        if response.ok:
-            return response.json()
-        else:
-            raise RuntimeError(
-                f"Failed to get automations for {workspace_id}. "
-                + f"{response.status_code}: {response.text}"
+        with self._api_rate_limiter:
+            response: requests.Response = self._api.get_automations(
+                workspace_id
             )
+            if response.ok:
+                return response.json()
+            else:
+                raise RuntimeError(
+                    f"Failed to get automations for {workspace_id}. "
+                    + f"{response.status_code}: {response.text}"
+                )
 
     def _store_automations(self, export_path: Path, workspace_id: str) -> None:
         """Stores the automations in the specified export path."""
@@ -183,7 +190,8 @@ class BackupManager:
     ) -> None:
         """Stores the filter views in the specified export path."""
         # Get the filter views YAML files from the API
-        self._api.store_declarative_filter_views(workspace_id, export_path)
+        with self._api_rate_limiter:
+            self._api.store_declarative_filter_views(workspace_id, export_path)
 
         # Move filter views to the subfolder containing the analytics model
         self._move_folder(
@@ -231,7 +239,10 @@ class BackupManager:
                 # the SDK. That way we could save and package all the declarations
                 # directly instead of reorganizing the folder structures. That should
                 # be more transparent/readable and possibly safer for threading
-                self._api.store_declarative_workspace(workspace_id, export_path)
+                with self._api_rate_limiter:
+                    self._api.store_declarative_workspace(
+                        workspace_id, export_path
+                    )
                 self.store_declarative_filter_views(export_path, workspace_id)
                 self._store_automations(export_path, workspace_id)
 
@@ -291,7 +302,6 @@ class BackupManager:
     def _process_batch(
         self,
         batch: BackupBatch,
-        stop_event: threading.Event,
         retry_count: int = 0,
     ) -> None:
         """Processes a single batch of workspaces for backup.
@@ -299,10 +309,6 @@ class BackupManager:
         and retry with exponential backoff up to BackupSettings.MAX_RETRIES.
         The base wait time is defined by BackupSettings.RETRY_DELAY.
         """
-        if stop_event.is_set():
-            # If the stop_event flag is set, return. This will terminate the thread
-            return
-
         try:
             with tempfile.TemporaryDirectory() as tmpdir:
                 self._get_workspace_export(tmpdir, batch.list_of_ids)
@@ -314,10 +320,7 @@ class BackupManager:
                 self.storage.export(tmpdir, self.org_id)
 
         except Exception as e:
-            if stop_event.is_set():
-                return
-
-            elif retry_count < BackupSettings.MAX_RETRIES:
+            if retry_count < BackupSettings.MAX_RETRIES:
                 # Retry with exponential backoff until MAX_RETRIES
                 next_retry = retry_count + 1
                 wait_time = BackupSettings.RETRY_DELAY**next_retry
@@ -328,55 +331,28 @@ class BackupManager:
                 )
 
                 time.sleep(wait_time)
-                self._process_batch(batch, stop_event, next_retry)
+                self._process_batch(batch, next_retry)
             else:
                 # If the batch fails after MAX_RETRIES, raise the error
                 self.logger.error(f"Batch failed: {e.__class__.__name__}: {e}")
                 raise
 
-    def _process_batches_in_parallel(
+    def _process_batches(
         self,
         batches: list[BackupBatch],
     ) -> None:
         """
-        Processes batches in parallel using concurrent.futures. Will stop the processing
-        if any one of the batches fails.
+        Processes batches sequentially to avoid overloading the API.
+        If any batch fails, the processing will stop.
         """
-
-        # Create a threading flag to control the threads that have already been started
-        stop_event = threading.Event()
-
-        with ThreadPoolExecutor(
-            max_workers=self.config.max_workers
-        ) as executor:
-            # Set the futures tasks.
-            futures = []
-            for batch in batches:
-                futures.append(
-                    executor.submit(
-                        self._process_batch,
-                        batch,
-                        stop_event,
-                    )
-                )
-
-            # Process futures as they complete
-            for future in as_completed(futures):
-                try:
-                    future.result()
-                except Exception:
-                    # On failure, set the flag to True - signal running processes to stop
-                    stop_event.set()
-
-                    # Cancel unstarted threads
-                    for f in futures:
-                        if not f.done():
-                            f.cancel()
-
-                    raise
+        for i, batch in enumerate(batches, 1):
+            self.logger.info(f"Processing batch {i}/{len(batches)}...")
+            self._process_batch(batch)
 
     def backup_workspaces(
-        self, path_to_csv: str | None, workspace_ids: list[str] | None
+        self,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> None:
         """Runs the backup process for a list of workspace IDs.
 
@@ -391,7 +367,9 @@ class BackupManager:
         self._backup(InputType.LIST_OF_WORKSPACES, path_to_csv, workspace_ids)
 
     def backup_hierarchies(
-        self, path_to_csv: str | None, workspace_ids: list[str] | None
+        self,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> None:
         """Runs the backup process for a list of hierarchies.
 
@@ -436,7 +414,7 @@ class BackupManager:
                 f"Exporting {len(workspaces_to_export)} workspaces in {len(batches)} batches."
             )
 
-            self._process_batches_in_parallel(batches)
+            self._process_batches(batches)
 
             self.logger.info("Backup completed")
         except Exception as e:

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/gooddata_pipelines/backup_and_restore/constants.py

@@ -21,19 +21,15 @@ class DirNames:
     UDF = "user_data_filters"
 
 
-@dataclass(frozen=True)
-class ConcurrencyDefaults:
-    MAX_WORKERS = 1
-    DEFAULT_BATCH_SIZE = 100
-
-
 @dataclass(frozen=True)
 class ApiDefaults:
     DEFAULT_PAGE_SIZE = 100
+    DEFAULT_BATCH_SIZE = 100
+    DEFAULT_API_CALLS_PER_SECOND = 1.0
 
 
 @dataclass(frozen=True)
-class BackupSettings(ConcurrencyDefaults, ApiDefaults):
+class BackupSettings(ApiDefaults):
     MAX_RETRIES = 3
     RETRY_DELAY = 5  # seconds
     TIMESTAMP_SDK_FOLDER = (

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/gooddata_pipelines/backup_and_restore/models/storage.py

@@ -83,14 +83,13 @@ class BackupRestoreConfig(BaseModel):
             description="Batch size must be greater than 0",
         ),
     ] = Field(default=BackupSettings.DEFAULT_BATCH_SIZE)
-    max_workers: Annotated[
-        int,
+    api_calls_per_second: Annotated[
+        float,
         Field(
             gt=0,
-            lt=3,
-            description="Max workers must be greater than 0 and less than 3",
+            description="Maximum API calls per second (rate limiting)",
         ),
-    ] = Field(default=BackupSettings.MAX_WORKERS)
+    ] = Field(default=BackupSettings.DEFAULT_API_CALLS_PER_SECOND)
 
     @classmethod
     def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig":

{gooddata_pipelines-1.49.1.dev2 → gooddata_pipelines-1.50.1.dev1}/gooddata_pipelines/provisioning/entities/users/models/permissions.py

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
-from abc import abstractmethod
+
 from enum import Enum
-from typing import Any, Iterator, TypeAlias, TypeVar
+from typing import Iterator, TypeAlias
 
 import attrs
 from gooddata_sdk.catalog.identifier import CatalogAssigneeIdentifier
@@ -14,85 +14,29 @@ from pydantic import BaseModel
 from gooddata_pipelines.provisioning.utils.exceptions import BaseUserException
 
 TargetsPermissionDict: TypeAlias = dict[str, dict[str, bool]]
-ConstructorType = TypeVar("ConstructorType", bound="ConstructorMixin")
 
 
-class PermissionType(str, Enum):
+class EntityType(str, Enum):
     # NOTE: Start using StrEnum with Python 3.11
     user = "user"
     user_group = "userGroup"
 
 
-class ConstructorMixin:
-    @staticmethod
-    def _get_id_and_type(
-        permission: dict[str, Any],
-    ) -> tuple[str, PermissionType]:
-        user_id: str | None = permission.get("user_id")
-        user_group_id: str | None = permission.get("ug_id")
-        if user_id and user_group_id:
-            raise ValueError("Only one of user_id or ug_id must be present")
-        elif user_id:
-            return user_id, PermissionType.user
-        elif user_group_id:
-            return user_group_id, PermissionType.user_group
-        else:
-            raise ValueError("Either user_id or ug_id must be present")
-
-    @classmethod
-    def from_list_of_dicts(
-        cls: type[ConstructorType], data: list[dict[str, Any]]
-    ) -> list[ConstructorType]:
-        """Creates a list of instances from list of dicts."""
-        # NOTE: We can use typing.Self for the return type in Python 3.11
-        permissions = []
-        for permission in data:
-            permissions.append(cls.from_dict(permission))
-        return permissions
-
-    @classmethod
-    @abstractmethod
-    def from_dict(cls, data: dict[str, Any]) -> Any:
-        """Construction form a dictionary to be implemented by subclasses."""
-        pass
-
-
-class PermissionIncrementalLoad(BaseModel, ConstructorMixin):
+class BasePermission(BaseModel):
     permission: str
     workspace_id: str
-    id_: str
-    type_: PermissionType
-    is_active: bool
+    entity_id: str
+    entity_type: EntityType
 
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> "PermissionIncrementalLoad":
-        """Returns an instance of PermissionIncrementalLoad from a dictionary."""
-        id_, target_type = cls._get_id_and_type(data)
-        return cls(
-            permission=data["ws_permissions"],
-            workspace_id=data["ws_id"],
-            id_=id_,
-            type_=target_type,
-            is_active=data["is_active"],
-        )
 
+class PermissionFullLoad(BasePermission):
+    """Input validator for full load of workspace permissions provisioning."""
 
-class PermissionFullLoad(BaseModel, ConstructorMixin):
-    permission: str
-    workspace_id: str
-    id_: str
-    type_: PermissionType
 
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> "PermissionFullLoad":
-        """Returns an instance of PermissionFullLoad from a dictionary."""
-        id_, target_type = cls._get_id_and_type(data)
-        return cls(
-            permission=data["ws_permissions"],
-            workspace_id=data["ws_id"],
-            id_=id_,
-            type_=target_type,
-        )
+class PermissionIncrementalLoad(BasePermission):
+    """Input validator for incremental load of workspace permissions provisioning."""
+
+    is_active: bool
 
 
 @attrs.define
@@ -117,7 +61,7 @@ class PermissionDeclaration:
                 permission.assignee.id,
             )
 
-            if permission_type == PermissionType.user.value:
+            if permission_type == EntityType.user.value:
                 target_dict = users
             else:
                 target_dict = user_groups
@@ -170,7 +114,7 @@ class PermissionDeclaration:
 
         for user_id, permissions in self.users.items():
             assignee = CatalogAssigneeIdentifier(
-                id=user_id, type=PermissionType.user.value
+                id=user_id, type=EntityType.user.value
             )
             for declaration in self._permissions_for_target(
                 permissions, assignee
@@ -179,7 +123,7 @@ class PermissionDeclaration:
 
         for ug_id, permissions in self.user_groups.items():
             assignee = CatalogAssigneeIdentifier(
-                id=ug_id, type=PermissionType.user_group.value
+                id=ug_id, type=EntityType.user_group.value
             )
             for declaration in self._permissions_for_target(
                 permissions, assignee
@@ -200,15 +144,15 @@ class PermissionDeclaration:
         """
         target_dict = (
             self.users
-            if permission.type_ == PermissionType.user
+            if permission.entity_type == EntityType.user
             else self.user_groups
         )
 
-        if permission.id_ not in target_dict:
-            target_dict[permission.id_] = {}
+        if permission.entity_id not in target_dict:
+            target_dict[permission.entity_id] = {}
 
         is_active = permission.is_active
-        target_permissions = target_dict[permission.id_]
+        target_permissions = target_dict[permission.entity_id]
         permission_value = permission.permission
 
         if permission_value not in target_permissions:
@@ -233,14 +177,14 @@ class PermissionDeclaration:
         """
         target_dict = (
             self.users
-            if permission.type_ == PermissionType.user
+            if permission.entity_type == EntityType.user
             else self.user_groups
         )
 
-        if permission.id_ not in target_dict:
-            target_dict[permission.id_] = {}
+        if permission.entity_id not in target_dict:
+            target_dict[permission.entity_id] = {}
 
-        target_permissions = target_dict[permission.id_]
+        target_permissions = target_dict[permission.entity_id]
         permission_value = permission.permission
 
         if permission_value not in target_permissions:

gooddata_pipelines-1.50.1.dev1/gooddata_pipelines/provisioning/entities/users/models/user_groups.py

@@ -0,0 +1,37 @@
+# (C) 2025 GoodData Corporation
+
+from pydantic import BaseModel, Field, ValidationInfo, field_validator
+
+
+class UserGroupBase(BaseModel):
+    user_group_id: str
+    user_group_name: str
+    parent_user_groups: list[str] = Field(default_factory=list)
+
+    @field_validator("user_group_name", mode="before")
+    @classmethod
+    def validate_user_group_name(
+        cls, v: str | None, info: ValidationInfo
+    ) -> str:
+        """If user_group_name is None or empty, default to user_group_id."""
+        if not v:  # handles None and empty string
+            return info.data.get("user_group_id", "")
+        return v
+
+    @field_validator("parent_user_groups", mode="before")
+    @classmethod
+    def validate_parent_user_groups(cls, v: list[str] | None) -> list[str]:
+        """If parent_user_groups is None or empty, default to empty list."""
+        if not v:
+            return []
+        return v
+
+
+class UserGroupFullLoad(UserGroupBase):
+    """Input validator for full load of user group provisioning."""
+
+
+class UserGroupIncrementalLoad(UserGroupBase):
+    """Input validator for incremental load of user group provisioning."""
+
+    is_active: bool