gooddata-pipelines 1.52.1.dev2__tar.gz → 1.52.1.dev3__tar.gz

{gooddata_pipelines-1.52.1.dev2 → gooddata_pipelines-1.52.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.52.1.dev2
+Version: 1.52.1.dev3
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License: MIT
@@ -8,7 +8,7 @@ License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.52.1.dev2
+Requires-Dist: gooddata-sdk~=1.52.1.dev3
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326
@@ -27,9 +27,10 @@ You can use the package to manage following resources in GDC:
    - User/Group permissions
    - User Data Filters
    - Child workspaces (incl. Workspace Data Filter settings)
-1. _[PLANNED]:_ Backup and restore of workspaces
-1. _[PLANNED]:_ Custom fields management
-   - extend the Logical Data Model of a child workspace
+1. Backup and restore of workspaces
+   - Create and backup snapshots of workspace metadata.
+1. LDM Extension
+   - extend the Logical Data Model of a child workspace with custom datasets and fields
 
 In case you are not interested in incorporating a library in your own program but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
 

{gooddata_pipelines-1.52.1.dev2 → gooddata_pipelines-1.52.1.dev3}/README.md

@@ -10,9 +10,10 @@ You can use the package to manage following resources in GDC:
    - User/Group permissions
    - User Data Filters
    - Child workspaces (incl. Workspace Data Filter settings)
-1. _[PLANNED]:_ Backup and restore of workspaces
-1. _[PLANNED]:_ Custom fields management
-   - extend the Logical Data Model of a child workspace
+1. Backup and restore of workspaces
+   - Create and backup snapshots of workspace metadata.
+1. LDM Extension
+   - extend the Logical Data Model of a child workspace with custom datasets and fields
 
 In case you are not interested in incorporating a library in your own program but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
 

{gooddata_pipelines-1.52.1.dev2 → gooddata_pipelines-1.52.1.dev3}/gooddata_pipelines/__init__.py

@@ -10,6 +10,10 @@ from .backup_and_restore.models.storage import (
     S3StorageConfig,
     StorageType,
 )
+from .backup_and_restore.restore_manager import (
+    RestoreManager,
+    WorkspaceToRestore,
+)
 from .backup_and_restore.storage.local_storage import LocalStorage
 from .backup_and_restore.storage.s3_storage import S3Storage
 
@@ -57,6 +61,8 @@ from .provisioning.generic.provision import provision
 
 __all__ = [
     "BackupManager",
+    "RestoreManager",
+    "WorkspaceToRestore",
     "BackupRestoreConfig",
     "StorageType",
     "LocalStorage",

{gooddata_pipelines-1.52.1.dev2 → gooddata_pipelines-1.52.1.dev3}/gooddata_pipelines/api/gooddata_api.py

@@ -167,6 +167,17 @@ class ApiMethods:
         endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters"
         return self._get(endpoint)
 
+    def put_user_data_filters(
+        self, workspace_id: str, user_data_filters: dict[str, Any]
+    ) -> requests.Response:
+        """Puts the user data filters into GoodData workspace."""
+        headers = {**self.headers, "Content-Type": "application/json"}
+        return self._put(
+            f"/layout/workspaces/{workspace_id}/userDataFilters",
+            user_data_filters,
+            headers,
+        )
+
     def get_automations(self, workspace_id: str) -> requests.Response:
         """Gets the automations for a given workspace."""
         endpoint = (
@@ -174,6 +185,22 @@ class ApiMethods:
         )
         return self._get(endpoint)
 
+    def post_automation(
+        self, workspace_id: str, automation: dict[str, Any]
+    ) -> requests.Response:
+        """Posts an automation for a given workspace."""
+        endpoint = f"/entities/workspaces/{workspace_id}/automations"
+        return self._post(endpoint, automation)
+
+    def delete_automation(
+        self, workspace_id: str, automation_id: str
+    ) -> requests.Response:
+        """Deletes an automation for a given workspace."""
+        endpoint = (
+            f"/entities/workspaces/{workspace_id}/automations/{automation_id}"
+        )
+        return self._delete(endpoint)
+
     def get_all_metrics(self, workspace_id: str) -> requests.Response:
         """Get all metrics from the specified workspace.
 

{gooddata_pipelines-1.52.1.dev2 → gooddata_pipelines-1.52.1.dev3}/gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -1,23 +1,20 @@
 # (C) 2025 GoodData Corporation
 
-import json
 import os
 import shutil
 import tempfile
 import time
 import traceback
 from pathlib import Path
-from typing import Any, Type
+from typing import Any
 
 import attrs
 import requests
-import yaml
-from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
 
-from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataApi
 from gooddata_pipelines.backup_and_restore.backup_input_processor import (
     BackupInputProcessor,
 )
+from gooddata_pipelines.backup_and_restore.base_manager import BaseManager
 from gooddata_pipelines.backup_and_restore.constants import (
     BackupSettings,
     DirNames,
@@ -25,18 +22,10 @@ from gooddata_pipelines.backup_and_restore.constants import (
 from gooddata_pipelines.backup_and_restore.models.input_type import InputType
 from gooddata_pipelines.backup_and_restore.models.storage import (
     BackupRestoreConfig,
-    StorageType,
 )
 from gooddata_pipelines.backup_and_restore.storage.base_storage import (
     BackupStorage,
 )
-from gooddata_pipelines.backup_and_restore.storage.local_storage import (
-    LocalStorage,
-)
-from gooddata_pipelines.backup_and_restore.storage.s3_storage import (
-    S3Storage,
-)
-from gooddata_pipelines.logger import LogObserver
 from gooddata_pipelines.utils.rate_limiter import RateLimiter
 
 
@@ -45,16 +34,12 @@ class BackupBatch:
     list_of_ids: list[str]
 
 
-class BackupManager:
+class BackupManager(BaseManager):
     storage: BackupStorage
 
     def __init__(self, host: str, token: str, config: BackupRestoreConfig):
-        self._api = GoodDataApi(host, token)
-        self.logger = LogObserver()
-
-        self.config = config
+        super().__init__(host, token, config)
 
-        self.storage = self._get_storage(self.config)
         self.org_id = self._api.get_organization_id()
 
         self.loader = BackupInputProcessor(self._api, self.config.api_page_size)
@@ -63,39 +48,6 @@ class BackupManager:
             calls_per_second=self.config.api_calls_per_second,
         )
 
-    @classmethod
-    def create(
-        cls: Type["BackupManager"],
-        config: BackupRestoreConfig,
-        host: str,
-        token: str,
-    ) -> "BackupManager":
-        """Creates a backup worker instance using the provided host and token."""
-        return cls(host=host, token=token, config=config)
-
-    @classmethod
-    def create_from_profile(
-        cls: Type["BackupManager"],
-        config: BackupRestoreConfig,
-        profile: str = "default",
-        profiles_path: Path = PROFILES_FILE_PATH,
-    ) -> "BackupManager":
-        """Creates a backup worker instance using a GoodData profile file."""
-        content = profile_content(profile, profiles_path)
-        return cls(**content, config=config)
-
-    @staticmethod
-    def _get_storage(conf: BackupRestoreConfig) -> BackupStorage:
-        """Returns the storage class based on the storage type."""
-        if conf.storage_type == StorageType.S3:
-            return S3Storage(conf)
-        elif conf.storage_type == StorageType.LOCAL:
-            return LocalStorage(conf)
-        else:
-            raise RuntimeError(
-                f'Unsupported storage type "{conf.storage_type.value}".'
-            )
-
     def get_user_data_filters(self, ws_id: str) -> dict:
         """Returns the user data filters for the specified workspace."""
         with self._api_rate_limiter:
@@ -133,19 +85,13 @@ class BackupManager:
                 "user_data_filters",
                 filter["id"] + ".yaml",
             )
-            self._write_to_yaml(udf_file_path, filter)
+            self.yaml_utils.dump(udf_file_path, filter)
 
     @staticmethod
     def _move_folder(source: Path, destination: Path) -> None:
         """Moves the source folder to the destination."""
         shutil.move(source, destination)
 
-    @staticmethod
-    def _write_to_yaml(path: str, source: Any) -> None:
-        """Writes the source to a YAML file."""
-        with open(path, "w") as outfile:
-            yaml.dump(source, outfile)
-
     def _get_automations_from_api(self, workspace_id: str) -> Any:
         """Returns automations for the workspace as JSON."""
         with self._api_rate_limiter:
@@ -182,8 +128,7 @@ class BackupManager:
 
         # Store the automations in a JSON file
         if len(automations["data"]) > 0:
-            with open(automations_file_path, "w") as f:
-                json.dump(automations, f)
+            self.json_utils.dump(automations_file_path, automations)
 
     def store_declarative_filter_views(
         self, export_path: Path, workspace_id: str

gooddata_pipelines-1.52.1.dev3/gooddata_pipelines/backup_and_restore/base_manager.py

@@ -0,0 +1,73 @@
+# (C) 2025 GoodData Corporation
+
+import abc
+from pathlib import Path
+from typing import Type, TypeVar
+
+from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
+
+from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataApi
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    StorageType,
+)
+from gooddata_pipelines.backup_and_restore.storage.base_storage import (
+    BackupStorage,
+)
+from gooddata_pipelines.backup_and_restore.storage.local_storage import (
+    LocalStorage,
+)
+from gooddata_pipelines.backup_and_restore.storage.s3_storage import S3Storage
+from gooddata_pipelines.logger import LogObserver
+from gooddata_pipelines.utils.file_utils import JsonUtils, YamlUtils
+
+ManagerT = TypeVar("ManagerT", bound="BaseManager")
+
+
+class BaseManager(abc.ABC):
+    """Base class to provide constructors for backup and restore managers."""
+
+    storage: BackupStorage
+
+    def __init__(self, host: str, token: str, config: BackupRestoreConfig):
+        self.config = config
+
+        self._api: GoodDataApi = GoodDataApi(host, token)
+        self.logger: LogObserver = LogObserver()
+
+        self.storage = self._get_storage(self.config)
+
+        self.yaml_utils = YamlUtils()
+        self.json_utils = JsonUtils()
+
+    def _get_storage(self, conf: BackupRestoreConfig) -> BackupStorage:
+        """Returns the storage class based on the storage type."""
+        if conf.storage_type == StorageType.S3:
+            return S3Storage(conf)
+        elif conf.storage_type == StorageType.LOCAL:
+            return LocalStorage(conf)
+        else:
+            raise RuntimeError(
+                f'Unsupported storage type "{conf.storage_type.value}".'
+            )
+
+    @classmethod
+    def create(
+        cls: Type[ManagerT],
+        config: BackupRestoreConfig,
+        host: str,
+        token: str,
+    ) -> ManagerT:
+        """Creates a backup worker instance using the provided host and token."""
+        return cls(host=host, token=token, config=config)
+
+    @classmethod
+    def create_from_profile(
+        cls: Type[ManagerT],
+        config: BackupRestoreConfig,
+        profile: str = "default",
+        profiles_path: Path = PROFILES_FILE_PATH,
+    ) -> ManagerT:
+        """Creates a backup worker instance using a GoodData profile file."""
+        content = profile_content(profile, profiles_path)
+        return cls(host=content["host"], token=content["token"], config=config)

{gooddata_pipelines-1.52.1.dev2 → gooddata_pipelines-1.52.1.dev3}/gooddata_pipelines/backup_and_restore/constants.py

@@ -23,13 +23,14 @@ class DirNames:
 
 @attrs.frozen
 class ApiDefaults:
-    DEFAULT_PAGE_SIZE = 100
-    DEFAULT_BATCH_SIZE = 100
-    DEFAULT_API_CALLS_PER_SECOND = 1.0
+    PAGE_SIZE = 100
+    BATCH_SIZE = 100
+    CALLS_PER_SECOND = 1.0
 
 
 @attrs.frozen
-class BackupSettings(ApiDefaults):
+class BackupSettings:
+    API = ApiDefaults()
     MAX_RETRIES = 3
     RETRY_DELAY = 5  # seconds
     TIMESTAMP_SDK_FOLDER = (

gooddata_pipelines-1.52.1.dev3/gooddata_pipelines/backup_and_restore/models/storage.py

@@ -0,0 +1,156 @@
+# (C) 2025 GoodData Corporation
+
+from enum import Enum
+from typing import Annotated
+
+import yaml
+from pydantic import BaseModel, Field, model_validator
+
+from gooddata_pipelines.backup_and_restore.constants import BackupSettings
+
+
+class StorageType(Enum):
+    """Type of storage."""
+
+    S3 = "s3"
+    LOCAL = "local"
+
+
+class S3StorageConfig(BaseModel):
+    """Configuration for S3 storage.
+
+    Can be created using the following constructor methods:
+    - `from_iam_role`
+    - `from_aws_credentials`
+    - `from_aws_profile`
+    """
+
+    backup_path: str
+    bucket: str
+    profile: str | None = None
+    aws_access_key_id: str | None = None
+    aws_secret_access_key: str | None = None
+    aws_default_region: str = "us-east-1"
+
+    @classmethod
+    def from_iam_role(cls, backup_path: str, bucket: str) -> "S3StorageConfig":
+        """Use default IAM role or environment credentials.
+
+        Args:
+            backup_path: The path to the backup directory.
+            bucket: The name of the S3 bucket.
+
+        Returns:
+            S3StorageConfig: The S3 storage configuration.
+        """
+        return cls(backup_path=backup_path, bucket=bucket)
+
+    @classmethod
+    def from_aws_credentials(
+        cls,
+        backup_path: str,
+        bucket: str,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        aws_default_region: str,
+    ) -> "S3StorageConfig":
+        """Use explicit AWS access keys and region.
+
+        Args:
+            backup_path: The path to the backup directory.
+            bucket: The name of the S3 bucket.
+            aws_access_key_id: The AWS access key ID.
+            aws_secret_access_key: The AWS secret access key.
+            aws_default_region: The AWS default region.
+
+        Returns:
+            S3StorageConfig: The S3 storage configuration.
+        """
+        return cls(
+            backup_path=backup_path,
+            bucket=bucket,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            aws_default_region=aws_default_region,
+        )
+
+    @classmethod
+    def from_aws_profile(
+        cls, backup_path: str, bucket: str, profile: str
+    ) -> "S3StorageConfig":
+        """Use a named AWS CLI profile.
+
+        Args:
+            backup_path: The path to the backup directory.
+            bucket: The name of the S3 bucket.
+            profile: The name of the AWS profile.
+
+        Returns:
+            S3StorageConfig: The S3 storage configuration.
+        """
+        return cls(backup_path=backup_path, bucket=bucket, profile=profile)
+
+
+class LocalStorageConfig(BaseModel):
+    """Placeholder for local storage config."""
+
+    backup_path: str = Field(default="local_backups")
+
+
+class BackupRestoreConfig(BaseModel):
+    """Configuration for backup and restore.
+
+    Args:
+        storage_type: The type of storage to use. Defaults to `StorageType.LOCAL`.
+        storage: Storage configuration. Either `S3StorageConfig` or `LocalStorageConfig`. Defaults to `LocalStorageConfig()`.
+        api_page_size: The page size for fetching workspace relationships. Defaults to `BackupSettings.API.PAGE_SIZE`.
+        batch_size: The batch size for fetching workspace relationships. Defaults to `BackupSettings.API.BATCH_SIZE`.
+        api_calls_per_second: The maximum API calls per second (rate limiting). Defaults to `BackupSettings.API.CALLS_PER_SECOND`.
+    """
+
+    storage_type: StorageType = Field(default=StorageType.LOCAL)
+    storage: S3StorageConfig | LocalStorageConfig = Field(
+        default_factory=LocalStorageConfig
+    )
+    api_page_size: Annotated[
+        int,
+        Field(
+            gt=0,
+            description="Page size must be greater than 0",
+        ),
+    ] = Field(default=BackupSettings.API.PAGE_SIZE)
+    batch_size: Annotated[
+        int,
+        Field(
+            gt=0,
+            description="Batch size must be greater than 0",
+        ),
+    ] = Field(default=BackupSettings.API.BATCH_SIZE)
+    api_calls_per_second: Annotated[
+        float,
+        Field(
+            gt=0,
+            description="Maximum API calls per second (rate limiting)",
+        ),
+    ] = Field(default=BackupSettings.API.CALLS_PER_SECOND)
+
+    @classmethod
+    def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig":
+        with open(conf_path, "r") as stream:
+            conf: dict = yaml.safe_load(stream)
+        return cls(**conf)
+
+    @model_validator(mode="after")
+    def validate_storage(self) -> "BackupRestoreConfig":
+        """Check that the storage gets correct configuration when using S3 storage"""
+        if self.storage_type == StorageType.S3:
+            if not isinstance(self.storage, S3StorageConfig):
+                raise ValueError(
+                    "S3 storage must be configured with S3StorageConfig object"
+                )
+        elif self.storage_type == StorageType.LOCAL:
+            if not isinstance(self.storage, LocalStorageConfig):
+                raise ValueError(
+                    "Local storage must be configured with LocalStorageConfig object"
+                )
+        return self