gooddata-pipelines 1.52.1.dev1__py3-none-any.whl → 1.52.1.dev3__py3-none-any.whl

gooddata_pipelines/__init__.py

@@ -10,6 +10,10 @@ from .backup_and_restore.models.storage import (
     S3StorageConfig,
     StorageType,
 )
+from .backup_and_restore.restore_manager import (
+    RestoreManager,
+    WorkspaceToRestore,
+)
 from .backup_and_restore.storage.local_storage import LocalStorage
 from .backup_and_restore.storage.s3_storage import S3Storage
 
@@ -57,6 +61,8 @@ from .provisioning.generic.provision import provision
 
 __all__ = [
     "BackupManager",
+    "RestoreManager",
+    "WorkspaceToRestore",
     "BackupRestoreConfig",
     "StorageType",
     "LocalStorage",

gooddata_pipelines/api/gooddata_api.py

@@ -167,6 +167,17 @@ class ApiMethods:
         endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters"
         return self._get(endpoint)
 
+    def put_user_data_filters(
+        self, workspace_id: str, user_data_filters: dict[str, Any]
+    ) -> requests.Response:
+        """Puts the user data filters into GoodData workspace."""
+        headers = {**self.headers, "Content-Type": "application/json"}
+        return self._put(
+            f"/layout/workspaces/{workspace_id}/userDataFilters",
+            user_data_filters,
+            headers,
+        )
+
     def get_automations(self, workspace_id: str) -> requests.Response:
         """Gets the automations for a given workspace."""
         endpoint = (
@@ -174,6 +185,22 @@ class ApiMethods:
         )
         return self._get(endpoint)
 
+    def post_automation(
+        self, workspace_id: str, automation: dict[str, Any]
+    ) -> requests.Response:
+        """Posts an automation for a given workspace."""
+        endpoint = f"/entities/workspaces/{workspace_id}/automations"
+        return self._post(endpoint, automation)
+
+    def delete_automation(
+        self, workspace_id: str, automation_id: str
+    ) -> requests.Response:
+        """Deletes an automation for a given workspace."""
+        endpoint = (
+            f"/entities/workspaces/{workspace_id}/automations/{automation_id}"
+        )
+        return self._delete(endpoint)
+
     def get_all_metrics(self, workspace_id: str) -> requests.Response:
         """Get all metrics from the specified workspace.
 

gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -1,23 +1,20 @@
 # (C) 2025 GoodData Corporation
 
-import json
 import os
 import shutil
 import tempfile
 import time
 import traceback
 from pathlib import Path
-from typing import Any, Type
+from typing import Any
 
 import attrs
 import requests
-import yaml
-from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
 
-from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataApi
 from gooddata_pipelines.backup_and_restore.backup_input_processor import (
     BackupInputProcessor,
 )
+from gooddata_pipelines.backup_and_restore.base_manager import BaseManager
 from gooddata_pipelines.backup_and_restore.constants import (
     BackupSettings,
     DirNames,
@@ -25,18 +22,10 @@ from gooddata_pipelines.backup_and_restore.constants import (
 from gooddata_pipelines.backup_and_restore.models.input_type import InputType
 from gooddata_pipelines.backup_and_restore.models.storage import (
     BackupRestoreConfig,
-    StorageType,
 )
 from gooddata_pipelines.backup_and_restore.storage.base_storage import (
     BackupStorage,
 )
-from gooddata_pipelines.backup_and_restore.storage.local_storage import (
-    LocalStorage,
-)
-from gooddata_pipelines.backup_and_restore.storage.s3_storage import (
-    S3Storage,
-)
-from gooddata_pipelines.logger import LogObserver
 from gooddata_pipelines.utils.rate_limiter import RateLimiter
 
 
@@ -45,16 +34,12 @@ class BackupBatch:
     list_of_ids: list[str]
 
 
-class BackupManager:
+class BackupManager(BaseManager):
     storage: BackupStorage
 
     def __init__(self, host: str, token: str, config: BackupRestoreConfig):
-        self._api = GoodDataApi(host, token)
-        self.logger = LogObserver()
-
-        self.config = config
+        super().__init__(host, token, config)
 
-        self.storage = self._get_storage(self.config)
         self.org_id = self._api.get_organization_id()
 
         self.loader = BackupInputProcessor(self._api, self.config.api_page_size)
@@ -63,39 +48,6 @@ class BackupManager:
             calls_per_second=self.config.api_calls_per_second,
         )
 
-    @classmethod
-    def create(
-        cls: Type["BackupManager"],
-        config: BackupRestoreConfig,
-        host: str,
-        token: str,
-    ) -> "BackupManager":
-        """Creates a backup worker instance using the provided host and token."""
-        return cls(host=host, token=token, config=config)
-
-    @classmethod
-    def create_from_profile(
-        cls: Type["BackupManager"],
-        config: BackupRestoreConfig,
-        profile: str = "default",
-        profiles_path: Path = PROFILES_FILE_PATH,
-    ) -> "BackupManager":
-        """Creates a backup worker instance using a GoodData profile file."""
-        content = profile_content(profile, profiles_path)
-        return cls(**content, config=config)
-
-    @staticmethod
-    def _get_storage(conf: BackupRestoreConfig) -> BackupStorage:
-        """Returns the storage class based on the storage type."""
-        if conf.storage_type == StorageType.S3:
-            return S3Storage(conf)
-        elif conf.storage_type == StorageType.LOCAL:
-            return LocalStorage(conf)
-        else:
-            raise RuntimeError(
-                f'Unsupported storage type "{conf.storage_type.value}".'
-            )
-
     def get_user_data_filters(self, ws_id: str) -> dict:
         """Returns the user data filters for the specified workspace."""
         with self._api_rate_limiter:
@@ -133,19 +85,13 @@ class BackupManager:
                 "user_data_filters",
                 filter["id"] + ".yaml",
             )
-            self._write_to_yaml(udf_file_path, filter)
+            self.yaml_utils.dump(udf_file_path, filter)
 
     @staticmethod
     def _move_folder(source: Path, destination: Path) -> None:
         """Moves the source folder to the destination."""
         shutil.move(source, destination)
 
-    @staticmethod
-    def _write_to_yaml(path: str, source: Any) -> None:
-        """Writes the source to a YAML file."""
-        with open(path, "w") as outfile:
-            yaml.dump(source, outfile)
-
     def _get_automations_from_api(self, workspace_id: str) -> Any:
         """Returns automations for the workspace as JSON."""
         with self._api_rate_limiter:
@@ -182,8 +128,7 @@ class BackupManager:
 
         # Store the automations in a JSON file
         if len(automations["data"]) > 0:
-            with open(automations_file_path, "w") as f:
-                json.dump(automations, f)
+            self.json_utils.dump(automations_file_path, automations)
 
     def store_declarative_filter_views(
         self, export_path: Path, workspace_id: str

gooddata_pipelines/backup_and_restore/base_manager.py

@@ -0,0 +1,73 @@
+# (C) 2025 GoodData Corporation
+
+import abc
+from pathlib import Path
+from typing import Type, TypeVar
+
+from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
+
+from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataApi
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    StorageType,
+)
+from gooddata_pipelines.backup_and_restore.storage.base_storage import (
+    BackupStorage,
+)
+from gooddata_pipelines.backup_and_restore.storage.local_storage import (
+    LocalStorage,
+)
+from gooddata_pipelines.backup_and_restore.storage.s3_storage import S3Storage
+from gooddata_pipelines.logger import LogObserver
+from gooddata_pipelines.utils.file_utils import JsonUtils, YamlUtils
+
+ManagerT = TypeVar("ManagerT", bound="BaseManager")
+
+
+class BaseManager(abc.ABC):
+    """Base class to provide constructors for backup and restore managers."""
+
+    storage: BackupStorage
+
+    def __init__(self, host: str, token: str, config: BackupRestoreConfig):
+        self.config = config
+
+        self._api: GoodDataApi = GoodDataApi(host, token)
+        self.logger: LogObserver = LogObserver()
+
+        self.storage = self._get_storage(self.config)
+
+        self.yaml_utils = YamlUtils()
+        self.json_utils = JsonUtils()
+
+    def _get_storage(self, conf: BackupRestoreConfig) -> BackupStorage:
+        """Returns the storage class based on the storage type."""
+        if conf.storage_type == StorageType.S3:
+            return S3Storage(conf)
+        elif conf.storage_type == StorageType.LOCAL:
+            return LocalStorage(conf)
+        else:
+            raise RuntimeError(
+                f'Unsupported storage type "{conf.storage_type.value}".'
+            )
+
+    @classmethod
+    def create(
+        cls: Type[ManagerT],
+        config: BackupRestoreConfig,
+        host: str,
+        token: str,
+    ) -> ManagerT:
+        """Creates a backup worker instance using the provided host and token."""
+        return cls(host=host, token=token, config=config)
+
+    @classmethod
+    def create_from_profile(
+        cls: Type[ManagerT],
+        config: BackupRestoreConfig,
+        profile: str = "default",
+        profiles_path: Path = PROFILES_FILE_PATH,
+    ) -> ManagerT:
+        """Creates a backup worker instance using a GoodData profile file."""
+        content = profile_content(profile, profiles_path)
+        return cls(host=content["host"], token=content["token"], config=config)

gooddata_pipelines/backup_and_restore/constants.py

@@ -23,13 +23,14 @@ class DirNames:
 
 @attrs.frozen
 class ApiDefaults:
-    DEFAULT_PAGE_SIZE = 100
-    DEFAULT_BATCH_SIZE = 100
-    DEFAULT_API_CALLS_PER_SECOND = 1.0
+    PAGE_SIZE = 100
+    BATCH_SIZE = 100
+    CALLS_PER_SECOND = 1.0
 
 
 @attrs.frozen
-class BackupSettings(ApiDefaults):
+class BackupSettings:
+    API = ApiDefaults()
     MAX_RETRIES = 3
     RETRY_DELAY = 5  # seconds
     TIMESTAMP_SDK_FOLDER = (

gooddata_pipelines/backup_and_restore/models/storage.py

@@ -1,10 +1,10 @@
 # (C) 2025 GoodData Corporation
 
 from enum import Enum
-from typing import Annotated, TypeAlias, Optional
+from typing import Annotated
 
 import yaml
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 
 from gooddata_pipelines.backup_and_restore.constants import BackupSettings
 
@@ -17,18 +17,32 @@ class StorageType(Enum):
 
 
 class S3StorageConfig(BaseModel):
-    """Configuration for S3 storage."""
+    """Configuration for S3 storage.
+
+    Can be created using the following constructor methods:
+    - `from_iam_role`
+    - `from_aws_credentials`
+    - `from_aws_profile`
+    """
 
     backup_path: str
     bucket: str
-    profile: Optional[str] = None
-    aws_access_key_id: Optional[str] = None
-    aws_secret_access_key: Optional[str] = None
-    aws_default_region: Optional[str] = "us-east-1"
+    profile: str | None = None
+    aws_access_key_id: str | None = None
+    aws_secret_access_key: str | None = None
+    aws_default_region: str = "us-east-1"
 
     @classmethod
     def from_iam_role(cls, backup_path: str, bucket: str) -> "S3StorageConfig":
-        """Use default IAM role or environment credentials."""
+        """Use default IAM role or environment credentials.
+
+        Args:
+            backup_path: The path to the backup directory.
+            bucket: The name of the S3 bucket.
+
+        Returns:
+            S3StorageConfig: The S3 storage configuration.
+        """
         return cls(backup_path=backup_path, bucket=bucket)
 
     @classmethod
@@ -40,7 +54,18 @@ class S3StorageConfig(BaseModel):
         aws_secret_access_key: str,
         aws_default_region: str,
     ) -> "S3StorageConfig":
-        """Use explicit AWS access keys and region."""
+        """Use explicit AWS access keys and region.
+
+        Args:
+            backup_path: The path to the backup directory.
+            bucket: The name of the S3 bucket.
+            aws_access_key_id: The AWS access key ID.
+            aws_secret_access_key: The AWS secret access key.
+            aws_default_region: The AWS default region.
+
+        Returns:
+            S3StorageConfig: The S3 storage configuration.
+        """
         return cls(
             backup_path=backup_path,
             bucket=bucket,
@@ -53,46 +78,79 @@ class S3StorageConfig(BaseModel):
     def from_aws_profile(
         cls, backup_path: str, bucket: str, profile: str
     ) -> "S3StorageConfig":
-        """Use a named AWS CLI profile."""
+        """Use a named AWS CLI profile.
+
+        Args:
+            backup_path: The path to the backup directory.
+            bucket: The name of the S3 bucket.
+            profile: The name of the AWS profile.
+
+        Returns:
+            S3StorageConfig: The S3 storage configuration.
+        """
         return cls(backup_path=backup_path, bucket=bucket, profile=profile)
 
 
 class LocalStorageConfig(BaseModel):
     """Placeholder for local storage config."""
 
-
-StorageConfig: TypeAlias = S3StorageConfig | LocalStorageConfig
+    backup_path: str = Field(default="local_backups")
 
 
 class BackupRestoreConfig(BaseModel):
-    """Configuration for backup and restore."""
-
-    storage_type: StorageType
-    storage: StorageConfig | None = Field(default=None)
+    """Configuration for backup and restore.
+
+    Args:
+        storage_type: The type of storage to use. Defaults to `StorageType.LOCAL`.
+        storage: Storage configuration. Either `S3StorageConfig` or `LocalStorageConfig`. Defaults to `LocalStorageConfig()`.
+        api_page_size: The page size for fetching workspace relationships. Defaults to `BackupSettings.API.PAGE_SIZE`.
+        batch_size: The batch size for fetching workspace relationships. Defaults to `BackupSettings.API.BATCH_SIZE`.
+        api_calls_per_second: The maximum API calls per second (rate limiting). Defaults to `BackupSettings.API.CALLS_PER_SECOND`.
+    """
+
+    storage_type: StorageType = Field(default=StorageType.LOCAL)
+    storage: S3StorageConfig | LocalStorageConfig = Field(
+        default_factory=LocalStorageConfig
+    )
     api_page_size: Annotated[
         int,
         Field(
             gt=0,
             description="Page size must be greater than 0",
         ),
-    ] = Field(default=BackupSettings.DEFAULT_PAGE_SIZE)
+    ] = Field(default=BackupSettings.API.PAGE_SIZE)
     batch_size: Annotated[
         int,
         Field(
             gt=0,
             description="Batch size must be greater than 0",
         ),
-    ] = Field(default=BackupSettings.DEFAULT_BATCH_SIZE)
+    ] = Field(default=BackupSettings.API.BATCH_SIZE)
     api_calls_per_second: Annotated[
         float,
         Field(
             gt=0,
             description="Maximum API calls per second (rate limiting)",
         ),
-    ] = Field(default=BackupSettings.DEFAULT_API_CALLS_PER_SECOND)
+    ] = Field(default=BackupSettings.API.CALLS_PER_SECOND)
 
     @classmethod
     def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig":
         with open(conf_path, "r") as stream:
             conf: dict = yaml.safe_load(stream)
         return cls(**conf)
+
+    @model_validator(mode="after")
+    def validate_storage(self) -> "BackupRestoreConfig":
+        """Check that the storage gets correct configuration when using S3 storage"""
+        if self.storage_type == StorageType.S3:
+            if not isinstance(self.storage, S3StorageConfig):
+                raise ValueError(
+                    "S3 storage must be configured with S3StorageConfig object"
+                )
+        elif self.storage_type == StorageType.LOCAL:
+            if not isinstance(self.storage, LocalStorageConfig):
+                raise ValueError(
+                    "Local storage must be configured with LocalStorageConfig object"
+                )
+        return self

gooddata_pipelines/backup_and_restore/restore_manager.py

@@ -0,0 +1,266 @@
+# (C) 2025 GoodData Corporation
+
+import os
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any
+
+import attrs
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.analytics_model.analytics_model import (
+    CatalogDeclarativeAnalytics,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.automation import (
+    CatalogDeclarativeAutomation,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm import (
+    CatalogDeclarativeModel,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.workspace import (
+    CatalogDeclarativeFilterView,
+)
+from pydantic import BaseModel, ConfigDict
+
+from gooddata_pipelines.backup_and_restore.base_manager import BaseManager
+from gooddata_pipelines.backup_and_restore.constants import DirNames
+from gooddata_pipelines.utils.decorators import log_and_reraise_exception
+
+
+@attrs.define
+class WorkspaceModel:
+    logical_data_model: CatalogDeclarativeModel
+    analytics_model: CatalogDeclarativeAnalytics
+
+
+class WorkspaceToRestore(BaseModel):
+    """Workspace to restore.
+
+    Args:
+        id: The ID of the workspace to restore.
+        path: The path to the folder containing the `gooddata_layouts.zip` file
+            to restore. Should be a continuation of the `backup_path` specified
+            in the storage configuration. Typically, it would look something like
+            `organization_id/workspace_id/backup_timestamp`
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    path: str
+
+
+class RestoreManager(BaseManager):
+    """Restores previsouly created backups of workspace metadata."""
+
+    @log_and_reraise_exception("Failed to extract backup from zip archive.")
+    def _extract_zip_archive(
+        self, file_to_extract: Path, destination: Path
+    ) -> None:
+        """Extracts the backup from zip archive."""
+        with zipfile.ZipFile(file_to_extract, "r") as zip_ref:
+            zip_ref.extractall(destination)
+
+    def _check_workspace_is_valid(self, workspace_root_dir_path: Path) -> None:
+        """Checks if the workspace layout is valid."""
+        if (
+            not workspace_root_dir_path.exists()
+            or not workspace_root_dir_path.is_dir()
+        ):
+            self.logger.error(
+                "Invalid source path found upon backup fetch. "
+                f"Got {workspace_root_dir_path}. "
+                "Check if target zip contains gooddata_layouts directory."
+            )
+            raise RuntimeError("Invalid source path upon load.")
+
+        children = list(workspace_root_dir_path.iterdir())
+        am_path = workspace_root_dir_path / DirNames.AM
+        ldm_path = workspace_root_dir_path / DirNames.LDM
+        udf_path = workspace_root_dir_path / DirNames.UDF
+
+        if (
+            am_path not in children
+            or ldm_path not in children
+            or udf_path not in children
+        ):
+            self.logger.error(
+                f"{DirNames.AM} or {DirNames.LDM} directory missing in the "
+                + "workspace hierarchy. "
+            )
+            raise RuntimeError(
+                f"{DirNames.AM} or {DirNames.LDM} directory missing."
+            )
+
+    @log_and_reraise_exception("Failed to load workspace declaration.")
+    def _load_workspace_layout(
+        self, workspace_root_dir_path: Path
+    ) -> WorkspaceModel:
+        """Loads the workspace layout from the backup."""
+        sdk_catalog = self._api._sdk.catalog_workspace_content
+
+        ldm = sdk_catalog.load_ldm_from_disk(workspace_root_dir_path)
+        am = sdk_catalog.load_analytics_model_from_disk(workspace_root_dir_path)
+
+        return WorkspaceModel(logical_data_model=ldm, analytics_model=am)
+
+    @log_and_reraise_exception("Failed to load user data filters from folder.")
+    def _load_user_data_filters(self, workspace_root_dir_path: Path) -> dict:
+        user_data_filters: dict = {"userDataFilters": []}
+        user_data_filters_folder = os.path.join(
+            workspace_root_dir_path, DirNames.UDF
+        )
+        for filename in os.listdir(user_data_filters_folder):
+            file_path = os.path.join(user_data_filters_folder, filename)
+            user_data_filter = self.yaml_utils.safe_load(Path(file_path))
+            user_data_filters["userDataFilters"].append(user_data_filter)
+
+        return user_data_filters
+
+    @log_and_reraise_exception("Failed to put workspace layout into GoodData.")
+    def _put_workspace_layout(
+        self, workspace_id: str, workspace_model: WorkspaceModel
+    ) -> None:
+        """Puts the workspace layout into GoodData."""
+        self._api._sdk.catalog_workspace_content.put_declarative_ldm(
+            workspace_id, workspace_model.logical_data_model
+        )
+        self._api._sdk.catalog_workspace_content.put_declarative_analytics_model(
+            workspace_id, workspace_model.analytics_model
+        )
+
+    @log_and_reraise_exception("Failed to put user data filters.")
+    def _put_user_data_filters(
+        self, workspace_id: str, user_data_filters: dict
+    ) -> None:
+        """Puts the user data filters into GoodData workspace."""
+        response = self._api.put_user_data_filters(
+            workspace_id, user_data_filters
+        )
+        self._api.raise_if_response_not_ok(response)
+
+    def _load_and_put_filter_views(
+        self, workspace_id: str, workspace_root_dir_path: Path
+    ) -> None:
+        """Loads and puts filter views into GoodData workspace."""
+        filter_views: list[CatalogDeclarativeFilterView] = []
+        if not (workspace_root_dir_path / "filter_views").exists():
+            # Skip if the filter_views directory does not exist
+            return
+
+        for file in Path(workspace_root_dir_path / "filter_views").iterdir():
+            filter_view_content: dict[str, Any] = dict(
+                self.yaml_utils.safe_load(file)
+            )
+            filter_view: CatalogDeclarativeFilterView = (
+                CatalogDeclarativeFilterView.from_dict(filter_view_content)
+            )
+            filter_views.append(filter_view)
+
+        if filter_views:
+            self._api._sdk.catalog_workspace.put_declarative_filter_views(
+                workspace_id, filter_views
+            )
+
+    def _load_and_post_automations(
+        self, workspace_id: str, workspace_root_dir_path: Path
+    ) -> None:
+        """Loads automations from specified json file and creates them in the workspace."""
+        # Load automations from JSON
+        path_to_json: Path = Path(
+            workspace_root_dir_path, "automations", "automations.json"
+        )
+
+        # Both the folder and the file must exist, otherwise skip
+        if not (workspace_root_dir_path.exists() and path_to_json.exists()):
+            return
+
+        # Delete all automations from the workspace and restore the automations from the backup.
+        self._delete_all_automations(workspace_id)
+
+        data: dict = self.json_utils.load(path_to_json)
+        automations: list[dict] = data["data"]
+
+        for automation in automations:
+            self._post_automation(workspace_id, automation)
+
+    def _delete_all_automations(self, workspace_id: str) -> None:
+        """Deletes all automations in the workspace."""
+        automations: list[CatalogDeclarativeAutomation] = (
+            self._api._sdk.catalog_workspace.get_declarative_automations(
+                workspace_id
+            )
+        )
+        for automation in automations:
+            self._api.delete_automation(workspace_id, automation.id)
+
+    def _post_automation(self, workspace_id: str, automation: dict) -> None:
+        """Posts a scheduled export to the workspace."""
+        attributes: dict = automation["attributes"]
+        relationships: dict = automation["relationships"]
+        id: str = automation["id"]
+
+        if attributes.get("schedule"):
+            if attributes["schedule"].get("cronDescription"):
+                # The cron description attribute is causing a 500 ("No mapping found...")
+                # error. Known and reported issue.
+                del attributes["schedule"]["cronDescription"]
+
+        data = {
+            "data": {
+                "attributes": attributes,
+                "id": id,
+                "type": "automation",
+                "relationships": relationships,
+            }
+        }
+
+        response = self._api.post_automation(workspace_id, data)
+
+        if not response.ok:
+            self.logger.error(
+                f"Failed to post automation ({response.status_code}): {response.text}"
+            )
+
+    def _restore_backup(
+        self, workspace_to_restore: WorkspaceToRestore, tempdir_path: Path
+    ) -> None:
+        """Restores the backup of a workspace."""
+
+        zip_target = tempdir_path / f"{DirNames.LAYOUTS}.zip"
+        src_path = tempdir_path / DirNames.LAYOUTS
+
+        try:
+            self.storage.get_ws_declaration(
+                workspace_to_restore.path, str(zip_target)
+            )
+            self._extract_zip_archive(zip_target, tempdir_path)
+            self._check_workspace_is_valid(src_path)
+            workspace_model: WorkspaceModel = self._load_workspace_layout(
+                src_path
+            )
+            user_data_filters = self._load_user_data_filters(src_path)
+            self._put_workspace_layout(workspace_to_restore.id, workspace_model)
+            self._put_user_data_filters(
+                workspace_to_restore.id, user_data_filters
+            )
+            self._load_and_put_filter_views(workspace_to_restore.id, src_path)
+            self._load_and_post_automations(workspace_to_restore.id, src_path)
+            self.logger.info(
+                f"Finished backup restore of {workspace_to_restore.id} from {workspace_to_restore.path}."
+            )
+        except Exception as e:
+            self.logger.error(
+                f"Failed to restore backup of {workspace_to_restore.id} from {workspace_to_restore.path}. "
+                f"Error caused by {e.__class__.__name__}: {e}."
+            )
+
+    def restore(self, workspaces_to_restore: list[WorkspaceToRestore]) -> None:
+        """Restores the backups of workspaces.
+
+        Args:
+            workspaces_to_restore: List of workspaces to restore.
+        """
+        for workspace_to_restore in workspaces_to_restore:
+            with tempfile.TemporaryDirectory() as tempdir:
+                tempdir_path = Path(tempdir)
+                self._restore_backup(workspace_to_restore, tempdir_path)

gooddata_pipelines/backup_and_restore/storage/base_storage.py

@@ -12,7 +12,18 @@ class BackupStorage(abc.ABC):
     def __init__(self, conf: BackupRestoreConfig):
         self.logger = LogObserver()
 
+        suffix = "/" if not conf.storage.backup_path.endswith("/") else ""
+        self._backup_path = conf.storage.backup_path + suffix
+
     @abc.abstractmethod
     def export(self, folder: str, org_id: str) -> None:
         """Exports the content of the folder to the storage."""
         raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_ws_declaration(
+        self, target_path: str, local_target_path: str
+    ) -> None:
+        raise NotImplementedError(
+            "This method should be implemented by the subclass."
+        )

gooddata_pipelines/backup_and_restore/storage/local_storage.py

@@ -5,6 +5,7 @@ from pathlib import Path
 
 from gooddata_pipelines.backup_and_restore.models.storage import (
     BackupRestoreConfig,
+    LocalStorageConfig,
 )
 from gooddata_pipelines.backup_and_restore.storage.base_storage import (
     BackupStorage,
@@ -14,24 +15,34 @@ from gooddata_pipelines.backup_and_restore.storage.base_storage import (
 class LocalStorage(BackupStorage):
     def __init__(self, conf: BackupRestoreConfig):
         super().__init__(conf)
+        if not isinstance(conf.storage, LocalStorageConfig):
+            raise ValueError("Local storage config is required")
+        self._config: LocalStorageConfig = conf.storage
 
-    def _export(
-        self, folder: str, org_id: str, export_folder: str = "local_backups"
-    ) -> None:
+    def _export(self, folder: str, org_id: str, export_folder: str) -> None:
         """Copies the content of the folder to local storage as backup."""
+
         self.logger.info(f"Saving {org_id} to local storage")
         shutil.copytree(
             Path(folder), Path(Path.cwd(), export_folder), dirs_exist_ok=True
         )
 
-    def export(
-        self, folder: str, org_id: str, export_folder: str = "local_backups"
-    ) -> None:
+    def export(self, folder: str, org_id: str) -> None:
         """Copies the content of the folder to local storage as backup."""
         try:
-            self._export(folder, org_id, export_folder)
+            self._export(folder, org_id, self._config.backup_path)
         except Exception as e:
             self.logger.error(
-                f"Error exporting {folder} to {export_folder}: {e}"
+                f"Error exporting {folder} to {self._config.backup_path}: {e}"
             )
             raise
+
+    def get_ws_declaration(
+        self, target_path: str, local_target_path: str
+    ) -> None:
+        """Retrieves workspace declaration from local storage and copies to the local target path.
+
+        The local target should be a temporary directory.
+        """
+        file_to_copy = self._backup_path + target_path + "/gooddata_layouts.zip"
+        shutil.copy(file_to_copy, local_target_path)

gooddata_pipelines/backup_and_restore/storage/s3_storage.py

@@ -25,8 +25,6 @@ class S3Storage(BackupStorage):
         self._client = self._session.client("s3")
         self._resource = self._session.resource("s3")
         self._bucket = self._resource.Bucket(self._config.bucket)  # type: ignore [missing library stubs]
-        suffix = "/" if not self._config.backup_path.endswith("/") else ""
-        self._backup_path = self._config.backup_path + suffix
 
         self._verify_connection()
 
@@ -95,3 +93,28 @@ class S3Storage(BackupStorage):
                     self._client.put_object(
                         Bucket=self._config.bucket, Key=export_path, Body=data
                     )
+
+    def get_ws_declaration(
+        self, target_path: str, local_target_path: str
+    ) -> None:
+        """Retrieves workspace declaration from S3 bucket."""
+        target_s3_prefix = f"{self._backup_path}{target_path}"
+
+        objs_found = list(self._bucket.objects.filter(Prefix=target_s3_prefix))
+
+        # Remove the included directory (which equals prefix) on hit
+        objs_found = objs_found[1:] if len(objs_found) > 0 else objs_found
+
+        if not objs_found:
+            message = f"No target backup found for {target_s3_prefix}."
+            self.logger.error(message)
+            raise Exception(message)
+
+        if len(objs_found) > 1:
+            self.logger.warning(
+                f"Multiple backups found at {target_s3_prefix}."
+                " Continuing with the first one, ignoring the rest..."
+            )
+
+        s3_obj = objs_found[0]
+        self._bucket.download_file(s3_obj.key, local_target_path)

gooddata_pipelines/provisioning/utils/utils.py

@@ -26,10 +26,11 @@ class AttributesMixin:
         Returns:
             dict: Returns a dictionary of the objects' attributes.
         """
-        # TODO: This might not work great with nested objects, values  which are lists of objects etc.
-        # If we care about parsing the logs back from the string, we should consider some other approach
         attributes: dict[str, str] = {}
-        for context_object in objects:
+        for index, context_object in enumerate(objects):
+            if isinstance(context_object, str):
+                attributes[f"string_context_{index}"] = context_object
+
             if isinstance(context_object, Response):
                 # for request.Response objects, keys need to be renamed to match the log schema
                 attributes.update(
@@ -48,10 +49,12 @@ class AttributesMixin:
                     cast(attrs.AttrsInstance, context_object)
                 ).items():
                     self._add_to_dict(attributes, key, value)
-            else:
+            elif hasattr(context_object, "__dict__"):
                 # Generic handling for other objects
                 for key, value in context_object.__dict__.items():
                     self._add_to_dict(attributes, key, value)
+            else:
+                attributes[f"object_{index}"] = str(context_object)
 
         if overrides:
             attributes.update(overrides)

gooddata_pipelines/utils/decorators.py

@@ -0,0 +1,30 @@
+# (C) 2025 GoodData Corporation
+
+from typing import Any, Callable
+
+from gooddata_pipelines.logger.logger import LogObserver
+
+logger: LogObserver = LogObserver()
+
+
+def log_and_reraise_exception(message: str) -> Callable:
+    """
+    Decorator to log an exception and re-raise it.
+
+    Args:
+        message (str): The message to log.
+    """
+
+    def decorator(fn: Callable) -> Callable:
+        def wrapper(*method_args: Any, **method_kwargs: Any) -> Callable:
+            try:
+                return fn(*method_args, **method_kwargs)
+            except Exception:
+                logger.error(
+                    f"{message}, {fn.__name__}, Args: {method_args}, Kwargs: {method_kwargs}"
+                )
+                raise
+
+        return wrapper
+
+    return decorator

gooddata_pipelines/utils/file_utils.py

@@ -0,0 +1,63 @@
+# (C) 2025 GoodData Corporation
+
+import json
+from pathlib import Path
+from typing import Any
+
+import attrs
+import yaml
+
+
+class PathUtils:
+    """Handles common path operations."""
+
+    @staticmethod
+    def validate_path(path: str | Path) -> Path:
+        """Validates a path."""
+        if not isinstance(path, Path):
+            path = Path(path)
+
+        return path
+
+    def check_path_exists(self, path: Path) -> None:
+        """Checks if a path exists."""
+        if not self.validate_path(path).exists():
+            raise FileNotFoundError(f"File {path} does not exist.")
+
+
+@attrs.define
+class JsonUtils:
+    """Handles common JSON interactions."""
+
+    path_utils: PathUtils = attrs.field(factory=PathUtils)
+
+    def load(self, path: Path) -> Any:
+        """Loads a JSON file."""
+        self.path_utils.check_path_exists(path)
+
+        with open(path, "r") as f:
+            return json.load(f)
+
+    def dump(self, path: Path, data: Any) -> None:
+        """Writes the source to a JSON file."""
+        with open(path, "w") as output_file:
+            json.dump(data, output_file)
+
+
+@attrs.define
+class YamlUtils:
+    """Handles common YMAL interactions."""
+
+    path_utils: PathUtils = attrs.field(factory=PathUtils)
+
+    def safe_load(self, path: Path) -> Any:
+        """Safe loads a YAML file."""
+        self.path_utils.check_path_exists(path)
+
+        with open(path, "r") as f:
+            return yaml.safe_load(f)
+
+    def dump(self, path: str, data: Any) -> None:
+        """Writes the source to a YAML file."""
+        with open(path, "w") as output_file:
+            yaml.dump(data, output_file)

{gooddata_pipelines-1.52.1.dev1.dist-info → gooddata_pipelines-1.52.1.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.52.1.dev1
+Version: 1.52.1.dev3
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License: MIT
@@ -8,7 +8,7 @@ License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.52.1.dev1
+Requires-Dist: gooddata-sdk~=1.52.1.dev3
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326
@@ -27,9 +27,10 @@ You can use the package to manage following resources in GDC:
    - User/Group permissions
    - User Data Filters
    - Child workspaces (incl. Workspace Data Filter settings)
-1. _[PLANNED]:_ Backup and restore of workspaces
-1. _[PLANNED]:_ Custom fields management
-   - extend the Logical Data Model of a child workspace
+1. Backup and restore of workspaces
+   - Create and backup snapshots of workspace metadata.
+1. LDM Extension
+   - extend the Logical Data Model of a child workspace with custom datasets and fields
 
 In case you are not interested in incorporating a library in your own program but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
 

{gooddata_pipelines-1.52.1.dev1.dist-info → gooddata_pipelines-1.52.1.dev3.dist-info}/RECORD

@@ -1,25 +1,27 @@
-gooddata_pipelines/__init__.py,sha256=dGbyckzT8MrxL5SljSOj2jo5DFXZ2HYn0kBbAe0vO7s,2602
+gooddata_pipelines/__init__.py,sha256=2vDjg0bX5qOdAFbr3H9Q2IuoTk2_kH-bfRK_miVtcoY,2746
 gooddata_pipelines/_version.py,sha256=Zi8Ht5ofjFeSYGG5USixQtJNB1po6okh0Rez8VyAsFM,200
 gooddata_pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gooddata_pipelines/api/__init__.py,sha256=0WaBI2XMdkkZgnUsQ9kqipNzh2l2zamZvUt_qjp8xCk,106
 gooddata_pipelines/api/exceptions.py,sha256=rddQXfv8Ktckz7RONKBnKfm53M7dzPCh50Dl1k-8hqs,1545
-gooddata_pipelines/api/gooddata_api.py,sha256=8AZ5-mGGvo_4pPFjaf_DxkzQQqp2dRtiRPTM2sIdfYs,10934
+gooddata_pipelines/api/gooddata_api.py,sha256=-A_yqk8JbfNW16B4egXvJcWSSzYSS8fwVk1Bc2AsEEU,11956
 gooddata_pipelines/api/gooddata_api_wrapper.py,sha256=t7dFrXJ6X4yXS9XDthOmvd2CyzdnDDNPeIngTEW72YU,1152
 gooddata_pipelines/api/gooddata_sdk.py,sha256=wd5O4e9BQLWUawt6odrs5a51nqFGthBkvqh9WOiW36Q,13734
 gooddata_pipelines/api/utils.py,sha256=3QY_aYH17I9THoCINE3l-n5oj52k-gNeT1wv6Z_VxN8,1433
 gooddata_pipelines/backup_and_restore/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
 gooddata_pipelines/backup_and_restore/backup_input_processor.py,sha256=iUB-bvYMsOc6FGiHsIzHPWkz-TGM8VontudlVWzmBDI,7921
-gooddata_pipelines/backup_and_restore/backup_manager.py,sha256=ubYgePX72pg4hCMPXCeX8_T6QHHhQS8fMbHx__nbfso,15209
-gooddata_pipelines/backup_and_restore/constants.py,sha256=SpCll6C-QcWYYVhcYZOyOGG7ELP6M87ioarbUr1nN9I,833
+gooddata_pipelines/backup_and_restore/backup_manager.py,sha256=yhUqw_DnJ8PYBUPPH_10tgjaoGToWCbty6cLlG2PS-A,13351
+gooddata_pipelines/backup_and_restore/base_manager.py,sha256=dceZQ-ACSuLtrt-GO9wS9rnRv4uIEVrBCikD87LQlJs,2448
+gooddata_pipelines/backup_and_restore/constants.py,sha256=tEFpz-GavUvyZADMb_zGsoxbRs1wQ4CDuQkeXRGmpyA,816
 gooddata_pipelines/backup_and_restore/csv_reader.py,sha256=0Kw7mJT7REj3Gjqfsc6YT9MbhcqfCGNB_SKBwzTI1rk,1268
+gooddata_pipelines/backup_and_restore/restore_manager.py,sha256=itvhu7MjxwdX0fJxA4F-g8BjxIpe8cxG2CVsYmq37z0,10491
 gooddata_pipelines/backup_and_restore/models/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
 gooddata_pipelines/backup_and_restore/models/input_type.py,sha256=CBKJigKdmZ-NJD9MSfNhq89bo86W0AqCMMoyonbd1QA,239
-gooddata_pipelines/backup_and_restore/models/storage.py,sha256=BcgOGIk4u3EaH0u0gArDHQpDyIPjx_c3fmoc-i_Ptj4,2795
+gooddata_pipelines/backup_and_restore/models/storage.py,sha256=kESVWGM_qq6HC_BCbrxsN6IyAFE4G4nPC96ggQy05ZI,5107
 gooddata_pipelines/backup_and_restore/models/workspace_response.py,sha256=eQbYLgRQc17IRG0yPTAJVrD-Xs05SzuwtzoNrPT2DoY,833
 gooddata_pipelines/backup_and_restore/storage/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
-gooddata_pipelines/backup_and_restore/storage/base_storage.py,sha256=67wdItlG3neExeb_eCUDQhswdUB62X5Nyj9sOImB_Hg,487
-gooddata_pipelines/backup_and_restore/storage/local_storage.py,sha256=NvhPRzRAvuSpc5qCDyPqZaMB0i1jeZOZczaSwjUSGEg,1155
-gooddata_pipelines/backup_and_restore/storage/s3_storage.py,sha256=ZAysu4sPMAvdWs3RUroHHp2XZLHeU_LhJ5qBHlBQ7n4,3732
+gooddata_pipelines/backup_and_restore/storage/base_storage.py,sha256=lzkfjsp_WZQS1U-178iSlzDplZ-ejsoodLQscK1Vd9s,858
+gooddata_pipelines/backup_and_restore/storage/local_storage.py,sha256=LYM8uZCQKwe_SiItVSxKr7DqaoW-1ePhHjB3usQgFck,1703
+gooddata_pipelines/backup_and_restore/storage/s3_storage.py,sha256=mcd91cHB8NMgo2t_F_DxMXqsCiBhSXCbULxWgaVAXr0,4535
 gooddata_pipelines/ldm_extension/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
 gooddata_pipelines/ldm_extension/input_processor.py,sha256=lNIx6YfU4OJpSLyAitCoPwwf6eFIT6OyivRnqYX5O-o,11678
 gooddata_pipelines/ldm_extension/input_validator.py,sha256=sAl-tixrS69G_lP19U9CjKHiWZinXOcjeAqwiydVctQ,7459
@@ -58,10 +60,12 @@ gooddata_pipelines/provisioning/generic/provision.py,sha256=TgBFbOroG9nHVhHQkqAm
 gooddata_pipelines/provisioning/utils/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
 gooddata_pipelines/provisioning/utils/context_objects.py,sha256=HJoeumH_gXwM6X-GO3HkC4w-6RYozz6-aqQOhDnu7no,879
 gooddata_pipelines/provisioning/utils/exceptions.py,sha256=1WnAOlPhqOf0xRcvn70lxAlLb8Oo6m6WCYS4hj9uzDU,3630
-gooddata_pipelines/provisioning/utils/utils.py,sha256=u-nVVp6ykY4MZqRXBoPCKLrFlUunLF-cugF9SpSzL1E,2766
+gooddata_pipelines/provisioning/utils/utils.py,sha256=OSskOvxx4Eu5_khLxuby6t5YGR3qJa-h5G6m7pVOUgQ,2818
 gooddata_pipelines/utils/__init__.py,sha256=s9TtSjKqo1gSGWOVoGrXaGi1TsbRowjRDYKtjmKy7BY,155
+gooddata_pipelines/utils/decorators.py,sha256=k2v0Kk-fnTyVKXBBCkWI6f5cbv_vE9QXpCNL8Us11IY,775
+gooddata_pipelines/utils/file_utils.py,sha256=mr2rVicXjEaEfq8cV7oeIkXaYY1PdtuL7QBvhV4C-l4,1619
 gooddata_pipelines/utils/rate_limiter.py,sha256=owbcEZhUxlTnE7rRHiWQ8XBC-vML2fVPbt41EeGEM7o,2002
-gooddata_pipelines-1.52.1.dev1.dist-info/METADATA,sha256=Bq0BXaTIq36gNxCeBc63ijIms2O87QuZsDGDKg8T82o,3658
-gooddata_pipelines-1.52.1.dev1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-gooddata_pipelines-1.52.1.dev1.dist-info/licenses/LICENSE.txt,sha256=PNC7WXGIo6OKkNoPLRxlVrw6jaLcjSTUsSxy9Xcu9Jo,560365
-gooddata_pipelines-1.52.1.dev1.dist-info/RECORD,,
+gooddata_pipelines-1.52.1.dev3.dist-info/METADATA,sha256=kdehzVDqJEinfjJR4320aRSm5Ya5Qwhr9EZWU2aaVW4,3709
+gooddata_pipelines-1.52.1.dev3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gooddata_pipelines-1.52.1.dev3.dist-info/licenses/LICENSE.txt,sha256=PNC7WXGIo6OKkNoPLRxlVrw6jaLcjSTUsSxy9Xcu9Jo,560365
+gooddata_pipelines-1.52.1.dev3.dist-info/RECORD,,