gooddata-pipelines 1.48.1.dev1__tar.gz → 1.48.1.dev3__tar.gz

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.48.1.dev1
+Version: 1.48.1.dev3
 Author-email: GoodData <support@gooddata.com>
 License: MIT
 License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.48.1.dev1
+Requires-Dist: gooddata-sdk~=1.48.1.dev3
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326
@@ -22,9 +22,9 @@ Description-Content-Type: text/markdown
 
 # GoodData Pipelines
 
-A high level library for automating the lifecycle of GoodData Cloud (GDC).
+A high-level library for automating the lifecycle of GoodData Cloud (GDC).
 
-You can use the package to manage following resoursec in GDC:
+You can use the package to manage following resources in GDC:
 
 1. Provisioning (create, update, delete)
    - User profiles
@@ -36,7 +36,7 @@ You can use the package to manage following resoursec in GDC:
 1. _[PLANNED]:_ Custom fields management
    - extend the Logical Data Model of a child workspace
 
-In case you are not interested in incorporating a library in your own program, but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
+In case you are not interested in incorporating a library in your own program but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
 
 ## Provisioning
 
@@ -53,23 +53,20 @@ import os
 from csv import DictReader
 from pathlib import Path
 
-# Import the Entity Provisioner class and corresponing model from gooddata_pipelines library
+# Import the Entity Provisioner class and corresponding model from gooddata_pipelines library
 from gooddata_pipelines import UserFullLoad, UserProvisioner
+from gooddata_pipelines.logger.logger import LogObserver
 
-# Optional: you can set up logging and subscribe it to the Provisioner
-from utils.logger import setup_logging
-
-setup_logging()
+# Optionally, subscribe a standard Python logger to the LogObserver
+import logging
 logger = logging.getLogger(__name__)
+LogObserver().subscribe(logger)
 
 # Create the Provisioner instance - you can also create the instance from a GDC yaml profile
 provisioner = UserProvisioner(
     host=os.environ["GDC_HOSTNAME"], token=os.environ["GDC_AUTH_TOKEN"]
 )
 
-# Optional: subscribe to logs
-provisioner.logger.subscribe(logger)
-
 # Load your data from your data source
 source_data_path: Path = Path("path/to/some.csv")
 source_data_reader = DictReader(source_data_path.read_text().splitlines())
@@ -82,4 +79,4 @@ full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts(
 provisioner.full_load(full_load_data)
 ```
 
-Ready made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository
+Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/README.md

@@ -1,8 +1,8 @@
 # GoodData Pipelines
 
-A high level library for automating the lifecycle of GoodData Cloud (GDC).
+A high-level library for automating the lifecycle of GoodData Cloud (GDC).
 
-You can use the package to manage following resoursec in GDC:
+You can use the package to manage following resources in GDC:
 
 1. Provisioning (create, update, delete)
    - User profiles
@@ -14,7 +14,7 @@ You can use the package to manage following resoursec in GDC:
 1. _[PLANNED]:_ Custom fields management
    - extend the Logical Data Model of a child workspace
 
-In case you are not interested in incorporating a library in your own program, but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
+In case you are not interested in incorporating a library in your own program but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
 
 ## Provisioning
 
@@ -31,23 +31,20 @@ import os
 from csv import DictReader
 from pathlib import Path
 
-# Import the Entity Provisioner class and corresponing model from gooddata_pipelines library
+# Import the Entity Provisioner class and corresponding model from gooddata_pipelines library
 from gooddata_pipelines import UserFullLoad, UserProvisioner
+from gooddata_pipelines.logger.logger import LogObserver
 
-# Optional: you can set up logging and subscribe it to the Provisioner
-from utils.logger import setup_logging
-
-setup_logging()
+# Optionally, subscribe a standard Python logger to the LogObserver
+import logging
 logger = logging.getLogger(__name__)
+LogObserver().subscribe(logger)
 
 # Create the Provisioner instance - you can also create the instance from a GDC yaml profile
 provisioner = UserProvisioner(
     host=os.environ["GDC_HOSTNAME"], token=os.environ["GDC_AUTH_TOKEN"]
 )
 
-# Optional: subscribe to logs
-provisioner.logger.subscribe(logger)
-
 # Load your data from your data source
 source_data_path: Path = Path("path/to/some.csv")
 source_data_reader = DictReader(source_data_path.read_text().splitlines())
@@ -60,4 +57,4 @@ full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts(
 provisioner.full_load(full_load_data)
 ```
 
-Ready made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository
+Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/gooddata_pipelines/backup_and_restore/backup_input_processor.py

@@ -158,25 +158,33 @@ class BackupInputProcessor:
         return all_workspaces
 
     def get_ids_to_backup(
-        self, input_type: InputType, path_to_csv: str | None = None
+        self,
+        input_type: InputType,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> list[str]:
         """Returns the list of workspace IDs to back up based on the input type."""
 
         if input_type in (InputType.LIST_OF_WORKSPACES, InputType.HIERARCHY):
-            if path_to_csv is None:
+            if (path_to_csv is None) == (workspace_ids is None):
                 raise ValueError(
-                    f"Path to CSV is required for this input type: {input_type.value}"
+                    f"Path to CSV and list of workspace IDs must be specified exclusively for this input type: {input_type.value}"
                 )
 
             # If we're backing up based on the list, simply read it from the CSV
+            list_of_parents = []
+            if path_to_csv is not None:
+                list_of_parents = self.csv_reader.read_backup_csv(path_to_csv)
+            if workspace_ids is not None:
+                list_of_parents = workspace_ids
+
             if input_type == InputType.LIST_OF_WORKSPACES:
-                return self.csv_reader.read_backup_csv(path_to_csv)
+                return list_of_parents
             else:
                 # For hierarchy backup, we read the CSV and treat it as a list of
                 # parent workspace IDs. Then we retrieve the children of each parent,
                 # including their children, and so on. The parent workspaces are
                 # also included in the backup.
-                list_of_parents = self.csv_reader.read_backup_csv(path_to_csv)
                 list_of_children: list[str] = []
 
                 for parent in list_of_parents:

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -373,29 +373,37 @@ class BackupManager:
 
                     raise
 
-    def backup_workspaces(self, path_to_csv: str) -> None:
+    def backup_workspaces(
+        self, path_to_csv: str | None, workspace_ids: list[str] | None
+    ) -> None:
         """Runs the backup process for a list of workspace IDs.
 
-        Will read the list of workspace IDs from a CSV file and create backup for
-        each workspace in storage specified in the configuration.
+        Will take the list of workspace IDs or read the list of
+        workspace IDs from a CSV file and create backup for each
+        workspace in storage specified in the configuration.
 
         Args:
             path_to_csv (str): Path to a CSV file containing a list of workspace IDs.
+            workspace_ids (list[str]): List of workspace IDs
         """
-        self.backup(InputType.LIST_OF_WORKSPACES, path_to_csv)
+        self.backup(InputType.LIST_OF_WORKSPACES, path_to_csv, workspace_ids)
 
-    def backup_hierarchies(self, path_to_csv: str) -> None:
+    def backup_hierarchies(
+        self, path_to_csv: str | None, workspace_ids: list[str] | None
+    ) -> None:
         """Runs the backup process for a list of hierarchies.
 
-        Will read the list of workspace IDs from a CSV file and create backup for
-        each those workspaces' hierarchies in storage specified in the configuration.
+        Will take the list of workspace IDs or read the list of workspace IDs
+        from a CSV file and create backup for each those workspaces' hierarchies
+        in storage specified in the configuration.
         Workspace hierarchy means the workspace itself and all its direct and
         indirect children.
 
         Args:
             path_to_csv (str): Path to a CSV file containing a list of workspace IDs.
+            workspace_ids (list[str]): List of workspace IDs
         """
-        self.backup(InputType.HIERARCHY, path_to_csv)
+        self.backup(InputType.HIERARCHY, path_to_csv, workspace_ids)
 
     def backup_entire_organization(self) -> None:
         """Runs the backup process for the entire organization.
@@ -406,12 +414,17 @@ class BackupManager:
         self.backup(InputType.ORGANIZATION)
 
     def backup(
-        self, input_type: InputType, path_to_csv: str | None = None
+        self,
+        input_type: InputType,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> None:
         """Runs the backup process with selected input type."""
         try:
             workspaces_to_export: list[str] = self.loader.get_ids_to_backup(
-                input_type, path_to_csv
+                input_type,
+                path_to_csv,
+                workspace_ids,
             )
             batches = self.split_to_batches(
                 workspaces_to_export, self.config.batch_size

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/gooddata_pipelines/backup_and_restore/models/storage.py

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
 
 from enum import Enum
-from typing import Annotated, TypeAlias
+from typing import Annotated, TypeAlias, Optional
 
 import yaml
 from pydantic import BaseModel, Field
@@ -22,6 +22,9 @@ class S3StorageConfig(BaseModel):
     backup_path: str
     bucket: str
     profile: str = "default"
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    aws_default_region: Optional[str] = None
 
 
 class LocalStorageConfig(BaseModel):

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/gooddata_pipelines/backup_and_restore/storage/s3_storage.py

@@ -21,8 +21,7 @@ class S3Storage(BackupStorage):
             raise ValueError("S3 storage config is required")
 
         self._config = conf.storage
-        self._profile = self._config.profile
-        self._session = self._create_boto_session(self._profile)
+        self._session = self._create_boto_session(self._config)
         self._resource = self._session.resource("s3")
         self._bucket = self._resource.Bucket(self._config.bucket)  # type: ignore [missing library stubs]
         suffix = "/" if not self._config.backup_path.endswith("/") else ""
@@ -30,9 +29,25 @@ class S3Storage(BackupStorage):
 
         self._verify_connection()
 
-    def _create_boto_session(self, profile: str) -> boto3.Session:
+    def _create_boto_session(self, config: S3StorageConfig) -> boto3.Session:
+        if config.aws_access_key_id and config.aws_secret_access_key:
+            if not config.aws_default_region:
+                self.logger.warning(
+                    "No AWS region specified. Defaulting to us-east-1."
+                )
+            try:
+                return boto3.Session(
+                    aws_access_key_id=config.aws_access_key_id,
+                    aws_secret_access_key=config.aws_secret_access_key,
+                    region_name=config.aws_default_region,
+                )
+            except Exception:
+                self.logger.warning(
+                    "Failed to create boto3 session with supplied credentials. Falling back to profile..."
+                )
+
         try:
-            return boto3.Session(profile_name=profile)
+            return boto3.Session(profile_name=config.profile)
         except Exception:
             self.logger.warning(
                 'AWS profile "[default]" not found. Trying other fallback methods...'

{gooddata_pipelines-1.48.1.dev1 → gooddata_pipelines-1.48.1.dev3}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
 [project]
 name = "gooddata-pipelines"
-version = "1.48.1.dev1"
+version = "1.48.1.dev3"
 description = ""
 authors = [{ name = "GoodData", email = "support@gooddata.com" }]
 license = { text = "MIT" }
@@ -11,7 +11,7 @@ dependencies = [
     "pydantic (>=2.11.3,<3.0.0)",
     "requests (>=2.32.3,<3.0.0)",
     "types-requests (>=2.32.0,<3.0.0)",
-    "gooddata-sdk~=1.48.1.dev1",
+    "gooddata-sdk~=1.48.1.dev3",
     "boto3 (>=1.39.3,<2.0.0)",
     "boto3-stubs (>=1.39.3,<2.0.0)",
     "types-pyyaml (>=6.0.12.20250326,<7.0.0)",