gooddata-pipelines 1.48.1.dev1__py3-none-any.whl → 1.48.1.dev3__py3-none-any.whl

gooddata_pipelines/backup_and_restore/backup_input_processor.py

@@ -158,25 +158,33 @@ class BackupInputProcessor:
         return all_workspaces
 
     def get_ids_to_backup(
-        self, input_type: InputType, path_to_csv: str | None = None
+        self,
+        input_type: InputType,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> list[str]:
         """Returns the list of workspace IDs to back up based on the input type."""
 
         if input_type in (InputType.LIST_OF_WORKSPACES, InputType.HIERARCHY):
-            if path_to_csv is None:
+            if (path_to_csv is None) == (workspace_ids is None):
                 raise ValueError(
-                    f"Path to CSV is required for this input type: {input_type.value}"
+                    f"Path to CSV and list of workspace IDs must be specified exclusively for this input type: {input_type.value}"
                 )
 
             # If we're backing up based on the list, simply read it from the CSV
+            list_of_parents = []
+            if path_to_csv is not None:
+                list_of_parents = self.csv_reader.read_backup_csv(path_to_csv)
+            if workspace_ids is not None:
+                list_of_parents = workspace_ids
+
             if input_type == InputType.LIST_OF_WORKSPACES:
-                return self.csv_reader.read_backup_csv(path_to_csv)
+                return list_of_parents
             else:
                 # For hierarchy backup, we read the CSV and treat it as a list of
                 # parent workspace IDs. Then we retrieve the children of each parent,
                 # including their children, and so on. The parent workspaces are
                 # also included in the backup.
-                list_of_parents = self.csv_reader.read_backup_csv(path_to_csv)
                 list_of_children: list[str] = []
 
                 for parent in list_of_parents:

gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -373,29 +373,37 @@ class BackupManager:
 
                     raise
 
-    def backup_workspaces(self, path_to_csv: str) -> None:
+    def backup_workspaces(
+        self, path_to_csv: str | None, workspace_ids: list[str] | None
+    ) -> None:
         """Runs the backup process for a list of workspace IDs.
 
-        Will read the list of workspace IDs from a CSV file and create backup for
-        each workspace in storage specified in the configuration.
+        Will take the list of workspace IDs or read the list of
+        workspace IDs from a CSV file and create backup for each
+        workspace in storage specified in the configuration.
 
         Args:
             path_to_csv (str): Path to a CSV file containing a list of workspace IDs.
+            workspace_ids (list[str]): List of workspace IDs
         """
-        self.backup(InputType.LIST_OF_WORKSPACES, path_to_csv)
+        self.backup(InputType.LIST_OF_WORKSPACES, path_to_csv, workspace_ids)
 
-    def backup_hierarchies(self, path_to_csv: str) -> None:
+    def backup_hierarchies(
+        self, path_to_csv: str | None, workspace_ids: list[str] | None
+    ) -> None:
         """Runs the backup process for a list of hierarchies.
 
-        Will read the list of workspace IDs from a CSV file and create backup for
-        each those workspaces' hierarchies in storage specified in the configuration.
+        Will take the list of workspace IDs or read the list of workspace IDs
+        from a CSV file and create backup for each those workspaces' hierarchies
+        in storage specified in the configuration.
         Workspace hierarchy means the workspace itself and all its direct and
         indirect children.
 
         Args:
             path_to_csv (str): Path to a CSV file containing a list of workspace IDs.
+            workspace_ids (list[str]): List of workspace IDs
         """
-        self.backup(InputType.HIERARCHY, path_to_csv)
+        self.backup(InputType.HIERARCHY, path_to_csv, workspace_ids)
 
     def backup_entire_organization(self) -> None:
         """Runs the backup process for the entire organization.
@@ -406,12 +414,17 @@ class BackupManager:
         self.backup(InputType.ORGANIZATION)
 
     def backup(
-        self, input_type: InputType, path_to_csv: str | None = None
+        self,
+        input_type: InputType,
+        path_to_csv: str | None = None,
+        workspace_ids: list[str] | None = None,
     ) -> None:
         """Runs the backup process with selected input type."""
         try:
             workspaces_to_export: list[str] = self.loader.get_ids_to_backup(
-                input_type, path_to_csv
+                input_type,
+                path_to_csv,
+                workspace_ids,
             )
             batches = self.split_to_batches(
                 workspaces_to_export, self.config.batch_size

gooddata_pipelines/backup_and_restore/models/storage.py

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
 
 from enum import Enum
-from typing import Annotated, TypeAlias
+from typing import Annotated, TypeAlias, Optional
 
 import yaml
 from pydantic import BaseModel, Field
@@ -22,6 +22,9 @@ class S3StorageConfig(BaseModel):
     backup_path: str
     bucket: str
     profile: str = "default"
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    aws_default_region: Optional[str] = None
 
 
 class LocalStorageConfig(BaseModel):

gooddata_pipelines/backup_and_restore/storage/s3_storage.py

@@ -21,8 +21,7 @@ class S3Storage(BackupStorage):
             raise ValueError("S3 storage config is required")
 
         self._config = conf.storage
-        self._profile = self._config.profile
-        self._session = self._create_boto_session(self._profile)
+        self._session = self._create_boto_session(self._config)
         self._resource = self._session.resource("s3")
         self._bucket = self._resource.Bucket(self._config.bucket)  # type: ignore [missing library stubs]
         suffix = "/" if not self._config.backup_path.endswith("/") else ""
@@ -30,9 +29,25 @@ class S3Storage(BackupStorage):
 
         self._verify_connection()
 
-    def _create_boto_session(self, profile: str) -> boto3.Session:
+    def _create_boto_session(self, config: S3StorageConfig) -> boto3.Session:
+        if config.aws_access_key_id and config.aws_secret_access_key:
+            if not config.aws_default_region:
+                self.logger.warning(
+                    "No AWS region specified. Defaulting to us-east-1."
+                )
+            try:
+                return boto3.Session(
+                    aws_access_key_id=config.aws_access_key_id,
+                    aws_secret_access_key=config.aws_secret_access_key,
+                    region_name=config.aws_default_region,
+                )
+            except Exception:
+                self.logger.warning(
+                    "Failed to create boto3 session with supplied credentials. Falling back to profile..."
+                )
+
         try:
-            return boto3.Session(profile_name=profile)
+            return boto3.Session(profile_name=config.profile)
         except Exception:
             self.logger.warning(
                 'AWS profile "[default]" not found. Trying other fallback methods...'

{gooddata_pipelines-1.48.1.dev1.dist-info → gooddata_pipelines-1.48.1.dev3.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.48.1.dev1
+Version: 1.48.1.dev3
 Author-email: GoodData <support@gooddata.com>
 License: MIT
 License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.48.1.dev1
+Requires-Dist: gooddata-sdk~=1.48.1.dev3
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326
@@ -22,9 +22,9 @@ Description-Content-Type: text/markdown
 
 # GoodData Pipelines
 
-A high level library for automating the lifecycle of GoodData Cloud (GDC).
+A high-level library for automating the lifecycle of GoodData Cloud (GDC).
 
-You can use the package to manage following resoursec in GDC:
+You can use the package to manage following resources in GDC:
 
 1. Provisioning (create, update, delete)
    - User profiles
@@ -36,7 +36,7 @@ You can use the package to manage following resoursec in GDC:
 1. _[PLANNED]:_ Custom fields management
    - extend the Logical Data Model of a child workspace
 
-In case you are not interested in incorporating a library in your own program, but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
+In case you are not interested in incorporating a library in your own program but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools).
 
 ## Provisioning
 
@@ -53,23 +53,20 @@ import os
 from csv import DictReader
 from pathlib import Path
 
-# Import the Entity Provisioner class and corresponing model from gooddata_pipelines library
+# Import the Entity Provisioner class and corresponding model from gooddata_pipelines library
 from gooddata_pipelines import UserFullLoad, UserProvisioner
+from gooddata_pipelines.logger.logger import LogObserver
 
-# Optional: you can set up logging and subscribe it to the Provisioner
-from utils.logger import setup_logging
-
-setup_logging()
+# Optionally, subscribe a standard Python logger to the LogObserver
+import logging
 logger = logging.getLogger(__name__)
+LogObserver().subscribe(logger)
 
 # Create the Provisioner instance - you can also create the instance from a GDC yaml profile
 provisioner = UserProvisioner(
     host=os.environ["GDC_HOSTNAME"], token=os.environ["GDC_AUTH_TOKEN"]
 )
 
-# Optional: subscribe to logs
-provisioner.logger.subscribe(logger)
-
 # Load your data from your data source
 source_data_path: Path = Path("path/to/some.csv")
 source_data_reader = DictReader(source_data_path.read_text().splitlines())
@@ -82,4 +79,4 @@ full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts(
 provisioner.full_load(full_load_data)
 ```
 
-Ready made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository
+Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository

{gooddata_pipelines-1.48.1.dev1.dist-info → gooddata_pipelines-1.48.1.dev3.dist-info}/RECORD

@@ -8,18 +8,18 @@ gooddata_pipelines/api/gooddata_api_wrapper.py,sha256=t7dFrXJ6X4yXS9XDthOmvd2Cyz
 gooddata_pipelines/api/gooddata_sdk.py,sha256=wd5O4e9BQLWUawt6odrs5a51nqFGthBkvqh9WOiW36Q,13734
 gooddata_pipelines/api/utils.py,sha256=3QY_aYH17I9THoCINE3l-n5oj52k-gNeT1wv6Z_VxN8,1433
 gooddata_pipelines/backup_and_restore/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
-gooddata_pipelines/backup_and_restore/backup_input_processor.py,sha256=1-Sh0n4DrBrQ7M3Hzq_iiNe-2cxQx0KDu4QdDok618I,7667
-gooddata_pipelines/backup_and_restore/backup_manager.py,sha256=NS3dKc6C7VVfuFQxG0opQitcc23GwCntbLirAhWQei0,15441
+gooddata_pipelines/backup_and_restore/backup_input_processor.py,sha256=ex1tGwETdHDDBRJ_DGKZsZbH6uoRuOrbGbKOC976H5s,7940
+gooddata_pipelines/backup_and_restore/backup_manager.py,sha256=MZJIdaWp3KlhUFv0V0lh495X-LHGUsNSEsrxKfvuHgE,15898
 gooddata_pipelines/backup_and_restore/constants.py,sha256=AO4H6ngsLMs4bCV-RcT7xIi-VZu3smgZWA_3P7lVsQc,907
 gooddata_pipelines/backup_and_restore/csv_reader.py,sha256=0Kw7mJT7REj3Gjqfsc6YT9MbhcqfCGNB_SKBwzTI1rk,1268
 gooddata_pipelines/backup_and_restore/models/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
 gooddata_pipelines/backup_and_restore/models/input_type.py,sha256=CBKJigKdmZ-NJD9MSfNhq89bo86W0AqCMMoyonbd1QA,239
-gooddata_pipelines/backup_and_restore/models/storage.py,sha256=hIIj5CETEroaMTqezzj0ze0eMpPQrzoj5pyp_HBC-xk,1390
+gooddata_pipelines/backup_and_restore/models/storage.py,sha256=DcFH8iWz7LtZZIXoiScZ9ztG6uZHeI9-vLsD07FFnFY,1537
 gooddata_pipelines/backup_and_restore/models/workspace_response.py,sha256=eQbYLgRQc17IRG0yPTAJVrD-Xs05SzuwtzoNrPT2DoY,833
 gooddata_pipelines/backup_and_restore/storage/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pjdIvtf25ut0r8ZwZVbi4s,32
 gooddata_pipelines/backup_and_restore/storage/base_storage.py,sha256=67wdItlG3neExeb_eCUDQhswdUB62X5Nyj9sOImB_Hg,487
 gooddata_pipelines/backup_and_restore/storage/local_storage.py,sha256=NvhPRzRAvuSpc5qCDyPqZaMB0i1jeZOZczaSwjUSGEg,1155
-gooddata_pipelines/backup_and_restore/storage/s3_storage.py,sha256=STA0J-xaP7QQefyYK3OncqwcvWfrtJRRB--5PBfEKDU,2608
+gooddata_pipelines/backup_and_restore/storage/s3_storage.py,sha256=iRtMtDq_C1b_JBL92P9DP4t-BtIdIlmf7hMDE7a02Ps,3284
 gooddata_pipelines/logger/__init__.py,sha256=W-fJvMStnsDUY52AYFhx_LnS2cSCFNf3bB47Iew2j04,129
 gooddata_pipelines/logger/logger.py,sha256=yIMdvqsmOSGQLI4U_tQwxX5E2q_FXUu0Ko7Hv39slFM,3549
 gooddata_pipelines/provisioning/__init__.py,sha256=RZDEiv8nla4Jwa2TZXUdp1NSxg2_-lLqz4h7k2c4v5Y,854
@@ -48,7 +48,7 @@ gooddata_pipelines/provisioning/utils/__init__.py,sha256=-BG28PGDbalLyZGQjpFG0pj
 gooddata_pipelines/provisioning/utils/context_objects.py,sha256=sM22hMsxE0XLI1TU0Vs-2kK0vf4YrB1musoAg__4bjc,936
 gooddata_pipelines/provisioning/utils/exceptions.py,sha256=1WnAOlPhqOf0xRcvn70lxAlLb8Oo6m6WCYS4hj9uzDU,3630
 gooddata_pipelines/provisioning/utils/utils.py,sha256=_Tk-mFgbIGpCixDCF9e-3ZYd-g5Jb3SJiLSH465k4jY,2846
-gooddata_pipelines-1.48.1.dev1.dist-info/METADATA,sha256=jrqTSbfrm5L3YQ7Uqi4_T4llqFJ3QbR5_G_OUSCbU3Q,3773
-gooddata_pipelines-1.48.1.dev1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-gooddata_pipelines-1.48.1.dev1.dist-info/licenses/LICENSE.txt,sha256=PNC7WXGIo6OKkNoPLRxlVrw6jaLcjSTUsSxy9Xcu9Jo,560365
-gooddata_pipelines-1.48.1.dev1.dist-info/RECORD,,
+gooddata_pipelines-1.48.1.dev3.dist-info/METADATA,sha256=OFkLoPEtFhiA8JmlGqSt3PXEwMTCTiknrgm0ym47flA,3750
+gooddata_pipelines-1.48.1.dev3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gooddata_pipelines-1.48.1.dev3.dist-info/licenses/LICENSE.txt,sha256=PNC7WXGIo6OKkNoPLRxlVrw6jaLcjSTUsSxy9Xcu9Jo,560365
+gooddata_pipelines-1.48.1.dev3.dist-info/RECORD,,