gooddata-pipelines 1.47.1.dev1__py3-none-any.whl

gooddata_pipelines/backup_and_restore/storage/s3_storage.py

@@ -0,0 +1,71 @@
+# (C) 2025 GoodData Corporation
+
+import os
+
+import boto3
+
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    S3StorageConfig,
+)
+from gooddata_pipelines.backup_and_restore.storage.base_storage import (
+    BackupStorage,
+)
+
+
+class S3Storage(BackupStorage):
+    def __init__(self, conf: BackupRestoreConfig):
+        super().__init__(conf)
+
+        if not isinstance(conf.storage, S3StorageConfig):
+            raise ValueError("S3 storage config is required")
+
+        self._config = conf.storage
+        self._profile = self._config.profile
+        self._session = self._create_boto_session(self._profile)
+        self._resource = self._session.resource("s3")
+        self._bucket = self._resource.Bucket(self._config.bucket)  # type: ignore [missing library stubs]
+        suffix = "/" if not self._config.backup_path.endswith("/") else ""
+        self._backup_path = self._config.backup_path + suffix
+
+        self._verify_connection()
+
+    def _create_boto_session(self, profile: str) -> boto3.Session:
+        try:
+            return boto3.Session(profile_name=profile)
+        except Exception:
+            self.logger.warning(
+                'AWS profile "[default]" not found. Trying other fallback methods...'
+            )
+
+        return boto3.Session()
+
+    def _verify_connection(self) -> None:
+        """
+        Pings the S3 bucket to verify that the connection is working.
+        """
+        try:
+            # TODO: install boto3 s3 stubs
+            self._resource.meta.client.head_bucket(Bucket=self._config.bucket)
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to connect to S3 bucket {self._config.bucket}: {e}"
+            )
+
+    def export(self, folder: str, org_id: str) -> None:
+        """Uploads the content of the folder to S3 as backup."""
+        storage_path = f"{self._config.bucket}/{self._backup_path}"
+        self.logger.info(f"Uploading {org_id} to {storage_path}")
+        folder = f"{folder}/{org_id}"
+        for subdir, dirs, files in os.walk(folder):
+            full_path = os.path.join(subdir)
+            export_path = (
+                f"{self._backup_path}{org_id}/{full_path[len(folder) + 1 :]}/"
+            )
+            self._bucket.put_object(Key=export_path)
+
+            for file in files:
+                full_path = os.path.join(subdir, file)
+                with open(full_path, "rb") as data:
+                    export_path = f"{self._backup_path}{org_id}/{full_path[len(folder) + 1 :]}"
+                    self._bucket.put_object(Key=export_path, Body=data)

gooddata_pipelines/logger/__init__.py

@@ -0,0 +1,8 @@
+# (C) 2025 GoodData Corporation
+
+from .logger import LoggerLike, LogObserver
+
+__all__ = [
+    "LoggerLike",
+    "LogObserver",
+]

gooddata_pipelines/logger/logger.py

@@ -0,0 +1,115 @@
+# (C) 2025 GoodData Corporation
+
+"""Logging observer for the GoodData Pipelines SDK.
+
+This module provides a singleton observer class `LogObserver` that allows
+subscribing logger-like objects to receive log messages. The observer emits
+unformatted log messages to all subscribed objects, which should implement
+the `LoggerLike` protocol.
+"""
+
+from enum import Enum
+from typing import Any, Protocol
+
+
+class SingletonMeta(type):
+    _instances: dict = {}
+
+    def __call__(cls, *args: Any, **kwargs: Any) -> "SingletonMeta":
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+
+class Severity(Enum):
+    """Severity levels for logging."""
+
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+class LoggerLike(Protocol):
+    """A protocol for a logger-like object.
+
+    This protocol defines the methods that a logger-like object should implement
+    to be compatible with the `LogObserver`. It includes methods for logging
+    messages at different severity levels: info, warning, and error.
+    """
+
+    def info(self, *args: Any, **kwargs: Any) -> None: ...
+
+    def warning(self, *args: Any, **kwargs: Any) -> None: ...
+
+    def error(self, *args: Any, **kwargs: Any) -> None: ...
+
+
+class LogObserver(metaclass=SingletonMeta):
+    """Singleton observer class for logging messages.
+
+    Emits unformatted log messages to all subscribed logger-like objects.
+    """
+
+    # TODO: in future we might want to add a timestamp or other metadata
+    # (severity...)? Currently that is left out to subscribers to handle.
+
+    # TODO: with error we're dumping the context as string to the message
+    # that could be improved (either passing the context as a separate arg
+    # or handling the process here).
+
+    def __init__(self) -> None:
+        self.subscribers: list[LoggerLike] = []
+
+    def subscribe(self, subscriber: LoggerLike) -> None:
+        """Subscribe a logger-like object to receive log messages.
+
+        Args:
+            subscriber (LoggerLike): An object that implements the LoggerLike
+            protocol.
+        Returns:
+            None
+        """
+        self.subscribers.append(subscriber)
+
+    def unsubscribe(self, subscriber: LoggerLike) -> None:
+        """Unsubscribe a logger-like object from receiving log messages.
+
+        Args:
+            subscriber (LoggerLike): An object that implements the LoggerLike
+            protocol.
+
+        Returns:
+            None
+        """
+        self.subscribers.remove(subscriber)
+
+    def _notify(self, severity: Severity, msg: str) -> None:
+        """Notify all subscribers with a log message.
+
+        Args:
+            severity (Severity): The severity level of the log message.
+            msg (str): The log message to be sent to subscribers.
+
+        Returns:
+            None
+        """
+        for subscriber in self.subscribers:
+            if severity == Severity.INFO:
+                subscriber.info(msg)
+            elif severity == Severity.WARNING:
+                subscriber.warning(msg)
+            elif severity == Severity.ERROR:
+                subscriber.error(msg)
+
+    def info(self, msg: str) -> None:
+        """Log an info message."""
+        self._notify(Severity.INFO, msg)
+
+    def warning(self, msg: str) -> None:
+        """Log a warning message."""
+        self._notify(Severity.WARNING, msg)
+
+    def error(self, msg: str) -> None:
+        """Log an error message."""
+        self._notify(Severity.ERROR, msg)

gooddata_pipelines/provisioning/__init__.py

@@ -0,0 +1,31 @@
+# (C) 2025 GoodData Corporation
+
+from .entities.users.models.permissions import (
+    PermissionFullLoad,
+    PermissionIncrementalLoad,
+)
+from .entities.users.models.user_groups import (
+    UserGroupFullLoad,
+    UserGroupIncrementalLoad,
+)
+from .entities.users.models.users import (
+    UserFullLoad,
+    UserIncrementalLoad,
+)
+from .entities.users.permissions import PermissionProvisioner
+from .entities.users.user_groups import UserGroupProvisioner
+from .entities.users.users import UserProvisioner
+from .entities.workspaces.workspace import WorkspaceProvisioner
+
+__all__ = [
+    "PermissionFullLoad",
+    "PermissionIncrementalLoad",
+    "PermissionProvisioner",
+    "UserFullLoad",
+    "UserGroupFullLoad",
+    "UserIncrementalLoad",
+    "UserGroupIncrementalLoad",
+    "UserGroupProvisioner",
+    "UserProvisioner",
+    "WorkspaceProvisioner",
+]

gooddata_pipelines/provisioning/assets/wdf_setting.json

@@ -0,0 +1,14 @@
+{
+  "data": {
+    "attributes": {
+      "filterValues": []
+    },
+    "id": "<wdf_setting_id>",
+    "relationships": {
+      "workspaceDataFilter": {
+        "data": { "id": "<wdf_id>", "type": "workspaceDataFilter" }
+      }
+    },
+    "type": "workspaceDataFilterSetting"
+  }
+}

gooddata_pipelines/provisioning/entities/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation

gooddata_pipelines/provisioning/entities/user_data_filters/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation

gooddata_pipelines/provisioning/entities/user_data_filters/models/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation

gooddata_pipelines/provisioning/entities/user_data_filters/models/udf_models.py

@@ -0,0 +1,32 @@
+# (C) 2025 GoodData Corporation
+
+"""This module defines data models for user data filters in a GoodData workspace."""
+
+# TODO: consider using attrs instead of dataclasses for these models. Dataclasses
+# have different functionality per Python version (not package version).
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class UserDataFilterGroup:
+    udf_id: str
+    udf_values: list[str]
+
+
+@dataclass
+class WorkspaceUserDataFilters:
+    workspace_id: str
+    user_data_filters: list["UserDataFilterGroup"] = field(default_factory=list)
+
+
+@dataclass
+class UserDataFilterFullLoad:
+    workspace_id: str
+    udf_id: str
+    udf_value: str
+
+
+@dataclass
+class UserDataFilterIncrementalLoad(UserDataFilterFullLoad):
+    is_active: bool

gooddata_pipelines/provisioning/entities/user_data_filters/user_data_filters.py

@@ -0,0 +1,221 @@
+# (C) 2025 GoodData Corporation
+
+"""Module for provisioning user data filters in GoodData workspaces.
+
+This module provides the `UserDataFilterProvisioner` class, which is responsible
+for creating, updating, and deleting user data filters in GoodData workspaces.
+"""
+
+import re
+
+from gooddata_sdk.catalog.workspace.entity_model.user_data_filter import (
+    CatalogEntityIdentifier,
+    CatalogUserDataFilter,
+    CatalogUserDataFilterAttributes,
+    CatalogUserDataFilterRelationships,
+)
+
+from gooddata_pipelines.provisioning.entities.user_data_filters.models.udf_models import (
+    UserDataFilterFullLoad,
+    UserDataFilterGroup,
+    UserDataFilterIncrementalLoad,
+    WorkspaceUserDataFilters,
+)
+from gooddata_pipelines.provisioning.provisioning import Provisioning
+from gooddata_pipelines.provisioning.utils.exceptions import ContextException
+
+
+class UserDataFilterProvisioner(
+    Provisioning[UserDataFilterFullLoad, UserDataFilterIncrementalLoad]
+):
+    """Provisioning class for user data filters in GoodData workspaces.
+
+    This class handles the creation, update, and deletion of user data filters
+    based on the provided source data.
+
+    Requires setting the `ldm_column_name` and `maql_column_name`
+    attributes before calling the `provision` method.
+
+    Usage:
+    ```
+    provisioner = UserDataFilterProvisioner(api, source_group)
+    provisioner.set_ldm_column_name("ldm_column")
+    provisioner.set_maql_column_name("maql_column")
+    provisioner.provision()
+    ```
+    """
+
+    source_group_full: list[UserDataFilterFullLoad]
+    source_group_incremental: list[UserDataFilterIncrementalLoad]
+    ldm_column_name: str = ""
+    maql_column_name: str = ""
+
+    def set_ldm_column_name(self, ldm_column_name: str) -> None:
+        """Set the LDM column name for user data filters.
+
+        Args:
+            ldm_column_name (str): The LDM column name to set.
+        """
+        self.ldm_column_name = ldm_column_name
+
+    def set_maql_column_name(self, maql_column_name: str) -> None:
+        """Set the MAQL column name for user data filters.
+
+        Args:
+            maql_column_name (str): The MAQL column name to set.
+        """
+        self.maql_column_name = maql_column_name
+
+    @staticmethod
+    def _group_db_user_data_filters_by_ws_id(
+        user_data_filters: list[UserDataFilterFullLoad],
+    ) -> list[WorkspaceUserDataFilters]:
+        """Group user data filters by workspace ID and user ID."""
+        ws_map: dict[str, dict[str, set[str]]] = {}
+
+        for udf in user_data_filters:
+            ws_map.setdefault(udf.workspace_id, {}).setdefault(
+                udf.udf_id, set()
+            ).add(str(udf.udf_value))
+
+        result: list[WorkspaceUserDataFilters] = []
+
+        for ws_id, udf_dict in ws_map.items():
+            udf_groups = [
+                UserDataFilterGroup(udf_id=udf_id, udf_values=list(values))
+                for udf_id, values in udf_dict.items()
+            ]
+            result.append(
+                WorkspaceUserDataFilters(
+                    workspace_id=ws_id, user_data_filters=udf_groups
+                )
+            )
+        return result
+
+    @staticmethod
+    def _extract_numbers_from_maql(maql: str) -> list[str]:
+        """Extract numbers from a MAQL string."""
+        numbers = re.findall(r'"\d+"', maql)
+        return [number.strip('"') for number in numbers]
+
+    def _skip_user_data_filter_update(
+        self, existing_udf: list[CatalogUserDataFilter], udf_value: list[str]
+    ) -> bool:
+        """Check if the user data filter update can be skipped."""
+        if not existing_udf:
+            return False
+        existing_udfs = self._extract_numbers_from_maql(
+            existing_udf[0].attributes.maql
+        )
+        return set(udf_value) == set(existing_udfs)
+
+    def _create_user_data_filters(
+        self, user_data_filter_ids_to_create: list[WorkspaceUserDataFilters]
+    ) -> None:
+        """Create or update user data filters in GoodData workspaces."""
+        for workspace_user_data_filter in user_data_filter_ids_to_create:
+            workspace_id = workspace_user_data_filter.workspace_id
+            user_data_filters = workspace_user_data_filter.user_data_filters
+
+            gd_user_data_filters: list[CatalogUserDataFilter] = (
+                self._api.list_user_data_filters(workspace_id)
+            )
+
+            gd_udf_ids = {
+                user.relationships.user["data"].id
+                for user in gd_user_data_filters
+                if user.relationships and user.relationships.user
+            }
+
+            db_udf_ids = {udf.udf_id for udf in user_data_filters}
+
+            udf_ids_to_delete: set[str] = gd_udf_ids.difference(db_udf_ids)
+            self._delete_user_data_filters(workspace_id, udf_ids_to_delete)
+
+            udf_group: UserDataFilterGroup
+            for udf_group in user_data_filters:
+                udf_id: str = udf_group.udf_id
+                udf_values: list[str] = udf_group.udf_values
+
+                existing_udf: list[CatalogUserDataFilter] = [
+                    udf for udf in gd_user_data_filters if udf.id == udf_id
+                ]
+                if self._skip_user_data_filter_update(existing_udf, udf_values):
+                    continue
+
+                formatted_udf_values = '", "'.join(
+                    str(value) for value in udf_values
+                )
+                maql = f'{self.maql_column_name} IN ("{formatted_udf_values}")'
+
+                attributes = CatalogUserDataFilterAttributes(maql=maql)
+                relationships = CatalogUserDataFilterRelationships(
+                    labels={
+                        "data": [
+                            CatalogEntityIdentifier(
+                                id=self.ldm_column_name, type="label"
+                            )
+                        ]
+                    },
+                    user={
+                        "data": CatalogEntityIdentifier(id=udf_id, type="user")
+                    },
+                )
+                user_data_filter = CatalogUserDataFilter(
+                    id=udf_id,
+                    attributes=attributes,
+                    relationships=relationships,
+                )
+
+                try:
+                    self._api.create_or_update_user_data_filter(
+                        workspace_id, user_data_filter
+                    )
+                    self.logger.info(
+                        "Created or updated user data filters for user with id "
+                        + f"{udf_id} for client with id {workspace_id}"
+                    )
+                except Exception as e:
+                    raise ContextException(
+                        f"Failed to create user data filters: {e}",
+                        udf_group,
+                        user_data_filter,
+                    ) from e
+
+    def _delete_user_data_filters(
+        self, workspace_id: str, udf_ids_to_delete: set[str]
+    ) -> None:
+        """Delete user data filters in GoodData workspaces."""
+        for udf_id in udf_ids_to_delete:
+            try:
+                self._api.delete_user_data_filter(workspace_id, udf_id)
+                self.logger.info(
+                    f"Deleted user data filters for user with id {udf_id}"
+                )
+            except Exception as e:
+                raise ContextException(
+                    f"Failed to delete user data filters: {e}"
+                ) from e
+
+    def _provision_full_load(self) -> None:
+        """Provision user data filters in GoodData workspaces."""
+
+        if not self.maql_column_name:
+            raise ContextException(
+                "MAQL column name is not set. Please set it before provisioning."
+            )
+        if not self.ldm_column_name:
+            raise ContextException(
+                "LDM column name is not set. Please set it before provisioning."
+            )
+
+        grouped_db_user_data_filters = (
+            self._group_db_user_data_filters_by_ws_id(self.source_group_full)
+        )
+        self._create_user_data_filters(grouped_db_user_data_filters)
+
+        self.logger.info("User data filters provisioning completed")
+
+    def _provision_incremental_load(self) -> None:
+        """Provision user data filters in GoodData workspaces."""
+        raise NotImplementedError("Not implemented yet.")

gooddata_pipelines/provisioning/entities/users/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation

gooddata_pipelines/provisioning/entities/users/models/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation