gooddata-pipelines 1.50.1.dev1__tar.gz → 1.50.1.dev2__tar.gz

{gooddata_pipelines-1.50.1.dev1 → gooddata_pipelines-1.50.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.50.1.dev1
+Version: 1.50.1.dev2
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License: MIT
@@ -8,7 +8,7 @@ License-File: LICENSE.txt
 Requires-Python: >=3.10
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.50.1.dev1
+Requires-Dist: gooddata-sdk~=1.50.1.dev2
 Requires-Dist: pydantic<3.0.0,>=2.11.3
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326
@@ -74,4 +74,12 @@ full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts(
 provisioner.full_load(full_load_data)
 ```
 
-Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository
+## Bugs & Requests
+
+Please use the [GitHub issue tracker](https://github.com/gooddata/gooddata-python-sdk/issues) to submit bugs
+or request features.
+
+## Changelog
+
+See  [Github releases](https://github.com/gooddata/gooddata-python-sdk/releases) for released versions
+and a list of changes.

{gooddata_pipelines-1.50.1.dev1 → gooddata_pipelines-1.50.1.dev2}/README.md

@@ -57,4 +57,12 @@ full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts(
 provisioner.full_load(full_load_data)
 ```
 
-Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository
+## Bugs & Requests
+
+Please use the [GitHub issue tracker](https://github.com/gooddata/gooddata-python-sdk/issues) to submit bugs
+or request features.
+
+## Changelog
+
+See  [Github releases](https://github.com/gooddata/gooddata-python-sdk/releases) for released versions
+and a list of changes.

{gooddata_pipelines-1.50.1.dev1 → gooddata_pipelines-1.50.1.dev2}/gooddata_pipelines/__init__.py

@@ -6,11 +6,22 @@ from ._version import __version__
 from .backup_and_restore.backup_manager import BackupManager
 from .backup_and_restore.models.storage import (
     BackupRestoreConfig,
+    LocalStorageConfig,
+    S3StorageConfig,
     StorageType,
 )
 from .backup_and_restore.storage.local_storage import LocalStorage
 from .backup_and_restore.storage.s3_storage import S3Storage
 
+# -------- LDM Extension --------
+from .ldm_extension.ldm_extension_manager import LdmExtensionManager
+from .ldm_extension.models.custom_data_object import (
+    ColumnDataType,
+    CustomDatasetDefinition,
+    CustomFieldDefinition,
+    CustomFieldType,
+)
+
 # -------- Provisioning --------
 from .provisioning.entities.user_data_filters.models.udf_models import (
     UserDataFilterFullLoad,
@@ -51,6 +62,8 @@ __all__ = [
     "UserIncrementalLoad",
     "UserGroupIncrementalLoad",
     "PermissionFullLoad",
+    "LocalStorageConfig",
+    "S3StorageConfig",
     "PermissionIncrementalLoad",
     "UserFullLoad",
     "UserGroupFullLoad",
@@ -61,5 +74,10 @@ __all__ = [
     "UserDataFilterProvisioner",
     "UserDataFilterFullLoad",
     "EntityType",
+    "LdmExtensionManager",
+    "CustomDatasetDefinition",
+    "CustomFieldDefinition",
+    "ColumnDataType",
+    "CustomFieldType",
     "__version__",
 ]

{gooddata_pipelines-1.50.1.dev1 → gooddata_pipelines-1.50.1.dev2}/gooddata_pipelines/api/gooddata_api.py

@@ -174,6 +174,49 @@ class ApiMethods:
         )
         return self._get(endpoint)
 
+    def get_all_metrics(self, workspace_id: str) -> requests.Response:
+        """Get all metrics from the specified workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace to retrieve metrics from.
+        Returns:
+            requests.Response: The response containing the metrics.
+        """
+        endpoint = f"/entities/workspaces/{workspace_id}/metrics"
+        headers = {**self.headers, "X-GDC-VALIDATE-RELATIONS": "true"}
+        return self._get(endpoint, headers=headers)
+
+    def get_all_visualization_objects(
+        self, workspace_id: str
+    ) -> requests.Response:
+        """Get all visualizations from the specified workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace to retrieve visualizations from.
+        Returns:
+            requests.Response: The response containing the visualizations.
+        """
+        endpoint = f"/entities/workspaces/{workspace_id}/visualizationObjects"
+        headers = {**self.headers, "X-GDC-VALIDATE-RELATIONS": "true"}
+        return self._get(endpoint, headers=headers)
+
+    def get_all_dashboards(self, workspace_id: str) -> requests.Response:
+        """Get all dashboards from the specified workspace.
+
+        Args:
+            workspace_id (str): The ID of the workspace to retrieve dashboards from.
+        Returns:
+            requests.Response: The response containing the dashboards.
+        """
+        endpoint = f"/entities/workspaces/{workspace_id}/analyticalDashboards"
+        headers = {**self.headers, "X-GDC-VALIDATE-RELATIONS": "true"}
+        return self._get(endpoint, headers=headers)
+
+    def get_profile(self) -> requests.Response:
+        """Returns organization and current user information."""
+        endpoint = "/profile"
+        return self._get(endpoint)
+
     def _get(
         self, endpoint: str, headers: dict[str, str] | None = None
     ) -> requests.Response:
@@ -253,3 +296,15 @@ class ApiMethods:
         url = self._get_url(endpoint)
 
         return requests.delete(url, headers=self.headers, timeout=TIMEOUT)
+
+    @staticmethod
+    def raise_if_response_not_ok(*responses: requests.Response) -> None:
+        """Check if responses from API calls are OK.
+
+        Raises ValueError if any response is not OK (status code not 2xx).
+        """
+        for response in responses:
+            if not response.ok:
+                raise ValueError(
+                    f"Request to {response.url} failed with status code {response.status_code}: {response.text}"
+                )

gooddata_pipelines-1.50.1.dev2/gooddata_pipelines/ldm_extension/input_processor.py

@@ -0,0 +1,286 @@
+# (C) 2025 GoodData Corporation
+"""Module for processing validated custom datasets and fields data.
+
+This module is responsible for converting validated custom datasets and fields
+into objects defined in the GoodData Python SDK.
+"""
+
+from gooddata_sdk.catalog.identifier import (
+    CatalogDatasetWorkspaceDataFilterIdentifier,
+    CatalogGrainIdentifier,
+    CatalogReferenceIdentifier,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.data_filter_references import (
+    CatalogDeclarativeWorkspaceDataFilterReferences,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.dataset.dataset import (
+    CatalogDataSourceTableIdentifier,
+    CatalogDeclarativeAttribute,
+    CatalogDeclarativeDataset,
+    CatalogDeclarativeDatasetSql,
+    CatalogDeclarativeFact,
+    CatalogDeclarativeReference,
+    CatalogDeclarativeReferenceSource,
+    CatalogDeclarativeWorkspaceDataFilterColumn,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.date_dataset.date_dataset import (
+    CatalogDeclarativeDateDataset,
+    CatalogGranularitiesFormatting,
+)
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm import (
+    CatalogDeclarativeLdm,
+    CatalogDeclarativeModel,
+)
+
+from gooddata_pipelines.ldm_extension.models.aliases import DatasetId
+from gooddata_pipelines.ldm_extension.models.custom_data_object import (
+    ColumnDataType,
+    CustomDataset,
+    CustomFieldDefinition,
+    CustomFieldType,
+)
+
+
+class LdmExtensionDataProcessor:
+    """Create GoodData LDM from validated custom datasets and fields."""
+
+    DATE_GRANULARITIES: list[str] = [
+        "MINUTE",
+        "HOUR",
+        "DAY",
+        "WEEK",
+        "MONTH",
+        "QUARTER",
+        "YEAR",
+        "MINUTE_OF_HOUR",
+        "HOUR_OF_DAY",
+        "DAY_OF_WEEK",
+        "DAY_OF_MONTH",
+        "DAY_OF_YEAR",
+        "WEEK_OF_YEAR",
+        "MONTH_OF_YEAR",
+        "QUARTER_OF_YEAR",
+    ]
+
+    @staticmethod
+    def _attribute_from_field(
+        dataset_name: str,
+        custom_field: CustomFieldDefinition,
+    ) -> CatalogDeclarativeAttribute:
+        """Assign a declarative attribute from a custom field definition."""
+        return CatalogDeclarativeAttribute(
+            id=custom_field.custom_field_id,
+            title=custom_field.custom_field_name,
+            source_column=custom_field.custom_field_source_column,
+            labels=[],
+            source_column_data_type=custom_field.custom_field_source_column_data_type.value,
+            tags=[dataset_name],
+        )
+
+    @staticmethod
+    def _fact_from_field(
+        dataset_name: str,
+        custom_field: CustomFieldDefinition,
+    ) -> CatalogDeclarativeFact:
+        """Assign a declarative fact from a custom field definition."""
+        return CatalogDeclarativeFact(
+            id=custom_field.custom_field_id,
+            title=custom_field.custom_field_name,
+            source_column=custom_field.custom_field_source_column,
+            source_column_data_type=custom_field.custom_field_source_column_data_type.value,
+            tags=[dataset_name],
+        )
+
+    def _date_from_field(
+        self,
+        dataset_name: str,
+        custom_field: CustomFieldDefinition,
+    ) -> CatalogDeclarativeDateDataset:
+        """Assign a declarative date dataset from a custom field definition."""
+
+        return CatalogDeclarativeDateDataset(
+            id=custom_field.custom_field_id,
+            title=custom_field.custom_field_name,
+            granularities_formatting=CatalogGranularitiesFormatting(
+                title_base="",
+                title_pattern="%titleBase - %granularityTitle",
+            ),
+            granularities=self.DATE_GRANULARITIES,
+            tags=[dataset_name],
+        )
+
+    @staticmethod
+    def _date_ref_from_field(
+        custom_field: CustomFieldDefinition,
+    ) -> CatalogDeclarativeReference:
+        """Create a date reference from a custom field definition."""
+        return CatalogDeclarativeReference(
+            identifier=CatalogReferenceIdentifier(
+                id=custom_field.custom_field_id
+            ),
+            multivalue=False,
+            sources=[
+                CatalogDeclarativeReferenceSource(
+                    column=custom_field.custom_field_source_column,
+                    target=CatalogGrainIdentifier(
+                        id=custom_field.custom_field_id,
+                        type=CustomFieldType.DATE.value,
+                    ),
+                    data_type=custom_field.custom_field_source_column_data_type.value,
+                )
+            ],
+        )
+
+    @staticmethod
+    def _get_sources(
+        dataset: CustomDataset,
+    ) -> tuple[
+        CatalogDataSourceTableIdentifier | None,
+        CatalogDeclarativeDatasetSql | None,
+    ]:
+        """Get the data source table and SQL from the dataset definition."""
+        # We will have either a table id or a sql statement. Let's store
+        # whatever data is available to variables and pass it to the
+        # dataset. Both can be object instances or None, but at least one
+        # should be valid as per prior validation.
+        dataset_source_table_id = (
+            CatalogDataSourceTableIdentifier(
+                id=dataset.definition.dataset_source_table,
+                data_source_id=dataset.definition.dataset_datasource_id,
+                path=[dataset.definition.dataset_source_table],
+            )
+            if dataset.definition.dataset_source_table
+            else None
+        )
+
+        dataset_sql = (
+            CatalogDeclarativeDatasetSql(
+                statement=dataset.definition.dataset_source_sql,
+                data_source_id=dataset.definition.dataset_datasource_id,
+            )
+            if dataset.definition.dataset_source_sql
+            else None
+        )
+        return dataset_source_table_id, dataset_sql
+
+    def datasets_to_ldm(
+        self, datasets: dict[DatasetId, CustomDataset]
+    ) -> CatalogDeclarativeModel:
+        """Convert validated datasets to GoodData declarative model.
+
+        Args:
+            datasets (dict[DatasetId, CustomDataset]): Dictionary of validated
+                datasets.
+        Returns:
+            CatalogDeclarativeModel: GoodData declarative model representation
+                of the datasets.
+        """
+
+        declarative_datasets: list[CatalogDeclarativeDataset] = []
+
+        # Date dimensions are not stored in a dataset, but as a separate datasets
+        # in `date_instances` object on the LDM
+        date_instances: list[CatalogDeclarativeDateDataset] = []
+
+        for dataset in datasets.values():
+            date_references: list[CatalogDeclarativeReference] = []
+            attributes: list[CatalogDeclarativeAttribute] = []
+            facts: list[CatalogDeclarativeFact] = []
+
+            # Iterate through the custom fields and create the appropriate objects
+            for custom_field in dataset.custom_fields:
+                if custom_field.custom_field_type == CustomFieldType.ATTRIBUTE:
+                    attributes.append(
+                        self._attribute_from_field(
+                            dataset.definition.dataset_name, custom_field
+                        )
+                    )
+
+                elif custom_field.custom_field_type == CustomFieldType.FACT:
+                    facts.append(
+                        self._fact_from_field(
+                            dataset.definition.dataset_name, custom_field
+                        )
+                    )
+
+                # Process date dimensions and store them to date_instances. Date
+                # dimensions are not stored in a dataset, but as a separate dataset.
+                # However, they need to be referenced in the dataset references to
+                # create the connection between the dataset and the date dimension
+                # in the GoodData Logical Data Model.
+                elif custom_field.custom_field_type == CustomFieldType.DATE:
+                    # Add the date dimension to the date_instances
+                    date_instances.append(
+                        self._date_from_field(
+                            dataset.definition.dataset_name, custom_field
+                        )
+                    )
+
+                    # Create a reference so that the date dimension is connected
+                    # to the dataset in the GoodData Logical Data Model.
+                    date_references.append(
+                        self._date_ref_from_field(custom_field)
+                    )
+
+                else:
+                    raise ValueError(
+                        f"Unsupported custom field type: {custom_field.custom_field_type}"
+                    )
+
+            # Get the data source info
+            dataset_source_table_id, dataset_sql = self._get_sources(dataset)
+
+            # Construct the declarative dataset object and append it to the list.
+            declarative_datasets.append(
+                CatalogDeclarativeDataset(
+                    id=dataset.definition.dataset_id,
+                    title=dataset.definition.dataset_name,
+                    grain=[],
+                    references=[
+                        CatalogDeclarativeReference(
+                            identifier=CatalogReferenceIdentifier(
+                                id=dataset.definition.parent_dataset_reference,
+                            ),
+                            multivalue=True,
+                            sources=[
+                                CatalogDeclarativeReferenceSource(
+                                    column=dataset.definition.dataset_reference_source_column,
+                                    data_type=dataset.definition.dataset_reference_source_column_data_type.value,
+                                    target=CatalogGrainIdentifier(
+                                        id=dataset.definition.parent_dataset_reference_attribute_id,
+                                        type=CustomFieldType.ATTRIBUTE.value,
+                                    ),
+                                )
+                            ],
+                        ),
+                    ]
+                    + date_references,
+                    description=None,
+                    attributes=attributes,
+                    facts=facts,
+                    data_source_table_id=dataset_source_table_id,
+                    sql=dataset_sql,
+                    workspace_data_filter_columns=[
+                        CatalogDeclarativeWorkspaceDataFilterColumn(
+                            name=dataset.definition.workspace_data_filter_column_name,
+                            data_type=ColumnDataType.STRING.value,
+                        )
+                    ],
+                    workspace_data_filter_references=[
+                        CatalogDeclarativeWorkspaceDataFilterReferences(
+                            filter_id=CatalogDatasetWorkspaceDataFilterIdentifier(
+                                id=dataset.definition.workspace_data_filter_id
+                            ),
+                            filter_column=dataset.definition.workspace_data_filter_column_name,
+                            filter_column_data_type=ColumnDataType.STRING.value,
+                        )
+                    ],
+                    tags=[dataset.definition.dataset_name],
+                )
+            )
+
+        # Create the Logical Data Model from the datasets and the date instances.
+        ldm = CatalogDeclarativeLdm(
+            datasets=declarative_datasets, date_instances=date_instances
+        )
+        return CatalogDeclarativeModel(ldm=ldm)

gooddata_pipelines-1.50.1.dev2/gooddata_pipelines/ldm_extension/input_validator.py

@@ -0,0 +1,185 @@
+# (C) 2025 GoodData Corporation
+"""Module for validating custom fields input data.
+
+This module is responsible for validating custom fields input data checking for
+row level and aggregated constraints.
+"""
+
+from collections import Counter
+from typing import Any, TypeVar
+
+from pydantic import BaseModel
+
+from gooddata_pipelines.ldm_extension.models.aliases import (
+    DatasetId,
+    WorkspaceId,
+)
+from gooddata_pipelines.ldm_extension.models.custom_data_object import (
+    CustomDataset,
+    CustomDatasetDefinition,
+    CustomFieldDefinition,
+    CustomFieldType,
+)
+
+
+class LdmExtensionDataValidator:
+    ModelT = TypeVar("ModelT", bound=BaseModel)
+
+    def validate(
+        self,
+        dataset_definitions: list[CustomDatasetDefinition],
+        field_definitions: list[CustomFieldDefinition],
+    ) -> dict[WorkspaceId, dict[DatasetId, CustomDataset]]:
+        """Validate dataset and field definitions.
+
+        Validates the dataset definitions and field definitions by using Pydantic
+        models to check row level constraints, then aggregates the definitions
+        per workspace, while checking for integrity on aggregated level, i.e.,
+        uniqueness of combinations of identifieres on workspace level.
+
+        Args:
+            raw_dataset_definitions (list[dict[str, str]]): List of raw dataset definitions to validate.
+            raw_field_definitions (list[dict[str, str]]): List of raw field definitions to validate.
+        Returns:
+            dict[WorkspaceId, dict[DatasetId, CustomDataset]]:
+                Dictionary of validated dataset definitions per workspace,
+                where each dataset contains its custom fields:
+                ```python
+                {
+                    "workspace_id_1": {
+                        "dataset_id_1": CustomDataset(...),
+                        "dataset_id_2": CustomDataset(...),
+                    },
+                    ...
+                }
+                ```
+        """
+
+        # First, validate the dataset definitions and aggregate them per workspace.
+        validated_data = self._validate_dataset_definitions(dataset_definitions)
+
+        # Then validate the field definitions and connect them to the datasets
+        validated_data = self._validate_field_definitions(
+            validated_data, field_definitions
+        )
+
+        return validated_data
+
+    def _validate_dataset_definitions(
+        self,
+        dataset_definitions: list[CustomDatasetDefinition],
+    ) -> dict[WorkspaceId, dict[DatasetId, CustomDataset]]:
+        self._check_dataset_combinations(dataset_definitions)
+
+        validated_definitions: dict[
+            WorkspaceId, dict[DatasetId, CustomDataset]
+        ] = {}
+        for definition in dataset_definitions:
+            validated_definitions.setdefault(definition.workspace_id, {})[
+                definition.dataset_id
+            ] = CustomDataset(definition=definition, custom_fields=[])
+
+        return validated_definitions
+
+    def _check_dataset_combinations(
+        self, dataset_definitions: list[CustomDatasetDefinition]
+    ) -> None:
+        """Check integrity of provided dataset definitions.
+
+        Validation criteria:
+            - workspace_id + dataset_id must be unique across all dataset definitions.
+
+        Args:
+            dataset_definitions (list[CustomDatasetDefinition]): List of dataset definitions to check.
+        Raises:
+            ValueError: If there are duplicate dataset definitions based on workspace_id and dataset_id.
+        """
+        workspace_dataset_combinations = [
+            (definition.workspace_id, definition.dataset_id)
+            for definition in dataset_definitions
+        ]
+        if len(workspace_dataset_combinations) != len(
+            set(workspace_dataset_combinations)
+        ):
+            duplicates = self._get_duplicates(workspace_dataset_combinations)
+            raise ValueError(
+                "Duplicate dataset definitions found in the raw dataset "
+                + f"definitions (workspace_id, dataset_id): {duplicates}"
+            )
+
+    @staticmethod
+    def _get_duplicates(list_to_check: list[Any]) -> list[Any]:
+        """Get duplicates from a list.
+
+        Args:
+            list_to_check (list[Any]): List of items to check for duplicates.
+        Returns:
+            list[Any]: List of duplicate items.
+        """
+        counts = Counter(list_to_check)
+        return [item for item, count in counts.items() if count > 1]
+
+    def _check_field_combinations(
+        self, field_definitions: list[CustomFieldDefinition]
+    ) -> None:
+        """Check integrity of provided field definitions.
+
+        Validation criteria (per workspace):
+            - unique workspace_id + cf_id combinations (only for attribute and fact custom_field_type)
+            - there is no row with the same dataset_id and cf_id (only for date custom_field_type)
+
+        Args:
+            field_definitions (list[CustomFieldDefinition]): List of field definitions to check.
+        Raises:
+            ValueError: If there are duplicate field definitions based on workspace_id and cf_id.
+        """
+        workspace_field_combinations: set[tuple[str, str]] = set()
+        dataset_field_combinations: set[tuple[str, str]] = set()
+
+        for field in field_definitions:
+            if field.custom_field_type in [
+                CustomFieldType.ATTRIBUTE,
+                CustomFieldType.FACT,
+            ]:
+                combination = (field.workspace_id, field.custom_field_id)
+                if combination in workspace_field_combinations:
+                    raise ValueError(
+                        f"Duplicate custom field found for workspace {field.workspace_id} "
+                        + f"with field ID {field.custom_field_id}"
+                    )
+                workspace_field_combinations.add(combination)
+
+            elif field.custom_field_type == CustomFieldType.DATE:
+                combination = (field.dataset_id, field.custom_field_id)
+                if combination in dataset_field_combinations:
+                    raise ValueError(
+                        f"Duplicate custom field found for dataset {field.dataset_id} "
+                        + f"with field ID {field.custom_field_id}"
+                    )
+                dataset_field_combinations.add(combination)
+
+    def _validate_field_definitions(
+        self,
+        validated_definitions: dict[
+            WorkspaceId, dict[DatasetId, CustomDataset]
+        ],
+        field_definitions: list[CustomFieldDefinition],
+    ) -> dict[WorkspaceId, dict[DatasetId, CustomDataset]]:
+        """Validates custom field definitions amd connects them to the datasets.
+
+        Args:
+            validated_definitions (dict[WorkspaceId, dict[DatasetId, CustomDataset]]):
+                Dictionary of validated dataset definitions per workspace.
+            raw_field_definitions (list[dict[str, str]]): List of raw field definitions to validate.
+        Returns:
+            dict[WorkspaceId, dict[DatasetId, CustomDataset]]:
+                Updated dictionary of validated dataset definitions with custom fields added.
+        """
+        self._check_field_combinations(field_definitions)
+
+        for field_definition in field_definitions:
+            validated_definitions[field_definition.workspace_id][
+                field_definition.dataset_id
+            ].custom_fields.append(field_definition)
+
+        return validated_definitions