gooddata-pipelines 1.65.1.dev2__tar.gz → 1.65.1.dev3__tar.gz

{gooddata_pipelines-1.65.1.dev2 → gooddata_pipelines-1.65.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.65.1.dev2
+Version: 1.65.1.dev3
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License-Expression: MIT
@@ -10,7 +10,7 @@ Requires-Dist: azure-identity<2.0.0,>=1.15.0
 Requires-Dist: azure-storage-blob<13.0.0,>=12.19.0
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.65.1.dev2
+Requires-Dist: gooddata-sdk~=1.65.1.dev3
 Requires-Dist: pydantic<3.0.0,>=2.9.2
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326

{gooddata_pipelines-1.65.1.dev2 → gooddata_pipelines-1.65.1.dev3}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
 [project]
 name = "gooddata-pipelines"
-version = "1.65.1.dev2"
+version = "1.65.1.dev3"
 description = "GoodData Cloud lifecycle automation pipelines"
 authors = [{ name = "GoodData", email = "support@gooddata.com" }]
 license = "MIT"
@@ -11,7 +11,7 @@ dependencies = [
     "pydantic (>=2.9.2,<3.0.0)",
     "requests (>=2.32.3,<3.0.0)",
     "types-requests (>1.0.0,<3.0.0)",
-    "gooddata-sdk~=1.65.1.dev2",
+    "gooddata-sdk~=1.65.1.dev3",
     "boto3 (>=1.39.3,<2.0.0)",
     "boto3-stubs (>=1.39.3,<2.0.0)",
     "azure-storage-blob (>=12.19.0,<13.0.0)",

{gooddata_pipelines-1.65.1.dev2 → gooddata_pipelines-1.65.1.dev3}/src/gooddata_pipelines/__init__.py

@@ -26,6 +26,7 @@ from .ldm_extension.models.custom_data_object import (
     CustomDatasetDefinition,
     CustomFieldDefinition,
     CustomFieldType,
+    ParentDatasetReference,
 )
 
 # -------- Provisioning --------
@@ -93,6 +94,7 @@ __all__ = [
     "CustomFieldDefinition",
     "ColumnDataType",
     "CustomFieldType",
+    "ParentDatasetReference",
     "provision",
     "WorkflowType",
     "__version__",

{gooddata_pipelines-1.65.1.dev2 → gooddata_pipelines-1.65.1.dev3}/src/gooddata_pipelines/ldm_extension/input_processor.py

@@ -154,6 +154,46 @@ class LdmExtensionDataProcessor:
             ],
         )
 
+    @staticmethod
+    def _build_parent_reference_sources(
+        definition: CustomDatasetDefinition,
+    ) -> list[CatalogDeclarativeReferenceSource]:
+        """Build the reference sources from either the new list or the legacy triple."""
+        if definition.parent_dataset_references:
+            return [
+                CatalogDeclarativeReferenceSource(
+                    column=ref.source_column,
+                    data_type=ref.data_type.value,
+                    target=CatalogGrainIdentifier(
+                        id=ref.attribute_id,
+                        type=CustomFieldType.ATTRIBUTE.value,
+                    ),
+                )
+                for ref in definition.parent_dataset_references
+            ]
+
+        # `check_reference_form` on the model guarantees all three legacy
+        # fields are set when `parent_dataset_references` is empty.
+        if (
+            definition.parent_dataset_reference_attribute_id is None
+            or definition.dataset_reference_source_column is None
+            or definition.dataset_reference_source_column_data_type is None
+        ):
+            raise ValueError(
+                "Legacy reference fields must be set when "
+                "`parent_dataset_references` is not provided."
+            )
+        return [
+            CatalogDeclarativeReferenceSource(
+                column=definition.dataset_reference_source_column,
+                data_type=definition.dataset_reference_source_column_data_type.value,
+                target=CatalogGrainIdentifier(
+                    id=definition.parent_dataset_reference_attribute_id,
+                    type=CustomFieldType.ATTRIBUTE.value,
+                ),
+            )
+        ]
+
     @staticmethod
     def _get_sources(
         dataset: CustomDataset,
@@ -253,6 +293,39 @@ class LdmExtensionDataProcessor:
             # Get the data source info
             dataset_source_table_id, dataset_sql = self._get_sources(dataset)
 
+            parent_reference_sources = self._build_parent_reference_sources(
+                dataset.definition
+            )
+
+            wdf_columns: (
+                list[CatalogDeclarativeWorkspaceDataFilterColumn] | None
+            ) = None
+            wdf_references: (
+                list[CatalogDeclarativeWorkspaceDataFilterReferences] | None
+            ) = None
+            wdf_id = dataset.definition.workspace_data_filter_id
+            wdf_column_name = (
+                dataset.definition.workspace_data_filter_column_name
+            )
+            # `check_wdf_pair` on the model guarantees both fields are set
+            # together or both omitted.
+            if wdf_id is not None and wdf_column_name is not None:
+                wdf_columns = [
+                    CatalogDeclarativeWorkspaceDataFilterColumn(
+                        name=wdf_column_name,
+                        data_type=ColumnDataType.STRING.value,
+                    )
+                ]
+                wdf_references = [
+                    CatalogDeclarativeWorkspaceDataFilterReferences(
+                        filter_id=CatalogDatasetWorkspaceDataFilterIdentifier(
+                            id=wdf_id
+                        ),
+                        filter_column=wdf_column_name,
+                        filter_column_data_type=ColumnDataType.STRING.value,
+                    )
+                ]
+
             # Construct the declarative dataset object and append it to the list.
             declarative_datasets.append(
                 CatalogDeclarativeDataset(
@@ -265,16 +338,7 @@ class LdmExtensionDataProcessor:
                                 id=dataset.definition.parent_dataset_reference,
                             ),
                             multivalue=True,
-                            sources=[
-                                CatalogDeclarativeReferenceSource(
-                                    column=dataset.definition.dataset_reference_source_column,
-                                    data_type=dataset.definition.dataset_reference_source_column_data_type.value,
-                                    target=CatalogGrainIdentifier(
-                                        id=dataset.definition.parent_dataset_reference_attribute_id,
-                                        type=CustomFieldType.ATTRIBUTE.value,
-                                    ),
-                                )
-                            ],
+                            sources=parent_reference_sources,
                         ),
                     ]
                     + date_references,
@@ -283,21 +347,8 @@ class LdmExtensionDataProcessor:
                     facts=facts,
                     data_source_table_id=dataset_source_table_id,
                     sql=dataset_sql,
-                    workspace_data_filter_columns=[
-                        CatalogDeclarativeWorkspaceDataFilterColumn(
-                            name=dataset.definition.workspace_data_filter_column_name,
-                            data_type=ColumnDataType.STRING.value,
-                        )
-                    ],
-                    workspace_data_filter_references=[
-                        CatalogDeclarativeWorkspaceDataFilterReferences(
-                            filter_id=CatalogDatasetWorkspaceDataFilterIdentifier(
-                                id=dataset.definition.workspace_data_filter_id
-                            ),
-                            filter_column=dataset.definition.workspace_data_filter_column_name,
-                            filter_column_data_type=ColumnDataType.STRING.value,
-                        )
-                    ],
+                    workspace_data_filter_columns=wdf_columns,
+                    workspace_data_filter_references=wdf_references,
                     tags=_effective_dataset_tags(dataset.definition),
                 )
             )

gooddata_pipelines-1.65.1.dev3/src/gooddata_pipelines/ldm_extension/models/custom_data_object.py

@@ -0,0 +1,184 @@
+# (C) 2025 GoodData Corporation
+"""This module defines enums and models used to represent the input data.
+
+Models defined here are used to validate and structure the input data before
+further processing.
+"""
+
+from enum import Enum
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class CustomFieldType(str, Enum):
+    """GoodData field types."""
+
+    # NOTE: Start using StrEnum with Python 3.11
+    ATTRIBUTE = "attribute"
+    FACT = "fact"
+    DATE = "date"
+
+
+class ColumnDataType(str, Enum):
+    """Supported data types"""
+
+    # NOTE: Start using StrEnum with Python 3.11
+    INT = "INT"
+    STRING = "STRING"
+    DATE = "DATE"
+    NUMERIC = "NUMERIC"
+    TIMESTAMP = "TIMESTAMP"
+    TIMESTAMP_TZ = "TIMESTAMP_TZ"
+    BOOLEAN = "BOOLEAN"
+
+
+class CustomFieldDefinition(BaseModel):
+    """Input model for custom field definition."""
+
+    workspace_id: str
+    dataset_id: str
+    custom_field_id: str
+    custom_field_name: str
+    custom_field_type: CustomFieldType
+    custom_field_source_column: str
+    custom_field_source_column_data_type: ColumnDataType
+    description: str | None = Field(
+        default=None,
+        description="Declarative description on the attribute, fact, or date dataset.",
+    )
+    tags: list[str] | None = Field(
+        default=None,
+        description="If set, replaces the default tag list (dataset display name only).",
+    )
+
+    @model_validator(mode="after")
+    def check_ids_not_equal(self) -> "CustomFieldDefinition":
+        """Check that custom field ID is not the same as dataset ID."""
+        if self.custom_field_id == self.dataset_id:
+            raise ValueError(
+                f"Custom field ID {self.custom_field_id} cannot be the same as dataset ID {self.dataset_id}"
+            )
+        return self
+
+
+class ParentDatasetReference(BaseModel):
+    """One column of a (possibly composite) join to the parent dataset.
+
+    A list of these on ``CustomDatasetDefinition.parent_dataset_references``
+    supports multi-column foreign keys. Each entry binds a source column on the
+    new dataset to a grain attribute on the parent.
+    """
+
+    attribute_id: str = Field(
+        description="Attribute ID on the parent dataset that this column joins to.",
+    )
+    source_column: str = Field(
+        description="Column name on this dataset used to join to the parent.",
+    )
+    data_type: ColumnDataType = Field(
+        description="Data type of the source column.",
+    )
+
+
+class CustomDatasetDefinition(BaseModel):
+    """Input model for custom dataset definition."""
+
+    workspace_id: str
+    dataset_id: str
+    dataset_name: str
+    dataset_datasource_id: str
+    dataset_source_table: str | None
+    dataset_source_sql: str | None
+    parent_dataset_reference: str
+    parent_dataset_reference_attribute_id: str | None = Field(
+        default=None,
+        deprecated=(
+            "Use `parent_dataset_references` instead. "
+            "This field will be removed in a future release."
+        ),
+    )
+    dataset_reference_source_column: str | None = Field(
+        default=None,
+        deprecated=(
+            "Use `parent_dataset_references` instead. "
+            "This field will be removed in a future release."
+        ),
+    )
+    dataset_reference_source_column_data_type: ColumnDataType | None = Field(
+        default=None,
+        deprecated=(
+            "Use `parent_dataset_references` instead. "
+            "This field will be removed in a future release."
+        ),
+    )
+    parent_dataset_references: list[ParentDatasetReference] | None = Field(
+        default=None,
+        description="List of references to the parent dataset.",
+    )
+    workspace_data_filter_id: str | None = None
+    workspace_data_filter_column_name: str | None = None
+    dataset_description: str | None = Field(
+        default=None,
+        description="Declarative description on the custom dataset.",
+    )
+    dataset_tags: list[str] | None = Field(
+        default=None,
+        description="If set, replaces the default tag list (dataset display name only).",
+    )
+
+    @model_validator(mode="after")
+    def check_source(self) -> "CustomDatasetDefinition":
+        """At least one of dataset_source_table or dataset_source_sql is provided."""
+        if not (self.dataset_source_table or self.dataset_source_sql):
+            raise ValueError(
+                "One of dataset_source_table and dataset_source_sql must be provided"
+            )
+        if self.dataset_source_table and self.dataset_source_sql:
+            raise ValueError(
+                "Only one of dataset_source_table and dataset_source_sql can be provided"
+            )
+        return self
+
+    @model_validator(mode="after")
+    def check_reference_form(self) -> "CustomDatasetDefinition":
+        """Exactly one reference form must be set: either the new list or the legacy triple."""
+        has_new = bool(self.parent_dataset_references)
+        has_legacy = (
+            self.parent_dataset_reference_attribute_id is not None
+            or self.dataset_reference_source_column is not None
+            or self.dataset_reference_source_column_data_type is not None
+        )
+        if has_new and has_legacy:
+            raise ValueError(
+                "Set either `parent_dataset_references` or the legacy single-column "
+                "fields (`parent_dataset_reference_attribute_id`, "
+                "`dataset_reference_source_column`, "
+                "`dataset_reference_source_column_data_type`), not both."
+            )
+        if not has_new and not has_legacy:
+            raise ValueError(
+                "Provide either `parent_dataset_references` or the legacy single-column "
+                "fields (`parent_dataset_reference_attribute_id`, "
+                "`dataset_reference_source_column`, "
+                "`dataset_reference_source_column_data_type`)."
+            )
+        return self
+
+    @model_validator(mode="after")
+    def check_wdf_pair(self) -> "CustomDatasetDefinition":
+        """Workspace data filter id and column name must be provided together or both omitted."""
+        has_id = self.workspace_data_filter_id is not None
+        has_col = self.workspace_data_filter_column_name is not None
+        if has_id != has_col:
+            raise ValueError(
+                "workspace_data_filter_id and workspace_data_filter_column_name "
+                "must both be set or both be omitted"
+            )
+        return self
+
+
+class CustomDataset(BaseModel):
+    """Custom dataset with its definition and custom fields."""
+
+    definition: CustomDatasetDefinition
+    custom_fields: list[CustomFieldDefinition]

{gooddata_pipelines-1.65.1.dev2 → gooddata_pipelines-1.65.1.dev3}/tests/test_ldm_extension/test_input_processor.py

@@ -129,3 +129,52 @@ def test_datasets_to_ldm(mock_custom_dataset):
     assert ds.workspace_data_filter_references[0].filter_id.id == "wdf1"
     assert len(ldm.date_instances) == 1
     assert ldm.date_instances[0].id == "date1"
+
+
+def test_datasets_to_ldm_parent_dataset_references_composite():
+    """Multi-column references via `parent_dataset_references` produce N sources."""
+    from gooddata_pipelines.ldm_extension.models.custom_data_object import (
+        CustomDatasetDefinition,
+        ParentDatasetReference,
+    )
+
+    definition = CustomDatasetDefinition(
+        workspace_id="workspace1",
+        dataset_id="ds_composite",
+        dataset_name="Composite Dataset",
+        dataset_source_table="table1",
+        dataset_datasource_id="ds_source",
+        dataset_source_sql=None,
+        parent_dataset_reference="parent_ds",
+        parent_dataset_references=[
+            ParentDatasetReference(
+                attribute_id="parent_pk1",
+                source_column="src_col1",
+                data_type=ColumnDataType.STRING,
+            ),
+            ParentDatasetReference(
+                attribute_id="parent_pk2",
+                source_column="src_col2",
+                data_type=ColumnDataType.INT,
+            ),
+        ],
+        workspace_data_filter_id="wdf1",
+        workspace_data_filter_column_name="col1",
+    )
+    ds = CustomDataset(definition=definition, custom_fields=[])
+    processor = LdmExtensionDataProcessor()
+    model = processor.datasets_to_ldm({"ds_composite": ds})
+    parent_ref = model.ldm.datasets[0].references[0]
+    assert len(parent_ref.sources) == 2
+    assert [s.column for s in parent_ref.sources] == ["src_col1", "src_col2"]
+
+
+def test_datasets_to_ldm_legacy_reference_fallback(mock_dataset_definition):
+    """When `parent_dataset_references` is not set, fall back to legacy fields."""
+    mock_dataset_definition.parent_dataset_references = None
+    ds = CustomDataset(definition=mock_dataset_definition, custom_fields=[])
+    processor = LdmExtensionDataProcessor()
+    model = processor.datasets_to_ldm({"ds1": ds})
+    parent_ref = model.ldm.datasets[0].references[0]
+    assert len(parent_ref.sources) == 1
+    assert parent_ref.sources[0].column == "ref_col"

gooddata_pipelines-1.65.1.dev3/tests/test_ldm_extension/test_models/test_custom_data_object.py

@@ -0,0 +1,208 @@
+# (C) 2025 GoodData Corporation
+import pytest
+from pydantic import ValidationError
+
+from gooddata_pipelines.ldm_extension.models.custom_data_object import (
+    ColumnDataType,
+    CustomDataset,
+    CustomDatasetDefinition,
+    CustomFieldDefinition,
+    CustomFieldType,
+    ParentDatasetReference,
+)
+
+
+def make_valid_field_def(**kwargs):
+    data = {
+        "workspace_id": "ws1",
+        "dataset_id": "ds1",
+        "custom_field_id": "cf1",
+        "custom_field_name": "Custom Field",
+        "custom_field_type": CustomFieldType.ATTRIBUTE,
+        "custom_field_source_column": "col1",
+        "custom_field_source_column_data_type": ColumnDataType.STRING,
+    }
+    data.update(kwargs)
+    return data
+
+
+def make_valid_dataset_def(**kwargs):
+    data = {
+        "workspace_id": "ws1",
+        "dataset_id": "ds1",
+        "dataset_name": "Dataset",
+        "dataset_datasource_id": "dsrc1",
+        "dataset_source_table": "table1",
+        "dataset_source_sql": None,
+        "parent_dataset_reference": "parent_ds",
+        "parent_dataset_reference_attribute_id": "parent_attr",
+        "dataset_reference_source_column": "src_col",
+        "dataset_reference_source_column_data_type": ColumnDataType.STRING,
+        "workspace_data_filter_id": "wdf1",
+        "workspace_data_filter_column_name": "col1",
+    }
+    data.update(kwargs)
+    return data
+
+
+def test_custom_field_definition_valid():
+    field = CustomFieldDefinition(**make_valid_field_def())
+    assert field.custom_field_id == "cf1"
+    assert field.custom_field_type == CustomFieldType.ATTRIBUTE
+
+
+def test_custom_field_definition_cf_id_equals_dataset_id_raises():
+    data = make_valid_field_def(custom_field_id="ds1")
+    with pytest.raises(ValidationError) as exc:
+        CustomFieldDefinition(**data)
+    assert "cannot be the same as dataset ID" in str(exc.value)
+
+
+def test_custom_dataset_definition_valid_table():
+    ds = CustomDatasetDefinition(**make_valid_dataset_def())
+    assert ds.dataset_source_table == "table1"
+    assert ds.dataset_source_sql is None
+
+
+def test_custom_dataset_definition_valid_sql():
+    data = make_valid_dataset_def(
+        dataset_source_table=None, dataset_source_sql="SELECT 1"
+    )
+    ds = CustomDatasetDefinition(**data)
+    assert ds.dataset_source_sql == "SELECT 1"
+    assert ds.dataset_source_table is None
+
+
+def test_custom_dataset_definition_both_none_raises():
+    data = make_valid_dataset_def(
+        dataset_source_table=None, dataset_source_sql=None
+    )
+    with pytest.raises(ValidationError) as exc:
+        CustomDatasetDefinition(**data)
+    assert "must be provided" in str(exc.value)
+
+
+def test_custom_dataset_definition_both_provided_raises():
+    data = make_valid_dataset_def(
+        dataset_source_table="table1", dataset_source_sql="SELECT 1"
+    )
+    with pytest.raises(ValidationError) as exc:
+        CustomDatasetDefinition(**data)
+    assert (
+        "Only one of dataset_source_table and dataset_source_sql can be provided"
+        in str(exc.value)
+    )
+
+
+def test_custom_dataset_model():
+    ds_def = CustomDatasetDefinition(**make_valid_dataset_def())
+    field_def = CustomFieldDefinition(**make_valid_field_def())
+    dataset = CustomDataset(definition=ds_def, custom_fields=[field_def])
+    assert dataset.definition.dataset_id == "ds1"
+    assert len(dataset.custom_fields) == 1
+    assert dataset.custom_fields[0].custom_field_id == "cf1"
+
+
+def test_custom_dataset_definition_parent_dataset_references_optional():
+    """The new composite-reference field is optional and defaults to None."""
+    ds = CustomDatasetDefinition(**make_valid_dataset_def())
+    assert ds.parent_dataset_references is None
+
+
+def test_custom_dataset_definition_parent_dataset_references_accepted():
+    """Composite references can be provided via the new list field."""
+    refs = [
+        ParentDatasetReference(
+            attribute_id="parent_pk1",
+            source_column="src_col1",
+            data_type=ColumnDataType.STRING,
+        ),
+        ParentDatasetReference(
+            attribute_id="parent_pk2",
+            source_column="src_col2",
+            data_type=ColumnDataType.INT,
+        ),
+    ]
+    data = make_valid_dataset_def(
+        parent_dataset_reference_attribute_id=None,
+        dataset_reference_source_column=None,
+        dataset_reference_source_column_data_type=None,
+        parent_dataset_references=refs,
+    )
+    ds = CustomDatasetDefinition(**data)
+    assert ds.parent_dataset_references is not None
+    assert len(ds.parent_dataset_references) == 2
+    assert ds.parent_dataset_references[1].data_type == ColumnDataType.INT
+
+
+def test_custom_dataset_definition_no_reference_form_raises():
+    """Providing neither the legacy fields nor `parent_dataset_references` is rejected."""
+    data = make_valid_dataset_def(
+        parent_dataset_reference_attribute_id=None,
+        dataset_reference_source_column=None,
+        dataset_reference_source_column_data_type=None,
+    )
+    with pytest.raises(ValidationError) as exc:
+        CustomDatasetDefinition(**data)
+    assert "Provide either" in str(exc.value)
+
+
+def test_custom_dataset_definition_mixed_reference_forms_raises():
+    """Setting both legacy fields and `parent_dataset_references` is rejected."""
+    data = make_valid_dataset_def(
+        parent_dataset_references=[
+            ParentDatasetReference(
+                attribute_id="parent_pk",
+                source_column="src_col",
+                data_type=ColumnDataType.STRING,
+            )
+        ],
+    )
+    with pytest.raises(ValidationError) as exc:
+        CustomDatasetDefinition(**data)
+    assert "not both" in str(exc.value)
+
+
+def test_custom_dataset_definition_legacy_reference_fields_optional():
+    data = make_valid_dataset_def(
+        parent_dataset_reference_attribute_id=None,
+        dataset_reference_source_column=None,
+        dataset_reference_source_column_data_type=None,
+        parent_dataset_references=[
+            ParentDatasetReference(
+                attribute_id="parent_pk",
+                source_column="src_col",
+                data_type=ColumnDataType.STRING,
+            )
+        ],
+    )
+    ds = CustomDatasetDefinition(**data)
+    assert ds.dataset_reference_source_column is None
+    assert ds.parent_dataset_references is not None
+
+
+def test_custom_dataset_definition_wdf_optional_both_none():
+    data = make_valid_dataset_def(
+        workspace_data_filter_id=None, workspace_data_filter_column_name=None
+    )
+    ds = CustomDatasetDefinition(**data)
+    assert ds.workspace_data_filter_id is None
+    assert ds.workspace_data_filter_column_name is None
+
+
+def test_custom_dataset_definition_wdf_only_id_raises():
+    data = make_valid_dataset_def(
+        workspace_data_filter_id="wdf1", workspace_data_filter_column_name=None
+    )
+    with pytest.raises(ValidationError) as exc:
+        CustomDatasetDefinition(**data)
+    assert "both be set or both be omitted" in str(exc.value)
+
+
+def test_custom_dataset_definition_wdf_only_column_raises():
+    data = make_valid_dataset_def(
+        workspace_data_filter_id=None, workspace_data_filter_column_name="col1"
+    )
+    with pytest.raises(ValidationError) as exc:
+        CustomDatasetDefinition(**data)
+    assert "both be set or both be omitted" in str(exc.value)