gooddata-pipelines 1.64.1.dev1__tar.gz → 1.64.1.dev2__tar.gz

{gooddata_pipelines-1.64.1.dev1 → gooddata_pipelines-1.64.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-pipelines
-Version: 1.64.1.dev1
+Version: 1.64.1.dev2
 Summary: GoodData Cloud lifecycle automation pipelines
 Author-email: GoodData <support@gooddata.com>
 License-Expression: MIT
@@ -10,7 +10,7 @@ Requires-Dist: azure-identity<2.0.0,>=1.15.0
 Requires-Dist: azure-storage-blob<13.0.0,>=12.19.0
 Requires-Dist: boto3-stubs<2.0.0,>=1.39.3
 Requires-Dist: boto3<2.0.0,>=1.39.3
-Requires-Dist: gooddata-sdk~=1.64.1.dev1
+Requires-Dist: gooddata-sdk~=1.64.1.dev2
 Requires-Dist: pydantic<3.0.0,>=2.9.2
 Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-pyyaml<7.0.0,>=6.0.12.20250326

{gooddata_pipelines-1.64.1.dev1 → gooddata_pipelines-1.64.1.dev2}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2025 GoodData Corporation
 [project]
 name = "gooddata-pipelines"
-version = "1.64.1.dev1"
+version = "1.64.1.dev2"
 description = "GoodData Cloud lifecycle automation pipelines"
 authors = [{ name = "GoodData", email = "support@gooddata.com" }]
 license = "MIT"
@@ -11,7 +11,7 @@ dependencies = [
     "pydantic (>=2.9.2,<3.0.0)",
     "requests (>=2.32.3,<3.0.0)",
     "types-requests (>1.0.0,<3.0.0)",
-    "gooddata-sdk~=1.64.1.dev1",
+    "gooddata-sdk~=1.64.1.dev2",
     "boto3 (>=1.39.3,<2.0.0)",
     "boto3-stubs (>=1.39.3,<2.0.0)",
     "azure-storage-blob (>=12.19.0,<13.0.0)",

{gooddata_pipelines-1.64.1.dev1 → gooddata_pipelines-1.64.1.dev2}/src/gooddata_pipelines/ldm_extension/input_processor.py

@@ -5,6 +5,8 @@ This module is responsible for converting validated custom datasets and fields
 into objects defined in the GoodData Python SDK.
 """
 
+import copy
+
 from gooddata_sdk.catalog.identifier import (
     CatalogDatasetWorkspaceDataFilterIdentifier,
     CatalogGrainIdentifier,
@@ -36,11 +38,26 @@ from gooddata_pipelines.ldm_extension.models.aliases import DatasetId
 from gooddata_pipelines.ldm_extension.models.custom_data_object import (
     ColumnDataType,
     CustomDataset,
+    CustomDatasetDefinition,
     CustomFieldDefinition,
     CustomFieldType,
 )
 
 
+def _effective_field_tags(
+    dataset_name: str, custom_field: CustomFieldDefinition
+) -> list[str]:
+    if custom_field.tags is not None:
+        return list(custom_field.tags)
+    return [dataset_name]
+
+
+def _effective_dataset_tags(definition: CustomDatasetDefinition) -> list[str]:
+    if definition.dataset_tags is not None:
+        return list(definition.dataset_tags)
+    return [definition.dataset_name]
+
+
 class LdmExtensionDataProcessor:
     """Create GoodData LDM from validated custom datasets and fields."""
 
@@ -77,7 +94,8 @@ class LdmExtensionDataProcessor:
             source_column=custom_field.custom_field_source_column,
             labels=[],
             source_column_data_type=custom_field.custom_field_source_column_data_type.value,
-            tags=[dataset_name],
+            description=custom_field.description,
+            tags=_effective_field_tags(dataset_name, custom_field),
         )
 
     @staticmethod
@@ -91,7 +109,8 @@ class LdmExtensionDataProcessor:
             title=custom_field.custom_field_name,
             source_column=custom_field.custom_field_source_column,
             source_column_data_type=custom_field.custom_field_source_column_data_type.value,
-            tags=[dataset_name],
+            description=custom_field.description,
+            tags=_effective_field_tags(dataset_name, custom_field),
         )
 
     def _date_from_field(
@@ -109,7 +128,8 @@ class LdmExtensionDataProcessor:
                 title_pattern="%titleBase - %granularityTitle",
             ),
             granularities=self.DATE_GRANULARITIES,
-            tags=[dataset_name],
+            description=custom_field.description,
+            tags=_effective_field_tags(dataset_name, custom_field),
         )
 
     @staticmethod
@@ -258,7 +278,7 @@ class LdmExtensionDataProcessor:
                         ),
                     ]
                     + date_references,
-                    description=None,
+                    description=dataset.definition.dataset_description,
                     attributes=attributes,
                     facts=facts,
                     data_source_table_id=dataset_source_table_id,
@@ -278,7 +298,7 @@ class LdmExtensionDataProcessor:
                             filter_column_data_type=ColumnDataType.STRING.value,
                         )
                     ],
-                    tags=[dataset.definition.dataset_name],
+                    tags=_effective_dataset_tags(dataset.definition),
                 )
             )
 
@@ -287,3 +307,60 @@ class LdmExtensionDataProcessor:
             datasets=declarative_datasets, date_instances=date_instances
         )
         return CatalogDeclarativeModel(ldm=ldm)
+
+    def merge_custom_ldm_into_existing(
+        self,
+        existing: CatalogDeclarativeModel,
+        custom_datasets: dict[DatasetId, CustomDataset],
+        *,
+        remove_managed_datasets_missing_from_input: bool = False,
+        management_tag: str | None = None,
+    ) -> CatalogDeclarativeModel:
+        """Merge datasets produced from ``custom_datasets`` into an existing declarative LDM.
+
+        Custom datasets and date instances that share an ``id`` with the fragment replace
+        their previous definitions. When ``remove_managed_datasets_missing_from_input`` is
+        set, datasets that carry ``management_tag`` but are absent from the incoming
+        fragment are removed first (typical for tooling-owned extension datasets).
+
+        Any other pre-existing LDM objects (previously uploaded extensions whose ids
+        are not in the incoming fragment) are preserved unchanged.
+        """
+        fragment = self.datasets_to_ldm(custom_datasets)
+        fragment_ldm = fragment.ldm or CatalogDeclarativeLdm(
+            datasets=[], date_instances=[]
+        )
+
+        result = copy.deepcopy(existing)
+        result_ldm = result.ldm or CatalogDeclarativeLdm(
+            datasets=[], date_instances=[]
+        )
+        result.ldm = result_ldm
+
+        incoming_dataset_ids = {d.id for d in fragment_ldm.datasets}
+        incoming_date_ids = {d.id for d in fragment_ldm.date_instances}
+
+        datasets = list(result_ldm.datasets)
+        if remove_managed_datasets_missing_from_input and management_tag:
+            datasets = [
+                d
+                for d in datasets
+                if not (
+                    d.tags
+                    and management_tag in d.tags
+                    and d.id not in incoming_dataset_ids
+                )
+            ]
+        datasets = [d for d in datasets if d.id not in incoming_dataset_ids]
+        datasets.extend(fragment_ldm.datasets)
+        result_ldm.datasets = datasets
+
+        date_instances = [
+            d
+            for d in result_ldm.date_instances
+            if d.id not in incoming_date_ids
+        ]
+        date_instances.extend(fragment_ldm.date_instances)
+        result_ldm.date_instances = date_instances
+
+        return result

{gooddata_pipelines-1.64.1.dev1 → gooddata_pipelines-1.64.1.dev2}/src/gooddata_pipelines/ldm_extension/ldm_extension_manager.py

@@ -3,6 +3,9 @@
 
 from pathlib import Path
 
+from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm import (
+    CatalogDeclarativeModel,
+)
 from gooddata_sdk.sdk import GoodDataSdk
 from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
 
@@ -147,9 +150,35 @@ class LdmExtensionManager:
         # If the set of new invalid relations is a subset of the current one,
         return set_new_invalid_relations.issubset(set_current_invalid_relations)
 
+    def _ldm_payload_for_workspace(
+        self,
+        workspace_id: str,
+        datasets: dict[DatasetId, CustomDataset],
+        *,
+        merge_into_existing_ldm: bool,
+        remove_managed_datasets_missing_from_input: bool,
+        management_tag: str | None,
+    ) -> CatalogDeclarativeModel:
+        """Build the declarative LDM payload to upload for one workspace."""
+        if not merge_into_existing_ldm:
+            return self._processor.datasets_to_ldm(datasets)
+        current = self._sdk.catalog_workspace_content.get_declarative_ldm(
+            workspace_id
+        )
+        return self._processor.merge_custom_ldm_into_existing(
+            current,
+            datasets,
+            remove_managed_datasets_missing_from_input=remove_managed_datasets_missing_from_input,
+            management_tag=management_tag,
+        )
+
     def _process_with_relations_check(
         self,
         validated_data: dict[WorkspaceId, dict[DatasetId, CustomDataset]],
+        *,
+        merge_into_existing_ldm: bool = False,
+        remove_managed_datasets_missing_from_input: bool = False,
+        management_tag: str | None = None,
     ) -> None:
         """Check whether relations of analytical objects are valid before and after
         updating the LDM in the GoodData workspace.
@@ -173,7 +202,13 @@ class LdmExtensionManager:
             # Put the LDM with custom datasets into the GoodData workspace.
             self._sdk.catalog_workspace_content.put_declarative_ldm(
                 workspace_id=workspace_id,
-                ldm=self._processor.datasets_to_ldm(datasets),
+                ldm=self._ldm_payload_for_workspace(
+                    workspace_id,
+                    datasets,
+                    merge_into_existing_ldm=merge_into_existing_ldm,
+                    remove_managed_datasets_missing_from_input=remove_managed_datasets_missing_from_input,
+                    management_tag=management_tag,
+                ),
             )
 
             # Get a set of objects with invalid relations from the new workspace state
@@ -232,13 +267,23 @@ class LdmExtensionManager:
     def _process_without_relations_check(
         self,
         validated_data: dict[WorkspaceId, dict[DatasetId, CustomDataset]],
+        *,
+        merge_into_existing_ldm: bool = False,
+        remove_managed_datasets_missing_from_input: bool = False,
+        management_tag: str | None = None,
     ) -> None:
         """Update the LDM in the GoodData workspace without checking relations."""
         for workspace_id, datasets in validated_data.items():
             # Put the LDM with custom datasets into the GoodData workspace.
             self._sdk.catalog_workspace_content.put_declarative_ldm(
                 workspace_id=workspace_id,
-                ldm=self._processor.datasets_to_ldm(datasets),
+                ldm=self._ldm_payload_for_workspace(
+                    workspace_id,
+                    datasets,
+                    merge_into_existing_ldm=merge_into_existing_ldm,
+                    remove_managed_datasets_missing_from_input=remove_managed_datasets_missing_from_input,
+                    management_tag=management_tag,
+                ),
             )
             self._log_success_message(workspace_id)
 
@@ -251,6 +296,9 @@ class LdmExtensionManager:
         custom_datasets: list[CustomDatasetDefinition],
         custom_fields: list[CustomFieldDefinition],
         check_relations: bool = True,
+        merge_into_existing_ldm: bool = False,
+        remove_managed_datasets_missing_from_input: bool = False,
+        management_tag: str | None = None,
     ) -> None:
         """Create custom datasets and fields in GoodData workspaces.
 
@@ -266,6 +314,14 @@ class LdmExtensionManager:
                 after updating the LDM. If the number of invalid relations increases,
                 the LDM will be reverted to its previous state. If False, the check
                 is skiped and the LDM is updated directly. Defaults to True.
+            merge_into_existing_ldm (bool): When True, load the workspace LDM first and merge
+                the generated custom datasets and date instances into it instead of uploading
+                only the extension fragment. Defaults to False for backward compatibility.
+            remove_managed_datasets_missing_from_input (bool): When ``merge_into_existing_ldm``
+                is True, remove existing datasets that contain ``management_tag`` but whose
+                dataset id is not present in this ``process`` call (tooling cleanup).
+            management_tag (str | None): Tag value used with
+                ``remove_managed_datasets_missing_from_input``.
 
         Raises:
             ValueError: If there are validation errors in the dataset or field definitions.
@@ -278,6 +334,16 @@ class LdmExtensionManager:
 
         if check_relations:
             # Process the validated data with relations check.
-            self._process_with_relations_check(validated_data)
+            self._process_with_relations_check(
+                validated_data,
+                merge_into_existing_ldm=merge_into_existing_ldm,
+                remove_managed_datasets_missing_from_input=remove_managed_datasets_missing_from_input,
+                management_tag=management_tag,
+            )
         else:
-            self._process_without_relations_check(validated_data)
+            self._process_without_relations_check(
+                validated_data,
+                merge_into_existing_ldm=merge_into_existing_ldm,
+                remove_managed_datasets_missing_from_input=remove_managed_datasets_missing_from_input,
+                management_tag=management_tag,
+            )

{gooddata_pipelines-1.64.1.dev1 → gooddata_pipelines-1.64.1.dev2}/src/gooddata_pipelines/ldm_extension/models/custom_data_object.py

@@ -7,7 +7,7 @@ further processing.
 
 from enum import Enum
 
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel, Field, model_validator
 
 
 class CustomFieldType(str, Enum):
@@ -42,6 +42,14 @@ class CustomFieldDefinition(BaseModel):
     custom_field_type: CustomFieldType
     custom_field_source_column: str
     custom_field_source_column_data_type: ColumnDataType
+    description: str | None = Field(
+        default=None,
+        description="Declarative description on the attribute, fact, or date dataset.",
+    )
+    tags: list[str] | None = Field(
+        default=None,
+        description="If set, replaces the default tag list (dataset display name only).",
+    )
 
     @model_validator(mode="after")
     def check_ids_not_equal(self) -> "CustomFieldDefinition":
@@ -68,6 +76,14 @@ class CustomDatasetDefinition(BaseModel):
     dataset_reference_source_column_data_type: ColumnDataType
     workspace_data_filter_id: str
     workspace_data_filter_column_name: str
+    dataset_description: str | None = Field(
+        default=None,
+        description="Declarative description on the custom dataset.",
+    )
+    dataset_tags: list[str] | None = Field(
+        default=None,
+        description="If set, replaces the default tag list (dataset display name only).",
+    )
 
     @model_validator(mode="after")
     def check_source(self) -> "CustomDatasetDefinition":

gooddata_pipelines-1.64.1.dev2/tests/test_ldm_extension/conftest.py

@@ -0,0 +1,84 @@
+# (C) 2025 GoodData Corporation
+import pytest
+
+from gooddata_pipelines.ldm_extension.models.custom_data_object import (
+    ColumnDataType,
+    CustomDataset,
+    CustomDatasetDefinition,
+    CustomFieldDefinition,
+    CustomFieldType,
+)
+
+
+@pytest.fixture
+def mock_custom_field_attribute():
+    return CustomFieldDefinition(
+        workspace_id="workspace1",
+        dataset_id="ds1",
+        custom_field_id="attr1",
+        custom_field_name="Attribute 1",
+        custom_field_type=CustomFieldType.ATTRIBUTE,
+        custom_field_source_column="col_attr1",
+        custom_field_source_column_data_type=ColumnDataType.STRING,
+    )
+
+
+@pytest.fixture
+def mock_custom_field_fact():
+    return CustomFieldDefinition(
+        workspace_id="workspace1",
+        dataset_id="ds1",
+        custom_field_id="fact1",
+        custom_field_name="Fact 1",
+        custom_field_type=CustomFieldType.FACT,
+        custom_field_source_column="col_fact1",
+        custom_field_source_column_data_type=ColumnDataType.INT,
+    )
+
+
+@pytest.fixture
+def mock_custom_field_date():
+    return CustomFieldDefinition(
+        workspace_id="workspace1",
+        dataset_id="ds1",
+        custom_field_id="date1",
+        custom_field_name="Date 1",
+        custom_field_type=CustomFieldType.DATE,
+        custom_field_source_column="col_date1",
+        custom_field_source_column_data_type=ColumnDataType.DATE,
+    )
+
+
+@pytest.fixture
+def mock_dataset_definition():
+    return CustomDatasetDefinition(
+        workspace_id="workspace1",
+        dataset_id="ds1",
+        dataset_name="Dataset 1",
+        dataset_source_table="table1",
+        dataset_datasource_id="ds_source",
+        dataset_source_sql=None,
+        parent_dataset_reference="parent_ds",
+        parent_dataset_reference_attribute_id="parent_attr",
+        dataset_reference_source_column="ref_col",
+        dataset_reference_source_column_data_type=ColumnDataType.STRING,
+        workspace_data_filter_id="wdf1",
+        workspace_data_filter_column_name="col1",
+    )
+
+
+@pytest.fixture
+def mock_custom_dataset(
+    mock_dataset_definition,
+    mock_custom_field_attribute,
+    mock_custom_field_fact,
+    mock_custom_field_date,
+):
+    return CustomDataset(
+        definition=mock_dataset_definition,
+        custom_fields=[
+            mock_custom_field_attribute,
+            mock_custom_field_fact,
+            mock_custom_field_date,
+        ],
+    )

{gooddata_pipelines-1.64.1.dev1 → gooddata_pipelines-1.64.1.dev2}/tests/test_ldm_extension/test_input_processor.py

@@ -1,21 +1,29 @@
 # (C) 2025 GoodData Corporation
-import pytest
-
 from gooddata_pipelines.ldm_extension.input_processor import (
     LdmExtensionDataProcessor,
 )
 from gooddata_pipelines.ldm_extension.models.custom_data_object import (
     ColumnDataType,
     CustomDataset,
-    CustomDatasetDefinition,
     CustomFieldDefinition,
     CustomFieldType,
 )
 
 
-@pytest.fixture
-def mock_custom_field_attribute():
-    return CustomFieldDefinition(
+def test_attribute_from_field(mock_custom_field_attribute):
+    attr = LdmExtensionDataProcessor._attribute_from_field(
+        "dataset_name", mock_custom_field_attribute
+    )
+    assert attr.id == "attr1"
+    assert attr.title == "Attribute 1"
+    assert attr.source_column == "col_attr1"
+    assert attr.source_column_data_type == ColumnDataType.STRING.value
+    assert attr.tags == ["dataset_name"]
+    assert attr.description is None
+
+
+def test_attribute_from_field_custom_tags_and_description():
+    field = CustomFieldDefinition(
         workspace_id="workspace1",
         dataset_id="ds1",
         custom_field_id="attr1",
@@ -23,79 +31,14 @@ def mock_custom_field_attribute():
         custom_field_type=CustomFieldType.ATTRIBUTE,
         custom_field_source_column="col_attr1",
         custom_field_source_column_data_type=ColumnDataType.STRING,
+        tags=["t1", "t2"],
+        description="Attr desc",
     )
-
-
-@pytest.fixture
-def mock_custom_field_fact():
-    return CustomFieldDefinition(
-        workspace_id="workspace1",
-        dataset_id="ds1",
-        custom_field_id="fact1",
-        custom_field_name="Fact 1",
-        custom_field_type=CustomFieldType.FACT,
-        custom_field_source_column="col_fact1",
-        custom_field_source_column_data_type=ColumnDataType.INT,
-    )
-
-
-@pytest.fixture
-def mock_custom_field_date():
-    return CustomFieldDefinition(
-        workspace_id="workspace1",
-        dataset_id="ds1",
-        custom_field_id="date1",
-        custom_field_name="Date 1",
-        custom_field_type=CustomFieldType.DATE,
-        custom_field_source_column="col_date1",
-        custom_field_source_column_data_type=ColumnDataType.DATE,
-    )
-
-
-@pytest.fixture
-def mock_dataset_definition():
-    return CustomDatasetDefinition(
-        workspace_id="workspace1",
-        dataset_id="ds1",
-        dataset_name="Dataset 1",
-        dataset_source_table="table1",
-        dataset_datasource_id="ds_source",
-        dataset_source_sql=None,
-        parent_dataset_reference="parent_ds",
-        parent_dataset_reference_attribute_id="parent_attr",
-        dataset_reference_source_column="ref_col",
-        dataset_reference_source_column_data_type=ColumnDataType.STRING,
-        workspace_data_filter_id="wdf1",
-        workspace_data_filter_column_name="col1",
-    )
-
-
-@pytest.fixture
-def mock_custom_dataset(
-    mock_dataset_definition,
-    mock_custom_field_attribute,
-    mock_custom_field_fact,
-    mock_custom_field_date,
-):
-    return CustomDataset(
-        definition=mock_dataset_definition,
-        custom_fields=[
-            mock_custom_field_attribute,
-            mock_custom_field_fact,
-            mock_custom_field_date,
-        ],
-    )
-
-
-def test_attribute_from_field(mock_custom_field_attribute):
     attr = LdmExtensionDataProcessor._attribute_from_field(
-        "dataset_name", mock_custom_field_attribute
+        "dataset_name", field
     )
-    assert attr.id == "attr1"
-    assert attr.title == "Attribute 1"
-    assert attr.source_column == "col_attr1"
-    assert attr.source_column_data_type == ColumnDataType.STRING.value
-    assert attr.tags == ["dataset_name"]
+    assert attr.tags == ["t1", "t2"]
+    assert attr.description == "Attr desc"
 
 
 def test_fact_from_field(mock_custom_field_fact):
@@ -149,6 +92,20 @@ def test_get_sources_sql_only(mock_dataset_definition):
     assert sql.statement == "SELECT * FROM foo"
 
 
+def test_datasets_to_ldm_dataset_tags_and_description(mock_dataset_definition):
+    mock_dataset_definition.dataset_tags = ["managed", "extra"]
+    mock_dataset_definition.dataset_description = "DS desc"
+    mock_dataset_definition.dataset_source_sql = "SELECT 1"
+    mock_dataset_definition.dataset_source_table = None
+    ds = CustomDataset(definition=mock_dataset_definition, custom_fields=[])
+    processor = LdmExtensionDataProcessor()
+    model = processor.datasets_to_ldm({"ds1": ds})
+    d = model.ldm.datasets[0]
+    assert d.description == "DS desc"
+    assert d.tags == ["managed", "extra"]
+    assert d.sql is not None
+
+
 def test_datasets_to_ldm(mock_custom_dataset):
     print(mock_custom_dataset)
     processor = LdmExtensionDataProcessor()