cognite-toolkit 0.6.99__tar.gz → 0.6.100__tar.gz

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.99
+Version: 0.6.100
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cdf.toml

@@ -32,7 +32,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.99"
+version = "0.6.100"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cognite_toolkit/_cdf_tk/constants.py

@@ -161,6 +161,17 @@ COGNITE_TIME_SERIES_CONTAINER = ContainerId("cdf_cdm", "CogniteTimeSeries")
 COGNITE_FILE_CONTAINER = ContainerId("cdf_cdm", "CogniteFile")
 CDF_UNIT_SPACE = "cdf_cdm_units"
 
+
+# Container properties that are read-only in DMS needs to be handled with extra care, as this aspect is not currently exposed by the API.
+READONLY_CONTAINER_PROPERTIES = {
+    ContainerId(space="cdf_cdm", external_id="CogniteAsset"): {
+        "assetHierarchy_path_last_updated_time",
+        "assetHierarchy_path",
+        "assetHierarchy_root",
+    },
+    ContainerId(space="cdf_cdm", external_id="CogniteFile"): {"isUploaded", "uploadedTime"},
+}
+
 # Data Plugin Constants
 DATA_DEFAULT_DIR = "data"
 DATA_RESOURCE_DIR = "resources"

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py

@@ -45,6 +45,7 @@ from cognite.client.data_classes.data_modeling import (
     EdgeApplyList,
     EdgeApplyResultList,
     EdgeList,
+    MappedProperty,
     Node,
     NodeApply,
     NodeApplyList,
@@ -74,6 +75,7 @@ from rich.console import Console
 from rich.markup import escape
 from rich.panel import Panel
 
+from cognite_toolkit._cdf_tk import constants
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.graphql_data_models import (
     GraphQLDataModel,
@@ -669,6 +671,26 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
                 self._view_by_id[view.as_id()] = view
         return {view_id: self._view_by_id[view_id] for view_id in view_ids if view_id in self._view_by_id}
 
+    def get_readonly_properties(self, view_id: ViewId) -> set[str]:
+        """Retrieve the set of read-only properties for a given view."""
+
+        readonly_properties: set[str] = set()
+
+        # Retrieve the view to check its properties
+        view = self._lookup_views([view_id]).get(view_id)
+        if view is None:
+            return readonly_properties
+
+        # Check each property in the view
+        for property_identifier, property in view.properties.items():
+            if isinstance(
+                property, MappedProperty
+            ) and property.container_property_identifier in constants.READONLY_CONTAINER_PROPERTIES.get(
+                property.container, set()
+            ):
+                readonly_properties.add(property_identifier)
+        return readonly_properties
+
     def topological_sort(self, view_ids: list[ViewId]) -> list[ViewId]:
         """Sorts the views in topological order based on their implements and through properties."""
         view_by_ids = self._lookup_views(view_ids)

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cognite_toolkit/_cdf_tk/storageio/_instances.py

@@ -4,15 +4,18 @@ from typing import ClassVar
 
 from cognite.client.data_classes.aggregations import Count
 from cognite.client.data_classes.data_modeling import (
+    ContainerId,
     ContainerList,
     EdgeApply,
     NodeApply,
     SpaceList,
+    ViewId,
     ViewList,
 )
 from cognite.client.data_classes.data_modeling.instances import Instance, InstanceApply
 from cognite.client.utils._identifier import InstanceId
 
+from cognite_toolkit._cdf_tk import constants
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceList
 from cognite_toolkit._cdf_tk.cruds import ContainerCRUD, SpaceCRUD, ViewCRUD
@@ -54,10 +57,35 @@ class InstanceIO(
     def __init__(self, client: ToolkitClient, remove_existing_version: bool = True) -> None:
         super().__init__(client)
         self._remove_existing_version = remove_existing_version
+        # Cache for view to read-only properties mapping
+        self._view_readonly_properties_cache: dict[ViewId, set[str]] = {}
+        self._view_crud = ViewCRUD.create_loader(self.client)
 
     def as_id(self, item: Instance) -> str:
         return f"{item.space}:{item.external_id}"
 
+    def _filter_readonly_properties(self, instance: InstanceApply) -> None:
+        """
+        Filter out read-only properties from the instance.
+
+        Args:
+            instance: The instance to filter readonly properties from
+        """
+
+        for source in instance.sources:
+            readonly_properties = set()
+            if isinstance(source.source, ViewId):
+                if source.source not in self._view_readonly_properties_cache:
+                    self._view_readonly_properties_cache[source.source] = self._view_crud.get_readonly_properties(
+                        source.source
+                    )
+                readonly_properties = self._view_readonly_properties_cache[source.source]
+            elif isinstance(source.source, ContainerId):
+                if source.source in constants.READONLY_CONTAINER_PROPERTIES:
+                    readonly_properties = constants.READONLY_CONTAINER_PROPERTIES[source.source]
+
+            source.properties = {k: v for k, v in source.properties.items() if k not in readonly_properties}
+
     def stream_data(self, selector: InstanceSelector, limit: int | None = None) -> Iterable[Page]:
         if isinstance(selector, InstanceViewSelector | InstanceSpaceSelector):
             chunk = InstanceList([])
@@ -133,12 +161,16 @@ class InstanceIO(
         item_to_load = dict(item_json)
         if self._remove_existing_version and "existingVersion" in item_to_load:
             del item_to_load["existingVersion"]
+        instance: InstanceApply
         if instance_type == "node":
-            return NodeApply._load(item_to_load, cognite_client=self.client)
+            instance = NodeApply._load(item_to_load, cognite_client=self.client)
         elif instance_type == "edge":
-            return EdgeApply._load(item_to_load, cognite_client=self.client)
+            instance = EdgeApply._load(item_to_load, cognite_client=self.client)
         else:
             raise ValueError(f"Unknown instance type {instance_type!r}")
+        # Filter out read-only properties if applicable
+        self._filter_readonly_properties(instance)
+        return instance
 
     def configurations(self, selector: InstanceSelector) -> Iterable[StorageIOConfig]:
         if not isinstance(selector, InstanceViewSelector | InstanceSpaceSelector):

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.99
+      image: cognite/toolkit:0.6.100
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.99
+      image: cognite/toolkit:0.6.100
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.99"
+version = "0.6.100"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.100/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.100"

cognite_toolkit-0.6.100/last_changelog_entry.md

@@ -0,0 +1,12 @@
+## cdf 
+
+### Improved
+
+- [alpha] When uploading instances using the `cdf data upload` command,
+Toolkit now skips read-only properties as these cannot be set through
+the API. This means, for example, CogniteAsset and CogniteFile instances
+can now be uploaded and will not fail anymore.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.100/last_git_message.txt

@@ -0,0 +1,33 @@
+[CDF-25328] Upload: Add filtering for read-only properties in data modeling (#2166)
+
+# Description
+
+This PR adds automatic filtering of read-only properties when uploading
+data model instances. A new method `_filter_readonly_properties`,
+detects and removes read-only properties from instance data before
+upload, preventing API errors that occur when attempting to write to
+properties like `assetHierarchy_path`. This is solved by using a
+constant with a list of the relevant properties, since the API currently
+does not expose whether a property is read-only or not.
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Improved
+
+- [alpha] When uploading instances using the `cdf data upload` command,
+Toolkit now skips read-only properties as these cannot be set through
+the API. This means, for example, CogniteAsset and CogniteFile instances
+can now be uploaded and will not fail anymore.
+
+## templates
+
+No changes.
+
+---------
+
+Co-authored-by: Magnus Schjølberg <magnus.schjolberg@cognitedata.com>

cognite_toolkit-0.6.100/last_version.txt

@@ -0,0 +1 @@
+0.6.99

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.99"
+version = "0.6.100"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/tests/test_unit/test_cdf_tk/test_cruds/test_view.py

@@ -99,3 +99,37 @@ class TestViewLoader:
         actual = ViewCRUD.get_dependent_items(item)
 
         assert list(actual) == expected
+
+    @pytest.mark.parametrize(
+        "view_id,expected_readonly_props",
+        [
+            pytest.param(
+                dm.ViewId("cdf_cdm", "CogniteAsset", "v1"),
+                {"pathLastUpdatedTime", "path", "root"},
+                id="CogniteAsset_has_readonly_properties",
+            ),
+            pytest.param(
+                dm.ViewId("cdf_cdm", "CogniteFile", "v1"),
+                {"isUploaded", "uploadedTime"},
+                id="CogniteFile_has_readonly_properties",
+            ),
+            pytest.param(
+                dm.ViewId("cdf_cdm", "CogniteSourceSystem", "v1"),
+                set(),
+                id="CogniteSourceSystem_no_readonly_properties",
+            ),
+        ],
+    )
+    def test_get_readonly_properties(
+        self,
+        toolkit_client_approval: ApprovalToolkitClient,
+        cognite_core_no_3D: dm.DataModel,
+        view_id: dm.ViewId,
+        expected_readonly_props: set[str],
+    ) -> None:
+        """Test that get_readonly_properties identifies readonly properties from containers."""
+        loader = ViewCRUD.create_loader(toolkit_client_approval.mock_client)
+        toolkit_client_approval.append(dm.View, cognite_core_no_3D.views)
+
+        readonly_props = loader.get_readonly_properties(view_id)
+        assert readonly_props == expected_readonly_props

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/tests/test_unit/test_cdf_tk/test_storageio/test_instance_io.py

@@ -5,7 +5,8 @@ import httpx
 import pytest
 import responses
 import respx
-from cognite.client.data_classes.data_modeling import EdgeApply, Node, NodeApply
+from cognite.client.data_classes import View
+from cognite.client.data_classes.data_modeling import DataModel, EdgeApply, Node, NodeApply
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
 from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList, InstanceList
@@ -13,6 +14,7 @@ from cognite_toolkit._cdf_tk.commands import DownloadCommand, UploadCommand
 from cognite_toolkit._cdf_tk.storageio import InstanceIO
 from cognite_toolkit._cdf_tk.storageio.selectors import InstanceSpaceSelector, InstanceViewSelector, SelectedView
 from cognite_toolkit._cdf_tk.utils.http_client import FailedResponseItems, HTTPClient, SuccessResponseItems
+from tests.test_unit.approval_client import ApprovalToolkitClient
 
 
 class TestInstanceIO:
@@ -286,3 +288,81 @@ class TestInstanceIO:
         )
 
         assert len(respx_mock.calls) == 1
+
+    @pytest.mark.parametrize(
+        "item_json,expected_properties",
+        [
+            pytest.param(
+                {
+                    "space": "my_space",
+                    "externalId": "asset_node",
+                    "instanceType": "node",
+                    "sources": [
+                        {
+                            "source": {"type": "container", "space": "cdf_cdm", "externalId": "CogniteAsset"},
+                            "properties": {
+                                "name": "Asset",
+                                "assetHierarchy_path": ["root", "child"],
+                                "assetHierarchy_root": "root",
+                                "description": "Test",
+                            },
+                        }
+                    ],
+                },
+                {"name": "Asset", "description": "Test"},
+                id="CogniteAsset container filters readonly props",
+            ),
+            pytest.param(
+                {
+                    "space": "my_space",
+                    "externalId": "file_node",
+                    "instanceType": "node",
+                    "sources": [
+                        {
+                            "source": {"type": "container", "space": "cdf_cdm", "externalId": "CogniteFile"},
+                            "properties": {"name": "File", "isUploaded": True, "uploadedTime": 123, "mimeType": "pdf"},
+                        }
+                    ],
+                },
+                {"name": "File", "mimeType": "pdf"},
+                id="CogniteFile container filters readonly props",
+            ),
+            pytest.param(
+                {
+                    "space": "my_space",
+                    "externalId": "asset_node",
+                    "instanceType": "node",
+                    "sources": [
+                        {
+                            "source": {
+                                "type": "view",
+                                "space": "cdf_cdm",
+                                "externalId": "CogniteAsset",
+                                "version": "v1",
+                            },
+                            "properties": {
+                                "name": "Asset",
+                                "path": ["root", "child"],
+                                "pathLastUpdatedTime": 123456789,
+                            },
+                        }
+                    ],
+                },
+                {"name": "Asset"},
+                id="CogniteAsset view filters readonly props",
+            ),
+        ],
+    )
+    def test_json_to_resource_filters_readonly_properties(
+        self,
+        toolkit_client_approval: ApprovalToolkitClient,
+        item_json: dict,
+        expected_properties: dict,
+        cognite_core_no_3D: DataModel,
+    ) -> None:
+        """Test that json_to_resource filters out read-only properties from containers."""
+        toolkit_client_approval.append(View, cognite_core_no_3D.views)
+        instance_io = InstanceIO(toolkit_client_approval.mock_client)
+        instance = instance_io.json_to_resource(item_json)
+
+        assert dict(instance.sources[0].properties) == expected_properties

{cognite_toolkit-0.6.99 → cognite_toolkit-0.6.100}/uv.lock

@@ -272,7 +272,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.6.99"
+version = "0.6.100"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.6.99/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.6.99"

cognite_toolkit-0.6.99/last_changelog_entry.md

@@ -1,11 +0,0 @@
-## cdf 
-
-### Changed
-
-- [alpha] The clean command requires the module name to be passed as a
-parameter. If not, the user will be prompted to select modules
-interactively.
-
-## templates
-
-No changes.

cognite_toolkit-0.6.99/last_git_message.txt

@@ -1,31 +0,0 @@
-[CDF-26131] Clean module limit (#2191)
-
-# Description
-
-From v0.7, the clean command will require a `-m`/`--module` parameter.
-If not provided, the user will be required to select one or more modules
-interactively. This is to reduce the damage potential of an unintended
-`cdf clean` command.
-
-
-## Changelog
-
-- [x] Patch
-- [ ] Skip
-
-## cdf
-
-### Changed
-
-- [alpha] The clean command requires the module name to be passed as a
-parameter. If not, the user will be prompted to select modules
-interactively.
-
-## templates
-
-No changes.
-
----------
-
-Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com>
-Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

cognite_toolkit-0.6.99/last_version.txt

@@ -1 +0,0 @@
-0.6.98