cognite-toolkit 0.7.21__tar.gz → 0.7.22__tar.gz

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.21
+Version: 0.7.22
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/cdf.toml

@@ -23,7 +23,7 @@ data = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable, Sequence
+from typing import Any
 
 from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
     IndustrialCanvas,
@@ -6,7 +7,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
 )
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList, ChartWrite
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
-from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, SimpleBodyRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
@@ -218,17 +219,22 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
         references = dumped.get("containerReferences", [])
         if not isinstance(references, list):
             return dumped
+        new_container_references: list[Any] = []
         for container_ref in references:
             if not isinstance(container_ref, dict):
+                new_container_references.append(container_ref)
                 continue
             sources = container_ref.get("sources", [])
             if not isinstance(sources, list) or len(sources) == 0:
+                new_container_references.append(container_ref)
                 continue
             source = sources[0]
             if not isinstance(source, dict) or "properties" not in source:
+                new_container_references.append(container_ref)
                 continue
             properties = source["properties"]
             if not isinstance(properties, dict):
+                new_container_references.append(container_ref)
                 continue
             reference_type = properties.get("containerReferenceType")
             if (
@@ -238,9 +244,13 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
                     "dataGrid",
                 }
             ):  # These container reference types are special cases with a resourceId statically set to -1, which is why we skip them
+                new_container_references.append(container_ref)
                 continue
             resource_id = properties.pop("resourceId", None)
             if not isinstance(resource_id, int):
+                HighSeverityWarning(
+                    f"Invalid resourceId {resource_id!r} in Canvas {canvas.canvas.name}. Skipping."
+                ).print_warning(console=self.client.console)
                 continue
             if reference_type == "asset":
                 external_id = self.client.lookup.assets.external_id(resource_id)
@@ -251,9 +261,16 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             elif reference_type == "file":
                 external_id = self.client.lookup.files.external_id(resource_id)
             else:
+                new_container_references.append(container_ref)
                 continue
-            if external_id is not None:
-                properties["resourceExternalId"] = external_id
+            if external_id is None:
+                HighSeverityWarning(
+                    f"Failed to look-up {reference_type} external ID for resource ID {resource_id!r}. Skipping resource in Canvas {canvas.canvas.name}"
+                ).print_warning(console=self.client.console)
+                continue
+            properties["resourceExternalId"] = external_id
+            new_container_references.append(container_ref)
+        dumped["containerReferences"] = new_container_references
         return dumped
 
     def json_chunk_to_data(
@@ -312,23 +329,30 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
         return self._load_resource(item_json)
 
     def _load_resource(self, item_json: dict[str, JsonVal]) -> IndustrialCanvasApply:
+        name = self._get_name(item_json)
         references = item_json.get("containerReferences", [])
         if not isinstance(references, list):
             return IndustrialCanvasApply._load(item_json)
+        new_container_references: list[Any] = []
         for container_ref in references:
             if not isinstance(container_ref, dict):
+                new_container_references.append(container_ref)
                 continue
             sources = container_ref.get("sources", [])
             if not isinstance(sources, list) or len(sources) == 0:
+                new_container_references.append(container_ref)
                 continue
             source = sources[0]
             if not isinstance(source, dict) or "properties" not in source:
+                new_container_references.append(container_ref)
                 continue
             properties = source["properties"]
             if not isinstance(properties, dict):
+                new_container_references.append(container_ref)
                 continue
             resource_external_id = properties.pop("resourceExternalId", None)
             if not isinstance(resource_external_id, str):
+                new_container_references.append(container_ref)
                 continue
             reference_type = properties.get("containerReferenceType")
             if reference_type == "asset":
@@ -340,8 +364,24 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             elif reference_type == "file":
                 resource_id = self.client.lookup.files.id(resource_external_id)
             else:
+                new_container_references.append(container_ref)
                 continue
-            if resource_id is not None:
-                properties["resourceId"] = resource_id
-
-        return IndustrialCanvasApply._load(item_json)
+            if resource_id is None:
+                # Failed look-up, skip the resourceId setting
+                HighSeverityWarning(
+                    f"Failed to look-up {reference_type} ID for external ID {resource_external_id!r}. Skipping resource in Canvas {name}"
+                ).print_warning(console=self.client.console)
+                continue
+            properties["resourceId"] = resource_id
+            new_container_references.append(container_ref)
+        new_item = dict(item_json)
+        new_item["containerReferences"] = new_container_references
+
+        return IndustrialCanvasApply._load(new_item)
+
+    @classmethod
+    def _get_name(cls, item_json: dict[str, JsonVal]) -> str:
+        try:
+            return item_json["canvas"]["sources"][0]["properties"]["name"]  # type: ignore[index,return-value, call-overload]
+        except (KeyError, IndexError, TypeError):
+            return "<unknown>"

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.21
+      image: cognite/toolkit:0.7.22
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.21
+      image: cognite/toolkit:0.7.22
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"
 
 
 [plugins]

cognite_toolkit-0.7.22/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.7.22"

cognite_toolkit-0.7.22/last_changelog_entry.md

@@ -0,0 +1,8 @@
+## Changelog 
+
+### Fixed
+
+- Using `cdf data upload dir` no longer fails on Canvas that have
+missing resources.
+- Using `cdf data download canvas` now skips references that does not
+exists in CDF.

cognite_toolkit-0.7.22/last_git_message.txt

@@ -0,0 +1,19 @@
+🐛Canvas fail on missing resource (#2284)
+
+# Description
+
+Handle missing resources. 
+
+## Bump
+
+- [x] Patch
+- [ ] Skip
+
+## Changelog
+
+### Fixed
+
+- Using `cdf data upload dir` no longer fails on Canvas that have
+missing resources.
+- Using `cdf data download canvas` now skips references that does not
+exists in CDF.

cognite_toolkit-0.7.22/last_version.txt

@@ -0,0 +1 @@
+0.7.21

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.7.21"
+version = "0.7.22"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/tests/data/build_group_with_unknown_acl/_build_environment.yaml

@@ -4,4 +4,4 @@ project: pytest-project
 type: dev
 selected:
 - modules
-cdf_toolkit_version: 0.7.21
+cdf_toolkit_version: 0.7.22

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/tests/data/cdf_toml_data/cdf.toml

@@ -1,7 +1,7 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"
 
 [plugins]
 graphql = true

{cognite_toolkit-0.7.21/tests/data/complete_org_alpha_flags → cognite_toolkit-0.7.22/tests/data/complete_org}/cdf.toml

@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"

{cognite_toolkit-0.7.21/tests/data/complete_org_only_identifier → cognite_toolkit-0.7.22/tests/data/complete_org_alpha_flags}/cdf.toml

@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"

{cognite_toolkit-0.7.21/tests/data/naughty_project → cognite_toolkit-0.7.22/tests/data/complete_org_only_identifier}/cdf.toml

@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"

{cognite_toolkit-0.7.21/tests/data/complete_org → cognite_toolkit-0.7.22/tests/data/naughty_project}/cdf.toml

@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"

cognite_toolkit-0.7.22/tests/data/project_no_cognite_modules/cdf.toml

@@ -0,0 +1,4 @@
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.7.22"

cognite_toolkit-0.7.22/tests/data/project_with_bad_modules/cdf.toml

@@ -0,0 +1,4 @@
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.7.22"

cognite_toolkit-0.7.22/tests/data/project_with_duplicates/cdf.toml

@@ -0,0 +1,4 @@
+[modules]
+# This is the version of the modules. It should not be changed manually.
+# It will be updated by the 'cdf modules upgrade' command.
+version = "0.7.22"

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/tests/data/run_data/cdf.toml

@@ -1,7 +1,7 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.22"
 
 [modules.packages]
 cdf_infield = [

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/tests/test_unit/test_cdf_tk/test_storageio/test_applications.py

@@ -1,11 +1,16 @@
 import json
+from datetime import datetime
 
 import pytest
 import responses
-from cognite.client.data_classes.data_modeling import NodeList
+from cognite.client.data_classes.data_modeling import NodeList, NodeListWithCursor
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
-from cognite_toolkit._cdf_tk.client.data_classes.canvas import IndustrialCanvas
+from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
+    Canvas,
+    ContainerReference,
+    IndustrialCanvas,
+)
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList
 from cognite_toolkit._cdf_tk.client.data_classes.charts_data import ChartData
 from cognite_toolkit._cdf_tk.client.data_classes.migration import InstanceSource
@@ -198,3 +203,93 @@ class TestCanvasIO:
             restored_canvas = restored_canvases[0]
             # The restored canvas should match the original without existingVersion fields
             assert restored_canvas.item.dump() == canvas.as_write().dump(keep_existing_version=False)
+
+    def test_load_canvas_missing_resource(self) -> None:
+        with monkeypatch_toolkit_client() as client:
+            client.lookup.assets.id.return_value = None
+            canvas_json = {
+                "canvas": {
+                    "externalId": "test_canvas",
+                    "instanceType": "node",
+                    "sources": [
+                        {
+                            "properties": {
+                                "createdBy": "doctrino",
+                                "name": "Test Canvas",
+                                "updatedAt": "2025-12-04T11:40:17.676",
+                                "updatedBy": "doctrino",
+                            },
+                            "source": {
+                                "externalId": "Canvas",
+                                "space": "cdf_industrial_canvas",
+                                "type": "view",
+                                "version": "v7",
+                            },
+                        }
+                    ],
+                    "space": "IndustrialCanvasInstanceSpace",
+                },
+                "containerReferences": [
+                    {
+                        "externalId": "asset_ref",
+                        "instanceType": "node",
+                        "sources": [
+                            {
+                                "properties": {
+                                    "containerReferenceType": "asset",
+                                    "resourceExternalId": "myAssetId",
+                                },
+                                "source": {
+                                    "externalId": "ContainerReference",
+                                    "space": "cdf_industrial_canvas",
+                                    "type": "view",
+                                    "version": "v2",
+                                },
+                            }
+                        ],
+                        "space": "IndustrialCanvasInstanceSpace",
+                    }
+                ],
+            }
+
+            io = CanvasIO(client)
+            loaded = io._load_resource(canvas_json)
+            assert len(loaded.container_references) == 0, "Container reference with missing resource should be skipped."
+
+    def test_dump_canvas_missing_resource(self) -> None:
+        with monkeypatch_toolkit_client() as client:
+            client.lookup.assets.external_id.return_value = None
+
+            canvas = IndustrialCanvas(
+                canvas=Canvas(
+                    "my_space",
+                    "test_canvas",
+                    version=1,
+                    last_updated_time=1,
+                    created_by=1,
+                    name="Test Canvas",
+                    updated_by="doctrino",
+                    updated_at=datetime.now(),
+                    created_time=123,
+                ),
+                container_references=NodeListWithCursor[ContainerReference](
+                    [
+                        ContainerReference(
+                            external_id="asset_ref",
+                            container_reference_type="asset",
+                            resource_id=12345,
+                            space="my_space",
+                            version=1,
+                            last_updated_time=1,
+                            created_time=1,
+                        ),
+                    ],
+                    cursor=None,
+                ),
+            )
+
+            io = CanvasIO(client)
+            dumped = io._dump_resource(canvas)
+            assert len(dumped["containerReferences"]) == 0, (
+                "Container reference with missing resource should be skipped."
+            )

{cognite_toolkit-0.7.21 → cognite_toolkit-0.7.22}/uv.lock

@@ -271,7 +271,7 @@ wheels = [
 
 [[package]]
 name = "cognite-toolkit"
-version = "0.7.21"
+version = "0.7.22"
 source = { editable = "." }
 dependencies = [
     { name = "cognite-sdk" },

cognite_toolkit-0.7.21/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.7.21"

cognite_toolkit-0.7.21/last_changelog_entry.md

@@ -1,7 +0,0 @@
-## Changelog 
-
-### Added
-
-- [alpha] When you download files with content, `cdf data download files
---include-file-contents ...`, you can now upload them with `cdf data
-upload dir`.

cognite_toolkit-0.7.21/last_git_message.txt

@@ -1,35 +0,0 @@
-[CDF-26479]  Uploading file content (#2278)
-
-# Description
-
-To support upload when specifying individual files I have done the
-following changes.
-
-Downloading files now includes `$FILEPATH` 
-<img width="823" height="306" alt="image"
-src="https://github.com/user-attachments/assets/997f26b5-c68e-4914-855f-357b2e5c6854"
-/>
-I have done this by introducing the extra class `MetadataWithFilePath`.
-[ref](https://github.com/cognitedata/toolkit/pull/2278/files#diff-ce79b5b2ac0aaf38f25fa828a794531924903d6eb0d9abb461be8cd561155272R54)
-
-The upload logic no longer assumes that you always have a template
-manifest,
-[here](https://github.com/cognitedata/toolkit/pull/2278/files#diff-ce79b5b2ac0aaf38f25fa828a794531924903d6eb0d9abb461be8cd561155272R411)
-and
-[here](https://github.com/cognitedata/toolkit/pull/2278/files#diff-ce79b5b2ac0aaf38f25fa828a794531924903d6eb0d9abb461be8cd561155272R259)
-
-
-Note this is also tested manually on a real customer.
-
-## Bump
-
-- [x] Patch
-- [ ] Skip
-
-## Changelog
-
-### Added
-
-- [alpha] When you download files with content, `cdf data download files
---include-file-contents ...`, you can now upload them with `cdf data
-upload dir`.

cognite_toolkit-0.7.21/last_version.txt

@@ -1 +0,0 @@
-0.7.20

cognite_toolkit-0.7.21/tests/data/project_no_cognite_modules/cdf.toml

@@ -1,4 +0,0 @@
-[modules]
-# This is the version of the modules. It should not be changed manually.
-# It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"

cognite_toolkit-0.7.21/tests/data/project_with_bad_modules/cdf.toml

@@ -1,4 +0,0 @@
-[modules]
-# This is the version of the modules. It should not be changed manually.
-# It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"

cognite_toolkit-0.7.21/tests/data/project_with_duplicates/cdf.toml

@@ -1,4 +0,0 @@
-[modules]
-# This is the version of the modules. It should not be changed manually.
-# It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"