cognite-toolkit 0.7.21__py3-none-any.whl → 0.7.23__py3-none-any.whl

cognite_toolkit/_cdf_tk/resource_classes/migration.py

@@ -1,19 +1,20 @@
-import re
 from typing import Literal
 
-from pydantic import Field, field_validator
+from pydantic import Field
 
-from cognite_toolkit._cdf_tk.utils import humanize_collection
+from cognite_toolkit._cdf_tk.constants import INSTANCE_EXTERNAL_ID_PATTERN
 
 from .base import ToolkitResource
 from .view_field_definitions import ViewReference
 
-JSON_PATH_PATTERN = re.compile(
-    r"^([a-zA-Z_][a-zA-Z0-9_]*|\[\d+\])(\.[a-zA-Z_][a-zA-Z0-9_]*|\[\d+\]|\[\'[^\']*\'\]|\[\"[^\"]*\"\])*$"
-)
-
 
 class ResourceViewMappingYAML(ToolkitResource):
+    external_id: str = Field(
+        description="External-id of the mapping.",
+        min_length=1,
+        max_length=256,
+        pattern=INSTANCE_EXTERNAL_ID_PATTERN,
+    )
     resource_type: Literal["asset", "event", "file", "timeSeries", "sequence", "assetAnnotation", "fileAnnotation"] = (
         Field(
             description="The type of the resource to map to the view.",
@@ -25,13 +26,3 @@ class ResourceViewMappingYAML(ToolkitResource):
     property_mapping: dict[str, str] = Field(
         description="A dictionary mapping from resource property to view property.",
     )
-
-    @field_validator("property_mapping")
-    @classmethod
-    def validate_json_paths(cls, value: dict[str, str]) -> dict[str, str]:
-        if not isinstance(value, dict):
-            return value
-        not_matching_keys = [k for k in value.keys() if not re.match(JSON_PATH_PATTERN, k)]
-        if not_matching_keys:
-            raise ValueError(f"Invalid JSON paths: {humanize_collection(not_matching_keys)}")
-        return value

cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable, Sequence
+from typing import Any
 
 from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
     IndustrialCanvas,
@@ -6,7 +7,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
 )
 from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList, ChartWrite
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
-from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, SimpleBodyRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
@@ -218,17 +219,22 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
         references = dumped.get("containerReferences", [])
         if not isinstance(references, list):
             return dumped
+        new_container_references: list[Any] = []
         for container_ref in references:
             if not isinstance(container_ref, dict):
+                new_container_references.append(container_ref)
                 continue
             sources = container_ref.get("sources", [])
             if not isinstance(sources, list) or len(sources) == 0:
+                new_container_references.append(container_ref)
                 continue
             source = sources[0]
             if not isinstance(source, dict) or "properties" not in source:
+                new_container_references.append(container_ref)
                 continue
             properties = source["properties"]
             if not isinstance(properties, dict):
+                new_container_references.append(container_ref)
                 continue
             reference_type = properties.get("containerReferenceType")
             if (
@@ -238,9 +244,13 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
                     "dataGrid",
                 }
             ):  # These container reference types are special cases with a resourceId statically set to -1, which is why we skip them
+                new_container_references.append(container_ref)
                 continue
             resource_id = properties.pop("resourceId", None)
             if not isinstance(resource_id, int):
+                HighSeverityWarning(
+                    f"Invalid resourceId {resource_id!r} in Canvas {canvas.canvas.name}. Skipping."
+                ).print_warning(console=self.client.console)
                 continue
             if reference_type == "asset":
                 external_id = self.client.lookup.assets.external_id(resource_id)
@@ -251,9 +261,16 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             elif reference_type == "file":
                 external_id = self.client.lookup.files.external_id(resource_id)
             else:
+                new_container_references.append(container_ref)
                 continue
-            if external_id is not None:
-                properties["resourceExternalId"] = external_id
+            if external_id is None:
+                HighSeverityWarning(
+                    f"Failed to look-up {reference_type} external ID for resource ID {resource_id!r}. Skipping resource in Canvas {canvas.canvas.name}"
+                ).print_warning(console=self.client.console)
+                continue
+            properties["resourceExternalId"] = external_id
+            new_container_references.append(container_ref)
+        dumped["containerReferences"] = new_container_references
         return dumped
 
     def json_chunk_to_data(
@@ -312,23 +329,30 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
         return self._load_resource(item_json)
 
     def _load_resource(self, item_json: dict[str, JsonVal]) -> IndustrialCanvasApply:
+        name = self._get_name(item_json)
         references = item_json.get("containerReferences", [])
         if not isinstance(references, list):
             return IndustrialCanvasApply._load(item_json)
+        new_container_references: list[Any] = []
         for container_ref in references:
             if not isinstance(container_ref, dict):
+                new_container_references.append(container_ref)
                 continue
             sources = container_ref.get("sources", [])
             if not isinstance(sources, list) or len(sources) == 0:
+                new_container_references.append(container_ref)
                 continue
             source = sources[0]
             if not isinstance(source, dict) or "properties" not in source:
+                new_container_references.append(container_ref)
                 continue
             properties = source["properties"]
             if not isinstance(properties, dict):
+                new_container_references.append(container_ref)
                 continue
             resource_external_id = properties.pop("resourceExternalId", None)
             if not isinstance(resource_external_id, str):
+                new_container_references.append(container_ref)
                 continue
             reference_type = properties.get("containerReferenceType")
             if reference_type == "asset":
@@ -340,8 +364,24 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             elif reference_type == "file":
                 resource_id = self.client.lookup.files.id(resource_external_id)
             else:
+                new_container_references.append(container_ref)
                 continue
-            if resource_id is not None:
-                properties["resourceId"] = resource_id
-
-        return IndustrialCanvasApply._load(item_json)
+            if resource_id is None:
+                # Failed look-up, skip the resourceId setting
+                HighSeverityWarning(
+                    f"Failed to look-up {reference_type} ID for external ID {resource_external_id!r}. Skipping resource in Canvas {name}"
+                ).print_warning(console=self.client.console)
+                continue
+            properties["resourceId"] = resource_id
+            new_container_references.append(container_ref)
+        new_item = dict(item_json)
+        new_item["containerReferences"] = new_container_references
+
+        return IndustrialCanvasApply._load(new_item)
+
+    @classmethod
+    def _get_name(cls, item_json: dict[str, JsonVal]) -> str:
+        try:
+            return item_json["canvas"]["sources"][0]["properties"]["name"]  # type: ignore[index,return-value, call-overload]
+        except (KeyError, IndexError, TypeError):
+            return "<unknown>"

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.21
+      image: cognite/toolkit:0.7.23
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.21
+      image: cognite/toolkit:0.7.23
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.21"
+version = "0.7.23"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.21"
+__version__ = "0.7.23"

{cognite_toolkit-0.7.21.dist-info → cognite_toolkit-0.7.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.21
+Version: 0.7.23
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.7.21.dist-info → cognite_toolkit-0.7.23.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=sefGD2JQuOTBZhEqSj_ECbNZ7nTRN4AwGwX1pSUhoow,5636
-cognite_toolkit/_version.py,sha256=dsLz3w-bEQrCd7hoih-4i3AFeJ0J6O2MZqka_Yl9j0A,23
+cognite_toolkit/_version.py,sha256=JGnAqfJascCxCeW2HUmSEq-6cTv1v4JA85VtarqlsUk,23
 cognite_toolkit/config.dev.yaml,sha256=M33FiIKdS3XKif-9vXniQ444GTZ-bLXV8aFH86u9iUQ,332
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
@@ -216,7 +216,7 @@ cognite_toolkit/_cdf_tk/resource_classes/infield_v1.py,sha256=XtstgUuhYhWgsxSoQ9
 cognite_toolkit/_cdf_tk/resource_classes/instance.py,sha256=FdPCDz4LlKZ2SDBGH2-z-JiPN0V53Gg3FrxxjcjrJZc,2935
 cognite_toolkit/_cdf_tk/resource_classes/labels.py,sha256=rh4SV_MBEiS9cjpz1g4_0SAnKGwiIepThD_wR1_bMkY,649
 cognite_toolkit/_cdf_tk/resource_classes/location.py,sha256=C4vhHs_kpLa3abfQIZiotfldnKoEq8YAbU3gvG7XDMA,3221
-cognite_toolkit/_cdf_tk/resource_classes/migration.py,sha256=Y-NOY3NdfjwWZEJ-gfDxPZ71l3qe2Gbtd7z1vtwESjs,1316
+cognite_toolkit/_cdf_tk/resource_classes/migration.py,sha256=ThMKmJSaMx9DKtqkwcY2Y_Y4hY0xk-UzyF7HC49qlnA,903
 cognite_toolkit/_cdf_tk/resource_classes/raw_database_table.py,sha256=2It1o8-zsH1YMCXTnI_KMjjnuN4bnZ_2PS3ZyBUrh0I,538
 cognite_toolkit/_cdf_tk/resource_classes/relationship.py,sha256=tZQvy_ofmxSry0e9BITKOcB-L8fNocQBl56BJjAgThc,2074
 cognite_toolkit/_cdf_tk/resource_classes/search_config.py,sha256=KhHLKs6-tljU7tYaCc2u3qyHXbNDqCXI3OmntSsKW8E,837
@@ -244,7 +244,7 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Q
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=h5Wr4i7zNIgsslrsRJxmp7ls4bNRKl0uZzQ7GLRMP7g,1920
 cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=JI_g18_Y9S7pbc9gm6dZMyo3Z-bCndJXF9C2lOva0bQ,4848
-cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=VlTRHqp8jVu16SW7LtrN05BNYZHSPtJ_wf9EsBAAsvE,16419
+cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=ozWeuTqay1_GSFuQZUxXJspsBhLEBIS7EksAi93B8_4,18722
 cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=TirKLSNPoLqKjczsw0djWAsR0VvopwmU23aUxrBOJN8,32464
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=ElvqhIEBnhcz0yY1Ds164wVN0_7CFNK-uT0-z7LcR9U,13067
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
@@ -305,13 +305,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=7RefWun5puw2176WrJSLEVlc9W2-3RR4hZoUZbCamn8,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=KcvP5c2MmkhPJj4kDXu8xRIH98Iyj-VOTzb7gdSMoJ8,2430
-cognite_toolkit/_resources/cdf.toml,sha256=bWCeSoZrULc_3jDfIFKhHbtEmAu5Yt7tuSrGS2LeEkg,475
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=9OAOoiEnbwN-vEuPQmI59u7_5bHwKKgJVd2b3wmXZZU,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=vVpoAsIznpwy42v_665Hg5E5N6pbI0i0Q8vQDjHCBeg,2430
+cognite_toolkit/_resources/cdf.toml,sha256=rDE7YGIeaEnxBZX8qUEFV8gr-Gh8wq8YArsxlDEPjIQ,475
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.7.21.dist-info/METADATA,sha256=_5E4GZxcoVkuzuabVj4_txqit-4Sl4YD62ORO8PkI8U,4501
-cognite_toolkit-0.7.21.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cognite_toolkit-0.7.21.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.7.21.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.7.21.dist-info/RECORD,,
+cognite_toolkit-0.7.23.dist-info/METADATA,sha256=nbev1eMp2ATVbkZUvRaDAlUsUOj5-KKDSdANwZ0gsK4,4501
+cognite_toolkit-0.7.23.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cognite_toolkit-0.7.23.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.7.23.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.7.23.dist-info/RECORD,,