cognite-toolkit 0.6.116__py3-none-any.whl → 0.6.118__py3-none-any.whl

cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -8,6 +8,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.charts import Chart, ChartList,
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
 from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
+from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, SimpleBodyRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
 from ._base import Page, UploadableStorageIO, UploadItem
@@ -159,6 +160,28 @@ class CanvasIO(UploadableStorageIO[CanvasSelector, IndustrialCanvas, IndustrialC
             raise ToolkitNotImplementedError(f"Unsupported selector type {type(selector).__name__!r} for CanvasIO")
         return len(selector.external_ids)
 
+    def upload_items(
+        self,
+        data_chunk: Sequence[UploadItem[IndustrialCanvasApply]],
+        http_client: HTTPClient,
+        selector: CanvasSelector | None = None,
+    ) -> Sequence[HTTPMessage]:
+        config = http_client.config
+        results: list[HTTPMessage] = []
+        for item in data_chunk:
+            instances = item.item.as_instances()
+            responses = http_client.request_with_retries(
+                message=SimpleBodyRequest(
+                    endpoint_url=config.create_api_url("/models/instances"),
+                    method="POST",
+                    # MyPy does not understand that .dump is valid json
+                    body_content={"items": [instance.dump() for instance in instances]},
+                )
+            )
+            results.extend(responses.as_item_responses(item.source_id))
+
+        return results
+
     def data_to_json_chunk(
         self, data_chunk: Sequence[IndustrialCanvas], selector: CanvasSelector | None = None
     ) -> list[dict[str, JsonVal]]:

cognite_toolkit/_cdf_tk/storageio/_file_content.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from typing import cast
 
 from cognite.client.data_classes import FileMetadata, FileMetadataWrite
+from cognite.client.data_classes.data_modeling import NodeId, ViewId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.cruds import FileMetadataCRUD
@@ -15,16 +16,20 @@ from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import (
     DataBodyRequest,
     ErrorDetails,
+    FailedResponse,
     FailedResponseItems,
     HTTPClient,
     HTTPMessage,
+    ResponseList,
     SimpleBodyRequest,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
 from ._base import Page, UploadableStorageIO, UploadItem
 from .selectors import FileContentSelector, FileMetadataTemplateSelector
-from .selectors._file_content import FILEPATH
+from .selectors._file_content import FILEPATH, FileDataModelingTemplateSelector
+
+COGNITE_FILE_VIEW = ViewId("cdf_cdm", "CogniteFile", "v1")
 
 
 @dataclass
@@ -94,35 +99,20 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
         http_client: HTTPClient,
         selector: FileContentSelector | None = None,
     ) -> Sequence[HTTPMessage]:
-        if not isinstance(selector, FileMetadataTemplateSelector):
-            raise ToolkitNotImplementedError("Only uploading of file metadata is currently supported.")
-        config = http_client.config
         results: MutableSequence[HTTPMessage] = []
-        for item in cast(Sequence[UploadFileContentItem], data_chunk):
-            responses = http_client.request_with_retries(
-                message=SimpleBodyRequest(
-                    endpoint_url=config.create_api_url(self.UPLOAD_ENDPOINT),
-                    method="POST",
-                    # MyPy does not understand that .dump is valid json
-                    body_content=item.dump(),  # type: ignore[arg-type]
-                )
+        if isinstance(selector, FileMetadataTemplateSelector):
+            upload_url_getter = self._upload_url_asset_centric
+        elif isinstance(selector, FileDataModelingTemplateSelector):
+            upload_url_getter = self._upload_url_data_modeling
+        elif selector is None:
+            raise ToolkitNotImplementedError("Selector must be provided for FileContentIO upload")
+        else:
+            raise ToolkitNotImplementedError(
+                f"Upload for the given selector, {type(selector).__name__}, is not supported for FileContentIO"
             )
-            try:
-                body = responses.get_first_body()
-            except ValueError:
-                results.extend(responses.as_item_responses(item.as_id()))
-                continue
-            try:
-                upload_url = cast(str, body["uploadUrl"])
-            except (KeyError, IndexError):
-                results.append(
-                    FailedResponseItems(
-                        status_code=200,
-                        body=json.dumps(body),
-                        error=ErrorDetails(code=200, message="Malformed response"),
-                        ids=[item.as_id()],
-                    )
-                )
+
+        for item in cast(Sequence[UploadFileContentItem], data_chunk):
+            if not (upload_url := upload_url_getter(item, http_client, results)):
                 continue
 
             upload_response = http_client.request_with_retries(
@@ -136,6 +126,119 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
             results.extend(upload_response.as_item_responses(item.as_id()))
         return results
 
+    def _upload_url_asset_centric(
+        self, item: UploadFileContentItem, http_client: HTTPClient, results: MutableSequence[HTTPMessage]
+    ) -> str | None:
+        responses = http_client.request_with_retries(
+            message=SimpleBodyRequest(
+                endpoint_url=http_client.config.create_api_url(self.UPLOAD_ENDPOINT),
+                method="POST",
+                # MyPy does not understand that .dump is valid json
+                body_content=item.dump(),  # type: ignore[arg-type]
+            )
+        )
+        return self._parse_upload_link_response(responses, item, results)
+
+    def _upload_url_data_modeling(
+        self,
+        item: UploadFileContentItem,
+        http_client: HTTPClient,
+        results: MutableSequence[HTTPMessage],
+        created_node: bool = False,
+    ) -> str | None:
+        """Get upload URL for data modeling file upload.
+
+        We first try to get the upload link assuming the CogniteFile node already exists.
+        If we get a "not found" error, we create the CogniteFile node and try again.
+
+        Args:
+            item: The upload item containing file metadata.
+            http_client: The HTTP client to use for requests.
+            results: A mutable sequence to collect HTTP messages and errors.
+            created_node: A flag indicating whether the CogniteFile node has already been created.
+                This prevents infinite recursion.
+
+        Returns:
+            The upload URL as a string, or None if there was an error.
+
+        """
+        # We know that instance_id is always set for data modeling uploads
+        instance_id = cast(NodeId, item.item.instance_id)
+        responses = http_client.request_with_retries(
+            message=SimpleBodyRequest(
+                endpoint_url=http_client.config.create_api_url("/files/uploadlink"),
+                method="POST",
+                body_content={"items": [{"instanceId": instance_id.dump(include_instance_type=False)}]},  # type: ignore[dict-item]
+            )
+        )
+        # We know there is only one response since we only requested one upload link
+        response = responses[0]
+        if isinstance(response, FailedResponse) and response.error.missing and not created_node:
+            if self._create_cognite_file_node(instance_id, http_client, item.as_id(), results):
+                return self._upload_url_data_modeling(item, http_client, results, created_node=True)
+            else:
+                return None
+
+        return self._parse_upload_link_response(responses, item, results)
+
+    @classmethod
+    def _create_cognite_file_node(
+        cls, instance_id: NodeId, http_client: HTTPClient, upload_id: str, results: MutableSequence[HTTPMessage]
+    ) -> bool:
+        node_creation = http_client.request_with_retries(
+            message=SimpleBodyRequest(
+                endpoint_url=http_client.config.create_api_url("/models/instances"),
+                method="POST",
+                body_content={
+                    "items": [
+                        {
+                            "space": instance_id.space,
+                            "externalId": instance_id.external_id,
+                            "instanceType": "node",
+                            # When we create a node with properties in CogniteFile View even with empty properties,
+                            # CDF will fill in empty values for all properties defined in the view (note this is only
+                            # possible because CogniteFile view has all properties as optional). This includes properties
+                            # in the CogniteFile container, which will trigger the file syncer to create a FileMetadata
+                            # and link it to the CogniteFile node.
+                            "sources": [{"source": COGNITE_FILE_VIEW.dump(include_type=True), "properties": {}}],  # type: ignore[dict-item]
+                        }
+                    ]
+                },
+            )
+        )
+        try:
+            _ = node_creation.get_first_body()
+        except ValueError:
+            results.extend(node_creation.as_item_responses(upload_id))
+            return False
+        return True
+
+    @classmethod
+    def _parse_upload_link_response(
+        cls, responses: ResponseList, item: UploadFileContentItem, results: MutableSequence[HTTPMessage]
+    ) -> str | None:
+        try:
+            body = responses.get_first_body()
+        except ValueError:
+            results.extend(responses.as_item_responses(item.as_id()))
+            return None
+
+        if "items" in body and isinstance(body["items"], list) and len(body["items"]) > 0:
+            body = body["items"][0]  # type: ignore[assignment]
+        try:
+            upload_url = cast(str, body["uploadUrl"])
+        except (KeyError, IndexError):
+            results.append(
+                FailedResponseItems(
+                    status_code=200,
+                    body=json.dumps(body),
+                    error=ErrorDetails(code=200, message="Malformed response"),
+                    ids=[item.as_id()],
+                )
+            )
+            return None
+        return upload_url
+
     @classmethod
     def read_chunks(
         cls, reader: MultiFileReader, selector: FileContentSelector

cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -15,6 +15,7 @@ from ._datapoints import (
 )
 from ._file_content import (
     FileContentSelector,
+    FileDataModelingTemplate,
     FileDataModelingTemplateSelector,
     FileMetadataTemplate,
     FileMetadataTemplateSelector,
@@ -64,6 +65,7 @@ __all__ = [
     "DataSetSelector",
     "ExternalIdColumn",
     "FileContentSelector",
+    "FileDataModelingTemplate",
     "FileDataModelingTemplateSelector",
     "FileMetadataTemplate",
     "FileMetadataTemplateSelector",

cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Literal
 
-from pydantic import ConfigDict, field_validator
+from pydantic import ConfigDict, field_validator, model_validator
 
 from ._base import DataSelector, SelectorObject
 from ._instances import SelectedView
@@ -64,7 +64,7 @@ class FileMetadataTemplateSelector(FileContentSelector):
         return self.template.create_instance(filepath.name)
 
 
-class FileDataModelingTemplate(FileTemplate):
+class TemplateNodeId(SelectorObject):
     space: str
     external_id: str
 
@@ -79,9 +79,23 @@ class FileDataModelingTemplate(FileTemplate):
         return v
 
 
+class FileDataModelingTemplate(FileTemplate):
+    instance_id: TemplateNodeId
+    # Name is required for FileMetadata but not for CogniteFiles. This is the same default behavior as in CDF.
+    name: str = "untitled"
+
+    @model_validator(mode="before")
+    def _move_space_external_id(cls, data: dict[str, Any]) -> dict[str, Any]:
+        if "space" in data and "externalId" in data:
+            data["instanceId"] = {"space": data.pop("space"), "externalId": data.pop("externalId")}
+        elif "space" in data and "external_id" in data:
+            data["instance_id"] = {"space": data.pop("space"), "external_id": data.pop("external_id")}
+        return data
+
+
 class FileDataModelingTemplateSelector(FileContentSelector):
     type: Literal["fileDataModelingTemplate"] = "fileDataModelingTemplate"
-    view_id: SelectedView
+    view_id: SelectedView = SelectedView(space="cdf_cdm", external_id="CogniteFile", version="v1")
     template: FileDataModelingTemplate
 
     @property

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.116
+      image: cognite/toolkit:0.6.118
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.116
+      image: cognite/toolkit:0.6.118
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.116"
+version = "0.6.118"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.116"
+__version__ = "0.6.118"

{cognite_toolkit-0.6.116.dist-info → cognite_toolkit-0.6.118.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.116
+Version: 0.6.118
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.116.dist-info → cognite_toolkit-0.6.118.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=qr31QC3AhJukM-9rBeWHBTLuNU01XIXAtV5dzqOU3iA,5958
-cognite_toolkit/_version.py,sha256=ezn27UkIAZOSq_y3A672Sb3NhZFS1DBg7KeB8M57Z6Y,24
+cognite_toolkit/_version.py,sha256=FxRZpIykG7tzNEEpbz77BWxMHeGPZF7cZeJus_vCRsM,24
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
 cognite_toolkit/_cdf_tk/constants.py,sha256=3UpFZ60xXdqgPqqpqCITQuAvjnVExH_IlbASxoelvu8,7236
@@ -240,21 +240,21 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Q
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=T2PzfD2Tf5khE0cmSDLcqTlRy9YEKTU-raKePIFSMNQ,1910
 cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=JI_g18_Y9S7pbc9gm6dZMyo3Z-bCndJXF9C2lOva0bQ,4848
-cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=TuhO5d465RG_maukaAazXi7P7upiJeSofxXrf9guJQE,15113
+cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=vena8BW7erki6i-hGdcdiOmDbMU3P9M_Kn_5dPG6Yzw,16084
 cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=mRwcVvBzUES21zNb-mDH7XqZWN37p4GUs0JRx7lbRyw,32817
 cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=cllSyg3rRGov_X45CoJ_isXhz0cU9Mt7OyV7_t_SeWw,12156
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=AGTQm9CBRbu1oXbBh0X7UGzFrHnlWZExqNvAohT0hM0,8641
-cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=HKf3Dg3PMH_QBbPrMhpC75H74laDRVx9R0KiEGKXikI,6103
+cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=g8HDDHbBPBcb3g6Zp-vHc8uvNZYrZXWs5G0jq_eOc4w,10884
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
 cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=5WjAFiVR0KKRhMqCy1IRy1TQFWj86D7nGu5WSFNLp6U,3869
-cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=h7fpasWnxINhwe5prJ_ZhxJw-aPQcOvPqlI_1oHmfI8,2163
+cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=ELUCirQmhDR52PVIhLRd_2M1DWYURwL44U6WiN9_hEA,2225
 cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py,sha256=7Iv_ccVX6Vzt3ZLFZ0Er3hN92iEsFTm9wgF-yermOWE,1467
 cognite_toolkit/_cdf_tk/storageio/selectors/_base.py,sha256=hjFkbmNGsK3QIW-jnJV_8YNmvVROERxzG82qIZhU7SM,3065
 cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py,sha256=E9S-wr-JUqRosI_2cSCfR0tF8MdIFTrMxDItuWRcuO4,597
 cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py,sha256=lQHuNtF3i6SEIMPAlziMm0QlqRcvZJ7MKIug6HMTDrs,1012
 cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py,sha256=EHVkWJYJ_HCs2i4Ur6Fj98UwuDvf67u0HOzrJXAHNt0,1719
-cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py,sha256=bDcU0Unbb_qG6z004aKTKgeAVhRnFQ4mz-rjNuouXu4,2881
+cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py,sha256=J_lP04QsvqCOIiw_u2rGMqIa_7obIxCaFkC70ukUjpI,3628
 cognite_toolkit/_cdf_tk/storageio/selectors/_instances.py,sha256=NCFSJrAw52bNX6UTfOali8PvNjlqHnvxzL0hYBr7ZmA,4934
 cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py,sha256=sZq9C4G9DMe3S46_usKet0FphQ6ow7cWM_PfXrEAakk,503
 cognite_toolkit/_cdf_tk/tk_warnings/__init__.py,sha256=U9bT-G2xKrX6mmtZ7nZ1FfQeCjNKfKP_p7pev90dwOE,2316
@@ -302,13 +302,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=a_46iFbHKFVtTSG94JW2WVJAMz37jDv-6-TpY6CNy3o,668
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=cVSBKRvJRYyuUIk9jJxRHlq2eqgqYS_O8V3JTXJkKFw,2431
-cognite_toolkit/_resources/cdf.toml,sha256=H8z6SinwW4X2em4YbzS4Sqmu9RIF4DulHm6kfrC5RLk,488
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=PGM98-0CQ8bWKL8L5NSU9sv_5zfyyJf0XDStcM1l2d4,668
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=GBFqqyqAWXty1N_ympFvlHCHYbN7hdNDjyhGS-rQm54,2431
+cognite_toolkit/_resources/cdf.toml,sha256=Yiqq2erdwbz67Z0VH02p-Q9iQ-mglcF6_kU0rethUms,488
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.116.dist-info/METADATA,sha256=5-qZwwRh2_UO6KZ09QsM5etUMGICQzMiT4La7MS_1ag,4502
-cognite_toolkit-0.6.116.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.116.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.116.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.116.dist-info/RECORD,,
+cognite_toolkit-0.6.118.dist-info/METADATA,sha256=dA8GSJXTE0SIcTlrNbxDJLJ39N79pWa6P6K9PMKYnIo,4502
+cognite_toolkit-0.6.118.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.118.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.118.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.118.dist-info/RECORD,,