cognite-toolkit 0.6.103__py3-none-any.whl → 0.6.104__py3-none-any.whl

cognite_toolkit/_cdf_tk/client/testing.py

@@ -35,7 +35,14 @@ from .api.lookup import (
     SecurityCategoriesLookUpAPI,
     TimeSeriesLookUpAPI,
 )
-from .api.migration import CreatedSourceSystemAPI, InstanceSourceAPI, MigrationAPI, ResourceViewMappingAPI
+from .api.migration import (
+    CreatedSourceSystemAPI,
+    InstanceSourceAPI,
+    LookupAPI,
+    MigrationAPI,
+    MigrationLookupAPI,
+    ResourceViewMappingAPI,
+)
 from .api.project import ProjectAPI
 from .api.robotics import RoboticsAPI
 from .api.robotics.capabilities import CapabilitiesAPI
@@ -96,6 +103,11 @@ class ToolkitClientMock(CogniteClientMock):
         self.lookup.functions = MagicMock(spec_set=FunctionLookUpAPI)
         self.migration = MagicMock(spec=MigrationAPI)
         self.migration.instance_source = MagicMock(spec_set=InstanceSourceAPI)
+        self.migration.lookup = MagicMock(spec=MigrationLookupAPI)
+        self.migration.lookup.assets = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.events = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.files = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.time_series = MagicMock(spec_set=LookupAPI)
         self.migration.resource_view_mapping = MagicMock(spec_set=ResourceViewMappingAPI)
         self.migration.created_source_system = MagicMock(spec_set=CreatedSourceSystemAPI)
         self.raw = MagicMock(spec=ExtendedRawAPI)

cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -157,10 +157,11 @@ class MigrationCommand(ToolkitCommand):
         log_file: NDJsonWriter,
     ) -> Callable[[Sequence[T_CogniteResource]], Sequence[UploadItem[T_WriteCogniteResource]]]:
         def track_mapping(source: Sequence[T_CogniteResource]) -> list[UploadItem[T_WriteCogniteResource]]:
+            mapped = mapper.map(source)
             issues: list[Chunk] = []
             targets: list[UploadItem[T_WriteCogniteResource]] = []
-            for item in source:
-                target, issue = mapper.map(item)
+
+            for (target, issue), item in zip(mapped, source):
                 id_ = data.as_id(item)
                 result: Status = "failed" if target is None else "success"
                 tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)

cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -1,6 +1,5 @@
-from collections.abc import Mapping, Set
-from dataclasses import dataclass
-from typing import Any, ClassVar
+from collections.abc import Iterable, Mapping, Set
+from typing import Any, ClassVar, cast
 
 from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
 from cognite.client.data_classes.data_modeling import (
@@ -15,6 +14,7 @@ from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeO
 from cognite.client.data_classes.data_modeling.views import ViewProperty
 from cognite.client.utils._identifier import InstanceId
 
+from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
 from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
 from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
@@ -30,61 +30,134 @@ from .data_model import INSTANCE_SOURCE_VIEW_ID
 from .issues import ConversionIssue, FailedConversion, InvalidPropertyDataType
 
 
-@dataclass
 class DirectRelationCache:
     """Cache for direct relation references to look up target of direct relations.
 
-    This is used when creating direct relations from asset-centric resources to CogniteAsset and CogniteSourceSystem.
-
-    Attributes:
-        asset: Mapping[int, DirectRelationReference]
-            A mapping from asset IDs to their corresponding DirectRelationReference in the data model.
-        source: Mapping[str, DirectRelationReference]
-            A mapping from source strings to their corresponding DirectRelationReference in the data model.
-
-    Methods:
-        get(resource_type: AssetCentric, property_id: str) -> Mapping[str | int, DirectRelationReference]:
-            Get the appropriate mapping based on the resource type and property ID.
-
-    Note:
-        The ASSET_REFERENCE_PROPERTIES and SOURCE_REFERENCE_PROPERTIES class variables define which properties
-        in asset-centric resources reference assets and sources, respectively.
-
+    This is used when creating direct relations from asset-centric resources to assets, files, and source systems.
     """
 
-    ASSET_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    class TableName:
+        ASSET_ID = "assetId"
+        SOURCE_NAME = "source"
+        FILE_ID = "fileId"
+        ASSET_EXTERNAL_ID = "assetExternalId"
+        FILE_EXTERNAL_ID = "fileExternalId"
+
+    ASSET_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("timeseries", "assetId"),
         ("file", "assetIds"),
         ("event", "assetIds"),
         ("sequence", "assetId"),
-        ("asset", "parentId"),
         ("annotation", "data.assetRef.id"),
+        ("asset", "parentId"),
     }
-    SOURCE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    SOURCE_NAME_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("asset", "source"),
         ("event", "source"),
         ("file", "source"),
     }
-    FILE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    FILE_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("annotation", "data.fileRef.id"),
         ("annotation", "annotatedResourceId"),
     }
+    ASSET_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.assetRef.externalId")}
+    FILE_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.fileRef.externalId")}
+
+    def __init__(self, client: ToolkitClient) -> None:
+        self._client = client
+        self._cache_map: dict[
+            tuple[str, str] | str, dict[str, DirectRelationReference] | dict[int, DirectRelationReference]
+        ] = {}
+        # Constructing the cache map to be accessed by both table name and property id
+        for table_name, properties in [
+            (self.TableName.ASSET_ID, self.ASSET_ID_PROPERTIES),
+            (self.TableName.SOURCE_NAME, self.SOURCE_NAME_PROPERTIES),
+            (self.TableName.FILE_ID, self.FILE_ID_PROPERTIES),
+            (self.TableName.ASSET_EXTERNAL_ID, self.ASSET_EXTERNAL_ID_PROPERTIES),
+            (self.TableName.FILE_EXTERNAL_ID, self.FILE_EXTERNAL_ID_PROPERTIES),
+        ]:
+            cache: dict[str, DirectRelationReference] | dict[int, DirectRelationReference] = {}
+            self._cache_map[table_name] = cache
+            for key in properties:
+                self._cache_map[key] = cache
+
+    def update(self, resources: Iterable[AssetCentricResourceExtended]) -> None:
+        """Update the cache with direct relation references for the given asset-centric resources.
+
+        This is used to bulk update the cache for a chunk of resources before converting them to data model instances.
+        """
+        asset_ids: set[int] = set()
+        source_ids: set[str] = set()
+        file_ids: set[int] = set()
+        asset_external_ids: set[str] = set()
+        file_external_ids: set[str] = set()
+        for resource in resources:
+            if isinstance(resource, Annotation):
+                if resource.annotated_resource_type == "file" and resource.annotated_resource_id:
+                    file_ids.add(resource.annotated_resource_id)
+                if "assetRef" in resource.data:
+                    asset_ref = resource.data["assetRef"]
+                    if isinstance(asset_id := asset_ref.get("id"), int):
+                        asset_ids.add(asset_id)
+                    if isinstance(asset_external_id := asset_ref.get("externalId"), str):
+                        asset_external_ids.add(asset_external_id)
+                if "fileRef" in resource.data:
+                    file_ref = resource.data["fileRef"]
+                    if isinstance(file_id := file_ref.get("id"), int):
+                        file_ids.add(file_id)
+                    if isinstance(file_external_id := file_ref.get("externalId"), str):
+                        file_external_ids.add(file_external_id)
+            elif isinstance(resource, Asset):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.parent_id is not None:
+                    asset_ids.add(resource.parent_id)
+            elif isinstance(resource, FileMetadata):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, Event):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, TimeSeries):
+                if resource.asset_id is not None:
+                    asset_ids.add(resource.asset_id)
+        if asset_ids:
+            self._update_cache(self._client.migration.lookup.assets(id=list(asset_ids)), self.TableName.ASSET_ID)
+        if source_ids:
+            # SourceSystems are not cached in the client, so we have to handle the caching ourselves.
+            cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
+            missing = set(source_ids) - set(cache.keys())
+            if missing:
+                source_systems = self._client.migration.created_source_system.retrieve(list(missing))
+                for source_system in source_systems:
+                    cache[source_system.source] = source_system.as_direct_relation_reference()
+        if file_ids:
+            self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
+        if asset_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.assets(external_id=list(asset_external_ids)),
+                self.TableName.ASSET_EXTERNAL_ID,
+            )
+        if file_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.files(external_id=list(file_external_ids)),
+                self.TableName.FILE_EXTERNAL_ID,
+            )
 
-    asset: Mapping[int, DirectRelationReference]
-    source: Mapping[str, DirectRelationReference]
-    file: Mapping[int, DirectRelationReference]
+    def _update_cache(self, instance_id_by_id: dict[int, NodeId] | dict[str, NodeId], table_name: str) -> None:
+        cache = self._cache_map[table_name]
+        for identifier, instance_id in instance_id_by_id.items():
+            cache[identifier] = DirectRelationReference(space=instance_id.space, external_id=instance_id.external_id)  # type: ignore[index]
 
-    def get(
+    def get_cache(
         self, resource_type: AssetCentricTypeExtended, property_id: str
-    ) -> Mapping[str | int, DirectRelationReference]:
-        key = resource_type, property_id
-        if key in self.ASSET_REFERENCE_PROPERTIES:
-            return self.asset  # type: ignore[return-value]
-        if key in self.SOURCE_REFERENCE_PROPERTIES:
-            return self.source  # type: ignore[return-value]
-        if key in self.FILE_REFERENCE_PROPERTIES:
-            return self.file  # type: ignore[return-value]
-        return {}
+    ) -> Mapping[str | int, DirectRelationReference] | None:
+        """Get the cache for the given resource type and property ID."""
+        return self._cache_map.get((resource_type, property_id))  # type: ignore[return-value]
 
 
 def asset_centric_to_dm(
@@ -92,9 +165,7 @@ def asset_centric_to_dm(
     instance_id: InstanceId,
     view_source: ResourceViewMapping,
     view_properties: dict[str, ViewProperty],
-    asset_instance_id_by_id: Mapping[int, DirectRelationReference],
-    source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
-    file_instance_id_by_id: Mapping[int, DirectRelationReference],
+    direct_relation_cache: DirectRelationCache,
 ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
     """Convert an asset-centric resource to a data model instance.
 
@@ -103,22 +174,11 @@ def asset_centric_to_dm(
         instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
         view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
         view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
-        asset_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from asset IDs to their corresponding
-            DirectRelationReference in the data model. This is used to create direct relations for resources that
-            reference assets.
-        source_instance_id_by_external_id (dict[str, DirectRelationReference]): A mapping from source strings to their
-            corresponding DirectRelationReference in the data model. This is used to create direct relations for resources
-            that reference sources.
-        file_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from file IDs to their corresponding
-            DirectRelationReference in the data model. This is used to create direct relations for resources that
-            reference files.
+        direct_relation_cache (DirectRelationCache): Cache for direct relation references.
 
     Returns:
         tuple[NodeApply | EdgeApply, ConversionIssue]: A tuple containing the converted NodeApply and any ConversionIssue encountered.
     """
-    cache = DirectRelationCache(
-        asset=asset_instance_id_by_id, source=source_instance_id_by_external_id, file=file_instance_id_by_id
-    )
     resource_type = _lookup_resource_type(resource)
     dumped = resource.dump()
     try:
@@ -138,7 +198,7 @@ def asset_centric_to_dm(
         view_source.property_mapping,
         resource_type,
         issue=issue,
-        direct_relation_cache=cache,
+        direct_relation_cache=direct_relation_cache,
     )
     sources: list[NodeOrEdgeData] = []
     if properties:
@@ -156,7 +216,7 @@ def asset_centric_to_dm(
     instance: NodeApply | EdgeApply
     if isinstance(instance_id, EdgeId):
         edge_properties = create_edge_properties(
-            dumped, view_source.property_mapping, resource_type, issue, cache, instance_id.space
+            dumped, view_source.property_mapping, resource_type, issue, direct_relation_cache, instance_id.space
         )
         if any(key not in edge_properties for key in ("start_node", "end_node", "type")):
             # Failed conversion of edge properties
@@ -240,7 +300,7 @@ def create_properties(
                 dm_prop.nullable,
                 destination_container_property=(dm_prop.container, dm_prop.container_property_identifier),
                 source_property=(resource_type, prop_json_path),
-                direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
+                direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
             )
         except (ValueError, TypeError, NotImplementedError) as e:
             issue.failed_conversions.append(
@@ -288,7 +348,7 @@ def create_edge_properties(
                     flatten_dump[prop_json_path],
                     DirectRelation(),
                     False,
-                    direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
+                    direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
                 )
             except (ValueError, TypeError, NotImplementedError) as e:
                 issue.failed_conversions.append(

cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,14 +1,21 @@
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from typing import Generic
 
 from cognite.client.data_classes._base import (
     T_CogniteResource,
 )
-from cognite.client.data_classes.data_modeling import DirectRelationReference, InstanceApply, View, ViewId
+from cognite.client.data_classes.data_modeling import (
+    EdgeApply,
+    InstanceApply,
+    NodeApply,
+    View,
+    ViewId,
+)
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
-from cognite_toolkit._cdf_tk.commands._migrate.conversion import asset_centric_to_dm
+from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
 from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
@@ -16,7 +23,9 @@ from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
 from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.useful_types import T_AssetCentricResource
+from cognite_toolkit._cdf_tk.utils.useful_types import (
+    T_AssetCentricResource,
+)
 
 
 class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource], ABC):
@@ -31,7 +40,9 @@ class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource],
         pass
 
     @abstractmethod
-    def map(self, source: T_CogniteResource) -> tuple[T_WriteCogniteResource | None, MigrationIssue]:
+    def map(
+        self, source: Sequence[T_CogniteResource]
+    ) -> Sequence[tuple[T_WriteCogniteResource | None, MigrationIssue]]:
         """Map a chunk of source data to the target format.
 
         Args:
@@ -51,10 +62,7 @@ class AssetCentricMapper(
         self.client = client
         self._ingestion_view_by_id: dict[ViewId, View] = {}
         self._view_mapping_by_id: dict[str, ResourceViewMapping] = {}
-        # This is used to keep track of already mapped assets, such that we can creat direct relations
-        # to them from files, events, and time series.
-        self._asset_mapping_by_id: dict[int, DirectRelationReference] = {}
-        self._source_system_mapping_by_id: dict[str, DirectRelationReference] = {}
+        self._direct_relation_cache = DirectRelationCache(client)
 
     def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
         ingestion_view_ids = source_selector.get_ingestion_mappings()
@@ -74,22 +82,23 @@ class AssetCentricMapper(
             raise ToolkitValueError(
                 f"The following ingestion views were not found in Data Modeling: {humanize_collection(missing_views)}"
             )
-        # We just look-up all source system for now. This can be optimized to only
-        # look-up the ones that are actually used in the ingestion views. However, SourceSystem is typically in
-        # the order ~10 instances, so this is not a big deal for now. See task CDF-25974.
-        source_systems = self.client.migration.created_source_system.list(limit=-1)
-        self._source_system_mapping_by_id = {
-            source_system.source: source_system.as_direct_relation_reference() for source_system in source_systems
-        }
-
-        # We look-up all existing asset mappings to be able to create direct relations to already mapped assets.
-        # This is needed in case the migration is run multiple times, or if assets are mapped
-        asset_mappings = self.client.migration.instance_source.list(resource_type="asset", limit=-1)
-        self._asset_mapping_by_id = {mapping.id_: mapping.as_direct_relation_reference() for mapping in asset_mappings}
-
-    def map(self, source: AssetCentricMapping[T_AssetCentricResource]) -> tuple[InstanceApply | None, ConversionIssue]:
+
+    def map(
+        self, source: Sequence[AssetCentricMapping[T_AssetCentricResource]]
+    ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
-        mapping = source.mapping
+        # We update the direct relation cache in bulk for all resources in the chunk.
+        self._direct_relation_cache.update(item.resource for item in source)
+        output: list[tuple[InstanceApply | None, ConversionIssue]] = []
+        for item in source:
+            instance, conversion_issue = self._map_single_item(item)
+            output.append((instance, conversion_issue))
+        return output
+
+    def _map_single_item(
+        self, item: AssetCentricMapping[T_AssetCentricResource]
+    ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
+        mapping = item.mapping
         ingestion_view = mapping.get_ingestion_view()
         try:
             view_source = self._view_mapping_by_id[ingestion_view]
@@ -99,19 +108,12 @@ class AssetCentricMapper(
                 f"Failed to lookup mapping or view for ingestion view '{ingestion_view}'. Did you forget to call .prepare()?"
             ) from e
         instance, conversion_issue = asset_centric_to_dm(
-            source.resource,
+            item.resource,
             instance_id=mapping.instance_id,
             view_source=view_source,
             view_properties=view_properties,
-            asset_instance_id_by_id=self._asset_mapping_by_id,
-            source_instance_id_by_external_id=self._source_system_mapping_by_id,
-            file_instance_id_by_id={},  # Todo implement file direct relations
+            direct_relation_cache=self._direct_relation_cache,
         )
         if mapping.instance_id.space == MISSING_INSTANCE_SPACE:
             conversion_issue.missing_instance_space = f"Missing instance space for dataset ID {mapping.data_set_id!r}"
-
-        if mapping.resource_type == "asset":
-            self._asset_mapping_by_id[mapping.id] = DirectRelationReference(
-                space=mapping.instance_id.space, external_id=mapping.instance_id.external_id
-            )
         return instance, conversion_issue

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -82,9 +82,9 @@ class UploadCommand(ToolkitCommand):
         │   │   └── table2.Table.yaml
         │   └── ...
         ├── datafile1.kind.ndjson # Data file of a specific kind
-        ├── datafile1.Metadata.yaml       # Metadata file for datafile1
+        ├── datafile1.Manifest.yaml       # Manifest for datafile1
         ├── datafile2.kind2.ndjson # Another data file of the same or different kind
-        ├── datafile2.Metadata.yaml       # Metadata file for datafile2
+        ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
         console = Console()

cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -9,7 +9,7 @@ from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
-from ._base import Page, UploadableStorageIO
+from ._base import Page, UploadableStorageIO, UploadItem
 from .selectors import AllChartsSelector, CanvasSelector, ChartExternalIdSelector, ChartOwnerSelector, ChartSelector
 
 
@@ -20,6 +20,9 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
     SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
     CHUNK_SIZE = 10
     BASE_SELECTOR = ChartSelector
+    UPLOAD_ENDPOINT_TYPE = "app"
+    UPLOAD_ENDPOINT_METHOD = "PUT"
+    UPLOAD_ENDPOINT = "/storage/charts/charts"
 
     def as_id(self, item: Chart) -> str:
         return item.external_id
@@ -75,7 +78,41 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
                         item["tsExternalId"] = ts_external_id
         return dumped
 
+    def json_chunk_to_data(self, data_chunk: list[tuple[str, dict[str, JsonVal]]]) -> Sequence[UploadItem[ChartWrite]]:
+        self._populate_timeseries_external_id_cache([item_json for _, item_json in data_chunk])
+        return super().json_chunk_to_data(data_chunk)
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> ChartWrite:
+        return self._load_resource(item_json)
+
+    def _populate_timeseries_external_id_cache(self, item_jsons: Sequence[dict[str, JsonVal]]) -> None:
+        timeseries_external_ids: set[str] = set()
+        for item_json in item_jsons:
+            if isinstance(data := item_json.get("data"), dict) and isinstance(
+                collection := data.get("timeSeriesCollection"), list
+            ):
+                for item in collection:
+                    if not isinstance(item, dict):
+                        continue
+                    ts_external_id = item.get("tsExternalId")
+                    if isinstance(ts_external_id, str):
+                        timeseries_external_ids.add(ts_external_id)
+        if timeseries_external_ids:
+            self.client.lookup.time_series.id(list(timeseries_external_ids))
+
+    def _load_resource(self, item_json: dict[str, JsonVal]) -> ChartWrite:
+        if isinstance(data := item_json.get("data"), dict) and isinstance(
+            collection := data.get("timeSeriesCollection"), list
+        ):
+            for item in collection:
+                if not isinstance(item, dict):
+                    continue
+                ts_external_id = item.get("tsExternalId")
+                if isinstance(ts_external_id, str) and item.get("tsId") is None:
+                    # We only look-up the internalID if it is missing
+                    ts_id = self.client.lookup.time_series.id(ts_external_id)
+                    if ts_id is not None:
+                        item["tsId"] = ts_id
         return ChartWrite._load(item_json)
 
 

cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Mapping, Sequence, Sized
 from dataclasses import dataclass
-from typing import ClassVar, Generic, TypeVar
+from typing import ClassVar, Generic, Literal, TypeVar
 
 from cognite.client.data_classes._base import T_CogniteResource
 
@@ -145,6 +145,8 @@ class UploadableStorageIO(
 
     KIND: ClassVar[str]
     SUPPORTED_READ_FORMATS: ClassVar[frozenset[str]]
+    UPLOAD_ENDPOINT_TYPE: Literal["app", "api"] = "api"
+    UPLOAD_ENDPOINT_METHOD: Literal["GET", "POST", "PATCH", "DELETE", "PUT"] = "POST"
     UPLOAD_ENDPOINT: ClassVar[str]
     UPLOAD_EXTRA_ARGS: ClassVar[Mapping[str, JsonVal] | None] = None
 
@@ -170,10 +172,17 @@ class UploadableStorageIO(
             raise ValueError(f"Data chunk size {len(data_chunk)} exceeds the maximum CHUNK_SIZE of {self.CHUNK_SIZE}.")
 
         config = http_client.config
+        if self.UPLOAD_ENDPOINT_TYPE == "api":
+            url = config.create_api_url(self.UPLOAD_ENDPOINT)
+        elif self.UPLOAD_ENDPOINT_TYPE == "app":
+            url = config.create_app_url(self.UPLOAD_ENDPOINT)
+        else:
+            raise ToolkitNotImplementedError(f"Unsupported UPLOAD_ENDPOINT_TYPE {self.UPLOAD_ENDPOINT_TYPE!r}.")
+
         return http_client.request_with_retries(
             message=ItemsRequest(
-                endpoint_url=config.create_api_url(self.UPLOAD_ENDPOINT),
-                method="POST",
+                endpoint_url=url,
+                method=self.UPLOAD_ENDPOINT_METHOD,
                 items=list(data_chunk),
                 extra_body_fields=dict(self.UPLOAD_EXTRA_ARGS or {}),
             )

cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py

@@ -87,7 +87,7 @@ class RequestMessage(HTTPMessage):
     """Base class for HTTP request messages"""
 
     endpoint_url: str
-    method: Literal["GET", "POST", "PATCH", "DELETE"]
+    method: Literal["GET", "POST", "PATCH", "DELETE", "PUT"]
     connect_attempt: int = 0
     read_attempt: int = 0
     status_attempt: int = 0

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.103
+      image: cognite/toolkit:0.6.104
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.103
+      image: cognite/toolkit:0.6.104
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.103"
+version = "0.6.104"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.103"
+__version__ = "0.6.104"

{cognite_toolkit-0.6.103.dist-info → cognite_toolkit-0.6.104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.103
+Version: 0.6.104
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.103.dist-info → cognite_toolkit-0.6.104.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=0abeQr1Tfk4lkGaoXyrnFC28wDSlR_8UGrh10noGduQ,6085
-cognite_toolkit/_version.py,sha256=bc53ksubP4hX6ZFjw8Qz9xxUaOPhtPL-MFyFCBVOWBI,24
+cognite_toolkit/_version.py,sha256=1DxGUOUNv3aZzpO7zqLfCeyAgR320ErlCl6Oiqlf_rw,24
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
 cognite_toolkit/_cdf_tk/constants.py,sha256=Z0nsRXBrqVUgalMa7MZFiwPp26KUfaHqX2h89Daox-M,6936
@@ -40,7 +40,7 @@ cognite_toolkit/_cdf_tk/client/_constants.py,sha256=COUGcea37mDF2sf6MGqJXWmecTY_
 cognite_toolkit/_cdf_tk/client/_toolkit_client.py,sha256=InfVdwtj5WA9hfa2kUr4sdhTCBhE08aDlVg2_QMKP5k,2928
 cognite_toolkit/_cdf_tk/client/api_client.py,sha256=CQdD_gfDqQkz5OYHrTnKvBvEvzHPdHDB1BkZPWRoahg,440
 cognite_toolkit/_cdf_tk/client/config.py,sha256=weMR43z-gqHMn-Jqvfmh_nJ0HbgEdyeCGtISuEf3OuY,4269
-cognite_toolkit/_cdf_tk/client/testing.py,sha256=5ZZI4k9efHlBH97zvV-gmYiHtG9wSqXJ57TGoN_F4RY,6313
+cognite_toolkit/_cdf_tk/client/testing.py,sha256=axKaW6rdzq_TI7DD4ijnLeHC4a74WKWDAxQ3QyZ7Kvg,6720
 cognite_toolkit/_cdf_tk/client/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/client/api/canvas.py,sha256=i2NwyhvmklTPx3e-yd4lvSxyn6JEjSpv8WXa1SxtmV8,8789
 cognite_toolkit/_cdf_tk/client/api/charts.py,sha256=t-VOrRGwpjmYUtUqGObQWYwGb5gOHVp4cHZBm8ZVGn0,4953
@@ -108,7 +108,7 @@ cognite_toolkit/_cdf_tk/commands/_download.py,sha256=OBKPM_HGGA1i32th1SAgkQM_81C
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
 cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=RadQHsmkPez3fZ5HCP9b82o2_fBx8P_-bTo7prkvWXU,32525
 cognite_toolkit/_cdf_tk/commands/_questionary_style.py,sha256=h-w7fZKkGls3TrzIGBKjsZSGoXJJIYchgD1StfA40r8,806
-cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=ZK9h6RlA-p348x64HMOY-Dnff4rdxDgCNQFDIV_fthY,14427
+cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=axrdDIjW59LxByIwq379tfCEkoBSdq8kVx8ORmsOQxo,14422
 cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
 cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
 cognite_toolkit/_cdf_tk/commands/auth.py,sha256=PLjfxfJJKaqux1eB2fycIRlwwSMCbM3qxWDnFX-blJU,31720
@@ -126,11 +126,11 @@ cognite_toolkit/_cdf_tk/commands/repo.py,sha256=MNy8MWphTklIZHvQOROCweq8_SYxGv6B
 cognite_toolkit/_cdf_tk/commands/run.py,sha256=JyX9jLEQej9eRrHVCCNlw4GuF80qETSol3-T5CCofgw,37331
 cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=i5ldcTah59K0E4fH5gHTV0GRvtDCEvVses9WQzn9Lno,226
 cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py,sha256=R-z0yfOFcJZj-zRLhN-7z_-SLxqzSmONMgrbzNF9dGs,8843
-cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=059GfizsxZzsO7CJP3uEtyvfbeKt2F5eYnvq9GvHxNE,14212
-cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=9hoKY_SNeVwxmQI7C1SRIvSaK5LNWEi7mpM8dPpjCFI,13524
+cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=jNoqqq81lbdfDTAQ5w2ctaYSUueLhZe0qjUKjCezk6s,14234
+cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=_okrcPRaNtM2d8u4Aba1hUYN8CgSTAcDgcugd76aXWA,16775
 cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=FTu7w3G8KyPY8pagG3KdPpOmpLcjehaAg2auEy6iM7A,9605
 cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=vOdvDPVibAbqLQt8hg7pmdHN1Fd6JUoyOZ-fCzfeEH8,9925
-cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=7m9uj_W11iokGmbWD979z5UK5KwOYv7L9HiY5PcpIN4,6231
+cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=5gqVRJGa7rh3c9pwPdipxKfl-zXV4mmU0G_h3ehPe2Q,5339
 cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=i1eUsNX6Dueol9STIEwyksBnBsWUk13O8qHIjW964pM,7860
 cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py,sha256=KkSq_4R6hQ15ccG-jHy7vVgPwC5IDd5OaXZLvz5mIZs,5547
 cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=lWSnuS3CfRDbA7i1g12gJ2reJnQcLmZWxHDK19-Wxkk,5772
@@ -239,9 +239,9 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Q
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=SSMV-W_uqMwS9I0xazBfAyNRqKWlAuLlABropMBEa50,2434
 cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=N5z2umaPwBo3OKIyGhNx2geRTBYhAeG3YhCAYm2hyao,4655
-cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=nqFB54C06UA-u5hoTtVza91XrscOktYt0mMsCkh1x-o,5091
+cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=CdqJueM9ZmXVh8RUme2lAgNasjAM8QTQDAfeJMm2ZYo,7026
 cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=DbTvIneN8Hw3ByhdH1kXkS7Gw68oXEWtIqlZGZgLMg0,33704
-cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=6f4akYlaOp8t22ecb8ZEt1rcBU58oD-iPm8UdwLdPuE,11716
+cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=at-5wERENsFRLO1cUOMKW2poVqmkvxSe9JuXpojz4NM,12196
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
 cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=nV8jaF5YLvMKhDPU3euf554GvSmfNYkzC9ZvEF7kbP8,8660
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
@@ -291,7 +291,7 @@ cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=mBf0-8JFwLfyTGJH8nWpbn89
 cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=T9Owx1XQ2bN1voBBLs7hUpTXihCiXgHwEhpIv_XB9xc,17732
 cognite_toolkit/_cdf_tk/utils/http_client/__init__.py,sha256=G8b7Bg4yIet5R4Igh3dS2SntWzE6I0iTGBeNlNsSxkQ,857
 cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=NTRfloXkCiS_rl5Vl1D_hsyTTowMKWDsiIR4oGwTADI,11208
-cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py,sha256=GaQ0DcJ-veToh1JNHt6ARY-ci1RP-Cvy6yIe1lzdTKc,12968
+cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py,sha256=PIlSmv3spObHeoylpSzz2fqUFAjIoE89qzvc0uSOGw0,12975
 cognite_toolkit/_cdf_tk/utils/http_client/_exception.py,sha256=fC9oW6BN0HbUe2AkYABMP7Kj0-9dNYXVFBY5RQztq2c,126
 cognite_toolkit/_cdf_tk/utils/http_client/_tracker.py,sha256=EBBnd-JZ7nc_jYNFJokCHN2UZ9sx0McFLZvlceUYYic,1215
 cognite_toolkit/_repo_files/.env.tmpl,sha256=UmgKZVvIp-OzD8oOcYuwb_6c7vSJsqkLhuFaiVgK7RI,972
@@ -299,13 +299,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=35BV9thL8pJRVLbKuXEoT0_j4R2SToeQQ81mQF75Q1c,668
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=xBsffXwMEnOPTRX6_J8g2plbgXKOL9VuDPDTG3sUIK0,2431
-cognite_toolkit/_resources/cdf.toml,sha256=TgAvDSPHTIqkgLiKA0gt61-nzIH3np8GflSw1mWWyjo,488
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=qEUo28LXoLaLmKW3HxlFAv-kLCe7pJW79ozjNA9-tTQ,668
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=vrp-6DByUjEy_Beovft4LKYkEdh_Sz6hcSNrS1fT1j4,2431
+cognite_toolkit/_resources/cdf.toml,sha256=m5sMGtmA5ileSZWahz8VRHfFSeORhRMaSKJeFZlEbts,488
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.103.dist-info/METADATA,sha256=eHyjnA0JjNbvpCRCwQYPzPTSYjgHHQEYSiS-cz0WLA8,4502
-cognite_toolkit-0.6.103.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.103.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.103.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.103.dist-info/RECORD,,
+cognite_toolkit-0.6.104.dist-info/METADATA,sha256=76Rem3olFNldH17SpzSmjJLYlDv4z_EJ0kaSuRzM_3U,4502
+cognite_toolkit-0.6.104.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.104.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.104.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.104.dist-info/RECORD,,