cognite-toolkit 0.6.103__tar.gz → 0.6.104__tar.gz

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.103
+Version: 0.6.104
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cdf.toml

@@ -32,7 +32,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.103"
+version = "0.6.104"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/client/testing.py

@@ -35,7 +35,14 @@ from .api.lookup import (
     SecurityCategoriesLookUpAPI,
     TimeSeriesLookUpAPI,
 )
-from .api.migration import CreatedSourceSystemAPI, InstanceSourceAPI, MigrationAPI, ResourceViewMappingAPI
+from .api.migration import (
+    CreatedSourceSystemAPI,
+    InstanceSourceAPI,
+    LookupAPI,
+    MigrationAPI,
+    MigrationLookupAPI,
+    ResourceViewMappingAPI,
+)
 from .api.project import ProjectAPI
 from .api.robotics import RoboticsAPI
 from .api.robotics.capabilities import CapabilitiesAPI
@@ -96,6 +103,11 @@ class ToolkitClientMock(CogniteClientMock):
         self.lookup.functions = MagicMock(spec_set=FunctionLookUpAPI)
         self.migration = MagicMock(spec=MigrationAPI)
         self.migration.instance_source = MagicMock(spec_set=InstanceSourceAPI)
+        self.migration.lookup = MagicMock(spec=MigrationLookupAPI)
+        self.migration.lookup.assets = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.events = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.files = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.time_series = MagicMock(spec_set=LookupAPI)
         self.migration.resource_view_mapping = MagicMock(spec_set=ResourceViewMappingAPI)
         self.migration.created_source_system = MagicMock(spec_set=CreatedSourceSystemAPI)
         self.raw = MagicMock(spec=ExtendedRawAPI)

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -157,10 +157,11 @@ class MigrationCommand(ToolkitCommand):
         log_file: NDJsonWriter,
     ) -> Callable[[Sequence[T_CogniteResource]], Sequence[UploadItem[T_WriteCogniteResource]]]:
         def track_mapping(source: Sequence[T_CogniteResource]) -> list[UploadItem[T_WriteCogniteResource]]:
+            mapped = mapper.map(source)
             issues: list[Chunk] = []
             targets: list[UploadItem[T_WriteCogniteResource]] = []
-            for item in source:
-                target, issue = mapper.map(item)
+
+            for (target, issue), item in zip(mapped, source):
                 id_ = data.as_id(item)
                 result: Status = "failed" if target is None else "success"
                 tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -1,6 +1,5 @@
-from collections.abc import Mapping, Set
-from dataclasses import dataclass
-from typing import Any, ClassVar
+from collections.abc import Iterable, Mapping, Set
+from typing import Any, ClassVar, cast
 
 from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
 from cognite.client.data_classes.data_modeling import (
@@ -15,6 +14,7 @@ from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeO
 from cognite.client.data_classes.data_modeling.views import ViewProperty
 from cognite.client.utils._identifier import InstanceId
 
+from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
 from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
 from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
@@ -30,61 +30,134 @@ from .data_model import INSTANCE_SOURCE_VIEW_ID
 from .issues import ConversionIssue, FailedConversion, InvalidPropertyDataType
 
 
-@dataclass
 class DirectRelationCache:
     """Cache for direct relation references to look up target of direct relations.
 
-    This is used when creating direct relations from asset-centric resources to CogniteAsset and CogniteSourceSystem.
-
-    Attributes:
-        asset: Mapping[int, DirectRelationReference]
-            A mapping from asset IDs to their corresponding DirectRelationReference in the data model.
-        source: Mapping[str, DirectRelationReference]
-            A mapping from source strings to their corresponding DirectRelationReference in the data model.
-
-    Methods:
-        get(resource_type: AssetCentric, property_id: str) -> Mapping[str | int, DirectRelationReference]:
-            Get the appropriate mapping based on the resource type and property ID.
-
-    Note:
-        The ASSET_REFERENCE_PROPERTIES and SOURCE_REFERENCE_PROPERTIES class variables define which properties
-        in asset-centric resources reference assets and sources, respectively.
-
+    This is used when creating direct relations from asset-centric resources to assets, files, and source systems.
     """
 
-    ASSET_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    class TableName:
+        ASSET_ID = "assetId"
+        SOURCE_NAME = "source"
+        FILE_ID = "fileId"
+        ASSET_EXTERNAL_ID = "assetExternalId"
+        FILE_EXTERNAL_ID = "fileExternalId"
+
+    ASSET_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("timeseries", "assetId"),
         ("file", "assetIds"),
         ("event", "assetIds"),
         ("sequence", "assetId"),
-        ("asset", "parentId"),
         ("annotation", "data.assetRef.id"),
+        ("asset", "parentId"),
     }
-    SOURCE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    SOURCE_NAME_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("asset", "source"),
         ("event", "source"),
         ("file", "source"),
     }
-    FILE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    FILE_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("annotation", "data.fileRef.id"),
         ("annotation", "annotatedResourceId"),
     }
+    ASSET_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.assetRef.externalId")}
+    FILE_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.fileRef.externalId")}
+
+    def __init__(self, client: ToolkitClient) -> None:
+        self._client = client
+        self._cache_map: dict[
+            tuple[str, str] | str, dict[str, DirectRelationReference] | dict[int, DirectRelationReference]
+        ] = {}
+        # Constructing the cache map to be accessed by both table name and property id
+        for table_name, properties in [
+            (self.TableName.ASSET_ID, self.ASSET_ID_PROPERTIES),
+            (self.TableName.SOURCE_NAME, self.SOURCE_NAME_PROPERTIES),
+            (self.TableName.FILE_ID, self.FILE_ID_PROPERTIES),
+            (self.TableName.ASSET_EXTERNAL_ID, self.ASSET_EXTERNAL_ID_PROPERTIES),
+            (self.TableName.FILE_EXTERNAL_ID, self.FILE_EXTERNAL_ID_PROPERTIES),
+        ]:
+            cache: dict[str, DirectRelationReference] | dict[int, DirectRelationReference] = {}
+            self._cache_map[table_name] = cache
+            for key in properties:
+                self._cache_map[key] = cache
+
+    def update(self, resources: Iterable[AssetCentricResourceExtended]) -> None:
+        """Update the cache with direct relation references for the given asset-centric resources.
+
+        This is used to bulk update the cache for a chunk of resources before converting them to data model instances.
+        """
+        asset_ids: set[int] = set()
+        source_ids: set[str] = set()
+        file_ids: set[int] = set()
+        asset_external_ids: set[str] = set()
+        file_external_ids: set[str] = set()
+        for resource in resources:
+            if isinstance(resource, Annotation):
+                if resource.annotated_resource_type == "file" and resource.annotated_resource_id:
+                    file_ids.add(resource.annotated_resource_id)
+                if "assetRef" in resource.data:
+                    asset_ref = resource.data["assetRef"]
+                    if isinstance(asset_id := asset_ref.get("id"), int):
+                        asset_ids.add(asset_id)
+                    if isinstance(asset_external_id := asset_ref.get("externalId"), str):
+                        asset_external_ids.add(asset_external_id)
+                if "fileRef" in resource.data:
+                    file_ref = resource.data["fileRef"]
+                    if isinstance(file_id := file_ref.get("id"), int):
+                        file_ids.add(file_id)
+                    if isinstance(file_external_id := file_ref.get("externalId"), str):
+                        file_external_ids.add(file_external_id)
+            elif isinstance(resource, Asset):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.parent_id is not None:
+                    asset_ids.add(resource.parent_id)
+            elif isinstance(resource, FileMetadata):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, Event):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, TimeSeries):
+                if resource.asset_id is not None:
+                    asset_ids.add(resource.asset_id)
+        if asset_ids:
+            self._update_cache(self._client.migration.lookup.assets(id=list(asset_ids)), self.TableName.ASSET_ID)
+        if source_ids:
+            # SourceSystems are not cached in the client, so we have to handle the caching ourselves.
+            cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
+            missing = set(source_ids) - set(cache.keys())
+            if missing:
+                source_systems = self._client.migration.created_source_system.retrieve(list(missing))
+                for source_system in source_systems:
+                    cache[source_system.source] = source_system.as_direct_relation_reference()
+        if file_ids:
+            self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
+        if asset_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.assets(external_id=list(asset_external_ids)),
+                self.TableName.ASSET_EXTERNAL_ID,
+            )
+        if file_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.files(external_id=list(file_external_ids)),
+                self.TableName.FILE_EXTERNAL_ID,
+            )
 
-    asset: Mapping[int, DirectRelationReference]
-    source: Mapping[str, DirectRelationReference]
-    file: Mapping[int, DirectRelationReference]
+    def _update_cache(self, instance_id_by_id: dict[int, NodeId] | dict[str, NodeId], table_name: str) -> None:
+        cache = self._cache_map[table_name]
+        for identifier, instance_id in instance_id_by_id.items():
+            cache[identifier] = DirectRelationReference(space=instance_id.space, external_id=instance_id.external_id)  # type: ignore[index]
 
-    def get(
+    def get_cache(
         self, resource_type: AssetCentricTypeExtended, property_id: str
-    ) -> Mapping[str | int, DirectRelationReference]:
-        key = resource_type, property_id
-        if key in self.ASSET_REFERENCE_PROPERTIES:
-            return self.asset  # type: ignore[return-value]
-        if key in self.SOURCE_REFERENCE_PROPERTIES:
-            return self.source  # type: ignore[return-value]
-        if key in self.FILE_REFERENCE_PROPERTIES:
-            return self.file  # type: ignore[return-value]
-        return {}
+    ) -> Mapping[str | int, DirectRelationReference] | None:
+        """Get the cache for the given resource type and property ID."""
+        return self._cache_map.get((resource_type, property_id))  # type: ignore[return-value]
 
 
 def asset_centric_to_dm(
@@ -92,9 +165,7 @@ def asset_centric_to_dm(
     instance_id: InstanceId,
     view_source: ResourceViewMapping,
     view_properties: dict[str, ViewProperty],
-    asset_instance_id_by_id: Mapping[int, DirectRelationReference],
-    source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
-    file_instance_id_by_id: Mapping[int, DirectRelationReference],
+    direct_relation_cache: DirectRelationCache,
 ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
     """Convert an asset-centric resource to a data model instance.
 
@@ -103,22 +174,11 @@ def asset_centric_to_dm(
         instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
         view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
         view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
-        asset_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from asset IDs to their corresponding
-            DirectRelationReference in the data model. This is used to create direct relations for resources that
-            reference assets.
-        source_instance_id_by_external_id (dict[str, DirectRelationReference]): A mapping from source strings to their
-            corresponding DirectRelationReference in the data model. This is used to create direct relations for resources
-            that reference sources.
-        file_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from file IDs to their corresponding
-            DirectRelationReference in the data model. This is used to create direct relations for resources that
-            reference files.
+        direct_relation_cache (DirectRelationCache): Cache for direct relation references.
 
     Returns:
         tuple[NodeApply | EdgeApply, ConversionIssue]: A tuple containing the converted NodeApply and any ConversionIssue encountered.
     """
-    cache = DirectRelationCache(
-        asset=asset_instance_id_by_id, source=source_instance_id_by_external_id, file=file_instance_id_by_id
-    )
     resource_type = _lookup_resource_type(resource)
     dumped = resource.dump()
     try:
@@ -138,7 +198,7 @@ def asset_centric_to_dm(
         view_source.property_mapping,
         resource_type,
         issue=issue,
-        direct_relation_cache=cache,
+        direct_relation_cache=direct_relation_cache,
     )
     sources: list[NodeOrEdgeData] = []
     if properties:
@@ -156,7 +216,7 @@ def asset_centric_to_dm(
     instance: NodeApply | EdgeApply
     if isinstance(instance_id, EdgeId):
         edge_properties = create_edge_properties(
-            dumped, view_source.property_mapping, resource_type, issue, cache, instance_id.space
+            dumped, view_source.property_mapping, resource_type, issue, direct_relation_cache, instance_id.space
         )
         if any(key not in edge_properties for key in ("start_node", "end_node", "type")):
             # Failed conversion of edge properties
@@ -240,7 +300,7 @@ def create_properties(
                 dm_prop.nullable,
                 destination_container_property=(dm_prop.container, dm_prop.container_property_identifier),
                 source_property=(resource_type, prop_json_path),
-                direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
+                direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
             )
         except (ValueError, TypeError, NotImplementedError) as e:
             issue.failed_conversions.append(
@@ -288,7 +348,7 @@ def create_edge_properties(
                     flatten_dump[prop_json_path],
                     DirectRelation(),
                     False,
-                    direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
+                    direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
                 )
             except (ValueError, TypeError, NotImplementedError) as e:
                 issue.failed_conversions.append(

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,14 +1,21 @@
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from typing import Generic
 
 from cognite.client.data_classes._base import (
     T_CogniteResource,
 )
-from cognite.client.data_classes.data_modeling import DirectRelationReference, InstanceApply, View, ViewId
+from cognite.client.data_classes.data_modeling import (
+    EdgeApply,
+    InstanceApply,
+    NodeApply,
+    View,
+    ViewId,
+)
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
-from cognite_toolkit._cdf_tk.commands._migrate.conversion import asset_centric_to_dm
+from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
 from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
@@ -16,7 +23,9 @@ from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
 from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.useful_types import T_AssetCentricResource
+from cognite_toolkit._cdf_tk.utils.useful_types import (
+    T_AssetCentricResource,
+)
 
 
 class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource], ABC):
@@ -31,7 +40,9 @@ class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource],
         pass
 
     @abstractmethod
-    def map(self, source: T_CogniteResource) -> tuple[T_WriteCogniteResource | None, MigrationIssue]:
+    def map(
+        self, source: Sequence[T_CogniteResource]
+    ) -> Sequence[tuple[T_WriteCogniteResource | None, MigrationIssue]]:
         """Map a chunk of source data to the target format.
 
         Args:
@@ -51,10 +62,7 @@ class AssetCentricMapper(
         self.client = client
         self._ingestion_view_by_id: dict[ViewId, View] = {}
         self._view_mapping_by_id: dict[str, ResourceViewMapping] = {}
-        # This is used to keep track of already mapped assets, such that we can creat direct relations
-        # to them from files, events, and time series.
-        self._asset_mapping_by_id: dict[int, DirectRelationReference] = {}
-        self._source_system_mapping_by_id: dict[str, DirectRelationReference] = {}
+        self._direct_relation_cache = DirectRelationCache(client)
 
     def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
         ingestion_view_ids = source_selector.get_ingestion_mappings()
@@ -74,22 +82,23 @@ class AssetCentricMapper(
             raise ToolkitValueError(
                 f"The following ingestion views were not found in Data Modeling: {humanize_collection(missing_views)}"
             )
-        # We just look-up all source system for now. This can be optimized to only
-        # look-up the ones that are actually used in the ingestion views. However, SourceSystem is typically in
-        # the order ~10 instances, so this is not a big deal for now. See task CDF-25974.
-        source_systems = self.client.migration.created_source_system.list(limit=-1)
-        self._source_system_mapping_by_id = {
-            source_system.source: source_system.as_direct_relation_reference() for source_system in source_systems
-        }
-
-        # We look-up all existing asset mappings to be able to create direct relations to already mapped assets.
-        # This is needed in case the migration is run multiple times, or if assets are mapped
-        asset_mappings = self.client.migration.instance_source.list(resource_type="asset", limit=-1)
-        self._asset_mapping_by_id = {mapping.id_: mapping.as_direct_relation_reference() for mapping in asset_mappings}
-
-    def map(self, source: AssetCentricMapping[T_AssetCentricResource]) -> tuple[InstanceApply | None, ConversionIssue]:
+
+    def map(
+        self, source: Sequence[AssetCentricMapping[T_AssetCentricResource]]
+    ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
-        mapping = source.mapping
+        # We update the direct relation cache in bulk for all resources in the chunk.
+        self._direct_relation_cache.update(item.resource for item in source)
+        output: list[tuple[InstanceApply | None, ConversionIssue]] = []
+        for item in source:
+            instance, conversion_issue = self._map_single_item(item)
+            output.append((instance, conversion_issue))
+        return output
+
+    def _map_single_item(
+        self, item: AssetCentricMapping[T_AssetCentricResource]
+    ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
+        mapping = item.mapping
         ingestion_view = mapping.get_ingestion_view()
         try:
             view_source = self._view_mapping_by_id[ingestion_view]
@@ -99,19 +108,12 @@ class AssetCentricMapper(
                 f"Failed to lookup mapping or view for ingestion view '{ingestion_view}'. Did you forget to call .prepare()?"
             ) from e
         instance, conversion_issue = asset_centric_to_dm(
-            source.resource,
+            item.resource,
             instance_id=mapping.instance_id,
             view_source=view_source,
             view_properties=view_properties,
-            asset_instance_id_by_id=self._asset_mapping_by_id,
-            source_instance_id_by_external_id=self._source_system_mapping_by_id,
-            file_instance_id_by_id={},  # Todo implement file direct relations
+            direct_relation_cache=self._direct_relation_cache,
         )
         if mapping.instance_id.space == MISSING_INSTANCE_SPACE:
             conversion_issue.missing_instance_space = f"Missing instance space for dataset ID {mapping.data_set_id!r}"
-
-        if mapping.resource_type == "asset":
-            self._asset_mapping_by_id[mapping.id] = DirectRelationReference(
-                space=mapping.instance_id.space, external_id=mapping.instance_id.external_id
-            )
         return instance, conversion_issue

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -82,9 +82,9 @@ class UploadCommand(ToolkitCommand):
         │   │   └── table2.Table.yaml
         │   └── ...
         ├── datafile1.kind.ndjson # Data file of a specific kind
-        ├── datafile1.Metadata.yaml       # Metadata file for datafile1
+        ├── datafile1.Manifest.yaml       # Manifest for datafile1
         ├── datafile2.kind2.ndjson # Another data file of the same or different kind
-        ├── datafile2.Metadata.yaml       # Metadata file for datafile2
+        ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
         console = Console()

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/storageio/_applications.py

@@ -9,7 +9,7 @@ from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
 from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
-from ._base import Page, UploadableStorageIO
+from ._base import Page, UploadableStorageIO, UploadItem
 from .selectors import AllChartsSelector, CanvasSelector, ChartExternalIdSelector, ChartOwnerSelector, ChartSelector
 
 
@@ -20,6 +20,9 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
     SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
     CHUNK_SIZE = 10
     BASE_SELECTOR = ChartSelector
+    UPLOAD_ENDPOINT_TYPE = "app"
+    UPLOAD_ENDPOINT_METHOD = "PUT"
+    UPLOAD_ENDPOINT = "/storage/charts/charts"
 
     def as_id(self, item: Chart) -> str:
         return item.external_id
@@ -75,7 +78,41 @@ class ChartIO(UploadableStorageIO[ChartSelector, Chart, ChartWrite]):
                         item["tsExternalId"] = ts_external_id
         return dumped
 
+    def json_chunk_to_data(self, data_chunk: list[tuple[str, dict[str, JsonVal]]]) -> Sequence[UploadItem[ChartWrite]]:
+        self._populate_timeseries_external_id_cache([item_json for _, item_json in data_chunk])
+        return super().json_chunk_to_data(data_chunk)
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> ChartWrite:
+        return self._load_resource(item_json)
+
+    def _populate_timeseries_external_id_cache(self, item_jsons: Sequence[dict[str, JsonVal]]) -> None:
+        timeseries_external_ids: set[str] = set()
+        for item_json in item_jsons:
+            if isinstance(data := item_json.get("data"), dict) and isinstance(
+                collection := data.get("timeSeriesCollection"), list
+            ):
+                for item in collection:
+                    if not isinstance(item, dict):
+                        continue
+                    ts_external_id = item.get("tsExternalId")
+                    if isinstance(ts_external_id, str):
+                        timeseries_external_ids.add(ts_external_id)
+        if timeseries_external_ids:
+            self.client.lookup.time_series.id(list(timeseries_external_ids))
+
+    def _load_resource(self, item_json: dict[str, JsonVal]) -> ChartWrite:
+        if isinstance(data := item_json.get("data"), dict) and isinstance(
+            collection := data.get("timeSeriesCollection"), list
+        ):
+            for item in collection:
+                if not isinstance(item, dict):
+                    continue
+                ts_external_id = item.get("tsExternalId")
+                if isinstance(ts_external_id, str) and item.get("tsId") is None:
+                    # We only look-up the internalID if it is missing
+                    ts_id = self.client.lookup.time_series.id(ts_external_id)
+                    if ts_id is not None:
+                        item["tsId"] = ts_id
         return ChartWrite._load(item_json)
 
 

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Mapping, Sequence, Sized
 from dataclasses import dataclass
-from typing import ClassVar, Generic, TypeVar
+from typing import ClassVar, Generic, Literal, TypeVar
 
 from cognite.client.data_classes._base import T_CogniteResource
 
@@ -145,6 +145,8 @@ class UploadableStorageIO(
 
     KIND: ClassVar[str]
     SUPPORTED_READ_FORMATS: ClassVar[frozenset[str]]
+    UPLOAD_ENDPOINT_TYPE: Literal["app", "api"] = "api"
+    UPLOAD_ENDPOINT_METHOD: Literal["GET", "POST", "PATCH", "DELETE", "PUT"] = "POST"
     UPLOAD_ENDPOINT: ClassVar[str]
     UPLOAD_EXTRA_ARGS: ClassVar[Mapping[str, JsonVal] | None] = None
 
@@ -170,10 +172,17 @@ class UploadableStorageIO(
             raise ValueError(f"Data chunk size {len(data_chunk)} exceeds the maximum CHUNK_SIZE of {self.CHUNK_SIZE}.")
 
         config = http_client.config
+        if self.UPLOAD_ENDPOINT_TYPE == "api":
+            url = config.create_api_url(self.UPLOAD_ENDPOINT)
+        elif self.UPLOAD_ENDPOINT_TYPE == "app":
+            url = config.create_app_url(self.UPLOAD_ENDPOINT)
+        else:
+            raise ToolkitNotImplementedError(f"Unsupported UPLOAD_ENDPOINT_TYPE {self.UPLOAD_ENDPOINT_TYPE!r}.")
+
         return http_client.request_with_retries(
             message=ItemsRequest(
-                endpoint_url=config.create_api_url(self.UPLOAD_ENDPOINT),
-                method="POST",
+                endpoint_url=url,
+                method=self.UPLOAD_ENDPOINT_METHOD,
                 items=list(data_chunk),
                 extra_body_fields=dict(self.UPLOAD_EXTRA_ARGS or {}),
             )

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py

@@ -87,7 +87,7 @@ class RequestMessage(HTTPMessage):
     """Base class for HTTP request messages"""
 
     endpoint_url: str
-    method: Literal["GET", "POST", "PATCH", "DELETE"]
+    method: Literal["GET", "POST", "PATCH", "DELETE", "PUT"]
     connect_attempt: int = 0
     read_attempt: int = 0
     status_attempt: int = 0

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.103
+      image: cognite/toolkit:0.6.104
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.103
+      image: cognite/toolkit:0.6.104
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.103"
+version = "0.6.104"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit-0.6.104/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.6.104"

cognite_toolkit-0.6.104/last_changelog_entry.md

@@ -0,0 +1,9 @@
+## cdf 
+
+### Added
+
+- [alpha] `cdf data upload <directory>` can now upload charts.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.104/last_git_message.txt

@@ -0,0 +1,20 @@
+[CDF-24844] 📉 Upload charts (#2211)
+
+# Description
+
+Minor adjustments were necessary to upload charts.
+
+## Changelog
+
+- [x] Patch
+- [ ] Skip
+
+## cdf
+
+### Added
+
+- [alpha] `cdf data upload <directory>` can now upload charts.
+
+## templates
+
+No changes.

cognite_toolkit-0.6.104/last_version.txt

@@ -0,0 +1 @@
+0.6.103

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.6.103"
+version = "0.6.104"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.104}/tests/test_unit/test_cdf_tk/test_commands/test_migration_cmd/test_command.py

@@ -194,12 +194,6 @@ class TestMigrationCommand:
             json={"items": [asset.dump() for asset in assets]},
             status=200,
         )
-        # Lookup CogniteSourceSystem (no source systems defined)
-        rsps.post(
-            config.create_api_url("/models/instances/list"),
-            json={"items": []},
-            status=200,
-        )
         # Instance creation
         respx.post(
             config.create_api_url("/models/instances"),