cognite-toolkit 0.6.103__tar.gz → 0.6.105__tar.gz

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.103
+Version: 0.6.105
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cdf.toml

@@ -22,6 +22,7 @@ upload = true
 function-cognite-file = true
 external-libraries = true
 v07 = true
+streams = true
 
 [plugins]
 run = true
@@ -32,7 +33,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.103"
+version = "0.6.105"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/client/data_classes/streams.py

@@ -29,13 +29,12 @@ class StreamRequestList(UserList[StreamRequest], ResourceRequestListProtocol):
     """List of Stream request resources."""
 
     _RESOURCE = StreamRequest
-    items: list[StreamRequest]
 
     def __init__(self, initlist: list[StreamRequest] | None = None, **_: Any) -> None:
         super().__init__(initlist or [])
 
     def dump(self, camel_case: bool = True) -> list[dict[str, Any]]:
-        return [item.dump(camel_case) for item in self.items]
+        return [item.dump(camel_case) for item in self.data]
 
     @classmethod
     def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> "StreamRequestList":
@@ -78,19 +77,25 @@ class StreamResponse(ResponseResource["StreamRequest"]):
 
     external_id: str
     created_time: int
-    created_from_template: StreamTemplateName | None = None
-    type: Literal["Mutable", "Immutable"] | None = None
+    created_from_template: StreamTemplateName
+    type: Literal["Mutable", "Immutable"]
     settings: StreamSettings | None = None
 
     def as_request_resource(self) -> StreamRequest:
-        template_name = self.created_from_template or "BasicArchive"
-        return StreamRequest(
-            external_id=self.external_id,
-            settings={"template": {"name": template_name}},
+        return StreamRequest.model_validate(
+            {
+                "externalId": self.external_id,
+                "settings": {"template": {"name": self.created_from_template}},
+            }
         )
 
-    def as_write(self) -> Self:
-        return self
+    def as_write(self) -> StreamRequest:
+        return StreamRequest.model_validate(
+            {
+                "externalId": self.external_id,
+                "settings": {"template": {"name": self.created_from_template}},
+            }
+        )
 
 
 class StreamResponseList(UserList[StreamResponse], ResourceResponseListProtocol):

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/client/testing.py

@@ -35,7 +35,14 @@ from .api.lookup import (
     SecurityCategoriesLookUpAPI,
     TimeSeriesLookUpAPI,
 )
-from .api.migration import CreatedSourceSystemAPI, InstanceSourceAPI, MigrationAPI, ResourceViewMappingAPI
+from .api.migration import (
+    CreatedSourceSystemAPI,
+    InstanceSourceAPI,
+    LookupAPI,
+    MigrationAPI,
+    MigrationLookupAPI,
+    ResourceViewMappingAPI,
+)
 from .api.project import ProjectAPI
 from .api.robotics import RoboticsAPI
 from .api.robotics.capabilities import CapabilitiesAPI
@@ -96,6 +103,11 @@ class ToolkitClientMock(CogniteClientMock):
         self.lookup.functions = MagicMock(spec_set=FunctionLookUpAPI)
         self.migration = MagicMock(spec=MigrationAPI)
         self.migration.instance_source = MagicMock(spec_set=InstanceSourceAPI)
+        self.migration.lookup = MagicMock(spec=MigrationLookupAPI)
+        self.migration.lookup.assets = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.events = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.files = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.time_series = MagicMock(spec_set=LookupAPI)
         self.migration.resource_view_mapping = MagicMock(spec_set=ResourceViewMappingAPI)
         self.migration.created_source_system = MagicMock(spec_set=CreatedSourceSystemAPI)
         self.raw = MagicMock(spec=ExtendedRawAPI)

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -157,10 +157,11 @@ class MigrationCommand(ToolkitCommand):
         log_file: NDJsonWriter,
     ) -> Callable[[Sequence[T_CogniteResource]], Sequence[UploadItem[T_WriteCogniteResource]]]:
         def track_mapping(source: Sequence[T_CogniteResource]) -> list[UploadItem[T_WriteCogniteResource]]:
+            mapped = mapper.map(source)
             issues: list[Chunk] = []
             targets: list[UploadItem[T_WriteCogniteResource]] = []
-            for item in source:
-                target, issue = mapper.map(item)
+
+            for (target, issue), item in zip(mapped, source):
                 id_ = data.as_id(item)
                 result: Status = "failed" if target is None else "success"
                 tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -1,6 +1,5 @@
-from collections.abc import Mapping, Set
-from dataclasses import dataclass
-from typing import Any, ClassVar
+from collections.abc import Iterable, Mapping, Set
+from typing import Any, ClassVar, cast
 
 from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
 from cognite.client.data_classes.data_modeling import (
@@ -15,6 +14,7 @@ from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeO
 from cognite.client.data_classes.data_modeling.views import ViewProperty
 from cognite.client.utils._identifier import InstanceId
 
+from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
 from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
 from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
@@ -30,61 +30,134 @@ from .data_model import INSTANCE_SOURCE_VIEW_ID
 from .issues import ConversionIssue, FailedConversion, InvalidPropertyDataType
 
 
-@dataclass
 class DirectRelationCache:
     """Cache for direct relation references to look up target of direct relations.
 
-    This is used when creating direct relations from asset-centric resources to CogniteAsset and CogniteSourceSystem.
-
-    Attributes:
-        asset: Mapping[int, DirectRelationReference]
-            A mapping from asset IDs to their corresponding DirectRelationReference in the data model.
-        source: Mapping[str, DirectRelationReference]
-            A mapping from source strings to their corresponding DirectRelationReference in the data model.
-
-    Methods:
-        get(resource_type: AssetCentric, property_id: str) -> Mapping[str | int, DirectRelationReference]:
-            Get the appropriate mapping based on the resource type and property ID.
-
-    Note:
-        The ASSET_REFERENCE_PROPERTIES and SOURCE_REFERENCE_PROPERTIES class variables define which properties
-        in asset-centric resources reference assets and sources, respectively.
-
+    This is used when creating direct relations from asset-centric resources to assets, files, and source systems.
     """
 
-    ASSET_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    class TableName:
+        ASSET_ID = "assetId"
+        SOURCE_NAME = "source"
+        FILE_ID = "fileId"
+        ASSET_EXTERNAL_ID = "assetExternalId"
+        FILE_EXTERNAL_ID = "fileExternalId"
+
+    ASSET_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("timeseries", "assetId"),
         ("file", "assetIds"),
         ("event", "assetIds"),
         ("sequence", "assetId"),
-        ("asset", "parentId"),
         ("annotation", "data.assetRef.id"),
+        ("asset", "parentId"),
     }
-    SOURCE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    SOURCE_NAME_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("asset", "source"),
         ("event", "source"),
         ("file", "source"),
     }
-    FILE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
+    FILE_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
         ("annotation", "data.fileRef.id"),
         ("annotation", "annotatedResourceId"),
     }
+    ASSET_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.assetRef.externalId")}
+    FILE_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.fileRef.externalId")}
+
+    def __init__(self, client: ToolkitClient) -> None:
+        self._client = client
+        self._cache_map: dict[
+            tuple[str, str] | str, dict[str, DirectRelationReference] | dict[int, DirectRelationReference]
+        ] = {}
+        # Constructing the cache map to be accessed by both table name and property id
+        for table_name, properties in [
+            (self.TableName.ASSET_ID, self.ASSET_ID_PROPERTIES),
+            (self.TableName.SOURCE_NAME, self.SOURCE_NAME_PROPERTIES),
+            (self.TableName.FILE_ID, self.FILE_ID_PROPERTIES),
+            (self.TableName.ASSET_EXTERNAL_ID, self.ASSET_EXTERNAL_ID_PROPERTIES),
+            (self.TableName.FILE_EXTERNAL_ID, self.FILE_EXTERNAL_ID_PROPERTIES),
+        ]:
+            cache: dict[str, DirectRelationReference] | dict[int, DirectRelationReference] = {}
+            self._cache_map[table_name] = cache
+            for key in properties:
+                self._cache_map[key] = cache
+
+    def update(self, resources: Iterable[AssetCentricResourceExtended]) -> None:
+        """Update the cache with direct relation references for the given asset-centric resources.
+
+        This is used to bulk update the cache for a chunk of resources before converting them to data model instances.
+        """
+        asset_ids: set[int] = set()
+        source_ids: set[str] = set()
+        file_ids: set[int] = set()
+        asset_external_ids: set[str] = set()
+        file_external_ids: set[str] = set()
+        for resource in resources:
+            if isinstance(resource, Annotation):
+                if resource.annotated_resource_type == "file" and resource.annotated_resource_id:
+                    file_ids.add(resource.annotated_resource_id)
+                if "assetRef" in resource.data:
+                    asset_ref = resource.data["assetRef"]
+                    if isinstance(asset_id := asset_ref.get("id"), int):
+                        asset_ids.add(asset_id)
+                    if isinstance(asset_external_id := asset_ref.get("externalId"), str):
+                        asset_external_ids.add(asset_external_id)
+                if "fileRef" in resource.data:
+                    file_ref = resource.data["fileRef"]
+                    if isinstance(file_id := file_ref.get("id"), int):
+                        file_ids.add(file_id)
+                    if isinstance(file_external_id := file_ref.get("externalId"), str):
+                        file_external_ids.add(file_external_id)
+            elif isinstance(resource, Asset):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.parent_id is not None:
+                    asset_ids.add(resource.parent_id)
+            elif isinstance(resource, FileMetadata):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, Event):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, TimeSeries):
+                if resource.asset_id is not None:
+                    asset_ids.add(resource.asset_id)
+        if asset_ids:
+            self._update_cache(self._client.migration.lookup.assets(id=list(asset_ids)), self.TableName.ASSET_ID)
+        if source_ids:
+            # SourceSystems are not cached in the client, so we have to handle the caching ourselves.
+            cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
+            missing = set(source_ids) - set(cache.keys())
+            if missing:
+                source_systems = self._client.migration.created_source_system.retrieve(list(missing))
+                for source_system in source_systems:
+                    cache[source_system.source] = source_system.as_direct_relation_reference()
+        if file_ids:
+            self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
+        if asset_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.assets(external_id=list(asset_external_ids)),
+                self.TableName.ASSET_EXTERNAL_ID,
+            )
+        if file_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.files(external_id=list(file_external_ids)),
+                self.TableName.FILE_EXTERNAL_ID,
+            )
 
-    asset: Mapping[int, DirectRelationReference]
-    source: Mapping[str, DirectRelationReference]
-    file: Mapping[int, DirectRelationReference]
+    def _update_cache(self, instance_id_by_id: dict[int, NodeId] | dict[str, NodeId], table_name: str) -> None:
+        cache = self._cache_map[table_name]
+        for identifier, instance_id in instance_id_by_id.items():
+            cache[identifier] = DirectRelationReference(space=instance_id.space, external_id=instance_id.external_id)  # type: ignore[index]
 
-    def get(
+    def get_cache(
         self, resource_type: AssetCentricTypeExtended, property_id: str
-    ) -> Mapping[str | int, DirectRelationReference]:
-        key = resource_type, property_id
-        if key in self.ASSET_REFERENCE_PROPERTIES:
-            return self.asset  # type: ignore[return-value]
-        if key in self.SOURCE_REFERENCE_PROPERTIES:
-            return self.source  # type: ignore[return-value]
-        if key in self.FILE_REFERENCE_PROPERTIES:
-            return self.file  # type: ignore[return-value]
-        return {}
+    ) -> Mapping[str | int, DirectRelationReference] | None:
+        """Get the cache for the given resource type and property ID."""
+        return self._cache_map.get((resource_type, property_id))  # type: ignore[return-value]
 
 
 def asset_centric_to_dm(
@@ -92,9 +165,7 @@ def asset_centric_to_dm(
     instance_id: InstanceId,
     view_source: ResourceViewMapping,
     view_properties: dict[str, ViewProperty],
-    asset_instance_id_by_id: Mapping[int, DirectRelationReference],
-    source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
-    file_instance_id_by_id: Mapping[int, DirectRelationReference],
+    direct_relation_cache: DirectRelationCache,
 ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
     """Convert an asset-centric resource to a data model instance.
 
@@ -103,22 +174,11 @@ def asset_centric_to_dm(
         instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
         view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
         view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
-        asset_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from asset IDs to their corresponding
-            DirectRelationReference in the data model. This is used to create direct relations for resources that
-            reference assets.
-        source_instance_id_by_external_id (dict[str, DirectRelationReference]): A mapping from source strings to their
-            corresponding DirectRelationReference in the data model. This is used to create direct relations for resources
-            that reference sources.
-        file_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from file IDs to their corresponding
-            DirectRelationReference in the data model. This is used to create direct relations for resources that
-            reference files.
+        direct_relation_cache (DirectRelationCache): Cache for direct relation references.
 
     Returns:
         tuple[NodeApply | EdgeApply, ConversionIssue]: A tuple containing the converted NodeApply and any ConversionIssue encountered.
     """
-    cache = DirectRelationCache(
-        asset=asset_instance_id_by_id, source=source_instance_id_by_external_id, file=file_instance_id_by_id
-    )
     resource_type = _lookup_resource_type(resource)
     dumped = resource.dump()
     try:
@@ -138,7 +198,7 @@ def asset_centric_to_dm(
         view_source.property_mapping,
         resource_type,
         issue=issue,
-        direct_relation_cache=cache,
+        direct_relation_cache=direct_relation_cache,
     )
     sources: list[NodeOrEdgeData] = []
     if properties:
@@ -156,7 +216,7 @@ def asset_centric_to_dm(
     instance: NodeApply | EdgeApply
     if isinstance(instance_id, EdgeId):
         edge_properties = create_edge_properties(
-            dumped, view_source.property_mapping, resource_type, issue, cache, instance_id.space
+            dumped, view_source.property_mapping, resource_type, issue, direct_relation_cache, instance_id.space
         )
         if any(key not in edge_properties for key in ("start_node", "end_node", "type")):
             # Failed conversion of edge properties
@@ -240,7 +300,7 @@ def create_properties(
                 dm_prop.nullable,
                 destination_container_property=(dm_prop.container, dm_prop.container_property_identifier),
                 source_property=(resource_type, prop_json_path),
-                direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
+                direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
             )
         except (ValueError, TypeError, NotImplementedError) as e:
             issue.failed_conversions.append(
@@ -288,7 +348,7 @@ def create_edge_properties(
                     flatten_dump[prop_json_path],
                     DirectRelation(),
                     False,
-                    direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
+                    direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
                 )
             except (ValueError, TypeError, NotImplementedError) as e:
                 issue.failed_conversions.append(

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,14 +1,21 @@
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from typing import Generic
 
 from cognite.client.data_classes._base import (
     T_CogniteResource,
 )
-from cognite.client.data_classes.data_modeling import DirectRelationReference, InstanceApply, View, ViewId
+from cognite.client.data_classes.data_modeling import (
+    EdgeApply,
+    InstanceApply,
+    NodeApply,
+    View,
+    ViewId,
+)
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
-from cognite_toolkit._cdf_tk.commands._migrate.conversion import asset_centric_to_dm
+from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
 from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
@@ -16,7 +23,9 @@ from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
 from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.useful_types import T_AssetCentricResource
+from cognite_toolkit._cdf_tk.utils.useful_types import (
+    T_AssetCentricResource,
+)
 
 
 class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource], ABC):
@@ -31,7 +40,9 @@ class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource],
         pass
 
     @abstractmethod
-    def map(self, source: T_CogniteResource) -> tuple[T_WriteCogniteResource | None, MigrationIssue]:
+    def map(
+        self, source: Sequence[T_CogniteResource]
+    ) -> Sequence[tuple[T_WriteCogniteResource | None, MigrationIssue]]:
         """Map a chunk of source data to the target format.
 
         Args:
@@ -51,10 +62,7 @@ class AssetCentricMapper(
         self.client = client
         self._ingestion_view_by_id: dict[ViewId, View] = {}
         self._view_mapping_by_id: dict[str, ResourceViewMapping] = {}
-        # This is used to keep track of already mapped assets, such that we can creat direct relations
-        # to them from files, events, and time series.
-        self._asset_mapping_by_id: dict[int, DirectRelationReference] = {}
-        self._source_system_mapping_by_id: dict[str, DirectRelationReference] = {}
+        self._direct_relation_cache = DirectRelationCache(client)
 
     def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
         ingestion_view_ids = source_selector.get_ingestion_mappings()
@@ -74,22 +82,23 @@ class AssetCentricMapper(
             raise ToolkitValueError(
                 f"The following ingestion views were not found in Data Modeling: {humanize_collection(missing_views)}"
             )
-        # We just look-up all source system for now. This can be optimized to only
-        # look-up the ones that are actually used in the ingestion views. However, SourceSystem is typically in
-        # the order ~10 instances, so this is not a big deal for now. See task CDF-25974.
-        source_systems = self.client.migration.created_source_system.list(limit=-1)
-        self._source_system_mapping_by_id = {
-            source_system.source: source_system.as_direct_relation_reference() for source_system in source_systems
-        }
-
-        # We look-up all existing asset mappings to be able to create direct relations to already mapped assets.
-        # This is needed in case the migration is run multiple times, or if assets are mapped
-        asset_mappings = self.client.migration.instance_source.list(resource_type="asset", limit=-1)
-        self._asset_mapping_by_id = {mapping.id_: mapping.as_direct_relation_reference() for mapping in asset_mappings}
-
-    def map(self, source: AssetCentricMapping[T_AssetCentricResource]) -> tuple[InstanceApply | None, ConversionIssue]:
+
+    def map(
+        self, source: Sequence[AssetCentricMapping[T_AssetCentricResource]]
+    ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
-        mapping = source.mapping
+        # We update the direct relation cache in bulk for all resources in the chunk.
+        self._direct_relation_cache.update(item.resource for item in source)
+        output: list[tuple[InstanceApply | None, ConversionIssue]] = []
+        for item in source:
+            instance, conversion_issue = self._map_single_item(item)
+            output.append((instance, conversion_issue))
+        return output
+
+    def _map_single_item(
+        self, item: AssetCentricMapping[T_AssetCentricResource]
+    ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
+        mapping = item.mapping
         ingestion_view = mapping.get_ingestion_view()
         try:
             view_source = self._view_mapping_by_id[ingestion_view]
@@ -99,19 +108,12 @@ class AssetCentricMapper(
                 f"Failed to lookup mapping or view for ingestion view '{ingestion_view}'. Did you forget to call .prepare()?"
             ) from e
         instance, conversion_issue = asset_centric_to_dm(
-            source.resource,
+            item.resource,
             instance_id=mapping.instance_id,
             view_source=view_source,
             view_properties=view_properties,
-            asset_instance_id_by_id=self._asset_mapping_by_id,
-            source_instance_id_by_external_id=self._source_system_mapping_by_id,
-            file_instance_id_by_id={},  # Todo implement file direct relations
+            direct_relation_cache=self._direct_relation_cache,
         )
         if mapping.instance_id.space == MISSING_INSTANCE_SPACE:
             conversion_issue.missing_instance_space = f"Missing instance space for dataset ID {mapping.data_set_id!r}"
-
-        if mapping.resource_type == "asset":
-            self._asset_mapping_by_id[mapping.id] = DirectRelationReference(
-                space=mapping.instance_id.space, external_id=mapping.instance_id.external_id
-            )
         return instance, conversion_issue

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -82,9 +82,9 @@ class UploadCommand(ToolkitCommand):
         │   │   └── table2.Table.yaml
         │   └── ...
         ├── datafile1.kind.ndjson # Data file of a specific kind
-        ├── datafile1.Metadata.yaml       # Metadata file for datafile1
+        ├── datafile1.Manifest.yaml       # Manifest for datafile1
         ├── datafile2.kind2.ndjson # Another data file of the same or different kind
-        ├── datafile2.Metadata.yaml       # Metadata file for datafile2
+        ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
         console = Console()

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/constants.py

@@ -182,4 +182,7 @@ MISSING_INSTANCE_SPACE = "<InstanceSpaceMissing>"
 MISSING_EXTERNAL_ID = "INTERNAL_ID_project_{project}_id_{id}"
 
 # Stream Template names
+STREAM_IMMUTABLE_TEMPLATE_NAME = frozenset(("ImmutableTestStream", "BasicArchive"))
+STREAM_MUTABLE_TEMPLATE_NAME = frozenset(("BasicLiveData",))
+
 StreamTemplateName: TypeAlias = Literal["ImmutableTestStream", "BasicArchive", "BasicLiveData"]

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -60,6 +60,7 @@ from ._resource_cruds import (
     SequenceCRUD,
     SequenceRowCRUD,
     SpaceCRUD,
+    StreamCRUD,
     StreamlitCRUD,
     ThreeDModelCRUD,
     TimeSeriesCRUD,
@@ -85,6 +86,8 @@ if not FeatureFlag.is_enabled(Flags.MIGRATE):
     _EXCLUDED_CRUDS.add(ResourceViewMappingCRUD)
 if not FeatureFlag.is_enabled(Flags.SEARCH_CONFIG):
     _EXCLUDED_CRUDS.add(SearchConfigCRUD)
+if not FeatureFlag.is_enabled(Flags.STREAMS):
+    _EXCLUDED_CRUDS.add(StreamCRUD)
 
 
 CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
@@ -132,6 +135,7 @@ ResourceTypes: TypeAlias = Literal[
     "functions",
     "raw",
     "robotics",
+    "streams",
     "streamlit",
     "workflows",
 ]

{cognite_toolkit-0.6.103 → cognite_toolkit-0.6.105}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py

@@ -35,6 +35,7 @@ from .robotics import (
     RoboticMapCRUD,
     RoboticsDataPostProcessingCRUD,
 )
+from .streams import StreamCRUD
 from .three_d_model import ThreeDModelCRUD
 from .timeseries import DatapointSubscriptionCRUD, TimeSeriesCRUD
 from .transformation import TransformationCRUD, TransformationNotificationCRUD, TransformationScheduleCRUD
@@ -82,6 +83,7 @@ __all__ = [
     "SequenceCRUD",
     "SequenceRowCRUD",
     "SpaceCRUD",
+    "StreamCRUD",
     "StreamlitCRUD",
     "ThreeDModelCRUD",
     "TimeSeriesCRUD",

cognite_toolkit-0.6.105/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -0,0 +1,99 @@
+from collections.abc import Hashable, Iterable, Sequence
+from typing import Any, final
+
+from cognite.client.data_classes.capabilities import Capability, StreamsAcl
+from cognite.client.utils.useful_types import SequenceNotStr
+
+from cognite_toolkit._cdf_tk.client.data_classes.streams import (
+    StreamRequest,
+    StreamRequestList,
+    StreamResponse,
+    StreamResponseList,
+)
+from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.resource_classes import StreamYAML
+from cognite_toolkit._cdf_tk.utils.http_client import ToolkitAPIError
+
+from .datamodel import ContainerCRUD
+
+
+@final
+class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse, StreamRequestList, StreamResponseList]):
+    folder_name = "streams"
+    filename_pattern = r"^.*\.Streams$"
+    filetypes = frozenset({"yaml", "yml"})
+    resource_cls = StreamResponse
+    resource_write_cls = StreamRequest
+    list_cls = StreamResponseList
+    list_write_cls = StreamRequestList
+    kind = "Streams"
+    yaml_cls = StreamYAML
+    dependencies = frozenset({ContainerCRUD})
+    _doc_url = "Streams/operation/createStream"
+    support_update = False
+
+    @property
+    def display_name(self) -> str:
+        return "streams"
+
+    @classmethod
+    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> str:
+        if isinstance(item, dict):
+            return item["externalId"]
+        return item.external_id
+
+    @classmethod
+    def dump_id(cls, id: str) -> dict[str, Any]:
+        return {"externalId": id}
+
+    @classmethod
+    def get_required_capability(
+        cls, items: Sequence[StreamRequest] | None, read_only: bool
+    ) -> Capability | list[Capability]:
+        if not items and items is not None:
+            return []
+
+        actions = (
+            [StreamsAcl.Action.Read]
+            if read_only
+            else [StreamsAcl.Action.Read, StreamsAcl.Action.Create, StreamsAcl.Action.Delete]
+        )
+        return StreamsAcl(actions, StreamsAcl.Scope.All())
+
+    def create(self, items: StreamRequestList) -> StreamResponseList:
+        created = self.client.streams.create(list(items))
+        return StreamResponseList(created)
+
+    def retrieve(self, ids: SequenceNotStr[str]) -> StreamResponseList:
+        retrieved: list[StreamResponse] = []
+        for _id in ids:
+            try:
+                _resp = self.client.streams.retrieve(_id)
+            except ToolkitAPIError:
+                continue
+            if _resp is not None:
+                retrieved.append(_resp)
+        return StreamResponseList(retrieved)
+
+    def delete(self, ids: SequenceNotStr[str]) -> int:
+        count = 0
+        for _id in ids:
+            try:
+                self.client.streams.delete(_id)
+            except ToolkitAPIError:
+                continue
+            count += 1
+        return count
+
+    def _iterate(
+        self,
+        data_set_external_id: str | None = None,
+        space: str | None = None,
+        parent_ids: list[Hashable] | None = None,
+    ) -> Iterable[StreamResponse]:
+        if data_set_external_id or space or parent_ids:
+            # These filters are not supported for Streams
+            return iter([])
+
+        all_streams = self.client.streams.list()
+        return iter(all_streams)