cognite-toolkit 0.6.104__tar.gz → 0.6.106__tar.gz

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.104
+Version: 0.6.106
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cdf.toml

@@ -22,6 +22,7 @@ upload = true
 function-cognite-file = true
 external-libraries = true
 v07 = true
+streams = true
 
 [plugins]
 run = true
@@ -32,7 +33,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.104"
+version = "0.6.106"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/client/data_classes/streams.py

@@ -29,13 +29,12 @@ class StreamRequestList(UserList[StreamRequest], ResourceRequestListProtocol):
     """List of Stream request resources."""
 
     _RESOURCE = StreamRequest
-    items: list[StreamRequest]
 
     def __init__(self, initlist: list[StreamRequest] | None = None, **_: Any) -> None:
         super().__init__(initlist or [])
 
     def dump(self, camel_case: bool = True) -> list[dict[str, Any]]:
-        return [item.dump(camel_case) for item in self.items]
+        return [item.dump(camel_case) for item in self.data]
 
     @classmethod
     def load(cls, data: list[dict[str, Any]], cognite_client: CogniteClient | None = None) -> "StreamRequestList":
@@ -78,19 +77,25 @@ class StreamResponse(ResponseResource["StreamRequest"]):
 
     external_id: str
     created_time: int
-    created_from_template: StreamTemplateName | None = None
-    type: Literal["Mutable", "Immutable"] | None = None
+    created_from_template: StreamTemplateName
+    type: Literal["Mutable", "Immutable"]
     settings: StreamSettings | None = None
 
     def as_request_resource(self) -> StreamRequest:
-        template_name = self.created_from_template or "BasicArchive"
-        return StreamRequest(
-            external_id=self.external_id,
-            settings={"template": {"name": template_name}},
+        return StreamRequest.model_validate(
+            {
+                "externalId": self.external_id,
+                "settings": {"template": {"name": self.created_from_template}},
+            }
         )
 
-    def as_write(self) -> Self:
-        return self
+    def as_write(self) -> StreamRequest:
+        return StreamRequest.model_validate(
+            {
+                "externalId": self.external_id,
+                "settings": {"template": {"name": self.created_from_template}},
+            }
+        )
 
 
 class StreamResponseList(UserList[StreamResponse], ResourceResponseListProtocol):

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -15,7 +15,10 @@ from cognite.client.data_classes.data_modeling.views import ViewProperty
 from cognite.client.utils._identifier import InstanceId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.migration import (
+    AssetCentricId,
+    ResourceViewMappingApply,
+)
 from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
 from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
     asset_centric_convert_to_primary_property,
@@ -163,7 +166,7 @@ class DirectRelationCache:
 def asset_centric_to_dm(
     resource: AssetCentricResourceExtended,
     instance_id: InstanceId,
-    view_source: ResourceViewMapping,
+    view_source: ResourceViewMappingApply,
     view_properties: dict[str, ViewProperty],
     direct_relation_cache: DirectRelationCache,
 ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
@@ -172,7 +175,7 @@ def asset_centric_to_dm(
     Args:
         resource (CogniteResource): The asset-centric resource to convert.
         instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
-        view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
+        view_source (ResourceViewMappingApply): The view source defining how to map the resource to the data model.
         view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
         direct_relation_cache (DirectRelationCache): Cache for direct relation references.
 

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -14,7 +14,11 @@ from pydantic import BaseModel, BeforeValidator, field_validator, model_validato
 from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
 from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
-from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
+    ASSET_ANNOTATIONS_ID,
+    FILE_ANNOTATIONS_ID,
+    create_default_mappings,
+)
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
 from cognite_toolkit._cdf_tk.utils.useful_types import (
@@ -182,6 +186,18 @@ class FileMapping(MigrationMapping):
 class AnnotationMapping(MigrationMapping):
     resource_type: Literal["annotation"] = "annotation"
     instance_id: EdgeId
+    annotation_type: Literal["diagrams.AssetLink", "diagrams.FileLink"] | None = None
+
+    def get_ingestion_view(self) -> str:
+        """Get the ingestion view for the mapping. If not specified, return the default ingestion view."""
+        if self.ingestion_view:
+            return self.ingestion_view
+        elif self.annotation_type == "diagrams.AssetLink":
+            return ASSET_ANNOTATIONS_ID
+        elif self.annotation_type == "diagrams.FileLink":
+            return FILE_ANNOTATIONS_ID
+        else:
+            raise ToolkitValueError("Cannot determine default ingestion view for annotation without annotation_type")
 
     @field_validator("instance_id", mode="before")
     def _validate_instance_id(cls, v: Any) -> Any:

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -14,9 +14,10 @@ from cognite.client.data_classes.data_modeling import (
 )
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
 from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
@@ -61,20 +62,22 @@ class AssetCentricMapper(
     def __init__(self, client: ToolkitClient) -> None:
         self.client = client
         self._ingestion_view_by_id: dict[ViewId, View] = {}
-        self._view_mapping_by_id: dict[str, ResourceViewMapping] = {}
+        self._view_mapping_by_id: dict[str, ResourceViewMappingApply] = {}
         self._direct_relation_cache = DirectRelationCache(client)
 
     def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
         ingestion_view_ids = source_selector.get_ingestion_mappings()
         ingestion_views = self.client.migration.resource_view_mapping.retrieve(ingestion_view_ids)
-        self._view_mapping_by_id = {view.external_id: view for view in ingestion_views}
+        defaults = {mapping.external_id: mapping for mapping in create_default_mappings()}
+        # Custom mappings from CDF override the default mappings
+        self._view_mapping_by_id = defaults | {view.external_id: view.as_write() for view in ingestion_views}
         missing_mappings = set(ingestion_view_ids) - set(self._view_mapping_by_id.keys())
         if missing_mappings:
             raise ToolkitValueError(
                 f"The following ingestion views were not found: {humanize_collection(missing_mappings)}"
             )
 
-        view_ids = list({view.view_id for view in ingestion_views})
+        view_ids = list({mapping.view_id for mapping in self._view_mapping_by_id.values()})
         views = self.client.data_modeling.views.retrieve(view_ids)
         self._ingestion_view_by_id = {view.as_id(): view for view in views}
         missing_views = set(view_ids) - set(self._ingestion_view_by_id.keys())

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py

@@ -93,7 +93,7 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
             view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
                 # We are ignoring the symbol region in the default mapping.
-                "annotatedResource.id": "edge.startNode",
+                "annotatedResourceId": "edge.startNode",
                 "annotationType": "edge.type.externalId",
                 "creatingUser": "sourceCreatedUser",
                 "creatingApp": "sourceId",
@@ -114,16 +114,16 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
         ResourceViewMappingApply(
             external_id=FILE_ANNOTATIONS_ID,
             resource_type="fileAnnotation",
-            view_id=ViewId("cdf_cdm", "CogniteFileAnnotation", "v1"),
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
-                "annotatedResource.id": "edge.startNode",
+                "annotatedResourceId": "edge.startNode",
                 "annotationType": "edge.type.externalId",
                 "creatingUser": "sourceCreatedUser",
                 "creatingApp": "sourceId",
                 "creatingAppVersion": "sourceContext",
                 "status": "status",
-                "data.fileRef.id": "edge.startNode",
-                "data.fileRef.externalId": "edge.startNode",
+                "data.fileRef.id": "edge.endNode",
+                "data.fileRef.externalId": "edge.endNode",
                 "data.description": "description",
                 "data.pageNumber": "startNodePageNumber",
                 "data.textRegion.confidence": "confidence",

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py

@@ -33,6 +33,7 @@ from .data_classes import (
     MigrationMappingList,
 )
 from .data_model import INSTANCE_SOURCE_VIEW_ID
+from .default_mappings import ASSET_ANNOTATIONS_ID, FILE_ANNOTATIONS_ID
 from .selectors import AssetCentricMigrationSelector, MigrateDataSetSelector, MigrationCSVFileSelector
 
 
@@ -213,6 +214,16 @@ class AssetCentricMigrationIO(
 class AnnotationMigrationIO(
     UploadableStorageIO[AssetCentricMigrationSelector, AssetCentricMapping[Annotation], InstanceApply]
 ):
+    """IO class for migrating Annotations.
+
+    Args:
+        client: The ToolkitClient to use for CDF interactions.
+        instance_space: The instance space to use for the migrated annotations.
+        default_asset_annotation_mapping: The default ingestion mapping to use for asset-linked annotations.
+        default_file_annotation_mapping: The default ingestion mappingto use for file-linked annotations.
+
+    """
+
     KIND = "AnnotationMigration"
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".parquet", ".csv", ".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
@@ -220,17 +231,28 @@ class AnnotationMigrationIO(
     CHUNK_SIZE = 1000
     UPLOAD_ENDPOINT = InstanceIO.UPLOAD_ENDPOINT
 
-    def __init__(self, client: ToolkitClient, instance_space: str | None = None) -> None:
+    def __init__(
+        self,
+        client: ToolkitClient,
+        instance_space: str | None = None,
+        default_asset_annotation_mapping: str = ASSET_ANNOTATIONS_ID,
+        default_file_annotation_mapping: str = FILE_ANNOTATIONS_ID,
+    ) -> None:
         super().__init__(client)
         self.annotation_io = AnnotationIO(client)
         self.instance_space = instance_space
+        self.default_asset_annotation_mapping = default_asset_annotation_mapping
+        self.default_file_annotation_mapping = default_file_annotation_mapping
 
     def as_id(self, item: AssetCentricMapping[Annotation]) -> str:
         return f"Annotation_{item.mapping.id}"
 
     def count(self, selector: AssetCentricMigrationSelector) -> int | None:
-        # There is no efficient way to count annotations in CDF.
-        return None
+        if isinstance(selector, MigrationCSVFileSelector):
+            return len(selector.items)
+        else:
+            # There is no efficient way to count annotations in CDF.
+            return None
 
     def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterable[Page]:
         if isinstance(selector, MigrateDataSetSelector):
@@ -253,8 +275,10 @@ class AnnotationMigrationIO(
                 mapping = AnnotationMapping(
                     instance_id=EdgeId(space=self.instance_space, external_id=f"annotation_{resource.id!r}"),
                     id=resource.id,
-                    ingestion_view=selector.ingestion_mapping,
+                    ingestion_view=self._get_mapping(selector.ingestion_mapping, resource),
                     preferred_consumer_view=selector.preferred_consumer_view,
+                    # The PySDK is poorly typed.
+                    annotation_type=resource.annotation_type,  # type: ignore[arg-type]
                 )
                 mapping_list.append(AssetCentricMapping(mapping=mapping, resource=resource))
             yield mapping_list
@@ -275,6 +299,7 @@ class AnnotationMigrationIO(
                 if resource is None:
                     not_found += 1
                     continue
+                mapping.ingestion_view = self._get_mapping(mapping.ingestion_view, resource)
                 chunk.append(AssetCentricMapping(mapping=mapping, resource=resource))
             if chunk:
                 yield chunk
@@ -284,6 +309,21 @@ class AnnotationMigrationIO(
                     f"Could not find {not_found} annotations referenced in the CSV file. They will be skipped during migration."
                 ).print_warning(include_timestamp=True, console=self.client.console)
 
+    def _get_mapping(self, current_mapping: str | None, resource: Annotation) -> str:
+        try:
+            return (
+                current_mapping
+                or {
+                    "diagrams.AssetLink": self.default_asset_annotation_mapping,
+                    "diagrams.FileLink": self.default_file_annotation_mapping,
+                }[resource.annotation_type]
+            )
+        except KeyError as e:
+            raise ToolkitValueError(
+                f"Could not determine default ingestion view for annotation type '{resource.annotation_type}'. "
+                "Please specify the ingestion view explicitly in the CSV file."
+            ) from e
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> InstanceApply:
         raise NotImplementedError("Deserializing Annotation Migrations from JSON is not supported.")
 

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -27,7 +27,7 @@ from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSever
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_READ_CLS_BY_FORMAT, FileReader
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
@@ -87,7 +87,7 @@ class UploadCommand(ToolkitCommand):
         ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
-        console = Console()
+        console = client.console
         data_files_by_selector = self._find_data_files(input_dir, kind)
 
         self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
@@ -225,49 +225,44 @@ class UploadCommand(ToolkitCommand):
                 io = self._create_selected_io(selector, datafiles[0], client)
                 if io is None:
                     continue
-                for data_file in datafiles:
-                    file_display = self._path_as_display_name(data_file)
-                    if verbose:
-                        console.print(f"{action} {selector.display_name} from {file_display.as_posix()!r}")
-                    reader = FileReader.from_filepath(data_file)
-                    is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
-                    if is_table and not isinstance(io, TableUploadableStorageIO):
-                        raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
-                    tracker = ProgressTracker[str]([self._UPLOAD])
-                    executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
-                        download_iterable=io.read_chunks(reader),
-                        process=partial(io.rows_to_data, selector=selector)
-                        if is_table and isinstance(io, TableUploadableStorageIO)
-                        else io.json_chunk_to_data,
-                        write=partial(
-                            self._upload_items,
-                            upload_client=upload_client,
-                            io=io,
-                            dry_run=dry_run,
-                            selector=selector,
-                            tracker=tracker,
-                            console=console,
-                        ),
-                        iteration_count=None,
-                        max_queue_size=self._MAX_QUEUE_SIZE,
-                        download_description=f"Reading {file_count:,}/{total_file_count + 1:,}: {file_display.as_posix()!s}",
-                        process_description="Processing",
-                        write_description=f"{action} {selector.display_name!r}",
+                reader = MultiFileReader(datafiles)
+                if reader.is_table and not isinstance(io, TableUploadableStorageIO):
+                    raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                tracker = ProgressTracker[str]([self._UPLOAD])
+                executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
+                    download_iterable=io.read_chunks(reader),
+                    process=partial(io.rows_to_data, selector=selector)
+                    if reader.is_table and isinstance(io, TableUploadableStorageIO)
+                    else io.json_chunk_to_data,
+                    write=partial(
+                        self._upload_items,
+                        upload_client=upload_client,
+                        io=io,
+                        dry_run=dry_run,
+                        selector=selector,
+                        tracker=tracker,
                         console=console,
-                    )
-                    executor.run()
-                    file_count += 1
-                    executor.raise_on_error()
-                    final_action = "Uploaded" if not dry_run else "Would upload"
-                    suffix = " successfully" if not dry_run else ""
-                    results = tracker.aggregate()
-                    success = results.get((self._UPLOAD, "success"), 0)
-                    failed = results.get((self._UPLOAD, "failed"), 0)
-                    if failed > 0:
-                        suffix += f", {failed:,} failed"
-                    console.print(
-                        f"{final_action} {success:,} {selector.display_name} from {file_display.as_posix()!r}{suffix}."
-                    )
+                    ),
+                    iteration_count=None,
+                    max_queue_size=self._MAX_QUEUE_SIZE,
+                    download_description=f"Reading {selector.display_name!r} files",
+                    process_description="Processing",
+                    write_description=f"{action} {selector.display_name!r}",
+                    console=console,
+                )
+                executor.run()
+                file_count += len(datafiles)
+                executor.raise_on_error()
+                final_action = "Uploaded" if not dry_run else "Would upload"
+                suffix = " successfully" if not dry_run else ""
+                results = tracker.aggregate()
+                success = results.get((self._UPLOAD, "success"), 0)
+                failed = results.get((self._UPLOAD, "failed"), 0)
+                if failed > 0:
+                    suffix += f", {failed:,} failed"
+                console.print(
+                    f"{final_action} {success:,} {selector.display_name} from {len(datafiles)} files{suffix}."
+                )
 
     @staticmethod
     def _path_as_display_name(input_path: Path, cwd: Path = Path.cwd()) -> Path:

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/constants.py

@@ -182,4 +182,7 @@ MISSING_INSTANCE_SPACE = "<InstanceSpaceMissing>"
 MISSING_EXTERNAL_ID = "INTERNAL_ID_project_{project}_id_{id}"
 
 # Stream Template names
+STREAM_IMMUTABLE_TEMPLATE_NAME = frozenset(("ImmutableTestStream", "BasicArchive"))
+STREAM_MUTABLE_TEMPLATE_NAME = frozenset(("BasicLiveData",))
+
 StreamTemplateName: TypeAlias = Literal["ImmutableTestStream", "BasicArchive", "BasicLiveData"]

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -60,6 +60,7 @@ from ._resource_cruds import (
     SequenceCRUD,
     SequenceRowCRUD,
     SpaceCRUD,
+    StreamCRUD,
     StreamlitCRUD,
     ThreeDModelCRUD,
     TimeSeriesCRUD,
@@ -85,6 +86,8 @@ if not FeatureFlag.is_enabled(Flags.MIGRATE):
     _EXCLUDED_CRUDS.add(ResourceViewMappingCRUD)
 if not FeatureFlag.is_enabled(Flags.SEARCH_CONFIG):
     _EXCLUDED_CRUDS.add(SearchConfigCRUD)
+if not FeatureFlag.is_enabled(Flags.STREAMS):
+    _EXCLUDED_CRUDS.add(StreamCRUD)
 
 
 CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
@@ -132,6 +135,7 @@ ResourceTypes: TypeAlias = Literal[
     "functions",
     "raw",
     "robotics",
+    "streams",
     "streamlit",
     "workflows",
 ]

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py

@@ -35,6 +35,7 @@ from .robotics import (
     RoboticMapCRUD,
     RoboticsDataPostProcessingCRUD,
 )
+from .streams import StreamCRUD
 from .three_d_model import ThreeDModelCRUD
 from .timeseries import DatapointSubscriptionCRUD, TimeSeriesCRUD
 from .transformation import TransformationCRUD, TransformationNotificationCRUD, TransformationScheduleCRUD
@@ -82,6 +83,7 @@ __all__ = [
     "SequenceCRUD",
     "SequenceRowCRUD",
     "SpaceCRUD",
+    "StreamCRUD",
     "StreamlitCRUD",
     "ThreeDModelCRUD",
     "TimeSeriesCRUD",

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py

@@ -328,6 +328,7 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
 
     def dump_resource(self, resource: Container, local: dict[str, Any] | None = None) -> dict[str, Any]:
         dumped = resource.as_write().dump()
+        has_local = local is not None
         local = local or {}
         for key in ["constraints", "indexes"]:
             if not dumped.get(key) and key not in local:
@@ -339,15 +340,16 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
                 continue
             local_prop = local_prop_by_id[prop_id]
             for key, default in [("immutable", False), ("autoIncrement", False), ("nullable", True)]:
-                if cdf_prop.get(key) is default and key not in local_prop:
+                if has_local and cdf_prop.get(key) is default and key not in local_prop:
                     cdf_prop.pop(key, None)
             cdf_type = cdf_prop.get("type", {})
             local_type = local_prop.get("type", {})
             for key, type_default in [("list", False), ("collation", "ucs_basic")]:
-                if cdf_type.get(key) == type_default and key not in local_type:
+                if has_local and cdf_type.get(key) == type_default and key not in local_type:
                     cdf_type.pop(key, None)
-        if "usedFor" not in local:
-            dumped.pop("usedFor", None)
+            if has_local and "usedFor" not in local and dumped.get("usedFor") == "node":
+                # Only drop if set to default by server.
+                dumped.pop("usedFor", None)
         return dumped
 
     def create(self, items: Sequence[ContainerApply]) -> ContainerList:

cognite_toolkit-0.6.106/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -0,0 +1,99 @@
+from collections.abc import Hashable, Iterable, Sequence
+from typing import Any, final
+
+from cognite.client.data_classes.capabilities import Capability, StreamsAcl
+from cognite.client.utils.useful_types import SequenceNotStr
+
+from cognite_toolkit._cdf_tk.client.data_classes.streams import (
+    StreamRequest,
+    StreamRequestList,
+    StreamResponse,
+    StreamResponseList,
+)
+from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
+from cognite_toolkit._cdf_tk.resource_classes import StreamYAML
+from cognite_toolkit._cdf_tk.utils.http_client import ToolkitAPIError
+
+from .datamodel import ContainerCRUD
+
+
+@final
+class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse, StreamRequestList, StreamResponseList]):
+    folder_name = "streams"
+    filename_pattern = r"^.*\.Streams$"
+    filetypes = frozenset({"yaml", "yml"})
+    resource_cls = StreamResponse
+    resource_write_cls = StreamRequest
+    list_cls = StreamResponseList
+    list_write_cls = StreamRequestList
+    kind = "Streams"
+    yaml_cls = StreamYAML
+    dependencies = frozenset({ContainerCRUD})
+    _doc_url = "Streams/operation/createStream"
+    support_update = False
+
+    @property
+    def display_name(self) -> str:
+        return "streams"
+
+    @classmethod
+    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> str:
+        if isinstance(item, dict):
+            return item["externalId"]
+        return item.external_id
+
+    @classmethod
+    def dump_id(cls, id: str) -> dict[str, Any]:
+        return {"externalId": id}
+
+    @classmethod
+    def get_required_capability(
+        cls, items: Sequence[StreamRequest] | None, read_only: bool
+    ) -> Capability | list[Capability]:
+        if not items and items is not None:
+            return []
+
+        actions = (
+            [StreamsAcl.Action.Read]
+            if read_only
+            else [StreamsAcl.Action.Read, StreamsAcl.Action.Create, StreamsAcl.Action.Delete]
+        )
+        return StreamsAcl(actions, StreamsAcl.Scope.All())
+
+    def create(self, items: StreamRequestList) -> StreamResponseList:
+        created = self.client.streams.create(list(items))
+        return StreamResponseList(created)
+
+    def retrieve(self, ids: SequenceNotStr[str]) -> StreamResponseList:
+        retrieved: list[StreamResponse] = []
+        for _id in ids:
+            try:
+                _resp = self.client.streams.retrieve(_id)
+            except ToolkitAPIError:
+                continue
+            if _resp is not None:
+                retrieved.append(_resp)
+        return StreamResponseList(retrieved)
+
+    def delete(self, ids: SequenceNotStr[str]) -> int:
+        count = 0
+        for _id in ids:
+            try:
+                self.client.streams.delete(_id)
+            except ToolkitAPIError:
+                continue
+            count += 1
+        return count
+
+    def _iterate(
+        self,
+        data_set_external_id: str | None = None,
+        space: str | None = None,
+        parent_ids: list[Hashable] | None = None,
+    ) -> Iterable[StreamResponse]:
+        if data_set_external_id or space or parent_ids:
+            # These filters are not supported for Streams
+            return iter([])
+
+        all_streams = self.client.streams.list()
+        return iter(all_streams)

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/resource_classes/__init__.py

@@ -41,6 +41,7 @@ from .securitycategories import SecurityCategoriesYAML
 from .sequence import SequenceRowYAML, SequenceYAML
 from .space import SpaceYAML
 from .streamlit_ import StreamlitYAML
+from .streams import StreamYAML
 from .threedmodels import ThreeDModelYAML
 from .timeseries import TimeSeriesYAML
 from .transformation_notification import TransformationNotificationYAML
@@ -92,6 +93,7 @@ __all__ = [
     "SequenceRowYAML",
     "SequenceYAML",
     "SpaceYAML",
+    "StreamYAML",
     "StreamlitYAML",
     "TableYAML",
     "ThreeDModelYAML",

{cognite_toolkit-0.6.104 → cognite_toolkit-0.6.106}/cognite_toolkit/_cdf_tk/storageio/_annotations.py

@@ -26,21 +26,23 @@ class AnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
     def stream_data(self, selector: AssetCentricSelector, limit: int | None = None) -> Iterable[Page[Annotation]]:
         total = 0
         for file_chunk in FileMetadataIO(self.client).stream_data(selector, None):
-            # Todo Support pagination. This is missing in the SDK.
-            results = self.client.annotations.list(
-                filter=AnnotationFilter(
-                    annotated_resource_type="file",
-                    annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+            for annotation_type in ["diagrams.AssetLink", "diagrams.FileLink"]:
+                # Todo Support pagination. This is missing in the SDK.
+                results = self.client.annotations.list(
+                    filter=AnnotationFilter(
+                        annotated_resource_type="file",
+                        annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+                        annotation_type=annotation_type,
+                    )
                 )
-            )
-            if limit is not None and total + len(results) > limit:
-                results = results[: limit - total]
-
-            for chunk in chunker_sequence(results, self.CHUNK_SIZE):
-                yield Page(worker_id="main", items=chunk)
-                total += len(chunk)
-            if limit is not None and total >= limit:
-                break
+                if limit is not None and total + len(results) > limit:
+                    results = results[: limit - total]
+
+                for chunk in chunker_sequence(results, self.CHUNK_SIZE):
+                    yield Page(worker_id="main", items=chunk)
+                    total += len(chunk)
+                if limit is not None and total >= limit:
+                    break
 
     def count(self, selector: AssetCentricSelector) -> int | None:
         """There is no efficient way to count annotations in CDF."""