cognite-toolkit 0.6.105__py3-none-any.whl → 0.6.107__py3-none-any.whl

cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -4,6 +4,7 @@ from typing import Annotated, Any
 
 import questionary
 import typer
+from cognite.client.data_classes import Annotation
 from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
@@ -19,6 +20,7 @@ from cognite_toolkit._cdf_tk.commands._migrate.creators import (
 )
 from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
+    AnnotationMigrationIO,
     AssetCentricMigrationIO,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
@@ -31,6 +33,7 @@ from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
+    FileMetadataInteractiveSelect,
     ResourceViewMappingInteractiveSelect,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
@@ -49,6 +52,7 @@ class MigrateApp(typer.Typer):
         self.command("events")(self.events)
         self.command("timeseries")(self.timeseries)
         self.command("files")(self.files)
+        self.command("annotations")(self.annotations)
         self.command("canvas")(self.canvas)
         # Uncomment when infield v2 config migration is ready
         # self.command("infield-configs")(self.infield_configs)
@@ -694,6 +698,149 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @classmethod
+    def annotations(
+        cls,
+        ctx: typer.Context,
+        mapping_file: Annotated[
+            Path | None,
+            typer.Option(
+                "--mapping-file",
+                "-m",
+                help="Path to the mapping file that contains the mapping from Annotations to CogniteDiagramAnnotation. "
+                "This file is expected to have the following columns: [id, space, externalId, ingestionView].",
+            ),
+        ] = None,
+        data_set_id: Annotated[
+            str | None,
+            typer.Option(
+                "--data-set-id",
+                "-s",
+                help="The data set ID to select for the annotations to migrate. If not provided and the mapping file is not provided, "
+                "an interactive selection will be performed to select the data set to migrate annotations from.",
+            ),
+        ] = None,
+        instance_space: Annotated[
+            str | None,
+            typer.Option(
+                "--instance-space",
+                "-i",
+                help="The instance space to use for the migrated annotations. Required when using --data-set-id.",
+            ),
+        ] = None,
+        asset_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--asset-annotation-mapping",
+                "-a",
+                help="The ingestion mapping to use for asset-linked annotations. If not provided, "
+                "the default mapping (cdf_asset_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        file_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--file-annotation-mapping",
+                "-f",
+                help="The ingestion mapping to use for file-linked annotations. If not provided, "
+                "the default mapping (cdf_file_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where logs will be stored. If the directory does not exist, it will be created.",
+            ),
+        ] = Path(f"migration_logs_{TODAY!s}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Annotations to CogniteDiagramAnnotation edges in data modeling.
+
+        Annotations are diagram annotations that link assets or files to other resources. This command
+        migrates them to edges in the data modeling space, preserving the relationships and metadata.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+
+        if data_set_id is not None and mapping_file is not None:
+            raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
+        elif mapping_file is not None:
+            selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(
+                datafile=mapping_file, kind="Annotations"
+            )
+            annotation_io = AnnotationMigrationIO(client)
+        elif data_set_id is not None:
+            if instance_space is None:
+                raise typer.BadParameter("--instance-space is required when using --data-set-id")
+            selected = MigrateDataSetSelector(data_set_external_id=data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+        else:
+            # Interactive selection
+            selector = FileMetadataInteractiveSelect(client, "migrate")
+            selected_data_set_id = selector.select_data_set(allow_empty=False)
+            dm_selector = DataModelingSelect(client, "migrate")
+            selected_instance_space = dm_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the annotations?",
+                include_empty=True,
+            )
+            if selected_instance_space is None:
+                raise typer.Abort()
+            asset_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate asset annotations")
+            asset_annotation_mapping = asset_annotations_selector.select_resource_view_mapping(
+                resource_type="assetAnnotation",
+            ).external_id
+            file_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate file annotations")
+            file_annotation_mapping = file_annotations_selector.select_resource_view_mapping(
+                resource_type="fileAnnotation",
+            ).external_id
+
+            selected = MigrateDataSetSelector(data_set_external_id=selected_data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=selected_instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [dry_run, verbose]):
+                raise typer.Abort()
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=selected,
+                data=annotation_io,
+                mapper=AssetCentricMapper[Annotation](client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def canvas(
         ctx: typer.Context,

cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -15,7 +15,10 @@ from cognite.client.data_classes.data_modeling.views import ViewProperty
 from cognite.client.utils._identifier import InstanceId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.migration import (
+    AssetCentricId,
+    ResourceViewMappingApply,
+)
 from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
 from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
     asset_centric_convert_to_primary_property,
@@ -163,7 +166,7 @@ class DirectRelationCache:
 def asset_centric_to_dm(
     resource: AssetCentricResourceExtended,
     instance_id: InstanceId,
-    view_source: ResourceViewMapping,
+    view_source: ResourceViewMappingApply,
     view_properties: dict[str, ViewProperty],
     direct_relation_cache: DirectRelationCache,
 ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
@@ -172,7 +175,7 @@ def asset_centric_to_dm(
     Args:
         resource (CogniteResource): The asset-centric resource to convert.
         instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
-        view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
+        view_source (ResourceViewMappingApply): The view source defining how to map the resource to the data model.
         view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
         direct_relation_cache (DirectRelationCache): Cache for direct relation references.
 

cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -14,7 +14,11 @@ from pydantic import BaseModel, BeforeValidator, field_validator, model_validato
 from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
 from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
-from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
+    ASSET_ANNOTATIONS_ID,
+    FILE_ANNOTATIONS_ID,
+    create_default_mappings,
+)
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
 from cognite_toolkit._cdf_tk.utils.useful_types import (
@@ -182,6 +186,18 @@ class FileMapping(MigrationMapping):
 class AnnotationMapping(MigrationMapping):
     resource_type: Literal["annotation"] = "annotation"
     instance_id: EdgeId
+    annotation_type: Literal["diagrams.AssetLink", "diagrams.FileLink"] | None = None
+
+    def get_ingestion_view(self) -> str:
+        """Get the ingestion view for the mapping. If not specified, return the default ingestion view."""
+        if self.ingestion_view:
+            return self.ingestion_view
+        elif self.annotation_type == "diagrams.AssetLink":
+            return ASSET_ANNOTATIONS_ID
+        elif self.annotation_type == "diagrams.FileLink":
+            return FILE_ANNOTATIONS_ID
+        else:
+            raise ToolkitValueError("Cannot determine default ingestion view for annotation without annotation_type")
 
     @field_validator("instance_id", mode="before")
     def _validate_instance_id(cls, v: Any) -> Any:

cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -14,9 +14,10 @@ from cognite.client.data_classes.data_modeling import (
 )
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
 from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
@@ -24,7 +25,7 @@ from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
-    T_AssetCentricResource,
+    T_AssetCentricResourceExtended,
 )
 
 
@@ -56,25 +57,27 @@ class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource],
 
 
 class AssetCentricMapper(
-    DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResource], InstanceApply]
+    DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResourceExtended], InstanceApply]
 ):
     def __init__(self, client: ToolkitClient) -> None:
         self.client = client
         self._ingestion_view_by_id: dict[ViewId, View] = {}
-        self._view_mapping_by_id: dict[str, ResourceViewMapping] = {}
+        self._view_mapping_by_id: dict[str, ResourceViewMappingApply] = {}
         self._direct_relation_cache = DirectRelationCache(client)
 
     def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
         ingestion_view_ids = source_selector.get_ingestion_mappings()
         ingestion_views = self.client.migration.resource_view_mapping.retrieve(ingestion_view_ids)
-        self._view_mapping_by_id = {view.external_id: view for view in ingestion_views}
+        defaults = {mapping.external_id: mapping for mapping in create_default_mappings()}
+        # Custom mappings from CDF override the default mappings
+        self._view_mapping_by_id = defaults | {view.external_id: view.as_write() for view in ingestion_views}
         missing_mappings = set(ingestion_view_ids) - set(self._view_mapping_by_id.keys())
         if missing_mappings:
             raise ToolkitValueError(
                 f"The following ingestion views were not found: {humanize_collection(missing_mappings)}"
             )
 
-        view_ids = list({view.view_id for view in ingestion_views})
+        view_ids = list({mapping.view_id for mapping in self._view_mapping_by_id.values()})
         views = self.client.data_modeling.views.retrieve(view_ids)
         self._ingestion_view_by_id = {view.as_id(): view for view in views}
         missing_views = set(view_ids) - set(self._ingestion_view_by_id.keys())
@@ -84,7 +87,7 @@ class AssetCentricMapper(
             )
 
     def map(
-        self, source: Sequence[AssetCentricMapping[T_AssetCentricResource]]
+        self, source: Sequence[AssetCentricMapping[T_AssetCentricResourceExtended]]
     ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
         # We update the direct relation cache in bulk for all resources in the chunk.
@@ -96,7 +99,7 @@ class AssetCentricMapper(
         return output
 
     def _map_single_item(
-        self, item: AssetCentricMapping[T_AssetCentricResource]
+        self, item: AssetCentricMapping[T_AssetCentricResourceExtended]
     ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
         mapping = item.mapping
         ingestion_view = mapping.get_ingestion_view()

cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py

@@ -93,7 +93,7 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
             view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
                 # We are ignoring the symbol region in the default mapping.
-                "annotatedResource.id": "edge.startNode",
+                "annotatedResourceId": "edge.startNode",
                 "annotationType": "edge.type.externalId",
                 "creatingUser": "sourceCreatedUser",
                 "creatingApp": "sourceId",
@@ -114,16 +114,16 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
         ResourceViewMappingApply(
             external_id=FILE_ANNOTATIONS_ID,
             resource_type="fileAnnotation",
-            view_id=ViewId("cdf_cdm", "CogniteFileAnnotation", "v1"),
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
-                "annotatedResource.id": "edge.startNode",
+                "annotatedResourceId": "edge.startNode",
                 "annotationType": "edge.type.externalId",
                 "creatingUser": "sourceCreatedUser",
                 "creatingApp": "sourceId",
                 "creatingAppVersion": "sourceContext",
                 "status": "status",
-                "data.fileRef.id": "edge.startNode",
-                "data.fileRef.externalId": "edge.startNode",
+                "data.fileRef.id": "edge.endNode",
+                "data.fileRef.externalId": "edge.endNode",
                 "data.description": "description",
                 "data.pageNumber": "startNodePageNumber",
                 "data.textRegion.confidence": "confidence",

cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py

@@ -33,6 +33,7 @@ from .data_classes import (
     MigrationMappingList,
 )
 from .data_model import INSTANCE_SOURCE_VIEW_ID
+from .default_mappings import ASSET_ANNOTATIONS_ID, FILE_ANNOTATIONS_ID
 from .selectors import AssetCentricMigrationSelector, MigrateDataSetSelector, MigrationCSVFileSelector
 
 
@@ -213,6 +214,16 @@ class AssetCentricMigrationIO(
 class AnnotationMigrationIO(
     UploadableStorageIO[AssetCentricMigrationSelector, AssetCentricMapping[Annotation], InstanceApply]
 ):
+    """IO class for migrating Annotations.
+
+    Args:
+        client: The ToolkitClient to use for CDF interactions.
+        instance_space: The instance space to use for the migrated annotations.
+        default_asset_annotation_mapping: The default ingestion mapping to use for asset-linked annotations.
+        default_file_annotation_mapping: The default ingestion mappingto use for file-linked annotations.
+
+    """
+
     KIND = "AnnotationMigration"
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".parquet", ".csv", ".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
@@ -220,17 +231,28 @@ class AnnotationMigrationIO(
     CHUNK_SIZE = 1000
     UPLOAD_ENDPOINT = InstanceIO.UPLOAD_ENDPOINT
 
-    def __init__(self, client: ToolkitClient, instance_space: str | None = None) -> None:
+    def __init__(
+        self,
+        client: ToolkitClient,
+        instance_space: str | None = None,
+        default_asset_annotation_mapping: str | None = None,
+        default_file_annotation_mapping: str | None = None,
+    ) -> None:
         super().__init__(client)
         self.annotation_io = AnnotationIO(client)
         self.instance_space = instance_space
+        self.default_asset_annotation_mapping = default_asset_annotation_mapping or ASSET_ANNOTATIONS_ID
+        self.default_file_annotation_mapping = default_file_annotation_mapping or FILE_ANNOTATIONS_ID
 
     def as_id(self, item: AssetCentricMapping[Annotation]) -> str:
         return f"Annotation_{item.mapping.id}"
 
     def count(self, selector: AssetCentricMigrationSelector) -> int | None:
-        # There is no efficient way to count annotations in CDF.
-        return None
+        if isinstance(selector, MigrationCSVFileSelector):
+            return len(selector.items)
+        else:
+            # There is no efficient way to count annotations in CDF.
+            return None
 
     def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterable[Page]:
         if isinstance(selector, MigrateDataSetSelector):
@@ -253,8 +275,10 @@ class AnnotationMigrationIO(
                 mapping = AnnotationMapping(
                     instance_id=EdgeId(space=self.instance_space, external_id=f"annotation_{resource.id!r}"),
                     id=resource.id,
-                    ingestion_view=selector.ingestion_mapping,
+                    ingestion_view=self._get_mapping(selector.ingestion_mapping, resource),
                     preferred_consumer_view=selector.preferred_consumer_view,
+                    # The PySDK is poorly typed.
+                    annotation_type=resource.annotation_type,  # type: ignore[arg-type]
                 )
                 mapping_list.append(AssetCentricMapping(mapping=mapping, resource=resource))
             yield mapping_list
@@ -275,6 +299,7 @@ class AnnotationMigrationIO(
                 if resource is None:
                     not_found += 1
                     continue
+                mapping.ingestion_view = self._get_mapping(mapping.ingestion_view, resource)
                 chunk.append(AssetCentricMapping(mapping=mapping, resource=resource))
             if chunk:
                 yield chunk
@@ -284,6 +309,21 @@ class AnnotationMigrationIO(
                     f"Could not find {not_found} annotations referenced in the CSV file. They will be skipped during migration."
                 ).print_warning(include_timestamp=True, console=self.client.console)
 
+    def _get_mapping(self, current_mapping: str | None, resource: Annotation) -> str:
+        try:
+            return (
+                current_mapping
+                or {
+                    "diagrams.AssetLink": self.default_asset_annotation_mapping,
+                    "diagrams.FileLink": self.default_file_annotation_mapping,
+                }[resource.annotation_type]
+            )
+        except KeyError as e:
+            raise ToolkitValueError(
+                f"Could not determine default ingestion view for annotation type '{resource.annotation_type}'. "
+                "Please specify the ingestion view explicitly in the CSV file."
+            ) from e
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> InstanceApply:
         raise NotImplementedError("Deserializing Annotation Migrations from JSON is not supported.")
 

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -27,7 +27,7 @@ from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSever
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_READ_CLS_BY_FORMAT, FileReader
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
@@ -87,7 +87,7 @@ class UploadCommand(ToolkitCommand):
         ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
-        console = Console()
+        console = client.console
         data_files_by_selector = self._find_data_files(input_dir, kind)
 
         self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
@@ -225,49 +225,44 @@ class UploadCommand(ToolkitCommand):
                 io = self._create_selected_io(selector, datafiles[0], client)
                 if io is None:
                     continue
-                for data_file in datafiles:
-                    file_display = self._path_as_display_name(data_file)
-                    if verbose:
-                        console.print(f"{action} {selector.display_name} from {file_display.as_posix()!r}")
-                    reader = FileReader.from_filepath(data_file)
-                    is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
-                    if is_table and not isinstance(io, TableUploadableStorageIO):
-                        raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
-                    tracker = ProgressTracker[str]([self._UPLOAD])
-                    executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
-                        download_iterable=io.read_chunks(reader),
-                        process=partial(io.rows_to_data, selector=selector)
-                        if is_table and isinstance(io, TableUploadableStorageIO)
-                        else io.json_chunk_to_data,
-                        write=partial(
-                            self._upload_items,
-                            upload_client=upload_client,
-                            io=io,
-                            dry_run=dry_run,
-                            selector=selector,
-                            tracker=tracker,
-                            console=console,
-                        ),
-                        iteration_count=None,
-                        max_queue_size=self._MAX_QUEUE_SIZE,
-                        download_description=f"Reading {file_count:,}/{total_file_count + 1:,}: {file_display.as_posix()!s}",
-                        process_description="Processing",
-                        write_description=f"{action} {selector.display_name!r}",
+                reader = MultiFileReader(datafiles)
+                if reader.is_table and not isinstance(io, TableUploadableStorageIO):
+                    raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                tracker = ProgressTracker[str]([self._UPLOAD])
+                executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
+                    download_iterable=io.read_chunks(reader),
+                    process=partial(io.rows_to_data, selector=selector)
+                    if reader.is_table and isinstance(io, TableUploadableStorageIO)
+                    else io.json_chunk_to_data,
+                    write=partial(
+                        self._upload_items,
+                        upload_client=upload_client,
+                        io=io,
+                        dry_run=dry_run,
+                        selector=selector,
+                        tracker=tracker,
                         console=console,
-                    )
-                    executor.run()
-                    file_count += 1
-                    executor.raise_on_error()
-                    final_action = "Uploaded" if not dry_run else "Would upload"
-                    suffix = " successfully" if not dry_run else ""
-                    results = tracker.aggregate()
-                    success = results.get((self._UPLOAD, "success"), 0)
-                    failed = results.get((self._UPLOAD, "failed"), 0)
-                    if failed > 0:
-                        suffix += f", {failed:,} failed"
-                    console.print(
-                        f"{final_action} {success:,} {selector.display_name} from {file_display.as_posix()!r}{suffix}."
-                    )
+                    ),
+                    iteration_count=None,
+                    max_queue_size=self._MAX_QUEUE_SIZE,
+                    download_description=f"Reading {selector.display_name!r} files",
+                    process_description="Processing",
+                    write_description=f"{action} {selector.display_name!r}",
+                    console=console,
+                )
+                executor.run()
+                file_count += len(datafiles)
+                executor.raise_on_error()
+                final_action = "Uploaded" if not dry_run else "Would upload"
+                suffix = " successfully" if not dry_run else ""
+                results = tracker.aggregate()
+                success = results.get((self._UPLOAD, "success"), 0)
+                failed = results.get((self._UPLOAD, "failed"), 0)
+                if failed > 0:
+                    suffix += f", {failed:,} failed"
+                console.print(
+                    f"{final_action} {success:,} {selector.display_name} from {len(datafiles)} files{suffix}."
+                )
 
     @staticmethod
     def _path_as_display_name(input_path: Path, cwd: Path = Path.cwd()) -> Path:

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py

@@ -328,6 +328,7 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
 
     def dump_resource(self, resource: Container, local: dict[str, Any] | None = None) -> dict[str, Any]:
         dumped = resource.as_write().dump()
+        has_local = local is not None
         local = local or {}
         for key in ["constraints", "indexes"]:
             if not dumped.get(key) and key not in local:
@@ -339,15 +340,16 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
                 continue
             local_prop = local_prop_by_id[prop_id]
             for key, default in [("immutable", False), ("autoIncrement", False), ("nullable", True)]:
-                if cdf_prop.get(key) is default and key not in local_prop:
+                if has_local and cdf_prop.get(key) is default and key not in local_prop:
                     cdf_prop.pop(key, None)
             cdf_type = cdf_prop.get("type", {})
             local_type = local_prop.get("type", {})
             for key, type_default in [("list", False), ("collation", "ucs_basic")]:
-                if cdf_type.get(key) == type_default and key not in local_type:
+                if has_local and cdf_type.get(key) == type_default and key not in local_type:
                     cdf_type.pop(key, None)
-        if "usedFor" not in local:
-            dumped.pop("usedFor", None)
+            if has_local and "usedFor" not in local and dumped.get("usedFor") == "node":
+                # Only drop if set to default by server.
+                dumped.pop("usedFor", None)
         return dumped
 
     def create(self, items: Sequence[ContainerApply]) -> ContainerList:

cognite_toolkit/_cdf_tk/storageio/_annotations.py

@@ -26,21 +26,23 @@ class AnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
     def stream_data(self, selector: AssetCentricSelector, limit: int | None = None) -> Iterable[Page[Annotation]]:
         total = 0
         for file_chunk in FileMetadataIO(self.client).stream_data(selector, None):
-            # Todo Support pagination. This is missing in the SDK.
-            results = self.client.annotations.list(
-                filter=AnnotationFilter(
-                    annotated_resource_type="file",
-                    annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+            for annotation_type in ["diagrams.AssetLink", "diagrams.FileLink"]:
+                # Todo Support pagination. This is missing in the SDK.
+                results = self.client.annotations.list(
+                    filter=AnnotationFilter(
+                        annotated_resource_type="file",
+                        annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+                        annotation_type=annotation_type,
+                    )
                 )
-            )
-            if limit is not None and total + len(results) > limit:
-                results = results[: limit - total]
-
-            for chunk in chunker_sequence(results, self.CHUNK_SIZE):
-                yield Page(worker_id="main", items=chunk)
-                total += len(chunk)
-            if limit is not None and total >= limit:
-                break
+                if limit is not None and total + len(results) > limit:
+                    results = results[: limit - total]
+
+                for chunk in chunker_sequence(results, self.CHUNK_SIZE):
+                    yield Page(worker_id="main", items=chunk)
+                    total += len(chunk)
+                if limit is not None and total >= limit:
+                    break
 
     def count(self, selector: AssetCentricSelector) -> int | None:
         """There is no efficient way to count annotations in CDF."""

cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -8,8 +8,7 @@ from cognite.client.data_classes._base import T_CogniteResource
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
 from cognite_toolkit._cdf_tk.utils.collection import chunker
-from cognite_toolkit._cdf_tk.utils.fileio import FileReader, SchemaColumn
-from cognite_toolkit._cdf_tk.utils.fileio._readers import TableReader
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemsRequest
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal, T_WriteCogniteResource
 
@@ -217,8 +216,8 @@ class UploadableStorageIO(
         raise NotImplementedError()
 
     @classmethod
-    def read_chunks(cls, reader: FileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
-        data_name = "row" if isinstance(reader, TableReader) else "line"
+    def read_chunks(cls, reader: MultiFileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+        data_name = "row" if reader.is_table else "line"
         # Include name of line for better error messages
         iterable = ((f"{data_name} {line_no}", item) for line_no, item in reader.read_chunks_with_line_numbers())
 

cognite_toolkit/_cdf_tk/storageio/_datapoints.py

@@ -19,8 +19,7 @@ from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
     _TextConverter,
     _ValueConverter,
 )
-from cognite_toolkit._cdf_tk.utils.fileio import FileReader
-from cognite_toolkit._cdf_tk.utils.fileio._readers import TableReader
+from cognite_toolkit._cdf_tk.utils.fileio._readers import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import DataBodyRequest, HTTPClient, HTTPMessage
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
@@ -165,8 +164,8 @@ class DatapointsIO(TableUploadableStorageIO[DataPointsFileSelector, DataPointLis
         )
 
     @classmethod
-    def read_chunks(cls, reader: FileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
-        if not isinstance(reader, TableReader):
+    def read_chunks(cls, reader: MultiFileReader) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
+        if not reader.is_table:
             raise RuntimeError("DatapointsIO can only read from TableReader instances.")
         iterator = iter(reader.read_chunks_with_line_numbers())
         try:

cognite_toolkit/_cdf_tk/utils/fileio/__init__.py

@@ -12,6 +12,7 @@ from ._readers import (
     CSVReader,
     FailedParsing,
     FileReader,
+    MultiFileReader,
     NDJsonReader,
     ParquetReader,
     YAMLReader,
@@ -45,6 +46,7 @@ __all__ = [
     "FileReader",
     "FileWriter",
     "GzipCompression",
+    "MultiFileReader",
     "NDJsonReader",
     "NDJsonWriter",
     "ParquetReader",

cognite_toolkit/_cdf_tk/utils/fileio/_base.py

@@ -14,7 +14,11 @@ T_IO = TypeVar("T_IO", bound=IO)
 
 
 class FileIO(ABC):
-    format: ClassVar[str]
+    FORMAT: ClassVar[str]
+
+    @property
+    def format(self) -> str:
+        return self.FORMAT
 
 
 @dataclass(frozen=True)

cognite_toolkit/_cdf_tk/utils/fileio/_readers.py

@@ -1,5 +1,6 @@
 import csv
 import json
+import re
 from abc import ABC, abstractmethod
 from collections import Counter, defaultdict
 from collections.abc import Callable, Iterator, Mapping, Sequence
@@ -40,7 +41,7 @@ class FileReader(FileIO, ABC):
         raise NotImplementedError("This method should be implemented in subclasses.")
 
     @classmethod
-    def from_filepath(cls, filepath: Path) -> "FileReader":
+    def from_filepath(cls, filepath: Path) -> "type[FileReader]":
         if len(filepath.suffixes) == 0:
             raise ToolkitValueError(
                 f"File has no suffix. Available formats: {humanize_collection(FILE_READ_CLS_BY_FORMAT.keys())}."
@@ -55,15 +56,57 @@ class FileReader(FileIO, ABC):
                 )
 
         if suffix in FILE_READ_CLS_BY_FORMAT:
-            return FILE_READ_CLS_BY_FORMAT[suffix](input_file=filepath)
+            return FILE_READ_CLS_BY_FORMAT[suffix]
 
         raise ToolkitValueError(
             f"Unknown file format: {suffix}. Available formats: {humanize_collection(FILE_READ_CLS_BY_FORMAT.keys())}."
         )
 
 
+class MultiFileReader(FileReader):
+    """Reads multiple files and yields chunks from each file sequentially.
+
+    Args:
+        input_files (Sequence[Path]): The list of file paths to read.
+    """
+
+    PART_PATTERN = re.compile(r"part-(\d{4})$")
+
+    def __init__(self, input_files: Sequence[Path]) -> None:
+        super().__init__(input_file=input_files[0])
+        self.input_files = input_files
+        reader_classes = Counter([FileReader.from_filepath(input_file) for input_file in self.input_files])
+        if len(reader_classes) > 1:
+            raise ToolkitValueError(
+                "All input files must be of the same format. "
+                f"Found formats: {humanize_collection([cls.FORMAT for cls in reader_classes.keys()])}."
+            )
+        self.reader_class = reader_classes.most_common(1)[0][0]
+
+    @property
+    def is_table(self) -> bool:
+        return issubclass(self.reader_class, TableReader)
+
+    @property
+    def format(self) -> str:
+        return self.reader_class.FORMAT
+
+    def read_chunks(self) -> Iterator[dict[str, JsonVal]]:
+        for input_file in sorted(self.input_files, key=self._part_no):
+            yield from self.reader_class(input_file).read_chunks()
+
+    def _part_no(self, path: Path) -> int:
+        match = self.PART_PATTERN.search(path.stem)
+        if match:
+            return int(match.group(1))
+        return 99999
+
+    def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
+        raise NotImplementedError("This method is not used in MultiFileReader.")
+
+
 class NDJsonReader(FileReader):
-    format = ".ndjson"
+    FORMAT = ".ndjson"
 
     def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
         for line in file:
@@ -77,11 +120,11 @@ class YAMLBaseReader(FileReader, ABC):
 
 
 class YAMLReader(YAMLBaseReader):
-    format = ".yaml"
+    FORMAT = ".yaml"
 
 
 class YMLReader(YAMLBaseReader):
-    format = ".yml"
+    FORMAT = ".yml"
 
 
 @dataclass
@@ -171,8 +214,8 @@ class TableReader(FileReader, ABC):
 
         if not input_file.exists():
             raise ToolkitFileNotFoundError(f"File not found: {input_file.as_posix()!r}.")
-        if input_file.suffix != cls.format:
-            raise ToolkitValueError(f"Expected a {cls.format} file got a {input_file.suffix!r} file instead.")
+        if input_file.suffix != cls.FORMAT:
+            raise ToolkitValueError(f"Expected a {cls.FORMAT} file got a {input_file.suffix!r} file instead.")
 
         column_names, sample_rows = cls._read_sample_rows(input_file, sniff_rows)
         cls._check_column_names(column_names)
@@ -213,7 +256,7 @@ class TableReader(FileReader, ABC):
 class CSVReader(TableReader):
     """Reads CSV files and yields each row as a dictionary."""
 
-    format = ".csv"
+    FORMAT = ".csv"
 
     def _read_chunks_from_file(self, file: TextIOWrapper) -> Iterator[dict[str, JsonVal]]:
         if self.keep_failed_cells and self.failed_cell:
@@ -257,7 +300,7 @@ class CSVReader(TableReader):
 
 
 class ParquetReader(TableReader):
-    format = ".parquet"
+    FORMAT = ".parquet"
 
     def __init__(self, input_file: Path) -> None:
         # Parquet files have their own schema, so we don't need to sniff or provide one.
@@ -312,19 +355,19 @@ class ParquetReader(TableReader):
 FILE_READ_CLS_BY_FORMAT: Mapping[str, type[FileReader]] = {}
 TABLE_READ_CLS_BY_FORMAT: Mapping[str, type[TableReader]] = {}
 for subclass in get_concrete_subclasses(FileReader):  # type: ignore[type-abstract]
-    if not getattr(subclass, "format", None):
+    if not getattr(subclass, "FORMAT", None):
         continue
-    if subclass.format in FILE_READ_CLS_BY_FORMAT:
+    if subclass.FORMAT in FILE_READ_CLS_BY_FORMAT:
         raise TypeError(
-            f"Duplicate file format {subclass.format!r} found for classes "
-            f"{FILE_READ_CLS_BY_FORMAT[subclass.format].__name__!r} and {subclass.__name__!r}."
+            f"Duplicate file format {subclass.FORMAT!r} found for classes "
+            f"{FILE_READ_CLS_BY_FORMAT[subclass.FORMAT].__name__!r} and {subclass.__name__!r}."
         )
     # We know we have a dict, but we want to expose FILE_READ_CLS_BY_FORMAT as a Mapping
-    FILE_READ_CLS_BY_FORMAT[subclass.format] = subclass  # type: ignore[index]
+    FILE_READ_CLS_BY_FORMAT[subclass.FORMAT] = subclass  # type: ignore[index]
     if issubclass(subclass, TableReader):
-        if subclass.format in TABLE_READ_CLS_BY_FORMAT:
+        if subclass.FORMAT in TABLE_READ_CLS_BY_FORMAT:
             raise TypeError(
-                f"Duplicate table file format {subclass.format!r} found for classes "
-                f"{TABLE_READ_CLS_BY_FORMAT[subclass.format].__name__!r} and {subclass.__name__!r}."
+                f"Duplicate table file format {subclass.FORMAT!r} found for classes "
+                f"{TABLE_READ_CLS_BY_FORMAT[subclass.FORMAT].__name__!r} and {subclass.__name__!r}."
             )
-        TABLE_READ_CLS_BY_FORMAT[subclass.format] = subclass  # type: ignore[index]
+        TABLE_READ_CLS_BY_FORMAT[subclass.FORMAT] = subclass  # type: ignore[index]

cognite_toolkit/_cdf_tk/utils/fileio/_writers.py

@@ -154,7 +154,7 @@ class TableWriter(FileWriter[T_IO], ABC):
 
 
 class NDJsonWriter(FileWriter[TextIOWrapper]):
-    format = ".ndjson"
+    FORMAT = ".ndjson"
 
     class _DateTimeEncoder(json.JSONEncoder):
         def default(self, obj: object) -> object:
@@ -181,15 +181,15 @@ class YAMLBaseWriter(FileWriter[TextIOWrapper], ABC):
 
 
 class YAMLWriter(YAMLBaseWriter):
-    format = ".yaml"
+    FORMAT = ".yaml"
 
 
 class YMLWriter(YAMLBaseWriter):
-    format = ".yml"
+    FORMAT = ".yml"
 
 
 class CSVWriter(TableWriter[TextIOWrapper]):
-    format = ".csv"
+    FORMAT = ".csv"
 
     def __init__(
         self,
@@ -241,7 +241,7 @@ class CSVWriter(TableWriter[TextIOWrapper]):
 
 
 class ParquetWriter(TableWriter["pq.ParquetWriter"]):
-    format = ".parquet"
+    FORMAT = ".parquet"
 
     def _create_writer(self, filepath: Path) -> "pq.ParquetWriter":
         import pyarrow.parquet as pq
@@ -411,19 +411,19 @@ class ParquetWriter(TableWriter["pq.ParquetWriter"]):
 FILE_WRITE_CLS_BY_FORMAT: Mapping[str, type[FileWriter]] = {}
 TABLE_WRITE_CLS_BY_FORMAT: Mapping[str, type[TableWriter]] = {}
 for subclass in get_concrete_subclasses(FileWriter):  # type: ignore[type-abstract]
-    if not getattr(subclass, "format", None):
+    if not getattr(subclass, "FORMAT", None):
         continue
-    if subclass.format in FILE_WRITE_CLS_BY_FORMAT:
+    if subclass.FORMAT in FILE_WRITE_CLS_BY_FORMAT:
         raise TypeError(
-            f"Duplicate file format {subclass.format!r} found for classes "
-            f"{FILE_WRITE_CLS_BY_FORMAT[subclass.format].__name__!r} and {subclass.__name__!r}."
+            f"Duplicate file format {subclass.FORMAT!r} found for classes "
+            f"{FILE_WRITE_CLS_BY_FORMAT[subclass.FORMAT].__name__!r} and {subclass.__name__!r}."
         )
     # We know we have a dict, but we want to expose FILE_WRITE_CLS_BY_FORMAT as a Mapping
-    FILE_WRITE_CLS_BY_FORMAT[subclass.format] = subclass  # type: ignore[index]
+    FILE_WRITE_CLS_BY_FORMAT[subclass.FORMAT] = subclass  # type: ignore[index]
     if issubclass(subclass, TableWriter):
-        if subclass.format in TABLE_WRITE_CLS_BY_FORMAT:
+        if subclass.FORMAT in TABLE_WRITE_CLS_BY_FORMAT:
             raise TypeError(
-                f"Duplicate table file format {subclass.format!r} found for classes "
-                f"{TABLE_WRITE_CLS_BY_FORMAT[subclass.format].__name__!r} and {subclass.__name__!r}."
+                f"Duplicate table file format {subclass.FORMAT!r} found for classes "
+                f"{TABLE_WRITE_CLS_BY_FORMAT[subclass.FORMAT].__name__!r} and {subclass.__name__!r}."
             )
-        TABLE_WRITE_CLS_BY_FORMAT[subclass.format] = subclass  # type: ignore[index]
+        TABLE_WRITE_CLS_BY_FORMAT[subclass.FORMAT] = subclass  # type: ignore[index]

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.6.105
+      image: cognite/toolkit:0.6.107
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.6.105
+      image: cognite/toolkit:0.6.107
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.105"
+version = "0.6.107"
 
 [alpha_flags]
 external-libraries = true

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.6.105"
+__version__ = "0.6.107"

{cognite_toolkit-0.6.105.dist-info → cognite_toolkit-0.6.107.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.105
+Version: 0.6.107
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.105.dist-info → cognite_toolkit-0.6.107.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf.py,sha256=0abeQr1Tfk4lkGaoXyrnFC28wDSlR_8UGrh10noGduQ,6085
-cognite_toolkit/_version.py,sha256=WfvaY_npde01GAAtt1MaxhE137U1_NM3UTrLq2lyxhA,24
+cognite_toolkit/_version.py,sha256=lu-o5QGcXmrHU8tes3QjIxzRHr_63kClROn7Lrtlq6s,24
 cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
 cognite_toolkit/_cdf_tk/constants.py,sha256=Gi7iGGzdUrOnBeIK6ix3XiBieHIwzLJO5BWjDI3a6l4,7082
@@ -19,7 +19,7 @@ cognite_toolkit/_cdf_tk/apps/_dev_app.py,sha256=q8DBr4BAK33AwsHW3gAWZWSjSaQRuCis
 cognite_toolkit/_cdf_tk/apps/_download_app.py,sha256=Io7mW3DhYpLiDrqJfThSQThHDWrXN9c8huu9zYd0u7E,19408
 cognite_toolkit/_cdf_tk/apps/_dump_app.py,sha256=Ec0aEqbKwCkxni09i06rfY31qZUyOVwbbvo7MHh4cf8,39056
 cognite_toolkit/_cdf_tk/apps/_landing_app.py,sha256=HxzSln3fJXs5NzulfQGUMropXcwMobUYpyePrCrQTQs,1502
-cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=xkXo1h3eQ14D5uQ0qQdq01etS1FV_fg6ZnvxMt8H-os,30580
+cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=JuzY1t5pkW94Dyygr-mT2lheluR7cL8AmD2MfaXhc8Y,37093
 cognite_toolkit/_cdf_tk/apps/_modules_app.py,sha256=95_H2zccRJl2mWn0oQ5mjCaEDnG63sPKOkB81IgWcIk,7637
 cognite_toolkit/_cdf_tk/apps/_profile_app.py,sha256=vSRJW54bEvIul8_4rOqyOYA7ztXx7TFOvZRZWZTxMbg,7007
 cognite_toolkit/_cdf_tk/apps/_purge.py,sha256=e8IgDK2Fib2u30l71Q2trbJ1az90zSLWr5TViTINmL0,15415
@@ -108,7 +108,7 @@ cognite_toolkit/_cdf_tk/commands/_download.py,sha256=OBKPM_HGGA1i32th1SAgkQM_81C
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
 cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=RadQHsmkPez3fZ5HCP9b82o2_fBx8P_-bTo7prkvWXU,32525
 cognite_toolkit/_cdf_tk/commands/_questionary_style.py,sha256=h-w7fZKkGls3TrzIGBKjsZSGoXJJIYchgD1StfA40r8,806
-cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=axrdDIjW59LxByIwq379tfCEkoBSdq8kVx8ORmsOQxo,14422
+cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=iwLRK53pXP68vmgIGBx7I1ODtlH6bks7_TanHpr04zw,13900
 cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
 cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
 cognite_toolkit/_cdf_tk/commands/auth.py,sha256=PLjfxfJJKaqux1eB2fycIRlwwSMCbM3qxWDnFX-blJU,31720
@@ -127,14 +127,14 @@ cognite_toolkit/_cdf_tk/commands/run.py,sha256=JyX9jLEQej9eRrHVCCNlw4GuF80qETSol
 cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=i5ldcTah59K0E4fH5gHTV0GRvtDCEvVses9WQzn9Lno,226
 cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py,sha256=R-z0yfOFcJZj-zRLhN-7z_-SLxqzSmONMgrbzNF9dGs,8843
 cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=jNoqqq81lbdfDTAQ5w2ctaYSUueLhZe0qjUKjCezk6s,14234
-cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=_okrcPRaNtM2d8u4Aba1hUYN8CgSTAcDgcugd76aXWA,16775
+cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=Ew9JRYrd-Ol9G9csTzpnhXAgCFnX67MwDYOTsdJLP3E,16803
 cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=FTu7w3G8KyPY8pagG3KdPpOmpLcjehaAg2auEy6iM7A,9605
-cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=vOdvDPVibAbqLQt8hg7pmdHN1Fd6JUoyOZ-fCzfeEH8,9925
-cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=5gqVRJGa7rh3c9pwPdipxKfl-zXV4mmU0G_h3ehPe2Q,5339
+cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=_vMS_qAPj4yup1VnmmojPVigAZtyPQH7PM0Raby5tao,10619
+cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=Y7MrE6FGa15uvboBjNyWNlslsBv4FpeP5WsrFsooxsA,5678
 cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=i1eUsNX6Dueol9STIEwyksBnBsWUk13O8qHIjW964pM,7860
-cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py,sha256=KkSq_4R6hQ15ccG-jHy7vVgPwC5IDd5OaXZLvz5mIZs,5547
+cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py,sha256=ERn3qFrJFXdtXaMjHq3Gk7MxH03MGFk3FrtWCOBJQts,5544
 cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=lWSnuS3CfRDbA7i1g12gJ2reJnQcLmZWxHDK19-Wxkk,5772
-cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py,sha256=xKjm2OKxfwjQ2DFrrLPFydQhzUjJJ20rDTdaFrkzan8,14372
+cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py,sha256=wrdBH5P6NgiZQSYLR0iJ3ZvqfQ5fY-_Ne2yKv9E1g4o,16277
 cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py,sha256=RfqaNoso5CyBwc-p6ckwcYqBfZXKhdJgdGIyd0TATaI,2635
 cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py,sha256=N1H_-rBpPUD6pbrlcofn1uEK1bA694EUXEe1zIXeqyo,2489
 cognite_toolkit/_cdf_tk/cruds/__init__.py,sha256=kxiB8gZo0Y4TyttWHGTLPCW5R1DUkN1uTZewTvaZRjo,6298
@@ -147,7 +147,7 @@ cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py,sha256=iGG2_btpEqip3o6OKpc
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=7RdiWvh6MLI1lLmt3gcqDQj61xbwREhsvoyjFuJn2F0,26402
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py,sha256=KrL7bj8q5q18mGB2V-NDkW5U5nfseZOyorXiUbp2uLw,6100
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py,sha256=iXn9iAtwA8mhH-7j9GF-MlLomTcaw3GhEbFY28Wx0iA,9927
-cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=kA9urLWU65fqskvdSpjJEluvuWzUcCVhWCWoho68OJY,65642
+cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=XaVUjYKfa2ceOicAgE_mPknMdQm_ldwiaNIs3wWjFA0,65823
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py,sha256=zv36HPO9goRmU3NM_i1wOvWQEdsgpQTI4bcAl-eis1g,18232
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=SnQMbxiZ3SSYkTLXQ_vIu2HVf_WyD1jplNRJuoeOUfA,16723
 cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py,sha256=F3n2FOWAPder4z3OTYs81VB-6C6r3oUzJsHvigdhaD0,15500
@@ -239,12 +239,12 @@ cognite_toolkit/_cdf_tk/resource_classes/robotics/frame.py,sha256=XmDqJ0pAxe_vAP
 cognite_toolkit/_cdf_tk/resource_classes/robotics/location.py,sha256=dbc9HT-bc2Qt15hHoR63SM7pg321BhNuTNjI7HHCwSA,468
 cognite_toolkit/_cdf_tk/resource_classes/robotics/map.py,sha256=j77z7CzCMiMj8r94BdUKCum9EuZRUjaSlUAy9K9DL_Q,942
 cognite_toolkit/_cdf_tk/storageio/__init__.py,sha256=SSMV-W_uqMwS9I0xazBfAyNRqKWlAuLlABropMBEa50,2434
-cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=N5z2umaPwBo3OKIyGhNx2geRTBYhAeG3YhCAYm2hyao,4655
+cognite_toolkit/_cdf_tk/storageio/_annotations.py,sha256=JI_g18_Y9S7pbc9gm6dZMyo3Z-bCndJXF9C2lOva0bQ,4848
 cognite_toolkit/_cdf_tk/storageio/_applications.py,sha256=CdqJueM9ZmXVh8RUme2lAgNasjAM8QTQDAfeJMm2ZYo,7026
 cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=DbTvIneN8Hw3ByhdH1kXkS7Gw68oXEWtIqlZGZgLMg0,33704
-cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=at-5wERENsFRLO1cUOMKW2poVqmkvxSe9JuXpojz4NM,12196
+cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=S52TFdNZuXXkMU_jUobnjW4COwkDwn47FFrKeSv64xs,12120
 cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
-cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=nV8jaF5YLvMKhDPU3euf554GvSmfNYkzC9ZvEF7kbP8,8660
+cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=1Cfch0lVPc4dyXH7RGfGaXudB1O3-f56_cfGw-e-ya0,8593
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
 cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=5WjAFiVR0KKRhMqCy1IRy1TQFWj86D7nGu5WSFNLp6U,3869
 cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=kvk7zdI_N2VobkrWTYRDuq1fSpy2Z99MsJp1sBa_KrQ,1715
@@ -285,11 +285,11 @@ cognite_toolkit/_cdf_tk/utils/text.py,sha256=1-LQMo633_hEhNhishQo7Buj-7np5Pe4qKk
 cognite_toolkit/_cdf_tk/utils/thread_safe_dict.py,sha256=NbRHcZvWpF9xHP5OkOMGFpxrPNbi0Q3Eea6PUNbGlt4,3426
 cognite_toolkit/_cdf_tk/utils/useful_types.py,sha256=oK88W6G_aK3hebORSQKZjWrq7jG-pO2lkLWSWYMlngM,1872
 cognite_toolkit/_cdf_tk/utils/validate_access.py,sha256=1puswcpgEDNCwdk91dhLqCBSu_aaUAd3Hsw21d-YVFs,21955
-cognite_toolkit/_cdf_tk/utils/fileio/__init__.py,sha256=ts5kYu_1Ks7xjnM6pIrVUrZe0nkYI6euYXeE4ox34xk,1199
-cognite_toolkit/_cdf_tk/utils/fileio/_base.py,sha256=MpWaD3lR9vrJ-kGzTiDOtChXhvFD7-xrP-Pzp7vjnLY,756
+cognite_toolkit/_cdf_tk/utils/fileio/__init__.py,sha256=0rJsL3jClj_smxh_Omqchf0K9xTi1DlKgmCDjBqJ38I,1243
+cognite_toolkit/_cdf_tk/utils/fileio/_base.py,sha256=eC6mRIwSD4LjyFa83BoBnhO0t3l-ctQMW295LIyxXLk,827
 cognite_toolkit/_cdf_tk/utils/fileio/_compression.py,sha256=8BAPgg5OKc3vkEEkqOvYsuyh12iXVNuEmC0omWwyJNQ,2355
-cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=mBf0-8JFwLfyTGJH8nWpbn89VPTj9UwP3GmZtx8t3A4,13969
-cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=T9Owx1XQ2bN1voBBLs7hUpTXihCiXgHwEhpIv_XB9xc,17732
+cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=plDxxRVipcddjhF8JOaVoDFeNFUmtl4fKMX8FLAU_eI,15527
+cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=4buAPp73Qfc0hw_LMyFI3g2DhdM4hbrasXuwMCiAcCQ,17732
 cognite_toolkit/_cdf_tk/utils/http_client/__init__.py,sha256=G8b7Bg4yIet5R4Igh3dS2SntWzE6I0iTGBeNlNsSxkQ,857
 cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=NTRfloXkCiS_rl5Vl1D_hsyTTowMKWDsiIR4oGwTADI,11208
 cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py,sha256=PIlSmv3spObHeoylpSzz2fqUFAjIoE89qzvc0uSOGw0,12975
@@ -300,13 +300,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=IwBGesItWryS0xfmEeOGZAePQ4mxHSR6CmBQ7n9i-mY,668
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=8Pg9SJbe4VbVP8EnGKE1hNds9TfSIGptB87N0X8xeNU,2431
-cognite_toolkit/_resources/cdf.toml,sha256=ATCi6pyPK0FQOsAgiYUaNz2YK2nyKdaeKVW1FOt1rGc,488
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=7ijoHGa5OFuDD3hYSllkChA4MU_iJgAnXmUX4sTlDkg,668
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=5yngla4gklkissfZ1vVMphWbfMfYmFs2gt3HnckqJrI,2431
+cognite_toolkit/_resources/cdf.toml,sha256=CkbioMspsFNI5FwfptPthjPiQiln-5d7B6Tm1eWW_YY,488
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.6.105.dist-info/METADATA,sha256=4MHffqZKT4SsNX4s4tz-bbPUm1wGxbGvrq93n4ul1Sw,4502
-cognite_toolkit-0.6.105.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_toolkit-0.6.105.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
-cognite_toolkit-0.6.105.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
-cognite_toolkit-0.6.105.dist-info/RECORD,,
+cognite_toolkit-0.6.107.dist-info/METADATA,sha256=jKo4OnTtq7P4OwIRv8ku2Exp5ZlhakpRrUtmHwbYWKM,4502
+cognite_toolkit-0.6.107.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_toolkit-0.6.107.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
+cognite_toolkit-0.6.107.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
+cognite_toolkit-0.6.107.dist-info/RECORD,,