cognite-toolkit 0.6.105__tar.gz → 0.6.107__tar.gz

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.6.105
+Version: 0.6.107
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
 Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cdf.toml

@@ -33,7 +33,7 @@ dev = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.105"
+version = "0.6.107"
 
 
 [library.toolkit-data]

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -4,6 +4,7 @@ from typing import Annotated, Any
 
 import questionary
 import typer
+from cognite.client.data_classes import Annotation
 from cognite.client.data_classes.data_modeling import ContainerId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
@@ -19,6 +20,7 @@ from cognite_toolkit._cdf_tk.commands._migrate.creators import (
 )
 from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
+    AnnotationMigrationIO,
     AssetCentricMigrationIO,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
@@ -31,6 +33,7 @@ from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
     AssetInteractiveSelect,
     DataModelingSelect,
+    FileMetadataInteractiveSelect,
     ResourceViewMappingInteractiveSelect,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
@@ -49,6 +52,7 @@ class MigrateApp(typer.Typer):
         self.command("events")(self.events)
         self.command("timeseries")(self.timeseries)
         self.command("files")(self.files)
+        self.command("annotations")(self.annotations)
         self.command("canvas")(self.canvas)
         # Uncomment when infield v2 config migration is ready
         # self.command("infield-configs")(self.infield_configs)
@@ -694,6 +698,149 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @classmethod
+    def annotations(
+        cls,
+        ctx: typer.Context,
+        mapping_file: Annotated[
+            Path | None,
+            typer.Option(
+                "--mapping-file",
+                "-m",
+                help="Path to the mapping file that contains the mapping from Annotations to CogniteDiagramAnnotation. "
+                "This file is expected to have the following columns: [id, space, externalId, ingestionView].",
+            ),
+        ] = None,
+        data_set_id: Annotated[
+            str | None,
+            typer.Option(
+                "--data-set-id",
+                "-s",
+                help="The data set ID to select for the annotations to migrate. If not provided and the mapping file is not provided, "
+                "an interactive selection will be performed to select the data set to migrate annotations from.",
+            ),
+        ] = None,
+        instance_space: Annotated[
+            str | None,
+            typer.Option(
+                "--instance-space",
+                "-i",
+                help="The instance space to use for the migrated annotations. Required when using --data-set-id.",
+            ),
+        ] = None,
+        asset_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--asset-annotation-mapping",
+                "-a",
+                help="The ingestion mapping to use for asset-linked annotations. If not provided, "
+                "the default mapping (cdf_asset_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        file_annotation_mapping: Annotated[
+            str | None,
+            typer.Option(
+                "--file-annotation-mapping",
+                "-f",
+                help="The ingestion mapping to use for file-linked annotations. If not provided, "
+                "the default mapping (cdf_file_annotations_mapping) will be used.",
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where logs will be stored. If the directory does not exist, it will be created.",
+            ),
+        ] = Path(f"migration_logs_{TODAY!s}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of what would be done is printed.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate Annotations to CogniteDiagramAnnotation edges in data modeling.
+
+        Annotations are diagram annotations that link assets or files to other resources. This command
+        migrates them to edges in the data modeling space, preserving the relationships and metadata.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+
+        if data_set_id is not None and mapping_file is not None:
+            raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
+        elif mapping_file is not None:
+            selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(
+                datafile=mapping_file, kind="Annotations"
+            )
+            annotation_io = AnnotationMigrationIO(client)
+        elif data_set_id is not None:
+            if instance_space is None:
+                raise typer.BadParameter("--instance-space is required when using --data-set-id")
+            selected = MigrateDataSetSelector(data_set_external_id=data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+        else:
+            # Interactive selection
+            selector = FileMetadataInteractiveSelect(client, "migrate")
+            selected_data_set_id = selector.select_data_set(allow_empty=False)
+            dm_selector = DataModelingSelect(client, "migrate")
+            selected_instance_space = dm_selector.select_instance_space(
+                multiselect=False,
+                message="In which instance space do you want to create the annotations?",
+                include_empty=True,
+            )
+            if selected_instance_space is None:
+                raise typer.Abort()
+            asset_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate asset annotations")
+            asset_annotation_mapping = asset_annotations_selector.select_resource_view_mapping(
+                resource_type="assetAnnotation",
+            ).external_id
+            file_annotations_selector = ResourceViewMappingInteractiveSelect(client, "migrate file annotations")
+            file_annotation_mapping = file_annotations_selector.select_resource_view_mapping(
+                resource_type="fileAnnotation",
+            ).external_id
+
+            selected = MigrateDataSetSelector(data_set_external_id=selected_data_set_id, kind="Annotations")
+            annotation_io = AnnotationMigrationIO(
+                client,
+                instance_space=selected_instance_space,
+                default_asset_annotation_mapping=asset_annotation_mapping,
+                default_file_annotation_mapping=file_annotation_mapping,
+            )
+
+            dry_run = questionary.confirm("Do you want to perform a dry run?", default=dry_run).ask()
+            verbose = questionary.confirm("Do you want verbose output?", default=verbose).ask()
+            if any(res is None for res in [dry_run, verbose]):
+                raise typer.Abort()
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=selected,
+                data=annotation_io,
+                mapper=AssetCentricMapper[Annotation](client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def canvas(
         ctx: typer.Context,

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -15,7 +15,10 @@ from cognite.client.data_classes.data_modeling.views import ViewProperty
 from cognite.client.utils._identifier import InstanceId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.migration import (
+    AssetCentricId,
+    ResourceViewMappingApply,
+)
 from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
 from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
     asset_centric_convert_to_primary_property,
@@ -163,7 +166,7 @@ class DirectRelationCache:
 def asset_centric_to_dm(
     resource: AssetCentricResourceExtended,
     instance_id: InstanceId,
-    view_source: ResourceViewMapping,
+    view_source: ResourceViewMappingApply,
     view_properties: dict[str, ViewProperty],
     direct_relation_cache: DirectRelationCache,
 ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
@@ -172,7 +175,7 @@ def asset_centric_to_dm(
     Args:
         resource (CogniteResource): The asset-centric resource to convert.
         instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
-        view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
+        view_source (ResourceViewMappingApply): The view source defining how to map the resource to the data model.
         view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
         direct_relation_cache (DirectRelationCache): Cache for direct relation references.
 

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -14,7 +14,11 @@ from pydantic import BaseModel, BeforeValidator, field_validator, model_validato
 from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
 from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
-from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
+    ASSET_ANNOTATIONS_ID,
+    FILE_ANNOTATIONS_ID,
+    create_default_mappings,
+)
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
 from cognite_toolkit._cdf_tk.utils.useful_types import (
@@ -182,6 +186,18 @@ class FileMapping(MigrationMapping):
 class AnnotationMapping(MigrationMapping):
     resource_type: Literal["annotation"] = "annotation"
     instance_id: EdgeId
+    annotation_type: Literal["diagrams.AssetLink", "diagrams.FileLink"] | None = None
+
+    def get_ingestion_view(self) -> str:
+        """Get the ingestion view for the mapping. If not specified, return the default ingestion view."""
+        if self.ingestion_view:
+            return self.ingestion_view
+        elif self.annotation_type == "diagrams.AssetLink":
+            return ASSET_ANNOTATIONS_ID
+        elif self.annotation_type == "diagrams.FileLink":
+            return FILE_ANNOTATIONS_ID
+        else:
+            raise ToolkitValueError("Cannot determine default ingestion view for annotation without annotation_type")
 
     @field_validator("instance_id", mode="before")
     def _validate_instance_id(cls, v: Any) -> Any:

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -14,9 +14,10 @@ from cognite.client.data_classes.data_modeling import (
 )
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
+from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
 from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
+from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
 from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
@@ -24,7 +25,7 @@ from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
-    T_AssetCentricResource,
+    T_AssetCentricResourceExtended,
 )
 
 
@@ -56,25 +57,27 @@ class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource],
 
 
 class AssetCentricMapper(
-    DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResource], InstanceApply]
+    DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResourceExtended], InstanceApply]
 ):
     def __init__(self, client: ToolkitClient) -> None:
         self.client = client
         self._ingestion_view_by_id: dict[ViewId, View] = {}
-        self._view_mapping_by_id: dict[str, ResourceViewMapping] = {}
+        self._view_mapping_by_id: dict[str, ResourceViewMappingApply] = {}
         self._direct_relation_cache = DirectRelationCache(client)
 
     def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
         ingestion_view_ids = source_selector.get_ingestion_mappings()
         ingestion_views = self.client.migration.resource_view_mapping.retrieve(ingestion_view_ids)
-        self._view_mapping_by_id = {view.external_id: view for view in ingestion_views}
+        defaults = {mapping.external_id: mapping for mapping in create_default_mappings()}
+        # Custom mappings from CDF override the default mappings
+        self._view_mapping_by_id = defaults | {view.external_id: view.as_write() for view in ingestion_views}
         missing_mappings = set(ingestion_view_ids) - set(self._view_mapping_by_id.keys())
         if missing_mappings:
             raise ToolkitValueError(
                 f"The following ingestion views were not found: {humanize_collection(missing_mappings)}"
             )
 
-        view_ids = list({view.view_id for view in ingestion_views})
+        view_ids = list({mapping.view_id for mapping in self._view_mapping_by_id.values()})
         views = self.client.data_modeling.views.retrieve(view_ids)
         self._ingestion_view_by_id = {view.as_id(): view for view in views}
         missing_views = set(view_ids) - set(self._ingestion_view_by_id.keys())
@@ -84,7 +87,7 @@ class AssetCentricMapper(
             )
 
     def map(
-        self, source: Sequence[AssetCentricMapping[T_AssetCentricResource]]
+        self, source: Sequence[AssetCentricMapping[T_AssetCentricResourceExtended]]
     ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
         # We update the direct relation cache in bulk for all resources in the chunk.
@@ -96,7 +99,7 @@ class AssetCentricMapper(
         return output
 
     def _map_single_item(
-        self, item: AssetCentricMapping[T_AssetCentricResource]
+        self, item: AssetCentricMapping[T_AssetCentricResourceExtended]
     ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
         mapping = item.mapping
         ingestion_view = mapping.get_ingestion_view()

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py

@@ -93,7 +93,7 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
             view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
                 # We are ignoring the symbol region in the default mapping.
-                "annotatedResource.id": "edge.startNode",
+                "annotatedResourceId": "edge.startNode",
                 "annotationType": "edge.type.externalId",
                 "creatingUser": "sourceCreatedUser",
                 "creatingApp": "sourceId",
@@ -114,16 +114,16 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
         ResourceViewMappingApply(
             external_id=FILE_ANNOTATIONS_ID,
             resource_type="fileAnnotation",
-            view_id=ViewId("cdf_cdm", "CogniteFileAnnotation", "v1"),
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
-                "annotatedResource.id": "edge.startNode",
+                "annotatedResourceId": "edge.startNode",
                 "annotationType": "edge.type.externalId",
                 "creatingUser": "sourceCreatedUser",
                 "creatingApp": "sourceId",
                 "creatingAppVersion": "sourceContext",
                 "status": "status",
-                "data.fileRef.id": "edge.startNode",
-                "data.fileRef.externalId": "edge.startNode",
+                "data.fileRef.id": "edge.endNode",
+                "data.fileRef.externalId": "edge.endNode",
                 "data.description": "description",
                 "data.pageNumber": "startNodePageNumber",
                 "data.textRegion.confidence": "confidence",

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py

@@ -33,6 +33,7 @@ from .data_classes import (
     MigrationMappingList,
 )
 from .data_model import INSTANCE_SOURCE_VIEW_ID
+from .default_mappings import ASSET_ANNOTATIONS_ID, FILE_ANNOTATIONS_ID
 from .selectors import AssetCentricMigrationSelector, MigrateDataSetSelector, MigrationCSVFileSelector
 
 
@@ -213,6 +214,16 @@ class AssetCentricMigrationIO(
 class AnnotationMigrationIO(
     UploadableStorageIO[AssetCentricMigrationSelector, AssetCentricMapping[Annotation], InstanceApply]
 ):
+    """IO class for migrating Annotations.
+
+    Args:
+        client: The ToolkitClient to use for CDF interactions.
+        instance_space: The instance space to use for the migrated annotations.
+        default_asset_annotation_mapping: The default ingestion mapping to use for asset-linked annotations.
+        default_file_annotation_mapping: The default ingestion mappingto use for file-linked annotations.
+
+    """
+
     KIND = "AnnotationMigration"
     SUPPORTED_DOWNLOAD_FORMATS = frozenset({".parquet", ".csv", ".ndjson"})
     SUPPORTED_COMPRESSIONS = frozenset({".gz"})
@@ -220,17 +231,28 @@ class AnnotationMigrationIO(
     CHUNK_SIZE = 1000
     UPLOAD_ENDPOINT = InstanceIO.UPLOAD_ENDPOINT
 
-    def __init__(self, client: ToolkitClient, instance_space: str | None = None) -> None:
+    def __init__(
+        self,
+        client: ToolkitClient,
+        instance_space: str | None = None,
+        default_asset_annotation_mapping: str | None = None,
+        default_file_annotation_mapping: str | None = None,
+    ) -> None:
         super().__init__(client)
         self.annotation_io = AnnotationIO(client)
         self.instance_space = instance_space
+        self.default_asset_annotation_mapping = default_asset_annotation_mapping or ASSET_ANNOTATIONS_ID
+        self.default_file_annotation_mapping = default_file_annotation_mapping or FILE_ANNOTATIONS_ID
 
     def as_id(self, item: AssetCentricMapping[Annotation]) -> str:
         return f"Annotation_{item.mapping.id}"
 
     def count(self, selector: AssetCentricMigrationSelector) -> int | None:
-        # There is no efficient way to count annotations in CDF.
-        return None
+        if isinstance(selector, MigrationCSVFileSelector):
+            return len(selector.items)
+        else:
+            # There is no efficient way to count annotations in CDF.
+            return None
 
     def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterable[Page]:
         if isinstance(selector, MigrateDataSetSelector):
@@ -253,8 +275,10 @@ class AnnotationMigrationIO(
                 mapping = AnnotationMapping(
                     instance_id=EdgeId(space=self.instance_space, external_id=f"annotation_{resource.id!r}"),
                     id=resource.id,
-                    ingestion_view=selector.ingestion_mapping,
+                    ingestion_view=self._get_mapping(selector.ingestion_mapping, resource),
                     preferred_consumer_view=selector.preferred_consumer_view,
+                    # The PySDK is poorly typed.
+                    annotation_type=resource.annotation_type,  # type: ignore[arg-type]
                 )
                 mapping_list.append(AssetCentricMapping(mapping=mapping, resource=resource))
             yield mapping_list
@@ -275,6 +299,7 @@ class AnnotationMigrationIO(
                 if resource is None:
                     not_found += 1
                     continue
+                mapping.ingestion_view = self._get_mapping(mapping.ingestion_view, resource)
                 chunk.append(AssetCentricMapping(mapping=mapping, resource=resource))
             if chunk:
                 yield chunk
@@ -284,6 +309,21 @@ class AnnotationMigrationIO(
                     f"Could not find {not_found} annotations referenced in the CSV file. They will be skipped during migration."
                 ).print_warning(include_timestamp=True, console=self.client.console)
 
+    def _get_mapping(self, current_mapping: str | None, resource: Annotation) -> str:
+        try:
+            return (
+                current_mapping
+                or {
+                    "diagrams.AssetLink": self.default_asset_annotation_mapping,
+                    "diagrams.FileLink": self.default_file_annotation_mapping,
+                }[resource.annotation_type]
+            )
+        except KeyError as e:
+            raise ToolkitValueError(
+                f"Could not determine default ingestion view for annotation type '{resource.annotation_type}'. "
+                "Please specify the ingestion view explicitly in the CSV file."
+            ) from e
+
     def json_to_resource(self, item_json: dict[str, JsonVal]) -> InstanceApply:
         raise NotImplementedError("Deserializing Annotation Migrations from JSON is not supported.")
 

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -27,7 +27,7 @@ from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSever
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_READ_CLS_BY_FORMAT, FileReader
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
@@ -87,7 +87,7 @@ class UploadCommand(ToolkitCommand):
         ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
-        console = Console()
+        console = client.console
         data_files_by_selector = self._find_data_files(input_dir, kind)
 
         self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
@@ -225,49 +225,44 @@ class UploadCommand(ToolkitCommand):
                 io = self._create_selected_io(selector, datafiles[0], client)
                 if io is None:
                     continue
-                for data_file in datafiles:
-                    file_display = self._path_as_display_name(data_file)
-                    if verbose:
-                        console.print(f"{action} {selector.display_name} from {file_display.as_posix()!r}")
-                    reader = FileReader.from_filepath(data_file)
-                    is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
-                    if is_table and not isinstance(io, TableUploadableStorageIO):
-                        raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
-                    tracker = ProgressTracker[str]([self._UPLOAD])
-                    executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
-                        download_iterable=io.read_chunks(reader),
-                        process=partial(io.rows_to_data, selector=selector)
-                        if is_table and isinstance(io, TableUploadableStorageIO)
-                        else io.json_chunk_to_data,
-                        write=partial(
-                            self._upload_items,
-                            upload_client=upload_client,
-                            io=io,
-                            dry_run=dry_run,
-                            selector=selector,
-                            tracker=tracker,
-                            console=console,
-                        ),
-                        iteration_count=None,
-                        max_queue_size=self._MAX_QUEUE_SIZE,
-                        download_description=f"Reading {file_count:,}/{total_file_count + 1:,}: {file_display.as_posix()!s}",
-                        process_description="Processing",
-                        write_description=f"{action} {selector.display_name!r}",
+                reader = MultiFileReader(datafiles)
+                if reader.is_table and not isinstance(io, TableUploadableStorageIO):
+                    raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+                tracker = ProgressTracker[str]([self._UPLOAD])
+                executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
+                    download_iterable=io.read_chunks(reader),
+                    process=partial(io.rows_to_data, selector=selector)
+                    if reader.is_table and isinstance(io, TableUploadableStorageIO)
+                    else io.json_chunk_to_data,
+                    write=partial(
+                        self._upload_items,
+                        upload_client=upload_client,
+                        io=io,
+                        dry_run=dry_run,
+                        selector=selector,
+                        tracker=tracker,
                         console=console,
-                    )
-                    executor.run()
-                    file_count += 1
-                    executor.raise_on_error()
-                    final_action = "Uploaded" if not dry_run else "Would upload"
-                    suffix = " successfully" if not dry_run else ""
-                    results = tracker.aggregate()
-                    success = results.get((self._UPLOAD, "success"), 0)
-                    failed = results.get((self._UPLOAD, "failed"), 0)
-                    if failed > 0:
-                        suffix += f", {failed:,} failed"
-                    console.print(
-                        f"{final_action} {success:,} {selector.display_name} from {file_display.as_posix()!r}{suffix}."
-                    )
+                    ),
+                    iteration_count=None,
+                    max_queue_size=self._MAX_QUEUE_SIZE,
+                    download_description=f"Reading {selector.display_name!r} files",
+                    process_description="Processing",
+                    write_description=f"{action} {selector.display_name!r}",
+                    console=console,
+                )
+                executor.run()
+                file_count += len(datafiles)
+                executor.raise_on_error()
+                final_action = "Uploaded" if not dry_run else "Would upload"
+                suffix = " successfully" if not dry_run else ""
+                results = tracker.aggregate()
+                success = results.get((self._UPLOAD, "success"), 0)
+                failed = results.get((self._UPLOAD, "failed"), 0)
+                if failed > 0:
+                    suffix += f", {failed:,} failed"
+                console.print(
+                    f"{final_action} {success:,} {selector.display_name} from {len(datafiles)} files{suffix}."
+                )
 
     @staticmethod
     def _path_as_display_name(input_path: Path, cwd: Path = Path.cwd()) -> Path:

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py

@@ -328,6 +328,7 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
 
     def dump_resource(self, resource: Container, local: dict[str, Any] | None = None) -> dict[str, Any]:
         dumped = resource.as_write().dump()
+        has_local = local is not None
         local = local or {}
         for key in ["constraints", "indexes"]:
             if not dumped.get(key) and key not in local:
@@ -339,15 +340,16 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
                 continue
             local_prop = local_prop_by_id[prop_id]
             for key, default in [("immutable", False), ("autoIncrement", False), ("nullable", True)]:
-                if cdf_prop.get(key) is default and key not in local_prop:
+                if has_local and cdf_prop.get(key) is default and key not in local_prop:
                     cdf_prop.pop(key, None)
             cdf_type = cdf_prop.get("type", {})
             local_type = local_prop.get("type", {})
             for key, type_default in [("list", False), ("collation", "ucs_basic")]:
-                if cdf_type.get(key) == type_default and key not in local_type:
+                if has_local and cdf_type.get(key) == type_default and key not in local_type:
                     cdf_type.pop(key, None)
-        if "usedFor" not in local:
-            dumped.pop("usedFor", None)
+            if has_local and "usedFor" not in local and dumped.get("usedFor") == "node":
+                # Only drop if set to default by server.
+                dumped.pop("usedFor", None)
         return dumped
 
     def create(self, items: Sequence[ContainerApply]) -> ContainerList:

{cognite_toolkit-0.6.105 → cognite_toolkit-0.6.107}/cognite_toolkit/_cdf_tk/storageio/_annotations.py

@@ -26,21 +26,23 @@ class AnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
     def stream_data(self, selector: AssetCentricSelector, limit: int | None = None) -> Iterable[Page[Annotation]]:
         total = 0
         for file_chunk in FileMetadataIO(self.client).stream_data(selector, None):
-            # Todo Support pagination. This is missing in the SDK.
-            results = self.client.annotations.list(
-                filter=AnnotationFilter(
-                    annotated_resource_type="file",
-                    annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+            for annotation_type in ["diagrams.AssetLink", "diagrams.FileLink"]:
+                # Todo Support pagination. This is missing in the SDK.
+                results = self.client.annotations.list(
+                    filter=AnnotationFilter(
+                        annotated_resource_type="file",
+                        annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
+                        annotation_type=annotation_type,
+                    )
                 )
-            )
-            if limit is not None and total + len(results) > limit:
-                results = results[: limit - total]
-
-            for chunk in chunker_sequence(results, self.CHUNK_SIZE):
-                yield Page(worker_id="main", items=chunk)
-                total += len(chunk)
-            if limit is not None and total >= limit:
-                break
+                if limit is not None and total + len(results) > limit:
+                    results = results[: limit - total]
+
+                for chunk in chunker_sequence(results, self.CHUNK_SIZE):
+                    yield Page(worker_id="main", items=chunk)
+                    total += len(chunk)
+                if limit is not None and total >= limit:
+                    break
 
     def count(self, selector: AssetCentricSelector) -> int | None:
         """There is no efficient way to count annotations in CDF."""